Skip to content

Commit 4377b66

Browse files
cheftakocheftako
committed
Initial change to add more KCP roles.
1 parent 340689d commit 4377b66

25 files changed

Lines changed: 391 additions & 32 deletions

File tree

‎cmd/kops/create_instancegroup.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,8 @@ func NewCmdCreateInstanceGroup(f *util.Factory, out io.Writer) *cobra.Command {
129129
if r == kopsapi.InstanceGroupRoleAPIServer && !featureflag.APIServerNodes.Enabled() {
130130
continue
131131
}
132+
// TODO: Can we GA the APIServerNodes feature flag?
133+
// TODO: Do we need feature flag for the new roles and multi role support?
132134
allRoles = append(allRoles, r.ToLowerString())
133135
}
134136

‎cmd/kops/reconcile_cluster.go‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,10 @@ func RunReconcileCluster(ctx context.Context, f *util.Factory, out io.Writer, op
141141
opt.InstanceGroupRoles = []string{
142142
string(kops.InstanceGroupRoleAPIServer),
143143
string(kops.InstanceGroupRoleControlPlane),
144+
string(kops.InstanceGroupRoleEtcd),
145+
string(kops.InstanceGroupRoleScheduler),
146+
string(kops.InstanceGroupRoleCloudControllerManager),
147+
string(kops.InstanceGroupRoleKubeControllerManager),
144148
}
145149
opt.Prune = false // Do not prune until after the last rolling update
146150
if _, err := RunCoreUpdateCluster(ctx, f, out, &opt); err != nil {
@@ -160,7 +164,7 @@ func RunReconcileCluster(ctx context.Context, f *util.Factory, out io.Writer, op
160164

161165
// filter the instance group to only include the control plane
162166
opt.filterInstanceGroups = func(ig *kops.InstanceGroup) bool {
163-
return ig.Spec.Role == kops.InstanceGroupRoleAPIServer || ig.Spec.Role == kops.InstanceGroupRoleControlPlane
167+
return ig.Spec.Role.IsControlPlaneType()
164168
}
165169

166170
// Ignore all pods, we just want to check the control plane is responding
@@ -182,6 +186,10 @@ func RunReconcileCluster(ctx context.Context, f *util.Factory, out io.Writer, op
182186
opt.InstanceGroupRoles = []string{
183187
string(kops.InstanceGroupRoleAPIServer),
184188
string(kops.InstanceGroupRoleControlPlane),
189+
string(kops.InstanceGroupRoleEtcd),
190+
string(kops.InstanceGroupRoleScheduler),
191+
string(kops.InstanceGroupRoleCloudControllerManager),
192+
string(kops.InstanceGroupRoleKubeControllerManager),
185193
}
186194
opt.Yes = c.Yes
187195
if err := RunRollingUpdateCluster(ctx, f, out, opt); err != nil {

‎pkg/apis/kops/instancegroup.go‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,16 +63,39 @@ const (
6363
InstanceGroupRoleBastion InstanceGroupRole = "Bastion"
6464
// InstanceGroupRoleAPIServer is an API server role.
6565
InstanceGroupRoleAPIServer InstanceGroupRole = "APIServer"
66+
// InstanceGroupRoleEtcd is an etcd role.
67+
InstanceGroupRoleEtcd InstanceGroupRole = "Etcd"
68+
// InstanceGroupRoleScheduler is a scheduler role.
69+
InstanceGroupRoleScheduler InstanceGroupRole = "Scheduler"
70+
// InstanceGroupRoleCloudControllerManager is a cloud controller manager role.
71+
InstanceGroupRoleCloudControllerManager InstanceGroupRole = "CloudControllerManager"
72+
// InstanceGroupRoleKubeControllerManager is a kubernetes controller manager role.
73+
InstanceGroupRoleKubeControllerManager InstanceGroupRole = "KubeControllerManager"
6674
)
6775

6876
// AllInstanceGroupRoles is a slice of all valid InstanceGroupRole values
6977
var AllInstanceGroupRoles = []InstanceGroupRole{
7078
InstanceGroupRoleControlPlane,
7179
InstanceGroupRoleAPIServer,
80+
InstanceGroupRoleEtcd,
81+
InstanceGroupRoleScheduler,
82+
InstanceGroupRoleCloudControllerManager,
83+
InstanceGroupRoleKubeControllerManager,
7284
InstanceGroupRoleNode,
7385
InstanceGroupRoleBastion,
7486
}
7587

88+
func (igr InstanceGroupRole) IsControlPlaneType() bool {
89+
switch igr {
90+
case InstanceGroupRoleControlPlane, InstanceGroupRoleAPIServer, InstanceGroupRoleEtcd, InstanceGroupRoleScheduler, InstanceGroupRoleCloudControllerManager, InstanceGroupRoleKubeControllerManager:
91+
return true
92+
case InstanceGroupRoleNode, InstanceGroupRoleBastion:
93+
return false
94+
}
95+
// TODO: Should we throw an error here?
96+
return false
97+
}
98+
7699
const (
77100
// BtfsFilesystem indicates a btfs filesystem
78101
BtfsFilesystem = "btfs"
@@ -363,6 +386,46 @@ func (g *InstanceGroup) IsAPIServerOnly() bool {
363386
}
364387
}
365388

389+
// IsEtcdOnly checks if instanceGroup runs only the Etcd
390+
func (g *InstanceGroup) IsEtcdOnly() bool {
391+
switch g.Spec.Role {
392+
case InstanceGroupRoleEtcd:
393+
return true
394+
default:
395+
return false
396+
}
397+
}
398+
399+
// IsSchedulerOnly checks if instanceGroup runs only the Scheduler
400+
func (g *InstanceGroup) IsSchedulerOnly() bool {
401+
switch g.Spec.Role {
402+
case InstanceGroupRoleScheduler:
403+
return true
404+
default:
405+
return false
406+
}
407+
}
408+
409+
// IsCloudControllerManagerOnly checks if instanceGroup runs only the Cloud Controller Manager
410+
func (g *InstanceGroup) IsCloudControllerManagerOnly() bool {
411+
switch g.Spec.Role {
412+
case InstanceGroupRoleCloudControllerManager:
413+
return true
414+
default:
415+
return false
416+
}
417+
}
418+
419+
// IsKubeControllerManagerOnly checks if instanceGroup runs only the Kube Controller Manager
420+
func (g *InstanceGroup) IsKubeControllerManagerOnly() bool {
421+
switch g.Spec.Role {
422+
case InstanceGroupRoleKubeControllerManager:
423+
return true
424+
default:
425+
return false
426+
}
427+
}
428+
366429
// hasAPIServer checks if instanceGroup runs an API Server
367430
func (g *InstanceGroup) HasAPIServer() bool {
368431
return g.IsControlPlane() || g.IsAPIServerOnly()

‎pkg/apis/kops/validation/instancegroup.go‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,10 @@ func ValidateInstanceGroup(g *kops.InstanceGroup, cloud fi.Cloud, strict bool) f
7373
case kops.InstanceGroupRoleNode:
7474
case kops.InstanceGroupRoleBastion:
7575
case kops.InstanceGroupRoleAPIServer:
76+
case kops.InstanceGroupRoleEtcd:
77+
case kops.InstanceGroupRoleScheduler:
78+
case kops.InstanceGroupRoleCloudControllerManager:
79+
case kops.InstanceGroupRoleKubeControllerManager:
7680
default:
7781
var supported []string
7882
for _, role := range kops.AllInstanceGroupRoles {
@@ -253,11 +257,18 @@ func validateVolumeMountSpec(path *field.Path, spec kops.VolumeMountSpec) field.
253257
func CrossValidateInstanceGroup(g *kops.InstanceGroup, cluster *kops.Cluster, cloud fi.Cloud, strict bool) field.ErrorList {
254258
allErrs := ValidateInstanceGroup(g, cloud, strict)
255259

256-
if g.Spec.Role == kops.InstanceGroupRoleControlPlane {
260+
switch g.Spec.Role {
261+
case kops.InstanceGroupRoleControlPlane:
257262
allErrs = append(allErrs, ValidateControlPlaneInstanceGroup(g, cluster)...)
258-
}
259-
260-
if g.Spec.Role == kops.InstanceGroupRoleAPIServer {
263+
case kops.InstanceGroupRoleEtcd:
264+
allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "role"), "Please implement ValidateEtcdInstanceGroup"))
265+
case kops.InstanceGroupRoleScheduler:
266+
allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "role"), "Please implement ValidateSchedulerInstanceGroup"))
267+
case kops.InstanceGroupRoleCloudControllerManager:
268+
allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "role"), "Please implement ValidateCloudControllerManagerInstanceGroup"))
269+
case kops.InstanceGroupRoleKubeControllerManager:
270+
allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "role"), "Please implement ValidateKubeControllerManagerInstanceGroup"))
271+
case kops.InstanceGroupRoleAPIServer:
261272
switch cluster.GetCloudProvider() {
262273
case kops.CloudProviderGCE:
263274
// Fully supported do nothing.

‎pkg/apis/kops/validation/instancegroup_test.go‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -408,6 +408,30 @@ func TestValidateInstanceGroupGVisorWorkerOnly(t *testing.T) {
408408
enabled: fi.PtrTo(true),
409409
expected: []string{"Forbidden::spec.containerd.gvisor"},
410410
},
411+
{
412+
name: "enabled on etcd",
413+
role: kops.InstanceGroupRoleEtcd,
414+
enabled: fi.PtrTo(true),
415+
expected: []string{"Forbidden::spec.containerd.gvisor"},
416+
},
417+
{
418+
name: "enabled on Scheduler",
419+
role: kops.InstanceGroupRoleScheduler,
420+
enabled: fi.PtrTo(true),
421+
expected: []string{"Forbidden::spec.containerd.gvisor"},
422+
},
423+
{
424+
name: "enabled on cloud controller manager",
425+
role: kops.InstanceGroupRoleCloudControllerManager,
426+
enabled: fi.PtrTo(true),
427+
expected: []string{"Forbidden::spec.containerd.gvisor"},
428+
},
429+
{
430+
name: "enabled on kube controller manager",
431+
role: kops.InstanceGroupRoleKubeControllerManager,
432+
enabled: fi.PtrTo(true),
433+
expected: []string{"Forbidden::spec.containerd.gvisor"},
434+
},
411435
{
412436
name: "enabled on bastion",
413437
role: kops.InstanceGroupRoleBastion,
@@ -419,6 +443,26 @@ func TestValidateInstanceGroupGVisorWorkerOnly(t *testing.T) {
419443
role: kops.InstanceGroupRoleAPIServer,
420444
enabled: fi.PtrTo(false),
421445
},
446+
{
447+
name: "disabled on etcd",
448+
role: kops.InstanceGroupRoleEtcd,
449+
enabled: fi.PtrTo(false),
450+
},
451+
{
452+
name: "disabled on Scheduler",
453+
role: kops.InstanceGroupRoleScheduler,
454+
enabled: fi.PtrTo(false),
455+
},
456+
{
457+
name: "disabled on cloud controller manager",
458+
role: kops.InstanceGroupRoleCloudControllerManager,
459+
enabled: fi.PtrTo(false),
460+
},
461+
{
462+
name: "disabled on kube controller manager",
463+
role: kops.InstanceGroupRoleKubeControllerManager,
464+
enabled: fi.PtrTo(false),
465+
},
422466
} {
423467
t.Run(test.name, func(t *testing.T) {
424468
ig := createMinimalInstanceGroup()
@@ -474,6 +518,70 @@ func TestValidInstanceGroup(t *testing.T) {
474518
ExpectedErrors: []string{},
475519
Description: "Valid API Server instance group failed to validate",
476520
},
521+
{
522+
IG: &kops.InstanceGroup{
523+
ObjectMeta: v1.ObjectMeta{
524+
Name: "eu-central-1a",
525+
},
526+
Spec: kops.InstanceGroupSpec{
527+
Role: kops.InstanceGroupRoleEtcd,
528+
Subnets: []string{"eu-central-1a"},
529+
MaxSize: fi.PtrTo(int32(1)),
530+
MinSize: fi.PtrTo(int32(1)),
531+
Image: "my-image",
532+
},
533+
},
534+
ExpectedErrors: []string{},
535+
Description: "Valid Etcd instance group failed to validate",
536+
},
537+
{
538+
IG: &kops.InstanceGroup{
539+
ObjectMeta: v1.ObjectMeta{
540+
Name: "eu-central-1a",
541+
},
542+
Spec: kops.InstanceGroupSpec{
543+
Role: kops.InstanceGroupRoleScheduler,
544+
Subnets: []string{"eu-central-1a"},
545+
MaxSize: fi.PtrTo(int32(1)),
546+
MinSize: fi.PtrTo(int32(1)),
547+
Image: "my-image",
548+
},
549+
},
550+
ExpectedErrors: []string{},
551+
Description: "Valid Scheduler instance group failed to validate",
552+
},
553+
{
554+
IG: &kops.InstanceGroup{
555+
ObjectMeta: v1.ObjectMeta{
556+
Name: "eu-central-1a",
557+
},
558+
Spec: kops.InstanceGroupSpec{
559+
Role: kops.InstanceGroupRoleCloudControllerManager,
560+
Subnets: []string{"eu-central-1a"},
561+
MaxSize: fi.PtrTo(int32(1)),
562+
MinSize: fi.PtrTo(int32(1)),
563+
Image: "my-image",
564+
},
565+
},
566+
ExpectedErrors: []string{},
567+
Description: "Valid Cloud Controller Manager instance group failed to validate",
568+
},
569+
{
570+
IG: &kops.InstanceGroup{
571+
ObjectMeta: v1.ObjectMeta{
572+
Name: "eu-central-1a",
573+
},
574+
Spec: kops.InstanceGroupSpec{
575+
Role: kops.InstanceGroupRoleKubeControllerManager,
576+
Subnets: []string{"eu-central-1a"},
577+
MaxSize: fi.PtrTo(int32(1)),
578+
MinSize: fi.PtrTo(int32(1)),
579+
Image: "my-image",
580+
},
581+
},
582+
ExpectedErrors: []string{},
583+
Description: "Valid Kube Controller Manager instance group failed to validate",
584+
},
477585
{
478586
IG: &kops.InstanceGroup{
479587
ObjectMeta: v1.ObjectMeta{

‎pkg/apis/nodeup/config_test.go‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,26 @@ func TestNewConfigGVisorWorkerOnly(t *testing.T) {
131131
role: kops.InstanceGroupRoleAPIServer,
132132
containerd: &kops.ContainerdConfig{GVisor: &kops.GVisorConfig{Enabled: ptrToBool(true)}},
133133
},
134+
{
135+
name: "cluster config on etcd",
136+
role: kops.InstanceGroupRoleEtcd,
137+
containerd: &kops.ContainerdConfig{GVisor: &kops.GVisorConfig{Enabled: ptrToBool(true)}},
138+
},
139+
{
140+
name: "cluster config on scheduler",
141+
role: kops.InstanceGroupRoleScheduler,
142+
containerd: &kops.ContainerdConfig{GVisor: &kops.GVisorConfig{Enabled: ptrToBool(true)}},
143+
},
144+
{
145+
name: "cluster config on cloud controller manager",
146+
role: kops.InstanceGroupRoleCloudControllerManager,
147+
containerd: &kops.ContainerdConfig{GVisor: &kops.GVisorConfig{Enabled: ptrToBool(true)}},
148+
},
149+
{
150+
name: "cluster config on kube controller manager",
151+
role: kops.InstanceGroupRoleKubeControllerManager,
152+
containerd: &kops.ContainerdConfig{GVisor: &kops.GVisorConfig{Enabled: ptrToBool(true)}},
153+
},
134154
{
135155
name: "cluster config on bastion",
136156
role: kops.InstanceGroupRoleBastion,

‎pkg/instancegroups/instancegroups.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -712,8 +712,7 @@ func (c *RollingUpdateCluster) drainNode(ctx context.Context, u *cloudinstances.
712712
if !c.Options.DeregisterControlPlaneNodes {
713713
if u.CloudInstanceGroup != nil && u.CloudInstanceGroup.InstanceGroup != nil {
714714
role := u.CloudInstanceGroup.InstanceGroup.Spec.Role
715-
switch role {
716-
case api.InstanceGroupRoleAPIServer, api.InstanceGroupRoleControlPlane:
715+
if role.IsControlPlaneType() {
717716
klog.Infof("skipping deregistration of instance %q, as part of instancegroup with role %q", u.ID, role)
718717
shouldDeregister = false
719718
}

‎pkg/instancegroups/rollingupdate.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,9 @@ func (c *RollingUpdateCluster) RollingUpdate(ctx context.Context, groups map[str
126126
nodeGroups[k] = group
127127
case api.InstanceGroupRoleAPIServer:
128128
apiServerGroups[k] = group
129+
case api.InstanceGroupRoleEtcd, api.InstanceGroupRoleScheduler, api.InstanceGroupRoleCloudControllerManager, api.InstanceGroupRoleKubeControllerManager:
130+
// TODO: Break out and handle each of these cases
131+
return fmt.Errorf("Still need to implement handling for group %q, type %q", group.InstanceGroup.ObjectMeta.Name, group.InstanceGroup.Spec.Role)
129132
case api.InstanceGroupRoleControlPlane:
130133
masterGroups[k] = group
131134
case api.InstanceGroupRoleBastion:

‎pkg/model/awsmodel/iam.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -272,6 +272,14 @@ func (b *IAMModelBuilder) roleKey(role iam.Subject) (string, bool) {
272272
return "master", false
273273
case *iam.NodeRoleAPIServer:
274274
return strings.ToLower(string(kops.InstanceGroupRoleAPIServer)), false
275+
case *iam.NodeRoleEtcd:
276+
return strings.ToLower(string(kops.InstanceGroupRoleEtcd)), false
277+
case *iam.NodeRoleScheduler:
278+
return strings.ToLower(string(kops.InstanceGroupRoleScheduler)), false
279+
case *iam.NodeRoleCloudControllerManager:
280+
return strings.ToLower(string(kops.InstanceGroupRoleCloudControllerManager)), false
281+
case *iam.NodeRoleKubeControllerManager:
282+
return strings.ToLower(string(kops.InstanceGroupRoleKubeControllerManager)), false
275283
case *iam.NodeRoleNode:
276284
return strings.ToLower(string(kops.InstanceGroupRoleNode)), false
277285
case *iam.NodeRoleBastion:

‎pkg/model/components/kubescheduler/model.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,8 @@ func (b *KubeSchedulerBuilder) Build(c *fi.CloudupModelBuilderContext) error {
5959
b.AssetBuilder.AddStaticFile(&assets.StaticFile{
6060
Path: KubeSchedulerConfigPath,
6161
Content: string(configYAML),
62-
Roles: []kops.InstanceGroupRole{kops.InstanceGroupRoleControlPlane, kops.InstanceGroupRoleAPIServer},
62+
// TODO: Why add the static file to APIServer only?
63+
Roles: []kops.InstanceGroupRole{kops.InstanceGroupRoleControlPlane, kops.InstanceGroupRoleAPIServer, kops.InstanceGroupRoleScheduler},
6364
})
6465
return nil
6566
}

0 commit comments

Comments
 (0)