Skip to content

Commit c993469

Browse files
author
Ubuntu
committed
feat(gce): add regional MIG support for multi-AZ instance groups
Add support for GCE Regional Managed Instance Groups, allowing a single autoscaling InstanceGroup to span multiple zones natively instead of splitting across per-zone zonal MIGs. Changes: - Add RegionInstanceGroupManagerClient interface and implementation to the GCE compute client - Add RegionInstanceGroupManager task with GCE API and Terraform rendering (google_compute_region_instance_group_manager) - Add mock client for testing - Add GCERegionalMIG feature flag (default: false) for backwards compatibility with existing zonal MIG clusters - Refactor AutoscalingGroupModelBuilder.Build() to create a single regional MIG with distribution_policy_zones when flag is enabled - Add NameForRegionInstanceGroupManager naming helper Resolves: #18411
1 parent 5449914 commit c993469

8 files changed

Lines changed: 506 additions & 10 deletions

File tree

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,3 +98,4 @@ charts/
9898

9999
# Ignore asdf .tool-versions file
100100
.tool-versions
101+
go1.26.3.linux-amd64.tar.gz

cloudmock/gce/mockcompute/api.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ type MockClient struct {
4141

4242
instanceTemplateClient *instanceTemplateClient
4343
instanceGroupManagerClient *instanceGroupManagerClient
44+
regionInstanceGroupManagerClient *regionInstanceGroupManagerClient
4445
targetPoolClient *targetPoolClient
4546

4647
diskClient *diskClient
@@ -70,6 +71,7 @@ func NewMockClient(project string) *MockClient {
7071

7172
instanceTemplateClient: newInstanceTemplateClient(),
7273
instanceGroupManagerClient: newInstanceGroupManagerClient(instanceClient),
74+
regionInstanceGroupManagerClient: newRegionInstanceGroupManagerClient(),
7375
targetPoolClient: newTargetPoolClient(),
7476

7577
diskClient: newDiskClient(),
@@ -173,6 +175,10 @@ func (c *MockClient) InstanceGroupManagers() gce.InstanceGroupManagerClient {
173175
return c.instanceGroupManagerClient
174176
}
175177

178+
func (c *MockClient) RegionInstanceGroupManagers() gce.RegionInstanceGroupManagerClient {
179+
return c.regionInstanceGroupManagerClient
180+
}
181+
176182
func (c *MockClient) TargetPools() gce.TargetPoolClient {
177183
return c.targetPoolClient
178184
}
Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
/*
2+
Copyright 2026 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package mockcompute
18+
19+
import (
20+
"context"
21+
"fmt"
22+
"sync"
23+
24+
compute "google.golang.org/api/compute/v1"
25+
"k8s.io/kops/upup/pkg/fi/cloudup/gce"
26+
)
27+
28+
type regionInstanceGroupManagerClient struct {
29+
// regionInstanceGroupManagers keyed by project, region, and name.
30+
regionInstanceGroupManagers map[string]map[string]map[string]*compute.InstanceGroupManager
31+
sync.Mutex
32+
}
33+
34+
var _ gce.RegionInstanceGroupManagerClient = &regionInstanceGroupManagerClient{}
35+
36+
func newRegionInstanceGroupManagerClient() *regionInstanceGroupManagerClient {
37+
return &regionInstanceGroupManagerClient{
38+
regionInstanceGroupManagers: map[string]map[string]map[string]*compute.InstanceGroupManager{},
39+
}
40+
}
41+
42+
func (c *regionInstanceGroupManagerClient) All() map[string]interface{} {
43+
c.Lock()
44+
defer c.Unlock()
45+
m := map[string]interface{}{}
46+
for _, regions := range c.regionInstanceGroupManagers {
47+
for _, igms := range regions {
48+
for n, igm := range igms {
49+
m[n] = igm
50+
}
51+
}
52+
}
53+
return m
54+
}
55+
56+
func (c *regionInstanceGroupManagerClient) Insert(project, region string, igm *compute.InstanceGroupManager) (*compute.Operation, error) {
57+
c.Lock()
58+
defer c.Unlock()
59+
igmRegions, ok := c.regionInstanceGroupManagers[project]
60+
if !ok {
61+
igmRegions = map[string]map[string]*compute.InstanceGroupManager{}
62+
c.regionInstanceGroupManagers[project] = igmRegions
63+
}
64+
igms, ok := igmRegions[region]
65+
if !ok {
66+
igms = map[string]*compute.InstanceGroupManager{}
67+
igmRegions[region] = igms
68+
}
69+
igm.SelfLink = fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/regions/%s/instanceGroupManagers/%s", project, region, igm.Name)
70+
igms[igm.Name] = igm
71+
return doneOperation(), nil
72+
}
73+
74+
func (c *regionInstanceGroupManagerClient) Delete(project, region, name string) (*compute.Operation, error) {
75+
c.Lock()
76+
defer c.Unlock()
77+
regions, ok := c.regionInstanceGroupManagers[project]
78+
if !ok {
79+
return nil, notFoundError()
80+
}
81+
igms, ok := regions[region]
82+
if !ok {
83+
return nil, notFoundError()
84+
}
85+
if _, ok := igms[name]; !ok {
86+
return nil, notFoundError()
87+
}
88+
delete(igms, name)
89+
return doneOperation(), nil
90+
}
91+
92+
func (c *regionInstanceGroupManagerClient) Get(project, region, name string) (*compute.InstanceGroupManager, error) {
93+
c.Lock()
94+
defer c.Unlock()
95+
regions, ok := c.regionInstanceGroupManagers[project]
96+
if !ok {
97+
return nil, notFoundError()
98+
}
99+
igms, ok := regions[region]
100+
if !ok {
101+
return nil, notFoundError()
102+
}
103+
igm, ok := igms[name]
104+
if !ok {
105+
return nil, notFoundError()
106+
}
107+
return igm, nil
108+
}
109+
110+
func (c *regionInstanceGroupManagerClient) List(ctx context.Context, project, region string) ([]*compute.InstanceGroupManager, error) {
111+
c.Lock()
112+
defer c.Unlock()
113+
regions, ok := c.regionInstanceGroupManagers[project]
114+
if !ok {
115+
return nil, nil
116+
}
117+
igms, ok := regions[region]
118+
if !ok {
119+
return nil, nil
120+
}
121+
var l []*compute.InstanceGroupManager
122+
for _, d := range igms {
123+
l = append(l, d)
124+
}
125+
return l, nil
126+
}
127+
128+
func (c *regionInstanceGroupManagerClient) SetTargetPools(project, region, name string, targetPools []string) (*compute.Operation, error) {
129+
return doneOperation(), nil
130+
}
131+
132+
func (c *regionInstanceGroupManagerClient) SetInstanceTemplate(project, region, name, instanceTemplateURL string) (*compute.Operation, error) {
133+
return doneOperation(), nil
134+
}
135+
136+
func (c *regionInstanceGroupManagerClient) Resize(project, region, name string, newSize int64) (*compute.Operation, error) {
137+
return doneOperation(), nil
138+
}

pkg/featureflag/featureflag.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ var (
6969
SpotinstController = new("SpotinstController", Bool(true))
7070
// VPCSkipEnableDNSSupport if set will make that a VPC does not need DNSSupport enabled.
7171
VPCSkipEnableDNSSupport = new("VPCSkipEnableDNSSupport", Bool(false))
72+
GCERegionalMIG = new("GCERegionalMIG", Bool(false))
7273
// SkipEtcdVersionCheck will bypass the check that etcd-manager is using a supported etcd version
7374
SkipEtcdVersionCheck = new("SkipEtcdVersionCheck", Bool(false))
7475
// EtcdEventsHTTP enables HTTP (non-TLS) for the events etcd cluster.

pkg/model/gcemodel/autoscalinggroup.go

Lines changed: 55 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import (
2323

2424
"k8s.io/klog/v2"
2525
"k8s.io/kops/pkg/apis/kops"
26+
"k8s.io/kops/pkg/featureflag"
2627
"k8s.io/kops/pkg/model"
2728
"k8s.io/kops/pkg/model/defaults"
2829
"k8s.io/kops/pkg/model/iam"
@@ -334,26 +335,34 @@ func (b *AutoscalingGroupModelBuilder) Build(c *fi.CloudupModelBuilderContext) e
334335
}
335336
c.AddTask(instanceTemplate)
336337

337-
instanceCountByZone, err := b.splitToZones(ig)
338-
if err != nil {
339-
return err
340-
}
338+
if featureflag.GCERegionalMIG.Enabled() {
339+
// Use a single regional MIG spanning all zones
340+
zones, err := b.FindZonesForInstanceGroup(ig)
341+
if err != nil {
342+
return err
343+
}
341344

342-
for zone, targetSize := range instanceCountByZone {
343-
name := gce.NameForInstanceGroupManager(b.Cluster.ObjectMeta.Name, ig.ObjectMeta.Name, zone)
345+
minSize := 1
346+
if ig.Spec.MinSize != nil {
347+
minSize = int(fi.ValueOf(ig.Spec.MinSize))
348+
} else if ig.Spec.Role == kops.InstanceGroupRoleNode {
349+
minSize = 2
350+
}
351+
352+
name := gce.NameForRegionInstanceGroupManager(b.Cluster.ObjectMeta.Name, ig.ObjectMeta.Name)
344353

345-
t := &gcetasks.InstanceGroupManager{
354+
t := &gcetasks.RegionInstanceGroupManager{
346355
Name: s(name),
347356
Lifecycle: b.Lifecycle,
348-
Zone: s(zone),
349-
TargetSize: fi.PtrTo(int64(targetSize)),
357+
Region: s(b.Region),
358+
TargetSize: fi.PtrTo(int64(minSize)),
350359
UpdatePolicy: &gcetasks.UpdatePolicy{MinimalAction: "REPLACE", Type: "OPPORTUNISTIC"},
351360
BaseInstanceName: s(ig.ObjectMeta.Name),
352361
InstanceTemplate: instanceTemplate,
353362
ListManagedInstancesResults: "PAGINATED",
363+
DistributionPolicyZones: zones,
354364
}
355365

356-
// Attach API server instances to load balancer if we're using one
357366
if ig.HasAPIServer() {
358367
if b.UseLoadBalancerForAPI() {
359368
lbSpec := b.Cluster.Spec.API.LoadBalancer
@@ -369,6 +378,42 @@ func (b *AutoscalingGroupModelBuilder) Build(c *fi.CloudupModelBuilderContext) e
369378
}
370379

371380
c.AddTask(t)
381+
} else {
382+
instanceCountByZone, err := b.splitToZones(ig)
383+
if err != nil {
384+
return err
385+
}
386+
387+
for zone, targetSize := range instanceCountByZone {
388+
name := gce.NameForInstanceGroupManager(b.Cluster.ObjectMeta.Name, ig.ObjectMeta.Name, zone)
389+
390+
t := &gcetasks.InstanceGroupManager{
391+
Name: s(name),
392+
Lifecycle: b.Lifecycle,
393+
Zone: s(zone),
394+
TargetSize: fi.PtrTo(int64(targetSize)),
395+
UpdatePolicy: &gcetasks.UpdatePolicy{MinimalAction: "REPLACE", Type: "OPPORTUNISTIC"},
396+
BaseInstanceName: s(ig.ObjectMeta.Name),
397+
InstanceTemplate: instanceTemplate,
398+
ListManagedInstancesResults: "PAGINATED",
399+
}
400+
401+
if ig.HasAPIServer() {
402+
if b.UseLoadBalancerForAPI() {
403+
lbSpec := b.Cluster.Spec.API.LoadBalancer
404+
if lbSpec != nil {
405+
switch lbSpec.Type {
406+
case kops.LoadBalancerTypePublic:
407+
t.TargetPools = append(t.TargetPools, b.LinkToTargetPool("api"))
408+
case kops.LoadBalancerTypeInternal:
409+
klog.Warningf("Not hooking the instance group manager up to anything.")
410+
}
411+
}
412+
}
413+
}
414+
415+
c.AddTask(t)
416+
}
372417
}
373418
}
374419

upup/pkg/fi/cloudup/gce/compute.go

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ type ComputeClient interface {
3939
Instances() InstanceClient
4040
InstanceTemplates() InstanceTemplateClient
4141
InstanceGroupManagers() InstanceGroupManagerClient
42+
RegionInstanceGroupManagers() RegionInstanceGroupManagerClient
4243
TargetPools() TargetPoolClient
4344
Disks() DiskClient
4445
RegionBackendServices() RegionBackendServiceClient
@@ -843,3 +844,67 @@ func (c *diskClientImpl) SetLabels(project, zone, name string, req *compute.Zone
843844
_, err := c.srv.SetLabels(project, zone, name, req).Do()
844845
return err
845846
}
847+
848+
// RegionInstanceGroupManagerClient wraps the GCE RegionInstanceGroupManagers API.
849+
type RegionInstanceGroupManagerClient interface {
850+
Insert(project, region string, i *compute.InstanceGroupManager) (*compute.Operation, error)
851+
Delete(project, region, name string) (*compute.Operation, error)
852+
Get(project, region, name string) (*compute.InstanceGroupManager, error)
853+
List(ctx context.Context, project, region string) ([]*compute.InstanceGroupManager, error)
854+
SetTargetPools(project, region, name string, targetPools []string) (*compute.Operation, error)
855+
SetInstanceTemplate(project, region, name, instanceTemplateURL string) (*compute.Operation, error)
856+
Resize(project, region, name string, newSize int64) (*compute.Operation, error)
857+
}
858+
859+
type regionInstanceGroupManagerClientImpl struct {
860+
srv *compute.RegionInstanceGroupManagersService
861+
}
862+
863+
var _ RegionInstanceGroupManagerClient = (*regionInstanceGroupManagerClientImpl)(nil)
864+
865+
func (c *regionInstanceGroupManagerClientImpl) Insert(project, region string, i *compute.InstanceGroupManager) (*compute.Operation, error) {
866+
return c.srv.Insert(project, region, i).Do()
867+
}
868+
869+
func (c *regionInstanceGroupManagerClientImpl) Delete(project, region, name string) (*compute.Operation, error) {
870+
return c.srv.Delete(project, region, name).Do()
871+
}
872+
873+
func (c *regionInstanceGroupManagerClientImpl) Get(project, region, name string) (*compute.InstanceGroupManager, error) {
874+
return c.srv.Get(project, region, name).Do()
875+
}
876+
877+
func (c *regionInstanceGroupManagerClientImpl) List(ctx context.Context, project, region string) ([]*compute.InstanceGroupManager, error) {
878+
var ms []*compute.InstanceGroupManager
879+
if err := c.srv.List(project, region).Pages(ctx, func(page *compute.RegionInstanceGroupManagerList) error {
880+
ms = append(ms, page.Items...)
881+
return nil
882+
}); err != nil {
883+
return nil, err
884+
}
885+
return ms, nil
886+
}
887+
888+
func (c *regionInstanceGroupManagerClientImpl) SetTargetPools(project, region, name string, targetPools []string) (*compute.Operation, error) {
889+
req := &compute.RegionInstanceGroupManagersSetTargetPoolsRequest{
890+
TargetPools: targetPools,
891+
}
892+
return c.srv.SetTargetPools(project, region, name, req).Do()
893+
}
894+
895+
func (c *regionInstanceGroupManagerClientImpl) SetInstanceTemplate(project, region, name, instanceTemplateURL string) (*compute.Operation, error) {
896+
req := &compute.RegionInstanceGroupManagersSetTemplateRequest{
897+
InstanceTemplate: instanceTemplateURL,
898+
}
899+
return c.srv.SetInstanceTemplate(project, region, name, req).Do()
900+
}
901+
902+
func (c *regionInstanceGroupManagerClientImpl) Resize(project, region, name string, newSize int64) (*compute.Operation, error) {
903+
return c.srv.Resize(project, region, name, newSize).Do()
904+
}
905+
906+
func (c *computeClientImpl) RegionInstanceGroupManagers() RegionInstanceGroupManagerClient {
907+
return &regionInstanceGroupManagerClientImpl{
908+
srv: c.srv.RegionInstanceGroupManagers,
909+
}
910+
}

upup/pkg/fi/cloudup/gce/instancegroups.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -322,3 +322,9 @@ func addCloudInstanceData(cm *cloudinstances.CloudInstance, instance *compute.In
322322
}
323323
}
324324
}
325+
326+
// NameForRegionInstanceGroupManager builds a name for a regional InstanceGroupManager.
327+
func NameForRegionInstanceGroupManager(clusterName, instanceGroupName string) string {
328+
name := SafeObjectName(instanceGroupName, clusterName)
329+
return name
330+
}

0 commit comments

Comments
 (0)