chore: Wrap operator and remove deps on cloudprovider.Context (#3733

)

cmd/controller/main.go

@@ -18,52 +18,49 @@ import (
 	"github.com/samber/lo"
 
 	"github.com/aws/karpenter/pkg/cloudprovider"
-	"github.com/aws/karpenter/pkg/context"
 	"github.com/aws/karpenter/pkg/controllers"
+	"github.com/aws/karpenter/pkg/operator"
 	"github.com/aws/karpenter/pkg/webhooks"
 
-	corecloudprovider "github.com/aws/karpenter-core/pkg/cloudprovider"
 	"github.com/aws/karpenter-core/pkg/cloudprovider/metrics"
 	corecontrollers "github.com/aws/karpenter-core/pkg/controllers"
 	"github.com/aws/karpenter-core/pkg/controllers/state"
-	"github.com/aws/karpenter-core/pkg/operator"
+	coreoperator "github.com/aws/karpenter-core/pkg/operator"
 	corewebhooks "github.com/aws/karpenter-core/pkg/webhooks"
 )
 
 func main() {
-	ctx, operator := operator.NewOperator()
-	awsCtx := context.NewOrDie(corecloudprovider.Context{
-		Context:             ctx,
-		Clock:               operator.Clock,
-		RESTConfig:          operator.RESTConfig,
-		KubeClient:          operator.GetClient(),
-		KubernetesInterface: operator.KubernetesInterface,
-		EventRecorder:       operator.EventRecorder,
-		StartAsync:          operator.Elected(),
-	})
+	ctx, op := operator.NewOperator(coreoperator.NewOperator())
 	awsCloudProvider := cloudprovider.New(
-		awsCtx.InstanceTypesProvider,
-		awsCtx.InstanceProvider,
-		awsCtx.KubeClient,
-		awsCtx.AMIProvider,
+		op.InstanceTypesProvider,
+		op.InstanceProvider,
+		op.GetClient(),
+		op.AMIProvider,
 	)
-	lo.Must0(operator.AddHealthzCheck("cloud-provider", awsCloudProvider.LivenessProbe))
+	lo.Must0(op.AddHealthzCheck("cloud-provider", awsCloudProvider.LivenessProbe))
 	cloudProvider := metrics.Decorate(awsCloudProvider)
 
-	operator.
+	op.
 		WithControllers(ctx, corecontrollers.NewControllers(
 			ctx,
-			operator.Clock,
-			operator.GetClient(),
-			operator.KubernetesInterface,
-			state.NewCluster(operator.Clock, operator.GetClient(), cloudProvider),
-			operator.EventRecorder,
+			op.Clock,
+			op.GetClient(),
+			op.KubernetesInterface,
+			state.NewCluster(op.Clock, op.GetClient(), cloudProvider),
+			op.EventRecorder,
 			cloudProvider,
 		)...).
 		WithWebhooks(corewebhooks.NewWebhooks()...).
 		WithControllers(ctx, controllers.NewControllers(
-			awsCtx,
-			awsCloudProvider,
+			ctx,
+			op.Session,
+			op.Clock,
+			op.GetClient(),
+			op.EventRecorder,
+			op.UnavailableOfferingsCache,
+			op.SubnetProvider,
+			op.SecurityGroupProvider,
+			op.PricingProvider,
 		)...).
 		WithWebhooks(webhooks.NewWebhooks()...).
 		Start(ctx)

go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/PuerkitoBio/goquery v1.8.1
 	github.com/avast/retry-go v3.0.0+incompatible
 	github.com/aws/aws-sdk-go v1.44.195
-	github.com/aws/karpenter-core v0.27.2-0.20230407175913-394c11ceb55a
+	github.com/aws/karpenter-core v0.27.2-0.20230407191512-7a015fc49bc3
 	github.com/go-playground/validator/v10 v10.11.2
 	github.com/imdario/mergo v0.3.15
 	github.com/mitchellh/hashstructure/v2 v2.0.2

go.sum

@@ -70,8 +70,8 @@ github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHS
 github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY=
 github.com/aws/aws-sdk-go v1.44.195 h1:d5xFL0N83Fpsq2LFiHgtBUHknCRUPGHdOlCWt/jtOJs=
 github.com/aws/aws-sdk-go v1.44.195/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
-github.com/aws/karpenter-core v0.27.2-0.20230407175913-394c11ceb55a h1:t8sVxSVAnLwNbc+K0uAhE9EPNq7NBoDZTuI/zlLvt4U=
-github.com/aws/karpenter-core v0.27.2-0.20230407175913-394c11ceb55a/go.mod h1:NQOEY9xxhPi6xUTknEA0KaY9pPbXzS5N4EKkZ5f85d8=
+github.com/aws/karpenter-core v0.27.2-0.20230407191512-7a015fc49bc3 h1:wOfc+9Cc81befzMuibe+oZzpN0EA+oPReAE0zMATNsM=
+github.com/aws/karpenter-core v0.27.2-0.20230407191512-7a015fc49bc3/go.mod h1:NQOEY9xxhPi6xUTknEA0KaY9pPbXzS5N4EKkZ5f85d8=
 github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=

hack/docs/instancetypes_gen_docs.go

@@ -32,14 +32,16 @@ import (
 	"k8s.io/apimachinery/pkg/api/resource"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/rest"
+	"sigs.k8s.io/controller-runtime/pkg/client/config"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
 
 	coresettings "github.com/aws/karpenter-core/pkg/apis/settings"
 	"github.com/aws/karpenter-core/pkg/apis/v1alpha5"
+	coreoperator "github.com/aws/karpenter-core/pkg/operator"
 	coretest "github.com/aws/karpenter-core/pkg/test"
 	"github.com/aws/karpenter/pkg/apis/settings"
 	awscloudprovider "github.com/aws/karpenter/pkg/cloudprovider"
-	awscontext "github.com/aws/karpenter/pkg/context"
+	"github.com/aws/karpenter/pkg/operator"
 	"github.com/aws/karpenter/pkg/test"
 
 	"github.com/aws/karpenter-core/pkg/cloudprovider"
@@ -53,8 +55,9 @@ func main() {
 		log.Fatalf("Usage: %s path/to/markdown.md", os.Args[0])
 	}
 
-	os.Setenv("AWS_SDK_LOAD_CONFIG", "true")
-	os.Setenv("AWS_REGION", "us-east-1")
+	lo.Must0(os.Setenv("SYSTEM_NAMESPACE", "karpenter"))
+	lo.Must0(os.Setenv("AWS_SDK_LOAD_CONFIG", "true"))
+	lo.Must0(os.Setenv("AWS_REGION", "us-east-1"))
 
 	ctx := coresettings.ToContext(context.Background(), coretest.Settings())
 	ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{
@@ -63,7 +66,13 @@ func main() {
 		IsolatedVPC:     lo.ToPtr(true), // disable pricing lookup
 	}))
 
-	cp := NewAWSCloudProviderForCodeGen(ctx)
+	restConfig := config.GetConfigOrDie()
+	ctx, op := operator.NewOperator(ctx, &coreoperator.Operator{
+		Manager:             lo.Must(manager.New(restConfig, manager.Options{})),
+		KubernetesInterface: kubernetes.NewForConfigOrDie(restConfig),
+	})
+	cp := awscloudprovider.New(op.InstanceTypesProvider, op.InstanceProvider, op.GetClient(), op.AMIProvider)
+
 	provider := v1alpha1.AWS{SubnetSelector: map[string]string{
 		"*": "*",
 	}}
@@ -236,12 +245,3 @@ func (f kubeDnsTransport) RoundTrip(request *http.Request) (*http.Response, erro
 		Body:       io.NopCloser(bytes.NewBufferString(kubeDNS)),
 	}, nil
 }
-
-func NewAWSCloudProviderForCodeGen(ctx context.Context) *awscloudprovider.CloudProvider {
-	context := awscontext.NewOrDie(cloudprovider.Context{
-		Context:             ctx,
-		RESTConfig:          &rest.Config{},
-		KubernetesInterface: lo.Must(kubernetes.NewForConfigAndClient(&rest.Config{}, &http.Client{Transport: &kubeDnsTransport{}})),
-	})
-	return awscloudprovider.New(context.InstanceTypesProvider, context.InstanceProvider, context.KubeClient, context.AMIProvider)
-}

hack/docs/metrics_gen_docs.go

@@ -266,6 +266,7 @@ func getIdentMapping(identName string) (string, error) {
 		"interruptionSubsystem":   "interruption",
 		"nodeTemplateSubsystem":   "nodetemplate",
 		"deprovisioningSubsystem": "deprovisioning",
+		"consistencySubsystem":    "consistency",
 		"batcherSubsystem":        "cloudprovider_batcher",
 	}
 	if v, ok := identMapping[identName]; ok {

pkg/controllers/controllers.go

@@ -15,33 +15,43 @@ limitations under the License.
 package controllers
 
 import (
+	"context"
+
+	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/sqs"
+	"k8s.io/utils/clock"
 	"knative.dev/pkg/logging"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 
+	"github.com/aws/karpenter-core/pkg/events"
 	"github.com/aws/karpenter/pkg/apis/settings"
-	"github.com/aws/karpenter/pkg/cloudprovider"
-	awscontext "github.com/aws/karpenter/pkg/context"
+	"github.com/aws/karpenter/pkg/cache"
 	"github.com/aws/karpenter/pkg/controllers/interruption"
 	"github.com/aws/karpenter/pkg/controllers/nodetemplate"
 	"github.com/aws/karpenter/pkg/providers/pricing"
+	"github.com/aws/karpenter/pkg/providers/securitygroup"
+	"github.com/aws/karpenter/pkg/providers/subnet"
 	"github.com/aws/karpenter/pkg/utils/project"
 
 	"github.com/aws/karpenter-core/pkg/operator/controller"
 )
 
-func NewControllers(ctx awscontext.Context, cloudProvider *cloudprovider.CloudProvider) []controller.Controller {
+func NewControllers(ctx context.Context, sess *session.Session, clk clock.Clock, kubeClient client.Client, recorder events.Recorder,
+	unavailableOfferings *cache.UnavailableOfferings, subnetProvider *subnet.Provider,
+	securityGroupProvider *securitygroup.Provider, pricingProvider *pricing.Provider) []controller.Controller {
+
 	logging.FromContext(ctx).With("version", project.Version).Debugf("discovered version")
 
 	controllers := []controller.Controller{
-		nodetemplate.NewController(ctx.KubeClient, ctx.SubnetProvider, ctx.SecurityGroupProvider),
+		nodetemplate.NewController(kubeClient, subnetProvider, securityGroupProvider),
 	}
 	if settings.FromContext(ctx).InterruptionQueueName != "" {
-		controllers = append(controllers, interruption.NewController(ctx.KubeClient, ctx.Clock, ctx.EventRecorder, interruption.NewSQSProvider(sqs.New(ctx.Session)), ctx.UnavailableOfferingsCache))
+		controllers = append(controllers, interruption.NewController(kubeClient, clk, recorder, interruption.NewSQSProvider(sqs.New(sess)), unavailableOfferings))
 	}
 	if settings.FromContext(ctx).IsolatedVPC {
 		logging.FromContext(ctx).Infof("assuming isolated VPC, pricing information will not be updated")
 	} else {
-		controllers = append(controllers, pricing.NewController(ctx.PricingProvider))
+		controllers = append(controllers, pricing.NewController(pricingProvider))
 	}
 	return controllers
 }

pkg/context/context.go → pkg/operator/operator.go

@@ -12,7 +12,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package context
+package operator
 
 import (
 	"context"
@@ -23,7 +23,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
-	"github.com/aws/aws-sdk-go/aws/client"
+	awsclient "github.com/aws/aws-sdk-go/aws/client"
 	"github.com/aws/aws-sdk-go/aws/ec2metadata"
 	"github.com/aws/aws-sdk-go/aws/endpoints"
 	"github.com/aws/aws-sdk-go/aws/request"
@@ -42,6 +42,7 @@ import (
 	"knative.dev/pkg/logging"
 	"knative.dev/pkg/ptr"
 
+	"github.com/aws/karpenter-core/pkg/operator"
 	"github.com/aws/karpenter/pkg/apis/settings"
 	awscache "github.com/aws/karpenter/pkg/cache"
 	"github.com/aws/karpenter/pkg/providers/amifamily"
@@ -52,13 +53,11 @@ import (
 	"github.com/aws/karpenter/pkg/providers/securitygroup"
 	"github.com/aws/karpenter/pkg/providers/subnet"
 	"github.com/aws/karpenter/pkg/utils/project"
-
-	"github.com/aws/karpenter-core/pkg/cloudprovider"
 )
 
-// Context is injected into the AWS CloudProvider's factories
-type Context struct {
-	cloudprovider.Context
+// Operator is injected into the AWS CloudProvider's factories
+type Operator struct {
+	*operator.Operator
 
 	Session                   *session.Session
 	UnavailableOfferingsCache *awscache.UnavailableOfferings
@@ -73,12 +72,11 @@ type Context struct {
 	InstanceProvider          *instance.Provider
 }
 
-func NewOrDie(ctx cloudprovider.Context) Context {
-	ctx.Context = logging.WithLogger(ctx, logging.FromContext(ctx).Named("aws"))
+func NewOperator(ctx context.Context, operator *operator.Operator) (context.Context, *Operator) {
 	sess := withUserAgent(session.Must(session.NewSession(
 		request.WithRetryer(
 			&aws.Config{STSRegionalEndpoint: endpoints.RegionalSTSEndpoint},
-			client.DefaultRetryer{NumMaxRetries: client.DefaultRetryerMaxNumRetries},
+			awsclient.DefaultRetryer{NumMaxRetries: awsclient.DefaultRetryerMaxNumRetries},
 		),
 	)))
 	if *sess.Config.Region == "" {
@@ -98,7 +96,7 @@ func NewOrDie(ctx cloudprovider.Context) Context {
 		logging.FromContext(ctx).With("cluster-endpoint", clusterEndpoint).Debugf("discovered cluster endpoint")
 	}
 	// We perform best-effort on resolving the kube-dns IP
-	kubeDNSIP, err := kubeDNSIP(ctx, ctx.KubernetesInterface)
+	kubeDNSIP, err := kubeDNSIP(ctx, operator.KubernetesInterface)
 	if err != nil {
 		// If we fail to get the kube-dns IP, we don't want to crash because this causes issues with custom DNS setups
 		// https://github.com/aws/karpenter/issues/2787
@@ -116,17 +114,17 @@ func NewOrDie(ctx cloudprovider.Context) Context {
 		ec2api,
 		*sess.Config.Region,
 	)
-	amiProvider := amifamily.NewProvider(ctx.KubeClient, ctx.KubernetesInterface, ssm.New(sess), ec2api,
+	amiProvider := amifamily.NewProvider(operator.GetClient(), operator.KubernetesInterface, ssm.New(sess), ec2api,
 		cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval), cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval), cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval))
-	amiResolver := amifamily.New(ctx.KubeClient, amiProvider)
+	amiResolver := amifamily.New(operator.GetClient(), amiProvider)
 	launchTemplateProvider := launchtemplate.NewProvider(
 		ctx,
 		cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval),
 		ec2api,
 		amiResolver,
 		securityGroupProvider,
-		lo.Must(getCABundle(ctx.RESTConfig)),
-		ctx.StartAsync,
+		lo.Must(getCABundle(operator.GetConfig())),
+		operator.Elected(),
 		kubeDNSIP,
 		clusterEndpoint,
 	)
@@ -148,8 +146,8 @@ func NewOrDie(ctx cloudprovider.Context) Context {
 		launchTemplateProvider,
 	)
 
-	return Context{
-		Context:                   ctx,
+	return ctx, &Operator{
+		Operator:                  operator,
 		Session:                   sess,
 		UnavailableOfferingsCache: unavailableOfferingsCache,
 		EC2API:                    ec2api,

pkg/context/suite_test.go → pkg/operator/suite_test.go

@@ -12,7 +12,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package context_test
+package operator_test
 
 import (
 	"context"
@@ -27,8 +27,8 @@ import (
 
 	"github.com/aws/karpenter/pkg/apis"
 	"github.com/aws/karpenter/pkg/apis/settings"
-	awscontext "github.com/aws/karpenter/pkg/context"
 	"github.com/aws/karpenter/pkg/fake"
+	awscontext "github.com/aws/karpenter/pkg/operator"
 	"github.com/aws/karpenter/pkg/test"
 
 	coresettings "github.com/aws/karpenter-core/pkg/apis/settings"
@@ -70,7 +70,7 @@ var _ = AfterEach(func() {
 	ExpectCleanedUp(ctx, env.Client)
 })
 
-var _ = Describe("Context", func() {
+var _ = Describe("Operator", func() {
 
 	It("should resolve endpoint if set via configuration", func() {
 		ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{

test/go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/aws/aws-sdk-go v1.44.234
 	github.com/aws/aws-sdk-go-v2/config v1.18.19
 	github.com/aws/karpenter v0.22.0
-	github.com/aws/karpenter-core v0.27.2-0.20230407175913-394c11ceb55a
+	github.com/aws/karpenter-core v0.27.2-0.20230407191512-7a015fc49bc3
 	github.com/onsi/ginkgo/v2 v2.9.2
 	github.com/onsi/gomega v1.27.6
 	github.com/samber/lo v1.38.1

test/go.sum

@@ -82,8 +82,8 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6 h1:B8cauxOH1W1v7rd8RdI/MWno
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6/go.mod h1:Lh/bc9XUf8CfOY6Jp5aIkQtN+j1mc+nExc+KXj9jx2s=
 github.com/aws/aws-sdk-go-v2/service/sts v1.18.7 h1:bWNgNdRko2x6gqa0blfATqAZKZokPIeM1vfmQt2pnvM=
 github.com/aws/aws-sdk-go-v2/service/sts v1.18.7/go.mod h1:JuTnSoeePXmMVe9G8NcjjwgOKEfZ4cOjMuT2IBT/2eI=
-github.com/aws/karpenter-core v0.27.2-0.20230407175913-394c11ceb55a h1:t8sVxSVAnLwNbc+K0uAhE9EPNq7NBoDZTuI/zlLvt4U=
-github.com/aws/karpenter-core v0.27.2-0.20230407175913-394c11ceb55a/go.mod h1:NQOEY9xxhPi6xUTknEA0KaY9pPbXzS5N4EKkZ5f85d8=
+github.com/aws/karpenter-core v0.27.2-0.20230407191512-7a015fc49bc3 h1:wOfc+9Cc81befzMuibe+oZzpN0EA+oPReAE0zMATNsM=
+github.com/aws/karpenter-core v0.27.2-0.20230407191512-7a015fc49bc3/go.mod h1:NQOEY9xxhPi6xUTknEA0KaY9pPbXzS5N4EKkZ5f85d8=
 github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8=
 github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
 github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=