Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(eks): make kubectlLayer property required from optional #32930

Closed
wants to merge 7 commits into from
Closed
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import * as iam from 'aws-cdk-lib/aws-iam';
import * as sfn from 'aws-cdk-lib/aws-stepfunctions';
import * as cdk from 'aws-cdk-lib';
import * as integ from '@aws-cdk/integ-tests-alpha';
import { KubectlV31Layer } from '@aws-cdk/lambda-layer-kubectl-v31';
import { EksCall, HttpMethods } from 'aws-cdk-lib/aws-stepfunctions-tasks';
import { EC2_RESTRICT_DEFAULT_SECURITY_GROUP } from 'aws-cdk-lib/cx-api';

Expand All @@ -25,6 +26,7 @@ stack.node.setContext(EC2_RESTRICT_DEFAULT_SECURITY_GROUP, false);
const cluster = new eks.Cluster(stack, 'EksCluster', {
version: eks.KubernetesVersion.V1_30,
clusterName: 'eksCluster',
kubectlLayer: new KubectlV31Layer(stack, 'KubectlLayer'),
});

const executionRole = new iam.Role(stack, 'Role', {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import * as iam from 'aws-cdk-lib/aws-iam';
import * as sfn from 'aws-cdk-lib/aws-stepfunctions';
import * as cdk from 'aws-cdk-lib';
import * as integ from '@aws-cdk/integ-tests-alpha';
import { KubectlV31Layer } from '@aws-cdk/lambda-layer-kubectl-v31';
import {
Classification, VirtualClusterInput, EksClusterInput, EmrContainersDeleteVirtualCluster,
EmrContainersCreateVirtualCluster, EmrContainersStartJobRun, ReleaseLabel,
Expand All @@ -29,6 +30,7 @@ const eksCluster = new eks.Cluster(stack, 'integration-test-eks-cluster', {
version: eks.KubernetesVersion.V1_30,
defaultCapacity: 3,
defaultCapacityInstance: ec2.InstanceType.of(ec2.InstanceClass.M5, ec2.InstanceSize.XLARGE),
kubectlLayer: new KubectlV31Layer(stack, 'KubectlLayer'),
});

const jobExecutionRole = new iam.Role(stack, 'JobExecutionRole', {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import * as sfn from 'aws-cdk-lib/aws-stepfunctions';
import * as cdk from 'aws-cdk-lib';
import { Aws } from 'aws-cdk-lib';
import * as integ from '@aws-cdk/integ-tests-alpha';
import { KubectlV31Layer } from '@aws-cdk/lambda-layer-kubectl-v31';
import { EmrContainersStartJobRun, ReleaseLabel, VirtualClusterInput } from 'aws-cdk-lib/aws-stepfunctions-tasks';
import { EC2_RESTRICT_DEFAULT_SECURITY_GROUP } from 'aws-cdk-lib/cx-api';

Expand All @@ -25,6 +26,7 @@ const eksCluster = new eks.Cluster(stack, 'integration-test-eks-cluster', {
version: eks.KubernetesVersion.V1_30,
defaultCapacity: 3,
defaultCapacityInstance: ec2.InstanceType.of(ec2.InstanceClass.M5, ec2.InstanceSize.XLARGE),
kubectlLayer: new KubectlV31Layer(stack, 'KubectlLayer'),
});

const virtualCluster = new cdk.CfnResource(stack, 'Virtual Cluster', {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,15 @@ import * as lambda from 'aws-cdk-lib/aws-lambda';
import * as cdk from 'aws-cdk-lib';
import * as cr from 'aws-cdk-lib/custom-resources';
import * as integ from '@aws-cdk/integ-tests-alpha';

import { KubectlLayer } from 'aws-cdk-lib/lambda-layer-kubectl';
import { KubectlV31Layer } from '@aws-cdk/lambda-layer-kubectl-v31';

/**
* Test verifies that kubectl and helm are invoked successfully inside Lambda runtime.
*/

const app = new cdk.App();
const stack = new cdk.Stack(app, 'lambda-layer-kubectl-integ-stack');
const layer = new KubectlLayer(stack, 'KubectlLayer');
const layer = new KubectlV31Layer(stack, 'KubectlLayer');

const runtimes = [
lambda.Runtime.PYTHON_3_9,
Expand Down
26 changes: 8 additions & 18 deletions packages/aws-cdk-lib/aws-eks/lib/cluster.ts
Original file line number Diff line number Diff line change
Expand Up @@ -142,9 +142,8 @@ export interface ICluster extends IResource, ec2.IConnectable {
/**
* An AWS Lambda layer that includes `kubectl` and `helm`
*
* If not defined, a default layer will be used containing Kubectl 1.20 and Helm 3.8
*/
readonly kubectlLayer?: lambda.ILayerVersion;
readonly kubectlLayer: lambda.ILayerVersion;

/**
* Specify which IP family is used to assign Kubernetes pod and service IP addresses.
Expand Down Expand Up @@ -375,19 +374,16 @@ export interface ClusterAttributes {
* This layer is used by the kubectl handler to apply manifests and install
* helm charts. You must pick an appropriate releases of one of the
* `@aws-cdk/layer-kubectl-vXX` packages, that works with the version of
* Kubernetes you have chosen. If you don't supply this value `kubectl`
* 1.20 will be used, but that version is most likely too old.
* Kubernetes you have chosen.
*
* The handler expects the layer to include the following executables:
*
* ```
* /opt/helm/helm
* /opt/kubectl/kubectl
* ```
*
* @default - a default layer with Kubectl 1.20 and helm 3.8.
*/
readonly kubectlLayer?: lambda.ILayerVersion;
readonly kubectlLayer: lambda.ILayerVersion;

/**
* An AWS Lambda layer that contains the `aws` CLI.
Expand Down Expand Up @@ -566,19 +562,16 @@ export interface ClusterOptions extends CommonClusterOptions {
* This layer is used by the kubectl handler to apply manifests and install
* helm charts. You must pick an appropriate releases of one of the
* `@aws-cdk/layer-kubectl-vXX` packages, that works with the version of
* Kubernetes you have chosen. If you don't supply this value `kubectl`
* 1.20 will be used, but that version is most likely too old.
* Kubernetes you have chosen.
*
* The handler expects the layer to include the following executables:
*
* ```
* /opt/helm/helm
* /opt/kubectl/kubectl
* ```
*
* @default - a default layer with Kubectl 1.20.
*/
readonly kubectlLayer?: lambda.ILayerVersion;
readonly kubectlLayer: lambda.ILayerVersion;

/**
* An AWS Lambda layer that contains the `aws` CLI.
Expand Down Expand Up @@ -1083,6 +1076,7 @@ abstract class ClusterBase extends Resource implements ICluster {
public abstract readonly ipFamily?: IpFamily;
public abstract readonly kubectlRole?: iam.IRole;
public abstract readonly kubectlLambdaRole?: iam.IRole;
public abstract readonly kubectlLayer: lambda.ILayerVersion;
public abstract readonly kubectlEnvironment?: { [key: string]: string };
public abstract readonly kubectlSecurityGroup?: ec2.ISecurityGroup;
public abstract readonly kubectlPrivateSubnets?: ec2.ISubnet[];
Expand Down Expand Up @@ -1474,9 +1468,8 @@ export class Cluster extends ClusterBase {
/**
* An AWS Lambda layer that includes `kubectl` and `helm`
*
* If not defined, a default layer will be used containing Kubectl 1.20 and Helm 3.8
*/
readonly kubectlLayer?: lambda.ILayerVersion;
readonly kubectlLayer: lambda.ILayerVersion;

/**
* An AWS Lambda layer that contains the `aws` CLI.
Expand Down Expand Up @@ -1578,9 +1571,6 @@ export class Cluster extends ClusterBase {
this.prune = props.prune ?? true;
this.vpc = props.vpc || new ec2.Vpc(this, 'DefaultVpc');

if (!props.kubectlLayer) {
Annotations.of(this).addWarningV2('@aws-cdk/aws-eks:clusterKubectlLayerNotSpecified', `You created a cluster with Kubernetes Version ${props.version.version} without specifying the kubectlLayer property. The property will become required instead of optional in 2025 Jan. Please update your CDK code to provide a kubectlLayer.`);
};
this.version = props.version;

// since this lambda role needs to be added to the trust policy of the creation role,
Expand Down Expand Up @@ -2402,7 +2392,7 @@ class ImportedCluster extends ClusterBase {
public readonly kubectlEnvironment?: { [key: string]: string } | undefined;
public readonly kubectlSecurityGroup?: ec2.ISecurityGroup | undefined;
public readonly kubectlPrivateSubnets?: ec2.ISubnet[] | undefined;
public readonly kubectlLayer?: lambda.ILayerVersion;
public readonly kubectlLayer: lambda.ILayerVersion;
public readonly ipFamily?: IpFamily;
public readonly awscliLayer?: lambda.ILayerVersion;
public readonly kubectlProvider?: IKubectlProvider;
Expand Down
3 changes: 1 addition & 2 deletions packages/aws-cdk-lib/aws-eks/lib/kubectl-provider.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import { Duration, Stack, NestedStack, Names, CfnCondition, Fn, Aws } from '../.
import { KubectlFunction } from '../../custom-resource-handlers/dist/aws-eks/kubectl-provider.generated';
import * as cr from '../../custom-resources';
import { AwsCliLayer } from '../../lambda-layer-awscli';
import { KubectlLayer } from '../../lambda-layer-kubectl';

/**
* Properties for a KubectlProvider
Expand Down Expand Up @@ -151,7 +150,7 @@ export class KubectlProvider extends NestedStack implements IKubectlProvider {

// allow user to customize the layers with the tools we need
handler.addLayers(props.cluster.awscliLayer ?? new AwsCliLayer(this, 'AwsCliLayer'));
handler.addLayers(props.cluster.kubectlLayer ?? new KubectlLayer(this, 'KubectlLayer'));
handler.addLayers(props.cluster.kubectlLayer);

this.handlerRole = handler.role!;

Expand Down
2 changes: 2 additions & 0 deletions packages/aws-cdk-lib/aws-eks/test/access-entry.test.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { KubectlV31Layer } from '@aws-cdk/lambda-layer-kubectl-v31';
import { Template } from '../../assertions';
import { App, Stack } from '../../core';
import {
Expand All @@ -18,6 +19,7 @@ describe('AccessEntry', () => {
cluster = new Cluster(stack, 'Cluster', {
version: KubernetesVersion.V1_29,
authenticationMode: AuthenticationMode.API,
kubectlLayer: new KubectlV31Layer(stack, 'KubectlLayer'),
});

mockAccessPolicies = [
Expand Down
4 changes: 3 additions & 1 deletion packages/aws-cdk-lib/aws-eks/test/addon.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { KubectlV31Layer } from '@aws-cdk/lambda-layer-kubectl-v31';
import { Template } from '../../assertions';
import { App, CfnOutput, Stack } from '../../core';
import { App, Stack } from '../../core';
import { Addon, KubernetesVersion, Cluster } from '../lib';

describe('Addon', () => {
Expand All @@ -12,6 +13,7 @@ describe('Addon', () => {
stack = new Stack(app, 'Stack');
cluster = new Cluster(stack, 'Cluster', {
version: KubernetesVersion.V1_30,
kubectlLayer: new KubectlV31Layer(stack, 'KubectlLayer'),
});
});

Expand Down
7 changes: 7 additions & 0 deletions packages/aws-cdk-lib/aws-eks/test/alb-controller.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import * as fs from 'fs';
import * as path from 'path';
import { KubectlV31Layer } from '@aws-cdk/lambda-layer-kubectl-v31';
import { testFixture } from './util';
import { Template, Match } from '../../assertions';
import * as iam from '../../aws-iam';
Expand All @@ -12,6 +13,7 @@ test.each(versions)('support AlbControllerVersion (%s)', (version) => {

const cluster = new Cluster(stack, 'Cluster', {
version: KubernetesVersion.V1_27,
kubectlLayer: new KubectlV31Layer(stack, 'KubectlLayer'),
});
AlbController.create(stack, {
cluster,
Expand Down Expand Up @@ -63,6 +65,7 @@ test('can configure a custom repository', () => {

const cluster = new Cluster(stack, 'Cluster', {
version: KubernetesVersion.V1_27,
kubectlLayer: new KubectlV31Layer(stack, 'KubectlLayer'),
});

AlbController.create(stack, {
Expand Down Expand Up @@ -96,6 +99,7 @@ test('throws when a policy is not defined for a custom version', () => {

const cluster = new Cluster(stack, 'Cluster', {
version: KubernetesVersion.V1_27,
kubectlLayer: new KubectlV31Layer(stack, 'KubectlLayer'),
});

expect(() => AlbController.create(stack, {
Expand All @@ -108,6 +112,7 @@ test.each(['us-gov-west-1', 'cn-north-1'])('stack does not include hard-coded pa
const { stack } = testFixture(region);
const cluster = new Cluster(stack, 'Cluster', {
version: KubernetesVersion.V1_27,
kubectlLayer: new KubectlV31Layer(stack, 'KubectlLayer'),
});

AlbController.create(stack, {
Expand All @@ -124,6 +129,7 @@ test('correct helm chart version is set for selected alb controller version', ()

const cluster = new Cluster(stack, 'Cluster', {
version: KubernetesVersion.V1_27,
kubectlLayer: new KubectlV31Layer(stack, 'KubectlLayer'),
});

AlbController.create(stack, {
Expand Down Expand Up @@ -159,6 +165,7 @@ describe('AlbController AwsAuth creation', () => {
const cluster = new Cluster(stack, 'Cluster', {
version: KubernetesVersion.V1_27,
authenticationMode,
kubectlLayer: new KubectlV31Layer(stack, 'KubectlLayer'),
});
AlbController.create(stack, {
cluster,
Expand Down
38 changes: 32 additions & 6 deletions packages/aws-cdk-lib/aws-eks/test/awsauth.test.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { KubectlV31Layer } from '@aws-cdk/lambda-layer-kubectl-v31';
import { testFixtureNoVpc } from './util';
import { Template } from '../../assertions';
import * as iam from '../../aws-iam';
Expand All @@ -15,7 +16,10 @@ describe('aws auth', () => {
const clusterStack = new cdk.Stack(app, 'ClusterStack');
const roleStack = new cdk.Stack(app, 'RoleStack');
const awsAuth = new AwsAuth(clusterStack, 'Auth', {
cluster: new Cluster(clusterStack, 'Cluster', { version: KubernetesVersion.V1_17 }),
cluster: new Cluster(clusterStack, 'Cluster', {
version: KubernetesVersion.V1_17,
kubectlLayer: new KubectlV31Layer(clusterStack, 'KubectlLayer'),
}),
});
const role = new iam.Role(roleStack, 'Role', { assumedBy: new iam.AnyPrincipal() });

Expand All @@ -31,7 +35,10 @@ describe('aws auth', () => {
const clusterStack = new cdk.Stack(app, 'ClusterStack');
const userStack = new cdk.Stack(app, 'UserStack');
const awsAuth = new AwsAuth(clusterStack, 'Auth', {
cluster: new Cluster(clusterStack, 'Cluster', { version: KubernetesVersion.V1_17 }),
cluster: new Cluster(clusterStack, 'Cluster', {
version: KubernetesVersion.V1_17,
kubectlLayer: new KubectlV31Layer(clusterStack, 'KubectlLayer'),
}),
});
const user = new iam.User(userStack, 'User');

Expand All @@ -45,7 +52,11 @@ describe('aws auth', () => {
test('empty aws-auth', () => {
// GIVEN
const { stack } = testFixtureNoVpc();
const cluster = new Cluster(stack, 'cluster', { version: CLUSTER_VERSION, prune: false });
const cluster = new Cluster(stack, 'cluster', {
version: CLUSTER_VERSION,
prune: false,
kubectlLayer: new KubectlV31Layer(stack, 'KubectlLayer'),
});

// WHEN
new AwsAuth(stack, 'AwsAuth', { cluster });
Expand All @@ -64,7 +75,11 @@ describe('aws auth', () => {
test('addRoleMapping and addUserMapping can be used to define the aws-auth ConfigMap', () => {
// GIVEN
const { stack } = testFixtureNoVpc();
const cluster = new Cluster(stack, 'Cluster', { version: CLUSTER_VERSION, prune: false });
const cluster = new Cluster(stack, 'Cluster', {
version: CLUSTER_VERSION,
prune: false,
kubectlLayer: new KubectlV31Layer(stack, 'KubectlLayer'),
});
const role = new iam.Role(stack, 'role', { assumedBy: new iam.AnyPrincipal() });
const user = new iam.User(stack, 'user');

Expand Down Expand Up @@ -142,7 +157,10 @@ describe('aws auth', () => {
test('imported users and roles can be also be used', () => {
// GIVEN
const { stack } = testFixtureNoVpc();
const cluster = new Cluster(stack, 'Cluster', { version: CLUSTER_VERSION });
const cluster = new Cluster(stack, 'Cluster', {
version: CLUSTER_VERSION,
kubectlLayer: new KubectlV31Layer(stack, 'KubectlLayer'),
});
const role = iam.Role.fromRoleArn(stack, 'imported-role', 'arn:aws:iam::123456789012:role/S3Access');
const user = iam.User.fromUserName(stack, 'import-user', 'MyUserName');

Expand Down Expand Up @@ -189,7 +207,11 @@ describe('aws auth', () => {
test('addMastersRole after addNodegroup correctly', () => {
// GIVEN
const { stack } = testFixtureNoVpc();
const cluster = new Cluster(stack, 'Cluster', { version: CLUSTER_VERSION, prune: false });
const cluster = new Cluster(stack, 'Cluster', {
version: CLUSTER_VERSION,
prune: false,
kubectlLayer: new KubectlV31Layer(stack, 'KubectlLayer'),
});
cluster.addNodegroupCapacity('NG');
const role = iam.Role.fromRoleArn(stack, 'imported-role', 'arn:aws:iam::123456789012:role/S3Access');

Expand Down Expand Up @@ -238,6 +260,7 @@ describe('aws auth', () => {
const cluster = new Cluster(clusterStack, 'Cluster', {
version: KubernetesVersion.V1_29,
authenticationMode: AuthenticationMode.API,
kubectlLayer: new KubectlV31Layer(clusterStack, 'KubectlLayer'),
});

// THEN
Expand All @@ -252,6 +275,7 @@ describe('aws auth', () => {
const cluster = new Cluster(clusterStack, 'Cluster', {
version: CLUSTER_VERSION,
authenticationMode: AuthenticationMode.API_AND_CONFIG_MAP,
kubectlLayer: new KubectlV31Layer(clusterStack, 'KubectlLayer'),
});

// THEN
Expand All @@ -266,6 +290,7 @@ describe('aws auth', () => {
const cluster = new Cluster(clusterStack, 'Cluster', {
version: CLUSTER_VERSION,
authenticationMode: AuthenticationMode.CONFIG_MAP,
kubectlLayer: new KubectlV31Layer(clusterStack, 'KubectlLayer'),
});

// THEN
Expand All @@ -279,6 +304,7 @@ describe('aws auth', () => {
const clusterStack = new cdk.Stack(app, 'test-stack');
const cluster = new Cluster(clusterStack, 'Cluster', {
version: CLUSTER_VERSION,
kubectlLayer: new KubectlV31Layer(clusterStack, 'KubectlLayer'),
});

// THEN
Expand Down
Loading
Loading