diff --git a/modules/aws-per-infra-iam/README.md b/modules/aws-per-infra-iam/README.md index 78d864c..88fbb72 100644 --- a/modules/aws-per-infra-iam/README.md +++ b/modules/aws-per-infra-iam/README.md @@ -51,6 +51,7 @@ recommended source. | `region` | The AWS region to deploy the BYOC infra into. | `string` | n/a | yes | | `external_id` | Unique identifier for role assumption, provided by ClickHouse. | `string` | n/a | yes | | `permissions_boundary` | ARN of an IAM policy to attach as the permissions boundary of every role this module creates. Set this if your organization mandates boundaries on every role. See the warning below. | `string` | `null` (no boundary) | no | +| `vpc_id` | ID of the VPC the BYOC infra is deployed into. Required when the VPC is owned by another account and shared into this one through AWS RAM. See below. | `string` | `null` (this account owns the VPC) | no | > `byoc_env` exists for internal ClickHouse use only. Leave it at its default > (`production`). See the note above. @@ -73,6 +74,15 @@ resources in AWS. AWS also does not accept a boundary on `eks-nodegroup` and `vpc-lattice`; an organization policy requiring a boundary on every `iam:Create*` call needs a carve-out for those. +## Shared VPC (AWS RAM) + +When the VPC is owned by another account (for example a network hub) and its +subnets are shared into this account through AWS RAM, set `vpc_id`. The subnets +of a shared VPC belong to the VPC owner, and IAM matches the owner's account ID +in subnet ARNs; the module looks the owner up from `vpc_id` so that Karpenter is +allowed to launch nodes into those subnets. Without it, Karpenter cannot launch +any nodes and ClickHouse services stay pending. + ## Outputs | Name | Description | diff --git a/modules/aws-per-infra-iam/main.tf b/modules/aws-per-infra-iam/main.tf index 3a4fb9c..303b7f0 100644 --- a/modules/aws-per-infra-iam/main.tf +++ b/modules/aws-per-infra-iam/main.tf @@ -4,6 +4,7 @@ locals { + vpc_owner_account_id = coalesce(one(data.aws_vpc.vpc[*].owner_id), data.aws_caller_identity.caller.account_id) byoc_account_map = { dev = 851725629656 staging = 767397831528 @@ -46,8 +47,18 @@ variable "permissions_boundary" { type = string default = null } + +variable "vpc_id" { + description = "ID of the VPC the BYOC infra is deployed into. Set it when the VPC is owned by another account and shared into this one through AWS RAM, so Karpenter is allowed to launch nodes into its subnets. Leave unset when this account owns the VPC." + type = string + default = null +} data "aws_caller_identity" "caller" { } +data "aws_vpc" "vpc" { + id = var.vpc_id + count = (var.vpc_id != null) ? 1 : 0 +} data "aws_iam_policy_document" "eks_pod_identity_assume_policy" { statement { actions = [ @@ -98,7 +109,7 @@ resource "aws_iam_role" "role_k8s_control_plane" { permissions_boundary = var.permissions_boundary tags = { clickhouse-byoc = "true" - version = "2.1.48-ffb5c32" + version = "2.1.53-a216b13" } } resource "aws_iam_role_policy_attachment" "managed_policy_k8s_control_plane_amazon_eks_cluster_policy" { @@ -155,7 +166,7 @@ resource "aws_iam_role" "role_k8s_worker" { permissions_boundary = var.permissions_boundary tags = { clickhouse-byoc = "true" - version = "2.1.48-ffb5c32" + version = "2.1.53-a216b13" } } resource "aws_iam_role_policy_attachment" "managed_policy_k8s_worker_amazon_eks_worker_node_policy" { @@ -192,7 +203,7 @@ resource "aws_iam_role" "role_load_balancer_controller" { permissions_boundary = var.permissions_boundary tags = { clickhouse-byoc = "true" - version = "2.1.48-ffb5c32" + version = "2.1.53-a216b13" } } data "aws_iam_policy_document" "inline_policy_load_balancer_controller_lb_controller_iam_policy" { @@ -493,7 +504,7 @@ resource "aws_iam_role" "role_ebs_csi_driver" { permissions_boundary = var.permissions_boundary tags = { clickhouse-byoc = "true" - version = "2.1.48-ffb5c32" + version = "2.1.53-a216b13" } } data "aws_iam_policy_document" "inline_policy_ebs_csi_driver_ebscsi_driver_policy" { @@ -759,7 +770,7 @@ resource "aws_iam_role" "role_cluster_autoscaler" { permissions_boundary = var.permissions_boundary tags = { clickhouse-byoc = "true" - version = "2.1.48-ffb5c32" + version = "2.1.53-a216b13" } } data "aws_iam_policy_document" "inline_policy_cluster_autoscaler_eks_autoscaler_policy" { @@ -789,10 +800,16 @@ resource "aws_iam_role_policy" "role_policy_cluster_autoscaler_eks_autoscaler_po resource "aws_iam_role" "role_karpenter_controller" { assume_role_policy = data.aws_iam_policy_document.eks_pod_identity_assume_policy.json name = "${var.spoken_name}-${var.region}-karpenter-controller" + path = "/clickhouse/" permissions_boundary = var.permissions_boundary tags = { clickhouse-byoc = "true" - version = "2.1.48-ffb5c32" + version = "2.1.53-a216b13" + } + lifecycle { + ignore_changes = [ + "path", + ] } } data "aws_iam_policy_document" "inline_policy_karpenter_controller_karpenter_controller_policy" { @@ -807,7 +824,7 @@ data "aws_iam_policy_document" "inline_policy_karpenter_controller_karpenter_con "arn:aws:ec2:${var.region}::snapshot/*", "arn:aws:ec2:${var.region}:${data.aws_caller_identity.caller.account_id}:spot-instances-request/*", "arn:aws:ec2:${var.region}:${data.aws_caller_identity.caller.account_id}:security-group/*", - "arn:aws:ec2:${var.region}:${data.aws_caller_identity.caller.account_id}:subnet/*", + "arn:aws:ec2:${var.region}:${local.vpc_owner_account_id}:subnet/*", "arn:aws:ec2:${var.region}:${data.aws_caller_identity.caller.account_id}:capacity-reservation/*", "arn:aws:ec2:${var.region}:${data.aws_caller_identity.caller.account_id}:placement-group/*" ] @@ -1170,7 +1187,7 @@ resource "aws_iam_role" "role_karpenter_node" { permissions_boundary = var.permissions_boundary tags = { clickhouse-byoc = "true" - version = "2.1.48-ffb5c32" + version = "2.1.53-a216b13" } } resource "aws_iam_instance_profile" "instance_profile_karpenter_node" { @@ -1178,7 +1195,7 @@ resource "aws_iam_instance_profile" "instance_profile_karpenter_node" { role = aws_iam_role.role_karpenter_node.name tags = { clickhouse-byoc = "true" - version = "2.1.48-ffb5c32" + version = "2.1.53-a216b13" } } resource "aws_iam_role_policy_attachment" "managed_policy_karpenter_node_amazon_eks_worker_node_policy" { @@ -1219,7 +1236,7 @@ resource "aws_iam_role" "role_state_exporter" { permissions_boundary = var.permissions_boundary tags = { clickhouse-byoc = "true" - version = "2.1.48-ffb5c32" + version = "2.1.53-a216b13" } } data "aws_iam_policy_document" "inline_policy_state_exporter_state_exporter_policy" { @@ -1245,7 +1262,7 @@ resource "aws_iam_role" "role_thanos" { permissions_boundary = var.permissions_boundary tags = { clickhouse-byoc = "true" - version = "2.1.48-ffb5c32" + version = "2.1.53-a216b13" } } data "aws_iam_policy_document" "inline_policy_thanos_thanos_policy" { @@ -1275,7 +1292,7 @@ resource "aws_iam_role" "role_clickhouse_scraper" { permissions_boundary = var.permissions_boundary tags = { clickhouse-byoc = "true" - version = "2.1.48-ffb5c32" + version = "2.1.53-a216b13" } } data "aws_iam_policy_document" "inline_policy_clickhouse_scraper_scraper_billing_bucket_assume_role_policy" { @@ -1298,10 +1315,16 @@ resource "aws_iam_role_policy" "role_policy_clickhouse_scraper_scraper_billing_b resource "aws_iam_role" "role_kube_metric_forwarder_asc" { assume_role_policy = data.aws_iam_policy_document.eks_pod_identity_assume_policy.json name = "${var.spoken_name}-${var.region}-kube-metric-forwarder-asc" + path = "/clickhouse/" permissions_boundary = var.permissions_boundary tags = { clickhouse-byoc = "true" - version = "2.1.48-ffb5c32" + version = "2.1.53-a216b13" + } + lifecycle { + ignore_changes = [ + "path", + ] } } data "aws_iam_policy_document" "inline_policy_kube_metric_forwarder_asc_kube_metric_forwarder_autoscale_bucket_assume_role_policy" { @@ -1327,7 +1350,7 @@ resource "aws_iam_role" "role_clickhouse_s3_access" { permissions_boundary = var.permissions_boundary tags = { clickhouse-byoc = "true" - version = "2.1.48-ffb5c32" + version = "2.1.53-a216b13" } } data "aws_iam_policy_document" "billing_bucket_assume_role_policy" { @@ -1343,6 +1366,31 @@ data "aws_iam_policy_document" "billing_bucket_assume_role_policy" { ] sid = "AllowAssumeroleRemote" } + statement { + actions = [ + "sts:AssumeRole", + "sts:TagSession" + ] + effect = "Allow" + resources = [ + "arn:aws:iam::*:role/*" + ] + sid = "AssumeRoleInOtherAccounts" + condition { + test = "StringNotEquals" + values = [ + "$${aws:PrincipalAccount}" + ] + variable = "aws:ResourceAccount" + } + condition { + test = "Null" + values = [ + "false" + ] + variable = "aws:ResourceAccount" + } + } } resource "aws_iam_role_policy" "role_policy_clickhouse_s3_access_billing_bucket_assume_role" { name = "BillingBucketAssumeRolePolicy" @@ -1431,7 +1479,7 @@ resource "aws_iam_role" "role_data_plane_management" { permissions_boundary = var.permissions_boundary tags = { clickhouse-byoc = "true" - version = "2.1.48-ffb5c32" + version = "2.1.53-a216b13" } } data "aws_iam_policy_document" "data_plane_management_policy" { @@ -1534,6 +1582,13 @@ data "aws_iam_policy_document" "tde_delegate_assume_role_policy" { ] effect = "Allow" sid = "ByocTDESharedPodRole" + condition { + test = "StringEquals" + values = [ + "${var.external_id}" + ] + variable = "sts:ExternalId" + } principals { identifiers = [ "arn:aws:iam::${data.aws_caller_identity.caller.account_id}:role/${var.spoken_name}-${var.region}-CH-S3-Role" @@ -1548,7 +1603,7 @@ resource "aws_iam_role" "role_tde_delegate" { permissions_boundary = var.permissions_boundary tags = { clickhouse-byoc = "true" - version = "2.1.48-ffb5c32" + version = "2.1.53-a216b13" } depends_on = [ "aws_iam_role.role_data_plane_management", diff --git a/modules/aws/main.tf b/modules/aws/main.tf index 6a3263b..9f37be3 100644 --- a/modules/aws/main.tf +++ b/modules/aws/main.tf @@ -95,7 +95,7 @@ resource "aws_iam_role" "clickhouse_management_role" { permissions_boundary = var.permissions_boundary tags = { clickhouse-byoc = "true" - version = "2.1.48-c74529e" + version = "2.1.53-6889cac" } } data "aws_iam_policy_document" "base_policy" {