Feat: Add Broker Node Security Groups

README.md

@@ -111,11 +111,15 @@ module "kafka" {
   name                   = "app"
   vpc_id                 = "vpc-XXXXXXXX"
   zone_id                = "Z14EN2YD427LRQ"
-  security_groups        = ["sg-XXXXXXXXX", "sg-YYYYYYYY"]
   subnet_ids             = ["subnet-XXXXXXXXX", "subnet-YYYYYYYY"]
   kafka_version          = "2.4.1"
   number_of_broker_nodes = 2 # this has to be a multiple of the # of subnet_ids
   broker_instance_type   = "kafka.m5.large"
+
+  # security groups to put on the cluster itself
+  broker_node_security_groups = ["sg-XXXXXXXXX", "sg-YYYYYYYY"]
+  # security groups to give access to the cluster
+  security_groups             = ["sg-XXXXXXXXX", "sg-YYYYYYYY"]
 }
 ```
 
@@ -146,7 +150,7 @@ Available targets:
 
 | Name | Version |
 |------|---------|
-| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 0.13.0 |
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 0.14.0 |
 | <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 3.0 |
 | <a name="requirement_random"></a> [random](#requirement\_random) | >= 2.2 |
 
@@ -160,6 +164,7 @@ Available targets:
 
 | Name | Source | Version |
 |------|--------|---------|
+| <a name="module_broker_security_group"></a> [broker\_security\_group](#module\_broker\_security\_group) | cloudposse/security-group/aws | 0.4.2 |
 | <a name="module_hostname"></a> [hostname](#module\_hostname) | cloudposse/route53-cluster-hostname/aws | 0.12.2 |
 | <a name="module_this"></a> [this](#module\_this) | cloudposse/label/null | 0.25.0 |
 
@@ -172,10 +177,6 @@ Available targets:
 | [aws_msk_cluster.default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/msk_cluster) | resource |
 | [aws_msk_configuration.config](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/msk_configuration) | resource |
 | [aws_msk_scram_secret_association.default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/msk_scram_secret_association) | resource |
-| [aws_security_group.default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource |
-| [aws_security_group_rule.egress](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource |
-| [aws_security_group_rule.ingress_cidr_blocks](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource |
-| [aws_security_group_rule.ingress_security_groups](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource |
 
 ## Inputs
 
@@ -185,6 +186,7 @@ Available targets:
 | <a name="input_allowed_cidr_blocks"></a> [allowed\_cidr\_blocks](#input\_allowed\_cidr\_blocks) | List of CIDR blocks to be allowed to connect to the cluster | `list(string)` | `[]` | no |
 | <a name="input_attributes"></a> [attributes](#input\_attributes) | ID element. Additional attributes (e.g. `workers` or `cluster`) to add to `id`,<br>in the order they appear in the list. New attributes are appended to the<br>end of the list. The elements of the list are joined by the `delimiter`<br>and treated as a single ID element. | `list(string)` | `[]` | no |
 | <a name="input_broker_instance_type"></a> [broker\_instance\_type](#input\_broker\_instance\_type) | The instance type to use for the Kafka brokers | `string` | n/a | yes |
+| <a name="input_broker_node_security_groups"></a> [broker\_node\_security\_groups](#input\_broker\_node\_security\_groups) | List of broker node security group IDs to be associated with the elastic network interfaces to control who can communicate with the cluster | `list(string)` | `[]` | no |
 | <a name="input_broker_volume_size"></a> [broker\_volume\_size](#input\_broker\_volume\_size) | The size in GiB of the EBS volume for the data drive on each broker node | `number` | `1000` | no |
 | <a name="input_certificate_authority_arns"></a> [certificate\_authority\_arns](#input\_certificate\_authority\_arns) | List of ACM Certificate Authority Amazon Resource Names (ARNs) to be used for TLS client authentication | `list(string)` | `[]` | no |
 | <a name="input_client_broker"></a> [client\_broker](#input\_client\_broker) | Encryption setting for data in transit between clients and brokers. Valid values: `TLS`, `TLS_PLAINTEXT`, and `PLAINTEXT` | `string` | `"TLS"` | no |

README.yaml

@@ -86,11 +86,15 @@ usage: |-
     name                   = "app"
     vpc_id                 = "vpc-XXXXXXXX"
     zone_id                = "Z14EN2YD427LRQ"
-    security_groups        = ["sg-XXXXXXXXX", "sg-YYYYYYYY"]
     subnet_ids             = ["subnet-XXXXXXXXX", "subnet-YYYYYYYY"]
     kafka_version          = "2.4.1"
     number_of_broker_nodes = 2 # this has to be a multiple of the # of subnet_ids
     broker_instance_type   = "kafka.m5.large"
+
+    # security groups to put on the cluster itself
+    broker_node_security_groups = ["sg-XXXXXXXXX", "sg-YYYYYYYY"]
+    # security groups to give access to the cluster
+    security_groups             = ["sg-XXXXXXXXX", "sg-YYYYYYYY"]
   }
   ```
 

docs/terraform.md

@@ -3,7 +3,7 @@
 
 | Name | Version |
 |------|---------|
-| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 0.13.0 |
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 0.14.0 |
 | <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 3.0 |
 | <a name="requirement_random"></a> [random](#requirement\_random) | >= 2.2 |
 
@@ -17,6 +17,7 @@
 
 | Name | Source | Version |
 |------|--------|---------|
+| <a name="module_broker_security_group"></a> [broker\_security\_group](#module\_broker\_security\_group) | cloudposse/security-group/aws | 0.4.2 |
 | <a name="module_hostname"></a> [hostname](#module\_hostname) | cloudposse/route53-cluster-hostname/aws | 0.12.2 |
 | <a name="module_this"></a> [this](#module\_this) | cloudposse/label/null | 0.25.0 |
 
@@ -29,10 +30,6 @@
 | [aws_msk_cluster.default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/msk_cluster) | resource |
 | [aws_msk_configuration.config](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/msk_configuration) | resource |
 | [aws_msk_scram_secret_association.default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/msk_scram_secret_association) | resource |
-| [aws_security_group.default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource |
-| [aws_security_group_rule.egress](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource |
-| [aws_security_group_rule.ingress_cidr_blocks](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource |
-| [aws_security_group_rule.ingress_security_groups](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource |
 
 ## Inputs
 
@@ -42,6 +39,7 @@
 | <a name="input_allowed_cidr_blocks"></a> [allowed\_cidr\_blocks](#input\_allowed\_cidr\_blocks) | List of CIDR blocks to be allowed to connect to the cluster | `list(string)` | `[]` | no |
 | <a name="input_attributes"></a> [attributes](#input\_attributes) | ID element. Additional attributes (e.g. `workers` or `cluster`) to add to `id`,<br>in the order they appear in the list. New attributes are appended to the<br>end of the list. The elements of the list are joined by the `delimiter`<br>and treated as a single ID element. | `list(string)` | `[]` | no |
 | <a name="input_broker_instance_type"></a> [broker\_instance\_type](#input\_broker\_instance\_type) | The instance type to use for the Kafka brokers | `string` | n/a | yes |
+| <a name="input_broker_node_security_groups"></a> [broker\_node\_security\_groups](#input\_broker\_node\_security\_groups) | List of broker node security group IDs to be associated with the elastic network interfaces to control who can communicate with the cluster | `list(string)` | `[]` | no |
 | <a name="input_broker_volume_size"></a> [broker\_volume\_size](#input\_broker\_volume\_size) | The size in GiB of the EBS volume for the data drive on each broker node | `number` | `1000` | no |
 | <a name="input_certificate_authority_arns"></a> [certificate\_authority\_arns](#input\_certificate\_authority\_arns) | List of ACM Certificate Authority Amazon Resource Names (ARNs) to be used for TLS client authentication | `list(string)` | `[]` | no |
 | <a name="input_client_broker"></a> [client\_broker](#input\_client\_broker) | Encryption setting for data in transit between clients and brokers. Valid values: `TLS`, `TLS_PLAINTEXT`, and `PLAINTEXT` | `string` | `"TLS"` | no |

examples/complete/versions.tf

@@ -1,21 +1,14 @@
 terraform {
-  required_version = ">= 0.13.0"
+  required_version = ">= 0.14.0"
 
   required_providers {
     aws = {
       source  = "hashicorp/aws"
       version = ">= 3.0"
     }
-    template = {
-      source  = "hashicorp/template"
-      version = ">= 2.0"
-    }
-    local = {
-      source  = "hashicorp/local"
-      version = ">= 1.3"
-    }
     random = {
-      source = "hashicorp/random"
+      source  = "hashicorp/random"
+      version = ">= 2.2"
     }
   }
 }

main.tf

@@ -12,47 +12,81 @@ locals {
   bootstrap_brokers_combined_list = concat(local.bootstrap_brokers_list, local.bootstrap_brokers_tls_list, local.bootstrap_brokers_scram_list, local.bootstrap_brokers_iam_list)
   # If var.storage_autoscaling_max_capacity is not set, don't autoscale past current size
   broker_volume_size_max = coalesce(var.storage_autoscaling_max_capacity, var.broker_volume_size)
-}
-
-resource "aws_security_group" "default" {
-  count       = local.enabled ? 1 : 0
-  vpc_id      = var.vpc_id
-  name        = module.this.id
-  description = "Allow inbound traffic from Security Groups and CIDRs. Allow all outbound traffic"
-  tags        = module.this.tags
-}
 
-resource "aws_security_group_rule" "ingress_security_groups" {
-  count                    = local.enabled ? length(var.security_groups) : 0
-  description              = "Allow inbound traffic from Security Groups"
-  type                     = "ingress"
-  from_port                = 0
-  to_port                  = 65535
-  protocol                 = "tcp"
-  source_security_group_id = var.security_groups[count.index]
-  security_group_id        = join("", aws_security_group.default.*.id)
+  # var.client_broker types
+  plaintext     = "PLAINTEXT"
+  tls_plaintext = "TLS_PLAINTEXT"
+  tls           = "TLS"
+
+  # The following ports are not configurable. See: https://docs.aws.amazon.com/msk/latest/developerguide/client-access.html#port-info
+  protocols = {
+    plaintext = {
+      name = "plaintext"
+      # https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/msk_cluster#bootstrap_brokers
+      enabled = contains([local.plaintext, local.tls_plaintext], var.client_broker)
+      port    = 9092
+    }
+    tls = {
+      name = "TLS"
+      # https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/msk_cluster#bootstrap_brokers_tls
+      enabled = contains([local.tls_plaintext, local.tls], var.client_broker)
+      port    = 9094
+    }
+    sasl_scram = {
+      name = "SASL/SCRAM"
+      # https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/msk_cluster#bootstrap_brokers_sasl_scram
+      enabled = var.client_sasl_scram_enabled && contains([local.tls_plaintext, local.tls], var.client_broker)
+      port    = 9096
+    }
+    sasl_iam = {
+      name = "SASL/IAM"
+      # https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/msk_cluster#bootstrap_brokers_sasl_iam
+      enabled = var.client_sasl_iam_enabled && contains([local.tls_plaintext, local.tls], var.client_broker)
+      port    = 9098
+    }
+    # The following two protocols are always enabled.
+    # See: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/msk_cluster#zookeeper_connect_string
+    # and https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/msk_cluster#zookeeper_connect_string_tls
+    zookeeper_plaintext = {
+      name    = "Zookeeper plaintext"
+      enabled = true
+      port    = 2181
+    }
+    zookeeper_tls = {
+      name    = "Zookeeper TLS"
+      enabled = true
+      port    = 2182
+    }
+  }
 }
 
-resource "aws_security_group_rule" "ingress_cidr_blocks" {
-  count             = local.enabled && length(var.allowed_cidr_blocks) > 0 ? 1 : 0
-  description       = "Allow inbound traffic from CIDR blocks"
-  type              = "ingress"
-  from_port         = 0
-  to_port           = 65535
-  protocol          = "tcp"
-  cidr_blocks       = var.allowed_cidr_blocks
-  security_group_id = join("", aws_security_group.default.*.id)
-}
+module "broker_security_group" {
+  source  = "cloudposse/security-group/aws"
+  version = "0.4.2"
+
+  attributes = ["broker"]
+
+  security_group_description = "Allow inbound MSK-related traffic from Security Groups and CIDRs. Allow all outbound traffic"
+  allow_all_egress           = true
+  rule_matrix = [
+    {
+      source_security_group_ids = var.security_groups
+      cidr_blocks               = var.allowed_cidr_blocks
+      rules = [
+        for protocol_key, protocol in local.protocols : {
+          key         = protocol_key
+          type        = "ingress"
+          from_port   = protocol.port
+          to_port     = protocol.port
+          protocol    = "tcp"
+          description = "Allow inbound ${protocol.name} traffic"
+        } if protocol.enabled
+      ]
+    }
+  ]
+  vpc_id = var.vpc_id
 
-resource "aws_security_group_rule" "egress" {
-  count             = local.enabled ? 1 : 0
-  description       = "Allow all egress traffic"
-  type              = "egress"
-  from_port         = 0
-  to_port           = 65535
-  protocol          = "tcp"
-  cidr_blocks       = ["0.0.0.0/0"]
-  security_group_id = join("", aws_security_group.default.*.id)
+  context = module.this.context
 }
 
 resource "aws_msk_configuration" "config" {
@@ -65,6 +99,7 @@ resource "aws_msk_configuration" "config" {
 }
 
 resource "aws_msk_cluster" "default" {
+  #bridgecrew:skip=BC_AWS_LOGGING_18:Skipping `Amazon MSK cluster logging is not enabled` check since it can be enabled with cloudwatch_logs_enabled = true
   count                  = local.enabled ? 1 : 0
   cluster_name           = module.this.id
   kafka_version          = var.kafka_version
@@ -75,7 +110,7 @@ resource "aws_msk_cluster" "default" {
     instance_type   = var.broker_instance_type
     ebs_volume_size = var.broker_volume_size
     client_subnets  = var.subnet_ids
-    security_groups = aws_security_group.default.*.id
+    security_groups = concat(var.broker_node_security_groups, [module.broker_security_group.id])
   }
 
   configuration_info {

outputs.tf

@@ -55,10 +55,10 @@ output "cluster_name" {
 
 output "security_group_id" {
   description = "The ID of the security group rule"
-  value       = join("", aws_security_group.default.*.id)
+  value       = module.broker_security_group.id
 }
 
 output "security_group_name" {
   description = "The name of the security group rule"
-  value       = join("", aws_security_group.default.*.name)
+  value       = module.broker_security_group.name
 }

variables.tf

@@ -41,6 +41,12 @@ variable "security_groups" {
   description = "List of security group IDs to be allowed to connect to the cluster"
 }
 
+variable "broker_node_security_groups" {
+  type        = list(string)
+  default     = []
+  description = "List of broker node security group IDs to be associated with the elastic network interfaces to control who can communicate with the cluster"
+}
+
 variable "allowed_cidr_blocks" {
   type        = list(string)
   default     = []

versions.tf

@@ -1,5 +1,5 @@
 terraform {
-  required_version = ">= 0.13.0"
+  required_version = ">= 0.14.0"
 
   required_providers {
     aws = {