Skip to content

Commit

Permalink
feat: use operator
Browse files Browse the repository at this point in the history
  • Loading branch information
johanneswuerbach committed Jun 13, 2024
1 parent 727d51a commit 82da85d
Show file tree
Hide file tree
Showing 18 changed files with 382 additions and 39 deletions.
4 changes: 3 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -221,8 +221,10 @@ Once you are finished with the reference architecture, you can remove all provis
| github | ~> 5.38 |
| helm | ~> 2.12 |
| humanitec | ~> 1.0 |
| kubectl | ~> 2.0 |
| kubernetes | ~> 2.25 |
| random | ~> 3.5 |
| tls | ~> 4.0 |

### Providers

Expand Down Expand Up @@ -254,7 +256,7 @@ Once you are finished with the reference architecture, you can remove all provis
| aws\_region | AWS region | `string` | n/a | yes |
| disk\_size | Disk size in GB to use for EKS nodes | `number` | `20` | no |
| github\_org\_id | GitHub org id (required for Backstage) | `string` | `null` | no |
| humanitec\_org\_id | Humanitec Organization ID (required for Backstage) | `string` | `null` | no |
| humanitec\_org\_id | Humanitec Organization ID | `string` | `null` | no |
| instance\_types | List of EC2 instances types to use for EKS nodes | `list(string)` | <pre>[<br> "t3.large"<br>]</pre> | no |
| with\_backstage | Deploy Backstage | `bool` | `false` | no |
<!-- END_TF_DOCS -->
Expand Down
9 changes: 6 additions & 3 deletions main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,11 @@
module "base" {
source = "./modules/base"

region = var.aws_region
instance_types = var.instance_types
disk_size = var.disk_size
humanitec_org_id = var.humanitec_org_id
aws_account_id = var.aws_account_id
aws_region = var.aws_region
instance_types = var.instance_types
disk_size = var.disk_size
}

# User used for scaffolding and deploying apps
Expand Down Expand Up @@ -62,6 +64,7 @@ module "portal_backstage" {

humanitec_org_id = var.humanitec_org_id
humanitec_ci_service_user_token = humanitec_service_user_token.deployer[0].token
humanitec_secret_store_id = module.base.humanitec_secret_store_id

github_org_id = var.github_org_id
github_app_client_id = module.github_app[0].client_id
Expand Down
36 changes: 31 additions & 5 deletions modules/base/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,21 +10,25 @@ Module that provides the reference architecture.
| Name | Version |
|------|---------|
| terraform | >= 1.3.0 |
| aws | >= 4.50 |
| aws | ~> 5.17 |
| helm | ~> 2.12 |
| humanitec | ~> 1.0 |
| kubernetes | >= 2.0.3 |
| kubectl | ~> 2.0 |
| kubernetes | ~> 2.0 |
| random | ~> 3.5 |
| tls | ~> 4.0 |

### Providers

| Name | Version |
|------|---------|
| aws | >= 4.50 |
| aws | ~> 5.17 |
| helm | ~> 2.12 |
| humanitec | ~> 1.0 |
| kubernetes | >= 2.0.3 |
| kubectl | ~> 2.0 |
| kubernetes | ~> 2.0 |
| random | ~> 3.5 |
| tls | ~> 4.0 |

### Modules

Expand All @@ -40,18 +44,37 @@ Module that provides the reference architecture.

| Name | Type |
|------|------|
| [aws_eks_pod_identity_association.humanitec_operator](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_pod_identity_association) | resource |
| [aws_iam_policy.humanitec_operator](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
| [aws_iam_role.humanitec_operator](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
| [aws_iam_role.humanitec_svc](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
| [aws_iam_role_policy_attachment.humanitec_operator](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
| [aws_iam_role_policy_attachment.humanitec_svc](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
| [helm_release.humanitec_agent](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
| [helm_release.humanitec_operator](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
| [helm_release.ingress_nginx](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
| [humanitec_agent.agent](https://registry.terraform.io/providers/humanitec/humanitec/latest/docs/resources/agent) | resource |
| [humanitec_key.operator_public_key](https://registry.terraform.io/providers/humanitec/humanitec/latest/docs/resources/key) | resource |
| [humanitec_resource_account.cluster_account](https://registry.terraform.io/providers/humanitec/humanitec/latest/docs/resources/resource_account) | resource |
| [humanitec_resource_definition.agent](https://registry.terraform.io/providers/humanitec/humanitec/latest/docs/resources/resource_definition) | resource |
| [humanitec_resource_definition.k8s_cluster_driver](https://registry.terraform.io/providers/humanitec/humanitec/latest/docs/resources/resource_definition) | resource |
| [humanitec_resource_definition.k8s_namespace](https://registry.terraform.io/providers/humanitec/humanitec/latest/docs/resources/resource_definition) | resource |
| [humanitec_resource_definition_criteria.agent](https://registry.terraform.io/providers/humanitec/humanitec/latest/docs/resources/resource_definition_criteria) | resource |
| [humanitec_resource_definition_criteria.default_mysql](https://registry.terraform.io/providers/humanitec/humanitec/latest/docs/resources/resource_definition_criteria) | resource |
| [humanitec_resource_definition_criteria.default_postgres](https://registry.terraform.io/providers/humanitec/humanitec/latest/docs/resources/resource_definition_criteria) | resource |
| [humanitec_resource_definition_criteria.k8s_cluster_driver](https://registry.terraform.io/providers/humanitec/humanitec/latest/docs/resources/resource_definition_criteria) | resource |
| [humanitec_resource_definition_criteria.k8s_namespace](https://registry.terraform.io/providers/humanitec/humanitec/latest/docs/resources/resource_definition_criteria) | resource |
| [humanitec_secretstore.main](https://registry.terraform.io/providers/humanitec/humanitec/latest/docs/resources/secretstore) | resource |
| [kubectl_manifest.humanitec_operator_secret_store](https://registry.terraform.io/providers/alekc/kubectl/latest/docs/resources/manifest) | resource |
| [kubernetes_namespace.agent-namespace](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/namespace) | resource |
| [kubernetes_namespace.humanitec_operator](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/namespace) | resource |
| [kubernetes_secret.humanitec_operator](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/secret) | resource |
| [random_password.external_id](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/password) | resource |
| [tls_private_key.agent_private_key](https://registry.terraform.io/providers/hashicorp/tls/latest/docs/resources/private_key) | resource |
| [tls_private_key.operator_private_key](https://registry.terraform.io/providers/hashicorp/tls/latest/docs/resources/private_key) | resource |
| [aws_elb_hosted_zone_id.main](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/elb_hosted_zone_id) | data source |
| [aws_iam_policy_document.assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
| [aws_iam_policy_document.humanitec_operator](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
| [aws_iam_policy_document.instance_assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source |
| [kubernetes_service.ingress_nginx_controller](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/data-sources/service) | data source |
Expand All @@ -60,7 +83,10 @@ Module that provides the reference architecture.

| Name | Description | Type | Default | Required |
|------|-------------|------|---------|:--------:|
| aws\_account\_id | AWS Account (ID) to use | `string` | n/a | yes |
| humanitec\_org\_id | Humanitec Organization ID | `string` | n/a | yes |
| additional\_k8s\_access\_entries | Additional access entries add to the k8s aws-auth configmap | <pre>list(object({<br> id = string<br> principal_arn = string<br> groups = list(string)<br> }))</pre> | `[]` | no |
| aws\_region | AWS Region to deploy into | `string` | `"us-east-1"` | no |
| capacity\_type | Defines whether to use ON\_DEMAND or SPOT EC2 instances for EKS nodes | `string` | `"ON_DEMAND"` | no |
| cluster\_name | Name for the EKS cluster | `string` | `"ref-arch"` | no |
| cluster\_version | Version of the EKS cluster to deploy | `string` | `null` | no |
Expand All @@ -74,7 +100,6 @@ Module that provides the reference architecture.
| node\_group\_desired\_size | Desired number of nodes for the EKS node group | `number` | `3` | no |
| node\_group\_max\_size | Maximum number of nodes for the EKS node group | `number` | `3` | no |
| node\_group\_min\_size | Minimum number of nodes for the EKS node group | `number` | `2` | no |
| region | AWS Region to deploy into | `string` | `"us-east-1"` | no |
| vpc\_name | AWS VPC name | `string` | `"ref-arch"` | no |

### Outputs
Expand All @@ -88,6 +113,7 @@ Module that provides the reference architecture.
| eks\_oidc\_provider\_arn | The ARN of the OIDC Provider |
| environment | Name of the environment to be deployed into |
| humanitec\_resource\_account\_id | Humanitec resource account id for the cluster |
| humanitec\_secret\_store\_id | Humanitec secret store id |
| ingress\_nginx\_external\_dns | External DNS entry for the Nginx ingress controller |
| vpc\_id | VPC id |
<!-- END_TF_DOCS -->
81 changes: 81 additions & 0 deletions modules/base/humanitec-agent.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
# Installs the humanitec-agent into the cluster

# More details https://developer.humanitec.com/integration-and-extensions/humanitec-agent/overview/

resource "kubernetes_namespace" "agent-namespace" {
metadata {
labels = {
"app.kubernetes.io/name" = "humanitec-operator"
"app.kubernetes.io/instance" = "humanitec-operator"
}

name = "humanitec-agent"
}
}

resource "tls_private_key" "agent_private_key" {
algorithm = "RSA"
rsa_bits = 4096
}

locals {
agent_id = "${local.res_def_prefix}agent"
}

resource "humanitec_agent" "agent" {
id = local.agent_id
description = "reference-architecture-aws"
public_keys = [{
key = tls_private_key.agent_private_key.public_key_pem
}]
}

resource "helm_release" "humanitec_agent" {
name = "humanitec-agent"
namespace = kubernetes_namespace.agent-namespace.id

repository = "oci://ghcr.io/humanitec/charts"
chart = "humanitec-agent"
version = "1.1.0"
wait = true
timeout = 300

set {
name = "humanitec.org"
value = var.humanitec_org_id
}

set {
name = "humanitec.privateKey"
value = tls_private_key.agent_private_key.private_key_pem
}

depends_on = [
humanitec_agent.agent
]
}

resource "humanitec_resource_definition" "agent" {
id = local.agent_id
name = local.agent_id
type = "agent"

driver_type = "humanitec/agent"
driver_inputs = {
values_string = jsonencode({
id = local.agent_id
})
}

depends_on = [
helm_release.humanitec_agent
]
}

resource "humanitec_resource_definition_criteria" "agent" {
resource_definition_id = humanitec_resource_definition.agent.id
res_id = "agent"
env_type = var.environment

force_delete = true
}
138 changes: 138 additions & 0 deletions modules/base/humanitec-operator.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,138 @@
# Installs the humanitec-operator into the cluster

# More details https://developer.humanitec.com/integration-and-extensions/humanitec-operator/overview/

resource "kubernetes_namespace" "humanitec_operator" {
metadata {
labels = {
"app.kubernetes.io/name" = "humanitec-operator"
"app.kubernetes.io/instance" = "humanitec-operator"
}

name = "humanitec-operator"
}
}


resource "tls_private_key" "operator_private_key" {
algorithm = "RSA"
rsa_bits = 4096
}

resource "humanitec_key" "operator_public_key" {
key = tls_private_key.operator_private_key.public_key_pem
}

resource "kubernetes_secret" "humanitec_operator" {
metadata {
name = "humanitec-operator-private-key"
namespace = kubernetes_namespace.humanitec_operator.id
}

data = {
privateKey = tls_private_key.operator_private_key.private_key_pem
humanitecOrganisationID = var.humanitec_org_id
}
}

resource "helm_release" "humanitec_operator" {
name = "humanitec-operator"
namespace = kubernetes_namespace.humanitec_operator.id

repository = "oci://ghcr.io/humanitec/charts"
chart = "humanitec-operator"
version = "0.2.4"
wait = true
timeout = 300

depends_on = [
humanitec_key.operator_public_key,
kubernetes_secret.humanitec_operator
]
}

# Configure the operator to be able to store secrets

locals {
humanitec_operator_k8s_sa_name = "humanitec-operator-controller-manager"
}

data "aws_iam_policy_document" "assume_role_policy" {
version = "2012-10-17"

statement {
actions = ["sts:AssumeRole", "sts:TagSession"]

principals {
type = "Service"
identifiers = ["pods.eks.amazonaws.com"]
}
}
}

resource "aws_iam_role" "humanitec_operator" {
name = "humanitec-operator"
description = "Humanitec Operator EKS service account"
assume_role_policy = data.aws_iam_policy_document.assume_role_policy.json
}

resource "aws_eks_pod_identity_association" "humanitec_operator" {
cluster_name = module.aws_eks.cluster_name
namespace = kubernetes_namespace.humanitec_operator.id
service_account = local.humanitec_operator_k8s_sa_name
role_arn = aws_iam_role.humanitec_operator.arn
}

data "aws_iam_policy_document" "humanitec_operator" {
version = "2012-10-17"

statement {
actions = [
"secretsmanager:GetSecretValue",
"secretsmanager:CreateSecret",
"secretsmanager:DeleteSecret",
"secretsmanager:PutSecretValue"
]

resources = ["arn:aws:secretsmanager:${var.aws_region}:${var.aws_account_id}:secret:*"]
}
}

resource "aws_iam_policy" "humanitec_operator" {
name = "humanitec-operator"
description = "Humanitec Operator EKS service account policy"
policy = data.aws_iam_policy_document.humanitec_operator.json
}

resource "aws_iam_role_policy_attachment" "humanitec_operator" {
role = aws_iam_role.humanitec_operator.name
policy_arn = aws_iam_policy.humanitec_operator.arn
}


# Configure a primary secret store

locals {
humanitec_secret_store_id = var.cluster_name
}

resource "kubectl_manifest" "humanitec_operator_secret_store" {
yaml_body = templatefile("${path.module}/manifests/humanitec-secret-store.yaml", {
SECRET_STORE_ID = local.humanitec_secret_store_id,
SECRETS_MANAGER_REGION = var.aws_region
})
override_namespace = kubernetes_namespace.humanitec_operator.id
wait = true

depends_on = [
helm_release.humanitec_operator
]
}

resource "humanitec_secretstore" "main" {
id = local.humanitec_secret_store_id
primary = true
awssm = {
region = var.aws_region
}
}
5 changes: 4 additions & 1 deletion modules/base/humanitec.tf
Original file line number Diff line number Diff line change
Expand Up @@ -35,11 +35,14 @@ resource "humanitec_resource_definition" "k8s_cluster_driver" {

driver_account = humanitec_resource_account.cluster_account.id
driver_inputs = {
secrets_string = jsonencode({
"agent_url" = "$${resources['agent#agent'].outputs.url}"
})
values_string = jsonencode({
"name" = module.aws_eks.cluster_name
"loadbalancer" = local.ingress_address
"loadbalancer_hosted_zone" = data.aws_elb_hosted_zone_id.main.id
"region" = var.region
"region" = var.aws_region
})
}
}
Expand Down
11 changes: 11 additions & 0 deletions modules/base/manifests/humanitec-secret-store.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
apiVersion: humanitec.io/v1alpha1
kind: SecretStore
metadata:
name: ${SECRET_STORE_ID}
namespace: humanitec-operator
labels:
app.humanitec.io/default-store: "true"
spec:
awssm:
region: ${SECRETS_MANAGER_REGION}
auth: {}
5 changes: 5 additions & 0 deletions modules/base/outputs.tf
Original file line number Diff line number Diff line change
Expand Up @@ -53,3 +53,8 @@ output "humanitec_resource_account_id" {
description = "Humanitec resource account id for the cluster"
value = humanitec_resource_account.cluster_account.id
}

output "humanitec_secret_store_id" {
description = "Humanitec secret store id"
value = humanitec_secretstore.main.id
}
Loading

0 comments on commit 82da85d

Please sign in to comment.