Append the nginx ingress http redirect to https
- Add extra port to nginx deployment for example: 8080, look like following
1ports:
2 - name: http
3 containerPort: 80
4 protocol: TCP
5 - name: https
6 containerPort: 443
7 protocol: TCP
8 - name: webhook
9 containerPort: 8443
10 protocol: TCP
11 - name: http-redirect
12 containerPort: 8080
- Add section
http-snippet to ConfigMap ingress-nginx-controller
1data:
2 allow-snippet-annotations: 'true'
3 compute-full-forwarded-for: 'true'
4 use-forwarded-headers: 'true'
5 server-tokens: 'false'
6 http-snippet: |
7 server {
8 listen 8080;
9 return 308 https://$host$request_uri;
10 }
- Modify the ingress service loadbalance configure, let http-80 forward to http-redirect-8080
1- name: http
2 protocol: TCP
3 appProtocol: http
4 port: 80
5 targetPort: http-redirect
6 nodePort: 32009
Using Nginx-Ingress Controller in AWS Eks environment
- Need manually enable the Proxy protocol V2 in NLB target group just only TCP https 443 listener Rather than TCP http 80 listener
- Multiple SSL certificate need manually add to NLB
- The http redirect to https, need to change some paramets
1 # use http-snippet and open proxy protocol
2 use-proxy-protocol: 'true'
3 use-forwarded-headers: 'true'
4 server-tokens: 'false'
5 http-snippet: |
6 server {
7 listen 8000;
8 return 308 https://$host$request_uri;
9 }
10 proxy-real-ip-cidr: ${join(",", var.internal_cidr)}
11 # Add special port in controller container
12 set {
13 type = "string"
14 name = "controller.containerPort.special"
15 value = "8000"
16 }
17 # Let https redirect to http port and http port redirect to special port , the special port listener in `http-snippet` section
18 set {
19 type = "string"
20 name = "controller.service.targetPorts.https"
21 value = "http"
22 }
23
24set {
25type = "string"
26name = "controller.service.targetPorts.http"
27value = "special"
28}
- The complete Terraform configuration is shown below
1 resource "helm_release" "ingress_nginx_controller" {
2 name = "nginx-ingress-controller"
3 repository = "https://kubernetes.github.io/ingress-nginx"
4 chart = "ingress-nginx"
5 version = "3.8.0"
6 namespace = "kube-system"
7
8 values = [
9 <<-EOF
10 controller:
11 config:
12 use-proxy-protocol: 'true'
13 use-forwarded-headers: 'true'
14 server-tokens: 'false'
15 http-snippet: |
16 server {
17 listen 8000;
18 return 308 https://$host$request_uri;
19 }
20 proxy-real-ip-cidr: ${join(",", var.internal_cidr)}
21 EOF
22 ]
23
24 set {
25 type = "string"
26 name = "controller.resources.requests.memory"
27 value = "256Mi"
28 }
29
30 set {
31 type = "string"
32 name = "controller.resources.limits.memory"
33 value = "500Mi"
34 }
35
36 set {
37 name = "fullnameOverride"
38 value = "nginx-ingress"
39 }
40
41 set {
42 type = "string"
43 name = "controller.service.targetPorts.https"
44 value = "http"
45 }
46
47 set {
48 type = "string"
49 name = "controller.service.targetPorts.http"
50 value = "special"
51 }
52
53 dynamic "set" {
54 for_each = concat(var.office_cidr, var.internal_cidr)
55 content {
56 name = join("", ["controller.service.loadBalancerSourceRanges[", set.key, "]"])
57 value = set.value
58 }
59 }
60
61 set {
62 type = "string"
63 name = "controller.containerPort.special"
64 value = "8000"
65 }
66
67 set {
68 type = "string"
69 name = "controller.service.annotations.service\\.beta\\.kubernetes\\.io/aws-load-balancer-ssl-cert"
70 value = var.alb_certificate
71 }
72
73 set {
74 type = "string"
75 name = "controller.service.annotations.service\\.beta\\.kubernetes\\.io/aws-load-balancer-proxy-protocol"
76 value = "*"
77 }
78
79 set {
80 type = "string"
81 name = "controller.service.annotations.service\\.beta\\.kubernetes\\.io/aws-load-balancer-backend-protocol"
82 value = "tcp"
83 }
84
85 set{
86 type = "string"
87 name = "controller.service.annotations.service\\.beta\\.kubernetes\\.io/aws-load-balancer-ssl-ports"
88 value = "https"
89 }
90
91 set {
92 type = "string"
93 name = "controller.service.annotations.service\\.beta\\.kubernetes\\.io/aws-load-balancer-connection-idle-timeout"
94 value = "120"
95 }
96
97 set {
98 type = "string"
99 name = "controller.service.annotations.service\\.beta\\.kubernetes\\.io/aws-load-balancer-type"
100 value = "nlb"
101 }
102
103 set {
104 type = "string"
105 name = "controller.service.annotations.service\\.beta\\.kubernetes\\.io/aws-load-balancer-cross-zone-load-balancing-enabled"
106 value = "true"
107 }
108}
Using Alb-ingress-controller in AWS Eks
- Every ingress should create application loadbalance it will cost a lot
- Alb-ingress controller need aws IAM policy to create ALB
- The ALB-ingress controller Terraform configurate
1 resource "kubernetes_cluster_role" "ingress" {
2 metadata {
3 name = "alb-ingress-controller"
4 labels = {
5 "app.kubernetes.io/name" = "alb-ingress-controller"
6 "app.kubernetes.io/managed-by" = "terraform"
7 }
8 }
9
10 rule {
11 api_groups = ["", "extensions"]
12 resources = ["configmaps", "endpoints", "events", "ingresses", "ingresses/status", "services"]
13 verbs = ["create", "get", "list", "update", "watch", "patch"]
14 }
15
16 rule {
17 api_groups = ["", "extensions"]
18 resources = ["nodes", "pods", "secrets", "services", "namespaces"]
19 verbs = ["get", "list", "watch"]
20 }
21
22 depends_on = [
23 aws_eks_cluster.main
24 ]
25}
26
27resource "kubernetes_cluster_role_binding" "ingress" {
28 metadata {
29 name = "alb-ingress-controller"
30 labels = {
31 "app.kubernetes.io/name" = "alb-ingress-controller"
32 "app.kubernetes.io/managed-by" = "terraform"
33 }
34 }
35 role_ref {
36 api_group = "rbac.authorization.k8s.io"
37 kind = "ClusterRole"
38 name = kubernetes_cluster_role.ingress.metadata[0].name
39 }
40 subject {
41 kind = "ServiceAccount"
42 name = kubernetes_service_account.ingress.metadata[0].name
43 namespace = kubernetes_service_account.ingress.metadata[0].namespace
44 }
45
46 depends_on = [
47 aws_eks_cluster.main,
48 kubernetes_cluster_role.ingress
49 ]
50}
51
52resource "kubernetes_service_account" "ingress" {
53 automount_service_account_token = true
54 metadata {
55 name = "alb-ingress-controller"
56 namespace = "kube-system"
57 labels = {
58 "app.kubernetes.io/name" = "alb-ingress-controller"
59 "app.kubernetes.io/managed-by" = "terraform"
60 }
61 annotations = {
62 "eks.amazonaws.com/role-arn" = aws_iam_role.eks_alb_ingress_controller.arn
63 }
64 }
65
66 depends_on = [
67 aws_eks_cluster.main
68 ]
69}
70
71resource "kubernetes_deployment" "ingress" {
72 metadata {
73 name = "alb-ingress-controller"
74 namespace = "kube-system"
75 labels = {
76 "app.kubernetes.io/name" = "alb-ingress-controller"
77 "app.kubernetes.io/version" = "v1.1.9"
78 "app.kubernetes.io/managed-by" = "terraform"
79 }
80 }
81
82 spec {
83 replicas = 1
84
85 selector {
86 match_labels = {
87 "app.kubernetes.io/name" = "alb-ingress-controller"
88 }
89 }
90
91 template {
92 metadata {
93 labels = {
94 "app.kubernetes.io/name" = "alb-ingress-controller"
95 "app.kubernetes.io/version" = "v1.1.9"
96 }
97 }
98
99 spec {
100 dns_policy = "ClusterFirst"
101 restart_policy = "Always"
102 service_account_name = kubernetes_service_account.ingress.metadata[0].name
103 termination_grace_period_seconds = 60
104
105 container {
106 name = "alb-ingress-controller"
107 image = "docker.io/amazon/aws-alb-ingress-controller:v1.1.9"
108 image_pull_policy = "Always"
109
110 args = [
111 "--ingress-class=alb",
112 "--cluster-name=${aws_eks_cluster.main.id}",
113 "--aws-vpc-id=${var.vpc_id}",
114 "--aws-region=${var.region}",
115 "--aws-max-retries=10",
116 ]
117
118 volume_mount {
119 mount_path = "/var/run/secrets/kubernetes.io/serviceaccount"
120 name = kubernetes_service_account.ingress.default_secret_name
121 read_only = true
122 }
123
124 port {
125 name = "health"
126 container_port = 10254
127 protocol = "TCP"
128 }
129
130 readiness_probe {
131 http_get {
132 path = "/healthz"
133 port = "health"
134 scheme = "HTTP"
135 }
136
137 initial_delay_seconds = 30
138 period_seconds = 60
139 timeout_seconds = 3
140 }
141
142 liveness_probe {
143 http_get {
144 path = "/healthz"
145 port = "health"
146 scheme = "HTTP"
147 }
148
149 initial_delay_seconds = 60
150 period_seconds = 60
151 }
152 }
153
154 volume {
155 name = kubernetes_service_account.ingress.default_secret_name
156
157 secret {
158 secret_name = kubernetes_service_account.ingress.default_secret_name
159 }
160 }
161 }
162 }
163 }
164
165 depends_on = [
166 kubernetes_cluster_role_binding.ingress,
167 aws_eks_cluster.main
168 ]
169}
- The Alb ingress resource should look likes following
1 resource "kubernetes_ingress" "example_ingress" {
2 metadata {
3 name = "example-ingress"
4 namespace = "default"
5 annotations = {
6 "kubernetes.io/ingress.class" = "alb"
7 "alb.ingress.kubernetes.io/scheme" = "internet-facing"
8 "alb.ingress.kubernetes.io/target-type" = "instance"
9 "alb.ingress.kubernetes.io/listen-ports" = "[{\"HTTPS\": 443},{\"HTTP\": 80}]"
10 "alb.ingress.kubernetes.io/actions.ssl-redirect" = "{\"Type\": \"redirect\", \"RedirectConfig\": { \"Protocol\": \"HTTPS\", \"Port\": \"443\", \"StatusCode\": \"HTTP_301\"}}"
11 "alb.ingress.kubernetes.io/security-groups" = var.alb_security_group_ids
12 "alb.ingress.kubernetes.io/certificate-arn" = var.alb_certificates
13 "alb.ingress.kubernetes.io/healthcheck-path" = "/"
14 "alb.ingress.kubernetes.io/success-codes" = "200,201,302,301"
15 "alb.ingress.kubernetes.io/unhealthy-threshold-count" = "5"
16 }
17 }
18 spec {
19 rule {
20 host = var.web_dns_name
21 http {
22 path {
23 backend {
24 service_name = "ssl-redirect"
25 service_port = "use-annotation"
26 }
27 path = "/*"
28 }
29 path {
30 backend {
31 service_name = "example-web"
32 service_port = 80
33 }
34 path = "/*"
35 }
36 }
37 }
38 }
39}
The Merge ingress it will only work if you create ingress in the same namespace
- The merge ingress helm chart need download to self github repo merge ingress
1resource "helm_release" "merge_ingress_controller" {
2 name = "merge-ingress"
3 chart = "${path.module}/../../../../charts/ingress-merge"
4 namespace = "kube-system"
5 }
- Should create merged_ingress configmap in every namespace if has ingress
1resource "kubernetes_config_map" "merged_ingress" {
2 metadata {
3 name = "merged-ingress"
4 namespace = "default"
5 }
6
7 data = {
8 annotations =<<DOC
9 "kubernetes.io/ingress.class": "alb"
10 "alb.ingress.kubernetes.io/scheme": "internet-facing"
11 "alb.ingress.kubernetes.io/target-type": "instance"
12 "alb.ingress.kubernetes.io/listen-ports": "[{\"HTTPS\": 443},{\"HTTP\": 80}]"
13 "alb.ingress.kubernetes.io/actions.ssl-redirect": "{\"Type\": \"redirect\", \"RedirectConfig\": { \"Protocol\": \"HTTPS\", \"Port\": \"443\", \"StatusCode\": \"HTTP_301\"}}"
14 "alb.ingress.kubernetes.io/security-groups": ${var.alb_security_group_ids}
15 "alb.ingress.kubernetes.io/certificate-arn": ${var.alb_certificates}
16 "alb.ingress.kubernetes.io/healthcheck-path": "/"
17 "alb.ingress.kubernetes.io/success-codes": "200,201,302,301"
18 "alb.ingress.kubernetes.io/unhealthy-threshold-count": "5"
19 DOC
20 }
21 }
- The ingress resource should be create in default namespace