2024-01-18|閱讀時間 ‧ 約 77 分鐘

Ingress Controller 101:輕鬆學會基本技巧

先說聲抱歉,本篇並不會多做太多ingress的說明,今天主要是要跟大家分享關於nginx ingress controller的基本操作。

raw-image

因為將程式碼也放上來,所以整體長度會比較長,也請大家見諒,但內容實務上有測試執行是可以運作的。

本文將說明以下部分 :

  1. 預先準備
  2. 流程
  3. 實際操作
  4. 將domain name 映射到Ingress LB IP
  5. 部署Demo App 驗證
  6. 結論

1.預先準備

1) Kubernetes cluster

(2) Admin access to kubernetes cluster

(3) 1個合法的domain,讓ingress controller Loadbalance IP來指向 (optional)

(4) 如果是On-prem的話,可以先建立類似metallb這樣的服務

(5) Nginx ingress controller有以下2種

  • kubernetes社群發佈的Nginx ingress controller
  • Nginx 發佈的Nginx ingress controller

2.流程

Step1. 建立 ingress-nginx namespace

Step2. 建立Nginx admission controller所需要的service account/Roles/ClusterRoles

Step3. 確認webhook設定

Step4. 建立/更新Webhook CA bundles的工作

Step5. 建立Nginx controller controller deployment所需要的service account/Roles/ClusterRoles

Step6. 建立Nginx controller configmap

Step7. 建立nginx controller與admission controller的service

Step8. 部署ingress controller


3.實際操作

#--------------------------------------------------
# S3-1. 建立namespace
#--------------------------------------------------
[root]# kubectl create ns ingress-nginx
#--------------------------------------------------
# S3-2. 建立Nginx admission controller role & SA
# 建立ROle/ClusterRole並綁定到SA
#--------------------------------------------------
[root]# vim admission-service-account.yaml
---
apiVersion: v1
kind: ServiceAccount
metadata:
labels:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
name: ingress-nginx-admission
namespace: ingress-nginx
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
annotations:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
name: ingress-nginx-admission
namespace: ingress-nginx
rules:
- apiGroups:
- ""
resources:
- secrets
verbs:
- get
- create
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
labels:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: nginx-nginx
name: ingress-nginx-admission
namespace: ingress-nginx
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: ingress-nginx-admission
subjects:
- kind: ServiceAccount
name: ingress-nginx-admission
namespace: ingress-nginx
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
name: ingress-nginx-admission
rules:
- apiGroups:
- admissionregistration.k8s.io
resources:
- validatingwebhookconfigurations
verbs:
- get
- update

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
labels:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
name: ingress-nginx-admission
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: ingress-nginx-admission
subjects:
- kind: ServiceAccount
name: ingress-nginx-admission
namespace: ingress-nginx
[root]# kubectl apply -f admission-service-account.yaml
serviceaccount/ingress-nginx-admission created
role.rbac.authorization.k8s.io/ingress-nginx-admission created
rolebinding.rbac.authorization.k8s.io/ingress-nginx-admission created
clusterrole.rbac.authorization.k8s.io/ingress-nginx-admission created
clusterrolebinding.rbac.authorization.k8s.io/ingress-nginx-admission created

kubernetes admission controller

負責驗證在k8s元件建立前的驗證或更新。以上讓admission controller來驗證ingress物件,並將admission controller程式碼寫在nginx controller內監聽8443 Port (也可以省略admission controller),有了這個驗證,可以確保建立出來的ingress物件的設定正確且不會打壞規則。


#--------------------------------------------------
# S3-3. 確認webhook設定
#--------------------------------------------------
[root]# vim validating-webhook.yaml
---
apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingWebhookConfiguration
metadata:
labels:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
name: ingress-nginx-admission
webhooks:
- admissionReviewVersions:
- v1
clientConfig:
service:
name: ingress-nginx-controller-admission
namespace: ingress-nginx
path: /networking/v1/ingresses
failurePolicy: Fail
matchPolicy: Equivalent
name: validate.nginx.ingress.kubernetes.io
rules:
- apiGroups:
- networking.k8s.io
apiVersions:
- v1
operations:
- CREATE
- UPDATE
resources:
- ingresses
sideEffects: None

[root]# kubectl apply -f validating-webhook.yaml
validatingwebhookconfiguration.admissionregistration.k8s.io/ingress-nginx-admission created
#--------------------------------------------------
# S3-4. 建立/更新Webhook CA bundles的工作
# 尤於ValidatingWebhookConfiguration只能在https下運行,故需要一個CA bundle
#--------------------------------------------------
[root]# vim jobs.yaml
---
apiVersion: batch/v1
kind: Job
metadata:
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
name: ingress-nginx-admission-create
namespace: ingress-nginx
spec:
template:
metadata:
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
name: ingress-nginx-admission-create
spec:
containers:
- args:
- create
- --host=ingress-nginx-controller-admission,ingress-nginx-controller-admission.$(POD_NAMESPACE).svc
- --namespace=$(POD_NAMESPACE)
- --secret-name=ingress-nginx-admission
env:
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
image: k8s.gcr.io/ingress-nginx/kube-webhook-certgen:v1.1.1
imagePullPolicy: IfNotPresent
name: create
securityContext:
allowPrivilegeEscalation: false
nodeSelector:
kubernetes.io/os: linux
restartPolicy: OnFailure
securityContext:
runAsNonRoot: true
runAsUser: 2000
serviceAccountName: ingress-nginx-admission
---
apiVersion: batch/v1
kind: Job
metadata:
labels:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
name: ingress-nginx-admission-patch
namespace: ingress-nginx
spec:
template:
metadata:
labels:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
name: ingress-nginx-admission-patch
spec:
containers:
- args:
- patch
- --webhook-name=ingress-nginx-admission
- --namespace=$(POD_NAMESPACE)
- --patch-mutating=false
- --secret-name=ingress-nginx-admission
- --patch-failure-policy=Fail
env:
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
image: k8s.gcr.io/ingress-nginx/kube-webhook-certgen:v1.1.1
imagePullPolicy: IfNotPresent
name: patch
securityContext:
allowPrivilegeEscalation: false
nodeSelector:
kubernetes.io/os: linux
restartPolicy: OnFailure
securityContext:
runAsNonRoot: true
runAsUser: 2000
serviceAccountName: ingress-nginx-admission

※ 執行後,可以再去確認`ValdatingWebhookConfiguration`有沒有patch bundle
[root]# kubectl describe ValidatingWebhookConfiguration ingress-nginx-admission
#--------------------------------------------------
# S3-5. 建立Nginx controller controller deployment所需要的service
# account/Roles/ClusterRoles
#--------------------------------------------------
[root]# vim ingress-service-account.yaml
---
apiVersion: v1
kind: ServiceAccount
metadata:
labels:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
name: ingress-nginx
namespace: ingress-nginx

---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
name: ingress-nginx
namespace: ingress-nginx
rules:
- apiGroups:
- ""
resources:
- namespaces
verbs:
- get
- apiGroups:
- ""
resources:
- configmaps
- pods
- secrets
- endpoints
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- services
verbs:
- get
- list
- watch
- apiGroups:
- networking.k8s.io
resources:
- ingresses
verbs:
- get
- list
- watch
- apiGroups:
- networking.k8s.io
resources:
- ingresses/status
verbs:
- update
- apiGroups:
- networking.k8s.io
resources:
- ingressclasses
verbs:
- get
- list
- watch
- apiGroups:
- ""
resourceNames:
- ingress-controller-leader
resources:
- configmaps
verbs:
- get
- update
- apiGroups:
- ""
resources:
- configmaps
verbs:
- create
- apiGroups:
- ""
resources:
- events
verbs:
- create
- patch

---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
name: ingress-nginx
namespace: ingress-nginx
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: ingress-nginx
subjects:
- kind: ServiceAccount
name: ingress-nginx
namespace: ingress-nginx

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
name: ingress-nginx
rules:
- apiGroups:
- ""
resources:
- configmaps
- endpoints
- nodes
- pods
- secrets
- namespaces
verbs:
- list
- watch
- apiGroups:
- ""
resources:
- nodes
verbs:
- get
- apiGroups:
- ""
resources:
- services
verbs:
- get
- list
- watch
- apiGroups:
- networking.k8s.io
resources:
- ingresses
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- events
verbs:
- create
- patch
- apiGroups:
- networking.k8s.io
resources:
- ingresses/status
verbs:
- update
- apiGroups:
- networking.k8s.io
resources:
- ingressclasses
verbs:
- get
- list
- watch

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
name: ingress-nginx
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: ingress-nginx
subjects:
- kind: ServiceAccount
name: ingress-nginx
namespace: ingress-nginx

[root]# kubectl apply -f ingress-service-account.yaml
serviceaccount/ingress-nginx created
role.rbac.authorization.k8s.io/ingress-nginx created
rolebinding.rbac.authorization.k8s.io/ingress-nginx created
clusterrole.rbac.authorization.k8s.io/ingress-nginx created
clusterrolebinding.rbac.authorization.k8s.io/ingress-nginx created
#--------------------------------------------------
# S3-6. 建立Nginx controller configmap
# (使用configmap就可以自訂nginx設定,例如自訂標頭…)
#--------------------------------------------------
[root]# vim configmap.yaml
---
apiVersion: v1
data:
allow-snippet-annotations: "true"
kind: ConfigMap
metadata:
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
name: ingress-nginx-controller
namespace: ingress-nginx

[root]# kubectl apply -f configmap.yaml
[root]# kubectl get all
#--------------------------------------------------
# S3-7. 建立nginx controller與admission controller的service
#--------------------------------------------------
[root]# vim services.yaml
---
apiVersion: v1
kind: Service
metadata:
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
name: ingress-nginx-controller
namespace: ingress-nginx
spec:
externalTrafficPolicy: Local
ipFamilies:
- IPv4
ipFamilyPolicy: SingleStack
ports:
- appProtocol: http
name: http
port: 80
protocol: TCP
targetPort: http
- appProtocol: https
name: https
port: 443
protocol: TCP
targetPort: https
selector:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
type: LoadBalancer
---
apiVersion: v1
kind: Service
metadata:
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
name: ingress-nginx-controller-admission
namespace: ingress-nginx
spec:
ports:
- appProtocol: https
name: https-webhook
port: 443
targetPort: webhook
selector:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
type: ClusterIP

[root]# kubectl apply -f services.yaml
service/ingress-nginx-controller created
service/ingress-nginx-controller-admission created

※ 建立出來的`ingress-nginx-controller`會在對應的cloud platform建立Loadbalancer
可以用以下方式取得load balancer的IP/DNS
[root]# kubectl --namespace ingress-nginx get services -o wide -w ingress-nginx-controller
--> 10.107.88.91
#--------------------------------------------------
# S3-8. 部署ingress controller
#--------------------------------------------------
[root]# vim deployment.yaml
---
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
name: ingress-nginx-controller
namespace: ingress-nginx
spec:
minReadySeconds: 0
revisionHistoryLimit: 10
selector:
matchLabels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
template:
metadata:
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
spec:
containers:
- args:
- /nginx-ingress-controller
- --publish-service=$(POD_NAMESPACE)/ingress-nginx-controller
- --election-id=ingress-controller-leader
- --controller-class=k8s.io/ingress-nginx
- --configmap=$(POD_NAMESPACE)/ingress-nginx-controller
- --validating-webhook=:8443
- --validating-webhook-certificate=/usr/local/certificates/cert
- --validating-webhook-key=/usr/local/certificates/key
env:
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: LD_PRELOAD
value: /usr/local/lib/libmimalloc.so
image: k8s.gcr.io/ingress-nginx/controller:v1.1.1
imagePullPolicy: IfNotPresent
lifecycle:
preStop:
exec:
command:
- /wait-shutdown
livenessProbe:
failureThreshold: 5
httpGet:
path: /healthz
port: 10254
scheme: HTTP
initialDelaySeconds: 10
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
name: controller
ports:
- containerPort: 80
name: http
protocol: TCP
- containerPort: 443
name: https
protocol: TCP
- containerPort: 8443
name: webhook
protocol: TCP
readinessProbe:
failureThreshold: 3
httpGet:
path: /healthz
port: 10254
scheme: HTTP
initialDelaySeconds: 10
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
resources:
requests:
cpu: 100m
memory: 90Mi
securityContext:
allowPrivilegeEscalation: true
capabilities:
add:
- NET_BIND_SERVICE
drop:
- ALL
runAsUser: 101
volumeMounts:
- mountPath: /usr/local/certificates/
name: webhook-cert
readOnly: true
dnsPolicy: ClusterFirst
nodeSelector:
kubernetes.io/os: linux
serviceAccountName: ingress-nginx
terminationGracePeriodSeconds: 300
volumes:
- name: webhook-cert
secret:
secretName: ingress-nginx-admission

[root]# kubectl apply -f deployment.yaml
[root]# kubectl get pod -n ingress-nginx
#--------------------------------------------------
# S3-09. 部署ingress class
#--------------------------------------------------
[root]# vim ingressclass.yaml
---
apiVersion: networking.k8s.io/v1
kind: IngressClass
metadata:
labels:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
name: nginx
spec:
controller: k8s.io/ingress-nginx
---
apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingWebhookConfiguration
metadata:
labels:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
name: ingress-nginx-admission
webhooks:
- admissionReviewVersions:
- v1
clientConfig:
service:
name: ingress-nginx-controller-admission
namespace: ingress-nginx
path: /networking/v1/ingresses
failurePolicy: Fail
matchPolicy: Equivalent
name: validate.nginx.ingress.kubernetes.io
rules:
- apiGroups:
- networking.k8s.io
apiVersions:
- v1
operations:
- CREATE
- UPDATE
resources:
- ingresses
sideEffects: None

[root]# kubectl create -f ingressclass.yaml
[root]# kubectl get ingressclass
NAME CONTROLLER PARAMETERS AGE
nginx k8s.io/ingress-nginx <none> 4m21s

4.將domain name 映射到Ingress LB IP

Ingress的主要目的是接收外部流量將其導至在K8S上運行的服務。所以實務上DNS要對應到Ingress controller的LB IP(此處為10.107.88.91, 內部建立的DNS要與其進行對應)。

可以透過對應的DNS來提供。

有二種方式:

  • Single DNS Mapping:直接把一整個domain全部mapp到ingress controller LB IP => 用這種方式可以實現一個domain給ingress controller與多個Path的流量路由
www.example.com ---> Loadbalancer IP
[OR]
http://www.example.com/app1
http://www.example.com/app2
http://www.example.com/app1/api
http://www.example.com/app2/api
  • Wildcard DNS Mapping:如果使用這種方式,就可以透過ingress使用dynamic DNS endpoints的功能。當使用wildcard在DNS時,需要加入ingress所需 要的DNS與nginx ingress controller要負責路由的service endpoint。
*.example.com --> Loadbalancer IP
*.apps.example.com --> Loadbalancer IP

這種方式可以單一ingress controller之下有多個dynamic subdomain,並且每個DNS可以有自已的path-based routing。

# URL one

http://demo1.example.com/api
http://demo1.example.com/api/v1
http://demo1.example.com/api/v2

# app specific urls

http://grafana.apps.example.com
http://prometheus.apps.example.com

# URL two

http://demo2.apps.example.com/api
http://demo2.apps.example.com/api/v1
http://demo2.apps.example.com/api/v2
#------------------------------------------------------
# S4-1. wildcard DNS Mapping
#------------------------------------------------------
[lb01]# vim /var/named/named.test.example.poc
--> Add
*.apps.test.example.poc. IN A 10.107.88.91

[lb01]# systemctl restart named

5.部署Demo App 驗證

#------------------------------------------------------
# S5-1. edit app yaml
#------------------------------------------------------
[root]# vim hello-app.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: hello-app
namespace: dev
spec:
selector:
matchLabels:
app: hello
replicas: 3
template:
metadata:
labels:
app: hello
spec:
containers:
- name: hello
image: "gcr.io/google-samples/hello-app:2.0"

[root]# kubectl create -f hello-app.yaml
[root]# kubectl get all -n dev
#------------------------------------------------------
# S5-2. create svc
#------------------------------------------------------
[root]# vim hello-svc.yaml
apiVersion: v1
kind: Service
metadata:
name: hello-service
namespace: dev
labels:
app: hello
spec:
type: ClusterIP
selector:
app: hello
ports:
- port: 80
targetPort: 8080
protocol: TCP

[root]# kubectl create -f hello-svc.yaml

※建立給App使用的Ingress

目標:建立ingress,並且外部可以透過DNS 以fqdn的方式存取到App (http://demo.test.example.poc)

運作:ingress controller pod會連接到ingress api 確認路由規則

#------------------------------------------------------
# S5-3. edit ingress.yaml
#------------------------------------------------------
[root]# vim ingress.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: test-ingress
namespace: dev
spec:
ingressClassName: nginx
rules:
- host: "demo.apps.test.example.poc"
http:
paths:
- pathType: Prefix
path: "/"
backend:
service:
name: hello-service
port:
number: 80
[root]# kubectl create -f ingress.yaml
[root]# kubectl describe ingress -n dev
==> http://demo.apps.test.example.poc
#------------------------------------------------------
# S5-4. 建立另一個服務,測試不同domain name都走到同一支ingress controller
#------------------------------------------------------
[root]# cat www-app.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: www
spec:
selector:
matchLabels:
app: www
replicas: 2 # tells deployment to run 2 pods matching the template
template:
metadata:
labels:
app: www
spec:
containers:
- name: nginx
image: gcr.io/google-samples/hello-app:2.0
ports:
- containerPort: 80

[root]# cat www-ingress.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: www-ingress
namespace: dev
spec:
ingressClassName: nginx
rules:
- host: "www.apps.test.example.poc"
http:
paths:
- pathType: Prefix
path: "/"
backend:
service:
name: www-svc
port:
number: 80
[root]# cat www-svc.yaml
apiVersion: v1
kind: Service
metadata:
name: www-svc
namespace: dev
labels:
app: www
spec:
selector:
app: www
ports:
- port: 80
targetPort: 8080
protocol: TCP

[root]# kubectl create -f www-app.yaml
[root]# kubectl create -f www-svc.yaml
[root]# kubectl create -f www-ingress.yaml
#------------------------------------------------------
# S5-5. 測試
#------------------------------------------------------
[lb01]# curl demo.apps.test.example.poc
[lb01]# curl www.apps.test.example.poc

6.結論

因為將程式碼的部分也直接給大家看內容,所以篇幅有點太長,但為了大家如果想要直接使用這些內容,就不用再跳到其他外部網站,在閱讀上比較方便,也請大家見諒。

在容器化的世界了解ingress是很重要的,本篇雖然沒有從頭開始講解,但透過實務上的操作之後,再回過頭了重新了解ingress的基礎也許會更有感覺。

今天的分享就到這邊為止,下期再見。






分享至
成為作者繼續創作的動力吧!
© 2024 vocus All rights reserved.