先說聲抱歉,本篇並不會多做太多ingress的說明,今天主要是要跟大家分享關於nginx ingress controller的基本操作。
因為將程式碼也放上來,所以整體長度會比較長,也請大家見諒,但內容實務上有測試執行是可以運作的。
本文將說明以下部分 :
1) Kubernetes cluster
(2) Admin access to kubernetes cluster
(3) 1個合法的domain,讓ingress controller Loadbalance IP來指向 (optional)
(4) 如果是On-prem的話,可以先建立類似metallb
這樣的服務
(5) Nginx ingress controller有以下2種
Step1. 建立 ingress-nginx namespace
Step2. 建立Nginx admission controller所需要的service account/Roles/ClusterRoles
Step3. 確認webhook設定
Step4. 建立/更新Webhook CA bundles的工作
Step5. 建立Nginx controller controller deployment所需要的service account/Roles/ClusterRoles
Step6. 建立Nginx controller configmap
Step7. 建立nginx controller與admission controller的service
Step8. 部署ingress controller
#--------------------------------------------------
# S3-1. 建立namespace
#--------------------------------------------------
[root]# kubectl create ns ingress-nginx
#--------------------------------------------------
# S3-2. 建立Nginx admission controller role & SA
# 建立ROle/ClusterRole並綁定到SA
#--------------------------------------------------
[root]# vim admission-service-account.yaml
---
apiVersion: v1
kind: ServiceAccount
metadata:
labels:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
name: ingress-nginx-admission
namespace: ingress-nginx
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
annotations:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
name: ingress-nginx-admission
namespace: ingress-nginx
rules:
- apiGroups:
- ""
resources:
- secrets
verbs:
- get
- create
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
labels:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: nginx-nginx
name: ingress-nginx-admission
namespace: ingress-nginx
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: ingress-nginx-admission
subjects:
- kind: ServiceAccount
name: ingress-nginx-admission
namespace: ingress-nginx
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
name: ingress-nginx-admission
rules:
- apiGroups:
- admissionregistration.k8s.io
resources:
- validatingwebhookconfigurations
verbs:
- get
- update
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
labels:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
name: ingress-nginx-admission
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: ingress-nginx-admission
subjects:
- kind: ServiceAccount
name: ingress-nginx-admission
namespace: ingress-nginx
[root]# kubectl apply -f admission-service-account.yaml
serviceaccount/ingress-nginx-admission created
role.rbac.authorization.k8s.io/ingress-nginx-admission created
rolebinding.rbac.authorization.k8s.io/ingress-nginx-admission created
clusterrole.rbac.authorization.k8s.io/ingress-nginx-admission created
clusterrolebinding.rbac.authorization.k8s.io/ingress-nginx-admission created
※ kubernetes admission controller:
負責驗證在k8s元件建立前的驗證或更新。以上讓admission controller來驗證ingress物件,並將admission controller程式碼寫在nginx controller內監聽8443 Port (也可以省略admission controller),有了這個驗證,可以確保建立出來的ingress物件的設定正確且不會打壞規則。
#--------------------------------------------------
# S3-3. 確認webhook設定
#--------------------------------------------------
[root]# vim validating-webhook.yaml
---
apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingWebhookConfiguration
metadata:
labels:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
name: ingress-nginx-admission
webhooks:
- admissionReviewVersions:
- v1
clientConfig:
service:
name: ingress-nginx-controller-admission
namespace: ingress-nginx
path: /networking/v1/ingresses
failurePolicy: Fail
matchPolicy: Equivalent
name: validate.nginx.ingress.kubernetes.io
rules:
- apiGroups:
- networking.k8s.io
apiVersions:
- v1
operations:
- CREATE
- UPDATE
resources:
- ingresses
sideEffects: None
[root]# kubectl apply -f validating-webhook.yaml
validatingwebhookconfiguration.admissionregistration.k8s.io/ingress-nginx-admission created
#--------------------------------------------------
# S3-4. 建立/更新Webhook CA bundles的工作
# 尤於ValidatingWebhookConfiguration只能在https下運行,故需要一個CA bundle
#--------------------------------------------------
[root]# vim jobs.yaml
---
apiVersion: batch/v1
kind: Job
metadata:
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
name: ingress-nginx-admission-create
namespace: ingress-nginx
spec:
template:
metadata:
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
name: ingress-nginx-admission-create
spec:
containers:
- args:
- create
- --host=ingress-nginx-controller-admission,ingress-nginx-controller-admission.$(POD_NAMESPACE).svc
- --namespace=$(POD_NAMESPACE)
- --secret-name=ingress-nginx-admission
env:
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
image: k8s.gcr.io/ingress-nginx/kube-webhook-certgen:v1.1.1
imagePullPolicy: IfNotPresent
name: create
securityContext:
allowPrivilegeEscalation: false
nodeSelector:
kubernetes.io/os: linux
restartPolicy: OnFailure
securityContext:
runAsNonRoot: true
runAsUser: 2000
serviceAccountName: ingress-nginx-admission
---
apiVersion: batch/v1
kind: Job
metadata:
labels:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
name: ingress-nginx-admission-patch
namespace: ingress-nginx
spec:
template:
metadata:
labels:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
name: ingress-nginx-admission-patch
spec:
containers:
- args:
- patch
- --webhook-name=ingress-nginx-admission
- --namespace=$(POD_NAMESPACE)
- --patch-mutating=false
- --secret-name=ingress-nginx-admission
- --patch-failure-policy=Fail
env:
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
image: k8s.gcr.io/ingress-nginx/kube-webhook-certgen:v1.1.1
imagePullPolicy: IfNotPresent
name: patch
securityContext:
allowPrivilegeEscalation: false
nodeSelector:
kubernetes.io/os: linux
restartPolicy: OnFailure
securityContext:
runAsNonRoot: true
runAsUser: 2000
serviceAccountName: ingress-nginx-admission
※ 執行後,可以再去確認`ValdatingWebhookConfiguration`有沒有patch bundle
[root]# kubectl describe ValidatingWebhookConfiguration ingress-nginx-admission
#--------------------------------------------------
# S3-5. 建立Nginx controller controller deployment所需要的service
# account/Roles/ClusterRoles
#--------------------------------------------------
[root]# vim ingress-service-account.yaml
---
apiVersion: v1
kind: ServiceAccount
metadata:
labels:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
name: ingress-nginx
namespace: ingress-nginx
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
name: ingress-nginx
namespace: ingress-nginx
rules:
- apiGroups:
- ""
resources:
- namespaces
verbs:
- get
- apiGroups:
- ""
resources:
- configmaps
- pods
- secrets
- endpoints
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- services
verbs:
- get
- list
- watch
- apiGroups:
- networking.k8s.io
resources:
- ingresses
verbs:
- get
- list
- watch
- apiGroups:
- networking.k8s.io
resources:
- ingresses/status
verbs:
- update
- apiGroups:
- networking.k8s.io
resources:
- ingressclasses
verbs:
- get
- list
- watch
- apiGroups:
- ""
resourceNames:
- ingress-controller-leader
resources:
- configmaps
verbs:
- get
- update
- apiGroups:
- ""
resources:
- configmaps
verbs:
- create
- apiGroups:
- ""
resources:
- events
verbs:
- create
- patch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
name: ingress-nginx
namespace: ingress-nginx
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: ingress-nginx
subjects:
- kind: ServiceAccount
name: ingress-nginx
namespace: ingress-nginx
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
name: ingress-nginx
rules:
- apiGroups:
- ""
resources:
- configmaps
- endpoints
- nodes
- pods
- secrets
- namespaces
verbs:
- list
- watch
- apiGroups:
- ""
resources:
- nodes
verbs:
- get
- apiGroups:
- ""
resources:
- services
verbs:
- get
- list
- watch
- apiGroups:
- networking.k8s.io
resources:
- ingresses
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- events
verbs:
- create
- patch
- apiGroups:
- networking.k8s.io
resources:
- ingresses/status
verbs:
- update
- apiGroups:
- networking.k8s.io
resources:
- ingressclasses
verbs:
- get
- list
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
name: ingress-nginx
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: ingress-nginx
subjects:
- kind: ServiceAccount
name: ingress-nginx
namespace: ingress-nginx
[root]# kubectl apply -f ingress-service-account.yaml
serviceaccount/ingress-nginx created
role.rbac.authorization.k8s.io/ingress-nginx created
rolebinding.rbac.authorization.k8s.io/ingress-nginx created
clusterrole.rbac.authorization.k8s.io/ingress-nginx created
clusterrolebinding.rbac.authorization.k8s.io/ingress-nginx created
#--------------------------------------------------
# S3-6. 建立Nginx controller configmap
# (使用configmap就可以自訂nginx設定,例如自訂標頭…)
#--------------------------------------------------
[root]# vim configmap.yaml
---
apiVersion: v1
data:
allow-snippet-annotations: "true"
kind: ConfigMap
metadata:
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
name: ingress-nginx-controller
namespace: ingress-nginx
[root]# kubectl apply -f configmap.yaml
[root]# kubectl get all
#--------------------------------------------------
# S3-7. 建立nginx controller與admission controller的service
#--------------------------------------------------
[root]# vim services.yaml
---
apiVersion: v1
kind: Service
metadata:
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
name: ingress-nginx-controller
namespace: ingress-nginx
spec:
externalTrafficPolicy: Local
ipFamilies:
- IPv4
ipFamilyPolicy: SingleStack
ports:
- appProtocol: http
name: http
port: 80
protocol: TCP
targetPort: http
- appProtocol: https
name: https
port: 443
protocol: TCP
targetPort: https
selector:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
type: LoadBalancer
---
apiVersion: v1
kind: Service
metadata:
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
name: ingress-nginx-controller-admission
namespace: ingress-nginx
spec:
ports:
- appProtocol: https
name: https-webhook
port: 443
targetPort: webhook
selector:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
type: ClusterIP
[root]# kubectl apply -f services.yaml
service/ingress-nginx-controller created
service/ingress-nginx-controller-admission created
※ 建立出來的`ingress-nginx-controller`會在對應的cloud platform建立Loadbalancer
可以用以下方式取得load balancer的IP/DNS:
[root]# kubectl --namespace ingress-nginx get services -o wide -w ingress-nginx-controller
--> 10.107.88.91
#--------------------------------------------------
# S3-8. 部署ingress controller
#--------------------------------------------------
[root]# vim deployment.yaml
---
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
name: ingress-nginx-controller
namespace: ingress-nginx
spec:
minReadySeconds: 0
revisionHistoryLimit: 10
selector:
matchLabels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
template:
metadata:
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
spec:
containers:
- args:
- /nginx-ingress-controller
- --publish-service=$(POD_NAMESPACE)/ingress-nginx-controller
- --election-id=ingress-controller-leader
- --controller-class=k8s.io/ingress-nginx
- --configmap=$(POD_NAMESPACE)/ingress-nginx-controller
- --validating-webhook=:8443
- --validating-webhook-certificate=/usr/local/certificates/cert
- --validating-webhook-key=/usr/local/certificates/key
env:
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: LD_PRELOAD
value: /usr/local/lib/libmimalloc.so
image: k8s.gcr.io/ingress-nginx/controller:v1.1.1
imagePullPolicy: IfNotPresent
lifecycle:
preStop:
exec:
command:
- /wait-shutdown
livenessProbe:
failureThreshold: 5
httpGet:
path: /healthz
port: 10254
scheme: HTTP
initialDelaySeconds: 10
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
name: controller
ports:
- containerPort: 80
name: http
protocol: TCP
- containerPort: 443
name: https
protocol: TCP
- containerPort: 8443
name: webhook
protocol: TCP
readinessProbe:
failureThreshold: 3
httpGet:
path: /healthz
port: 10254
scheme: HTTP
initialDelaySeconds: 10
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
resources:
requests:
cpu: 100m
memory: 90Mi
securityContext:
allowPrivilegeEscalation: true
capabilities:
add:
- NET_BIND_SERVICE
drop:
- ALL
runAsUser: 101
volumeMounts:
- mountPath: /usr/local/certificates/
name: webhook-cert
readOnly: true
dnsPolicy: ClusterFirst
nodeSelector:
kubernetes.io/os: linux
serviceAccountName: ingress-nginx
terminationGracePeriodSeconds: 300
volumes:
- name: webhook-cert
secret:
secretName: ingress-nginx-admission
[root]# kubectl apply -f deployment.yaml
[root]# kubectl get pod -n ingress-nginx
#--------------------------------------------------
# S3-09. 部署ingress class
#--------------------------------------------------
[root]# vim ingressclass.yaml
---
apiVersion: networking.k8s.io/v1
kind: IngressClass
metadata:
labels:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
name: nginx
spec:
controller: k8s.io/ingress-nginx
---
apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingWebhookConfiguration
metadata:
labels:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
name: ingress-nginx-admission
webhooks:
- admissionReviewVersions:
- v1
clientConfig:
service:
name: ingress-nginx-controller-admission
namespace: ingress-nginx
path: /networking/v1/ingresses
failurePolicy: Fail
matchPolicy: Equivalent
name: validate.nginx.ingress.kubernetes.io
rules:
- apiGroups:
- networking.k8s.io
apiVersions:
- v1
operations:
- CREATE
- UPDATE
resources:
- ingresses
sideEffects: None
[root]# kubectl create -f ingressclass.yaml
[root]# kubectl get ingressclass
NAME CONTROLLER PARAMETERS AGE
nginx k8s.io/ingress-nginx <none> 4m21s
Ingress的主要目的是接收外部流量將其導至在K8S上運行的服務
。所以實務上DNS要對應到Ingress controller的LB IP(此處為10.107.88.91, 內部建立的DNS要與其進行對應)。
可以透過對應的DNS來提供。
有二種方式:
一個domain給ingress controller與多個Path的流量路由
www.example.com ---> Loadbalancer IP
[OR]
http://www.example.com/app1
http://www.example.com/app2
http://www.example.com/app1/api
http://www.example.com/app2/api
dynamic DNS endpoints
的功能。當使用wildcard在DNS時,需要加入ingress所需 要的DNS與nginx ingress controller要負責路由的service endpoint。*.example.com --> Loadbalancer IP
*.apps.example.com --> Loadbalancer IP
這種方式可以單一ingress controller之下有多個dynamic subdomain,並且每個DNS可以有自已的path-based routing。
# URL one
http://demo1.example.com/api
http://demo1.example.com/api/v1
http://demo1.example.com/api/v2
# app specific urls
http://grafana.apps.example.com
http://prometheus.apps.example.com
# URL two
http://demo2.apps.example.com/api
http://demo2.apps.example.com/api/v1
http://demo2.apps.example.com/api/v2
#------------------------------------------------------
# S4-1. wildcard DNS Mapping
#------------------------------------------------------
[lb01]# vim /var/named/named.test.example.poc
--> Add
*.apps.test.example.poc. IN A 10.107.88.91
[lb01]# systemctl restart named
#------------------------------------------------------
# S5-1. edit app yaml
#------------------------------------------------------
[root]# vim hello-app.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: hello-app
namespace: dev
spec:
selector:
matchLabels:
app: hello
replicas: 3
template:
metadata:
labels:
app: hello
spec:
containers:
- name: hello
image: "gcr.io/google-samples/hello-app:2.0"
[root]# kubectl create -f hello-app.yaml
[root]# kubectl get all -n dev
#------------------------------------------------------
# S5-2. create svc
#------------------------------------------------------
[root]# vim hello-svc.yaml
apiVersion: v1
kind: Service
metadata:
name: hello-service
namespace: dev
labels:
app: hello
spec:
type: ClusterIP
selector:
app: hello
ports:
- port: 80
targetPort: 8080
protocol: TCP
[root]# kubectl create -f hello-svc.yaml
※建立給App使用的Ingress
目標:建立ingress,並且外部可以透過DNS 以fqdn的方式存取到App (http://demo.test.example.poc)
運作:ingress controller pod會連接到ingress api 確認路由規則
#------------------------------------------------------
# S5-3. edit ingress.yaml
#------------------------------------------------------
[root]# vim ingress.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: test-ingress
namespace: dev
spec:
ingressClassName: nginx
rules:
- host: "demo.apps.test.example.poc"
http:
paths:
- pathType: Prefix
path: "/"
backend:
service:
name: hello-service
port:
number: 80
[root]# kubectl create -f ingress.yaml
[root]# kubectl describe ingress -n dev
==> http://demo.apps.test.example.poc
#------------------------------------------------------
# S5-4. 建立另一個服務,測試不同domain name都走到同一支ingress controller
#------------------------------------------------------
[root]# cat www-app.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: www
spec:
selector:
matchLabels:
app: www
replicas: 2 # tells deployment to run 2 pods matching the template
template:
metadata:
labels:
app: www
spec:
containers:
- name: nginx
image: gcr.io/google-samples/hello-app:2.0
ports:
- containerPort: 80
[root]# cat www-ingress.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: www-ingress
namespace: dev
spec:
ingressClassName: nginx
rules:
- host: "www.apps.test.example.poc"
http:
paths:
- pathType: Prefix
path: "/"
backend:
service:
name: www-svc
port:
number: 80
[root]# cat www-svc.yaml
apiVersion: v1
kind: Service
metadata:
name: www-svc
namespace: dev
labels:
app: www
spec:
selector:
app: www
ports:
- port: 80
targetPort: 8080
protocol: TCP
[root]# kubectl create -f www-app.yaml
[root]# kubectl create -f www-svc.yaml
[root]# kubectl create -f www-ingress.yaml
#------------------------------------------------------
# S5-5. 測試
#------------------------------------------------------
[lb01]# curl demo.apps.test.example.poc
[lb01]# curl www.apps.test.example.poc
因為將程式碼的部分也直接給大家看內容,所以篇幅有點太長,但為了大家如果想要直接使用這些內容,就不用再跳到其他外部網站,在閱讀上比較方便,也請大家見諒。
在容器化的世界了解ingress是很重要的,本篇雖然沒有從頭開始講解,但透過實務上的操作之後,再回過頭了重新了解ingress的基礎也許會更有感覺。
今天的分享就到這邊為止,下期再見。