To benefit Kubernetes flexible deployment scaling, consider the following conditions:
- Do not rely on Nginx Ingress load balance method, specifically the
ip-hashfor balanced loads - An app should use shared storage accessible through all its instances, such for session storage or persistent files
The following deployment shows typical usage for containerized app which includes MySQL, MongoDB, and PHP. All app will be grouped in a namespace to simplify configuration and management.
Deploy Common Data Such as Persistent Storage, Certificate, and Host Key
-
Apply namespace
vi myapp/namespace.yamlkind: Namespace apiVersion: v1 metadata: name: myappkubectl apply -f myapp/namespace.yamlnamespace/myapp created -
Claim a persistent volume as code repository, a NFS export on
stagingvm must has been configured for/home/appexportsvi myapp/pvc.yamlapiVersion: v1 kind: PersistentVolume metadata: name: myapp-app-pv spec: capacity: storage: 20Gi accessModes: - ReadWriteMany nfs: server: 10.0.0.19 path: /home/app mountOptions: - nfsvers=4.2 --- apiVersion: v1 kind: PersistentVolumeClaim metadata: name: app-pvc namespace: myapp spec: accessModes: - ReadWriteMany storageClassName: "" resources: requests: storage: 20Gi volumeName: myapp-app-pvkubectl apply -f myapp/pvc.yamlpersistentvolume/myapp-app-pv created persistentvolumeclaim/app-pvc created -
Store TLS certificate for web server as ConfigMap
vi myapp/cert.yamlapiVersion: v1 kind: ConfigMap metadata: name: cert namespace: myapp data: cert.key: | -----BEGIN PRIVATE KEY----- PLACE PRIVATE KEY HERE -----END PRIVATE KEY----- cert.crt-combined: | -----BEGIN CERTIFICATE----- PLACE CERTIFICATE HERE -----END CERTIFICATE----- -----BEGIN CERTIFICATE----- PLACE CERTIFICATE HERE -----END CERTIFICATE-----kubectl apply -f myapp/cert.yamlconfigmap/cert created -
For SSH-ing into container passwordless, store host key as ConfigMap. SSH can be useful to run specific command from cron job
vi myapp/hostkey.yamlapiVersion: v1 kind: ConfigMap metadata: name: hostkey namespace: myapp data: id_rsa: | -----BEGIN OPENSSH PRIVATE KEY----- PLACE PRIVATE KEY HERE -----END OPENSSH PRIVATE KEY----- id_rsa.pub: | PLACE PUBLIC KEY HEREkubectl apply -f myapp/hostkey.yamlconfigmap/hostkey created
Deploy MySQL
-
Review MySQL cluster configuration
vi myapp/mysql.yamlapiVersion: mysql.oracle.com/v2 kind: InnoDBCluster metadata: name: mysql namespace: myapp spec: secretName: mysql-password instances: 1 router: instances: 1 tlsUseSelfSigned: true datadirVolumeClaimTemplate: accessModes: - ReadWriteMany resources: requests: storage: 100Gi storageClassName: nfs-data-1 podSpec: metadata: namespace: myap containers: - name: mysql resources: limits: cpu: "2.0" memory: 8192Mi - name: sidecar resources: limits: cpu: "0.5" memory: 512Mi initContainers: - name: fixdatadir resources: limits: cpu: "0.25" memory: 256Mi - name: initconf resources: limits: cpu: "0.25" memory: 256Mi - name: initmysql resources: limits: cpu: "0.5" memory: 512Mi affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchExpressions: - key: mysql.oracle.com/cluster operator: In values: - mysql topologyKey: kubernetes.io/hostname service: type: LoadBalancer annotations: metallb.universe.tf/allow-shared-ip: "shared-ip-myapp" metallb.universe.tf/loadBalancerIPs: 10.0.0.22 version: 8.0.35 --- apiVersion: v1 kind: Secret metadata: name: mysql-password namespace: myapp stringData: rootUser: root rootHost: '%' rootPassword: password -
Deploy MySQL cluster
kubectl apply -f myapp/mysql.yamlinnodbcluster.mysql.oracle.com/mysql created secret/mysql-password created
Deploy MongoDB
-
Apply RBAC for MongoDB in app namespace
kubectl apply -k mongodb-operator/config/rbac/ -n myappserviceaccount/mongodb-database created serviceaccount/mongodb-kubernetes-operator created role.rbac.authorization.k8s.io/mongodb-database created role.rbac.authorization.k8s.io/mongodb-kubernetes-operator created rolebinding.rbac.authorization.k8s.io/mongodb-database created rolebinding.rbac.authorization.k8s.io/mongodb-kubernetes-operator created -
Review MongoDB cluster configuration
vi myapp/mongodb.yamlapiVersion: mongodbcommunity.mongodb.com/v1 kind: MongoDBCommunity metadata: name: mongodb namespace: myapp spec: members: 1 type: ReplicaSet version: "7.0.4" security: authentication: modes: ["SCRAM"] users: - name: user db: admin passwordSecretRef: # a reference to the secret that will be used to generate the user's password name: mongodb-user-password roles: - name: root db: admin scramCredentialsSecretName: mongodb additionalMongodConfig: storage.wiredTiger.engineConfig.journalCompressor: zlib statefulSet: spec: volumeClaimTemplates: - metadata: name: data-volume spec: storageClassName: nfs-data-1 accessModes: - ReadWriteOnce resources: requests: storage: 100Gi - metadata: name: logs-volume spec: storageClassName: nfs-data-1 template: spec: containers: - name: mongod securityContext: runAsNonRoot: false runAsUser: 0 runAsGroup: 0 resources: limits: cpu: "1.0" memory: 2048Mi requests: cpu: "1.0" memory: 2048Mi - name: mongodb-agent securityContext: runAsNonRoot: false runAsUser: 0 runAsGroup: 0 resources: limits: cpu: "0.5" memory: 400Mi initContainers: - name: mongod-posthook resources: limits: cpu: "0.25" memory: 128Mi - name: mongodb-agent-readinessprobe resources: limits: cpu: "0.25" memory: 128Mi securityContext: fsGroup: 0 affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchExpressions: - key: app operator: In values: - mongodb-svc topologyKey: kubernetes.io/hostname # the user credentials will be generated from this secret # once the credentials are generated, this secret is no longer required --- apiVersion: v1 kind: Service metadata: name: mongodb namespace: myapp labels: app: mongodb-svc annotations: metallb.universe.tf/allow-shared-ip: "shared-ip-myapp" metallb.universe.tf/loadBalancerIPs: 10.0.0.22 spec: selector: app: mongodb-svc type: LoadBalancer ports: - name: mongodb port: 27017 targetPort: 27017 --- apiVersion: v1 kind: Secret metadata: name: mongodb-user-password namespace: myapp type: Opaque stringData: password: password -
Deploy MongoDB cluster
kubectl apply -f myapp/mongodb.yamlmongodbcommunity.mongodbcommunity.mongodb.com/mongodb created secret/mongodb-user-password created
Deploy PHP app
-
Prepare PHP code for app id
myappinstagingvm, its code should be located at/home/app/apps/myapp. It is encouraged to create distribution tarball/home/app/apps/myapp/dist/app.tgzfor container speed upcd /home/app/apps/myapp mkdir -p dist rm -rf dist/* tar -czf dist/app.tgz -p --exclude=./dist --exclude-vcs --exclude-vcs-ignores . -
Review PHP app configuration
vi myapp/webapp.yamlapiVersion: apps/v1 kind: Deployment metadata: name: webapp namespace: myapp labels: app: webapp tier: frontend spec: replicas: 5 selector: matchLabels: app: webapp tier: frontend template: metadata: namespace: myapp labels: app: webapp tier: frontend spec: containers: - name: php-apache image: docker.io/php:8.1.26-apache resources: limits: cpu: "2.0" memory: 2048Mi ports: - containerPort: 22 - containerPort: 443 volumeMounts: - name: app-data mountPath: /home/www - name: app-init-data mountPath: /data - name: app-repository mountPath: /repo - name: cert mountPath: /cert - name: hostkey mountPath: /hostkey command: - /bin/bash - "-c" - | RETRY=1 LOG=/var/log/install.log set_timezone() { ln -sf /usr/share/zoneinfo/${APP_TIMEZONE} /etc/localtime dpkg-reconfigure -f noninteractive tzdata } apt_mirror() { [ -f /etc/apt/sources.list.d/debian.sources ] && \ sed -i -e "s/deb.debian.org/kartolo.sby.datautama.net.id/g" /etc/apt/sources.list.d/debian.sources } apt_updates() { for i in {1..$RETRY}; do apt-get update done } apt_install() { for i in {1..$RETRY}; do apt-get install -y $@ done } # initialize set_timezone apt_mirror apt_updates>>$LOG # set apache site configuration rm -rf ${APACHE_CONFDIR}/sites-enabled/* ln -s /data/app.conf ${APACHE_CONFDIR}/sites-enabled/ # patch listen ports if [ -n "${APP_HTTP_PORT}" ]; then sed -i -e "s/Listen 80/Listen ${APP_HTTP_PORT}/g" ${APACHE_CONFDIR}/ports.conf fi if [ -n "${APP_HTTPS_PORT}" ]; then sed -i -e "s/Listen 443/Listen ${APP_HTTPS_PORT}/g" ${APACHE_CONFDIR}/ports.conf fi # adjust workers and connections if [ -n "${APP_HTTP_WORKERS}" ]; then sed -i -e "s/MaxRequestWorkers 150/MaxRequestWorkers ${APP_HTTP_WORKERS}/g" \ ${APACHE_CONFDIR}/mods-available/mpm_prefork.conf fi if [ -n "${APP_HTTP_CONNECTIONS}" ]; then sed -i -e "s/MaxConnectionsPerChild 0/MaxConnectionsPerChild ${APP_HTTP_CONNECTIONS}/g" \ ${APACHE_CONFDIR}/mods-available/mpm_prefork.conf fi # enable ssl and rewrite module a2enmod ssl rewrite # run initialization if [ -f /data/init.sh ]; then cp /data/init.sh ~/init.sh chmod +x ~/init.sh ~/init.sh & fi # run entrypoint docker-php-entrypoint apache2-foreground workingDir: /home/www env: - name: APP_ID valueFrom: configMapKeyRef: name: webapp-data key: app_id - name: APP_DEBUG valueFrom: configMapKeyRef: name: webapp-data key: app_debug - name: APP_TIMEZONE value: "Asia/Jakarta" - name: APP_HTTP_WORKERS value: "40" - name: APP_PERSISTENT_STORAGES value: "uploads" readinessProbe: exec: command: - cat - /tmp/.ready initialDelaySeconds: 60 periodSeconds: 60 failureThreshold: 10 initContainers: - name: sync-apps image: docker.io/instrumentisto/rsync-ssh:alpine resources: limits: cpu: "0.5" memory: 128Mi resources: limits: cpu: "0.5" memory: 128Mi command: - /bin/sh - "-c" - | APP_DIR=/home/www APP_SRC=/repo/apps/${APP_ID}/ mkdir -p ${APP_DIR} if [ "x$SYNC_DIST" = "xtrue" -a -f $APP_SRC/dist/app.tgz ]; then tar -xvzf $APP_SRC/dist/app.tgz --exclude-from=/data/exclude.txt -C ${APP_DIR} else rsync -avzi --exclude-from=/data/exclude.txt ${APP_SRC} ${APP_DIR} fi env: - name: APP_ID valueFrom: configMapKeyRef: name: webapp-data key: app_id - name: SYNC_DIST value: "true" volumeMounts: - name: app-data mountPath: /home/www - name: app-init-data mountPath: /data - name: app-repository mountPath: /repo volumes: - name: app-data emptyDir: {} - name: app-init-data configMap: name: webapp-data items: - key: exclude.txt path: exclude.txt - key: app.conf path: app.conf - key: init.sh path: init.sh - key: addons.lst path: addons.lst - key: sshd.sh path: sshd.sh - name: app-repository persistentVolumeClaim: claimName: app-pvc - name: cert configMap: name: cert items: - key: cert.key path: cert.key - key: cert.crt-combined path: cert.crt-combined - name: hostkey configMap: name: hostkey items: - key: id_rsa path: id_rsa - key: id_rsa.pub path: id_rsa.pub --- apiVersion: v1 kind: Service metadata: name: web namespace: myapp labels: app: web-svc tier: frontend spec: selector: app: webapp tier: frontend ports: - name: https port: 443 targetPort: 443 sessionAffinity: ClientIP --- apiVersion: v1 kind: Service metadata: name: ssh namespace: myapp labels: app: ssh-svc tier: frontend annotations: metallb.universe.tf/allow-shared-ip: "shared-ip-myapp" metallb.universe.tf/loadBalancerIPs: 10.0.0.22 spec: selector: app: webapp tier: frontend type: LoadBalancer ports: - name: ssh port: 22 targetPort: 22 sessionAffinity: ClientIP --- apiVersion: v1 kind: ConfigMap metadata: name: webapp-data namespace: myapp data: app_id: myapp app_debug: "false" init.sh: | #!/bin/bash echo "===basename $0===" LOG=/var/log/init.log RETRY=1 apt_install() { for i in {1..$RETRY}; do apt-get install -y $@>>$LOG done } get_php_ini() { KEY=$1 IFS="=>" read -r -a ARR <<<php -i | grep "^${KEY}"echo ${ARR[2]} | xargs } php_ext_enabled() { EXT_INI="$PHP_INI_DIR/conf.d/docker-php-ext-$1.ini" if [ -f $EXT_INI ]; then echo 1 else echo 0 fi } CACHE_DIR=/repo/lib/php-$(php -v | awk '/PHP ([0-9]\.[0-9]\.[0-9]+)/{print $2}')/$(uname -m) EXTENSIONS="gd mysqli pdo_mysql zip" PECL_EXTENSIONS="mongodb xdebug" PHP_INI_DIR=get_php_ini 'Configuration File (php.ini) Path'PHP_EXT_DIR=get_php_ini 'extension_dir'echo "PHP ini dir = ${PHP_INI_DIR}...">>$LOG echo "Extension dir = ${PHP_EXT_DIR}...">>$LOG mkdir -p ${CACHE_DIR}>>$LOG # install dependencies apt_install imagemagick # install PHP extensions for EXT in ${EXTENSIONS}; do if [php_ext_enabled ${EXT}-eq 1 ]; then echo "Extension ${EXT} already enabled, skipping..." continue fi if [ -f "${CACHE_DIR}/${EXT}.so" ]; then cp "${CACHE_DIR}/${EXT}.so" "${PHP_EXT_DIR}/${EXT}.so">>$LOG PACKAGES="" case "${EXT}" in gd) PACKAGES="${PACKAGES} libfreetype6 libjpeg62-turbo libpng16-16 libxpm4 libwebp7 zlib1g";; zip) PACKAGES="${PACKAGES} libzip4";; esac if [ -n "${PACKAGES}" ]; then apt_install ${PACKAGES} fi docker-php-ext-enable ${EXT}>>$LOG else PACKAGES="" CONFIGURES="" case "${EXT}" in gd) PACKAGES="${PACKAGES} libfreetype6-dev libjpeg62-turbo-dev libpng-dev libxpm-dev libwebp-dev zlib1g-dev" CONFIGURES="--with-freetype --with-jpeg --with-xpm --with-webp";; zip) PACKAGES="${PACKAGES} libzip-dev";; esac if [ -n "${PACKAGES}" ]; then apt_install ${PACKAGES} fi if [ -n "${CONFIGURES}" ]; then docker-php-ext-configure ${EXT} ${CONFIGURES}>>$LOG fi docker-php-ext-install -j$(nproc) ${EXT}>>$LOG cp "${PHP_EXT_DIR}/${EXT}.so" "${CACHE_DIR}/${EXT}.so">>$LOG fi done # install PHP pecl extensions for EXT in ${PECL_EXTENSIONS}; do if [php_ext_enabled ${EXT}-eq 1 ]; then echo "Extension ${EXT} already enabled, skipping..." continue fi if [ -f "${CACHE_DIR}/${EXT}.so" ]; then cp "${CACHE_DIR}/${EXT}.so" "${PHP_EXT_DIR}/${EXT}.so">>$LOG docker-php-ext-enable ${EXT}>>$LOG else PACKAGES="" case "${EXT}" in mongodb) PACKAGES="${PACKAGES} libssl-dev";; xdebug) if [ "${EXT}" == "xdebug" ]; then if [ "${APP_DEBUG}" != "true" ]; then continue fi fi;; esac if [ -n "${PACKAGES}" ]; then apt_install ${PACKAGES} fi pecl install ${EXT}>>$LOG docker-php-ext-enable ${EXT}>>$LOG cp "${PHP_EXT_DIR}/${EXT}.so" "${CACHE_DIR}/${EXT}.so">>$LOG fi done # prepare php.ini PHP_INI=${PHP_INI_DIR}/php.ini cp ${PHP_INI}-production ${PHP_INI} sed -i -e "s#memory_limit = 128M#memory_limit = 1024M#g" \ -e "s#max_execution_time = 30#max_execution_time = 60#g" \ -e "s#post_max_size = 8M#post_max_size = 0#g" \ -e "s#;date.timezone =#date.timezone = ${APP_TIMEZONE}#g" \ ${PHP_INI} # run add-ons if [ -f /data/addons.lst ]; then for ADDON incat /data/addons.lst; do if [ -f /data/${ADDON} ]; then echo "=== ${ADDON} ===" cp /data/${ADDON} ~/${ADDON} chmod +x ~/${ADDON} ~/${ADDON} fi done fi # reload apache /etc/init.d/apache2 reload>>$LOG # mark as ready touch /tmp/.ready addons.lst: | sshd.sh sshd.sh: | #!/bin/bash LOG=/var/log/sshd.log # start sshd if [ -f /hostkey/id_rsa.pub ]; then mkdir -p ~/.ssh cp /hostkey/id_rsa.pub ~/.ssh/authorized_keys chmod 0640 ~/.ssh/authorized_keys apt-get install -y openssh-server>>$LOG # prepare openssh-server configuration SSHD_CONFIG=/etc/ssh/sshd_config if [ -n "${APP_SSH_PORT}" ]; then sed -i -e "s/#Port 22/Port ${APP_SSH_PORT}/g" ${SSHD_CONFIG} fi sed -i -e "s/#PermitRootLogin prohibit-password/PermitRootLogin yes/g" ${SSHD_CONFIG} mkdir -p /run/sshd touch ~/.Xauthority /usr/sbin/sshd -D & fi exclude.txt: | .git/ app/cache/* app/log/* app.conf: |ServerName example.com ServerAlias www.example.com ServerAdmin user@example.com SSLEngine on SSLCertificateFile /cert/cert.crt-combined SSLCertificateKeyFile /cert/cert.key LogLevel warn ErrorLog /dev/stderr CustomLog /var/log/apache2/app.access.log combined DocumentRoot "/home/www/app/public/" Options FollowSymLinks AllowOverride None Options FollowSymLinks AllowOverride All Require all granted -
Deploy the app
kubectl apply -f myapp/webapp.yamldeployment.apps/webapp created service/web created service/ssh created configmap/webapp-data created -
Create Nginx Ingress
vi myapp/ingress.yamlkind: Ingress apiVersion: networking.k8s.io/v1 metadata: name: nginx-ingress-master namespace: myapp labels: app: nginx-master tier: frontend annotations: nginx.org/mergeable-ingress-type: "master" nginx.org/redirect-to-https: "true" spec: ingressClassName: nginx tls: - hosts: - example.com rules: - host: example.com --- kind: Ingress apiVersion: networking.k8s.io/v1 metadata: name: nginx-ingress namespace: myapp labels: app: nginx-ingress tier: frontend annotations: nginx.org/mergeable-ingress-type: "minion" nginx.org/ssl-services: "web" spec: ingressClassName: nginx rules: - host: example.com http: paths: - path: / pathType: Prefix backend: service: name: web port: name: httpskubectl apply -f myapp/ingress.yamlingress.networking.k8s.io/nginx-ingress-master created ingress.networking.k8s.io/nginx-ingress created -
Verify ingress is running
kubectl get ingress -n myapp -o wideNAME CLASS HOSTS ADDRESS PORTS AGE nginx-ingress nginx example.com 10.0.0.22 80 7d2h nginx-ingress-master nginx example.com 10.0.0.22 80, 443 7d2h
Create A Cron Job
In this example, assumed there is a task configured at /home/www/cron/deliver-notification in the web app, the following steps show how to add a cron job at specified time to run those task.
-
Review cron job configuration
vi myapp/cron.yamlapiVersion: batch/v1 kind: CronJob metadata: name: deliver-notification namespace: myapp labels: app: myapp-job spec: schedule: "0 8,16 * * *" timeZone: Asia/Jakarta successfulJobsHistoryLimit: 1 jobTemplate: spec: template: spec: containers: - name: cron image: docker.io/debian:bookworm-slim command: - /bin/bash - -c - | cp /cron/cron.sh ~/cron.sh chmod +x ~/cron.sh ~/cron.sh /home/www/cron/deliver-notification volumeMounts: - name: cron mountPath: /cron - name: hostkey mountPath: /hostkey volumes: - name: cron configMap: name: cron-data items: - key: cron.sh path: cron.sh - key: cron.var path: cron.var - name: hostkey configMap: name: hostkey items: - key: id_rsa path: id_rsa - key: id_rsa.pub path: id_rsa.pub restartPolicy: Never backoffLimit: 1 --- apiVersion: v1 kind: ConfigMap metadata: name: cron-data namespace: myapp data: cron.sh: | #!/bin/bash . /cron/cron.var # prepare hostkey FILES="id_rsa id_rsa.pub" for F in ${FILES}; do if [ -f /hostkey/${F} ]; then if [ ! -d ~/.ssh ]; then mkdir -p ~/.ssh; fi cp /hostkey/${F} ~/.ssh/${F} if [ "${F}" == "id_rsa" ]; then chmod 0600 ~/.ssh/${F} else chmod 0640 ~/.ssh/${F} fi fi done # bootstrap openssh-client [ -f /etc/apt/sources.list.d/debian.sources ] && \ sed -i -e "s/deb.debian.org/kartolo.sby.datautama.net.id/g" /etc/apt/sources.list.d/debian.sources apt-get update>/dev/null apt-get install -y openssh-client>/dev/null # pick host IFS=" " read -r -a ARR <<< ${APP_HOSTS} SZ=${#ARR[@]} IDX=$(($RANDOM % $SZ)) APP_HOST=${ARR[$IDX]} # execute command ssh-keyscan -H -p ${APP_PORT} -t ecdsa-sha2-nistp256 ${APP_HOST}> ~/.ssh/known_hosts ssh ${APP_USER}@${APP_HOST} -p ${APP_PORT} $1 sleep 10 cron.var: | APP_HOSTS=ssh APP_USER=root APP_PORT=22 -
Apply cron job
kubectl apply -f myapp/cron.yamlcronjob.batch/deliver-notification created configmap/cron-data created
Verify Deployment
-
Make sure all services is running
kubectl get service -n myapp -o wideNAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE SELECTOR mongodb LoadBalancer 10.102.152.114 10.0.0.22 27017:31898/TCP 28d app=mongodb-svc mongodb-svc ClusterIP None27017/TCP 327d app=mongodb-svc mysql LoadBalancer 10.103.254.22 10.0.0.22 3306:31695/TCP,33060:30351/TCP,6446:31628/TCP,6448:32739/TCP,6447:30746/TCP,6449:30332/TCP,6450:30904/TCP,8443:30520/TCP 298d component=mysqlrouter,mysql.oracle.com/cluster=mysql,tier=mysql mysql-instances ClusterIP None 3306/TCP,33060/TCP,33061/TCP 298d component=mysqld,mysql.oracle.com/cluster=mysql,tier=mysql ssh LoadBalancer 10.102.187.161 10.0.0.22 22:31788/TCP 7d13h app=webapp,tier=frontend web ClusterIP 10.106.154.197 443/TCP 7d13h app=webapp,tier=frontend
Exposing App using Reverse Proxy
Here is an example of Nginx configuration to serve app at https://example.com/.
server {
listen 443 ssl;
server_name example.com;
location / {
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $host;
proxy_http_version 1.1;
proxy_pass https://10.0.0.22/;
}
}What's Next
ā Scheduled backup
ā Operator and provisioner deployment