feat(server): add tls config for ingress gateway (#1569)

build/charts/laf-server/templates/deployment.yaml

@@ -77,6 +77,10 @@ spec:
               value: {{ .Values.default_region.runtime_domain }}
             - name: DEFAULT_REGION_WEBSITE_DOMAIN
               value: {{ .Values.default_region.website_domain }}
+            - name: DEFAULT_REGION_TLS_ENABLED
+              value: {{ .Values.default_region.tls.enabled }}
+            - name: DEFAULT_REGION_TLS_WILDCARD_CERTIFICATE_SECRET_NAME
+              value: {{ .Values.default_region.tls.wildcard_certificate_secret_name }}
             - name: DEFAULT_REGION_LOG_SERVER_URL
               value: {{ .Values.default_region.log_server_url }}
             - name: DEFAULT_REGION_LOG_SERVER_SECRET

build/charts/laf-server/values.yaml

@@ -19,7 +19,9 @@ default_region:
   minio_root_access_key: ""
   minio_root_secret_key: ""
   # gateway conf
-  tls: false
+  tls:
+    enabled: false
+    wildcard_certificate_secret_name: ""
   runtime_domain: ""
   website_domain: ""
   # log-server

build/start.sh

@@ -95,7 +95,7 @@ helm install server -n ${NAMESPACE} \
     --set default_region.minio_root_secret_key=${MINIO_ROOT_SECRET_KEY} \
     --set default_region.runtime_domain=${DOMAIN} \
     --set default_region.website_domain=site.${DOMAIN} \
-    --set default_region.tls=false \
+    --set default_region.tls.enabled=false \
     --set default_region.log_server_url=${LOG_SERVER_URL} \
     --set default_region.log_server_secret=${LOG_SERVER_SECRET} \
     --set default_region.log_server_database_url=${LOG_SERVER_DATABASE_URL} \

server/src/constants.ts

@@ -113,11 +113,25 @@ export class ServerConfig {
   }
 
   static get DEFAULT_REGION_RUNTIME_DOMAIN() {
-    return process.env.DEFAULT_REGION_RUNTIME_DOMAIN || 'localhost'
+    if (!process.env.DEFAULT_REGION_RUNTIME_DOMAIN) {
+      throw new Error('DEFAULT_REGION_RUNTIME_DOMAIN is not defined')
+    }
+    return process.env.DEFAULT_REGION_RUNTIME_DOMAIN
   }
 
   static get DEFAULT_REGION_WEBSITE_DOMAIN() {
-    return process.env.DEFAULT_REGION_WEBSITE_DOMAIN || 'localhost'
+    if (!process.env.DEFAULT_REGION_WEBSITE_DOMAIN) {
+      throw new Error('DEFAULT_REGION_WEBSITE_DOMAIN is not defined')
+    }
+    return process.env.DEFAULT_REGION_WEBSITE_DOMAIN
+  }
+
+  static get DEFAULT_REGION_TLS_ENABLED() {
+    return process.env.DEFAULT_REGION_TLS_ENABLED === 'true'
+  }
+
+  static get DEFAULT_REGION_TLS_WILDCARD_CERTIFICATE_SECRET_NAME() {
+    return process.env.DEFAULT_REGION_TLS_WILDCARD_CERTIFICATE_SECRET_NAME
   }
 
   static get DEFAULT_REGION_MINIO_DOMAIN() {

server/src/gateway/ingress/runtime-ingress.service.ts

@@ -1,4 +1,4 @@
-import { V1Ingress } from '@kubernetes/client-node'
+import { V1Ingress, V1IngressTLS } from '@kubernetes/client-node'
 import { Injectable, Logger } from '@nestjs/common'
 import { LABEL_KEY_APP_ID } from 'src/constants'
 import { ClusterService } from 'src/region/cluster/cluster.service'
@@ -55,11 +55,20 @@ export class RuntimeGatewayService {
     })
 
     // build tls
-    const tls = []
-    if (runtimeDomain.customDomain) {
-      const secretName =
-        this.certificate.getRuntimeCertificateName(runtimeDomain)
-      tls.push({ secretName, hosts })
+    const tls: Array<V1IngressTLS> = []
+    if (region.gatewayConf.tls.enabled) {
+      // add wildcardDomain tls
+      if (region.gatewayConf.tls.wildcardCertificateSecretName) {
+        const secretName = region.gatewayConf.tls.wildcardCertificateSecretName
+        tls.push({ secretName, hosts: [runtimeDomain.domain] })
+      }
+
+      // add customDomain tls
+      if (runtimeDomain.customDomain) {
+        const secretName =
+          this.certificate.getRuntimeCertificateName(runtimeDomain)
+        tls.push({ secretName, hosts: [runtimeDomain.customDomain] })
+      }
     }
 
     // create ingress

server/src/gateway/ingress/website-ingress.service.ts

@@ -1,16 +1,20 @@
-import { V1Ingress, V1IngressRule } from '@kubernetes/client-node'
+import { V1Ingress, V1IngressRule, V1IngressTLS } from '@kubernetes/client-node'
 import { Injectable, Logger } from '@nestjs/common'
 import { LABEL_KEY_APP_ID } from 'src/constants'
 import { ClusterService } from 'src/region/cluster/cluster.service'
 import { Region } from 'src/region/entities/region'
 import { GetApplicationNamespace } from 'src/utils/getter'
 import { WebsiteHosting } from 'src/website/entities/website'
+import { CertificateService } from '../certificate.service'
 
 @Injectable()
 export class WebsiteHostingGatewayService {
   private readonly logger = new Logger(WebsiteHostingGatewayService.name)
 
-  constructor(private readonly clusterService: ClusterService) {}
+  constructor(
+    private readonly clusterService: ClusterService,
+    private readonly certificate: CertificateService,
+  ) {}
 
   getIngressName(websiteHosting: WebsiteHosting) {
     return websiteHosting._id.toString()
@@ -44,6 +48,20 @@ export class WebsiteHostingGatewayService {
       http: { paths: [{ path: '/', pathType: 'Prefix', backend }] },
     }
 
+    // build tls
+    const tls: Array<V1IngressTLS> = []
+    if (region.gatewayConf.tls.enabled) {
+      if (website.isCustom) {
+        // add custom domain tls
+        const secretName = this.certificate.getWebsiteCertificateName(website)
+        tls.push({ secretName, hosts: [website.domain] })
+      } else if (region.gatewayConf.tls.wildcardCertificateSecretName) {
+        // add wildcardDomain tls
+        const secretName = region.gatewayConf.tls.wildcardCertificateSecretName
+        tls.push({ secretName, hosts: [website.domain] })
+      }
+    }
+
     // create ingress
     const ingressClassName = region.gatewayConf.driver
     const ingressBody: V1Ingress = {
@@ -62,7 +80,7 @@ export class WebsiteHostingGatewayService {
           'nginx.ingress.kubernetes.io/enable-cors': 'true',
         },
       },
-      spec: { ingressClassName, rules: [rule] },
+      spec: { ingressClassName, rules: [rule], tls },
     }
 
     const res = await this.clusterService.createIngress(region, ingressBody)

server/src/gateway/runtime-domain.service.ts

@@ -42,9 +42,7 @@ export class RuntimeDomainService {
   async checkResolved(appid: string, customDomain: string) {
     const runtimeDomain = await this.db
       .collection<RuntimeDomain>('RuntimeDomain')
-      .findOne({
-        appid,
-      })
+      .findOne({ appid })
 
     const cnameTarget = runtimeDomain.domain
 

server/src/gateway/website-task.service.ts

@@ -99,6 +99,7 @@ export class WebsiteTaskService {
     assert(bucketDomain, 'bucket domain not found')
 
     // create website custom certificate if custom domain is set
+    // Warning: create certificate before ingress, otherwise apisix ingress will not work
     if (site.isCustom && region.gatewayConf.tls.enabled) {
       const waitingTime = Date.now() - site.updatedAt.getTime()
 

server/src/initializer/initializer.service.ts

@@ -76,9 +76,10 @@ export class InitializerService {
         websiteDomain: ServerConfig.DEFAULT_REGION_WEBSITE_DOMAIN,
         port: 80,
         tls: {
-          enabled: false,
-          issuerRef: { name: 'laf-issuer', kind: 'ClusterIssuer' },
-          wildcardCertificateSecretName: null,
+          enabled: ServerConfig.DEFAULT_REGION_TLS_ENABLED,
+          issuerRef: { name: 'laf-issuer', kind: 'Issuer' },
+          wildcardCertificateSecretName:
+            ServerConfig.DEFAULT_REGION_TLS_WILDCARD_CERTIFICATE_SECRET_NAME,
         },
       },
       logServerConf: {