diff --git a/clusters/wb/infrastructure/kustomization.yaml b/clusters/wb/infrastructure/kustomization.yaml index 6cc40fa..0266a4f 100644 --- a/clusters/wb/infrastructure/kustomization.yaml +++ b/clusters/wb/infrastructure/kustomization.yaml @@ -10,6 +10,8 @@ resources: - ../../../infrastructure/goalert - ../../../infrastructure/kafka-exporter - ../../../infrastructure/postgres-exporter + - ../../../infrastructure/superset + - ../../../infrastructure/trino - ./node-exporter-vmnodescrape.yaml patches: - path: ./patches/istio-config.yaml @@ -75,3 +77,17 @@ patches: kind: HelmRelease name: postgres-exporter namespace: postgres-exporter + - path: ./patches/superset.yaml + target: + group: helm.toolkit.fluxcd.io + version: v2 + kind: HelmRelease + name: superset + namespace: superset + - path: ./patches/trino.yaml + target: + group: helm.toolkit.fluxcd.io + version: v2 + kind: HelmRelease + name: trino + namespace: trino diff --git a/clusters/wb/infrastructure/patches/istio-config.yaml b/clusters/wb/infrastructure/patches/istio-config.yaml index 2cd2460..67452de 100644 --- a/clusters/wb/infrastructure/patches/istio-config.yaml +++ b/clusters/wb/infrastructure/patches/istio-config.yaml @@ -51,6 +51,16 @@ spec: - openobserve-srx.wb.ru tls: credentialName: istio-gateway-wildcard-cert + superset: + name: superset-gw + namespace: gateway + selector: + istio: ingressgateway + servers: + - hosts: + - superset-srx.wb.ru + tls: + credentialName: istio-gateway-wildcard-cert vmalert: name: vmalert-gw namespace: gateway @@ -95,6 +105,17 @@ spec: prefix: / service: openobserve-web.openobserve.svc.cluster.local port: 5080 + superset-vs: + namespace: superset + hosts: + - superset-srx.wb.ru + gateways: + - gateway/superset-gw + routes: + - path: + prefix: / + service: superset.superset.svc.cluster.local + port: 8088 vmalert-vs: namespace: goalert hosts: diff --git a/clusters/wb/infrastructure/patches/superset.yaml b/clusters/wb/infrastructure/patches/superset.yaml new file mode 100644 index 0000000..f8de927 --- /dev/null +++ b/clusters/wb/infrastructure/patches/superset.yaml @@ -0,0 +1,102 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: superset + namespace: superset +spec: + interval: 5m + timeout: 20m + values: + fullnameOverride: superset + vault: + enabled: false + stronghold: + enabled: false + extraEnv: + GUNICORN_TIMEOUT: "300" + SERVER_WORKER_AMOUNT: "4" + BABEL_DEFAULT_LOCALE: ru + extraEnvRaw: + - name: ENABLE_PROXY_FIX + value: "true" + - name: DB_PASS + valueFrom: + secretKeyRef: + name: postgresql-auth + key: password + - name: SUPERSET_SECRET_KEY + valueFrom: + secretKeyRef: + name: superset-secret + key: secret.key + - name: JWT_SECRET + valueFrom: + secretKeyRef: + name: jwt-secret + key: jwt_secret + supersetNode: + connections: + redis_host: superset-redis-headless + redis_port: "6379" + redis_user: "" + redis_cache_db: "1" + redis_celery_db: "0" + db_host: sarex-vpsql-01.xc.wb.ru + db_port: "5432" + db_user: superset + db_name: ss + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + memory: 2Gi + supersetWorker: + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + memory: 2Gi + init: + loadExamples: false + postgresql: + enabled: false + redis: + enabled: true + architecture: standalone + auth: + enabled: false + existingSecret: "" + existingSecretKey: "" + password: "" + master: + persistence: + enabled: false + configOverrides: + feature_flags: | + FEATURE_FLAGS = { + "EMBEDDED_SUPERSET": True, + "ENABLE_TEMPLATE_PROCESSING": True, + } + iframe_config: | + X_FRAME_OPTIONS = None + HTTP_HEADERS = { + "Content-Security-Policy": "frame-ancestors https://*.wb.ru", + } + extend_timeout: | + SQLLAB_ASYNC_TIME_LIMIT_SEC = 300 + SUPERSET_WEBSERVER_TIMEOUT = 300 + SQLLAB_TIMEOUT = 600 + set_locale: | + BABEL_DEFAULT_LOCALE = "ru" + enable_oauth: "" + extraConfigs: + import_datasources.yaml: | + databases: + - database_name: trino + sqlalchemy_uri: trino://superset@trino.trino.svc.cluster.local:8080 + expose_in_sqllab: true + allow_ctas: true + allow_cvas: true + allow_dml: false diff --git a/clusters/wb/infrastructure/patches/trino.yaml b/clusters/wb/infrastructure/patches/trino.yaml new file mode 100644 index 0000000..afd3460 --- /dev/null +++ b/clusters/wb/infrastructure/patches/trino.yaml @@ -0,0 +1,112 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: trino + namespace: trino +spec: + interval: 5m + timeout: 20m + values: + nameOverride: trino + coordinatorNameOverride: trino-coordinator + workerNameOverride: trino-worker + gateway: + enabled: false + virtualService: + enabled: false + server: + workers: 1 + config: + authenticationType: "" + autoscaling: + enabled: false + auth: + passwordAuthSecret: "" + env: + - name: PROD_PG_RO_PASSWORD + valueFrom: + secretKeyRef: + name: postgres-secret + key: postgres_ro_password + catalogs: + sarex_db: | + connector.name=postgresql + connection-url=jdbc:postgresql://sarex-vpsql-01.xc.wb.ru:5432/sarex_db?sslmode=disable + connection-user=readonly + connection-password=${ENV:PROD_PG_RO_PASSWORD} + postgresql.array-mapping=AS_JSON + flows_db: | + connector.name=postgresql + connection-url=jdbc:postgresql://sarex-vpsql-01.xc.wb.ru:5432/flows_db?sslmode=disable + connection-user=readonly + connection-password=${ENV:PROD_PG_RO_PASSWORD} + postgresql.array-mapping=AS_JSON + issues_db: | + connector.name=postgresql + connection-url=jdbc:postgresql://sarex-vpsql-01.xc.wb.ru:5432/issues_db?sslmode=disable + connection-user=readonly + connection-password=${ENV:PROD_PG_RO_PASSWORD} + postgresql.array-mapping=AS_JSON + workspaces_db: | + connector.name=postgresql + connection-url=jdbc:postgresql://sarex-vpsql-01.xc.wb.ru:5432/workspaces_db?sslmode=disable + connection-user=readonly + connection-password=${ENV:PROD_PG_RO_PASSWORD} + postgresql.array-mapping=AS_JSON + resources_db: | + connector.name=postgresql + connection-url=jdbc:postgresql://sarex-vpsql-01.xc.wb.ru:5432/resources_db?sslmode=disable + connection-user=readonly + connection-password=${ENV:PROD_PG_RO_PASSWORD} + postgresql.array-mapping=AS_JSON + pm_db: | + connector.name=postgresql + connection-url=jdbc:postgresql://sarex-vpsql-01.xc.wb.ru:5432/pm_db?sslmode=disable + connection-user=readonly + connection-password=${ENV:PROD_PG_RO_PASSWORD} + postgresql.array-mapping=AS_JSON + eav_db: | + connector.name=postgresql + connection-url=jdbc:postgresql://sarex-vpsql-01.xc.wb.ru:5432/eav_db?sslmode=disable + connection-user=readonly + connection-password=${ENV:PROD_PG_RO_PASSWORD} + postgresql.array-mapping=AS_JSON + inspections_db: | + connector.name=postgresql + connection-url=jdbc:postgresql://sarex-vpsql-01.xc.wb.ru:5432/inspections_db?sslmode=disable + connection-user=readonly + connection-password=${ENV:PROD_PG_RO_PASSWORD} + postgresql.array-mapping=AS_JSON + documentations_db: | + connector.name=postgresql + connection-url=jdbc:postgresql://sarex-vpsql-01.xc.wb.ru:5432/documentations_db?sslmode=disable + connection-user=readonly + connection-password=${ENV:PROD_PG_RO_PASSWORD} + postgresql.array-mapping=AS_JSON + clickhouse_db: | + connector.name=tpch + tpch.splits-per-node=4 + hive: | + connector.name=tpch + tpch.splits-per-node=4 + coordinator: + resources: + requests: + cpu: 500m + memory: 2Gi + limits: + memory: 4Gi + worker: + resources: + requests: + cpu: 500m + memory: 2Gi + limits: + memory: 4Gi + nodeSelector: + dedicated: processing-light + tolerations: + - key: dedicated + operator: Equal + value: processing-light + effect: NoSchedule diff --git a/infrastructure/superset/base/helmrelease.yaml b/infrastructure/superset/base/helmrelease.yaml new file mode 100644 index 0000000..a08eb3e --- /dev/null +++ b/infrastructure/superset/base/helmrelease.yaml @@ -0,0 +1,22 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: superset + namespace: superset +spec: + interval: 10m + chart: + spec: + chart: superset-prod + version: "0.13.3" + sourceRef: + kind: HelmRepository + name: yc-oci-charts + namespace: flux-system + interval: 10m + install: + remediation: + retries: 3 + upgrade: + remediation: + retries: 3 diff --git a/infrastructure/superset/base/kustomization.yaml b/infrastructure/superset/base/kustomization.yaml new file mode 100644 index 0000000..ab0ff4d --- /dev/null +++ b/infrastructure/superset/base/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: superset +resources: + - namespace.yaml + - helmrelease.yaml diff --git a/infrastructure/superset/base/namespace.yaml b/infrastructure/superset/base/namespace.yaml new file mode 100644 index 0000000..646529a --- /dev/null +++ b/infrastructure/superset/base/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: superset diff --git a/infrastructure/superset/kustomization.yaml b/infrastructure/superset/kustomization.yaml new file mode 100644 index 0000000..3c2f51f --- /dev/null +++ b/infrastructure/superset/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./base diff --git a/infrastructure/trino/base/helmrelease.yaml b/infrastructure/trino/base/helmrelease.yaml new file mode 100644 index 0000000..10fa3d4 --- /dev/null +++ b/infrastructure/trino/base/helmrelease.yaml @@ -0,0 +1,22 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: trino + namespace: trino +spec: + interval: 10m + chart: + spec: + chart: trino-prod + version: "0.33.0" + sourceRef: + kind: HelmRepository + name: yc-oci-charts + namespace: flux-system + interval: 10m + install: + remediation: + retries: 3 + upgrade: + remediation: + retries: 3 diff --git a/infrastructure/trino/base/kustomization.yaml b/infrastructure/trino/base/kustomization.yaml new file mode 100644 index 0000000..0a1a7da --- /dev/null +++ b/infrastructure/trino/base/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: trino +resources: + - namespace.yaml + - helmrelease.yaml diff --git a/infrastructure/trino/base/namespace.yaml b/infrastructure/trino/base/namespace.yaml new file mode 100644 index 0000000..32162bc --- /dev/null +++ b/infrastructure/trino/base/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: trino diff --git a/infrastructure/trino/kustomization.yaml b/infrastructure/trino/kustomization.yaml new file mode 100644 index 0000000..3c2f51f --- /dev/null +++ b/infrastructure/trino/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./base