add ss + trino to wb

2026-07-22 13:10:48 +03:00 · 2026-07-22 13:10:48 +03:00 · c640705feb
commit c640705feb
parent 09c06170db
20 changed files with 8220 additions and 0 deletions
--- a/.tmp/wb-kustomize-rendered.yaml
+++ b/.tmp/wb-kustomize-rendered.yaml
--- a/.tmp/wb-superset-trino-secrets.README.md
+++ b/.tmp/wb-superset-trino-secrets.README.md
@ -0,0 +1,14 @@
+# wb superset/trino local secret files
+
+Do not commit this directory.
+
+- `.tmp/wb_superset_pg_password`
+  - target secret: `superset/postgresql-auth`
+  - target keys: `password`, `postgres-password`
+  - postgresql role: `superset`
+
+- `.tmp/wb_readonly_pg_password`
+  - target secret: `trino/postgres-secret`
+  - target key: `postgres_ro_password`
+  - postgresql role: `readonly`
+
--- a/.tmp/wb-superset-values.yaml
+++ b/.tmp/wb-superset-values.yaml
@ -0,0 +1,93 @@
+fullnameOverride: superset
+vault:
+  enabled: false
+stronghold:
+  enabled: false
+extraEnv:
+  GUNICORN_TIMEOUT: "300"
+  SERVER_WORKER_AMOUNT: "4"
+  BABEL_DEFAULT_LOCALE: ru
+extraEnvRaw:
+  - name: ENABLE_PROXY_FIX
+    value: "true"
+  - name: DB_PASS
+    valueFrom:
+      secretKeyRef:
+        name: postgresql-auth
+        key: password
+  - name: SUPERSET_SECRET_KEY
+    valueFrom:
+      secretKeyRef:
+        name: superset-secret
+        key: secret.key
+  - name: JWT_SECRET
+    valueFrom:
+      secretKeyRef:
+        name: jwt-secret
+        key: jwt_secret
+supersetNode:
+  connections:
+    redis_host: superset-redis-headless
+    redis_port: "6379"
+    redis_user: ""
+    redis_cache_db: "1"
+    redis_celery_db: "0"
+    db_host: sarex-vpsql-01.xc.wb.ru
+    db_port: "5432"
+    db_user: superset
+    db_name: ss
+  resources:
+    requests:
+      cpu: 500m
+      memory: 1Gi
+    limits:
+      memory: 2Gi
+supersetWorker:
+  resources:
+    requests:
+      cpu: 500m
+      memory: 1Gi
+    limits:
+      memory: 2Gi
+init:
+  loadExamples: false
+postgresql:
+  enabled: false
+redis:
+  enabled: true
+  architecture: standalone
+  auth:
+    enabled: false
+    existingSecret: ""
+    existingSecretKey: ""
+    password: ""
+  master:
+    persistence:
+      enabled: false
+configOverrides:
+  feature_flags: |
+    FEATURE_FLAGS = {
+        "EMBEDDED_SUPERSET": True,
+        "ENABLE_TEMPLATE_PROCESSING": True,
+    }
+  iframe_config: |
+    X_FRAME_OPTIONS = None
+    HTTP_HEADERS = {
+        "Content-Security-Policy": "frame-ancestors https://*.wb.ru",
+    }
+  extend_timeout: |
+    SQLLAB_ASYNC_TIME_LIMIT_SEC = 300
+    SUPERSET_WEBSERVER_TIMEOUT = 300
+    SQLLAB_TIMEOUT = 600
+  set_locale: |
+    BABEL_DEFAULT_LOCALE = "ru"
+  enable_oauth: ""
+extraConfigs:
+  import_datasources.yaml: |
+    databases:
+      - database_name: trino
+        sqlalchemy_uri: trino://superset@trino.trino.svc.cluster.local:8080
+        expose_in_sqllab: true
+        allow_ctas: true
+        allow_cvas: true
+        allow_dml: false
--- a/.tmp/wb-trino-values.yaml
+++ b/.tmp/wb-trino-values.yaml
@ -0,0 +1,103 @@
+nameOverride: trino
+coordinatorNameOverride: trino-coordinator
+workerNameOverride: trino-worker
+gateway:
+  enabled: false
+virtualService:
+  enabled: false
+server:
+  workers: 1
+  config:
+    authenticationType: ""
+  autoscaling:
+    enabled: false
+auth:
+  passwordAuthSecret: ""
+env:
+  - name: PROD_PG_RO_PASSWORD
+    valueFrom:
+      secretKeyRef:
+        name: postgres-secret
+        key: postgres_ro_password
+catalogs:
+  sarex_db: |
+    connector.name=postgresql
+    connection-url=jdbc:postgresql://sarex-vpsql-01.xc.wb.ru:5432/sarex_db?sslmode=disable
+    connection-user=readonly
+    connection-password=${ENV:PROD_PG_RO_PASSWORD}
+    postgresql.array-mapping=AS_JSON
+  flows_db: |
+    connector.name=postgresql
+    connection-url=jdbc:postgresql://sarex-vpsql-01.xc.wb.ru:5432/flows_db?sslmode=disable
+    connection-user=readonly
+    connection-password=${ENV:PROD_PG_RO_PASSWORD}
+    postgresql.array-mapping=AS_JSON
+  issues_db: |
+    connector.name=postgresql
+    connection-url=jdbc:postgresql://sarex-vpsql-01.xc.wb.ru:5432/issues_db?sslmode=disable
+    connection-user=readonly
+    connection-password=${ENV:PROD_PG_RO_PASSWORD}
+    postgresql.array-mapping=AS_JSON
+  workspaces_db: |
+    connector.name=postgresql
+    connection-url=jdbc:postgresql://sarex-vpsql-01.xc.wb.ru:5432/workspaces_db?sslmode=disable
+    connection-user=readonly
+    connection-password=${ENV:PROD_PG_RO_PASSWORD}
+    postgresql.array-mapping=AS_JSON
+  resources_db: |
+    connector.name=postgresql
+    connection-url=jdbc:postgresql://sarex-vpsql-01.xc.wb.ru:5432/resources_db?sslmode=disable
+    connection-user=readonly
+    connection-password=${ENV:PROD_PG_RO_PASSWORD}
+    postgresql.array-mapping=AS_JSON
+  pm_db: |
+    connector.name=postgresql
+    connection-url=jdbc:postgresql://sarex-vpsql-01.xc.wb.ru:5432/pm_db?sslmode=disable
+    connection-user=readonly
+    connection-password=${ENV:PROD_PG_RO_PASSWORD}
+    postgresql.array-mapping=AS_JSON
+  eav_db: |
+    connector.name=postgresql
+    connection-url=jdbc:postgresql://sarex-vpsql-01.xc.wb.ru:5432/eav_db?sslmode=disable
+    connection-user=readonly
+    connection-password=${ENV:PROD_PG_RO_PASSWORD}
+    postgresql.array-mapping=AS_JSON
+  inspections_db: |
+    connector.name=postgresql
+    connection-url=jdbc:postgresql://sarex-vpsql-01.xc.wb.ru:5432/inspections_db?sslmode=disable
+    connection-user=readonly
+    connection-password=${ENV:PROD_PG_RO_PASSWORD}
+    postgresql.array-mapping=AS_JSON
+  documentations_db: |
+    connector.name=postgresql
+    connection-url=jdbc:postgresql://sarex-vpsql-01.xc.wb.ru:5432/documentations_db?sslmode=disable
+    connection-user=readonly
+    connection-password=${ENV:PROD_PG_RO_PASSWORD}
+    postgresql.array-mapping=AS_JSON
+  clickhouse_db: |
+    connector.name=tpch
+    tpch.splits-per-node=4
+  hive: |
+    connector.name=tpch
+    tpch.splits-per-node=4
+coordinator:
+  resources:
+    requests:
+      cpu: 500m
+      memory: 2Gi
+    limits:
+      memory: 4Gi
+worker:
+  resources:
+    requests:
+      cpu: 500m
+      memory: 2Gi
+    limits:
+      memory: 4Gi
+  nodeSelector:
+    dedicated: processing-light
+  tolerations:
+    - key: dedicated
+      operator: Equal
+      value: processing-light
+      effect: NoSchedule
--- a/.tmp/wb_pg_admin_password
+++ b/.tmp/wb_pg_admin_password
@ -0,0 +1 @@
+pZpMmssSArrqHbeegnxTVU7YEbyKzgEg
--- a/.tmp/wb_pg_exporter_secret.json
+++ b/.tmp/wb_pg_exporter_secret.json
@ -0,0 +1,18 @@
+{
+    "apiVersion": "v1",
+    "data": {
+        "data_source_password": "cFpwTW1zc1NBcnJxSGJlZWdueFRWVTdZRWJ5S3pnRWc="
+    },
+    "kind": "Secret",
+    "metadata": {
+        "annotations": {
+            "kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"v1\",\"data\":{\"data_source_password\":\"cFpwTW1zc1NBcnJxSGJlZWdueFRWVTdZRWJ5S3pnRWc=\"},\"kind\":\"Secret\",\"metadata\":{\"annotations\":{},\"name\":\"postgres-exporter-prometheus-postgres-exporter\",\"namespace\":\"postgres-exporter\"},\"type\":\"Opaque\"}\n"
+        },
+        "creationTimestamp": "2026-05-27T10:32:32Z",
+        "name": "postgres-exporter-prometheus-postgres-exporter",
+        "namespace": "postgres-exporter",
+        "resourceVersion": "97334157",
+        "uid": "0f74002a-eb50-419c-91d7-2d285b505319"
+    },
+    "type": "Opaque"
+}
--- a/.tmp/wb_readonly_pg_password
+++ b/.tmp/wb_readonly_pg_password
@ -0,0 +1 @@
+1uCiCAN4kFsCwsz+Tx/7nB2nZk6JAyRoJoz9t+41jJzcQjjE
--- a/.tmp/wb_superset_pg_password
+++ b/.tmp/wb_superset_pg_password
@ -0,0 +1 @@
+BKw+QgsRaH+MizRiZTTSPkdH95zOdQ8Rod0vUw9N9EEuswgB
--- a/clusters/wb/infrastructure/kustomization.yaml
+++ b/clusters/wb/infrastructure/kustomization.yaml
@ -10,6 +10,8 @@ resources:
  - ../../../infrastructure/goalert
  - ../../../infrastructure/kafka-exporter
  - ../../../infrastructure/postgres-exporter
+  - ../../../infrastructure/superset
+  - ../../../infrastructure/trino
  - ./node-exporter-vmnodescrape.yaml
 patches:
  - path: ./patches/istio-config.yaml
@ -75,3 +77,17 @@ patches:
      kind: HelmRelease
      name: postgres-exporter
      namespace: postgres-exporter
+  - path: ./patches/superset.yaml
+    target:
+      group: helm.toolkit.fluxcd.io
+      version: v2
+      kind: HelmRelease
+      name: superset
+      namespace: superset
+  - path: ./patches/trino.yaml
+    target:
+      group: helm.toolkit.fluxcd.io
+      version: v2
+      kind: HelmRelease
+      name: trino
+      namespace: trino
--- a/clusters/wb/infrastructure/patches/istio-config.yaml
+++ b/clusters/wb/infrastructure/patches/istio-config.yaml
@ -51,6 +51,16 @@ spec:
                    - openobserve-srx.wb.ru
                  tls:
                    credentialName: istio-gateway-wildcard-cert
+            superset:
+              name: superset-gw
+              namespace: gateway
+              selector:
+                istio: ingressgateway
+              servers:
+                - hosts:
+                    - superset-srx.wb.ru
+                  tls:
+                    credentialName: istio-gateway-wildcard-cert
            vmalert:
              name: vmalert-gw
              namespace: gateway
@ -95,6 +105,17 @@ spec:
                    prefix: /
                  service: openobserve-web.openobserve.svc.cluster.local
                  port: 5080
+            superset-vs:
+              namespace: superset
+              hosts:
+                - superset-srx.wb.ru
+              gateways:
+                - gateway/superset-gw
+              routes:
+                - path:
+                    prefix: /
+                  service: superset.superset.svc.cluster.local
+                  port: 8088
            vmalert-vs:
              namespace: goalert
              hosts:
--- a/clusters/wb/infrastructure/patches/superset.yaml
+++ b/clusters/wb/infrastructure/patches/superset.yaml
@ -0,0 +1,102 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: superset
+  namespace: superset
+spec:
+  interval: 5m
+  timeout: 20m
+  values:
+    fullnameOverride: superset
+    vault:
+      enabled: false
+    stronghold:
+      enabled: false
+    extraEnv:
+      GUNICORN_TIMEOUT: "300"
+      SERVER_WORKER_AMOUNT: "4"
+      BABEL_DEFAULT_LOCALE: ru
+    extraEnvRaw:
+      - name: ENABLE_PROXY_FIX
+        value: "true"
+      - name: DB_PASS
+        valueFrom:
+          secretKeyRef:
+            name: postgresql-auth
+            key: password
+      - name: SUPERSET_SECRET_KEY
+        valueFrom:
+          secretKeyRef:
+            name: superset-secret
+            key: secret.key
+      - name: JWT_SECRET
+        valueFrom:
+          secretKeyRef:
+            name: jwt-secret
+            key: jwt_secret
+    supersetNode:
+      connections:
+        redis_host: superset-redis-headless
+        redis_port: "6379"
+        redis_user: ""
+        redis_cache_db: "1"
+        redis_celery_db: "0"
+        db_host: sarex-vpsql-01.xc.wb.ru
+        db_port: "5432"
+        db_user: superset
+        db_name: ss
+      resources:
+        requests:
+          cpu: 500m
+          memory: 1Gi
+        limits:
+          memory: 2Gi
+    supersetWorker:
+      resources:
+        requests:
+          cpu: 500m
+          memory: 1Gi
+        limits:
+          memory: 2Gi
+    init:
+      loadExamples: false
+    postgresql:
+      enabled: false
+    redis:
+      enabled: true
+      architecture: standalone
+      auth:
+        enabled: false
+        existingSecret: ""
+        existingSecretKey: ""
+        password: ""
+      master:
+        persistence:
+          enabled: false
+    configOverrides:
+      feature_flags: |
+        FEATURE_FLAGS = {
+            "EMBEDDED_SUPERSET": True,
+            "ENABLE_TEMPLATE_PROCESSING": True,
+        }
+      iframe_config: |
+        X_FRAME_OPTIONS = None
+        HTTP_HEADERS = {
+            "Content-Security-Policy": "frame-ancestors https://*.wb.ru",
+        }
+      extend_timeout: |
+        SQLLAB_ASYNC_TIME_LIMIT_SEC = 300
+        SUPERSET_WEBSERVER_TIMEOUT = 300
+        SQLLAB_TIMEOUT = 600
+      set_locale: |
+        BABEL_DEFAULT_LOCALE = "ru"
+      enable_oauth: ""
+    extraConfigs:
+      import_datasources.yaml: |
+        databases:
+          - database_name: trino
+            sqlalchemy_uri: trino://superset@trino.trino.svc.cluster.local:8080
+            expose_in_sqllab: true
+            allow_ctas: true
+            allow_cvas: true
+            allow_dml: false
--- a/clusters/wb/infrastructure/patches/trino.yaml
+++ b/clusters/wb/infrastructure/patches/trino.yaml
@ -0,0 +1,112 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: trino
+  namespace: trino
+spec:
+  interval: 5m
+  timeout: 20m
+  values:
+    nameOverride: trino
+    coordinatorNameOverride: trino-coordinator
+    workerNameOverride: trino-worker
+    gateway:
+      enabled: false
+    virtualService:
+      enabled: false
+    server:
+      workers: 1
+      config:
+        authenticationType: ""
+      autoscaling:
+        enabled: false
+    auth:
+      passwordAuthSecret: ""
+    env:
+      - name: PROD_PG_RO_PASSWORD
+        valueFrom:
+          secretKeyRef:
+            name: postgres-secret
+            key: postgres_ro_password
+    catalogs:
+      sarex_db: |
+        connector.name=postgresql
+        connection-url=jdbc:postgresql://sarex-vpsql-01.xc.wb.ru:5432/sarex_db?sslmode=disable
+        connection-user=readonly
+        connection-password=${ENV:PROD_PG_RO_PASSWORD}
+        postgresql.array-mapping=AS_JSON
+      flows_db: |
+        connector.name=postgresql
+        connection-url=jdbc:postgresql://sarex-vpsql-01.xc.wb.ru:5432/flows_db?sslmode=disable
+        connection-user=readonly
+        connection-password=${ENV:PROD_PG_RO_PASSWORD}
+        postgresql.array-mapping=AS_JSON
+      issues_db: |
+        connector.name=postgresql
+        connection-url=jdbc:postgresql://sarex-vpsql-01.xc.wb.ru:5432/issues_db?sslmode=disable
+        connection-user=readonly
+        connection-password=${ENV:PROD_PG_RO_PASSWORD}
+        postgresql.array-mapping=AS_JSON
+      workspaces_db: |
+        connector.name=postgresql
+        connection-url=jdbc:postgresql://sarex-vpsql-01.xc.wb.ru:5432/workspaces_db?sslmode=disable
+        connection-user=readonly
+        connection-password=${ENV:PROD_PG_RO_PASSWORD}
+        postgresql.array-mapping=AS_JSON
+      resources_db: |
+        connector.name=postgresql
+        connection-url=jdbc:postgresql://sarex-vpsql-01.xc.wb.ru:5432/resources_db?sslmode=disable
+        connection-user=readonly
+        connection-password=${ENV:PROD_PG_RO_PASSWORD}
+        postgresql.array-mapping=AS_JSON
+      pm_db: |
+        connector.name=postgresql
+        connection-url=jdbc:postgresql://sarex-vpsql-01.xc.wb.ru:5432/pm_db?sslmode=disable
+        connection-user=readonly
+        connection-password=${ENV:PROD_PG_RO_PASSWORD}
+        postgresql.array-mapping=AS_JSON
+      eav_db: |
+        connector.name=postgresql
+        connection-url=jdbc:postgresql://sarex-vpsql-01.xc.wb.ru:5432/eav_db?sslmode=disable
+        connection-user=readonly
+        connection-password=${ENV:PROD_PG_RO_PASSWORD}
+        postgresql.array-mapping=AS_JSON
+      inspections_db: |
+        connector.name=postgresql
+        connection-url=jdbc:postgresql://sarex-vpsql-01.xc.wb.ru:5432/inspections_db?sslmode=disable
+        connection-user=readonly
+        connection-password=${ENV:PROD_PG_RO_PASSWORD}
+        postgresql.array-mapping=AS_JSON
+      documentations_db: |
+        connector.name=postgresql
+        connection-url=jdbc:postgresql://sarex-vpsql-01.xc.wb.ru:5432/documentations_db?sslmode=disable
+        connection-user=readonly
+        connection-password=${ENV:PROD_PG_RO_PASSWORD}
+        postgresql.array-mapping=AS_JSON
+      clickhouse_db: |
+        connector.name=tpch
+        tpch.splits-per-node=4
+      hive: |
+        connector.name=tpch
+        tpch.splits-per-node=4
+    coordinator:
+      resources:
+        requests:
+          cpu: 500m
+          memory: 2Gi
+        limits:
+          memory: 4Gi
+    worker:
+      resources:
+        requests:
+          cpu: 500m
+          memory: 2Gi
+        limits:
+          memory: 4Gi
+      nodeSelector:
+        dedicated: processing-light
+      tolerations:
+        - key: dedicated
+          operator: Equal
+          value: processing-light
+          effect: NoSchedule
--- a/infrastructure/superset/base/helmrelease.yaml
+++ b/infrastructure/superset/base/helmrelease.yaml
@ -0,0 +1,22 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: superset
+  namespace: superset
+spec:
+  interval: 10m
+  chart:
+    spec:
+      chart: superset-prod
+      version: "0.13.3"
+      sourceRef:
+        kind: HelmRepository
+        name: yc-oci-charts
+        namespace: flux-system
+      interval: 10m
+  install:
+    remediation:
+      retries: 3
+  upgrade:
+    remediation:
+      retries: 3
--- a/infrastructure/superset/base/kustomization.yaml
+++ b/infrastructure/superset/base/kustomization.yaml
@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: superset
+resources:
+  - namespace.yaml
+  - helmrelease.yaml
--- a/infrastructure/superset/base/namespace.yaml
+++ b/infrastructure/superset/base/namespace.yaml
@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: superset
--- a/infrastructure/superset/kustomization.yaml
+++ b/infrastructure/superset/kustomization.yaml
@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./base
--- a/infrastructure/trino/base/helmrelease.yaml
+++ b/infrastructure/trino/base/helmrelease.yaml
@ -0,0 +1,22 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: trino
+  namespace: trino
+spec:
+  interval: 10m
+  chart:
+    spec:
+      chart: trino-prod
+      version: "0.33.0"
+      sourceRef:
+        kind: HelmRepository
+        name: yc-oci-charts
+        namespace: flux-system
+      interval: 10m
+  install:
+    remediation:
+      retries: 3
+  upgrade:
+    remediation:
+      retries: 3
--- a/infrastructure/trino/base/kustomization.yaml
+++ b/infrastructure/trino/base/kustomization.yaml
@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: trino
+resources:
+  - namespace.yaml
+  - helmrelease.yaml
--- a/infrastructure/trino/base/namespace.yaml
+++ b/infrastructure/trino/base/namespace.yaml
@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: trino
--- a/infrastructure/trino/kustomization.yaml
+++ b/infrastructure/trino/kustomization.yaml
@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./base
				`@ -0,0 +1 @@`
				`1uCiCAN4kFsCwsz+Tx/7nB2nZk6JAyRoJoz9t+41jJzcQjjE`
				`@ -0,0 +1 @@`
				`BKw+QgsRaH+MizRiZTTSPkdH95zOdQ8Rod0vUw9N9EEuswgB`