Add pg sanitizer multi workflow

infrastructure/pg-sanitizer/base/cronworkflow-multi.yaml

@@ -0,0 +1,39 @@
+apiVersion: argoproj.io/v1alpha1
+kind: CronWorkflow
+metadata:
+  name: company-dump-sanitizer-multi-daily
+  namespace: argo
+spec:
+  schedule: "0 4 * * *"
+  timezone: Europe/Moscow
+  suspend: true
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 10
+  failedJobsHistoryLimit: 10
+  workflowSpec:
+    workflowTemplateRef:
+      name: company-dump-sanitizer-multi
+    arguments:
+      parameters:
+        - name: company_ids
+          value: "330"
+        - name: exclude_company_ids
+          value: ""
+        - name: auto_discover_company_ids
+          value: "true"
+        - name: auto_company_limit
+          value: "5"
+        - name: reuse_base_databases
+          value: "false"
+        - name: run_fk_integrity_validation
+          value: "false"
+        - name: fk_integrity_dbs
+          value: ""
+        - name: bucket
+          value: prod-pg-dumps
+        - name: source_prefix
+          value: ""
+        - name: output_prefix
+          value: sanitized_dumps_multi
+        - name: s3_endpoint_url
+          value: https://storage.yandexcloud.net

infrastructure/pg-sanitizer/base/kustomization.yaml

@@ -5,3 +5,5 @@ resources:
   - rbac.yaml
   - workflowtemplate.yaml
   - cronworkflow.yaml
+  - workflowtemplate-multi.yaml
+  - cronworkflow-multi.yaml

infrastructure/pg-sanitizer/base/workflowtemplate-multi.yaml

@@ -0,0 +1,139 @@
+apiVersion: argoproj.io/v1alpha1
+kind: WorkflowTemplate
+metadata:
+  name: company-dump-sanitizer-multi
+  namespace: argo
+spec:
+  entrypoint: sanitize-multi
+  serviceAccountName: pg-sanitizer
+  activeDeadlineSeconds: 172800
+  ttlStrategy:
+    secondsAfterSuccess: 259200
+    secondsAfterFailure: 259200
+  podMetadata:
+    annotations:
+      vault.hashicorp.com/agent-init-first: "true"
+      vault.hashicorp.com/agent-inject: "true"
+      vault.hashicorp.com/agent-pre-populate-only: "true"
+      vault.hashicorp.com/auth-path: "auth/kubernetes"
+      vault.hashicorp.com/role: "pg-sanitizer"
+      vault.hashicorp.com/agent-inject-secret-pg-sanitizer-env: "secrets/data/yc-infra-prod/infra/pg_dumper/pg_dumper_secret"
+      vault.hashicorp.com/agent-inject-template-pg-sanitizer-env: |
+        {{ with secret "secrets/data/yc-infra-prod/infra/pg_dumper/pg_dumper_secret" }}
+        export AWS_ACCESS_KEY_ID={{ printf "%q" .Data.data.aws_access_key_id }}
+        export AWS_SECRET_ACCESS_KEY={{ printf "%q" .Data.data.aws_secret_access_key }}
+        {{ end }}
+        {{ with secret "secrets/data/yc-infra-prod/infra/postgresql_contour/postgresql_contour_secret" }}
+        export PGPASSWORD={{ printf "%q" (index .Data.data "postgres-password") }}
+        {{ end }}
+  arguments:
+    parameters:
+      - name: company_ids
+        value: "330"
+      - name: exclude_company_ids
+        value: ""
+      - name: auto_discover_company_ids
+        value: "true"
+      - name: auto_company_limit
+        value: "5"
+      - name: reuse_base_databases
+        value: "false"
+      - name: run_fk_integrity_validation
+        value: "false"
+      - name: fk_integrity_dbs
+        value: ""
+      - name: bucket
+        value: prod-pg-dumps
+      - name: source_prefix
+        value: ""
+      - name: output_prefix
+        value: sanitized_dumps_multi
+      - name: s3_endpoint_url
+        value: https://storage.yandexcloud.net
+      - name: db_items
+        value: >-
+          attachments:attachments resources:resources inspections:inspections issues:issues
+          notes:notes processing:processing workspaces:workspaces comparisons:comparisons
+          checklists:checklists contracts:contracts drawings:drawings remarks:remarks
+          subscriptions:subscriptions system-log:system-log transmittal:transmittal
+          pm_db:pm_db pulse_db:pulse_db eav:eav documentations:documentations
+          flows:restored_flow_db rfi:rfi sarex_db:sarex_db
+  templates:
+    - name: sanitize-multi
+      nodeSelector:
+        dedicated: sts
+      tolerations:
+        - key: dedicated
+          operator: Equal
+          value: sts
+          effect: NoSchedule
+      container:
+        image: cr.yandex/crp3ccidau046kdj8g9q/yc-pg-sanitizer-multi:0.1.3
+        imagePullPolicy: IfNotPresent
+        command:
+          - /usr/local/bin/multi_company_sanitize_from_s3.sh
+        env:
+          - name: VAULT_ENV_FILE
+            value: /vault/secrets/pg-sanitizer-env
+          - name: COMPANY_IDS
+            value: "{{workflow.parameters.company_ids}}"
+          - name: EXCLUDE_COMPANY_IDS
+            value: "{{workflow.parameters.exclude_company_ids}}"
+          - name: AUTO_DISCOVER_COMPANY_IDS
+            value: "{{workflow.parameters.auto_discover_company_ids}}"
+          - name: AUTO_COMPANY_LIMIT
+            value: "{{workflow.parameters.auto_company_limit}}"
+          - name: REUSE_BASE_DATABASES
+            value: "{{workflow.parameters.reuse_base_databases}}"
+          - name: RUN_FK_INTEGRITY_VALIDATION
+            value: "{{workflow.parameters.run_fk_integrity_validation}}"
+          - name: FK_INTEGRITY_DBS
+            value: "{{workflow.parameters.fk_integrity_dbs}}"
+          - name: S3_BUCKET
+            value: "{{workflow.parameters.bucket}}"
+          - name: SOURCE_PREFIX
+            value: "{{workflow.parameters.source_prefix}}"
+          - name: OUTPUT_PREFIX
+            value: "{{workflow.parameters.output_prefix}}"
+          - name: S3_ENDPOINT_URL
+            value: "{{workflow.parameters.s3_endpoint_url}}"
+          - name: RUN_ID
+            value: "{{workflow.name}}"
+          - name: PGHOST
+            value: postgresql.postgresql.svc.cluster.local
+          - name: PGPORT
+            value: "5432"
+          - name: PGUSER
+            value: postgres
+          - name: PGSSLMODE
+            value: disable
+          - name: PG_ALLOW_NONLOCAL
+            value: "false"
+          - name: PGRESTORE_JOBS
+            value: "6"
+          - name: DB_WORK_MEM
+            value: 64MB
+          - name: DB_MAINTENANCE_WORK_MEM
+            value: 1GB
+          - name: DB_TEMP_BUFFERS
+            value: 64MB
+          - name: RESTORE_VALIDATION_STRICT
+            value: "false"
+          - name: DROP_DATABASES_ON_FAILURE
+            value: "false"
+          - name: DB_ITEMS
+            value: "{{workflow.parameters.db_items}}"
+          - name: WORK_ROOT
+            value: /work
+        resources:
+          requests:
+            cpu: "4"
+            memory: 8Gi
+            ephemeral-storage: 120Gi
+        volumeMounts:
+          - name: work
+            mountPath: /work
+      volumes:
+        - name: work
+          emptyDir:
+            sizeLimit: 120Gi