sarex/iac
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add pg sanitizer multi workflow

Browse Source
This commit is contained in:
Kochetkov S 2026-06-20 21:31:59 +03:00
parent 7960ce5106
commit 2e94d2329c
3 changed files with 180 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
infrastructure/pg-sanitizer/base/cronworkflow-multi.yaml Normal file
View File

@ -0,0 +1,39 @@
apiVersion: argoproj.io/v1alpha1
kind: CronWorkflow
metadata:
name: company-dump-sanitizer-multi-daily
namespace: argo
spec:
schedule: "0 4 * * *"
timezone: Europe/Moscow
suspend: true
concurrencyPolicy: Forbid
successfulJobsHistoryLimit: 10
failedJobsHistoryLimit: 10
workflowSpec:
workflowTemplateRef:
name: company-dump-sanitizer-multi
arguments:
parameters:
- name: company_ids
value: "330"
- name: exclude_company_ids
value: ""
- name: auto_discover_company_ids
value: "true"
- name: auto_company_limit
value: "5"
- name: reuse_base_databases
value: "false"
- name: run_fk_integrity_validation
value: "false"
- name: fk_integrity_dbs
value: ""
- name: bucket
value: prod-pg-dumps
- name: source_prefix
value: ""
- name: output_prefix
value: sanitized_dumps_multi
- name: s3_endpoint_url
value: https://storage.yandexcloud.net

2
infrastructure/pg-sanitizer/base/kustomization.yaml
View File

@ -5,3 +5,5 @@ resources:
- rbac.yaml
- workflowtemplate.yaml
- cronworkflow.yaml
- workflowtemplate-multi.yaml
- cronworkflow-multi.yaml

139
infrastructure/pg-sanitizer/base/workflowtemplate-multi.yaml Normal file
View File

@ -0,0 +1,139 @@
apiVersion: argoproj.io/v1alpha1
kind: WorkflowTemplate
metadata:
name: company-dump-sanitizer-multi
namespace: argo
spec:
entrypoint: sanitize-multi
serviceAccountName: pg-sanitizer
activeDeadlineSeconds: 172800
ttlStrategy:
secondsAfterSuccess: 259200
secondsAfterFailure: 259200
podMetadata:
annotations:
vault.hashicorp.com/agent-init-first: "true"
vault.hashicorp.com/agent-inject: "true"
vault.hashicorp.com/agent-pre-populate-only: "true"
vault.hashicorp.com/auth-path: "auth/kubernetes"
vault.hashicorp.com/role: "pg-sanitizer"
vault.hashicorp.com/agent-inject-secret-pg-sanitizer-env: "secrets/data/yc-infra-prod/infra/pg_dumper/pg_dumper_secret"
vault.hashicorp.com/agent-inject-template-pg-sanitizer-env: |
{{ with secret "secrets/data/yc-infra-prod/infra/pg_dumper/pg_dumper_secret" }}
export AWS_ACCESS_KEY_ID={{ printf "%q" .Data.data.aws_access_key_id }}
export AWS_SECRET_ACCESS_KEY={{ printf "%q" .Data.data.aws_secret_access_key }}
{{ end }}
{{ with secret "secrets/data/yc-infra-prod/infra/postgresql_contour/postgresql_contour_secret" }}
export PGPASSWORD={{ printf "%q" (index .Data.data "postgres-password") }}
{{ end }}
arguments:
parameters:
- name: company_ids
value: "330"
- name: exclude_company_ids
value: ""
- name: auto_discover_company_ids
value: "true"
- name: auto_company_limit
value: "5"
- name: reuse_base_databases
value: "false"
- name: run_fk_integrity_validation
value: "false"
- name: fk_integrity_dbs
value: ""
- name: bucket
value: prod-pg-dumps
- name: source_prefix
value: ""
- name: output_prefix
value: sanitized_dumps_multi
- name: s3_endpoint_url
value: https://storage.yandexcloud.net
- name: db_items
value: >-
attachments:attachments resources:resources inspections:inspections issues:issues
notes:notes processing:processing workspaces:workspaces comparisons:comparisons
checklists:checklists contracts:contracts drawings:drawings remarks:remarks
subscriptions:subscriptions system-log:system-log transmittal:transmittal
pm_db:pm_db pulse_db:pulse_db eav:eav documentations:documentations
flows:restored_flow_db rfi:rfi sarex_db:sarex_db
templates:
- name: sanitize-multi
nodeSelector:
dedicated: sts
tolerations:
- key: dedicated
operator: Equal
value: sts
effect: NoSchedule
container:
image: cr.yandex/crp3ccidau046kdj8g9q/yc-pg-sanitizer-multi:0.1.3
imagePullPolicy: IfNotPresent
command:
- /usr/local/bin/multi_company_sanitize_from_s3.sh
env:
- name: VAULT_ENV_FILE
value: /vault/secrets/pg-sanitizer-env
- name: COMPANY_IDS
value: "{{workflow.parameters.company_ids}}"
- name: EXCLUDE_COMPANY_IDS
value: "{{workflow.parameters.exclude_company_ids}}"
- name: AUTO_DISCOVER_COMPANY_IDS
value: "{{workflow.parameters.auto_discover_company_ids}}"
- name: AUTO_COMPANY_LIMIT
value: "{{workflow.parameters.auto_company_limit}}"
- name: REUSE_BASE_DATABASES
value: "{{workflow.parameters.reuse_base_databases}}"
- name: RUN_FK_INTEGRITY_VALIDATION
value: "{{workflow.parameters.run_fk_integrity_validation}}"
- name: FK_INTEGRITY_DBS
value: "{{workflow.parameters.fk_integrity_dbs}}"
- name: S3_BUCKET
value: "{{workflow.parameters.bucket}}"
- name: SOURCE_PREFIX
value: "{{workflow.parameters.source_prefix}}"
- name: OUTPUT_PREFIX
value: "{{workflow.parameters.output_prefix}}"
- name: S3_ENDPOINT_URL
value: "{{workflow.parameters.s3_endpoint_url}}"
- name: RUN_ID
value: "{{workflow.name}}"
- name: PGHOST
value: postgresql.postgresql.svc.cluster.local
- name: PGPORT
value: "5432"
- name: PGUSER
value: postgres
- name: PGSSLMODE
value: disable
- name: PG_ALLOW_NONLOCAL
value: "false"
- name: PGRESTORE_JOBS
value: "6"
- name: DB_WORK_MEM
value: 64MB
- name: DB_MAINTENANCE_WORK_MEM
value: 1GB
- name: DB_TEMP_BUFFERS
value: 64MB
- name: RESTORE_VALIDATION_STRICT
value: "false"
- name: DROP_DATABASES_ON_FAILURE
value: "false"
- name: DB_ITEMS
value: "{{workflow.parameters.db_items}}"
- name: WORK_ROOT
value: /work
resources:
requests:
cpu: "4"
memory: 8Gi
ephemeral-storage: 120Gi
volumeMounts:
- name: work
mountPath: /work
volumes:
- name: work
emptyDir:
sizeLimit: 120Gi