diff --git a/infrastructure/pg-sanitizer/base/cronworkflow-multi.yaml b/infrastructure/pg-sanitizer/base/cronworkflow-multi.yaml new file mode 100644 index 0000000..dafd66b --- /dev/null +++ b/infrastructure/pg-sanitizer/base/cronworkflow-multi.yaml @@ -0,0 +1,39 @@ +apiVersion: argoproj.io/v1alpha1 +kind: CronWorkflow +metadata: + name: company-dump-sanitizer-multi-daily + namespace: argo +spec: + schedule: "0 4 * * *" + timezone: Europe/Moscow + suspend: true + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 10 + failedJobsHistoryLimit: 10 + workflowSpec: + workflowTemplateRef: + name: company-dump-sanitizer-multi + arguments: + parameters: + - name: company_ids + value: "330" + - name: exclude_company_ids + value: "" + - name: auto_discover_company_ids + value: "true" + - name: auto_company_limit + value: "5" + - name: reuse_base_databases + value: "false" + - name: run_fk_integrity_validation + value: "false" + - name: fk_integrity_dbs + value: "" + - name: bucket + value: prod-pg-dumps + - name: source_prefix + value: "" + - name: output_prefix + value: sanitized_dumps_multi + - name: s3_endpoint_url + value: https://storage.yandexcloud.net diff --git a/infrastructure/pg-sanitizer/base/kustomization.yaml b/infrastructure/pg-sanitizer/base/kustomization.yaml index 33d89c4..4f67448 100644 --- a/infrastructure/pg-sanitizer/base/kustomization.yaml +++ b/infrastructure/pg-sanitizer/base/kustomization.yaml @@ -5,3 +5,5 @@ resources: - rbac.yaml - workflowtemplate.yaml - cronworkflow.yaml + - workflowtemplate-multi.yaml + - cronworkflow-multi.yaml diff --git a/infrastructure/pg-sanitizer/base/workflowtemplate-multi.yaml b/infrastructure/pg-sanitizer/base/workflowtemplate-multi.yaml new file mode 100644 index 0000000..6f4db5c --- /dev/null +++ b/infrastructure/pg-sanitizer/base/workflowtemplate-multi.yaml @@ -0,0 +1,139 @@ +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: company-dump-sanitizer-multi + namespace: argo +spec: + entrypoint: sanitize-multi + serviceAccountName: pg-sanitizer + activeDeadlineSeconds: 172800 + ttlStrategy: + secondsAfterSuccess: 259200 + secondsAfterFailure: 259200 + podMetadata: + annotations: + vault.hashicorp.com/agent-init-first: "true" + vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/agent-pre-populate-only: "true" + vault.hashicorp.com/auth-path: "auth/kubernetes" + vault.hashicorp.com/role: "pg-sanitizer" + vault.hashicorp.com/agent-inject-secret-pg-sanitizer-env: "secrets/data/yc-infra-prod/infra/pg_dumper/pg_dumper_secret" + vault.hashicorp.com/agent-inject-template-pg-sanitizer-env: | + {{ with secret "secrets/data/yc-infra-prod/infra/pg_dumper/pg_dumper_secret" }} + export AWS_ACCESS_KEY_ID={{ printf "%q" .Data.data.aws_access_key_id }} + export AWS_SECRET_ACCESS_KEY={{ printf "%q" .Data.data.aws_secret_access_key }} + {{ end }} + {{ with secret "secrets/data/yc-infra-prod/infra/postgresql_contour/postgresql_contour_secret" }} + export PGPASSWORD={{ printf "%q" (index .Data.data "postgres-password") }} + {{ end }} + arguments: + parameters: + - name: company_ids + value: "330" + - name: exclude_company_ids + value: "" + - name: auto_discover_company_ids + value: "true" + - name: auto_company_limit + value: "5" + - name: reuse_base_databases + value: "false" + - name: run_fk_integrity_validation + value: "false" + - name: fk_integrity_dbs + value: "" + - name: bucket + value: prod-pg-dumps + - name: source_prefix + value: "" + - name: output_prefix + value: sanitized_dumps_multi + - name: s3_endpoint_url + value: https://storage.yandexcloud.net + - name: db_items + value: >- + attachments:attachments resources:resources inspections:inspections issues:issues + notes:notes processing:processing workspaces:workspaces comparisons:comparisons + checklists:checklists contracts:contracts drawings:drawings remarks:remarks + subscriptions:subscriptions system-log:system-log transmittal:transmittal + pm_db:pm_db pulse_db:pulse_db eav:eav documentations:documentations + flows:restored_flow_db rfi:rfi sarex_db:sarex_db + templates: + - name: sanitize-multi + nodeSelector: + dedicated: sts + tolerations: + - key: dedicated + operator: Equal + value: sts + effect: NoSchedule + container: + image: cr.yandex/crp3ccidau046kdj8g9q/yc-pg-sanitizer-multi:0.1.3 + imagePullPolicy: IfNotPresent + command: + - /usr/local/bin/multi_company_sanitize_from_s3.sh + env: + - name: VAULT_ENV_FILE + value: /vault/secrets/pg-sanitizer-env + - name: COMPANY_IDS + value: "{{workflow.parameters.company_ids}}" + - name: EXCLUDE_COMPANY_IDS + value: "{{workflow.parameters.exclude_company_ids}}" + - name: AUTO_DISCOVER_COMPANY_IDS + value: "{{workflow.parameters.auto_discover_company_ids}}" + - name: AUTO_COMPANY_LIMIT + value: "{{workflow.parameters.auto_company_limit}}" + - name: REUSE_BASE_DATABASES + value: "{{workflow.parameters.reuse_base_databases}}" + - name: RUN_FK_INTEGRITY_VALIDATION + value: "{{workflow.parameters.run_fk_integrity_validation}}" + - name: FK_INTEGRITY_DBS + value: "{{workflow.parameters.fk_integrity_dbs}}" + - name: S3_BUCKET + value: "{{workflow.parameters.bucket}}" + - name: SOURCE_PREFIX + value: "{{workflow.parameters.source_prefix}}" + - name: OUTPUT_PREFIX + value: "{{workflow.parameters.output_prefix}}" + - name: S3_ENDPOINT_URL + value: "{{workflow.parameters.s3_endpoint_url}}" + - name: RUN_ID + value: "{{workflow.name}}" + - name: PGHOST + value: postgresql.postgresql.svc.cluster.local + - name: PGPORT + value: "5432" + - name: PGUSER + value: postgres + - name: PGSSLMODE + value: disable + - name: PG_ALLOW_NONLOCAL + value: "false" + - name: PGRESTORE_JOBS + value: "6" + - name: DB_WORK_MEM + value: 64MB + - name: DB_MAINTENANCE_WORK_MEM + value: 1GB + - name: DB_TEMP_BUFFERS + value: 64MB + - name: RESTORE_VALIDATION_STRICT + value: "false" + - name: DROP_DATABASES_ON_FAILURE + value: "false" + - name: DB_ITEMS + value: "{{workflow.parameters.db_items}}" + - name: WORK_ROOT + value: /work + resources: + requests: + cpu: "4" + memory: 8Gi + ephemeral-storage: 120Gi + volumeMounts: + - name: work + mountPath: /work + volumes: + - name: work + emptyDir: + sizeLimit: 120Gi