config_ppp.yaml

---
# Platform Output Support Configuration File - PPP
work_path: ./work
log_level: DEBUG
release_uri: gs://open-targets-pipeline-runs/sz/26.03-ppp.1 # e.g gs://open-targets-pipeline-runs/szsz/25.06-testrun-4
scratchpad:
  # ================================= SET THESE VALUES ====================================
  release: 26.03 # e.g. 25.12
  data_source: gs://open-targets-pipeline-runs/sz/26.03-ppp.1 # same as release_uri
  database_namespace: ppp2603 # e.g. ppp2603
  # Google disk snapshot variables
  opensearch_disk_name: ppp-2603-os
  opensearch_disk_snapshot_name: ppp-2603-os
  clickhouse_disk_name: ppp-2603-ch
  clickhouse_disk_snapshot_name: ppp-2603-ch
  # ========================= THESE VALUES RARELY NEED CHANGING ===========================
  disk_snapshot_project_id: open-targets-eu-dev
  release_gcs_bucket: gs://open-targets-ppp-data-releases
  aws_address: s3://open-targets-public-data-releases/platform

  # temp file locations
  local_data: release_data
  prepared_data: prepared_data

  # opensearch
  opensearch_version: 3.5.0
  opensearch_java_opts: -Xms60g -Xmx300g
  opensearch_data: /mnt/opensearch/data
  opensearch_logs: /var/log/pos/opensearch
  opensearch_snapshot_repository: ot-os-snapshots
  opensearch_snapshot_bucket: open-targets-data-backends
  opensearch_snapshot_base_path: opensearch
  opensearch_indices_to_restore: '*,-.*' # e.g. '*,-.*' to restore all except hidden indices

  # clickhouse
  clickhouse_version: 25.8.18.1
  clickhouse_data: /mnt/clickhouse/data
  clickhouse_logs: /var/log/pos/clickhouse
  clickhouse_backup_base_path: https://storage.googleapis.com/open-targets-data-backends/clickhouse/

steps:
  ##################################################################################################
  # Full PPP backend
  ##################################################################################################
  backend:
    # sync data
    - name: sync_bucket etl output files
      source: ${data_source}/output
      destination: ${local_data}/output
    - name: sync_bucket etl view files
      source: ${data_source}/view
      destination: ${local_data}/view
    # ot croissant
    - name: ot_croissant generate release metadata
      requires:
        - sync_bucket etl output files
      ftp_address:
      gcp_address: ${release_gcs_bucket}/${release}/output/
      aws_address:
      dataset_path: ${local_data}/output
      output: croissant.json
      prepared_data_parent: ${prepared_data}
    # opensearch
    - name: open_search_start data loading instance
      volume_data: ${opensearch_data}
      volume_logs: ${opensearch_logs}
      opensearch_version: ${opensearch_version}
      opensearch_java_opts: ${opensearch_java_opts}
      startup_wait: '60'
    - name: explode prep and load all datasets
      requires:
        - sync_bucket etl output files
        - sync_bucket etl view files
        - ot_croissant generate release metadata
        - open_search_start data loading instance
      foreach: &opensearch_datasets
        - croissant
        - facet_search_disease
        - facet_search_target
        - search_disease
        - search_drug
        - search_study
        - search_target
        - search_variant
      do:
        - name: explode_data_prep ${each}
          dataset: ${each}
          parquet_parent: ${local_data}
          json_parent: ${prepared_data}
          step: opensearch
        - name: open_search_create_index ${each}
          requires:
            - explode_data_prep ${each}
          dataset: ${each}
          prefix: ${database_namespace}
        - name: open_search_load dataset ${each}
          requires:
            - open_search_create_index ${each}
          dataset: ${each}
          prefix: ${database_namespace}
          json_parent: ${prepared_data}
    - name: open_search_snapshot data loading instance
      requires:
        - explode prep and load all datasets
      snapshot_repository_name: ${opensearch_snapshot_repository}
      snapshot_name: ${database_namespace}
      snapshot_bucket: ${opensearch_snapshot_bucket}
      snapshot_base_path: ${opensearch_snapshot_base_path}/${database_namespace}
    - name: open_search_stop data loading instance
      requires:
        - open_search_snapshot data loading instance
      service_name: os-pos
    - name: create_gcp_disk_snapshot opensearch
      gcp_project_id: ${disk_snapshot_project_id}
      gcp_disk_name: ${opensearch_disk_name}
      gcp_snapshot_name: ${opensearch_disk_snapshot_name}
      gcp_disk_zone: europe-west1-d
      mount_point: ${opensearch_data}
      requires:
        - open_search_stop data loading instance
    # clickhouse
    - name: clickhouse_start data loading instance
      volume_data: ${clickhouse_data}
      volume_logs: ${clickhouse_logs}
      clickhouse_version: ${clickhouse_version}
      clickhouse_database: ${database_namespace}
    - name: explode load all datasets
      requires:
        - sync_bucket etl output files
        - sync_bucket etl view files
        - clickhouse_start data loading instance
      foreach: &clickhouse_datasets
        - baseline_expression
        - biosample
        - credible_sets
        - colocalisation
        - disease_hpo
        - drug
        - drug_warnings
        - enhancer_to_gene
        - evidence
        - expression
        - go
        - hpo
        - interaction_evidence
        - l2g_predictions
        - literature
        - mechanism_of_action
        - mouse_phenotypes
        - novelty
        - openfda_faers
        - otar_projects
        - pharmacogenomics
        - protein_coding_coords
        - so
        - target_essentiality
        - variants
        - w2v
        - clinical_indication
        - clinical_report
        - clinical_target
      do:
        - name: clickhouse_load clickhouse load ${each}
          dataset: ${each}
          clickhouse_database: ${database_namespace}
          data_dir_parent: ${local_data}
    - name: clickhouse_load clickhouse target_prioritisation
      requires:
        - clickhouse_load clickhouse load target_essentiality
      dataset: target_prioritisation
      clickhouse_database: ${database_namespace}
      data_dir_parent: ${local_data}
    - name: clickhouse_load clickhouse load interaction
      requires:
        - clickhouse_load clickhouse load interaction_evidence
      dataset: interaction
      clickhouse_database: ${database_namespace}
      data_dir_parent: ${local_data}
    - name: clickhouse_load clickhouse load studies
      requires:
        - clickhouse_load clickhouse load credible_sets
      dataset: studies
      clickhouse_database: ${database_namespace}
      data_dir_parent: ${local_data}
    - name: clickhouse_load clickhouse load targets
      requires:
        - clickhouse_load clickhouse load studies
      dataset: targets
      clickhouse_database: ${database_namespace}
      data_dir_parent: ${local_data}
    - name: clickhouse_load clickhouse load disease
      requires:
        - clickhouse_load clickhouse load studies
      dataset: disease
      clickhouse_database: ${database_namespace}
      data_dir_parent: ${local_data}
    - name: clickhouse_load clickhouse load aotf
      requires:
        - clickhouse_load clickhouse load disease
      dataset: aotf
      clickhouse_database: ${database_namespace}
      data_dir_parent: ${local_data}
    - name: explode backup clickhouse tables
      requires:
        - explode load all datasets
        - clickhouse_load clickhouse target_prioritisation
        - clickhouse_load clickhouse load interaction
        - clickhouse_load clickhouse load studies
        - clickhouse_load clickhouse load disease
        - clickhouse_load clickhouse load aotf
        - clickhouse_load clickhouse load targets
      foreach: &clickhouse_tables
        - associations_otf_target
        - associations_otf_disease
        - baseline_expression
        - biosample
        - colocalisation
        - credible_sets
        - credible_sets_by_variant
        - credible_sets_by_region
        - credible_sets_locus
        - disease_hpo
        - disease
        - drug_warnings
        - drug
        - enhancer_to_gene
        - evidence
        - evidence_by_variant
        - evidence_by_disease_and_target
        - expression
        - gene_ontology
        - hpo
        - interaction
        - l2g_predictions
        - literature
        - mechanism_of_action
        - ml_w2v
        - mouse_phenotypes
        - novelty
        - openfda_faers
        - otar_projects
        - pharmacogenomics_by_drug
        - pharmacogenomics_by_target
        - pharmacogenomics_by_variant
        - protein_coding_coords_by_target
        - protein_coding_coords_by_variant
        - sequence_ontology
        - studies
        - target_essentiality
        - target_prioritisation
        - targets
        - variants
        - clinical_indication_disease
        - clinical_indication_drug
        - clinical_report
        - clinical_target
      do:
        - name: clickhouse_backup table ${each}
          clickhouse_database: ${database_namespace}
          table: ${each}
          gcs_base_path: ${clickhouse_backup_base_path}
    - name: clickhouse_stop data loading instance
      requires:
        - explode backup clickhouse tables
      clickhouse_database_name: ${database_namespace}
    - name: create_gcp_disk_snapshot clickhouse
      requires:
        - clickhouse_stop data loading instance
      gcp_project_id: ${disk_snapshot_project_id}
      gcp_disk_name: ${clickhouse_disk_name}
      gcp_snapshot_name: ${clickhouse_disk_snapshot_name}
      gcp_disk_zone: europe-west1-d
      mount_point: ${clickhouse_data}
  # Tarballs
  ################################################################################################
  tarballs:
    - name: tarball opensearch
      source: /mnt/opensearch
      destination: disk_images/opensearch.tgz
    - name: tarball clickhouse
      source: /mnt/clickhouse
      destination: disk_images/clickhouse.tgz

  ################################################################################################
  # Sync data only
  ################################################################################################
  sync_data:
    - name: sync_bucket etl output files
      source: ${data_source}/output
      destination: ${local_data}/output
    - name: sync_bucket etl view files
      source: ${data_source}/view
      destination: ${local_data}/view

  ################################################################################################
  # OT Croissant
  ################################################################################################
  ot_croissant:
    - name: sync_bucket etl output files
      source: ${data_source}/output
      destination: ${local_data}/output
    - name: ot_croissant generate release metadata
      requires:
        - sync_bucket etl output files
      ftp_address:
      gcp_address: ${release_gcs_bucket}/${release}/output/
      aws_address:
      dataset_path: ${local_data}/output
      output: croissant.json
      prepared_data_parent: ${prepared_data}
  ################################################################################################
  # OpenSearch
  ################################################################################################
  opensearch:
    - name: sync_bucket etl output files
      source: ${data_source}/output
      destination: ${local_data}/output
    - name: sync_bucket etl view files
      source: ${data_source}/view
      destination: ${local_data}/view
    - name: ot_croissant generate release metadata
      requires:
        - sync_bucket etl output files
      ftp_address:
      gcp_address: ${release_gcs_bucket}/${release}/output/
      aws_address:
      dataset_path: ${local_data}/output
      output: croissant.json
      prepared_data_parent: ${prepared_data}
    - name: open_search_start data loading instance
      volume_data: ${opensearch_data}
      volume_logs: ${opensearch_logs}
      opensearch_version: ${opensearch_version}
      opensearch_java_opts: ${opensearch_java_opts}
      startup_wait: '60'
    - name: explode prep and load all datasets
      requires:
        - sync_bucket etl output files
        - sync_bucket etl view files
        - ot_croissant generate release metadata
        - open_search_start data loading instance
      foreach: *opensearch_datasets
      do:
        - name: explode_data_prep ${each}
          dataset: ${each}
          parquet_parent: ${local_data}
          json_parent: ${prepared_data}
          step: opensearch
        - name: open_search_create_index ${each}
          requires:
            - explode_data_prep ${each}
          dataset: ${each}
          prefix: ${database_namespace}
        - name: open_search_load dataset ${each}
          requires:
            - open_search_create_index ${each}
          dataset: ${each}
          prefix: ${database_namespace}
          json_parent: ${prepared_data}
    - name: open_search_stop data loading instance
      requires:
        - explode prep and load all datasets
      service_name: os-pos
    - name: create_gcp_disk_snapshot opensearch
      gcp_project_id: ${disk_snapshot_project_id}
      gcp_disk_name: ${opensearch_disk_name}
      gcp_snapshot_name: ${opensearch_disk_snapshot_name}
      gcp_disk_zone: europe-west1-d
      mount_point: ${opensearch_data}
      requires:
        - open_search_stop data loading instance
  ################################################################################################
  # OpenSearch - Modular Steps
  ################################################################################################
  opensearch_start:
    - name: open_search_start data loading instance
      service_name: os-pos
      volume_data: ${opensearch_data}
      volume_logs: ${opensearch_logs}
      opensearch_version: ${opensearch_version}
      opensearch_java_opts: ${opensearch_java_opts}
  opensearch_disk_snapshot:
    - name: create_gcp_disk_snapshot opensearch
      gcp_project_id: ${disk_snapshot_project_id}
      gcp_disk_name: ${opensearch_disk_name}
      gcp_snapshot_name: ${opensearch_disk_snapshot_name}
      gcp_disk_zone: europe-west1-d
      mount_point: ${opensearch_data}
  opensearch_snapshot: # create a snapshot of the opensearch data NOT the disk.
    - name: open_search_start data loading instance
      volume_data: ${opensearch_data}
      volume_logs: ${opensearch_logs}
      opensearch_version: ${opensearch_version}
      opensearch_java_opts: ${opensearch_java_opts}
      startup_wait: '60'
    - name: open_search_snapshot data loading instance
      requires:
        - open_search_start data loading instance
      snapshot_repository_name: ${opensearch_snapshot_repository}
      snapshot_name: ${database_namespace}
      snapshot_bucket: ${opensearch_snapshot_bucket}
      snapshot_base_path: ${opensearch_snapshot_base_path}/${database_namespace}
  opensearch_restore:
    - name: open_search_restore indices
      snapshot_repository_name: ${opensearch_snapshot_repository}
      snapshot_name: ${database_namespace}
      snapshot_bucket: ${opensearch_snapshot_bucket}
      snapshot_base_path: ${opensearch_snapshot_base_path}/${database_namespace}
      indices: ${opensearch_indices_to_restore}
  opensearch_stop:
    - name: open_search_stop data loading instance
      service_name: os-pos
  opensearch_tarball:
    - name: tarball opensearch
      source: /mnt/opensearch
      destination: disk_images/opensearch.tgz
  opensearch_prep_all:
    - name: sync_bucket pipeline output files
      source: ${data_source}/output
      destination: ${local_data}/output
    - name: sync_bucket pipeline view files
      source: ${data_source}/view/
      destination: ${local_data}/view
    - name: explode prep all datasets
      requires:
        - sync_bucket pipeline output files
        - sync_bucket pipeline view files
      foreach: *opensearch_datasets
      do:
        - name: explode_data_prep ${each}
          dataset: ${each}
          parquet_parent: ${local_data}
          json_parent: ${prepared_data}
          step: opensearch
  opensearch_load_all:
    - name: open_search_start data loading instance
      volume_data: ${opensearch_data}
      volume_logs: ${opensearch_logs}
      opensearch_version: ${opensearch_version}
      opensearch_java_opts: ${opensearch_java_opts}
    - name: explode load all datasets
      foreach: *opensearch_datasets
      do:
        - name: open_search_create_index ${each}
          requires:
            - open_search_start data loading instance
          dataset: ${each}
          prefix: ${database_namespace}
        - name: open_search_load dataset ${each}
          requires:
            - open_search_create_index ${each}
          dataset: ${each}
          prefix: ${database_namespace}
          json_parent: ${prepared_data}

  ################################################################################################
  # ClickHouse
  ################################################################################################
  clickhouse:
    - name: sync_bucket etl output files
      source: ${data_source}/output
      destination: ${local_data}/output
    - name: sync_bucket etl view files
      source: ${data_source}/view
      destination: ${local_data}/view
    - name: clickhouse_start data loading instance
      volume_data: ${clickhouse_data}
      volume_logs: ${clickhouse_logs}
      clickhouse_version: ${clickhouse_version}
      clickhouse_database: ${database_namespace}
    - name: explode load all datasets
      requires:
        - sync_bucket etl output files
        - sync_bucket etl view files
        - clickhouse_start data loading instance
      foreach: *clickhouse_datasets
      do:
        - name: clickhouse_load clickhouse load ${each}
          dataset: ${each}
          clickhouse_database: ${database_namespace}
          data_dir_parent: ${local_data}
    - name: clickhouse_load clickhouse target_prioritisation
      requires:
        - clickhouse_load clickhouse load target_essentiality
      dataset: target_prioritisation
      clickhouse_database: ${database_namespace}
      data_dir_parent: ${local_data}
    - name: clickhouse_load clickhouse load interaction
      requires:
        - clickhouse_load clickhouse load interaction_evidence
      dataset: interaction
      clickhouse_database: ${database_namespace}
      data_dir_parent: ${local_data}
    - name: clickhouse_load clickhouse load studies
      requires:
        - clickhouse_load clickhouse load credible_sets
      dataset: studies
      clickhouse_database: ${database_namespace}
      data_dir_parent: ${local_data}
    - name: clickhouse_load clickhouse load targets
      requires:
        - clickhouse_load clickhouse load studies
      dataset: targets
      clickhouse_database: ${database_namespace}
      data_dir_parent: ${local_data}
    - name: clickhouse_load clickhouse load disease
      requires:
        - clickhouse_load clickhouse load studies
      dataset: disease
      clickhouse_database: ${database_namespace}
      data_dir_parent: ${local_data}
    - name: clickhouse_load clickhouse load aotf
      requires:
        - clickhouse_load clickhouse load disease
      dataset: aotf
      clickhouse_database: ${database_namespace}
      data_dir_parent: ${local_data}
    - name: clickhouse_stop data loading instance
      requires:
        - explode load all datasets
        - clickhouse_load clickhouse target_prioritisation
        - clickhouse_load clickhouse load interaction
        - clickhouse_load clickhouse load studies
        - clickhouse_load clickhouse load disease
        - clickhouse_load clickhouse load aotf
        - clickhouse_load clickhouse load targets
      clickhouse_database_name: ${database_namespace}
    - name: create_gcp_disk_snapshot clickhouse
      requires:
        - clickhouse_stop data loading instance
      gcp_project_id: ${disk_snapshot_project_id}
      gcp_disk_name: ${clickhouse_disk_name}
      gcp_snapshot_name: ${clickhouse_disk_snapshot_name}
      gcp_disk_zone: europe-west1-d
      mount_point: ${clickhouse_data}
  ################################################################################################
  # ClickHouse - Modular Steps
  ################################################################################################

  clickhouse_start:
    - name: clickhouse_start data loading instance
      volume_data: ${clickhouse_data}
      volume_logs: ${clickhouse_logs}
      clickhouse_version: ${clickhouse_version}
      clickhouse_database: ${database_namespace}
  clickhouse_tarball:
    - name: tarball clickhouse
      source: /mnt/clickhouse
      destination: disk_images/clickhouse.tgz
  clickhouse_disk_snapshot:
    - name: create_gcp_disk_snapshot clickhouse
      gcp_project_id: ${disk_snapshot_project_id}
      gcp_disk_name: ${clickhouse_disk_name}
      gcp_snapshot_name: ${clickhouse_disk_snapshot_name}
      gcp_disk_zone: europe-west1-d
      mount_point: ${clickhouse_data}
  clickhouse_backup_all:
    - name: clickhouse_start data loading instance
      volume_data: ${clickhouse_data}
      volume_logs: ${clickhouse_logs}
      clickhouse_version: ${clickhouse_version}
      clickhouse_database: ${database_namespace}
    - name: explode backup clickhouse tables
      requires:
        - clickhouse_start data loading instance
      foreach: *clickhouse_tables
      do:
        - name: clickhouse_backup table ${each}
          requires:
            - clickhouse_start data loading instance
          clickhouse_database: ${database_namespace}
          table: ${each}
          gcs_base_path: ${clickhouse_backup_base_path}
  clickhouse_restore_all:
    - name: clickhouse_create_database for restoration
      clickhouse_database: ${database_namespace}
    - name: explode restore clickhouse tables
      requires:
        - clickhouse_create_database for restoration
      foreach: *clickhouse_tables
      do:
        - name: clickhouse_restore table ${each}
          clickhouse_database: ${database_namespace}
          table: ${each}
          gcs_base_path: ${clickhouse_backup_base_path}
  clickhouse_stop:
    - name: clickhouse_stop data loading instance
      clickhouse_database_name: ${database_namespace}
  clickhouse_load_all:
    - name: sync_bucket pipeline output files
      source: ${data_source}/output
      destination: ${local_data}/output
    - name: sync_bucket pipeline view files
      source: ${data_source}/view
      destination: ${local_data}/view
    - name: clickhouse_start data loading instance
      volume_data: ${clickhouse_data}
      volume_logs: ${clickhouse_logs}
      clickhouse_version: ${clickhouse_version}
      clickhouse_database: ${database_namespace}
    - name: explode load all datasets
      foreach: *clickhouse_datasets
      do:
        - name: clickhouse_load clickhouse load ${each}
          requires:
            - clickhouse_start data loading instance
            - sync_bucket pipeline output files
            - sync_bucket pipeline view files
          dataset: ${each}
          clickhouse_database: ${database_namespace}
          data_dir_parent: ${local_data}
    - name: clickhouse_load clickhouse load aotf
      requires:
        - clickhouse_load clickhouse load disease
      dataset: aotf
      clickhouse_database: ${database_namespace}
      data_dir_parent: ${local_data}
    - name: clickhouse_load clickhouse load targets
      requires:
        - clickhouse_load clickhouse load credible_sets
        - clickhouse_load clickhouse load studies
      dataset: targets
      clickhouse_database: ${database_namespace}
      data_dir_parent: ${local_data}
  ################################################################################################
  # gcs_sync
  gcs_sync:
    - name: sync_bucket gcs release data
      source: ${data_source}/
      destination: ${release_gcs_bucket}/${release}/