---

project:
  name: test-new-op

ha:
  enabled: true
  flinkConfig:
    high-availability: org.apache.flink.kubernetes.highavailability.KubernetesHaServicesFactory
    high-availability.storageDir: hdfs://t2ru-dl-hdfs-ha.corp.t2.ru:8020/warehouse/tds.db/sftp_tms_huawei/ha_state/test-new-op
  persistence:
    enabled: false

taskManager:
  resource:
    memory: "6g"
    cpu: 6

jobManager:
  replicas: 2
  image: artifactory.tdp.corp.t2.ru:443/infrastructure-docker-repo/flink:1.19.3-scala_2.12-java8-hadoop-libs-bdops
  flinkversion: v1_19
  resource:
    memory: "4000m"
    cpu: 1
  ingress:
    enabled: false
    template: |-  
      "flink-tst-{{name}}.bdo.corp.t2.ru"
    className: "nginx"
    annotations:
      nginx.ingress.kubernetes.io/proxy-body-size: "300m"
      nginx.ingress.kubernetes.io/client-max-body-size: "300m"
      nginx.ingress.kubernetes.io/auth-snippet: | 
        proxy_set_header Ldap-Allowed-Groups "SYS BDO Flink Cluster UI Users";
      nginx.ingress.kubernetes.io/auth-url: https://another-ldap.bdo.corp.t2.ru/auth
      nginx.ingress.kubernetes.io/server-snippet: |
        error_page 401 = @login;
        location @login {
          return 302 https://another-ldap.bdo.corp.t2.ru/?protocol=$pass_access_scheme&callback=$host;
        }
  flinkConfiguration:
    jobConfig.hosts: "10.216.15.132, 10.216.15.133, 10.216.15.134, 10.216.15.135, 10.216.15.136, 10.216.15.137, 10.216.15.138, 10.216.15.139, 10.216.15.140"
    jobConfig.remoteDir: "/store/mbb_ufdr"
    jobConfig.hdfsSinkPath: "/warehouse/tds.db/sftp_tms_huawei/data" # TODO
    # state.checkpoints.dir: hdfs://t2ru-dl-hdfs-ha.corp.t2.ru:8020/warehouse/tds.db/sftp_tms_huawei/checkpoints/tms_huawei/job=huawei_dpi_spb_ekt
    # state.checkpoints.num-retained: 3
    # execution.checkpointing.savepoint-dir: hdfs://t2ru-dl-hdfs-ha.corp.t2.ru:8020/warehouse/tds.db/sftp_tms_huawei/savepoints/tms_huawei/job=huawei_dpi_spb_ekt
    # execution.checkpointing.interval: 1500000
    # execution.checkpointing.timeout: 1800000
    # execution.checkpointing.tolerable-failed-checkpoints: 1
    taskmanager.numberOfTaskSlots: "6"
    taskmanager.memory.managed.fraction: 0.1

    # включение автоскейлера
    job.autoscaler.enabled: "false"
    # включение режима dry-run - скейлинг выключен, но оператор об этом не знает
    job.autoscaler.scaling.enabled: "true"
    # окно снятия показателей утилизации
    job.autoscaler.metrics.window: "15m"
    job.autoscaler.stabilization.interval: "1m"

    # метрика автоскейлинга
    job.autoscaler.target.utilization: "0.6"
    job.autoscaler.utilization.max: "0.8"
    job.autoscaler.utilization.min: "0.4"

    # за сколько должны догнать лаг
    job.autoscaler.catch-up.duration: "30m"
    # не включать - скейлит вниз heap per TM при расширении вверх parallelism 
    job.autoscaler.memory.tuning.enabled: "false"

    jobmanager.scheduler: adaptive

    # максимальный скейлинг по параллелизму
    pipeline.max-parallelism: "72"

    flink.hadoop.hadoop.security.authentication: kerberos
    flink.hadoop.hadoop.http.authentication.kerberos.principal: service.tdp_int_02@CORP.TELE2.RU
    flink.hadoop.hadoop.http.authentication.kerberos.keytab: /etc/keytab/service.tdp_int_02.keytab
    flink.hadoop.hadoop.kerberos.keytab.login.autorenewal.enabled: "true"
    execution.checkpointing.externalized-checkpoint-retention: RETAIN_ON_CANCELLATION
    web.cancel.enable: true
    security.kerberos.access.hadoopFileSystems: hdfs://t2ru-dl-hdfs-ha.corp.t2.ru:8020
    security.kerberos.fetch.delegation-token: false
    security.kerberos.login.principal: service.tdp_int_02@CORP.TELE2.RU
    security.kerberos.login.keytab: /etc/keytab/service.tdp_int_02.keytab
    security.kerberos.login.contexts: Client,Hadoop
    security.kerberos.login.use-ticket-cache: true
    akka.framesize: 31457280b
  podTemplate:
    kerberos: 
      enabled: false
    main:
      volumeMounts:
      - name: krb5
        mountPath: /etc/krb5.conf
        subPath: krb5.conf
      - name: jaas
        mountPath: /etc/jaas.conf
        subPath: jaas.conf
      env:
      - name: _JAVA_OPTIONS
        value: -Djavax.security.auth.useSubjectCredsOnly=false
      - name: HADOOP_CLASSPATH
        value: /opt/hadoop/etc/hadoop:/opt/hadoop/share/hadoop/common/lib/*:/opt/hadoop/share/hadoop/common/*:/opt/hadoop/share/hadoop/hdfs:/opt/hadoop/share/hadoop/hdfs/lib/*:/opt/hadoop/share/hadoop/hdfs/*:/opt/hadoop/share/hadoop/mapreduce/*:/opt/hadoop/share/hadoop/yarn:/opt/hadoop/share/hadoop/yarn/lib/*:/opt/hadoop/share/hadoop/yarn/*:/opt/flink/lib/*
      - name: SFTP_USER
        valueFrom:
          secretKeyRef:
            name: flink-sftp-spb-ros-secret
            key: SFTP_USER
      - name: SFTP_PASSWORD
        valueFrom:
          secretKeyRef:
            name: flink-sftp-spb-ros-secret
            key: SFTP_PASSWORD
    volumes:
    - name: krb5
      configMap:
        name: krb5.conf
        items:
        - key: krb5.conf
          path: krb5.conf
    - name: jaas
      configMap:
        name: jaas.conf
        items:
        - key: jaas.conf
          path: jaas.conf

jobs:
  - name: huawei-dpi-ros-test
    enabled: true
    jarURI: "https://artifactory.tdp.corp.t2.ru:443/artifactory/bdo-flink-jars-repo/sftp-to-hdfs-master-1dfc42d4.jar"
    parallelism: 1
    upgradeMode: stateless
    savepointTriggerNonce: 0
    # initialSavepointPath: hdfs://t2ru-dl-hdfs-ha.corp.t2.ru:8020/warehouse/stg/stg_ftp/checkpoints/tms_huawei/job=huawei_dpi_ros/4d1189facca1b15921457c16a3c759af/chk-483
    entryClass: "com.tele2.sftpflink.HuaweiSftpToHdfsApp"
    args:
      - "--jobName=huawei-dpi-ros-test"

quota:
  enabled: false