Загрузка данных


dmitriev-aal@VDI-Dmitriev-A:~$ kubectl --context rcsdstbl -n vector get cm vector-agent -o yaml | \
>   grep -A40 -B10 -Ei 'elasticsearch|opensearch|logstash|sink|index'
apiVersion: v1
data:
  vector.yaml: |
    api:
      address: 0.0.0.0:8686
      enabled: true
    data_dir: /var/lib/vector
    sinks:
      elasticsearch_journald_sink:
        api_version: v7
        auth:
          password: pe8jefowU5/huK4uc76fh+pHmRgYCdelW7MV3GMjCKc=
          strategy: basic
          user: fluentd
        batch:
          max_bytes: 10485760
        buffer:
          max_size: 314572800
          type: disk
          when_full: block
        bulk:
          index: journald-%Y.%m.%d
        compression: none
        endpoints:
        - https://elastic-data.rshbdev.ru
        inputs:
        - sanitizeJournalD
        mode: normal
        request:
          concurrency: adaptive
        tls:
          ca_file: /usr/share/cacert/rshb_bundle.pem
        type: elasticsearch
      elasticsearch_platform_sink:
        api_version: v7
        auth:
          password: pe8jefowU5/huK4uc76fh+pHmRgYCdelW7MV3GMjCKc=
          strategy: basic
          user: fluentd
        batch:
          max_bytes: 20971520
        buffer:
          max_size: 314572800
          type: disk
          when_full: block
        bulk:
          index: platform-%Y.%m.%d
        compression: none
        endpoints:
        - https://elastic-data.rshbdev.ru
        inputs:
        - platformLogs
        mode: normal
        request:
          concurrency: adaptive
        tls:
          ca_file: /usr/share/cacert/rshb_bundle.pem
        type: elasticsearch
      elasticsearch_sink:
        api_version: v7
        auth:
          password: pe8jefowU5/huK4uc76fh+pHmRgYCdelW7MV3GMjCKc=
          strategy: basic
          user: fluentd
        batch:
          max_bytes: 20971520
        buffer:
          max_size: 1610612736
          type: disk
          when_full: block
        bulk:
          index: logstash-%Y.%m.%d
        compression: none
        endpoints:
        - https://elastic-data.rshbdev.ru
        inputs:
        - throttle
        mode: normal
        request:
          concurrency: adaptive
        tls:
          ca_file: /usr/share/cacert/rshb_bundle.pem
        type: elasticsearch
      metrics:
        address: 0.0.0.0:9598
        inputs:
        - internal_metrics
        type: prometheus_exporter
    sources:
      internal_metrics:
        scrape_interval_secs: 5
        type: internal_metrics
      journaldSource:
        batch_size: 8
        current_boot_only: false
        include_matches:
          _SYSTEMD_UNIT:
          - kubelet.service
          - containerd.service
          - nginx.service
          _TRANSPORT:
          - kernel
        since_now: true
        type: journald
      kubernetesLogs:
        extra_label_selector: app.kubernetes.io/name!=grafana-operator,type!=adapter
        glob_minimum_cooldown_ms: 100
        max_line_bytes: 8388608
        max_read_bytes: 8388608
        pod_annotation_fields:
          pod_labels: kubernetes.labels
          pod_namespace: kubernetes.namespace_name
          pod_node_name: kubernetes.node_name
        type: kubernetes_logs
    transforms:
      parseJSON:
        inputs:
        - sanitizeEvents
        source: |
          json_message,err = parse_json(.message)

          ## https://gitlab.rshbdev.ru/rshbintech/integrations/ckpr/board/-/issues/2336
          ## Replace @timestamp with .timestamp, or use timestamp if it exists
dmitriev-aal@VDI-Dmitriev-A:~$ kubectl --context rcsdstbl -n vector logs \
>   vector-agent-xdb2q --since=24h | \
>   grep -Ei 'error|warn|quotes|897454e2|elastic|opensearch|sink|failed|retry'
dmitriev-aal@VDI-Dmitriev-A:~$ kubectl --context rcsdstbl -n vector exec \
>   vector-agent-xdb2q -- \
>   ls -la /var/log/pods | grep quotes
drwxr-xr-x  5 root root  4096 Oct  5 12:30 isys-sinv-int_quotes-5b5d6675f4-s5msn_897454e2-90d9-454a-bbe0-6281f34892ae
dmitriev-aal@VDI-Dmitriev-A:~$