https://pastein.ru/t/imW
Загрузка данных
Running with gitlab-runner 15.8.0 (12335144)
on gitlab-runner-gitlab-runner-64775b4cff-qpdc8 9cgwHtm5, system ID: r_FsAm41LqwxCr
Preparing the "kubernetes" executor
00:00
Using Kubernetes namespace: gitlab-runners
Using Kubernetes executor with image registry.rshbdev.ru/appfarm/infra/images/kube-client-apps:8.21.15 ...
Using attach strategy to execute scripts...
Preparing environment
00:09
Waiting for pod gitlab-runners/runner-9cgwhtm5-project-82622-concurrent-0fcr2p to be running, status is Pending
Waiting for pod gitlab-runners/runner-9cgwhtm5-project-82622-concurrent-0fcr2p to be running, status is Pending
ContainersNotInitialized: "containers with incomplete status: [init-permissions]"
ContainersNotReady: "containers with unready status: [build helper]"
ContainersNotReady: "containers with unready status: [build helper]"
Waiting for pod gitlab-runners/runner-9cgwhtm5-project-82622-concurrent-0fcr2p to be running, status is Pending
ContainersNotReady: "containers with unready status: [build helper]"
ContainersNotReady: "containers with unready status: [build helper]"
Running on runner-9cgwhtm5-project-82622-concurrent-0fcr2p via gitlab-runner-gitlab-runner-64775b4cff-qpdc8...
Getting source from Git repository
00:03
$ git config --global --add url."https://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}/".insteadOf "https://${CI_SERVER_HOST}" # collapsed multi-line command
Fetching changes with git depth set to 20...
Initialized empty Git repository in /builds/rshbintech/crft/ckof/ownfin/backend/prd/owf-ms-prd-debitcardapplication/.git/
Created fresh repository.
Checking out 885b636d as stress...
Skipping Git submodules setup
Downloading artifacts
00:01
Downloading artifacts for build (24172842)...
Downloading artifacts from coordinator... ok id=24172842 responseStatus=200 OK token=64_UY6JT
Downloading artifacts for dockerfilegen (24172841)...
Downloading artifacts from coordinator... ok id=24172841 responseStatus=200 OK token=64_UY6JT
Downloading artifacts for devsecops_antivirus_scan (24172840)...
Downloading artifacts from coordinator... ok id=24172840 responseStatus=200 OK token=64_UY6JT
Downloading artifacts for unit (24172844)...
Downloading artifacts from coordinator... ok id=24172844 responseStatus=200 OK token=64_UY6JT
Executing "step_script" stage of the job script
00:55
$ ( umask 0077; mkdir -p ~/.kube && echo "$KUBECONFIG_COMBINED" | base64 -d > ~/.kube/config )
$ if [[ ${IS_SENSITIVE_SYSTEM} = true ]]; then export K8S_CLUSTERS=$K8S_CLUSTERS_SENSITIVE; fi
$ echo 'IS_SENSITIVE_SYSTEM:' $IS_SENSITIVE_SYSTEM
IS_SENSITIVE_SYSTEM: true
$ echo 'TARGET_CLUSTERS:' $K8S_CLUSTERS
TARGET_CLUSTERS: rcsdstbl
$ set -x
++ echo '$ for CLUSTER in $K8S_CLUSTERS; do'
$ for CLUSTER in $K8S_CLUSTERS; do
++ for CLUSTER in $K8S_CLUSTERS
++ echo '$ export CLUSTER=$CLUSTER'
$ export CLUSTER=$CLUSTER
++ export CLUSTER=rcsdstbl
++ CLUSTER=rcsdstbl
++ echo '$ if [[ ${CI_ENVIRONMENT_SLUG} = production ]]; then export NAMESPACE=isys-${ISYS_NAME}; else export NAMESPACE=isys-${ISYS_NAME}-${CI_ENVIRONMENT_SLUG}; fi'
$ if [[ ${CI_ENVIRONMENT_SLUG} = production ]]; then export NAMESPACE=isys-${ISYS_NAME}; else export NAMESPACE=isys-${ISYS_NAME}-${CI_ENVIRONMENT_SLUG}; fi
++ [[ stress = production ]]
++ export NAMESPACE=isys-ownfin-stress
++ NAMESPACE=isys-ownfin-stress
++ echo '$ read -r REVISION STATUS < <(helm --kube-context ${CLUSTER:-default} -n $NAMESPACE history $CI_PROJECT_NAME | tail -1 | cut -f1,3) || true'
$ read -r REVISION STATUS < <(helm --kube-context ${CLUSTER:-default} -n $NAMESPACE history $CI_PROJECT_NAME | tail -1 | cut -f1,3) || true
++ read -r REVISION STATUS
+++ helm --kube-context rcsdstbl -n isys-ownfin-stress history owf-ms-prd-debitcardapplication
+++ tail -1
+++ cut -f1,3
++ echo '$ if [[ "$STATUS" =~ "pending" ]]; then helm --kube-context ${CLUSTER:-default} -n $NAMESPACE rollback $CI_PROJECT_NAME $REVISION || true; fi'
$ if [[ "$STATUS" =~ "pending" ]]; then helm --kube-context ${CLUSTER:-default} -n $NAMESPACE rollback $CI_PROJECT_NAME $REVISION || true; fi
++ [[ deployed =~ pending ]]
++ echo '$ helmfile ${HELMFILE_DEFAULT_NAMESPACE:+--namespace $HELMFILE_DEFAULT_NAMESPACE} --environment ${CLUSTER:-default} -f deploy/helmfile.yaml --log-level info apply --suppress-secrets --set "additionalWorkloadAnnotations.gitlabPipeline=${CI_PIPELINE_URL}" --set "additionalWorkloadAnnotations.gitlabCommit=${CI_COMMIT_SHA}"'
$ helmfile ${HELMFILE_DEFAULT_NAMESPACE:+--namespace $HELMFILE_DEFAULT_NAMESPACE} --environment ${CLUSTER:-default} -f deploy/helmfile.yaml --log-level info apply --suppress-secrets --set "additionalWorkloadAnnotations.gitlabPipeline=${CI_PIPELINE_URL}" --set "additionalWorkloadAnnotations.gitlabCommit=${CI_COMMIT_SHA}"
++ helmfile --environment rcsdstbl -f deploy/helmfile.yaml --log-level info apply --suppress-secrets --set additionalWorkloadAnnotations.gitlabPipeline=https://gitlab.rshbdev.ru/rshbintech/crft/ckof/ownfin/backend/prd/owf-ms-prd-debitcardapplication/-/pipelines/3472181 --set additionalWorkloadAnnotations.gitlabCommit=885b636dbc23ec1c8dc3f852fd977f19cbb39ab4
== /usr/local/link/helmfile: Initialize base deploy hierarchy in /builds/rshbintech/crft/ckof/ownfin/backend/prd/owf-ms-prd-debitcardapplication
no matches for path: envs/stress/rcsdstbl/helmfile.yaml.gotmpl
Adding repo rshb-charts https://nexus.rshbdev.ru/repository/charts/
"rshb-charts" has been added to your repositories
Comparing release=vault-secrets-owf-ms-prd-debitcardapplication, chart=rshb-charts/raw
Comparing release=psvc-owf-ms-prd-debitcardapplication, chart=rshb-charts/raw
Comparing release=platform-database-owf-ms-prd-debitcardapplication, chart=rshb-charts/raw
Comparing release=owf-ms-prd-debitcardapplication-grafana-dashboard, chart=rshb-charts/grafana-dashboards
Comparing release=owf-ms-prd-debitcardapplication-prometheus-rules, chart=rshb-charts/prometheus-rules
Comparing release=links-owf-ms-prd-debitcardapplication, chart=rshb-charts/raw
Comparing release=owf-ms-prd-debitcardapplication-rumsk1, chart=rshb-charts/base
isys-ownfin-stress, owf-ms-prd-debitcardapplication-rumsk1, Deployment (apps) has changed:
# Source: base/templates/workload.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: owf-ms-prd-debitcardapplication-rumsk1
labels:
app: owf-ms-prd-debitcardapplication
fullname: owf-ms-prd-debitcardapplication-rumsk1
chart: base-1.15.3
release: owf-ms-prd-debitcardapplication-rumsk1
heritage: Helm
isys: "ownfin-stress"
psvc: "owf-ms-prd-debitcardapplication"
version: "stress"
workload.topology.app.farm/zone: "rumsk1"
annotations:
- gitlabCommit: 59c7e02a8101a36cf130992e6ab2b2ce822eb66d
- gitlabPipeline: https://gitlab.rshbdev.ru/rshbintech/crft/ckof/ownfin/backend/prd/owf-ms-prd-debitcardapplication/-/pipelines/3471908
- platform.ckpr.integrations.rshbintech.ru/gitlab-commit-sha: 59c7e02a
- platform.ckpr.integrations.rshbintech.ru/gitlab-pipeline-id: "3471908"
+ gitlabCommit: 885b636dbc23ec1c8dc3f852fd977f19cbb39ab4
+ gitlabPipeline: https://gitlab.rshbdev.ru/rshbintech/crft/ckof/ownfin/backend/prd/owf-ms-prd-debitcardapplication/-/pipelines/3472181
+ platform.ckpr.integrations.rshbintech.ru/gitlab-commit-sha: 885b636d
+ platform.ckpr.integrations.rshbintech.ru/gitlab-pipeline-id: "3472181"
spec:
strategy:
type: RollingUpdate
replicas: 1
selector:
matchLabels:
app: owf-ms-prd-debitcardapplication
fullname: owf-ms-prd-debitcardapplication-rumsk1
release: owf-ms-prd-debitcardapplication-rumsk1
template:
metadata:
labels:
app: owf-ms-prd-debitcardapplication
fullname: owf-ms-prd-debitcardapplication-rumsk1
chart: base-1.15.3
release: owf-ms-prd-debitcardapplication-rumsk1
heritage: Helm
isys: "ownfin-stress"
psvc: "owf-ms-prd-debitcardapplication"
version: "stress"
workload.topology.app.farm/zone: "rumsk1"
annotations:
checksum/configMapsEnv: "4b5b1e1ad3903c8d70121843a80ac2f2abe5b957fe14d6fd86447d70458d92ad"
checksum/secretEnv: "99f21e84d2109bcb5b2bbda14b604556f4810215f47838ddc7545269fbc2db5c"
- ci/commithash: "59c7e02a"
+ ci/commithash: "885b636d"
inject.istio.io/templates: "sidecar,custom"
prometheus.io/path: "/metrics"
prometheus.io/port: "8080"
prometheus.io/scrape: "true"
sidecar.istio.io/proxyCPU: "500m"
sidecar.istio.io/proxyCPULimit: "600m"
sidecar.istio.io/proxyMemory: "512Mi"
sidecar.istio.io/proxyMemoryLimit: "768Mi"
sidecar.istio.io/userVolume: "[{\"name\": \"wasmfilters-dir\",\"configMap\": {\"name\": \"ownfin-stress-envoy-filters\"}}]"
sidecar.istio.io/userVolumeMount: "[{\"mountPath\":\"/var/local/lib/wasm-filters\",\"name\":\"wasmfilters-dir\"}]"
spec:
automountServiceAccountToken: false
serviceAccountName: default
nodeSelector:
node-role.kubernetes.io/wload: ""
workload.topology.app.farm/zone: rumsk1
priorityClassName: rumsk1
containers:
- name: app
image: registry.rshbdev.ru/rshbintech/crft/ckof/ownfin/backend/prd/owf-ms-prd-debitcardapplication:stress
imagePullPolicy: Always
envFrom:
- configMapRef:
name: owf-ms-prd-debitcardapplication-rumsk1-app-cm-env
- secretRef:
name: owf-ms-prd-debitcardapplication-rumsk1-app-secret-env
resources:
limits:
cpu: 2000m
memory: 2200Mi
requests:
cpu: 400m
memory: 1Gi
ports:
livenessProbe:
failureThreshold: 5
httpGet:
path: /health/liveness
port: 8080
initialDelaySeconds: 240
periodSeconds: 5
successThreshold: 1
timeoutSeconds: 1
readinessProbe:
failureThreshold: 3
httpGet:
path: /health/readiness
port: 8080
initialDelaySeconds: 60
periodSeconds: 4
successThreshold: 1
timeoutSeconds: 1
volumeMounts:
- name: app-vault-secrets
mountPath: /secrets
readOnly: true
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
privileged: false
procMount: Default
readOnlyRootFilesystem: false
runAsGroup: 1001
runAsNonRoot: true
runAsUser: 1001
securityContext:
fsGroup: 1001
fsGroupChangePolicy: OnRootMismatch
runAsGroup: 1001
runAsNonRoot: true
runAsUser: 1001
tolerations:
- effect: NoSchedule
key: node-role.kubernetes.io/wload
operator: Exists
- effect: NoSchedule
key: workload.topology.app.farm/zone
operator: Exists
hostNetwork: false
volumes:
- name: app-vault-secrets
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: owf-ms-prd-debitcardapplication-rumsk1-app-secret-provider
Comparing release=owf-ms-prd-debitcardapplication-rumsk2, chart=rshb-charts/base
isys-ownfin-stress, owf-ms-prd-debitcardapplication-rumsk2, Deployment (apps) has changed:
# Source: base/templates/workload.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: owf-ms-prd-debitcardapplication-rumsk2
labels:
app: owf-ms-prd-debitcardapplication
fullname: owf-ms-prd-debitcardapplication-rumsk2
chart: base-1.15.3
release: owf-ms-prd-debitcardapplication-rumsk2
heritage: Helm
isys: "ownfin-stress"
psvc: "owf-ms-prd-debitcardapplication"
version: "stress"
workload.topology.app.farm/zone: "rumsk2"
annotations:
- gitlabCommit: 59c7e02a8101a36cf130992e6ab2b2ce822eb66d
- gitlabPipeline: https://gitlab.rshbdev.ru/rshbintech/crft/ckof/ownfin/backend/prd/owf-ms-prd-debitcardapplication/-/pipelines/3471908
- platform.ckpr.integrations.rshbintech.ru/gitlab-commit-sha: 59c7e02a
- platform.ckpr.integrations.rshbintech.ru/gitlab-pipeline-id: "3471908"
+ gitlabCommit: 885b636dbc23ec1c8dc3f852fd977f19cbb39ab4
+ gitlabPipeline: https://gitlab.rshbdev.ru/rshbintech/crft/ckof/ownfin/backend/prd/owf-ms-prd-debitcardapplication/-/pipelines/3472181
+ platform.ckpr.integrations.rshbintech.ru/gitlab-commit-sha: 885b636d
+ platform.ckpr.integrations.rshbintech.ru/gitlab-pipeline-id: "3472181"
spec:
strategy:
type: RollingUpdate
replicas: 1
selector:
matchLabels:
app: owf-ms-prd-debitcardapplication
fullname: owf-ms-prd-debitcardapplication-rumsk2
release: owf-ms-prd-debitcardapplication-rumsk2
template:
metadata:
labels:
app: owf-ms-prd-debitcardapplication
fullname: owf-ms-prd-debitcardapplication-rumsk2
chart: base-1.15.3
release: owf-ms-prd-debitcardapplication-rumsk2
heritage: Helm
isys: "ownfin-stress"
psvc: "owf-ms-prd-debitcardapplication"
version: "stress"
workload.topology.app.farm/zone: "rumsk2"
annotations:
checksum/configMapsEnv: "8e3df1d0ce1d880a555e53465ea0c0e97e730734baa5a399b779e7baf8ec368e"
checksum/secretEnv: "36641515e91db76be31d8b14287e85822084fc006de31e65ec5e0219afff5b8c"
- ci/commithash: "59c7e02a"
+ ci/commithash: "885b636d"
inject.istio.io/templates: "sidecar,custom"
prometheus.io/path: "/metrics"
prometheus.io/port: "8080"
prometheus.io/scrape: "true"
sidecar.istio.io/proxyCPU: "500m"
sidecar.istio.io/proxyCPULimit: "600m"
sidecar.istio.io/proxyMemory: "512Mi"
sidecar.istio.io/proxyMemoryLimit: "768Mi"
sidecar.istio.io/userVolume: "[{\"name\": \"wasmfilters-dir\",\"configMap\": {\"name\": \"ownfin-stress-envoy-filters\"}}]"
sidecar.istio.io/userVolumeMount: "[{\"mountPath\":\"/var/local/lib/wasm-filters\",\"name\":\"wasmfilters-dir\"}]"
spec:
automountServiceAccountToken: false
serviceAccountName: default
nodeSelector:
node-role.kubernetes.io/wload: ""
workload.topology.app.farm/zone: rumsk2
priorityClassName: rumsk2
containers:
- name: app
image: registry.rshbdev.ru/rshbintech/crft/ckof/ownfin/backend/prd/owf-ms-prd-debitcardapplication:stress
imagePullPolicy: Always
envFrom:
- configMapRef:
name: owf-ms-prd-debitcardapplication-rumsk2-app-cm-env
- secretRef:
name: owf-ms-prd-debitcardapplication-rumsk2-app-secret-env
resources:
limits:
cpu: 2000m
memory: 2200Mi
requests:
cpu: 400m
memory: 1Gi
ports:
livenessProbe:
failureThreshold: 5
httpGet:
path: /health/liveness
port: 8080
initialDelaySeconds: 240
periodSeconds: 5
successThreshold: 1
timeoutSeconds: 1
readinessProbe:
failureThreshold: 3
httpGet:
path: /health/readiness
port: 8080
initialDelaySeconds: 60
periodSeconds: 4
successThreshold: 1
timeoutSeconds: 1
volumeMounts:
- name: app-vault-secrets
mountPath: /secrets
readOnly: true
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
privileged: false
procMount: Default
readOnlyRootFilesystem: false
runAsGroup: 1001
runAsNonRoot: true
runAsUser: 1001
securityContext:
fsGroup: 1001
fsGroupChangePolicy: OnRootMismatch
runAsGroup: 1001
runAsNonRoot: true
runAsUser: 1001
tolerations:
- effect: NoSchedule
key: node-role.kubernetes.io/wload
operator: Exists
- effect: NoSchedule
key: workload.topology.app.farm/zone
operator: Exists
hostNetwork: false
volumes:
- name: app-vault-secrets
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: owf-ms-prd-debitcardapplication-rumsk2-app-secret-provider
Comparing release=owf-ms-prd-debitcardapplication, chart=rshb-charts/raw
Listing releases matching ^pjob-owf-ms-prd-debitcardapplication$
Listing releases matching ^exsvc-owf-ms-prd-debitcardapplication$
Listing releases matching ^data-owf-ms-prd-debitcardapplication$
Listing releases matching ^assets-config-owf-ms-prd-debitcardapplication$
Listing releases matching ^owf-ms-prd-debitcardapplication-raw$
Upgrading release=owf-ms-prd-debitcardapplication-rumsk1, chart=rshb-charts/base
Upgrading release=owf-ms-prd-debitcardapplication-rumsk2, chart=rshb-charts/base
FAILED RELEASES:
NAME
owf-ms-prd-debitcardapplication-rumsk1
owf-ms-prd-debitcardapplication-rumsk2
in deploy/helmfile.yaml: 2 errors:
err 0: failed processing release owf-ms-prd-debitcardapplication-rumsk1: command "/usr/local/link/helm" exited with non-zero status:
PATH:
/usr/local/link/helm
ARGS:
0: helm (4 bytes)
1: --kube-context (14 bytes)
2: rcsdstbl (8 bytes)
3: upgrade (7 bytes)
4: --install (9 bytes)
5: --reset-values (14 bytes)
6: owf-ms-prd-debitcardapplication-rumsk1 (38 bytes)
7: rshb-charts/base (16 bytes)
8: --version (9 bytes)
9: 1.15.3 (6 bytes)
10: --wait (6 bytes)
11: --timeout (9 bytes)
12: 600s (4 bytes)
13: --atomic (8 bytes)
14: --kube-context (14 bytes)
15: rcsdstbl (8 bytes)
16: --namespace (11 bytes)
17: isys-ownfin-stress (18 bytes)
18: --values (8 bytes)
19: /tmp/helmfile2591309174/isys-ownfin-stress-owf-ms-prd-debitcardapplication-rumsk1-values-69cfb97789 (99 bytes)
20: --values (8 bytes)
21: /tmp/helmfile119359715/isys-ownfin-stress-owf-ms-prd-debitcardapplication-rumsk1-values-6549b99544 (98 bytes)
22: --values (8 bytes)
23: /tmp/helmfile669246869/isys-ownfin-stress-owf-ms-prd-debitcardapplication-rumsk1-values-6fcf5cdbb (97 bytes)
24: --set (5 bytes)
25: additionalWorkloadAnnotations.gitlabPipeline=https://gitlab.rshbdev.ru/rshbintech/crft/ckof/ownfin/backend/prd/owf-ms-prd-debitcardapplication/-/pipelines/3472181 (162 bytes)
26: --set (5 bytes)
27: additionalWorkloadAnnotations.gitlabCommit=885b636dbc23ec1c8dc3f852fd977f19cbb39ab4 (83 bytes)
28: --history-max (13 bytes)
29: 10 (2 bytes)
ERROR:
exit status 1
EXIT STATUS
1
STDERR:
Error: UPGRADE FAILED: another operation (install/upgrade/rollback) is in progress
COMBINED OUTPUT:
Error: UPGRADE FAILED: another operation (install/upgrade/rollback) is in progress
err 1: failed processing release owf-ms-prd-debitcardapplication-rumsk2: command "/usr/local/link/helm" exited with non-zero status:
PATH:
/usr/local/link/helm
ARGS:
0: helm (4 bytes)
1: --kube-context (14 bytes)
2: rcsdstbl (8 bytes)
3: upgrade (7 bytes)
4: --install (9 bytes)
5: --reset-values (14 bytes)
6: owf-ms-prd-debitcardapplication-rumsk2 (38 bytes)
7: rshb-charts/base (16 bytes)
8: --version (9 bytes)
9: 1.15.3 (6 bytes)
10: --wait (6 bytes)
11: --timeout (9 bytes)
12: 600s (4 bytes)
13: --atomic (8 bytes)
14: --kube-context (14 bytes)
15: rcsdstbl (8 bytes)
16: --namespace (11 bytes)
17: isys-ownfin-stress (18 bytes)
18: --values (8 bytes)
19: /tmp/helmfile3010291570/isys-ownfin-stress-owf-ms-prd-debitcardapplication-rumsk2-values-59fcdcbbf4 (99 bytes)
20: --values (8 bytes)
21: /tmp/helmfile3841461191/isys-ownfin-stress-owf-ms-prd-debitcardapplication-rumsk2-values-fddbf9679 (98 bytes)
22: --values (8 bytes)
23: /tmp/helmfile3846892374/isys-ownfin-stress-owf-ms-prd-debitcardapplication-rumsk2-values-7c689cf9d9 (99 bytes)
24: --set (5 bytes)
25: additionalWorkloadAnnotations.gitlabPipeline=https://gitlab.rshbdev.ru/rshbintech/crft/ckof/ownfin/backend/prd/owf-ms-prd-debitcardapplication/-/pipelines/3472181 (162 bytes)
26: --set (5 bytes)
27: additionalWorkloadAnnotations.gitlabCommit=885b636dbc23ec1c8dc3f852fd977f19cbb39ab4 (83 bytes)
28: --history-max (13 bytes)
29: 10 (2 bytes)
ERROR:
exit status 1
EXIT STATUS
1
STDERR:
Error: UPGRADE FAILED: another operation (install/upgrade/rollback) is in progress
COMBINED OUTPUT:
Error: UPGRADE FAILED: another operation (install/upgrade/rollback) is in progress
Uploading artifacts for failed job
00:01
Uploading artifacts...
deploy.env: found 1 matching artifact files and directories
Uploading artifacts as "dotenv" to coordinator... 201 Created id=24180536 responseStatus=201 Created token=64_UY6JT
Cleaning up project directory and file based variables
00:00
ERROR: Job failed: command terminated with exit code 1