Как мы внедряли отказоустойчивый GitLab Cluster с использованием Ansible и бесшовными обновлениями

картинку вставлю чуть позже

---     haproxy:       children:         haproxy_dev:           hosts:             ybd-bln-hpr01:               ansible_host: 10.0.220.189               ansible_port: 222     gitlab_cluster:       children:         gitlab_rails:           hosts:             ybd-git-rai01:               ansible_host: 10.0.220.180             ybd-git-rai02:               ansible_host: 10.0.220.40         gitlab_sidekiq:           hosts:             ybd-git-sid01:               ansible_host: 10.0.220.181             ybd-git-sid02:               ansible_host: 10.0.220.86         gitlab_praefect:           hosts:             ybd-git-prf01:               ansible_host: 10.0.220.182               gitlab_omnibus_praefect_override:                 auto_migrate: true             ybd-git-prf02:               ansible_host: 10.0.220.183             ybd-git-prf03:               ansible_host: 10.0.220.184         gitlab_gitaly:           hosts:             ybd-git-gtl01:               ansible_host: 10.0.220.185             ybd-git-gtl02:               ansible_host: 10.0.220.186             ybd-git-gtl03:               ansible_host: 10.0.220.187         gitlab_postgresql:           hosts:             ybd-git-psg01:               ansible_host: 10.0.220.188         gitlab_redis:           hosts:             ybd-git-red01:               ansible_host: 10.0.220.190               master: true 

--- haproxy_packet: "{{ haproxy_packet_name }}" haproxy_packet_name: "haproxy" haproxy_defaults_log: 127.0.0.1:5140 len 65535 local0 haproxy_global_nbthread: 2 haproxy_global_raw_options:   - stats socket /var/lib/haproxy/stats mode 660 level admin expose-fd listeners   - stats timeout 30s  haproxy_logformat_tcp: &haproxy_logformat_tcp   logformat: >-     '{"appname":"haproxy","@timestamp":"%Ts","pid":%pid,"haproxy_frontend_type":"tcp","haproxy_process_concurrent_connections":%ac,     "haproxy_frontend_concurrent_connections":%fc,"haproxy_backend_concurrent_connections":%bc,     "haproxy_server_concurrent_connections":%sc,"haproxy_backend_queue":%bq,"haproxy_server_queue":%sq,     "haproxy_queue_wait_time":%Tw,"haproxy_server_wait_time":%Tc,"response_time":%Td,"session_duration":%Tt,     "request_termination_state":"%tsc","haproxy_server_connection_retries":%rc,"remote_addr":"%ci","remote_port":%cp,     "frontend_addr":"%fi","frontend_port":%fp,"frontend_ssl_version":"%sslv","frontend_ssl_ciphers":"%sslc",     "haproxy_frontend_name":"%f","haproxy_backend_name":"%b","haproxy_server_name":"%s","response_size":%B,     "request_size":%U}'  haproxy_logformat_http: &haproxy_logformat_http   logformat: >-     '{"appname":"haproxy","@timestamp":"%Ts","pid":%pid,"haproxy_frontend_type":"http","haproxy_process_concurrent_connections":%ac,     "haproxy_frontend_concurrent_connections":%fc,"haproxy_backend_concurrent_connections":%bc,     "haproxy_server_concurrent_connections":%sc,"haproxy_backend_queue":%bq,"haproxy_server_queue":%sq,"haproxy_client_request_send_time":%Tq,     "haproxy_queue_wait_time":%Tw,"haproxy_server_wait_time":%Tc,"haproxy_server_response_send_time":%Tr,     "response_time":%Td,"session_duration":%Tt,"request_termination_state":"%tsc","haproxy_server_connection_retries":%rc,     "remote_addr":"%ci","remote_port":%cp,"frontend_addr":"%fi","frontend_port":%fp,"frontend_ssl_version":"%sslv",     "frontend_ssl_ciphers":"%sslc","request_method":"%HM","request_uri":"%[capture.req.uri,json(utf8s)]",     "request_http_version":"%HV","host":"%[capture.req.hdr(0)]","referer":"%[capture.req.hdr(1),json(utf8s)]",     "haproxy_frontend_name":"%f","haproxy_backend_name":"%b","haproxy_server_name":"%s","status":%ST,"response_size":%B,     "request_size":%U}' haproxy_defaults_option:   - forwardfor   - dontlognull  haproxy_default_raw_options:   - backlog 10000   - retries 3  haproxy_defaults_timeout:   - type: connect     timeout: 5000   - type: client     timeout: 50000   - type: server     timeout: 101000   - type: http-request     timeout: 10000   - type: queue     timeout: 30000  haproxy_frontend:   - name: http     description: Front-end for all HTTP traffic     bind:       - listen: "0.0.0.0:80"       - listen: "0.0.0.0:443 ssl crt /opt/testlab.com.pem crt /opt/testlab.org.pem crt /opt/pages.testlab.org.pem alpn h2,http/1.1"     mode: http     <<: *haproxy_logformat_http     raw_options:       - redirect scheme https unless { ssl_fc }       - http-request redirect scheme https code 301 unless { ssl_fc }     acl:       - string: host_gitlab-backup hdr(host) -i git-backup.testlab.org       - string: host_pages-backup hdr_end(host) -i pages-backup.testlab.org      http_request:       - action: capture         param: req.hdr(Host) len 1000       - action: capture         param: req.hdr(Referer) len 1000     use_backend:       - gitlab-backup if host_gitlab-backup       - pages-backup if host_pages-backup    - name: exporter     bind:       - listen: 0.0.0.0:8404     <<: *haproxy_logformat_http     option:       - http-use-htx     http_request:       - action: use-service         param: prometheus-exporter         cond: "if { path /metrics }"     raw_options:       - stats enable       - stats uri /stats       - stats refresh 10s   - name: gitlab-backup-ssh     description: gitlab-backup-ssh     bind:       - listen: "0.0.0.0:22"     mode: tcp     option:       - tcplog       - clitcpka     <<: *haproxy_logformat_tcp     default_backend: gitlab-backup-ssh   - name: praefect-back     description: praefect-back     bind:       - listen: "0.0.0.0:2305"     option:       - tcplog       - clitcpka     mode: tcp     <<: *haproxy_logformat_tcp     default_backend: git-prafect-back  haproxy_backend:   - name: git-prafect-back     description: git-prafect-back     mode: tcp     balance: roundrobin     options:       - srvtcpka       - tcp-check     server:       - name: ybd-git-prf01         listen: ybd-git-prf01:2305 check       - name: ybd-git-prf02         listen: ybd-git-prf02:2305 check       - name: ybd-git-prf03         listen: ybd-git-prf03:2305 check   - name: gitlab-backup     description: gitlab-backup http     mode: http     balance: roundrobin     http_check: expect string '{"status":"ok"}'     option:       - httpchk GET /-/liveness       - tcp-check       - srvtcpka     server:       - name: ybd-git-rai01         listen: ybd-git-rai01:80 check       - name: ybd-git-rai02         listen: ybd-git-rai02:80 check   - name: pages-backup     description: pages-backup http     mode: http     balance: roundrobin     http_check: expect string '{"status":"ok"}'     option:       - httpchk GET /-/liveness       - tcp-check       - srvtcpka     server:       - name: ybd-git-rai01         listen: ybd-git-rai01:8081 check port 80       - name: ybd-git-rai02         listen: ybd-git-rai02:8081 check port 80   - name: gitlab-backup-ssh     description: gitlab-backup-ssh     mode: tcp     balance: roundrobin     http_check: expect string '{"status":"ok"}'     option:       - httpchk GET /-/liveness       - tcp-check       - srvtcpka     server:       - name: ybd-git-rai01         listen: ybd-git-rai01:22 check port 80       - name: ybd-git-rai02         listen: ybd-git-rai02:22 check port 80  vector_version: "0.31.0" vector_configs:   - name: haproxy     config:       sources:         local_haproxy:           address: 127.0.0.1:5140           max_length: 102400           mode: udp           type: syslog       sinks:         sink_kafka:           type: kafka           inputs:             - local_haproxy           bootstrap_servers: "{{ vector_kafka_url }}"           topic: "infra-haproxy"           compression: none           encoding:             codec: text           tls:             enabled: true 

--- gitlab_omnibus_packet_version: 17.7.3-ce.0 gitlab_omnibus_restic_password: "{{ vault_gitlab_omnibus_restic_password }}" gitlab_omnibus_haproxy_hosts:   - ybd-bln-hpr01  gitlab_omnibus_gitaly:   data_hosts:     - ybd-git-gtl01:8075     - ybd-git-gtl02:8075     - ybd-git-gtl03:8075  gitlab_omnibus_shell:   secret_token: "{{ vault_gitlab_omnibus_shell_secret_token }}"  gitlab_omnibus_pages_external_url: https://pages-backup.testlab.org  gitlab_omnibus_pages:   gitlab_server: "https://git-backup.testlab.org"   gitlab_secret: "{{ vault_gitlab_omnibus_pages_gitlab_secret }}"   api_secret_key: "{{ vault_gitlab_omnibus_pages_api_secret_key }}"  gitlab_omnibus_praefect:   external_address: tcp://git-backup.testlab.org:2305   praefect_external_token: "{{ vault_gitlab_omnibus_praefect_praefect_external_token }}"   praefect_internal_token: "{{ vault_gitlab_omnibus_praefect_praefect_internal_token }}"   database_host: ybd-git-psg01   database_user: gitlab_praefect_backup   database_password: "{{ vault_gitlab_omnibus_praefect_database_password }}"   database_dbname: gitlab_praefect_backup  gitlab_omnibus_rails:   gitlab_email_from: noreply@testlab.com   gitlab_email_reply_to: noreply@testlab.com   smtp_authentication: login   smtp_domain: testlab.com   smtp_enable: false   gitlab_email_enabled: false   gitlab_shell_ssh_port: 22   smtp_address: lalala.testlab.com   smtp_port: 587   smtp_password: "{{ vault_gitlab_omnibus_rails_smtp_password }}"   smtp_openssl_verify_mode: peer   smtp_enable_starttls_auto: true   smtp_user_name: noreply@testlab.com   ldap_servers_main_password: "{{ vault_gitlab_omnibus_rails_ldap_servers_main_password }}"   ldap_servers_main_label: LDAP   ldap_servers_main_host: 'testlab.com'   ldap_servers_main_port: 636   ldap_servers_main_uid: 'sAMAccountName'   ldap_servers_main_encryption: 'simple_tls'   ldap_servers_main_bind_dn: 'CN=gitlab ldap,OU=Service Accounts,OU=Organization,DC=testlab,DC=com'   ldap_servers_main_base: 'OU=Organization,DC=testlab,DC=com'   initial_root_password: "{{ vault_gitlab_omnibus_rails_initial_root_password }}"   db_host: ybd-git-psg01   db_port: 5432   db_database: gitlab_rails_backup   db_username: gitlab_rails_backup   db_password: "{{ vault_gitlab_omnibus_rails_db_password }}"   external_url: https://git-backup.testlab.org   internal_api_url: https://git-backup.testlab.org   secret_key_base: "{{ vault_gitlab_omnibus_rails_secret_key_base }}"   otp_key_base: "{{ vault_gitlab_omnibus_rails_otp_key_base }}"   db_key_base: "{{ vault_gitlab_omnibus_rails_db_key_base }}"   encrypted_settings_key_base: "{{ vault_gitlab_omnibus_rails_encrypted_settings_key_base }}"   openid_connect_signing_key: "{{ vault_gitlab_omnibus_rails_openid_connect_signing_key }}"   ci_jwt_signing_key: "{{ vault_gitlab_omnibus_rails_ci_jwt_signing_key }}"   packages_enable: false   object_store:     enabled: true     proxy_download: true     connection:       aws_access_key_id: "{{ vault_gitlab_omnibus_rails_object_store_connection_aws_access_key_id }}"       aws_secret_access_key: "{{ vault_gitlab_omnibus_rails_object_store_connection_aws_secret_access_key }}"       endpoint: https://storage.yandexcloud.net       host: storage.yandexcloud.net     objects:       artifacts_bucket: gitlab-prod-artifacts       external_diffs_bucket: gitlab-prod-external-diffs       lfs_bucket: gitlab-prod-lfs       uploads_bucket: gitlab-prod-uploads       packages_bucket: crutch       dependency_proxy_bucket: crutch2       terraform_state_bucket: gitlab-prod-terraform-state       pages_bucket: gitlab-prod-pages       ci_secure_files_bucket: gitlab-prod-ci-secure-files       backup_bucket: backup-gitlab   redis:     sentinels:       - host: ybd-git-red01         port: 26379     password: "{{ vault_gitlab_omnibus_rails_redis_password }}"     master_name: gitlab_redis  postgresql_databases:   - name: "{{ gitlab_omnibus_rails.db_database }}"     owner: "{{ gitlab_omnibus_rails.db_username }}"   - name: "{{ gitlab_omnibus_praefect.database_dbname }}"     owner: "{{ gitlab_omnibus_praefect.database_user }}"  postgresql_users:   - name: "{{ gitlab_omnibus_rails.db_username }}"     role_attr_flags: superuser     password: "{{ gitlab_omnibus_rails.db_password }}"     db: "{{ gitlab_omnibus_rails.db_database }}"   - name: "{{ gitlab_omnibus_praefect.database_user }}"     role_attr_flags: createdb     password: "{{ gitlab_omnibus_praefect.database_password }}"     db: "{{ gitlab_omnibus_praefect.database_dbname }}"  postgresql_hba_entries:   - type: local     database: all     user: postgres     auth_method: peer   - type: local     database: all     user: all     auth_method: peer   - type: host     database: all     user: all     address: '127.0.0.1/32'     auth_method: "{{ postgresql_auth_method }}"   - type: host     database: all     user: all     address: '10.0.0.0/8'     auth_method: "{{ postgresql_auth_method }}"   - type: host     database: all     user: all     address: '172.16.0.0/12'     auth_method: "{{ postgresql_auth_method }}"   - type: host     database: all     user: all     address: '::1/128'     auth_method: "{{ postgresql_auth_method }}"  postgresql_global_config_options:   - option: listen_addresses     value: '*'   - option: max_connections     value: 500   - option: shared_buffers     value: 1GB   - option: effective_cache_size     value: 3GB   - option: maintenance_work_mem     value: 256MB   - option: checkpoint_completion_target     value: 0.9   - option: wal_buffers     value: 16MB   - option: default_statistics_target     value: 100   - option: random_page_cost     value: 1.1   - option: effective_io_concurrency     value: 200   - option: work_mem     value: 2621kB   - option: huge_pages     value: 'off'   - option: min_wal_size     value: 1GB   - option: max_wal_size     value: 4GB  gitlab_omnibius_ssh_keys:   - path: /etc/ssh/ssh_host_dsa_key     content: "{{ vault_gitlab_omnibus_gitlab_ssh_keys_ssh_host_dsa_key }}"     mode: '0600'   - path: /etc/ssh/ssh_host_dsa_key.pub     mode: '0644'     content: "{{ vault_gitlab_omnibus_gitlab_ssh_keys_ssh_host_dsa_key_pub }}"   - path: /etc/ssh/ssh_host_ecdsa_key     mode: '0600'     content: "{{ vault_gitlab_omnibus_gitlab_ssh_keys_ssh_host_ecdsa_key }}"   - path: /etc/ssh/ssh_host_ecdsa_key.pub     mode: '0644'     content: "{{ vault_gitlab_omnibus_gitlab_ssh_keys_ssh_host_ecdsa_key_pub }}"   - path: /etc/ssh/ssh_host_ed25519_key     mode: '0600'     content: "{{ vault_gitlab_omnibus_gitlab_ssh_keys_ssh_host_ed25519_key }}"   - path: /etc/ssh/ssh_host_ed25519_key.pub     mode: '0644'     content: "{{ vault_gitlab_omnibus_gitlab_ssh_keys_ssh_host_ed25519_key_pub }}"   - path: /etc/ssh/ssh_host_rsa_key     mode: '0600'     content: "{{ vault_gitlab_omnibus_gitlab_ssh_keys_ssh_host_rsa_key }}"   - path: /etc/ssh/ssh_host_rsa_key.pub     mode: '0644'     content: "{{ vault_gitlab_omnibus_gitlab_ssh_keys_ssh_host_rsa_key_pub }}"  redis_server_password: "{{ vault_gitlab_omnibus_rails_redis_password }}" redis_sentinel_password: "{{ vault_gitlab_omnibus_rails_redis_password }}" redis_node_roles:   - master   - sentinel redis_mastername: gitlab_redis redis_masteruser: gitlab_redis redis_masterauth: "{{ vault_gitlab_omnibus_rails_redis_password }}" redis_sentinel_monitors:   - name: gitlab_redis     host: ybd-git-red01     port: 6379     quorum: 2     auth_pass: "{{ vault_gitlab_omnibus_rails_redis_password }}"     down_after_milliseconds: 1000     parallel_syncs: 1     failover_timeout: 1000     notification_script: false     rename_commands: []  redis_sentinel_extra_config:   sentinel:     resolve_hostnames: "yes"     announce_hostnames: "yes" 

--- - name: pull gitlab secrets from vault   hosts: gitlab_cluster   tasks:     - name: Read the vault       community.hashi_vault.vault_kv2_get:         url: 'https://vault.testlab.org'         path: '{{ environments }}/gitlab_cluster'         auth_method: 'approle'         role_id: "{{ vault_role_id }}"         secret_id: "{{ vault_secret_id }}"         engine_mount_point: 'kv-devops'       register: secrets     - name: set env       ansible.builtin.set_fact:         "{{ item.key }}": "{{ item.value }}"       loop: "{{ secrets.secret | dict2items }}"       no_log: true   tags:     - gitlab     - postgresql     - redis-cluster  - hosts: gitlab_postgresql   become: true   roles:     - postgresql-local   tags:     - postgresql  - hosts: gitlab_redis   become: true   roles:     - redis-cluster   tags:     - redis-cluster  - hosts:     - gitlab_cluster:!gitlab_postgresql:!gitlab_redis   become: true   roles:     - gitlab-omnibus   tags:     - gitlab 

--- - name: pull gitlab secrets from vault   hosts: gitlab_cluster   tasks:     - name: Read the vault       community.hashi_vault.vault_kv2_get:         url: 'https://vault.testlab.org'         path: '{{ environments }}/gitlab_cluster'         auth_method: 'approle'         role_id: "{{ vault_role_id }}"         secret_id: "{{ vault_secret_id }}"         engine_mount_point: 'kv-devops'       register: secrets     - name: set env       ansible.builtin.set_fact:         "{{ item.key }}": "{{ item.value }}"       loop: "{{ secrets.secret | dict2items }}"       no_log: true  - name: upgrade_gitaly   hosts: gitlab_gitaly   become: true   roles:     - gitlab-omnibus   vars:     gitlab_omnibus_upgrade_gitaly: true   serial: 1  - name: upgrade_praefect   hosts: gitlab_praefect   become: true   roles:     - gitlab-omnibus   vars:     gitlab_omnibus_upgrade_praefect: true   serial: 1  - name: upgrade_rails[0]   hosts: gitlab_rails[0]   become: true   roles:     - gitlab-omnibus   vars:     gitlab_omnibus_upgrade_rails: true   serial: 1  - name: db_migrate over gitlab_rails[0]   hosts: gitlab_rails[0]   become: true   roles:     - gitlab-omnibus   vars:     gitlab_omnibus_db_migrate: true   serial: 1   run_once: true  - name: upgrade_rails[1]   hosts: gitlab_rails[1]   become: true   roles:     - gitlab-omnibus   vars:     gitlab_omnibus_upgrade_rails: true   serial: 1  - name: upgrade_sidekiq   hosts: gitlab_sidekiq   become: true   roles:     - gitlab-omnibus   vars:     gitlab_omnibus_upgrade_sidekiq: true   serial: 1