path: root/deployment/playbooks/gather_logs.yaml

# Expected vars:
#
# - 'config_filepath' - required. It is expected to be provided and must
# store filepath for the config file used by automated test caes.
# Current playbook will take all the nodes info from it.
#
# - 'output_artifacts_dir' - optional. It is directory where should be saved
# generated/gathered files.
#
# Command to run this playbook:
#
# $ tox -e ocp3.11 -- ansible-playbook -i 127.0.0.1, \
#     playbooks/gather_logs.yaml \
#     -e config_filepath=/path/to/the/cns-automation-config.yaml \
#     -e output_artifacts_dir=../cluster_logs/

---
- hosts: localhost
  connection: local
  gather_facts: no
  tasks:
  - name: Process config file and find all the nodes of an OpenShift cluster
    command: "python -c \"import yaml ;
        config = yaml.load(open('{{ config_filepath }}', 'r'));
        print(' '.join(list(config['ocp_servers']['master'].keys())));
        print(' '.join(list(config['ocp_servers']['nodes'].keys())));
        print(' '.join(list(config['gluster_servers'].keys())));
        print(config.get('openshift', config.get('cns',
            {}))['heketi_config']['heketi_server_url']);
        print(config.get('openshift', config.get('cns',
            {}))['heketi_config']['heketi_cli_user']);
        print(config.get('openshift', config.get('cns',
            {}))['heketi_config']['heketi_cli_key'])\""
    register: config_data
  - debug:
      msg: "{{ config_data }}"
  - name: Save config data to the host vars
    set_fact:
      master_nodes: "{{ config_data.stdout_lines[0].split(' ') }}"
      compute_nodes: "{{ config_data.stdout_lines[1].split(' ') }}"
      gluster_nodes: "{{ config_data.stdout_lines[2].split(' ') }}"
      heketi_server_url: "{{ config_data.stdout_lines[3] }}"
      heketi_cli_user: "{{ config_data.stdout_lines[4] }}"
      heketi_cli_key: "{{ config_data.stdout_lines[5] }}"
  - name: Print list of master nodes IP addresses
    debug:
      msg: "{{ master_nodes }}"
  - name: Print list of compute nodes IP addresses
    debug:
      msg: "{{ compute_nodes }}"
  - name: Print list of gluster nodes IP addresses
    debug:
      msg: "{{ gluster_nodes }}"
  - name: Add gathered master IP addresses to the Ansible host list
    add_host:
      hostname: "{{ item }}"
      ansible_host: "{{ item }}"
      groups: logs_ocp_nodes, logs_single_master_node, logs_master_nodes
    with_items: "{{ master_nodes }}"
  - name: Add gathered compute IP addresses to the Ansible host list
    add_host:
      hostname: "{{ item }}"
      ansible_host: "{{ item }}"
      groups: logs_ocp_nodes, logs_compute_nodes
    with_items: "{{ compute_nodes }}"
  - name: Add gathered gluster IP addresses to the Ansible host list
    add_host:
      hostname: "{{ item }}"
      ansible_host: "{{ item }}"
      groups: logs_ocp_nodes, logs_gluster_nodes
    with_items: "{{ gluster_nodes }}"

- hosts: all
  gather_facts: no
  tasks:
  - set_fact:
      output_artifacts_dir: "{{
          (output_artifacts_dir | default('../cluster_logs/')) }}"
  - set_fact:
      output_artifacts_dir: "{{ output_artifacts_dir
      }}{% if output_artifacts_dir[-1] != '/' %}{{ '/' }}{% endif %}"
  - name: Get hostname of the current host
    shell: "hostname"
    register: hostname
  - name: Save hostname to the var
    set_fact:
      current_hostname: "{{ hostname.stdout_lines[0].strip() }}"
      separator_line: "{{ '=' * 79 }}"
  - name: Create output artifacts directory if absent
    delegate_to: localhost
    file:
      path: "{{ output_artifacts_dir }}"
      state: directory
    run_once: yes
  - name: Install 'rsync' package which is required by 'synchronize' module
    yum:
      name: rsync
      state: installed
    ignore_errors: yes

# All nodes SOS reports
- hosts: logs_ocp_nodes
  gather_facts: no
  tasks:
  - name: Install 'sos' package if absent
    package:
      name: sos
      state: present
  - name: Run sosreport command
    shell: "sosreport --batch --verbose --tmp-dir . --label customuniquelabel"
    register: sosreport_output
  - name: Print the output of the sosreport command
    debug:
      msg: "{{ sosreport_output }}"
  - name: Get name of the generated sos-file
    shell: 'echo -e "{{ sosreport_output.stdout }}" | grep customuniquelabel'
    register: sos_filepath
  - name: Copy generated files to the localhost
    fetch:
      src: "{{ sos_filepath.stdout_lines[0].strip() }}"
      dest: "{{ output_artifacts_dir }}sosreports/"
      flat: yes
      fail_on_missing: yes

- hosts: logs_single_master_node
  gather_facts: no
  vars:
    master_package_list:
    - docker
    - heketi
    master_service_list:
    - docker
    - multipathd
    heketi_pod_package_list:
    - gluster
    - heketi
    gluster_pod_package_list:
    - gluster
    - heketi
    - targetcli
    - gluster-block
    - tcmu-runner
    - python-configshell
    - python-rtslib
    gluster_pod_service_list:
    - glusterd
    - heketi
    - gluster-blockd
    - gluster-block-target
    - tcmu-runner
    - rpcbind
  tasks:

  # Master node info
  - name: Get distro version
    shell: "uname -a"
    register: master_linux_kernel_version
  - name: Get Red Hat release info
    shell: "cat /etc/redhat-release"
    register: master_rh_release
  - name: Create grep filter with all the packages we are interested in
    set_fact:
      package_filter: "{{ package_filter | default('grep') + ' -e ' + item }}"
    with_items: "{{ master_package_list }}"
  - name: Get list of installed packages we are interested in
    shell: "rpm -qa | {{ package_filter }}"
    register: master_packages
  - name: Get status of services on OCP Master node
    shell: "systemctl list-units {{ master_service_list | join('.service ') }}.service
            --type=service --all --no-pager --no-legend"
    register: master_services
  - name: Get OpenShift client version
    shell: "(oc version | grep -e 'oc ' -e 'openshift' -e 'kube') ||
            echo failed_to_get_oc_version_info"
    register: master_oc_version
  - name: Get list of OCP nodes
    shell: "oc get nodes -o wide || echo failed_to_get_list_of_nodes"
    register: master_ocp_nodes
  - name: Get info about all the docker images used in OCP cluster
    shell: "(oc get pods --all-namespaces
            -o=custom-columns=:.status.containerStatuses[*].image | grep -v -e '^$' | uniq) ||
            echo failed_to_get_list_of_images"
    register: master_image_info
  - name: Write master data to the data file
    delegate_to: localhost
    yedit:
      src: "{{ output_artifacts_dir }}master_data.yaml"
      state: present
      edits:
      - key: master
        value:
          Linux kernel version: "{{ master_linux_kernel_version.stdout_lines }}"
          Red Hat release info: "{{ master_rh_release.stdout_lines }}"
          List of Packages: "{{ master_packages.stdout_lines }}"
          List of services: "{{ master_services.stdout_lines }}"
          OC Version: "{{ master_oc_version.stdout_lines }}"
          OCP nodes: "{{ master_ocp_nodes.stdout_lines }}"
          Images info: "{{ master_image_info.stdout_lines }}"

  # Heketi POD logs, config and DB dump
  - name: Get heketi POD
    shell: "oc get pods --all-namespaces -l heketi --no-headers
                -o=custom-columns=:.metadata.name,:.metadata.namespace"
    register: heketi_pods
    retries: 10
    delay: 6
    until: heketi_pods is succeeded
  - name: DEBUG HEKETI
    debug:
      msg: "{{ heketi_pods }}"
  - block:
    - name: Create var with destination dir path
      set_fact:
        dir_path: "{{ output_artifacts_dir }}heketi_pod/"
    - name: Create compute directory if absent
      delegate_to: localhost
      file:
        path: "{{ dir_path }}"
        state: directory
    - name: Set Heketi POD name and Heketi namespace as vars
      set_fact:
        heketi_pod_name: "{{
          (heketi_pods.stdout_lines[0].split(' ') | list)[0] }}"
        heketi_pod_namespace: "{{
          (heketi_pods.stdout_lines[0].split(' ') | list)[-1] }}"
    - name: Set Heketi pod command prefix
      set_fact:
        heketi_pod_cmd_prefix: "oc exec {{ heketi_pod_name
          }} --namespace {{ heketi_pod_namespace }} --"
    - name: Get the Heketi config from the Heketi POD
      shell: 'echo -e "$({{ heketi_pod_cmd_prefix
        }} cat /etc/heketi/heketi.json ||
        echo error_failed_to_get_the_heketi_config_file)" |
        tee /tmp/heketi_config.json'
    - name: Copy the Heketi config
      fetch:
        src: "/tmp/heketi_config.json"
        dest: "{{ dir_path }}"
        flat: yes
        fail_on_missing: yes
    - name: Save Heketi POD logs
      shell: "(oc logs {{ heketi_pod_name }} ||
        echo 'ERROR! Failed to get the Heketi logs.') > /tmp/heketi.log"
    - name: Copy the Heketi logs
      fetch:
        src: "/tmp/heketi.log"
        dest: "{{ dir_path }}"
        flat: yes
        fail_on_missing: yes
    - name: Dump the Heketi DB
      shell: 'echo -e "$({{ heketi_pod_cmd_prefix }} heketi-cli --server {{
        hostvars["localhost"]["heketi_server_url"] }} --user {{
        hostvars["localhost"]["heketi_cli_user"] }} --secret {{
        hostvars["localhost"]["heketi_cli_key"]
        }} db dump ||
        echo \{\"error\"\:\"failed_to_get_the_Heketi_db_dump\"\})" |
        python -m json.tool > /tmp/heketi_db_dump.json'
    - name: Copy the Heketi DB dump
      fetch:
        src: "/tmp/heketi_db_dump.json"
        dest: "{{ dir_path }}"
        flat: yes
        fail_on_missing: yes
    - name: Get storage release version from Heketi POD
      shell: "{{ heketi_pod_cmd_prefix }} cat /etc/redhat-storage-release ||
              echo failed_to_read_redhat_storage_release_info"
      register: heketi_pod_storage_release_version
    - name: Get info about packages on Heketi POD
      shell: "{{ heketi_pod_cmd_prefix }} rpm -qa | grep -e {{
              heketi_pod_package_list | join(' -e ') }} ||
              echo failed_to_read_list_of_installed_packages"
      register: heketi_pod_packages
    - name: Write Heketi data to the data file
      delegate_to: localhost
      yedit:
        src: "{{ dir_path }}heketi_pod_data.yaml"
        state: present
        edits:
        - key: "Storage_release_version"
          value: "{{ heketi_pod_storage_release_version.stdout_lines }}"
        - key: "List_of_Packages"
          value: "{{ heketi_pod_packages.stdout_lines }}"
    when: "((heketi_pods.stdout_lines | join('')).strip() | length) > 0"

  # Gluster PODs
  - name: Get list of Gluster PODs
    shell: "oc get pods --all-namespaces -l glusterfs-node --no-headers
                -o=custom-columns=:.metadata.name,:.metadata.namespace"
    register: gluster_pods
    retries: 10
    delay: 6
    until: gluster_pods is succeeded
  - name: DEBUG GLUSTER
    debug:
      msg: "{{ gluster_pods }}"
  - name: Create var describing the Gluster cluster deployment type
    set_fact:
      is_gluster_containerized: "{{
          ((gluster_pods.stdout_lines | join('')).strip() | length) > 0 }}"
    delegate_to: "{{ item }}"
    delegate_facts: yes
    with_items: "{{ groups['all'] }}"
  - block:
    - name: Create var with destination dir path
      set_fact:
        dir_path: "{{ output_artifacts_dir }}gluster_pods/"
    - name: Create directory if absent
      delegate_to: localhost
      file:
        path: "{{ dir_path }}"
        state: directory
    - name: Define storage namespace
      set_fact:
        storage_namespace: "{{ (gluster_pods.stdout_lines[0].split(
                            ' ') | list)[-1] }}"
    - name: Define gluster POD names
      set_fact:
        gluster_pod_names: "{{ (gluster_pod_names | default([])) +
                               [(item.split(' ') | list)[0]] }}"
      with_items: "{{ gluster_pods.stdout_lines[0:] }}"
    - debug:
        msg: "{{ gluster_pod_names }}"

    - name: Get storage release version from Gluster PODs
      shell: "(oc exec {{ item }} --namespace {{ storage_namespace }} --
          cat /etc/redhat-storage-release) ||
          echo failed_to_get_redhat_storage_release_info"
      with_items: "{{ gluster_pod_names }}"
      register: gluster_pod_storage_release_version_results
    - name: Process gluster PODs storage release versions results
      set_fact:
        gluster_pod_storage_release_version_processed: "{{
            gluster_pod_storage_release_version_processed | default({}) | combine(
                {(item.item.strip().split(' ')[0]): item.stdout_lines},
                recursive=True
            ) }}"
      with_items: "{{ gluster_pod_storage_release_version_results.results }}"
    - name: Get info about packages on Gluster PODs
      shell: "(oc exec {{ item }} --namespace {{ storage_namespace }} --
          rpm -qa | grep -e {{ gluster_pod_package_list | join(' -e ') }}) ||
          echo failed_to_get_packages_info_from_gluster_pod"
      with_items: "{{ gluster_pod_names }}"
      register: gluster_pod_package_list_results
    - name: Process gluster PODs package lists results
      set_fact:
        gluster_pod_package_list_processed: "{{
            gluster_pod_package_list_processed | default({}) | combine(
                {(item.item.strip().split(' ')[0]): item.stdout_lines},
                recursive=True
            ) }}"
      with_items: "{{ gluster_pod_package_list_results.results }}"
    - name: Get info about services on Gluster PODs
      shell: "(oc exec {{ item }} --namespace {{ storage_namespace }} --
          systemctl list-units {{
              gluster_pod_service_list | join('.service ') }}.service
          --type=service --all --no-pager --no-legend) ||
          echo failed_to_get_services_info_from_gluster_pod"
      with_items: "{{ gluster_pod_names }}"
      register: gluster_pod_service_list_results
    - name: Process gluster PODs service lists results
      set_fact:
        gluster_pod_service_list_processed: "{{
            gluster_pod_service_list_processed | default({}) | combine(
                {(item.item.strip().split(' ')[0]): item.stdout_lines},
                recursive=True
            ) }}"
      with_items: "{{ gluster_pod_service_list_results.results }}"
    - name: Write Gluster PODs data to the data file
      delegate_to: localhost
      yedit:
        src: "{{ dir_path }}gluster_pods_packages_and_services_data.yaml"
        state: present
        edits:
        - key: gluster_pods
          value:
            Storage release version: "{{
                gluster_pod_storage_release_version_processed }}"
            List of Packages: "{{ gluster_pod_package_list_processed }}"
            List of Services: "{{ gluster_pod_service_list_processed }}"

    - name: Get 'targetcli ls' output
      shell: "(oc exec {{ item }} --namespace {{ storage_namespace }} --
          targetcli ls) || echo failed_to_get_targetcli_ls_output"
      with_items: "{{ gluster_pod_names }}"
      register: gluster_pod_targetcli_ls_results
    - debug:
        msg: "{{ gluster_pod_targetcli_ls_results }}"
    - name: Write Gluster PODs data to the data file
      delegate_to: localhost
      no_log: yes
      copy:
        content: "{{ item.stdout }}"
        dest: "{{ dir_path }}{{ (item.item.strip().split(' ') | list)[0] }}-targetcli-ls"
      with_items: "{{ gluster_pod_targetcli_ls_results.results }}"

    - name: Create gluster log directories on the master node
      file:
        path: "/tmp/gluster/{{ item }}-var_log_glusterfs"
        state: directory
      with_items: "{{ gluster_pod_names }}"
    - name: Copy '/var/log/glusterfs/*' files to the master
      shell: "(oc cp {{ storage_namespace }}/{{ item }}:/var/log/glusterfs
          /tmp/gluster/{{ item }}-var_log_glusterfs) ||
          echo failed_to_copy_var_log_glusterfs_files"
      with_items: "{{ gluster_pod_names }}"
    - name: Copy '/var/log/glusterfs/*' files from the master to the localhost
      synchronize:
        src: "/tmp/gluster/{{ item }}-var_log_glusterfs/"
        dest: "{{ dir_path }}{{ item }}-var_log_glusterfs/"
        mode: pull
        recursive: yes
        use_ssh_args: yes
      with_items: "{{ gluster_pod_names }}"
      ignore_errors: yes

    - name: Get 'dmesg -T' info
      shell: "(oc exec {{ item }} --namespace {{ storage_namespace
              }} -- dmesg -T) || echo failed_toget_dmesg_-T_info"
      with_items: "{{ gluster_pod_names }}"
      register: gluster_pods_dmesg_results
    - name: Write Gluster PODs dmesg data to files
      delegate_to: localhost
      no_log: yes
      copy:
        content: "{{ item.stdout }}"
        dest: "{{ dir_path }}{{ (item.item.strip().split(' ') | list)[0] }}-dmesg"
      with_items: "{{ gluster_pods_dmesg_results.results }}"

    - name: Get list of processes and info for processes with 'D' stat
      shell: "(oc exec {{ item }} --namespace {{ storage_namespace }} -- ps aux ;
              oc exec {{ item }} --namespace {{ storage_namespace }} --
              ps -aux | tee /dev/tty | awk {'if ( $8 ~ \"D\" ) print $2'} |
              while read -r pid ;
                do echo -e \"\nRunning '/proc/$pid/stack' command:\";
                cat /proc/$pid/stack ;
              done) || echo failed_to_get_info_about_processes_with_D_stat"
      with_items: "{{ gluster_pod_names }}"
      register: gluster_pod_io_processes_info
    - name: Write Gluster PODs 'I\O' proccesses info to files
      delegate_to: localhost
      no_log: yes
      copy:
        content: "{{ item.stdout }}"
        dest: "{{ dir_path }}{{ (item.item.strip().split(' ') | list)[0] }}-ps"
      with_items: "{{ gluster_pod_io_processes_info.results }}"

    - name: List dirs and files in '/sys/module/dm_multipath'
      shell: "(oc exec {{ item }} --namespace {{ storage_namespace
              }} -- ls -l /sys/module/dm_multipath) ||
              echo failed_to_list_files_in__sys_module_dm_multipath"
      with_items: "{{ gluster_pod_names }}"
      register: sys_module_dm_multipath_results
    - name: Write Gluster PODs 'ls -l /sys/module/dm_multipath' output to files
      delegate_to: localhost
      no_log: yes
      copy:
        content: "{{ item.stdout }}"
        dest: "{{ dir_path }}{{ (item.item.strip().split(' ') | list)[0]
               }}-ls-sys_module_dm_multipath"
      with_items: "{{ sys_module_dm_multipath_results.results }}"

    - name: "Run 'lsmod | egrep target_core' command"
      shell: "(oc exec {{ item }} --namespace {{ storage_namespace
              }} -- lsmod | egrep target_core) ||
              echo failed_to_get_lsmod_info_for_target_core"
      with_items: "{{ gluster_pod_names }}"
      register: lsmod_target_core_results
    - name: Write Gluster PODs 'lsmod | egrep target_core' command results to files
      delegate_to: localhost
      no_log: yes
      copy:
        content: "{{ item.stdout }}"
        dest: "{{ dir_path }}{{
               (item.item.strip().split(' ') | list)[0] }}-lsmod_target_core"
      with_items: "{{ lsmod_target_core_results.results }}"

    - name: Get info about devices
      shell: '(oc exec {{ item }} --namespace {{ storage_namespace
              }} -- bash -c "echo -e \"{{ separator_line }}\nlsblk info:\"; lsblk;
              echo -e \"{{ separator_line }}\nPVs info:\"; pvs;
              echo -e \"{{ separator_line }}\nVGs info:\"; vgs;
              echo -e \"{{ separator_line }}\nLVs info:\"; lvs;
              echo -e \"{{ separator_line }}\"") ||
              echo failed_to_get_list_of_pvs_vgs_and_lvs'
      with_items: "{{ gluster_pod_names }}"
      register: lsblk_pvs_vgs_lvs_results
    - name: Write Gluster PODs info about PVs, VGs and LVs to files
      delegate_to: localhost
      no_log: yes
      copy:
        content: "{{ item.stdout }}"
        dest: "{{ dir_path }}{{ (item.item.strip().split(' ') | list)[0]
               }}-lsblk_pvs_vgs_lvs"
      with_items: "{{ lsblk_pvs_vgs_lvs_results.results }}"

    - name: Read 'journalctl' output
      shell: "(oc exec {{ item }} --namespace {{ storage_namespace
              }} -- journalctl) || echo failed_to_read_journalctl_output"
      with_items: "{{ gluster_pod_names }}"
      register: journalctl_results
    - name: Write Gluster PODs 'journalctl' output to files
      delegate_to: localhost
      no_log: yes
      copy:
        content: "{{ item.stdout }}"
        dest: "{{ dir_path }}{{ (item.item.strip().split(' ') | list)[0]
               }}-journalctl"
      with_items: "{{ journalctl_results.results }}"

    - name: Read 'mount' output
      shell: "(oc exec {{ item }} --namespace {{ storage_namespace
              }} -- mount) || echo failed_to_read_mount_output"
      with_items: "{{ gluster_pod_names }}"
      register: mount_results
    - name: Write Gluster PODs 'mount' output to files
      delegate_to: localhost
      no_log: yes
      copy:
        content: "{{ item.stdout }}"
        dest: "{{ dir_path }}{{ (item.item.strip().split(' ') | list)[0] }}-mount"
      with_items: "{{ mount_results.results }}"

    - name: Create archive from ' /etc/target/' dir
      shell: "(oc exec {{ item }} --namespace {{ storage_namespace
              }} -- tar -czvf etc_target.tar.gz /etc/target/ ) ||
              echo failed_to_archive__etc_target_dir"
      with_items: "{{ gluster_pod_names }}"
    - name: Copy archive of the '/etc/target/' dir to the master
      shell: "(oc cp {{ storage_namespace }}/{{ item }}:/etc_target.tar.gz
          /tmp/gluster/{{ item }}-etc_target.tar.gz) ||
          echo failed_to_copy_etc_target_file"
      with_items: "{{ gluster_pod_names }}"
    - name: Copy archive of the '/etc/target/' dir to the localhost
      fetch:
        src: "/tmp/gluster/{{ item }}-etc_target.tar.gz"
        dest: "{{ dir_path }}"
        flat: yes
        fail_on_missing: yes
      with_items: "{{ gluster_pod_names }}"

    - name: Create archive from '/sys/kernel/config/target/' dir
      shell: "(oc exec {{ item }} --namespace {{ storage_namespace
              }} -- tar -czvf sys_kernel_config_target.tar.gz /sys/kernel/config/target/ ) ||
              echo failed_to_archive__sys_kernel_config_target_dir"
      with_items: "{{ gluster_pod_names }}"
    - name: Copy archive of the '/sys/kernel/config/target/' dir to the master
      shell: "(oc cp {{ storage_namespace }}/{{ item }}:/sys_kernel_config_target.tar.gz
          /tmp/gluster/{{ item }}-sys_kernel_config_target.tar.gz) ||
          echo failed_to_copy_sys_kernel_config_target_file"
      with_items: "{{ gluster_pod_names }}"
    - name: Copy archive of the '/sys/kernel/config/target/' dir to the localhost
      fetch:
        src: "/tmp/gluster/{{ item }}-sys_kernel_config_target.tar.gz"
        dest: "{{ dir_path }}"
        flat: yes
        fail_on_missing: yes
      with_items: "{{ gluster_pod_names }}"
    when: "is_gluster_containerized"

# Gather info from gluster nodes in case of 'standalone' deployment
- hosts: logs_gluster_nodes
  gather_facts: no
  vars:
    gluster_package_list:
    - gluster
    - heketi
    - targetcli
    - gluster-block
    - tcmu-runner
    - python-configshell
    - python-rtslib
    gluster_service_list:
    - glusterd
    - heketi
    - gluster-blockd
    - gluster-block-target
    - tcmu-runner
    - rpcbind
  tasks:
  - block:
    - name: Create var with destination dir path
      set_fact:
        dir_path: "{{ output_artifacts_dir }}gluster_nodes/"
    - name: Create directory if absent
      delegate_to: localhost
      file:
        path: "{{ dir_path }}"
        state: directory
      run_once: yes

    - name: Get storage release version from Gluster node
      shell: "(cat /etc/redhat-storage-release) ||
              echo failed_to_get_redhat_storage_release_info"
      register: gluster_node_storage_release_version_results
    - name: Get info about packages on Gluster node
      shell: "(rpm -qa | grep -e {{ gluster_package_list | join(' -e ') }}) ||
          echo failed_to_get_packages_info_from_gluster_node"
      register: gluster_node_package_list_results
    - name: Get info about services on Gluster node
      shell: "(systemctl list-units {{
                  gluster_service_list | join('.service ') }}.service
              --type=service --all --no-pager --no-legend) ||
              echo failed_to_get_services_info_from_gluster_node"
      register: gluster_node_service_list_results
    - name: Write Gluster node data to the data file
      delegate_to: localhost
      yedit:
        src: "{{ dir_path }}gluster_nodes_packages_and_services_data.yaml"
        state: present
        edits:
        - key: "gluster_node_{{ current_hostname }}"
          value:
            Storage release version: "{{
                gluster_node_storage_release_version_results.stdout }}"
            List of Packages: "{{ gluster_node_package_list_results.stdout_lines }}"
            List of Services: "{{ gluster_node_service_list_results.stdout_lines }}"

    - name: Get 'targetcli ls' output
      shell: "targetcli ls || echo failed_to_get_targetcli_ls_output"
      register: gluster_node_targetcli_ls_results
    - name: Write Gluster node data to the data file
      delegate_to: localhost
      no_log: yes
      copy:
        content: "{{ gluster_node_targetcli_ls_results.stdout }}"
        dest: "{{ dir_path }}{{ current_hostname }}-targetcli-ls"

    - name: Copy '/var/log/glusterfs/*' files from the current gluster node
      synchronize:
        src: "/var/log/glusterfs/"
        dest: "{{ dir_path }}{{ current_hostname }}-var_log_glusterfs/"
        mode: pull
        recursive: yes
        use_ssh_args: yes
      ignore_errors: yes

    - name: Get info about space usage
      shell: '(echo -e "File system disk space usage on the {{
              current_hostname }} node:\n"; df -Th) ||echo failed_to_get_df_info'
      register: df_info
    - name: Save mounts info into a file
      delegate_to: localhost
      copy:
        content: "{{ df_info.stdout }}"
        dest: "{{ dir_path }}{{ current_hostname }}_df"

    - name: Get 'dmesg -T' info
      shell: "dmesg -T || echo failed_toget_dmesg_-T_info"
      register: gluster_node_dmesg_results
    - name: Write Gluster node dmesg data to files
      delegate_to: localhost
      no_log: yes
      copy:
        content: "{{ gluster_node_dmesg_results.stdout }}"
        dest: "{{ dir_path }}{{ current_hostname }}-dmesg"

    - name: Get list of processes and info for processes with 'D' stat
      shell: "( ps aux ;
              ps -aux | tee /dev/tty | awk {'if ( $8 ~ \"D\" ) print $2'} |
              while read -r pid ;
                do echo -e \"\nRunning '/proc/$pid/stack' command:\";
                cat /proc/$pid/stack ;
              done) || echo failed_to_get_info_about_processes_with_D_stat"
      register: gluster_node_io_processes_info
    - name: Write Gluster node 'I\O' proccesses info to a file
      delegate_to: localhost
      no_log: yes
      copy:
        content: "{{ gluster_node_io_processes_info.stdout }}"
        dest: "{{ dir_path }}{{ current_hostname }}-ps"

    - name: List dirs and files in '/sys/module/dm_multipath'
      shell: "ls -l /sys/module/dm_multipath ||
              echo failed_to_list_files_in__sys_module_dm_multipath"
      register: sys_module_dm_multipath_results
    - name: Write Gluster node 'ls -l /sys/module/dm_multipath' output to a file
      delegate_to: localhost
      no_log: yes
      copy:
        content: "{{ sys_module_dm_multipath_results.stdout }}"
        dest: "{{ dir_path }}{{ current_hostname }}-ls-sys_module_dm_multipath"

    - name: "Run 'lsmod | egrep target_core' command"
      shell: "(lsmod | egrep target_core) ||
              echo failed_to_get_lsmod_info_for_target_core"
      register: lsmod_target_core_results
    - name: Write Gluster node 'lsmod | egrep target_core' command results to a file
      delegate_to: localhost
      no_log: yes
      copy:
        content: "{{ lsmod_target_core_results.stdout }}"
        dest: "{{ dir_path }}{{ current_hostname }}-lsmod_target_core"

    - name: Get info about devices
      shell: '(echo -e "{{ separator_line }}\nlsblk info:"; lsblk;
              echo -e "{{ separator_line }}\nPVs info:"; pvs;
              echo -e "{{ separator_line }}\nVGs info:"; vgs;
              echo -e "{{ separator_line }}\nLVs info:"; lvs;
              echo -e "{{ separator_line }}\nll /dev/disk/by-path/ip-*:";
              ll /dev/disk/by-path/ip-*; echo {{ separator_line }}) ||
              echo failed_to_get_info'
      register: lsblk_pvs_vgs_lvs
    - name: Save devices info into a file
      delegate_to: localhost
      copy:
        content: "{{ lsblk_pvs_vgs_lvs.stdout }}"
        dest: "{{ dir_path }}{{ current_hostname }}_lsblk_pvs_vgs_lvs"

    - name: Read 'journalctl' output
      shell: "journalctl || echo failed_to_read_journalctl_output"
      register: journalctl_results
    - name: Write Gluster node 'journalctl' output to a file
      delegate_to: localhost
      no_log: yes
      copy:
        content: "{{ journalctl_results.stdout }}"
        dest: "{{ dir_path }}{{ current_hostname }}-journalctl"

    - name: Read 'mount' output
      shell: "mount || echo failed_to_read_mount_output"
      register: mount_results
    - name: Write Gluster node 'mount' output to a file
      delegate_to: localhost
      no_log: yes
      copy:
        content: "{{ mount_results.stdout }}"
        dest: "{{ dir_path }}{{ current_hostname }}-mount"

    - name: Create archive from ' /etc/target/' dir
      shell: "tar -czvf /tmp/gluster/{{ current_hostname
              }}-etc_target.tar.gz /etc/target/ ||
              echo failed_to_archive__etc_target_dir"
    - name: Copy archive of the '/etc/target/' dir to the localhost
      fetch:
        src: "/tmp/gluster/{{ current_hostname }}-etc_target.tar.gz"
        dest: "{{ dir_path }}"
        flat: yes
        fail_on_missing: yes

    - name: Create archive from '/sys/kernel/config/target/' dir
      shell: "tar -czvf /tmp/gluster/{{ current_hostname
              }}-sys_kernel_config_target.tar.gz /sys/kernel/config/target/ ||
              echo failed_to_archive__sys_kernel_config_target_dir"
    - name: Copy archive of the '/sys/kernel/config/target/' dir to the localhost
      fetch:
        src: "/tmp/gluster/{{ current_hostname }}-sys_kernel_config_target.tar.gz"
        dest: "{{ dir_path }}"
        flat: yes
        fail_on_missing: yes

    - name: Create archive from '/var/log/messages' dir
      shell: "tar -czvf var_log_messages.tar.gz /var/log/messages"
      retries: 15
      delay: 2
      register: result
      until: result is succeeded
      ignore_errors: yes
    - name: Copy the archive to the localhost
      fetch:
        src: "var_log_messages.tar.gz"
        dest: "{{ dir_path }}{{ current_hostname }}_var_log_messages.tar.gz"
        flat: yes
        fail_on_missing: yes
      ignore_errors: yes
    when: "not is_gluster_containerized"

# Gather info from compute nodes
- hosts: logs_compute_nodes
  gather_facts: no
  vars:
    compute_package_list:
    - docker
    - heketi
    - rpcbind
    compute_service_list:
    - docker
    - multipathd
    - rpcbind
    - iscsid
  tasks:
  - name: Create var with destination dir path
    set_fact:
      dir_path: "{{ output_artifacts_dir }}compute_nodes/"
  - name: Create compute directory if absent
    delegate_to: localhost
    file:
      path: "{{ dir_path }}"
      state: directory
    run_once: yes

  - name: Create grep filter with all the packages we are interested in
    set_fact:
      package_filter: "{{ package_filter | default('grep') + ' -e ' + item }}"
    with_items: "{{ compute_package_list }}"
  - name: Get list of installed packages we are interested in
    shell: "rpm -qa | {{ package_filter }} || echo failed_to_get_info"
    register: compute_packages
  - name: Write compute nodes package list to a file
    delegate_to: localhost
    copy:
      content: "{{ compute_packages.stdout }}"
      dest: "{{ dir_path }}{{ current_hostname }}_packages.yaml"

  - name: Get status of services on OCP Compute nodes
    shell: "systemctl list-units {{
            compute_service_list | join('.service ') }}.service --no-pager
            --type=service --all --no-legend || echo failed_to_get_info"
    register: compute_services
  - name: Write compute nodes service list to a file
    delegate_to: localhost
    copy:
      content: "{{ compute_services.stdout }}"
      dest: "{{ dir_path }}{{ current_hostname }}_services.yaml"

  - name: Get multipath info
    shell: "(echo 'Multipath config is following:'; cat /etc/multipath.conf;
            echo -e \"{{ separator_line }}\nResults of 'nmultipath -ll' command:\";
            multipath -ll; echo {{ separator_line }}) || echo failed_to_get_info"
    register: multipath_info
  - name: Save multipath info into a file
    delegate_to: localhost
    copy:
      content: "{{ multipath_info.stdout }}"
      dest: "{{ dir_path }}{{ current_hostname }}_multipath_info"

  - name: Get info about devices
    shell: '(echo -e "{{ separator_line }}\nlsblk info:"; lsblk;
            echo -e "{{ separator_line }}\nPVs info:"; pvs;
            echo -e "{{ separator_line }}\nVGs info:"; vgs;
            echo -e "{{ separator_line }}\nLVs info:"; lvs;
            echo -e "{{ separator_line }}\nll /dev/disk/by-path/ip-*:";
            ll /dev/disk/by-path/ip-*; echo {{ separator_line }}) ||
            echo failed_to_get_info'
    register: lsblk_pvs_vgs_lvs
  - name: Save devices info into a file
    delegate_to: localhost
    copy:
      content: "{{ lsblk_pvs_vgs_lvs.stdout }}"
      dest: "{{ dir_path }}{{ current_hostname }}_lsblk_pvs_vgs_lvs"

  - name: Get info about mounts
    shell: '(echo -e "Mounts on the {{ current_hostname }} node:\n"; mount) ||
            echo failed_to_get_info'
    register: mounts
  - name: Save mounts info into a file
    delegate_to: localhost
    copy:
      content: "{{ mounts.stdout }}"
      dest: "{{ dir_path }}{{ current_hostname }}_mount"

  - name: Get info about space usage
    shell: '(echo -e "File system disk space usage on the {{
            current_hostname }} node:\n"; df -Th) ||echo failed_to_get_df_info'
    register: df_info
  - name: Save mounts info into a file
    delegate_to: localhost
    copy:
      content: "{{ df_info.stdout }}"
      dest: "{{ dir_path }}{{ current_hostname }}_df"

  - name: Read 'dmesg -T' info
    shell: "dmesg -T || echo failed_to_get_info"
    register: dmesg_info
  - name: Save dmesg info into a file
    delegate_to: localhost
    copy:
      content: "{{ dmesg_info.stdout }}"
      dest: "{{ dir_path }}{{ current_hostname }}_dmesg"

  - name: Create archive from '/var/log/messages' dir
    shell: "tar -czvf var_log_messages.tar.gz /var/log/messages"
    retries: 15
    delay: 2
    register: result
    until: result is succeeded
    ignore_errors: yes
  - name: Copy the archive to the localhost
    fetch:
      src: "var_log_messages.tar.gz"
      dest: "{{ dir_path }}{{ current_hostname }}_var_log_messages.tar.gz"
      flat: yes
      fail_on_missing: yes
    ignore_errors: yes