diff options
-rw-r--r-- | deployment/playbooks/gather_logs.yaml | 883 | ||||
-rw-r--r-- | deployment/playbooks/get_ocp_info.yaml | 2 | ||||
-rw-r--r-- | deployment/playbooks/roles/vmware-guest-setup/tasks/main.yaml | 5 |
3 files changed, 890 insertions, 0 deletions
diff --git a/deployment/playbooks/gather_logs.yaml b/deployment/playbooks/gather_logs.yaml new file mode 100644 index 00000000..33b9114f --- /dev/null +++ b/deployment/playbooks/gather_logs.yaml @@ -0,0 +1,883 @@ +# Expected vars: +# +# - 'config_filepath' - required. It is expected to be provided and must +# store filepath for the config file used by automated test caes. +# Current playbook will take all the nodes info from it. +# +# - 'output_artifacts_dir' - optional. It is directory where should be saved +# generated/gathered files. +# +# Command to run this playbook: +# +# $ tox -e ocp3.11 -- ansible-playbook -i 127.0.0.1, \ +# playbooks/gather_logs.yaml \ +# -e config_filepath=/path/to/the/cns-automation-config.yaml \ +# -e output_artifacts_dir=../cluster_logs/ + +--- +- hosts: localhost + connection: local + gather_facts: no + tasks: + - name: Process config file and find all the nodes of an OpenShift cluster + command: "python -c \"import yaml ; + config = yaml.load(open('{{ config_filepath }}', 'r')); + print(' '.join(list(config['ocp_servers']['master'].keys()))); + print(' '.join(list(config['ocp_servers']['nodes'].keys()))); + print(' '.join(list(config['gluster_servers'].keys()))); + print(config.get('openshift', config.get('cns', + {}))['heketi_config']['heketi_server_url']); + print(config.get('openshift', config.get('cns', + {}))['heketi_config']['heketi_cli_user']); + print(config.get('openshift', config.get('cns', + {}))['heketi_config']['heketi_cli_key'])\"" + register: config_data + - debug: + msg: "{{ config_data }}" + - name: Save config data to the host vars + set_fact: + master_nodes: "{{ config_data.stdout_lines[0].split(' ') }}" + compute_nodes: "{{ config_data.stdout_lines[1].split(' ') }}" + gluster_nodes: "{{ config_data.stdout_lines[2].split(' ') }}" + heketi_server_url: "{{ config_data.stdout_lines[3] }}" + heketi_cli_user: "{{ config_data.stdout_lines[4] }}" + heketi_cli_key: "{{ config_data.stdout_lines[5] }}" + - name: Print list of master nodes IP addresses + debug: + msg: "{{ master_nodes }}" + - name: Print list of compute nodes IP addresses + debug: + msg: "{{ compute_nodes }}" + - name: Print list of gluster nodes IP addresses + debug: + msg: "{{ gluster_nodes }}" + - name: Add gathered master IP addresses to the Ansible host list + add_host: + hostname: "{{ item }}" + ansible_host: "{{ item }}" + groups: logs_ocp_nodes, logs_single_master_node, logs_master_nodes + with_items: "{{ master_nodes }}" + - name: Add gathered compute IP addresses to the Ansible host list + add_host: + hostname: "{{ item }}" + ansible_host: "{{ item }}" + groups: logs_ocp_nodes, logs_compute_nodes + with_items: "{{ compute_nodes }}" + - name: Add gathered gluster IP addresses to the Ansible host list + add_host: + hostname: "{{ item }}" + ansible_host: "{{ item }}" + groups: logs_ocp_nodes, logs_gluster_nodes + with_items: "{{ gluster_nodes }}" + +- hosts: all + gather_facts: no + tasks: + - set_fact: + output_artifacts_dir: "{{ + (output_artifacts_dir | default('../cluster_logs/')) }}" + - set_fact: + output_artifacts_dir: "{{ output_artifacts_dir + }}{% if output_artifacts_dir[-1] != '/' %}{{ '/' }}{% endif %}" + - name: Get hostname of the current host + shell: "hostname" + register: hostname + - name: Save hostname to the var + set_fact: + current_hostname: "{{ hostname.stdout_lines[0].strip() }}" + separator_line: "{{ '=' * 79 }}" + - name: Create output artifacts directory if absent + delegate_to: localhost + file: + path: "{{ output_artifacts_dir }}" + state: directory + run_once: yes + - name: Install 'rsync' package which is required by 'synchronize' module + yum: + name: rsync + state: installed + ignore_errors: yes + +# All nodes SOS reports +- hosts: logs_ocp_nodes + gather_facts: no + tasks: + - name: Install 'sos' package if absent + package: + name: sos + state: present + - name: Run sosreport command + shell: "sosreport --batch --verbose --tmp-dir . --label customuniquelabel" + register: sosreport_output + - name: Print the output of the sosreport command + debug: + msg: "{{ sosreport_output }}" + - name: Get name of the generated sos-file + shell: 'echo -e "{{ sosreport_output.stdout }}" | grep customuniquelabel' + register: sos_filepath + - name: Copy generated files to the localhost + fetch: + src: "{{ sos_filepath.stdout_lines[0].strip() }}" + dest: "{{ output_artifacts_dir }}sosreports/" + flat: yes + fail_on_missing: yes + +- hosts: logs_single_master_node + gather_facts: no + vars: + master_package_list: + - docker + - heketi + master_service_list: + - docker + - multipathd + heketi_pod_package_list: + - gluster + - heketi + gluster_pod_package_list: + - gluster + - heketi + - targetcli + - gluster-block + - tcmu-runner + - python-configshell + - python-rtslib + gluster_pod_service_list: + - glusterd + - heketi + - gluster-blockd + - gluster-block-target + - tcmu-runner + - rpcbind + tasks: + + # Master node info + - name: Get distro version + shell: "uname -a" + register: master_linux_kernel_version + - name: Get Red Hat release info + shell: "cat /etc/redhat-release" + register: master_rh_release + - name: Create grep filter with all the packages we are interested in + set_fact: + package_filter: "{{ package_filter | default('grep') + ' -e ' + item }}" + with_items: "{{ master_package_list }}" + - name: Get list of installed packages we are interested in + shell: "rpm -qa | {{ package_filter }}" + register: master_packages + - name: Get status of services on OCP Master node + shell: "systemctl list-units {{ master_service_list | join('.service ') }}.service + --type=service --all --no-pager --no-legend" + register: master_services + - name: Get OpenShift client version + shell: "(oc version | grep -e 'oc ' -e 'openshift' -e 'kube') || + echo failed_to_get_oc_version_info" + register: master_oc_version + - name: Get list of OCP nodes + shell: "oc get nodes -o wide || echo failed_to_get_list_of_nodes" + register: master_ocp_nodes + - name: Get info about all the docker images used in OCP cluster + shell: "(oc get pods --all-namespaces + -o=custom-columns=:.status.containerStatuses[*].image | grep -v -e '^$' | uniq) || + echo failed_to_get_list_of_images" + register: master_image_info + - name: Write master data to the data file + delegate_to: localhost + yedit: + src: "{{ output_artifacts_dir }}master_data.yaml" + state: present + edits: + - key: master + value: + Linux kernel version: "{{ master_linux_kernel_version.stdout_lines }}" + Red Hat release info: "{{ master_rh_release.stdout_lines }}" + List of Packages: "{{ master_packages.stdout_lines }}" + List of services: "{{ master_services.stdout_lines }}" + OC Version: "{{ master_oc_version.stdout_lines }}" + OCP nodes: "{{ master_ocp_nodes.stdout_lines }}" + Images info: "{{ master_image_info.stdout_lines }}" + + # Heketi POD logs, config and DB dump + - name: Get heketi POD + shell: "oc get pods --all-namespaces -l heketi --no-headers + -o=custom-columns=:.metadata.name,:.metadata.namespace" + register: heketi_pods + retries: 10 + delay: 6 + until: heketi_pods is succeeded + - name: DEBUG HEKETI + debug: + msg: "{{ heketi_pods }}" + - block: + - name: Create var with destination dir path + set_fact: + dir_path: "{{ output_artifacts_dir }}heketi_pod/" + - name: Create compute directory if absent + delegate_to: localhost + file: + path: "{{ dir_path }}" + state: directory + - name: Set Heketi POD name and Heketi namespace as vars + set_fact: + heketi_pod_name: "{{ + (heketi_pods.stdout_lines[0].split(' ') | list)[0] }}" + heketi_pod_namespace: "{{ + (heketi_pods.stdout_lines[0].split(' ') | list)[-1] }}" + - name: Set Heketi pod command prefix + set_fact: + heketi_pod_cmd_prefix: "oc exec {{ heketi_pod_name + }} --namespace {{ heketi_pod_namespace }} --" + - name: Get the Heketi config from the Heketi POD + shell: 'echo -e "$({{ heketi_pod_cmd_prefix + }} cat /etc/heketi/heketi.json || + echo error_failed_to_get_the_heketi_config_file)" | + tee /tmp/heketi_config.json' + - name: Copy the Heketi config + fetch: + src: "/tmp/heketi_config.json" + dest: "{{ dir_path }}" + flat: yes + fail_on_missing: yes + - name: Save Heketi POD logs + shell: "(oc logs {{ heketi_pod_name }} || + echo 'ERROR! Failed to get the Heketi logs.') > /tmp/heketi.log" + - name: Copy the Heketi logs + fetch: + src: "/tmp/heketi.log" + dest: "{{ dir_path }}" + flat: yes + fail_on_missing: yes + - name: Dump the Heketi DB + shell: 'echo -e "$({{ heketi_pod_cmd_prefix }} heketi-cli --server {{ + hostvars["localhost"]["heketi_server_url"] }} --user {{ + hostvars["localhost"]["heketi_cli_user"] }} --secret {{ + hostvars["localhost"]["heketi_cli_key"] + }} db dump || + echo \{\"error\"\:\"failed_to_get_the_Heketi_db_dump\"\})" | + python -m json.tool > /tmp/heketi_db_dump.json' + - name: Copy the Heketi DB dump + fetch: + src: "/tmp/heketi_db_dump.json" + dest: "{{ dir_path }}" + flat: yes + fail_on_missing: yes + - name: Get storage release version from Heketi POD + shell: "{{ heketi_pod_cmd_prefix }} cat /etc/redhat-storage-release || + echo failed_to_read_redhat_storage_release_info" + register: heketi_pod_storage_release_version + - name: Get info about packages on Heketi POD + shell: "{{ heketi_pod_cmd_prefix }} rpm -qa | grep -e {{ + heketi_pod_package_list | join(' -e ') }} || + echo failed_to_read_list_of_installed_packages" + register: heketi_pod_packages + - name: Write Heketi data to the data file + delegate_to: localhost + yedit: + src: "{{ dir_path }}heketi_pod_data.yaml" + state: present + edits: + - key: "Storage_release_version" + value: "{{ heketi_pod_storage_release_version.stdout_lines }}" + - key: "List_of_Packages" + value: "{{ heketi_pod_packages.stdout_lines }}" + when: "((heketi_pods.stdout_lines | join('')).strip() | length) > 0" + + # Gluster PODs + - name: Get list of Gluster PODs + shell: "oc get pods --all-namespaces -l glusterfs-node --no-headers + -o=custom-columns=:.metadata.name,:.metadata.namespace" + register: gluster_pods + retries: 10 + delay: 6 + until: gluster_pods is succeeded + - name: DEBUG GLUSTER + debug: + msg: "{{ gluster_pods }}" + - name: Create var describing the Gluster cluster deployment type + set_fact: + is_gluster_containerized: "{{ + ((gluster_pods.stdout_lines | join('')).strip() | length) > 0 }}" + delegate_to: "{{ item }}" + delegate_facts: yes + with_items: "{{ groups['all'] }}" + - block: + - name: Create var with destination dir path + set_fact: + dir_path: "{{ output_artifacts_dir }}gluster_pods/" + - name: Create directory if absent + delegate_to: localhost + file: + path: "{{ dir_path }}" + state: directory + - name: Define storage namespace + set_fact: + storage_namespace: "{{ (gluster_pods.stdout_lines[0].split( + ' ') | list)[-1] }}" + - name: Define gluster POD names + set_fact: + gluster_pod_names: "{{ (gluster_pod_names | default([])) + + [(item.split(' ') | list)[0]] }}" + with_items: "{{ gluster_pods.stdout_lines[0:] }}" + - debug: + msg: "{{ gluster_pod_names }}" + + - name: Get storage release version from Gluster PODs + shell: "(oc exec {{ item }} --namespace {{ storage_namespace }} -- + cat /etc/redhat-storage-release) || + echo failed_to_get_redhat_storage_release_info" + with_items: "{{ gluster_pod_names }}" + register: gluster_pod_storage_release_version_results + - name: Process gluster PODs storage release versions results + set_fact: + gluster_pod_storage_release_version_processed: "{{ + gluster_pod_storage_release_version_processed | default({}) | combine( + {(item.item.strip().split(' ')[0]): item.stdout_lines}, + recursive=True + ) }}" + with_items: "{{ gluster_pod_storage_release_version_results.results }}" + - name: Get info about packages on Gluster PODs + shell: "(oc exec {{ item }} --namespace {{ storage_namespace }} -- + rpm -qa | grep -e {{ gluster_pod_package_list | join(' -e ') }}) || + echo failed_to_get_packages_info_from_gluster_pod" + with_items: "{{ gluster_pod_names }}" + register: gluster_pod_package_list_results + - name: Process gluster PODs package lists results + set_fact: + gluster_pod_package_list_processed: "{{ + gluster_pod_package_list_processed | default({}) | combine( + {(item.item.strip().split(' ')[0]): item.stdout_lines}, + recursive=True + ) }}" + with_items: "{{ gluster_pod_package_list_results.results }}" + - name: Get info about services on Gluster PODs + shell: "(oc exec {{ item }} --namespace {{ storage_namespace }} -- + systemctl list-units {{ + gluster_pod_service_list | join('.service ') }}.service + --type=service --all --no-pager --no-legend) || + echo failed_to_get_services_info_from_gluster_pod" + with_items: "{{ gluster_pod_names }}" + register: gluster_pod_service_list_results + - name: Process gluster PODs service lists results + set_fact: + gluster_pod_service_list_processed: "{{ + gluster_pod_service_list_processed | default({}) | combine( + {(item.item.strip().split(' ')[0]): item.stdout_lines}, + recursive=True + ) }}" + with_items: "{{ gluster_pod_service_list_results.results }}" + - name: Write Gluster PODs data to the data file + delegate_to: localhost + yedit: + src: "{{ dir_path }}gluster_pods_packages_and_services_data.yaml" + state: present + edits: + - key: gluster_pods + value: + Storage release version: "{{ + gluster_pod_storage_release_version_processed }}" + List of Packages: "{{ gluster_pod_package_list_processed }}" + List of Services: "{{ gluster_pod_service_list_processed }}" + + - name: Get 'targetcli ls' output + shell: "(oc exec {{ item }} --namespace {{ storage_namespace }} -- + targetcli ls) || echo failed_to_get_targetcli_ls_output" + with_items: "{{ gluster_pod_names }}" + register: gluster_pod_targetcli_ls_results + - debug: + msg: "{{ gluster_pod_targetcli_ls_results }}" + - name: Write Gluster PODs data to the data file + delegate_to: localhost + no_log: yes + copy: + content: "{{ item.stdout }}" + dest: "{{ dir_path }}{{ (item.item.strip().split(' ') | list)[0] }}-targetcli-ls" + with_items: "{{ gluster_pod_targetcli_ls_results.results }}" + + - name: Create gluster log directories on the master node + file: + path: "/tmp/gluster/{{ item }}-var_log_glusterfs" + state: directory + with_items: "{{ gluster_pod_names }}" + - name: Copy '/var/log/glusterfs/*' files to the master + shell: "(oc cp {{ storage_namespace }}/{{ item }}:/var/log/glusterfs + /tmp/gluster/{{ item }}-var_log_glusterfs) || + echo failed_to_copy_var_log_glusterfs_files" + with_items: "{{ gluster_pod_names }}" + - name: Copy '/var/log/glusterfs/*' files from the master to the localhost + synchronize: + src: "/tmp/gluster/{{ item }}-var_log_glusterfs/" + dest: "{{ dir_path }}{{ item }}-var_log_glusterfs/" + mode: pull + recursive: yes + use_ssh_args: yes + with_items: "{{ gluster_pod_names }}" + ignore_errors: yes + + - name: Get 'dmesg -T' info + shell: "(oc exec {{ item }} --namespace {{ storage_namespace + }} -- dmesg -T) || echo failed_toget_dmesg_-T_info" + with_items: "{{ gluster_pod_names }}" + register: gluster_pods_dmesg_results + - name: Write Gluster PODs dmesg data to files + delegate_to: localhost + no_log: yes + copy: + content: "{{ item.stdout }}" + dest: "{{ dir_path }}{{ (item.item.strip().split(' ') | list)[0] }}-dmesg" + with_items: "{{ gluster_pods_dmesg_results.results }}" + + - name: Get list of processes and info for processes with 'D' stat + shell: "(oc exec {{ item }} --namespace {{ storage_namespace }} -- ps aux ; + oc exec {{ item }} --namespace {{ storage_namespace }} -- + ps -aux | tee /dev/tty | awk {'if ( $8 ~ \"D\" ) print $2'} | + while read -r pid ; + do echo -e \"\nRunning '/proc/$pid/stack' command:\"; + cat /proc/$pid/stack ; + done) || echo failed_to_get_info_about_processes_with_D_stat" + with_items: "{{ gluster_pod_names }}" + register: gluster_pod_io_processes_info + - name: Write Gluster PODs 'I\O' proccesses info to files + delegate_to: localhost + no_log: yes + copy: + content: "{{ item.stdout }}" + dest: "{{ dir_path }}{{ (item.item.strip().split(' ') | list)[0] }}-ps" + with_items: "{{ gluster_pod_io_processes_info.results }}" + + - name: List dirs and files in '/sys/module/dm_multipath' + shell: "(oc exec {{ item }} --namespace {{ storage_namespace + }} -- ls -l /sys/module/dm_multipath) || + echo failed_to_list_files_in__sys_module_dm_multipath" + with_items: "{{ gluster_pod_names }}" + register: sys_module_dm_multipath_results + - name: Write Gluster PODs 'ls -l /sys/module/dm_multipath' output to files + delegate_to: localhost + no_log: yes + copy: + content: "{{ item.stdout }}" + dest: "{{ dir_path }}{{ (item.item.strip().split(' ') | list)[0] + }}-ls-sys_module_dm_multipath" + with_items: "{{ sys_module_dm_multipath_results.results }}" + + - name: "Run 'lsmod | egrep target_core' command" + shell: "(oc exec {{ item }} --namespace {{ storage_namespace + }} -- lsmod | egrep target_core) || + echo failed_to_get_lsmod_info_for_target_core" + with_items: "{{ gluster_pod_names }}" + register: lsmod_target_core_results + - name: Write Gluster PODs 'lsmod | egrep target_core' command results to files + delegate_to: localhost + no_log: yes + copy: + content: "{{ item.stdout }}" + dest: "{{ dir_path }}{{ + (item.item.strip().split(' ') | list)[0] }}-lsmod_target_core" + with_items: "{{ lsmod_target_core_results.results }}" + + - name: Get info about devices + shell: '(oc exec {{ item }} --namespace {{ storage_namespace + }} -- bash -c "echo -e \"{{ separator_line }}\nlsblk info:\"; lsblk; + echo -e \"{{ separator_line }}\nPVs info:\"; pvs; + echo -e \"{{ separator_line }}\nVGs info:\"; vgs; + echo -e \"{{ separator_line }}\nLVs info:\"; lvs; + echo -e \"{{ separator_line }}\"") || + echo failed_to_get_list_of_pvs_vgs_and_lvs' + with_items: "{{ gluster_pod_names }}" + register: lsblk_pvs_vgs_lvs_results + - name: Write Gluster PODs info about PVs, VGs and LVs to files + delegate_to: localhost + no_log: yes + copy: + content: "{{ item.stdout }}" + dest: "{{ dir_path }}{{ (item.item.strip().split(' ') | list)[0] + }}-lsblk_pvs_vgs_lvs" + with_items: "{{ lsblk_pvs_vgs_lvs_results.results }}" + + - name: Read 'journalctl' output + shell: "(oc exec {{ item }} --namespace {{ storage_namespace + }} -- journalctl) || echo failed_to_read_journalctl_output" + with_items: "{{ gluster_pod_names }}" + register: journalctl_results + - name: Write Gluster PODs 'journalctl' output to files + delegate_to: localhost + no_log: yes + copy: + content: "{{ item.stdout }}" + dest: "{{ dir_path }}{{ (item.item.strip().split(' ') | list)[0] + }}-journalctl" + with_items: "{{ journalctl_results.results }}" + + - name: Read 'mount' output + shell: "(oc exec {{ item }} --namespace {{ storage_namespace + }} -- mount) || echo failed_to_read_mount_output" + with_items: "{{ gluster_pod_names }}" + register: mount_results + - name: Write Gluster PODs 'mount' output to files + delegate_to: localhost + no_log: yes + copy: + content: "{{ item.stdout }}" + dest: "{{ dir_path }}{{ (item.item.strip().split(' ') | list)[0] }}-mount" + with_items: "{{ mount_results.results }}" + + - name: Create archive from ' /etc/target/' dir + shell: "(oc exec {{ item }} --namespace {{ storage_namespace + }} -- tar -czvf etc_target.tar.gz /etc/target/ ) || + echo failed_to_archive__etc_target_dir" + with_items: "{{ gluster_pod_names }}" + - name: Copy archive of the '/etc/target/' dir to the master + shell: "(oc cp {{ storage_namespace }}/{{ item }}:/etc_target.tar.gz + /tmp/gluster/{{ item }}-etc_target.tar.gz) || + echo failed_to_copy_etc_target_file" + with_items: "{{ gluster_pod_names }}" + - name: Copy archive of the '/etc/target/' dir to the localhost + fetch: + src: "/tmp/gluster/{{ item }}-etc_target.tar.gz" + dest: "{{ dir_path }}" + flat: yes + fail_on_missing: yes + with_items: "{{ gluster_pod_names }}" + + - name: Create archive from '/sys/kernel/config/target/' dir + shell: "(oc exec {{ item }} --namespace {{ storage_namespace + }} -- tar -czvf sys_kernel_config_target.tar.gz /sys/kernel/config/target/ ) || + echo failed_to_archive__sys_kernel_config_target_dir" + with_items: "{{ gluster_pod_names }}" + - name: Copy archive of the '/sys/kernel/config/target/' dir to the master + shell: "(oc cp {{ storage_namespace }}/{{ item }}:/sys_kernel_config_target.tar.gz + /tmp/gluster/{{ item }}-sys_kernel_config_target.tar.gz) || + echo failed_to_copy_sys_kernel_config_target_file" + with_items: "{{ gluster_pod_names }}" + - name: Copy archive of the '/sys/kernel/config/target/' dir to the localhost + fetch: + src: "/tmp/gluster/{{ item }}-sys_kernel_config_target.tar.gz" + dest: "{{ dir_path }}" + flat: yes + fail_on_missing: yes + with_items: "{{ gluster_pod_names }}" + when: "is_gluster_containerized" + +# Gather info from gluster nodes in case of 'standalone' deployment +- hosts: logs_gluster_nodes + gather_facts: no + vars: + gluster_package_list: + - gluster + - heketi + - targetcli + - gluster-block + - tcmu-runner + - python-configshell + - python-rtslib + gluster_service_list: + - glusterd + - heketi + - gluster-blockd + - gluster-block-target + - tcmu-runner + - rpcbind + tasks: + - block: + - name: Create var with destination dir path + set_fact: + dir_path: "{{ output_artifacts_dir }}gluster_nodes/" + - name: Create directory if absent + delegate_to: localhost + file: + path: "{{ dir_path }}" + state: directory + run_once: yes + + - name: Get storage release version from Gluster node + shell: "(cat /etc/redhat-storage-release) || + echo failed_to_get_redhat_storage_release_info" + register: gluster_node_storage_release_version_results + - name: Get info about packages on Gluster node + shell: "(rpm -qa | grep -e {{ gluster_package_list | join(' -e ') }}) || + echo failed_to_get_packages_info_from_gluster_node" + register: gluster_node_package_list_results + - name: Get info about services on Gluster node + shell: "(systemctl list-units {{ + gluster_service_list | join('.service ') }}.service + --type=service --all --no-pager --no-legend) || + echo failed_to_get_services_info_from_gluster_node" + register: gluster_node_service_list_results + - name: Write Gluster node data to the data file + delegate_to: localhost + yedit: + src: "{{ dir_path }}gluster_nodes_packages_and_services_data.yaml" + state: present + edits: + - key: "gluster_node_{{ current_hostname }}" + value: + Storage release version: "{{ + gluster_node_storage_release_version_results.stdout }}" + List of Packages: "{{ gluster_node_package_list_results.stdout_lines }}" + List of Services: "{{ gluster_node_service_list_results.stdout_lines }}" + + - name: Get 'targetcli ls' output + shell: "targetcli ls || echo failed_to_get_targetcli_ls_output" + register: gluster_node_targetcli_ls_results + - name: Write Gluster node data to the data file + delegate_to: localhost + no_log: yes + copy: + content: "{{ gluster_node_targetcli_ls_results.stdout }}" + dest: "{{ dir_path }}{{ current_hostname }}-targetcli-ls" + + - name: Copy '/var/log/glusterfs/*' files from the current gluster node + synchronize: + src: "/var/log/glusterfs/" + dest: "{{ dir_path }}{{ current_hostname }}-var_log_glusterfs/" + mode: pull + recursive: yes + use_ssh_args: yes + ignore_errors: yes + + - name: Get info about space usage + shell: '(echo -e "File system disk space usage on the {{ + current_hostname }} node:\n"; df -Th) ||echo failed_to_get_df_info' + register: df_info + - name: Save mounts info into a file + delegate_to: localhost + copy: + content: "{{ df_info.stdout }}" + dest: "{{ dir_path }}{{ current_hostname }}_df" + + - name: Get 'dmesg -T' info + shell: "dmesg -T || echo failed_toget_dmesg_-T_info" + register: gluster_node_dmesg_results + - name: Write Gluster node dmesg data to files + delegate_to: localhost + no_log: yes + copy: + content: "{{ gluster_node_dmesg_results.stdout }}" + dest: "{{ dir_path }}{{ current_hostname }}-dmesg" + + - name: Get list of processes and info for processes with 'D' stat + shell: "( ps aux ; + ps -aux | tee /dev/tty | awk {'if ( $8 ~ \"D\" ) print $2'} | + while read -r pid ; + do echo -e \"\nRunning '/proc/$pid/stack' command:\"; + cat /proc/$pid/stack ; + done) || echo failed_to_get_info_about_processes_with_D_stat" + register: gluster_node_io_processes_info + - name: Write Gluster node 'I\O' proccesses info to a file + delegate_to: localhost + no_log: yes + copy: + content: "{{ gluster_node_io_processes_info.stdout }}" + dest: "{{ dir_path }}{{ current_hostname }}-ps" + + - name: List dirs and files in '/sys/module/dm_multipath' + shell: "ls -l /sys/module/dm_multipath || + echo failed_to_list_files_in__sys_module_dm_multipath" + register: sys_module_dm_multipath_results + - name: Write Gluster node 'ls -l /sys/module/dm_multipath' output to a file + delegate_to: localhost + no_log: yes + copy: + content: "{{ sys_module_dm_multipath_results.stdout }}" + dest: "{{ dir_path }}{{ current_hostname }}-ls-sys_module_dm_multipath" + + - name: "Run 'lsmod | egrep target_core' command" + shell: "(lsmod | egrep target_core) || + echo failed_to_get_lsmod_info_for_target_core" + register: lsmod_target_core_results + - name: Write Gluster node 'lsmod | egrep target_core' command results to a file + delegate_to: localhost + no_log: yes + copy: + content: "{{ lsmod_target_core_results.stdout }}" + dest: "{{ dir_path }}{{ current_hostname }}-lsmod_target_core" + + - name: Get info about devices + shell: '(echo -e "{{ separator_line }}\nlsblk info:"; lsblk; + echo -e "{{ separator_line }}\nPVs info:"; pvs; + echo -e "{{ separator_line }}\nVGs info:"; vgs; + echo -e "{{ separator_line }}\nLVs info:"; lvs; + echo -e "{{ separator_line }}\nll /dev/disk/by-path/ip-*:"; + ll /dev/disk/by-path/ip-*; echo {{ separator_line }}) || + echo failed_to_get_info' + register: lsblk_pvs_vgs_lvs + - name: Save devices info into a file + delegate_to: localhost + copy: + content: "{{ lsblk_pvs_vgs_lvs.stdout }}" + dest: "{{ dir_path }}{{ current_hostname }}_lsblk_pvs_vgs_lvs" + + - name: Read 'journalctl' output + shell: "journalctl || echo failed_to_read_journalctl_output" + register: journalctl_results + - name: Write Gluster node 'journalctl' output to a file + delegate_to: localhost + no_log: yes + copy: + content: "{{ journalctl_results.stdout }}" + dest: "{{ dir_path }}{{ current_hostname }}-journalctl" + + - name: Read 'mount' output + shell: "mount || echo failed_to_read_mount_output" + register: mount_results + - name: Write Gluster node 'mount' output to a file + delegate_to: localhost + no_log: yes + copy: + content: "{{ mount_results.stdout }}" + dest: "{{ dir_path }}{{ current_hostname }}-mount" + + - name: Create archive from ' /etc/target/' dir + shell: "tar -czvf /tmp/gluster/{{ current_hostname + }}-etc_target.tar.gz /etc/target/ || + echo failed_to_archive__etc_target_dir" + - name: Copy archive of the '/etc/target/' dir to the localhost + fetch: + src: "/tmp/gluster/{{ current_hostname }}-etc_target.tar.gz" + dest: "{{ dir_path }}" + flat: yes + fail_on_missing: yes + + - name: Create archive from '/sys/kernel/config/target/' dir + shell: "tar -czvf /tmp/gluster/{{ current_hostname + }}-sys_kernel_config_target.tar.gz /sys/kernel/config/target/ || + echo failed_to_archive__sys_kernel_config_target_dir" + - name: Copy archive of the '/sys/kernel/config/target/' dir to the localhost + fetch: + src: "/tmp/gluster/{{ current_hostname }}-sys_kernel_config_target.tar.gz" + dest: "{{ dir_path }}" + flat: yes + fail_on_missing: yes + + - name: Create archive from '/var/log/messages' dir + shell: "tar -czvf var_log_messages.tar.gz /var/log/messages" + retries: 15 + delay: 2 + register: result + until: result is succeeded + ignore_errors: yes + - name: Copy the archive to the localhost + fetch: + src: "var_log_messages.tar.gz" + dest: "{{ dir_path }}{{ current_hostname }}_var_log_messages.tar.gz" + flat: yes + fail_on_missing: yes + ignore_errors: yes + when: "not is_gluster_containerized" + +# Gather info from compute nodes +- hosts: logs_compute_nodes + gather_facts: no + vars: + compute_package_list: + - docker + - heketi + - rpcbind + compute_service_list: + - docker + - multipathd + - rpcbind + - iscsid + tasks: + - name: Create var with destination dir path + set_fact: + dir_path: "{{ output_artifacts_dir }}compute_nodes/" + - name: Create compute directory if absent + delegate_to: localhost + file: + path: "{{ dir_path }}" + state: directory + run_once: yes + + - name: Create grep filter with all the packages we are interested in + set_fact: + package_filter: "{{ package_filter | default('grep') + ' -e ' + item }}" + with_items: "{{ compute_package_list }}" + - name: Get list of installed packages we are interested in + shell: "rpm -qa | {{ package_filter }} || echo failed_to_get_info" + register: compute_packages + - name: Write compute nodes package list to a file + delegate_to: localhost + copy: + content: "{{ compute_packages.stdout }}" + dest: "{{ dir_path }}{{ current_hostname }}_packages.yaml" + + - name: Get status of services on OCP Compute nodes + shell: "systemctl list-units {{ + compute_service_list | join('.service ') }}.service --no-pager + --type=service --all --no-legend || echo failed_to_get_info" + register: compute_services + - name: Write compute nodes service list to a file + delegate_to: localhost + copy: + content: "{{ compute_services.stdout }}" + dest: "{{ dir_path }}{{ current_hostname }}_services.yaml" + + - name: Get multipath info + shell: "(echo 'Multipath config is following:'; cat /etc/multipath.conf; + echo -e \"{{ separator_line }}\nResults of 'nmultipath -ll' command:\"; + multipath -ll; echo {{ separator_line }}) || echo failed_to_get_info" + register: multipath_info + - name: Save multipath info into a file + delegate_to: localhost + copy: + content: "{{ multipath_info.stdout }}" + dest: "{{ dir_path }}{{ current_hostname }}_multipath_info" + + - name: Get info about devices + shell: '(echo -e "{{ separator_line }}\nlsblk info:"; lsblk; + echo -e "{{ separator_line }}\nPVs info:"; pvs; + echo -e "{{ separator_line }}\nVGs info:"; vgs; + echo -e "{{ separator_line }}\nLVs info:"; lvs; + echo -e "{{ separator_line }}\nll /dev/disk/by-path/ip-*:"; + ll /dev/disk/by-path/ip-*; echo {{ separator_line }}) || + echo failed_to_get_info' + register: lsblk_pvs_vgs_lvs + - name: Save devices info into a file + delegate_to: localhost + copy: + content: "{{ lsblk_pvs_vgs_lvs.stdout }}" + dest: "{{ dir_path }}{{ current_hostname }}_lsblk_pvs_vgs_lvs" + + - name: Get info about mounts + shell: '(echo -e "Mounts on the {{ current_hostname }} node:\n"; mount) || + echo failed_to_get_info' + register: mounts + - name: Save mounts info into a file + delegate_to: localhost + copy: + content: "{{ mounts.stdout }}" + dest: "{{ dir_path }}{{ current_hostname }}_mount" + + - name: Get info about space usage + shell: '(echo -e "File system disk space usage on the {{ + current_hostname }} node:\n"; df -Th) ||echo failed_to_get_df_info' + register: df_info + - name: Save mounts info into a file + delegate_to: localhost + copy: + content: "{{ df_info.stdout }}" + dest: "{{ dir_path }}{{ current_hostname }}_df" + + - name: Read 'dmesg -T' info + shell: "dmesg -T || echo failed_to_get_info" + register: dmesg_info + - name: Save dmesg info into a file + delegate_to: localhost + copy: + content: "{{ dmesg_info.stdout }}" + dest: "{{ dir_path }}{{ current_hostname }}_dmesg" + + - name: Create archive from '/var/log/messages' dir + shell: "tar -czvf var_log_messages.tar.gz /var/log/messages" + retries: 15 + delay: 2 + register: result + until: result is succeeded + ignore_errors: yes + - name: Copy the archive to the localhost + fetch: + src: "var_log_messages.tar.gz" + dest: "{{ dir_path }}{{ current_hostname }}_var_log_messages.tar.gz" + flat: yes + fail_on_missing: yes + ignore_errors: yes diff --git a/deployment/playbooks/get_ocp_info.yaml b/deployment/playbooks/get_ocp_info.yaml index dfce216c..7046ccc6 100644 --- a/deployment/playbooks/get_ocp_info.yaml +++ b/deployment/playbooks/get_ocp_info.yaml @@ -74,7 +74,9 @@ - glusterd - heketi - gluster-blockd + - gluster-block-target - tcmu-runner + - rpcbind heketi_pod_package_list: - gluster - heketi diff --git a/deployment/playbooks/roles/vmware-guest-setup/tasks/main.yaml b/deployment/playbooks/roles/vmware-guest-setup/tasks/main.yaml index c6c8cf27..159eb93e 100644 --- a/deployment/playbooks/roles/vmware-guest-setup/tasks/main.yaml +++ b/deployment/playbooks/roles/vmware-guest-setup/tasks/main.yaml @@ -15,6 +15,11 @@ when: s.stat.exists - block: + - name: Install 'sos' package + yum: + name: sos + state: installed + ignore_errors: yes - name: be sure all pre-req packages are installed yum: name={{item}} state=installed with_items: |