[Deployment] Fix OCP3.6 and 3.7 installation

By doing following changes:
- Fix the deployment logic using separate set of vars for
  installation of OCP3.6 and 3.7, which were broken by the too new
  config options, which started being supported only since OCP3.9.
- Add waiters for oc services after nodes reboot.
- Do not update docker for OCP3.6 and 3.7 versions. Because,
  docker version should be fixed for it.
- Add all cluster nodes to the 'OSEv3' Ansible host group,
  because it is required in some places.
- Do not specify 'openshift_use_crio' option for all OCP versions
  except OCP3.11 for 2 reasons:
-- It is supported only for this OCP version
-- Other versions try to run some CRI-O plays just having this option
   defined. In other words - avoid bugs.

Change-Id: I7ab3db30a828f0e965c907282a54b2ae9aead00a

12 files changed, 209 insertions, 36 deletions

diff --git a/deployment/add-node.py b/deployment/add-node.py
index 8ce1a6c5..2c6e0281 100755
--- a/deployment/add-node.py
+++ b/deployment/add-node.py
@@ -563,15 +563,16 @@ class VMWareAddNode(object):
         if self.docker_image_tag:
             playbook_vars_dict['openshift_image_tag'] = self.docker_image_tag
 
-        if self.openshift_use_crio:
-            playbook_vars_dict['openshift_use_crio'] = self.openshift_use_crio
-            playbook_vars_dict['openshift_use_crio_only'] = (
-                self.openshift_use_crio)
-            playbook_vars_dict['openshift_crio_enable_docker_gc'] = (
-                self.openshift_use_crio)
-        else:
-            playbook_vars_dict['openshift_use_crio'] = 'false'
-
+        if self.openshift_vers == 'v3_11':
+            if self.openshift_use_crio:
+                playbook_vars_dict['openshift_use_crio'] = (
+                    self.openshift_use_crio)
+                playbook_vars_dict['openshift_use_crio_only'] = (
+                    self.openshift_use_crio)
+                playbook_vars_dict['openshift_crio_enable_docker_gc'] = (
+                    self.openshift_use_crio)
+            else:
+                playbook_vars_dict['openshift_use_crio'] = 'false'
         if self.openshift_vers in ("v3_6", "v3_7", "v3_9"):
             for key in ('image', 'version',
                         'block_image', 'block_version',
diff --git a/deployment/ocp-on-vmware.py b/deployment/ocp-on-vmware.py
index a6089459..c54a9d17 100755
--- a/deployment/ocp-on-vmware.py
+++ b/deployment/ocp-on-vmware.py
@@ -441,14 +441,16 @@ class OCPOnVMWare(object):
         if self.web_console_install:
             playbook_vars_dict['openshift_web_console_install'] = (
                 self.web_console_install)
-        if self.openshift_use_crio:
-            playbook_vars_dict['openshift_use_crio'] = self.openshift_use_crio
-            playbook_vars_dict['openshift_use_crio_only'] = (
-                self.openshift_use_crio)
-            playbook_vars_dict['openshift_crio_enable_docker_gc'] = (
-                self.openshift_use_crio)
-        else:
-            playbook_vars_dict['openshift_use_crio'] = 'false'
+        if self.openshift_vers == 'v3_11':
+            if self.openshift_use_crio:
+                playbook_vars_dict['openshift_use_crio'] = (
+                    self.openshift_use_crio)
+                playbook_vars_dict['openshift_use_crio_only'] = (
+                    self.openshift_use_crio)
+                playbook_vars_dict['openshift_crio_enable_docker_gc'] = (
+                    self.openshift_use_crio)
+            else:
+                playbook_vars_dict['openshift_use_crio'] = 'false'
         if self.openshift_vers in ('v3_6', 'v3_7'):
             playbook_vars_dict['docker_version'] = '1.12.6'
 
diff --git a/deployment/playbooks/add-node.yaml b/deployment/playbooks/add-node.yaml
index 2f51bd50..51971644 100644
--- a/deployment/playbooks/add-node.yaml
+++ b/deployment/playbooks/add-node.yaml
@@ -130,3 +130,13 @@
   become: no
   roles:
   - yum-update-and-reboot
+
+- hosts: single_master
+  gather_facts: no
+  tasks:
+  - name: Make sure oc client is responsive
+    command: oc status
+    retries: 120
+    delay: 5
+    register: oc_status_result
+    until: oc_status_result is succeeded
diff --git a/deployment/playbooks/cns-node-setup.yaml b/deployment/playbooks/cns-node-setup.yaml
index 59b33720..c5293619 100644
--- a/deployment/playbooks/cns-node-setup.yaml
+++ b/deployment/playbooks/cns-node-setup.yaml
@@ -119,3 +119,13 @@
   become: no
   roles:
   - yum-update-and-reboot
+
+- hosts: single_master
+  gather_facts: no
+  tasks:
+  - name: Make sure oc client is responsive
+    command: oc status
+    retries: 120
+    delay: 5
+    register: oc_status_result
+    until: oc_status_result is succeeded
diff --git a/deployment/playbooks/cns-setup.yaml b/deployment/playbooks/cns-setup.yaml
index e51bf1c0..5147749b 100644
--- a/deployment/playbooks/cns-setup.yaml
+++ b/deployment/playbooks/cns-setup.yaml
@@ -38,6 +38,7 @@
       name: glusterfs-{{ glusterfs_name }}
       content:
         allowVolumeExpansion: true
+    when: openshift_vers not in ['v3_6', 'v3_7']
 
 - name: Get IP address of the node with router
   hosts: single_master
diff --git a/deployment/playbooks/crs-node-setup.yaml b/deployment/playbooks/crs-node-setup.yaml
index 21802a5a..c762f48a 100644
--- a/deployment/playbooks/crs-node-setup.yaml
+++ b/deployment/playbooks/crs-node-setup.yaml
@@ -111,3 +111,13 @@
   become: no
   roles:
   - yum-update-and-reboot
+
+- hosts: single_master
+  gather_facts: no
+  tasks:
+  - name: Make sure oc client is responsive
+    command: oc status
+    retries: 120
+    delay: 5
+    register: oc_status_result
+    until: oc_status_result is succeeded
diff --git a/deployment/playbooks/crs-setup.yaml b/deployment/playbooks/crs-setup.yaml
index ea84a27e..a4bbad3c 100644
--- a/deployment/playbooks/crs-setup.yaml
+++ b/deployment/playbooks/crs-setup.yaml
@@ -83,6 +83,7 @@
       name: glusterfs-{{ glusterfs_name }}
       content:
         allowVolumeExpansion: true
+    when: openshift_vers not in ['v3_6', 'v3_7']
 
 - name: Get IP address of the node with router
   hosts: single_master
diff --git a/deployment/playbooks/ocp-install.yaml b/deployment/playbooks/ocp-install.yaml
index 51e08721..43b92c2f 100644
--- a/deployment/playbooks/ocp-install.yaml
+++ b/deployment/playbooks/ocp-install.yaml
@@ -78,7 +78,133 @@
       - key: kubeletArguments.runtime-request-timeout
         value: ["10m"]
 
-- name: call openshift includes for installer
+- name: Call openshift includes for OCP3.6 and OCP3.7 installer
+  include: "{{
+      lookup('env', 'VIRTUAL_ENV')
+  }}/usr/share/ansible/openshift-ansible/playbooks/{{
+      (openshift_vers in ['v3_6', 'v3_7']) |
+          ternary('byo/config.yml', 'deploy_cluster.yml')
+  }}"
+  vars:
+    openshift_release: "v3.{{ openshift_vers.split('_')[-1] }}"
+    debug_level: 2
+    console_port: 8443
+    openshift_debug_level: "{{ debug_level }}"
+    openshift_node_debug_level: "{{ node_debug_level | default(debug_level, true) }}"
+    # NOTE(vponomar): following two can be changed to "true" when
+    # https://github.com/openshift/openshift-ansible/issues/6086 is fixed
+    openshift_enable_service_catalog: false
+    template_service_broker_install: false
+    osm_controller_args:
+      cloud-provider:
+      - "vsphere"
+      cloud-config:
+      - "/etc/origin/cloudprovider/vsphere.conf"
+    osm_api_server_args:
+      cloud-provider:
+      - "vsphere"
+      cloud-config:
+      - "/etc/origin/cloudprovider/vsphere.conf"
+    openshift_master_debug_level: "{{ master_debug_level | default(debug_level, true) }}"
+    openshift_master_access_token_max_seconds: 2419200
+    openshift_hosted_router_replicas: 1
+    openshift_hosted_registry_replicas: 1
+    openshift_master_api_port: "{{ console_port }}"
+    openshift_master_console_port: "{{ console_port }}"
+    openshift_node_local_quota_per_fsgroup: 512Mi
+    osm_cluster_network_cidr: 172.16.0.0/16
+    osm_use_cockpit: false
+    osm_default_node_selector: "role=compute"
+    openshift_registry_selector: "role=compute"
+    openshift_override_hostname_check: true
+    openshift_router_selector: "role=compute"
+    openshift_master_cluster_method: native
+    openshift_cloudprovider_kind: vsphere
+    openshift_cloudprovider_vsphere_host: "{{ vcenter_host }}"
+    openshift_cloudprovider_vsphere_username: "{{ vcenter_username }}"
+    openshift_cloudprovider_vsphere_password: "{{ vcenter_password }}"
+    openshift_cloudprovider_vsphere_datacenter: "{{ vcenter_datacenter }}"
+    openshift_cloudprovider_vsphere_datastore: "{{ vcenter_datastore }}"
+    openshift_cloudprovider_vsphere_folder: "{{ vcenter_folder }}"
+    wildcard_zone: "{{ app_dns_prefix }}.{{ dns_zone }}"
+    osm_default_subdomain: "{{ wildcard_zone }}"
+    openshift_master_default_subdomain: "{{osm_default_subdomain}}"
+    deployment_type: "{{ deployment_type }}"
+    load_balancer_hostname: "{{ lb_host }}"
+    openshift_master_cluster_hostname: "{{ load_balancer_hostname }}"
+    openshift_master_cluster_public_hostname: "{{ load_balancer_hostname }}"
+    os_sdn_network_plugin_name: "{{ openshift_sdn }}"
+    openshift_master_identity_providers:
+    - name: 'allow_all'
+      kind: 'AllowAllPasswordIdentityProvider'
+      login: True
+      challenge: True
+    openshift_crio_docker_gc_node_selector:
+      runtime: crio
+    # 'openshift_node_groups' is required for OCP3.10
+    openshift_node_groups:
+    - name: node-config-master
+      labels:
+      - 'node-role.kubernetes.io/master=true'
+      - 'role=master'
+      edits: []
+    - name: node-config-master-crio
+      labels:
+      - 'node-role.kubernetes.io/master=true'
+      - 'role=master'
+      - 'runtime=cri-o'
+      edits:
+      - key: kubeletArguments.container-runtime
+        value: ["remote"]
+      - key: kubeletArguments.container-runtime-endpoint
+        value: ["/var/run/crio/crio.sock"]
+      - key: kubeletArguments.image-service-endpoint
+        value: ["/var/run/crio/crio.sock"]
+      - key: kubeletArguments.runtime-request-timeout
+        value: ["10m"]
+    - name: node-config-compute
+      labels:
+      - 'node-role.kubernetes.io/compute=true'
+      - 'node-role.kubernetes.io/infra=true'
+      - 'role=compute'
+      edits: []
+    - name: node-config-compute-crio
+      labels:
+      - 'node-role.kubernetes.io/compute=true'
+      - 'node-role.kubernetes.io/infra=true'
+      - 'role=compute'
+      - 'runtime=cri-o'
+      edits:
+      - key: kubeletArguments.container-runtime
+        value: ["remote"]
+      - key: kubeletArguments.container-runtime-endpoint
+        value: ["/var/run/crio/crio.sock"]
+      - key: kubeletArguments.image-service-endpoint
+        value: ["/var/run/crio/crio.sock"]
+      - key: kubeletArguments.runtime-request-timeout
+        value: ["10m"]
+    - name: node-config-storage
+      labels:
+      - 'node-role.kubernetes.io/storage=true'
+      - 'role=storage'
+      edits: []
+    - name: node-config-storage-crio
+      labels:
+      - 'node-role.kubernetes.io/storage=true'
+      - 'role=storage'
+      - 'runtime=cri-o'
+      edits:
+      - key: kubeletArguments.container-runtime
+        value: ["remote"]
+      - key: kubeletArguments.container-runtime-endpoint
+        value: ["/var/run/crio/crio.sock"]
+      - key: kubeletArguments.image-service-endpoint
+        value: ["/var/run/crio/crio.sock"]
+      - key: kubeletArguments.runtime-request-timeout
+        value: ["10m"]
+  when: openshift_vers in ['v3_6', 'v3_7']
+
+- name: "Call openshift includes for OCP3.9+ installer"
   include: "{{
       lookup('env', 'VIRTUAL_ENV')
   }}/usr/share/ansible/openshift-ansible/playbooks/{{
@@ -212,6 +338,7 @@
         value: ["/var/run/crio/crio.sock"]
       - key: kubeletArguments.runtime-request-timeout
         value: ["10m"]
+  when: openshift_vers not in ['v3_6', 'v3_7']
 
 - hosts: allnodes
   gather_facts: no
@@ -226,3 +353,13 @@
   gather_facts: no
   roles:
   - yum-update-and-reboot
+
+- hosts: single_master
+  gather_facts: no
+  tasks:
+  - name: Make sure oc client is responsive
+    command: oc status
+    retries: 120
+    delay: 5
+    register: oc_status_result
+    until: oc_status_result is succeeded
diff --git a/deployment/playbooks/roles/docker-storage-setup/tasks/main.yaml b/deployment/playbooks/roles/docker-storage-setup/tasks/main.yaml
index 3736797c..70c04802 100644
--- a/deployment/playbooks/roles/docker-storage-setup/tasks/main.yaml
+++ b/deployment/playbooks/roles/docker-storage-setup/tasks/main.yaml
@@ -3,7 +3,7 @@
   file:
     path: /etc/sysconfig/docker-storage
     state: absent
-  when: not (openshift_use_crio | bool)
+  when: not (openshift_use_crio | default(false) | bool)
 
 - block:
     - name: create the docker-storage config file
@@ -16,7 +16,7 @@
   when:
     - ansible_distribution_version | version_compare('7.4', '>=')
     - ansible_distribution == "RedHat"
-    - not (openshift_use_crio | bool)
+    - not (openshift_use_crio | default(false) | bool)
 
 - block:
     - name: create the docker-storage-setup config file
@@ -29,11 +29,11 @@
   when:
     - ansible_distribution_version | version_compare('7.4', '<')
     - ansible_distribution == "RedHat"
-    - not (openshift_use_crio | bool)
+    - not (openshift_use_crio | default(false) | bool)
 
 - name: start docker
   service:
     name: docker
     state: started
     enabled: true
-  when: not (openshift_use_crio | bool)
+  when: not (openshift_use_crio | default(false) | bool)
diff --git a/deployment/playbooks/roles/instance-groups/tasks/main.yaml b/deployment/playbooks/roles/instance-groups/tasks/main.yaml
index 5607f4c7..f0f3c0f9 100644
--- a/deployment/playbooks/roles/instance-groups/tasks/main.yaml
+++ b/deployment/playbooks/roles/instance-groups/tasks/main.yaml
@@ -19,16 +19,16 @@
 - name: Add masters to requisite groups
   add_host:
     name: "{{ hostvars[item].inventory_hostname }}"
-    groups: allnodes, masters, etcd, nodes, cluster_hosts, master
+    groups: allnodes, masters, etcd, nodes, cluster_hosts, master, OSEv3
     openshift_node_group_name: "node-config-master{{
-      (openshift_use_crio | bool) | ternary('-crio', '') }}"
+      (openshift_use_crio | default(false) | bool) | ternary('-crio', '') }}"
   with_items: "{{ groups[cluster_id + '-master'] }}"
   when:
     - "openshift_vers not in ['v3_6', 'v3_7', 'v3_9', 'v3_10']"
 - name: Add masters to requisite groups
   add_host:
     name: "{{ hostvars[item].inventory_hostname }}"
-    groups: allnodes, masters, etcd, nodes, cluster_hosts, master
+    groups: allnodes, masters, etcd, nodes, cluster_hosts, master, OSEv3
     openshift_node_group_name: "node-config-master"
     openshift_node_labels:
       role: master
@@ -42,7 +42,7 @@
     name: "{{ hostvars[item].inventory_hostname }}"
     groups: single_master
     openshift_node_group_name: "node-config-master{{
-      (openshift_use_crio | bool) | ternary('-crio', '') }}"
+      (openshift_use_crio | default(false) | bool) | ternary('-crio', '') }}"
   with_items: "{{ groups[cluster_id + '-master'][0] }}"
   when:
     - "openshift_vers not in ['v3_6', 'v3_7', 'v3_9', 'v3_10']"
@@ -61,16 +61,16 @@
 - name: Add compute instances to host group
   add_host:
     name: "{{ hostvars[item].inventory_hostname }}"
-    groups: allnodes, nodes, cluster_hosts, schedulable_nodes, compute
+    groups: allnodes, nodes, cluster_hosts, schedulable_nodes, compute, OSEv3
     openshift_node_group_name: "node-config-compute{{
-      (openshift_use_crio | bool) | ternary('-crio', '') }}"
+      (openshift_use_crio | default(false) | bool) | ternary('-crio', '') }}"
   with_items: "{{ groups[cluster_id + '-compute'] }}"
   when:
     - "openshift_vers not in ['v3_6', 'v3_7', 'v3_9', 'v3_10']"
 - name: Add compute instances to host group
   add_host:
     name: "{{ hostvars[item].inventory_hostname }}"
-    groups: allnodes, nodes, cluster_hosts, schedulable_nodes, compute
+    groups: allnodes, nodes, cluster_hosts, schedulable_nodes, compute, OSEv3
     openshift_node_group_name: "node-config-compute"
     openshift_node_labels:
       role: compute
@@ -85,7 +85,7 @@
     name: "{{ hostvars[item].inventory_hostname }}"
     groups: allnodes, new_nodes
     openshift_node_group_name: "node-config-compute{{
-      (openshift_use_crio | bool) | ternary('-crio', '') }}"
+      (openshift_use_crio | default(false) | bool) | ternary('-crio', '') }}"
   with_items: "{{ groups.tag_provision_node | default([]) }}"
   when:
     - add_node is defined
@@ -107,15 +107,15 @@
 - name: Add cns instances to allnodes
   add_host:
     name: "{{ hostvars[item].inventory_hostname }}"
-    groups: allnodes
+    groups: allnodes, OSEv3
     openshift_node_group_name: "node-config-storage{{
-      (openshift_use_crio | bool) | ternary('-crio', '') }}"
+      (openshift_use_crio | default(false) | bool) | ternary('-crio', '') }}"
   with_items: "{{ groups[cluster_id + '-storage'] | default([]) }}"
 
 - name: Add crs instances to allnodes
   add_host:
     name: "{{ hostvars[item].inventory_hostname }}"
-    groups: allnodes
+    groups: allnodes, OSEv3
     openshift_node_group_name: "node-config-storage"
   with_items: "{{ groups[cluster_id + '-crs'] | default([]) }}"
 
@@ -124,7 +124,7 @@
     name: "{{ hostvars[item].inventory_hostname }}"
     groups: nodes, cluster_hosts, schedulable_nodes, storage
     openshift_node_group_name: "node-config-storage{{
-      (openshift_use_crio | bool) | ternary('-crio', '') }}"
+      (openshift_use_crio | default(false) | bool) | ternary('-crio', '') }}"
   with_items: "{{ groups[cluster_id + '-storage'] }}"
   when:
     - "'cns' in container_storage and add_node is defined and 'storage' in node_type"
diff --git a/deployment/playbooks/roles/vmware-guest-setup/tasks/main.yaml b/deployment/playbooks/roles/vmware-guest-setup/tasks/main.yaml
index 159eb93e..e640b861 100644
--- a/deployment/playbooks/roles/vmware-guest-setup/tasks/main.yaml
+++ b/deployment/playbooks/roles/vmware-guest-setup/tasks/main.yaml
@@ -45,7 +45,7 @@
       delay: 5
       register: result
       until: result is succeeded
-      when: not (openshift_use_crio | bool)
+      when: not (openshift_use_crio | default(false) | bool)
 
     - name: be sure openvmtools is running and enabled
       service: name=vmtoolsd state=started enabled=yes
diff --git a/deployment/playbooks/roles/yum-update-and-reboot/tasks/main.yaml b/deployment/playbooks/roles/yum-update-and-reboot/tasks/main.yaml
index c49bcc9d..826ff498 100644
--- a/deployment/playbooks/roles/yum-update-and-reboot/tasks/main.yaml
+++ b/deployment/playbooks/roles/yum-update-and-reboot/tasks/main.yaml
@@ -10,7 +10,8 @@
     when: "(hostnames_for_reboot is not defined) or hostnames_for_reboot | length < 1"
 
   - name: Run yum_update command
-    command: "yum update -y"
+    command: "yum update -y {{ (openshift_vers in ['v3_6', 'v3_7']) |
+      ternary('--exclude=*docker*', '') }}"
     delegate_to: "{{ item }}"
     with_items: "{{ hostnames_for_reboot }}"