R4.0~62 [vcenter only] : Provisioning fails due to python-urllib3 not being available

On a multi-node vcenter setup (with multiple controllers) running ubuntu 16.04 R4.0 build 62, provisioning fails due to python-urllib3 package not being available on 2 of them.

"2017-10-06 08:39:05,499-INFO-sm_ansible_callback.py:43-append(): fatal: [10.204.217.144] (item=python-urllib3) => {
    "failed": true,
    "invocation": {
        "module_args": {
            "allow_unauthenticated": false,
            "autoremove": null,
            "cache_valid_time": 0,
            "deb": null,
            "default_release": null,
            "dpkg_options": "force-confdef,force-confold",
            "force": false,
            "install_recommends": null,
            "name": "python-urllib3",
            "only_upgrade": false,
            "package": [
                "python-urllib3"
            ],
            "purge": false,
            "state": "latest",
            "update_cache": null,
            "upgrade": null
        }
    },
    "item": "python-urllib3",
    "msg": "No package matching 'python-urllib3' is available"
}"
"2017-10-06 08:39:05,511-INFO-sm_ansible_callback.py:43-append(): fatal: [10.204.217.140] (item=python-urllib3) => {
    "failed": true,
    "invocation": {
        "module_args": {
            "allow_unauthenticated": false,
            "autoremove": null,
            "cache_valid_time": 0,
            "deb": null,
            "default_release": null,
            "dpkg_options": "force-confdef,force-confold",
            "force": false,
            "install_recommends": null,
            "name": "python-urllib3",
            "only_upgrade": false,
            "package": [
                "python-urllib3"
            ],
            "purge": false,
            "state": "latest",
            "update_cache": null,
            "upgrade": null}
    },
    "item": "python-urllib3",
    "msg": "No package matching 'python-urllib3' is available"
}"

<10.204.217.140> (0, '\r\n{"msg": "No package matching \'python-urllib3\' is available", "failed": true, "invocation": {"module_args": {"dpkg_options": "force-confdef,force-confold", "upgrade": null, "force": false, "name": "python-urllib3", "package": ["python-urllib3"], "purge": false, "allow_unauthenticated": false, "state": "latest", "autoremove": null, "update_cache": null, "default_release": null, "only_upgrade": false, "cache_valid_time": 0, "deb": null, "install_recommends": null}}}\r\n', 'Shared connection to 10.204.217.140 closed.\r\n')

<10.204.217.144> (0, '\r\n{"msg": "No package matching \'python-urllib3\' is available", "failed": true, "invocation": {"module_args": {"dpkg_options": "force-confdef,force-confold", "upgrade": null, "force": false, "name": "python-urllib3", "package": ["python-urllib3"], "purge": false, "allow_unauthenticated": false, "state": "latest", "autoremove": null, "update_cache": null, "default_release": null, "only_upgrade": false, "cache_valid_time": 0, "deb": null, "install_recommends": null}}}\r\n', 'Shared connection to 10.204.217.144 closed.\r\n')

Workaround : On manually installing it on the nodes and rerunning provisioning, things worked fine.

Inventory details -
cat cluster-esxi-new.inv
[contrail-analyticsdb]
10.204.217.139
10.204.217.144
10.204.217.140

[contrail-controllers]
10.204.217.139
10.204.217.144
10.204.217.140

[contrail-analytics]
10.204.217.139
10.204.217.144
10.204.217.140

[all:children]
contrail-controllers
contrail-analytics
contrail-analyticsdb
contrail-vc-plugin
contrail-compute

[contrail-compute]
10.204.217.204 esxi_host=10.204.217.147
10.204.217.229 esxi_host=10.204.217.148

[all:vars]
ansible_user=root

docker_insecure_registries=10.204.217.139:5100

analytics_image=10.204.217.139:5100/ubuntu16vcenter62-contrail-analytics:62

vc_plugin_config=[{'username': '<email address hidden>', 'datacenter': 'i27_datacenter11', 'introspect_port': '8234', 'esxtocomputemap': '10.204.217.147:10.204.217.204,10.204.217.148:10.204.217.229,', 'ipfabricpg': 'contrail-fab-pg', 'mode': 'vcenter-only', 'vc_url': 'https://10.204.217.246/sdk', 'password': 'Contrail123!', 'dvs': 'vm_dvs2'}]

cluster_id=cluster-esxi-new

docker_network_bridge=False

rabbitmq_user=guest

ansible_log_path=/var/log/contrail-server-manager/debug.log

vcenterplugin_image=10.204.217.139:5100/ubuntu16vcenter62-contrail-vcenter-plugin:62

contrail_compute_mode=bare_metal

global_config={'analytics_ip': '10.204.217.139', 'controller_ip': '10.204.217.139', 'config_ip': '10.204.217.139'}

esxi_hosts=[{'username': 'root', 'datacenter': 'i27_datacenter11', 'std_switch_list': [{'pg_name': 'contrail-fab-pg', 'switch_name': 'vSwitch0'}], 'name': '10.204.217.147', 'contrail_vm': {'control_data_switch': 'vSwitch0', 'vmdk': '/root/vmdk_new/vmdk.tar', 'mode': 'vcenter', 'mgmt_pg': 'contrail-fab-pg', 'networks': [{'mac': '00:77:56:cd:bc:ba', 'sw_type': 'standard', 'role': 'mgmt', 'switch_name': 'vSwitch0', 'pg': 'contrail-fab-pg'}]}, 'cluster': 'i27_cluster11', 'id_in_sm': 'ContrailVM-i27_datacenter11-10.204.217.147', 'datastore': 'nodei35-ds', 'password': 'c0ntrail123', 'validate_certs': False, 'vcenter_server': {'username': '<email address hidden>', 'datacentername': 'i27_datacenter11', 'password': 'Contrail123!', 'vcenter_compute_ip': None, 'hostname': '10.204.217.246', 'dv_port_group_mgmt': {'dv_portgroup_name': u'', 'number_of_ports': u'', 'uplink': u''}, 'dv_switch_control_data': {'dv_switch_name': u''}, 'dv_switch_mgmt': {'dv_switch_name': u''}, 'clusternames': ['i27_cluster11', 'i27_cluster12'], 'dv_switch': {'dv_switch_name': 'vm_dvs2'}, 'dv_port_group_control_data': {'dv_portgroup_name': u'', 'number_of_ports': u'', 'uplink': u''}, 'validate_certs': False, 'dv_port_group': {'dv_portgroup_name': 'vm_dvs_pg2', 'number_of_ports': '3'}}}, {'username': 'root', 'datacenter': 'i27_datacenter11', 'std_switch_list': [{'pg_name': 'contrail-fab-pg', 'switch_name': 'vSwitch0'}], 'name': '10.204.217.148', 'contrail_vm': {'control_data_switch': 'vSwitch0', 'vmdk': '/root/vmdk_new/vmdk.tar', 'mode': 'vcenter', 'mgmt_pg': 'contrail-fab-pg', 'networks': [{'mac': '00:77:56:aa:ba:ba', 'sw_type': 'standard', 'role': 'mgmt', 'switch_name': 'vSwitch0', 'pg': 'contrail-fab-pg'}]}, 'cluster': 'i27_cluster11', 'id_in_sm': 'ContrailVM-i27_datacenter11-10.204.217.148', 'datastore': 'nodei36-ds', 'password': 'c0ntrail123', 'validate_certs': False, 'vcenter_server': {'username': '<email address hidden>', 'datacentername': 'i27_datacenter11', 'password': 'Contrail123!', 'vcenter_compute_ip': None, 'hostname': '10.204.217.246', 'dv_port_group_mgmt': {'dv_portgroup_name': u'', 'number_of_ports': u'', 'uplink': u''}, 'dv_switch_control_data': {'dv_switch_name': u''}, 'dv_switch_mgmt': {'dv_switch_name': u''}, 'clusternames': ['i27_cluster11', 'i27_cluster12'], 'dv_switch': {'dv_switch_name': 'vm_dvs2'}, 'dv_port_group_control_data': {'dv_portgroup_name': u'', 'number_of_ports': u'', 'uplink': u''}, 'validate_certs': False, 'dv_port_group': {'dv_portgroup_name': 'vm_dvs_pg2', 'number_of_ports': '3'}}}]

contrail_image_id=ubuntu16vcenter62

docker_registry=10.204.217.139:5100

ansible_srvr_port=9003

ansible_playbook=/opt/contrail/server_manager/ansible/playbooks/ubuntu16vcenter62/playbooks/site.yml

config_file=/opt/contrail/server_manager/sm-config.ini

enable_lbaas=True

controller_image=10.204.217.139:5100/ubuntu16vcenter62-contrail-controller:62

docker_package_name=docker-engine

ssl_certs_src_dir=/etc/contrail_smgr/puppet/ssl

ansible_srvr_ip=10.204.217.139

analyticsdb_image=10.204.217.139:5100/ubuntu16vcenter62-contrail-analyticsdb:62

docker_registry_insecure=True

docker_install_method=package

vcenter_servers=[{'server1_i27_datacenter11_0': {'username': '<email address hidden>', 'datacentername': 'i27_datacenter11', 'password': 'Contrail123!', 'vcenter_compute_ip': None, 'hostname': '10.204.217.246', 'dv_port_group_mgmt': {'dv_portgroup_name': u'', 'number_of_ports': u'', 'uplink': u''}, 'dv_switch_control_data': {'dv_switch_name': u''}, 'dv_switch_mgmt': {'dv_switch_name': u''}, 'clusternames': ['i27_cluster11', 'i27_cluster12'], 'dv_switch': {'dv_switch_name': 'vm_dvs2'}, 'dv_port_group_control_data': {'dv_portgroup_name': u'', 'number_of_ports': u'', 'uplink': u''}, 'validate_certs': False, 'dv_port_group': {'dv_portgroup_name': 'vm_dvs_pg2', 'number_of_ports': '3'}}}]

tor_ssl_certs_src_dir=/etc/contrail_smgr/puppet/ssl/tor

contrail_apt_repo=[arch=amd64] http://10.204.217.139/contrail/repo/ubuntu16vcenter62 contrail main

cloud_orchestrator=vcenter

rabbitmq_password=guest

ansible_password=c0ntrail123

[contrail-vc-plugin]
10.204.217.139
10.204.217.144
10.204.217.140