R5.0.1 - multicloud :: OnPrem :: OnPrem controller to OnPrem GW unreachable after provisioning.

Setup:-

OnPrem:-
nodec10 10.204.217.176 192.168.100.11/24 GW
nodec33 10.204.217.168 192.168.100.12/24 Controller
nodec28 10.204.217.13 192.168.100.13/24 Compute
nodeg37 10.204.217.77 192.168.100.14/24 Compute

**********************************************************************************************

**********************************************************************************************

AZURE Cloud:-
rg-gw-1 192.169.100.5
rg-compute-1 192.169.100.4

**********************************************************************************************

**********************************************************************************************

Provisioning Machine:-
a2s42 - 10.84.7.42

Provisioning dir:- - /root/ritam/AZURE/OnPrem/PATCHED/contrail-multi-cloud

**********************************************************************************************

**********************************************************************************************
Git pull of contrail-multicloud repo:-
root@a2s42:~/ritam/AZURE/OnPrem/PATCHED/contrail-multi-cloud# git branch
* master
root@a2s42:~/ritam/AZURE/OnPrem/PATCHED/contrail-multi-cloud# git log
commit e7b7bc91efb7ea70cba5258a6876238b839b39b4
Author: Damian Rakowski <email address hidden>
Date: Thu Aug 9 15:13:51 2018 +0200

    Simplified docker daemon config file generation (#194)

commit 12e8f219e6867c1f3ef8d7b791467f020fe52046
Merge: ce8b127 bd657d4
Author: Sanju Abraham <email address hidden>
Date: Thu Aug 9 03:22:35 2018 -0700

    Merge pull request #191 from Juniper/muli_subnet

    Muli subnet

commit bd657d4dcda5238f5d99e87ac23cf68153c2846b
Author: pawelkopka <email address hidden>
Date: Wed Aug 8 16:59:40 2018 +0200

    allow to run

    Modified multicloud_contrail_aap.py to parse the IP address with cidr

commit ce8b1274a17b47115c2dfee7bed35e730460eddb
Author: pawelkopka <email address hidden>
Date: Thu Aug 9 11:44:20 2018 +0200

    fix docker non-existing deamon.json (#193)

**********************************************************************************************

**********************************************************************************************
TOPOLOGY FILE:-

- provider: OnPrem
  organization: Juniper
  project: multicloud
  instances:
    - name: nodec10
      public_ip: 10.204.217.176
      private_ip: 192.168.100.11
      private_subnet: 192.168.100.0/24
      vip: 192.168.100.110
      roles:
        - gateway
      provision: true
      username: root
      password: c0ntrail123
      protocols_mode:
        - ssl_client
      interface: enp1s0f1
    - name: nodec33
      public_ip: 10.204.217.168
      private_ip: 192.168.100.12
      private_subnet: 192.168.100.0/24
      roles:
        - controller
        - k8s_master
      provision: true
      username: root
      password: c0ntrail123
      interface: enp1s0f1
    - name: nodec28
      public_ip: 10.204.217.13
      private_ip: 192.168.100.13
      private_subnet: 192.168.100.0/24
      roles:
        - compute_node
      provision: true
      username: root
      password: c0ntrail123
      interface: enp1s0f1
    - name: nodeg37
      public_ip: 10.204.217.77
      private_ip: 192.168.100.14
      private_subnet: 192.168.100.0/24
      roles:
        - compute_node
      provision: true
      username: root
      password: c0ntrail123
      interface: enp1s0f1

- provider: azure
  organization: Juniper
  project: multicloud
  regions:
    - name: SouthIndia
      resource_group: contrail-test-south-india
      clouds:
        - name: rg-vpc-1
          cidr_block: 192.169.100.0/24
          subnets:
            - name: rg-subnet-1
              cidr_block: 192.169.100.0/25
              security_group: rg-sg-1
          security_groups:
            - name: rg-sg-1
              rules:
                - name: rg-all_in
                  direction: inbound
                - name: rg-all_out
                  direction: outbound
          instances:
            - name: rg-gw-1
              provision: true
              username: ubuntu
              os: ubuntu16
              instance_type: Standard_F2
              subnets: rg-subnet-1
              interface: eth1
              roles:
               - gateway
              protocols_mode:
                - ssl_server
            - name: rg-compute-1
              provision: true
              username: ubuntu
              os: ubuntu16
              instance_type: Standard_F2
              subnets: rg-subnet-1
              interface: eth0
              roles:
               - compute_node

**********************************************************************************************

**********************************************************************************************

Logs on the Onperm controller:-

My OnPrem controller is on nodec33 and gw on nodec10. The routes seem to be fine on nodec33. I added a route to reach the gw. but ping to the OnPrem gw (nodec10) and the public cloud (Azure) 192.169.100 n/w is failing. This was supposed to be fixed through the commits made for https://bugs.launchpad.net/juniperopenstack/+bug/1786120.

root@nodec33:~# ip route
default via 10.204.217.254 dev enp1s0f0
10.204.217.0/24 dev enp1s0f0 proto kernel scope link src 10.204.217.168
172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1 linkdown
192.168.100.0/24 dev enp1s0f1 proto kernel scope link src 192.168.100.12
192.169.100.0/24 via 192.168.100.11 dev enp1s0f1
root@nodec33:~# ping -c 3 192.169.100.4
PING 192.169.100.4 (192.169.100.4) 56(84) bytes of data.

--- 192.169.100.4 ping statistics ---
3 packets transmitted, 0 received, 100% packet loss, time 2017ms

root@nodec33:~# ping -c 3 192.169.100.11
PING 192.169.100.11 (192.169.100.11) 56(84) bytes of data.

--- 192.169.100.11 ping statistics ---
3 packets transmitted, 0 received, 100% packet loss, time 1999ms

root@nodec33:~#

**********************************************************************************************

**********************************************************************************************

root@nodec10:~# ip route
default via 10.204.217.254 dev enp1s0f0
10.204.217.0/24 dev enp1s0f0 proto kernel scope link src 10.204.217.176
100.64.0.2 dev tap0 proto kernel scope link src 100.64.0.1
100.65.0.2 via 100.64.0.2 dev tap0 proto bird
169.254.0.1 dev vhost0 proto 109 scope link
192.168.100.0/24 dev vhost0 proto kernel scope link src 192.168.100.11
192.169.100.0/26 via 100.64.0.2 dev tap0 proto bird src 192.168.100.11
198.18.0.0/24 dev docker0 proto kernel scope link src 198.18.0.1 linkdown
root@nodec10:~#

**********************************************************************************************

**********************************************************************************************