updating sanity phy pod tests

Change-Id: Iba0b46e9152f0161b4984bc547f65c61f06f6f95
(cherry picked from commit 2f567a4174c8ef2781ef88818517477b959cc48b)
diff --git a/src/test/cord-api/Framework/utils/onosUtils.py b/src/test/cord-api/Framework/utils/onosUtils.py
new file mode 100644
index 0000000..06420a6
--- /dev/null
+++ b/src/test/cord-api/Framework/utils/onosUtils.py
@@ -0,0 +1,45 @@
+# Copyright 2017-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import paramiko
+
+def onos_command_execute(host, portNum, cmd, user='karaf', passwd='karaf'):
+    """
+    :param host: onos-cord or onos-fabric
+    :param portNum: 8102 or 8101
+    :param cmd: command to execute
+    :param user: onos/karaf
+    :param passwd: onos/karaf
+    :return: output of command executed inside onos karaf (shell)
+    """
+    try:
+        client = paramiko.SSHClient()
+        client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        client.connect(host, port=int(portNum), username=user, password=passwd)
+        stdin, stdout, stderr = client.exec_command(cmd)
+        while not stdout.channel.exit_status_ready():
+            if stdout.channel.recv_ready():
+                return stdout.read()
+    finally:
+        client.close()
+
+def get_compute_node_ip(compute_node):
+    """
+    :param compute_node: one compute node information from output of 'cordvtn-nodes'
+    :return: data_ip of that compute node
+    """
+    for line in compute_node.splitlines():
+        columns = line.split()
+        if len(columns) >= 2:
+            return columns[2].split("/")[0]
\ No newline at end of file
diff --git a/src/test/cord-api/Framework/utils/utils.robot b/src/test/cord-api/Framework/utils/utils.robot
index 2bf46f8..db559e5 100644
--- a/src/test/cord-api/Framework/utils/utils.robot
+++ b/src/test/cord-api/Framework/utils/utils.robot
@@ -89,10 +89,18 @@
     ${output}=    Run    ${cmd}
     [Return]    ${output}
 
+Execute ONOS Command
+    [Arguments]    ${onos}    ${port}    ${cmd}    ${user}=karaf    ${pass}=karaf
+    ${conn_id}=    SSHLibrary.Open Connection    ${onos}    port=${port}    prompt=onos>    timeout=300s
+    SSHLibrary.Login    ${user}    ${pass}
+    ${output}=    SSHLibrary.Execute Command    ${cmd}
+    SSHLibrary.Close Connection
+    [Return]    ${output}
+
 Get Docker Container ID
-    [Arguments]    ${system}    ${container_name}    ${user}=${USER}    ${password}=${PASSWD}
-    [Documentation]    Retrieves the id of the requested docker container running inside given ${HOST}
-    ${container_id}=    Execute Command on CIAB Server in Specific VM    ${system}    head1    docker ps | grep ${container_name} | awk '{print $1}'    ${user}    ${password}
+    [Arguments]    ${container_name}
+    [Documentation]    Retrieves the id of the requested docker container running inside headnode
+    ${container_id}=     Run    docker ps | grep ${container_name} | awk '{print $1}'
     Log    ${container_id}
     [Return]    ${container_id}
 
@@ -114,8 +122,16 @@
 Remove Value From List
     [Arguments]    ${list}    ${val}
     ${length}=    Get Length    ${list}
-    : FOR    ${INDEX}    IN RANGE    0    ${length}-1
+    : FOR    ${INDEX}    IN RANGE    0    ${length}
     \    Log    ${list[${INDEX}]}
     \    ${value}=    Get Dictionary Values    ${list[${INDEX}]}
     \    Log    ${value[0]}
     \    Run Keyword If    '${value[0]}' == '${val}'    Remove From List    ${list}    ${INDEX}
+    \    Run Keyword If    '${value[0]}' == '${val}'    Exit For Loop
+
+Test Ping
+    [Arguments]    ${interface}    ${host}
+    [Documentation]    Ping hosts to check connectivity
+    ${result}=   Run    ping -I ${interface} -c 5 ${host}
+    Should Contain    ${result}    64 bytes
+    Should Not Contain    ${result}    Destination Host Unreachable
\ No newline at end of file
diff --git a/src/test/cord-api/Tests/Ch_DefaultServiceCheck.txt b/src/test/cord-api/Tests/Ch_DefaultServiceCheck.txt
index 2613475..6b73369 100644
--- a/src/test/cord-api/Tests/Ch_DefaultServiceCheck.txt
+++ b/src/test/cord-api/Tests/Ch_DefaultServiceCheck.txt
@@ -29,6 +29,7 @@
     ##need to remove openstack and onos from xos_services list in each manifest as these services arent treated as typical xos synchronizers
     utils.Remove Value From List    ${dynamicServiceList}    openstack
     utils.Remove Value From List    ${dynamicServiceList}    onos
+    utils.Remove Value From List    ${dynamicServiceList}    exampleservice
     Log    ${dynamicServiceList}
 
 Verify Service Sanity
diff --git a/src/test/robot/SanityPhyPOD.robot b/src/test/robot/SanityPhyPOD.robot
index 5899cbe..359b1fd 100755
--- a/src/test/robot/SanityPhyPOD.robot
+++ b/src/test/robot/SanityPhyPOD.robot
@@ -1,4 +1,3 @@
-
 # Copyright 2017-present Open Networking Foundation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -28,30 +27,106 @@
 # limitations under the License.
 
 
-
-
-
 *** Settings ***
-Documentation     Test suite for checking default maas,xos and onos containers and fabric switch default services and maas cli commands 
-Library       OperatingSystem
-Resource      ../cord-api/Framework/utils/utils.robot
+Documentation     Test suite for checking default maas,xos and onos containers and fabric switch default services and maas cli commands
+Library           OperatingSystem
+Library           ../cord-api/Framework/utils/onosUtils.py
+Resource          ../cord-api/Framework/utils/utils.robot
 
 *** Variables ***
-@{MAAS_SERVICE_STATUS}    start/running    is running
-@{JUJU_SERVICE_STATUS}    active           is ready
-@{LXD_CONTAINER_STATUS}   RUNNING
-@{BOOT_RESOURCES_OUTPUT}  ubuntu/trusty
-${FABRIC_SWITCH_PROMPT}    #
-${FABRIC_SWITCH_USER}     root
-${FABRIC_SWITCH_PASSWD}    onl
-@{FABRIC_SERVICE_STATUS}    is running
-${IP_PATTERN}     (\\d+)\\.(\\d+)\\.(\\d+)\\.(\\d+)
+@{MAAS_SERVICE_STATUS}        start/running    is running
+@{JUJU_SERVICE_STATUS}        active           is ready    unknown
+@{LXD_CONTAINER_STATUS}       RUNNING
+@{BOOT_RESOURCES_OUTPUT}      ubuntu/trusty
+${FABRIC_SWITCH_PROMPT}       \#
+${FABRIC_SWITCH_USER}         root
+${FABRIC_SWITCH_PASSWD}       onl
+@{FABRIC_SERVICE_STATUS}      is running
+${IP_PATTERN}                 (\\d+)\\.(\\d+)\\.(\\d+)\\.(\\d+)
+${public_iface}               eth2
+${num_nodes} 		          2
+${num_of_switches}            4
 
 *** Test Cases ***
-Verify headnode interfaces detected
-    Verify HeadNode Interfaces
+Verify Headnode Interfaces
+    [Documentation]    Verifies the headnode interface is up and has external connectivity
+    Verify HeadNode Interfaces Detected
+    Test Ping    ${public_iface}    www.opennetworking.org
 
-Verify the state MAAS service
+Get Compute Node and Fabric Info
+    [Documentation]    Get all information pretaining to the compute nodes and fabric
+    ${nodes}=    Create List
+    ${hostnames}=    Create List
+    ${hostname_prefixes}=    Create List
+    ${node_ips}=    Create List
+    ${node_data_ips}=    Create List
+    ${node_count}    Run    cord prov list | grep node | wc -l
+    ${node_count}=    Convert To Integer    ${node_count}
+    Log    ${node_count}
+    ##Get hostname
+    : FOR    ${INDEX}    IN RANGE    1    ${node_count}+1
+    \    ${hostname}=    Run    cord prov list | grep node | awk '{print $2}' | sed -n ${INDEX}p
+    \    Append To List    ${hostnames}    ${hostname}
+    ##Get hostname prefixes
+    : FOR    ${INDEX}    IN RANGE    0    ${node_count}
+    \    ${hostname_prefix}=    Remove String    ${hostnames[${INDEX}]}    .cord.lab
+    \    Append To List    ${hostname_prefixes}    ${hostname_prefix}
+    ##Get compute node data ips
+    ${cordvtnnodes}=    ONOS Command Execute    onos-cord    8102    cordvtn-nodes | grep fabric
+    ${nds}=    Split To Lines    ${cordvtnnodes}
+    : FOR    ${i}    IN    @{nds}
+    \    ${data_ip}=    Get Compute Node IP    ${i}
+    \    Append To List    ${node_data_ips}    ${data_ip}
+    ##Get compute node ips
+    : FOR    ${i}    IN    @{hostname_prefixes}
+    \    ${node_ip}=    Run    cord harvest list | grep ${i} | awk '{print $4}'
+    \    Append To List    ${node_ips}    ${node_ip}
+    @{switch_ips}=    Discover FABRIC IPs
+    Set Suite Variable    ${switch_ips}
+    Set Suite Variable    ${hostnames}
+    Set Suite Variable    ${hostname_prefixes}
+    Set Suite Variable    ${node_ips}
+    Set Suite Variable    ${node_data_ips}
+
+Verify Compute Nodes Pingability
+    [Documentation]    Verifies that the two compute nodes can ping each other
+    [Tags]    fabric
+    ##Verify pingablilty across compute nodes
+    : FOR    ${i}    IN    @{node_data_ips}
+    \    ${result}=    Run    ssh -A ubuntu@${i} "ping -c 3 ${node_data_ips[0]}"
+    \    Should Contain    ${result}    64 bytes
+    \    Should Not Contain    ${result}    Destination Host Unreachable
+    \    ${result}=    Run    ssh -A ubuntu@${i} "ping -c 3 ${node_data_ips[1]}"
+    \    Should Contain    ${result}    64 bytes
+    \    Should Not Contain    ${result}    Destination Host Unreachable
+
+Verify Compute Nodes to Fabric Pingability
+    [Documentation]    Verifies that the two compute nodes can ping the switches
+    [Tags]    fabric
+    ##Verify pingability from compute nodes to fabric
+    ${switch_len}=    Get Length    ${switch_ips}
+    : FOR    ${INDEX}    IN RANGE    0    ${switch_len}
+    \    ${result}=    Run    ssh -A ubuntu@${node_data_ips[0]} "ping -c 3 ${switch_ips[${INDEX}]}"
+    \    Should Contain    ${result}    64 bytes
+    \    Should Not Contain    ${result}    Destination Host Unreachable
+    \    ${result}=    Run    ssh -A ubuntu@${node_data_ips[1]} "ping -c 3 ${switch_ips[${INDEX}]}"
+    \    Should Contain    ${result}    64 bytes
+    \    Should Not Contain    ${result}    Destination Host Unreachable
+
+Verify CordVTN Nodes
+    [Documentation]    Verifies that the cordvtn app running in onos identifies the nodes and devices (fabric)
+    ${nodes}=    Execute ONOS Command    onos-cord    8102    cordvtn-nodes
+    : FOR    ${i}    IN    @{node_ips}
+    \    ${node_1}=    Get Lines Containing String    ${nodes}    ${i}
+    \    Should Contain    ${node_1}    COMPLETE
+    \    Should Contain    ${node_1}    ${i}
+    ${ports}=    Execute ONOS Command    onos-cord    8102    cordvtn-ports
+    ${devices}=    Execute ONOS Command    onos-fabric    8101    devices
+    @{switch_ips}=    Discover FABRIC IPs
+    : FOR    ${i}    IN    @{switch_ips}
+    \    Should Contain    ${devices}    ${i}
+
+Verify MAAS Service State
     [Template]     Verify MAAS Service
     maas-dhcpd
     maas-regiond
@@ -59,46 +134,43 @@
     maas-proxy
     bind9
 
-Verify the state of MAAS Containers
-    [Template]      Verify Containers   
-    cord-maas-automation    
-    cord-maas-switchq    
-    cord-provisioner    
-    cord-ip-allocator    
-    cord-dhcp-harvester
-    config-generator    
+Verify MAAS Containers State
+    [Template]      Verify Containers
+    maas-automation
+    maas-switchq
+    maas-provisioner
+    maas-allocator
+    maas-harvester
+    maas-generator
 
-Verify the state of XOS Containers
-    [Template]      Verify Containers   
-    xos-gui    
-    xos-ws    
-    chameleon    
+Verify XOS Containers State
+    [Template]      Verify Containers
+    xos-gui
+    xos-ws
+    chameleon
     xos-ui
     onos-synchronizer
     vrouter-synchronizer
     exampleservice-synchronizer
-    vsg-synchronizer    
-    gui-extension-rcord
-    gui-extension-vtr
+    vsg-synchronizer
     vtn-synchronizer
     vtr-synchronizer
     fabric-synchronizer
     openstack-synchronizer
     xos-postgres
 
-Verify the state of ONOS Containers
-    [Template]      Verify Containers   
+Verify ONOS Containers State
+    [Template]      Verify Containers
     onosproject/onos
     xos/onos
- 
-Verify the state of other Containers
-    [Template]      Verify Containers   
-    redis 
+
+Verify Other Containers State
+    [Template]      Verify Containers
+    redis
     mavenrepo
     registry-mirror
-    registry
 
-Verify the state of juju services
+Verify Juju Services State
     [Template]    Verify JUJU Service
     ceilometer
     ceilometer-agent
@@ -106,115 +178,109 @@
     keystone
     mongodb
     nagios
-    neturon-api
+    neutron-api
     nova-cloud-controller
     nova-compute
     openstack-dashboard
     percona-cluster
     rabbitmq-server
 
-Verify the state of openstack lxd containers
-    [Template]    Verify Openstack LXD Containers 
+Verify Openstack LXD Containers State
+    [Template]    Verify Openstack LXD Containers
     ceilometer
     glance
     keystone
     mongodb
     nagios
-    neutron-api 
+    neutron-api
     nova-cloud-controller
     openstack-dashboard
     percona-cluster
     rabbitmq-server
-    #testclient
 
 Verify MAAS CLI commands
+    [Tags]    notready
     Login MAAS Server
     Verify MAAS CLI Commands   boot-resources read | jq 'map(select(.type == "Synced"))'    ubuntu/trusty
-    Verify MAAS CLI Commands   devices list | jq '.' | jq '.[]'.hostname | wc -l     3 
-    #Verify MAAS CLI Commands   events query | jq '.' | jq .events[].id | wc -l    100 
+    Verify MAAS CLI Commands   devices list | jq '.' | jq '.[]'.hostname | wc -l     ${num_of_switches}
+    #Verify MAAS CLI Commands   events query | jq '.' | jq .events[].id | wc -l    100
     Verify MAAS CLI Commands   fabrics read | jq '.' | jq .[].name | wc -l    4
     Verify MAAS CLI Commands   networks read | jq '.' | jq .[].name | wc -l    4
     Verify MAAS CLI Commands   node-groups list | jq '.' | jq .[].status | wc -l    1
     Verify MAAS CLI Commands   subnets read | jq '.' | jq .[].name | wc -l    4
     Verify MAAS CLI Commands   nodes list | jq '.' | jq .[].substatus_name | wc -l    1
-    Verify MAAS CLI Commands   zones read | jq '.' | jq .[].name | wc -l   2 
+    Verify MAAS CLI Commands   zones read | jq '.' | jq .[].name | wc -l   2
     Logout MAAS Server
 
 Verify Fabric Switch Service
-    ${fabric_ip}=    Discover FABRIC IP    ${FABRIC_SWITCH_MAC}   
-    Verify Fabric Switch Service    ${fabric_ip}    faultd
-    Verify Fabric Switch Service    ${fabric_ip}    netplug 
-    Verify Fabric Switch Service    ${fabric_ip}    ofdpa
-    Verify Fabric Switch Service    ${fabric_ip}    onlp-snmpd
-    Verify Fabric Switch Service    ${fabric_ip}    onlpd
-    Verify Fabric Switch Service    ${fabric_ip}    resolvconf 
-    Verify Fabric Switch Service    ${fabric_ip}    rsyslog
-    Verify Fabric Switch Service    ${fabric_ip}    snmpd 
-    Verify Fabric Switch Service    ${fabric_ip}    ssh 
-    Verify Fabric Switch Service    ${fabric_ip}    sxdkernel 
-    Verify Fabric Switch Service    ${fabric_ip}    udev 
-    Verify Fabric Switch Service    ${fabric_ip}    watchdog 
-
+    [Tags]    fabric
+    @{switch_ips}=    Discover FABRIC IPs
+    : FOR    ${i}    IN    @{switch_ips}
+    \    Verify Fabric Switch Service    ${i}    faultd
+    \    Verify Fabric Switch Service    ${i}    netplug
+    \    Verify Fabric Switch Service    ${i}    onlp-snmpd
+    \    Verify Fabric Switch Service    ${i}    onlpd
+    \    Verify Fabric Switch Service    ${i}    rsyslog
+    \    Verify Fabric Switch Service    ${i}    snmpd
+    \    Verify Fabric Switch Service    ${i}    ssh
+    \    Verify Fabric Switch Service    ${i}    udev
+    \    Verify Fabric Switch Service    ${i}    watchdog
 
 *** Keywords ***
-Verify HeadNode Interfaces
+Verify HeadNode Interfaces Detected
     ${cmd}=    Catenate    SEPARATOR=|   sudo ethtool mgmtbr    grep 'Link detected:'    awk '{ print $3 }'
-    ${output}=    Run     ${cmd} 
+    ${output}=    Run     ${cmd}
     Should Contain    ${output}    yes    msg= mgmtbr is not detected !!!. Reason:
     ${cmd}=    Catenate    SEPARATOR=|   sudo ethtool fabric    grep 'Link detected:'    awk '{ print $3 }'
-    ${output}=    Run     ${cmd} 
+    ${output}=    Run     ${cmd}
     Should Contain    ${output}    yes    msg= fabric interface is not detected !!!. Reason:
 
-
-Verify Containers 
-    [Arguments]    ${name} 
-    ${cmd}=    Catenate    SEPARATOR=|   docker ps -a  grep -v grep  grep ${name}    awk '{print $7,$8,$9,$10,$11}'     
-    ${output}=    Run     ${cmd} 
-    Should Contain    ${output}    Up    msg= ${name} is not running !!!. Reason:
+Verify Containers
+    [Arguments]    ${name}
+    ${container_id}=    Get Docker Container ID    ${name}
+    ${output}=    Run     docker inspect --format="{{ .State.Running }}" ${container_id}
+    Should Contain    ${output}    true    msg=${name} is not running !!!. Reason:
 
 Verify MAAS Service
     [Arguments]    ${name}
-    ${cmd}=    Catenate   sudo service    ${name}   status 
-    ${output}=    Run     ${cmd} 
+    ${cmd}=    Catenate   sudo service ${name} status
+    ${output}=    Run     ${cmd}
     Should Contain Any    ${output}    @{MAAS_SERVICE_STATUS}    msg= ${name} is not running !!!. Reason:
 
-
 Verify JUJU Service
     [Arguments]    ${name}
     ${cmd}    Catenate    SEPARATOR=|    juju status --format=tabular    grep -v grep    grep ${name}/0    awk '{ print $2,$7,$8,$9,$10}'
-    ${output}=    Run     ${cmd} 
+    ${output}=    Run     ${cmd}
     Should Contain Any    ${output}    @{JUJU_SERVICE_STATUS}    msg= ${name} is not running !!!. Reason:
 
 Verify Openstack LXD Containers
     [Arguments]    ${name}
     ${cmd}    Catenate    SEPARATOR=|    sudo lxc list    grep -v grep    grep ${name}-1    awk '{ print $2,$4 }'
-    ${output}=    Run     ${cmd} 
+    ${output}=    Run     ${cmd}
     Should Contain Any    ${output}    @{LXD_CONTAINER_STATUS}    msg= ${name} is not running !!!. Reason:
 
 Verify MAAS CLI Commands
     [Arguments]    ${name}    ${expected}
     ${cmd}    Catenate    maas cord ${name}
-    ${output}=    Run     ${cmd} 
-    Should Contain    ${output}    ${expected}    msg=Reason:   
+    ${output}=    Run     ${cmd}
+    Should Contain    ${output}    ${expected}    msg=Reason:
 
-Login MAAS Server 
-    ${cmd}   Catenate   maas login cord http://localhost/MAAS/api/1.0 $(sudo maas-region-admin apikey --user=cord) 
-    ${output}=    Run     ${cmd} 
+Login MAAS Server
+    ${cmd}   Catenate   maas login cord http://localhost/MAAS/api/1.0 $(sudo maas-region-admin apikey --user=cord)
+    ${output}=    Run     ${cmd}
     Should Contain   ${output}   You are now logged in to the MAAS  msg= MAAS login failure !!!. Reason:
 
-Logout MAAS Server 
-    ${cmd}   Catenate   maas logout cord 
-    ${output}=    Run     ${cmd} 
+Logout MAAS Server
+    ${cmd}   Catenate   maas logout cord
+    ${output}=    Run     ${cmd}
 
-Discover FABRIC IP
-    [Arguments]    ${fabric_mac}
-    ${cmd}   Catenate    SEPARATOR=|  cord switch list   grep -v IP   awk '{ print $3 }' 
-    ${output}=    Run     ${cmd} 
-    ${ret}=    Should Match Regexp    ${output}    ${IP_PATTERN}    msg="unable to get ip"   
-    [Return]   ${ret[0]} 
+Discover FABRIC IPs
+    ${switches}=    Run    cord prov list | grep fabric | awk '{print $4}'
+    @{switch_ips}=    Split To Lines    ${switches}
+    [Return]    ${switch_ips}
 
 Verify Fabric Switch Service
     [Arguments]    ${ip}    ${name}
-    ${cmd}=    Catenate    service   ${name}   status 
-    ${output}=    Run Command On Remote System    ${ip}    ${cmd}   ${FABRIC_SWITCH_USER}   ${FABRIC_SWITCH_PASSWD}    ${FABRIC_SWITCH_PROMPT}   60s   False 
-    Should Contain Any    ${output}    @{FABRIC_SERVICE_STATUS}    msg= ${name} is not running !!!. Reason:
+    ${cmd}=    Catenate    service   ${name}   status
+    ${output}=    Run Command On Remote System    ${ip}    ${cmd}   ${FABRIC_SWITCH_USER}   ${FABRIC_SWITCH_PASSWD}    ${FABRIC_SWITCH_PROMPT}   60s   False
+    Should Contain Any    ${output}    @{FABRIC_SERVICE_STATUS}    msg= ${name} is not running !!!. Reason:
\ No newline at end of file
diff --git a/src/test/setup/requirements.txt b/src/test/setup/requirements.txt
index 57cdaed..71676b7 100644
--- a/src/test/setup/requirements.txt
+++ b/src/test/setup/requirements.txt
@@ -12,11 +12,12 @@
 robotframework-requests
 robotframework-sshlibrary
 robotframework-httplibrary
-paramiko==1.10.1
+paramiko==2.3.1
 twisted
 pexpect
 apiclient
 pyyaml
+pyopenssl
 #python-libmaas
 #maasclient
 #maasutil