Continue on failure in POD sanity tests

Change-Id: Ib18c01e03d2d9c4d13c81494ffd363a301a91e67
diff --git a/src/test/diag/verifyCollectDiag.robot b/src/test/diag/verifyCollectDiag.robot
index df3e425..92ac5f4 100644
--- a/src/test/diag/verifyCollectDiag.robot
+++ b/src/test/diag/verifyCollectDiag.robot
@@ -31,14 +31,17 @@
     [Documentation]    Verify expected containers are up and running
     ${dockerContainersExpected}    utils.jsonToList    ${DOCKER_CONTAINERS_FILE}    docker-containers-${CORD_PROFILE}
     : FOR    ${container}    IN    @{dockerContainersExpected}
-    \    OperatingSystem.File Should Exist    /home/cord/diag-*/docker/${container}
+    \    Run Keyword And Continue On Failure    Verify Docker Container    ${container}
 
 Verify Synchronizer Logs
     [Documentation]    Verify synchronizer logs are correct
     ${synchronizerLogs}    utils.readFiles    /home/cord/diag-*/docker/*synchronizer*
     : FOR    ${key}    IN    @{synchronizerLogs.keys()}
-    \    ${value}=    Get From Dictionary    ${synchronizerLogs}    ${key}
-    \    Should Contain    ${value}    Waiting for event or timeout
+    \    @{nameWithSuffix}=    Split String    ${key}    cord_
+    \    @{name}=    Split String    @{nameWithSuffix}[1]    -synchronizer
+    \    ${synchronizerConfig}    utils.readFile    /opt/cord/orchestration/xos_services/*/xos/synchronizer/@{name}[0]_config.yaml
+    \    ${synchronizerLog}=    Get From Dictionary    ${synchronizerLogs}    ${key}
+    \    Run Keyword And Continue On Failure    Verify Synchronizer Log    ${synchronizerConfig}    ${synchronizerLog}
 
 Verify ONOS
     [Documentation]    Verify ONOS status, applications and logs
@@ -46,17 +49,26 @@
     Verify ONOS-CORD    ${CORD_PROFILE}
 
 *** Keywords ***
+Verify Docker Container
+    [Arguments]    ${container}
+    OperatingSystem.File Should Exist    /home/cord/diag-*/docker/${container}
+
+Verify Synchronizer Log
+    [Arguments]    ${config}    ${log}
+    Run Keyword If    'steps_dir' in '''${config}'''    Should Contain    ${log}    Waiting for event or timeout
+    Run Keyword If    'model_policies_dir' in '''${config}'''    Should Contain    ${log}    Loaded model policies
+
 Verify ONOS-Fabric
     [Arguments]    ${cord_profile}
-    Verify ONOS Status    onos-fabric
-    AND Verify ONOS Applications    onos-fabric    ${cord_profile}
-    AND Verify ONOS Logs    onos-fabric
+    Run Keyword And Continue On Failure    Verify ONOS Status    onos-fabric
+    Run Keyword And Continue On Failure    Verify ONOS Applications    onos-fabric    ${cord_profile}
+    Run Keyword And Continue On Failure    Verify ONOS Log    onos-fabric
 
 Verify ONOS-CORD
     [Arguments]    ${cord_profile}
-    Verify ONOS Status    onos-cord
-    AND Verify ONOS Applications    onos-cord    ${cord_profile}
-    AND Verify ONOS Logs    onos-cord
+    Run Keyword And Continue On Failure    Verify ONOS Status    onos-cord
+    Run Keyword And Continue On Failure    Verify ONOS Applications    onos-cord    ${cord_profile}
+    Run Keyword And Continue On Failure    Verify ONOS Log    onos-cord
 
 Verify ONOS Status
     [Arguments]    ${onosName}
@@ -70,7 +82,7 @@
     : FOR    ${app}    IN    @{onosAppsExpected}
     \    Should Contain    ${onosApps}    ${app}
 
-Verify ONOS Logs
+Verify ONOS Log
     [Arguments]    ${onosName}
     ${onosLog}    utils.readFile    /home/cord/diag-*/${onosName}/log_display
     Should Not Contain    ${onosLog}    ERROR
diff --git a/src/test/robot/SanityPhyPOD.robot b/src/test/robot/SanityPhyPOD.robot
index 1cc2e20..9e09072 100755
--- a/src/test/robot/SanityPhyPOD.robot
+++ b/src/test/robot/SanityPhyPOD.robot
@@ -33,6 +33,7 @@
 ${PUBLIC_IFACE}               eth2
 ${NUM_OF_SWITCHES}            4
 ${CORD_PROFILE}               rcord
+${FABRIC}                     on
 ${DOCKER_CONTAINERS_FILE}     ${CURDIR}/../diag/dockerContainers.json
 
 *** Test Cases ***
@@ -96,8 +97,8 @@
     ${nodes}=    Execute ONOS Command    onos-cord    8102    cordvtn-nodes
     : FOR    ${i}    IN    @{node_ips}
     \    ${node_1}=    Get Lines Containing String    ${nodes}    ${i}
-    \    Should Contain    ${node_1}    COMPLETE
-    \    Should Contain    ${node_1}    ${i}
+    \    Run Keyword If    "${FABRIC}" == "on"    Verify CordVTN Node    ${node_1}    COMPLETE    ${i}
+    \    Run Keyword If    "${FABRIC}" == "off"    Verify CordVTN Node    ${node_1}    DEVICE_CREATED    ${i}
     ${ports}=    Execute ONOS Command    onos-cord    8102    cordvtn-ports
     ${devices}=    Execute ONOS Command    onos-fabric    8101    devices
     @{switch_ips}=    Discover FABRIC IPs
@@ -182,6 +183,11 @@
     ${output}=    Run     ${cmd}
     Should Contain    ${output}    yes    msg= fabric interface is not detected !!!. Reason:
 
+Verify CordVTN Node
+    [Arguments]    ${node}    ${status}    ${ip}
+    Should Contain    ${node}    ${status}
+    Should Contain    ${node}    ${ip}
+
 Verify Containers
     [Arguments]    ${name}
     ${container_id}=    Get Docker Container ID    ${name}