Multiple cherrypicks to cord-tester 4.0 branch

Change-Id: Ibbeaef75635e697ad52da7ee23b77b353ab48ad7
diff --git a/src/test/diag/dockerContainers.json b/src/test/diag/dockerContainers.json
new file mode 100644
index 0000000..13c29c3
--- /dev/null
+++ b/src/test/diag/dockerContainers.json
@@ -0,0 +1,118 @@
+{
+    "docker-containers-rcord":
+    [
+        "allocator",
+        "generator",
+        "automation",
+        "provisioner",
+        "registry",
+        "harvester",
+        "registry-mirror",
+        "storage",
+        "switchq",
+        "mavenrepo",
+        "onosfabric_xos-onos_1",
+        "onoscord_xos-onos_1",
+        "rcord_consul_1",
+        "rcord_onos-synchronizer_1",
+        "rcord_xos_chameleon_1",
+        "rcord_xos_redis_1",
+        "rcord_xos_core_1",
+        "rcord_xos_tosca_1",
+        "rcord_registrator_1",
+        "rcord_xos_db_1",
+        "rcord_xos_ui_1",
+        "rcord_xos_gui_1",
+        "rcord_xos_ws_1",
+        "rcord_vrouter-synchronizer_1",
+        "rcord_exampleservice-synchronizer_1",
+        "rcord_vsg-synchronizer_1",
+        "rcord_vtn-synchronizer_1",
+        "rcord_vtr-synchronizer_1",
+        "rcord_fabric-synchronizer_1",
+        "rcord_openstack-synchronizer_1"
+    ],
+    "docker-containers-ecord-global":
+    [
+        "ecordglobal_consul_1",
+        "ecordglobal_onos-synchronizer_1",
+        "ecordglobal_xos_chameleon_1",
+        "ecordglobal_xos_redis_1",
+        "mavenrepo",
+        "ecordglobal_xos_core_1",
+        "ecordglobal_xos_tosca_1",
+        "onoscord_xos-onos_1",
+        "ecordglobal_registrator_1",
+        "ecordglobal_xos_db_1",
+        "ecordglobal_xos_ui_1",
+        "ecordglobal_vnaas-synchronizer_1",
+        "ecordglobal_xos_gui_1",
+        "ecordglobal_xos_ws_1"
+    ],
+    "docker-containers-ecord":
+    [
+        "allocator",
+        "generator",
+        "automation",
+        "provisioner",
+        "registry",
+        "harvester",
+        "registry-mirror",
+        "storage",
+        "switchq",
+        "ecord_onos-synchronizer_1",
+        "ecord_vrouter-synchronizer_1",
+        "ecord_addressmanager-synchronizer_1",
+        "ecord_consul_1",
+        "ecord_fabric-synchronizer_1",
+        "ecord_openstack-synchronizer_1",
+        "ecord_registrator_1",
+        "ecord_vee-synchronizer_1",
+        "ecord_veg-synchronizer_1",
+        "ecord_vrouter-synchronizer_1",
+        "ecord_vtn-synchronizer_1",
+        "ecord_xos_chameleon_1",
+        "ecord_xos_core_1",
+        "ecord_xos_db_1",
+        "ecord_xos_gui_1",
+        "ecord_xos_redis_1",
+        "ecord_xos_tosca_1",
+        "ecord_xos_ui_1",
+        "ecord_xos_ws_1",
+        "onoscord_xos-onos_1",
+        "onosfabric_xos-onos_1"
+    ],
+    "docker-containers-mcord-ng40":
+    [
+        "allocator",
+        "generator",
+        "automation",
+        "provisioner",
+        "registry",
+        "harvester",
+        "registry-mirror",
+        "storage",
+        "switchq",
+        "mavenrepo",
+        "mcordng40_xos_ws_1",
+        "mcordng40_xos_gui_1",
+        "mcordng40_xos_chameleon_1",
+	"mcordng40_xos_tosca_1",
+	"mcordng40_xos_ui_1",
+	"mcordng40_xos_core_1",
+	"mcordng40_openstack-synchronizer_1",
+	"mcordng40_vepc-synchronizer_1",
+	"mcordng40_vtn-synchronizer_1",
+	"mcordng40_vspgwc-synchronizer_1",
+	"mcordng40_fabric-synchronizer_1",
+	"mcordng40_venb-synchronizer_1",
+	"mcordng40_onos-synchronizer_1",
+	"mcordng40_vspgwu-synchronizer_1",
+	"mcordng40_xos_redis_1",
+	"mcordng40_xos_db_1",
+	"mcordng40_registrator_1",
+	"mcordng40_consul_1",
+	"onosfabric_xos-onos_1",
+	"onoscord_xos-onos_1"
+    ]
+}
diff --git a/src/test/diag/onosApps.json b/src/test/diag/onosApps.json
new file mode 100644
index 0000000..5ea01d7
--- /dev/null
+++ b/src/test/diag/onosApps.json
@@ -0,0 +1,93 @@
+{
+    "onos-fabric-rcord":
+    [
+        "org.onosproject.drivers",
+        "org.onosproject.segmentrouting",
+        "org.onosproject.optical-model",
+        "org.onosproject.fpm",
+        "org.onosproject.hostprovider",
+        "org.onosproject.lldpprovider",
+        "org.onosproject.fibinstaller",
+        "org.onosproject.cpr",
+        "org.onosproject.vrouter",
+        "org.onosproject.netcfghostprovider",
+        "org.onosproject.openflow-base",
+        "org.onosproject.openflow"
+    ],
+    "onos-cord-rcord":
+    [
+        "org.onosproject.drivers",
+        "org.onosproject.optical-model",
+        "org.onosproject.openflow-base",
+        "org.onosproject.ovsdb-base",
+        "org.onosproject.drivers.ovsdb",
+        "org.onosproject.dhcp",
+        "org.opencord.config",
+        "org.opencord.vtn"
+    ],
+    "onos-cord-ecord-global":
+    [
+        "org.onosproject.drivers",
+        "org.opencord.ce.api",
+        "org.opencord.ce.global.app"
+    ],
+    "onos-fabric-ecord":
+    [
+        "org.onosproject.drivers",
+        "org.onosproject.netcfghostprovider",
+        "org.onosproject.optical-model",
+        "org.onosproject.lldpprovider",
+        "org.onosproject.segmentrouting",
+        "org.onosproject.hostprovider",
+        "org.onosproject.openflow-base",
+        "org.onosproject.openflow",
+        "org.onosproject.fpm",
+        "org.opencord.ce.api",
+        "org.opencord.ce.local.bigswitch",
+        "org.opencord.ce.local.channel.http",
+        "org.opencord.ce.local.fabric"
+    ],
+    "onos-cord-ecord":
+    [
+        "org.onosproject.drivers",
+        "org.onosproject.ovsdb-base",
+        "org.onosproject.drivers.ovsdb",
+        "org.onosproject.faultmanagement",
+        "org.onosproject.netconf",
+        "org.onosproject.yang",
+        "org.onosproject.optical-model",
+        "org.onosproject.openflow-base",
+        "org.onosproject.config",
+        "org.onosproject.netconfsb",
+        "org.onosproject.drivers.netconf",
+        "org.onosproject.models.microsemi",
+        "org.onosproject.dhcp",
+        "org.opencord.ce.api",
+        "org.opencord.ce.local.bigswitch",
+        "org.opencord.ce.local.channel.http",
+        "org.opencord.ce.local.vee",
+        "org.opencord.config",
+        "org.opencord.vtn"
+    ],
+    "onos-fabric-mcord-ng40":
+    [
+        "org.onosproject.drivers",
+        "org.onosproject.optical-model",
+        "org.onosproject.openflow-base",
+        "org.onosproject.hostprovider",
+        "org.onosproject.netcfghostprovider",
+        "org.onosproject.lldpprovider",
+        "org.onosproject.openflow"
+    ],
+    "onos-cord-mcord-ng40":
+    [
+        "org.onosproject.drivers",
+        "org.onosproject.optical-model",
+        "org.onosproject.openflow-base",
+        "org.onosproject.dhcp",
+        "org.onosproject.ovsdb-base",
+        "org.onosproject.drivers.ovsdb",
+        "org.opencord.config",
+        "org.opencord.vtn"
+    ]
+}
diff --git a/src/test/diag/verifyCollectDiag.robot b/src/test/diag/verifyCollectDiag.robot
new file mode 100644
index 0000000..4dd3e6d
--- /dev/null
+++ b/src/test/diag/verifyCollectDiag.robot
@@ -0,0 +1,90 @@
+# Copyright 2017-present Radisys Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+*** Settings ***
+Documentation     Test suite for checking results collected by `make collect-diag` command
+Library           OperatingSystem
+Library           ../cord-api/Framework/utils/onosUtils.py
+Library           ../cord-api/Framework/utils/utils.py
+Resource          ../cord-api/Framework/utils/utils.robot
+
+*** Variables ***
+${DOCKER_CONTAINERS_FILE}       ${CURDIR}/dockerContainers.json
+${ONOS_APPS_FILE}       	${CURDIR}/onosApps.json
+${CORD_PROFILE}			rcord
+${CORD_SCENARIO}		cord
+
+*** Test Cases ***
+Verify Docker Containers
+    [Documentation]    Verify expected containers are up and running
+    ${dockerContainersExpected}    utils.jsonToList    ${DOCKER_CONTAINERS_FILE}    docker-containers-${CORD_PROFILE}
+    : FOR    ${container}    IN    @{dockerContainersExpected}
+    \    Run Keyword And Continue On Failure    Verify Docker Container    ${container}
+
+Verify Synchronizer Logs
+    [Documentation]    Verify synchronizer logs are correct
+    ${synchronizerLogs}    utils.readFiles    /home/cord/diag-*/docker/*synchronizer*
+    : FOR    ${key}    IN    @{synchronizerLogs.keys()}
+    \    @{name}=    Split String    ${key}    -synchronizer
+    \    @{name}=    Split String From Right   @{name}[0]    _    1
+    \    ${synchronizerLog}=    Get From Dictionary    ${synchronizerLogs}    ${key}
+    \    Run Keyword And Continue On Failure    Verify Synchronizer Log    ${name}    ${synchronizerLog}
+
+Verify ONOS
+    [Documentation]    Verify ONOS status, applications and logs
+    Run Keyword If    '${CORD_PROFILE}' != 'ecord-global'   Verify ONOS-Fabric    ${CORD_PROFILE}
+    Verify ONOS-CORD    ${CORD_PROFILE}
+
+*** Keywords ***
+Verify Docker Container
+    [Arguments]    ${container}
+    OperatingSystem.File Should Exist    /home/cord/diag-*/docker/${container}
+
+Verify Synchronizer Log
+    [Arguments]    ${name}    ${log}
+    ${config}    utils.readFile    /opt/cord/orchestration/xos_services/*/xos/synchronizer/@{name}[1]_config.yaml
+    ${match1}=    Get Lines Matching Regexp    ${config}    ^steps_dir: ".*"$
+    ${match2}=    Get Lines Matching Regexp    ${config}    ^model_policies_dir: ".*"$
+    Run Keyword If    '${match1}' != '${EMPTY}'    Should Contain    ${log}    Waiting for event or timeout    msg= "Waiting for event or timeout" not found in @{name}[1] synchronizer log
+    ...    ELSE IF    '${match2}' != '${EMPTY}'    Should Contain    ${log}    Loaded model policies    msg= "Loaded model policies" not found in @{name}[1] synchronizer log
+
+Verify ONOS-Fabric
+    [Arguments]    ${cord_profile}
+    Run Keyword And Continue On Failure    Verify ONOS Status    onos-fabric
+    Run Keyword And Continue On Failure    Verify ONOS Applications    onos-fabric    ${cord_profile}
+    Run Keyword And Continue On Failure    Verify ONOS Log    onos-fabric
+
+Verify ONOS-CORD
+    [Arguments]    ${cord_profile}
+    Run Keyword And Continue On Failure    Verify ONOS Status    onos-cord
+    Run Keyword And Continue On Failure    Verify ONOS Applications    onos-cord    ${cord_profile}
+    Run Keyword And Continue On Failure    Verify ONOS Log    onos-cord
+
+Verify ONOS Status
+    [Arguments]    ${onosName}
+    ${onosStatus}    utils.readFile    /home/cord/diag-*/${onosName}/nodes
+    Should Contain    ${onosStatus}    READY
+
+Verify ONOS Applications
+    [Arguments]    ${onosName}    ${cordProfile}
+    ${onosAppsExpected}    utils.jsonToList    ${ONOS_APPS_FILE}    ${onosName}-${cordProfile}
+    ${onosApps}    utils.readFile    /home/cord/diag-*/${onosName}/apps_-s_-a
+    : FOR    ${app}    IN    @{onosAppsExpected}
+    \    Should Contain    ${onosApps}    ${app}
+
+Verify ONOS Log
+    [Arguments]    ${onosName}
+    ${onosLog}    utils.readFile    /home/cord/diag-*/${onosName}/log_display
+    Should Not Contain    ${onosLog}    ERROR