VOL-1670 - test failure or rw-core

Validates a a scale down/up test of the rw-core
to determine if the system can recover and continue

Change-Id: I6dd937724d2dd75b77c61505e0e3ee7cd36cb0db
diff --git a/libraries/k8s.robot b/libraries/k8s.robot
index 7362470..dfc0727 100644
--- a/libraries/k8s.robot
+++ b/libraries/k8s.robot
@@ -28,7 +28,7 @@
 *** Keywords ***
 Lookup Service IP
     [Arguments]    ${namespace}    ${name}
-    [Documentation]    Uses kubeclt to resolve a service name to an IP
+    [Documentation]    Uses kubectl to resolve a service name to an IP
     ${rc}    ${ip}=    Run and Return Rc and Output
     ...    kubectl get svc -n ${namespace} ${name} -o jsonpath={.spec.clusterIP}
     Should Be Equal as Integers    ${rc}    0
@@ -36,7 +36,7 @@
 
 Lookup Service PORT
     [Arguments]    ${namespace}    ${name}
-    [Documentation]    Uses kubeclt to resolve a service name to an PORT
+    [Documentation]    Uses kubectl to resolve a service name to an PORT
     ${rc}    ${port}=    Run and Return Rc and Output
     ...    kubectl get svc -n ${namespace} ${name} -o jsonpath={.spec.ports[0].port}
     Should Be Equal as Integers    ${rc}    0
@@ -155,3 +155,64 @@
     END
     [Return]    ${errorPodList}
 
+Delete K8s Pod
+    [Arguments]    ${namespace}    ${name}
+    [Documentation]    Uses kubectl to delete a named POD
+    ${rc}    Run and Return Rc
+    ...    kubectl delete -n ${namespace} pod/${name}
+    Should Be Equal as Integers    ${rc}    0
+
+Scale K8s Deployment
+    [Arguments]    ${namespace}    ${name}    ${count}
+    [Documentation]    Uses kubectl to scale a named deployment
+    ${rc}    Run and Return Rc
+    ...    kubectl scale --replicas=${count} -n ${namespace} deploy/${name}
+    Should Be Equal as Integers    ${rc}    0
+
+Pod Exists
+    [Arguments]    ${namespace}    ${name}
+    [Documentation]    Succeeds it the named POD exists
+    ${rc}    ${count}    Run and Return Rc    kubectl get -n ${namespace} pod -o json | jq -r ".items[].metadata.name" | grep ${name}
+    Should Be True    ${count}>0
+
+Pod Does Not Exist
+    [Arguments]    ${namespace}    ${name}
+    [Documentation]    Succeeds if the named POD does not exist
+    ${rc}    ${count}    Run and Return Rc And Output
+    ...    kubectl get -n ${namespace} pod -o json | jq -r ".items[].metadata.name" | grep -c ${name}
+    Should Be Equal As Integers    ${count}    0
+    Should Be True    ${count}==0
+
+Get Available Deployment Replicas
+    [Arguments]    ${namespace}    ${name}
+    [Documentation]    Succeeds if the named POD exists and has a ready count > 0
+    ${rc}    ${count}    Run and Return Rc and Output    ${KUBECTL_CONFIG};kubectl get -n ${namespace} deploy/${name} -o jsonpath='{.status.availableReplicas}'
+    ${result}=    Run Keyword If    '${count}' == ''    Set Variable    0
+    ...    ELSE    Set Variable    ${count}
+    [Return]    ${result}
+
+Check Expected Available Deployment Replicas
+    [Arguments]    ${namespace}    ${name}    ${expected}
+    [Documentation]    Succeeds if the named deployment has the expected number of available replicas
+    ${count}=    Get Available Deployment Replicas    ${namespace}    ${name}
+    Should Be Equal As Integers    ${expected}    ${count}
+
+Get Deployment Replica Count
+    [Arguments]    ${namespace}    ${name}
+    [Documentation]    Uses kubectl to fetch the number of configured replicas on a deployment
+    ${rc}    ${value}    Run and Return Rc and Output
+    ...    kubectl -n ${namespace} get deploy/${name} -o 'jsonpath={.status.replicas}'
+    Should Be Equal as Integers    ${rc}    0
+    ${replicas}=    Run Keyword If    '${value}' == ''    Set Variable    0
+    ...    ELSE    Set Variable    ${value}
+    [Return]  ${replicas}
+
+Does Deployment Have Replicas
+    [Arguments]    ${namespace}    ${name}    ${expected_count}
+    [Documentation]    Uses kubectl to fetch the number of configured replicas on a deployment
+    ${rc}    ${value}    Run and Return Rc and Output
+    ...    kubectl -n ${namespace} get deploy/${name} -o 'jsonpath={.status.replicas}'
+    Should Be Equal as Integers    ${rc}    0
+    ${replicas}=    Run Keyword If    '${value}' == ''    Set Variable    0
+    ...    ELSE    Set Variable    ${value}
+    Should be Equal as Integers    ${replicas}    ${expected_count}
diff --git a/libraries/onos.robot b/libraries/onos.robot
index 67ae0af..e9dfa63 100644
--- a/libraries/onos.robot
+++ b/libraries/onos.robot
@@ -128,3 +128,22 @@
     ${allocations}=    Execute ONOS CLI Command    ${ip}    ${port}
     ...    dhcpl2relay-allocations | grep DHCPACK | grep ${onu_port}
     Should Not Be Empty    ${allocations}    ONU port ${onu_port} not found in dhcpl2relay-allocations
+
+Device Is Available In ONOS
+    [Arguments]    ${url}    ${dpid}
+    [Documentation]    Validates the device exists and it available in ONOS
+    ${rc}    ${json}    Run And Return Rc And Output    curl --fail -sSL ${url}/onos/v1/devices/${dpid}
+    Should Be Equal As Integers    0    ${rc}
+    ${rc}    ${value}    Run And Return Rc And Output    echo '${json}' | jq -r .available
+    Should Be Equal As Integers    0    ${rc}
+    Should Be Equal    'true'    '${value}'
+
+Remove All Devices From ONOS
+    [Arguments]    ${url}
+    [Documentation]    Executes the device-remove command on each device in ONOS
+    ${rc}    @{dpids}    Run And Return Rc And Output    curl --fail -sSL ${url}/onos/v1/devices | jq -r '.devices[].id'
+    Should Be Equal As Integers    ${rc}    0
+    ${count}=    Get length    ${dpids}
+    :FOR    ${dpid}    IN    @{dpids}
+    \    ${rc}=    Run Keyword If    '${dpid}' != ''    Run And Return Rc    curl -XDELETE --fail -sSL ${url}/onos/v1/devices/${dpid}
+    \    Run Keyword If    '${dpid}' != ''    Should Be Equal As Integers    ${rc}    0
diff --git a/libraries/voltctl.robot b/libraries/voltctl.robot
index 71da9a1..2fbd900 100644
--- a/libraries/voltctl.robot
+++ b/libraries/voltctl.robot
@@ -33,7 +33,7 @@
     ${jsondata}=    To Json    ${output}
     Log    ${jsondata}
     ${length}=    Get Length    ${jsondata}
-    [Return]  ${length}
+    Should Be Equal As Integers    ${length}    0
 
 Create Device
     [Arguments]    ${ip}    ${port}
@@ -56,6 +56,33 @@
     ${rc}    ${output}=    Run and Return Rc and Output    ${VOLTCTL_CONFIG}; voltctl device disable ${device_id}
     Should Be Equal As Integers    ${rc}    0
 
+Disable Devices In Voltha
+    [Documentation]    Disables all the known devices in voltha
+    [Arguments]    ${filter}
+    ${arg}=    Set Variable    ${EMPTY}
+    ${arg}=    Run Keyword If    len('${filter}'.strip()) != 0    Set Variable    --filter ${filter}
+    ${rc}    ${devices}=    Run and Return Rc and Output    ${VOLTCTL_CONFIG}; voltctl device list ${arg} --orderby Root -q | xargs echo -n
+    Should Be Equal As Integers    ${rc}    0
+    ${rc}    ${output}=    Run Keyword If    len('${devices}') != 0    Run and Return Rc and Output    ${VOLTCTL_CONFIG}; voltctl device disable ${devices}
+    Run Keyword If    len('${devices}') != 0    Should Be Equal As Integers    ${rc}    0
+
+Test Devices Disabled In Voltha
+    [Documentation]    Tests to verify that all devices in VOLTHA are disabled
+    [Arguments]    ${filter}
+    ${rc}    ${count}=    Run and Return Rc and Output    ${VOLTCTL_CONFIG}; voltctl device list --filter '${filter},AdminState!=DISABLED' -q | wc -l
+    Should Be Equal As Integers    ${rc}    0
+    Should Be Equal As Integers    ${count}    0
+
+Delete Devices In Voltha
+    [Documentation]    Disables all the known devices in voltha
+    [Arguments]    ${filter}
+    ${arg}=    Set Variable    ${EMPTY}
+    ${arg}=    Run Keyword If    len('${filter}'.strip()) != 0    Set Variable    --filter ${filter}
+    ${rc}    ${devices}=    Run and Return Rc and Output    ${VOLTCTL_CONFIG}; voltctl device list ${arg} --orderby Root -q | xargs echo -n
+    Should Be Equal As Integers    ${rc}    0
+    ${rc}    ${output}=    Run Keyword If    len('${devices}') != 0    Run and Return Rc and Output    ${VOLTCTL_CONFIG}; voltctl device delete ${devices}
+    Run Keyword If    len('${devices}') != 0    Should Be Equal As Integers    ${rc}    0
+
 Get Device Flows from Voltha
     [Arguments]    ${device_id}
     [Documentation]    Gets device flows from VOLTHA
diff --git a/libraries/voltha.robot b/libraries/voltha.robot
new file mode 100644
index 0000000..1009eda
--- /dev/null
+++ b/libraries/voltha.robot
@@ -0,0 +1,52 @@
+# Copyright 2019-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# voltha common functions
+
+*** Settings ***
+Documentation     Library for various utilities
+Library           SSHLibrary
+Library           HttpLibrary.HTTP
+Library           String
+Library           DateTime
+Library           Process
+Library           Collections
+Library           RequestsLibrary
+Library           OperatingSystem
+
+*** Keywords ***
+Lookup Pod That Owns Device
+    [Arguments]    ${device_id}
+    [Documentation]    Uses a utility script to lookup which RW Core has current ownership of an OLT
+    ${rc}    ${pod}=    Run and Return Rc and Output
+    ...    ../scripts/which_pod_owns_device.sh ${device_id}
+    Should Be Equal as Integers    ${rc}    0
+    [Return]    ${pod}
+
+Lookup Deployment That Owns Device
+    [Arguments]    ${device_id}
+    [Documentation]    Uses a utility script to lookup which RW Core has current ownership of an OLT
+    ${rc}    ${deploy}=    Run and Return Rc and Output
+    ...    which_deployment_owns_device.sh ${device_id}
+    Should Be Equal as Integers    ${rc}    0
+    [Return]    ${deploy}
+
+Restart VOLTHA Port Foward
+    [Arguments]    ${name}
+    [Documentation]    Uses a script to restart a kubectl port-forward
+    ${rc}    ${pid}    Run And Return Rc And Output
+    ...    ps e -ww | grep _TAG=${name} | grep -v grep | awk '{printf(\"%s %s\\n\",$1,$5)}' | grep -v bash | awk '{print $1}'
+    Should Be Equal as Integers    ${rc}    0
+    Run Keyword If    '${pid}' != ''    Run And Return Rc    kill -9 ${pid}
+    Should Be Equal as Integers    ${rc}    0