Refactor jenkinsfile for both zero-touch and top-down provisioning

Change-Id: I04d0688bb42ac9b9eff56af481d48172f201f130
diff --git a/Jenkinsfile-rcordlite-topdown b/Jenkinsfile-rcordlite-topdown
index c7cc429..a264f5b 100644
--- a/Jenkinsfile-rcordlite-topdown
+++ b/Jenkinsfile-rcordlite-topdown
@@ -12,218 +12,243 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-pod_config = null
-
 node ("${TestNodeName}") {
-    timeout (time: 240) {
-        stage ("Parse deployment configuration file") {
-            sh returnStdout: true, script: 'rm -rf ${configRepoBaseDir}'
-            sh returnStdout: true, script: 'git clone -b ${branch} ${configRepoUrl}'
-            deployment_config = readYaml file: "${configRepoBaseDir}${configRepoFile}"
-        }
-        
-        stage('Export pod config') {
-            timeout(10) {
-                try {
-                    sh """
-                    cd /home/cord
-                    rm -rf /home/cord/helm-charts
-                    git clone -b ${branch} https://gerrit.opencord.org/helm-charts
-                    export KUBECONFIG=/home/cord/${deployment_config.pod_config}
-                    kubectl get pods
-                    """
-                } catch(error) { currentBuild.result = 'FAILURE' }
+    timeout (100) {
+        try {
+            stage ("Parse deployment configuration file") {
+                sh returnStdout: true, script: "rm -rf ${configBaseDir}"
+                sh returnStdout: true, script: "git clone -b ${branch} ${cordRepoUrl}/${configBaseDir}"
+                deployment_config = readYaml file: "${configBaseDir}/${configDeploymentDir}/${configFileName}.yml"
             }
-        }
-        stage('Clean up') {
-            timeout(10) {
-                try {
-                    sh """
-                    cd /home/cord/helm-charts
-                    export KUBECONFIG=/home/cord/${deployment_config.pod_config}
-                    helm ls | grep onos-voltha && helm delete --purge onos-voltha || true
-                    helm ls | grep onos-fabric && helm delete --purge onos-fabric || true
-                    helm ls | grep voltha && helm delete --purge voltha || true
-                    helm ls | grep rcord-lite && helm delete --purge rcord-lite || true
-                    helm ls | grep xos-core && helm delete --purge xos-core || true
+            stage('Clean up') {
+                timeout(10) {
+                    sh returnStdout: true, script: """
+                    rm -rf helm-charts cord-tester
+                    git clone -b ${branch} ${cordRepoUrl}/helm-charts
+                    export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
+                    for hchart in \$(helm list -q);
+                    do
+                        echo "Purging chart: \${hchart}"
+                        helm delete --purge "\${hchart}"
+                    done
                     """
-                } catch(error) { currentBuild.result = 'FAILURE' }
-                // Wait for helm delete
-                timeout(5) {
-                    waitUntil {
-                        try {
-                            sh """
-                            export KUBECONFIG=/home/cord/${deployment_config.pod_config}
-                            ! helm ls | grep -E 'onos-voltha|onos-fabric|voltha |rcord-lite|xos-core'
-                            ! kubectl get pods --all-namespaces | grep -E 'onos-voltha|onos-fabric|voltha |rcord-lite-|xos-'
+                    timeout(5) {
+                        waitUntil {
+                            helm_deleted = sh returnStdout: true, script: """
+                            export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf &&
+                            helm ls | wc -l
                             """
-                            return true
-                        } catch (exception) {
-                            return false
+                            return helm_deleted.toInteger() == 0
+                        }
+                    }
+                    timeout(5) {
+                        waitUntil {
+                            kubectl_deleted = sh returnStdout: true, script: """
+                            export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf &&
+                            kubectl get pods --all-namespaces --no-headers | grep -v kube-system | wc -l
+                            """
+                            return kubectl_deleted.toInteger() == 0
                         }
                     }
                 }
             }
-        }
-        stage('Install xos-core and rcord-lite') {
-            timeout(10) {
-                try {
-                    sh """
-                    export KUBECONFIG=/home/cord/${deployment_config.pod_config}
-                    cd /home/cord/helm-charts
-                    helm dep update xos-core
-                    helm install -n xos-core xos-core
-                    helm dep update xos-profiles/rcord-lite
-                    helm install -n rcord-lite xos-profiles/rcord-lite
-                    """
-                } catch(error) { currentBuild.result = 'FAILURE' }
-            }
-        }
-        stage('Install voltha') {
-            timeout(10) {
-                try {
-                    sh """
-                    export KUBECONFIG=/home/cord/${deployment_config.pod_config}
-                    cd /home/cord/helm-charts
-                    helm repo add incubator https://kubernetes-charts-incubator.storage.googleapis.com/
-                    helm dep build voltha
-                    helm install -n voltha --set etcd-operator.customResources.createEtcdClusterCRD=false voltha
-                    helm upgrade --set etcd-operator.customResources.createEtcdClusterCRD=true voltha ./voltha
-                    """
-                } catch(error) { currentBuild.result = 'FAILURE' }
-            }
-        }
-        stage('Install ONOS-Voltha and ONOS-Fabric') {
-            timeout(10) {
-                try {
-                    sh """
-                    export KUBECONFIG=/home/cord/${deployment_config.pod_config}
-                    cd /home/cord/helm-charts
-                    helm install -n onos-fabric -f configs/onos-fabric.yaml onos
-                    helm install -n onos-voltha -f configs/onos-voltha.yaml onos
-                    sleep 60
-                    """
-                } catch(error) { currentBuild.result = 'FAILURE' }
-            }
-        }
-        /*
-        stage('Download cord-tester repo') {
-            timeout(10) {
-                try {
-                    sh """
-                    rm -rf /home/cord/cord-tester
-                    cd /home/cord/
-                    git clone -b ${branch} https://gerrit.opencord.org/cord-tester 
-                    """
-                } catch(error) { currentBuild.result = 'FAILURE' }
-            }
-        }
-        stage('Validate Helm Chart Install') {
-            timeout(10) {
-                try {
-                    sh """
-                    export KUBECONFIG=/home/cord/${deployment_config.pod_config}
-                    cd /home/cord/cord-tester/src/test/robot/
-                    rm -rf Log/ || true
-                    pybot -d Log -T SanityK8POD.robot || true
-                    """
-                } catch(error) { currentBuild.result = 'FAILURE' }
-            }
-        }*/
-        stage('Wait for PODs to complete') {
-            /*
-            timeout(5) {
-                try {
-                    sh """
-                    export KUBECONFIG=/home/cord/${deployment_config.pod_config}
-                    chameleon=\$(kubectl get pods | grep chameleon | cut -d' ' -f1)
-                    echo \$chameleon
-                    kubectl delete pod \$chameleon
-                    """
-                } catch(error) { currentBuild.result = 'FAILURE' }
-            }*/
-            timeout(5) {
-                waitUntil {
-                    try {
-                        sh """
-                        export KUBECONFIG=/home/cord/${deployment_config.pod_config}
-                        kubectl get pods | grep -i tosca-loader | grep -i complete
-                        kubectl get pods | grep -i chameleon | grep -i running
-                        kubectl get pods | grep -i onos-fabric | grep -i running
-                        kubectl get pods --namespace=voltha | grep -i voltha- | grep -i running
-                        kubectl get pods --namespace=voltha | grep -i onos-voltha | grep -i running
+            dir ("helm-charts") {
+                stage('Install voltha') {
+                    timeout(10) {
+                        sh returnStdout: true, script: """
+                        export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
+                        helm repo add incubator https://kubernetes-charts-incubator.storage.googleapis.com/
+                        helm dep build voltha
+                        helm install -n voltha -f ../${configBaseDir}/${configKubernetesDir}/${configFileName}.yml --set etcd-operator.customResources.createEtcdClusterCRD=false voltha
+                        helm upgrade --set etcd-operator.customResources.createEtcdClusterCRD=true voltha ./voltha
                         """
-                        return true
-                    } catch (exception) {
-                        return false
+                    }
+                    timeout(10) {
+                        waitUntil {
+                            voltha_completed = sh returnStdout: true, script: """
+                            export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf &&
+                            kubectl get pods -n voltha | grep -i running | grep 1/1 | wc -l
+                            """
+                            return voltha_completed.toInteger() == 8
+                        }
+                    }
+                }
+                stage('Install ONOS-Voltha') {
+                    timeout(10) {
+                        sh returnStdout: true, script: """
+                        export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
+                        helm install -n onos-voltha -f ../${configBaseDir}/${configKubernetesDir}/${configFileName}.yml -f configs/onos-voltha.yaml onos
+                        """
+                    }
+                    timeout(10) {
+                        waitUntil {
+                            onos_voltha_completed = sh returnStdout: true, script: """
+                            export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf &&
+                            kubectl get pods -n voltha | grep -i onos-voltha | grep -i running | grep 1/1 | wc -l
+                            """
+                            return onos_voltha_completed.toInteger() == 1
+                        }
+                    }
+                }
+                stage('Install ONOS-Fabric') {
+                    timeout(10) {
+                        sh returnStdout: true, script: """
+                        export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
+                        helm install -n onos-fabric -f ../${configBaseDir}/${configKubernetesDir}/${configFileName}.yml -f configs/onos-fabric.yaml onos
+                        """
+                    }
+                    timeout(10) {
+                        waitUntil {
+                            onos_fabric_completed = sh returnStdout: true, script: """
+                            export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf &&
+                            kubectl get pods | grep -i onos-fabric | grep -i running | grep 1/1 | wc -l
+                            """
+                            return onos_fabric_completed.toInteger() == 1
+                        }
+                    }
+                }
+                stage('Install xos-core') {
+                    timeout(10) {
+                        sh returnStdout: true, script: """
+                        export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
+                        helm dep update xos-core
+                        helm install -f ../${configBaseDir}/${configKubernetesDir}/${configFileName}.yml -n xos-core xos-core
+                        """
+                    }
+                    timeout(10) {
+                        waitUntil {
+                            xos_core_completed = sh returnStdout: true, script: """
+                            export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf &&
+                            kubectl get pods | grep -i xos | grep -i running | grep 1/1 | wc -l
+                            """
+                            return xos_core_completed.toInteger() == 7
+                        }
+                    }
+                }
+                stage('Install rcord-lite') {
+                    timeout(10) {
+                        sh returnStdout: true, script: """
+                        export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
+                        helm dep update xos-profiles/rcord-lite
+                        helm install -f ../${configBaseDir}/${configKubernetesDir}/${configFileName}.yml -n rcord-lite xos-profiles/rcord-lite
+                        """
+                    }
+                    timeout(10) {
+                        waitUntil {
+                            rcord_tosca_completed = sh returnStdout: true, script: """
+                            export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf &&
+                            kubectl get pods | grep -i tosca-loader | grep -i completed | wc -l
+                            """
+                            return rcord_tosca_completed.toInteger() == 1
+                        }
                     }
                 }
             }
-        }
-        stage('Restart OLT processes') {
-            timeout(5) {
-                try {
-                    sh """
-                    export KUBECONFIG=/home/cord/${deployment_config.pod_config}
-                    sshpass -p onl ssh -l root 10.192.4.218 'pkill bal_core_dist' || true
-                    sshpass -p onl ssh -l root 10.192.4.218 'pkill openolt' || true
-                    sshpass -p onl ssh -l root 10.192.4.218 'cd /broadcom; ./bal_core_dist -C 127.0.0.1:40000 -A 127.0.0.1:50000 < /dev/tty1 > ./bal.log 2>&1 &'
-                    sshpass -p onl ssh -l root 10.192.4.218 'cd /broadcom; ./openolt -C 127.0.0.1:40000 -A 127.0.0.1:50000 < /dev/tty1 > ./openolt.log 2>&1 &'
-                    sleep 120
+            stage('Reinstall OLT software') {
+                for(int i=0; i < deployment_config.olts.size(); i++) {
+                    sh returnStdout: true, script: """
+                    sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'dpkg --remove asfvolt16 && dpkg --purge asfvolt16'
                     """
-                } catch(error) { currentBuild.result = 'FAILURE' }
-            }
-        }
-        stage('Download cord-tester repo') {
-            timeout(10) {
-                try {
-                    sh """
-                    rm -rf /home/cord/cord-tester
-                    cd /home/cord/
-                    git clone -b ${branch} https://gerrit.opencord.org/cord-tester 
+                    timeout(5) {
+                        waitUntil {
+                            olt_sw_present = sh returnStdout: true, script: """
+                            sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'dpkg --list | grep asfvolt16 | wc -l'
+                            """
+                            return olt_sw_present.toInteger() == 0
+                        }
+                    }
+                    sh returnStdout: true, script: """
+                    sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} "dpkg --install ${oltDebVersion}"
                     """
-                } catch(error) { currentBuild.result = 'FAILURE' }
+                    timeout(5) {
+                        waitUntil {
+                            olt_sw_present = sh returnStdout: true, script: """
+                            sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'dpkg --list | grep asfvolt16 | wc -l'
+                            """
+                            return olt_sw_present.toInteger() == 1
+                        }
+                    }
+                    // If the OLT is connected to a 40G switch interface, set the NNI port to be downgraded
+                    if ("${deployment_config.olts[i].fortygig}" != null && "${deployment_config.olts[i].fortygig}" == 'true') {
+                        sh returnStdout: true, script: """
+                        sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'echo "port ce128 sp=40000" >> /broadcom/qax.soc'
+                        sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} '/opt/bcm68620/svk_init.sh'
+                        """
+                    }
+                }
             }
-        }
-        stage('Validate Helm Chart Install') {
-            timeout(10) {
-                try {
+            stage('Restart OLT processes') {
+                for(int i=0; i < deployment_config.olts.size(); i++) {
+                    timeout(5) {
+                        sh returnStdout: true, script: """
+                        ssh-keyscan -H ${deployment_config.olts[i].ip} >> ~/.ssh/known_hosts
+                        sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'pkill bal_core_dist' || true
+                        sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'pkill openolt' || true
+                        sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} '> /broadcom/bal.log'
+                        sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} '> /broadcom/openolt.log'
+                        sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'cd /broadcom; ./bal_core_dist -C :55001 < /dev/tty1 > ./bal.log 2>&1 &'
+                        sleep 5
+                        sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'cd /broadcom; ./openolt -C 127.0.0.1:55001 < /dev/tty1 > ./openolt.log 2>&1 &'
+                        """
+                    }
+                    timeout(15) {
+                        waitUntil {
+                            onu_discovered = sh returnStdout: true, script: "sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'cat /broadcom/openolt.log | grep \"oper_state:up\" | wc -l'"
+                            return onu_discovered.toInteger() > 0
+                        }
+                    }
+                }
+            }
+            stage('Download cord-tester repo') {
+                timeout(2) {
+                    sh returnStdout: true, script: """
+                    git clone -b ${branch} ${cordRepoUrl}/cord-tester
+                    """
+                }
+            }
+            stage('Validate installed PODs') {
+                timeout(10) {
                     sh """
-                    export KUBECONFIG=/home/cord/${deployment_config.pod_config}
-                    cd /home/cord/cord-tester/src/test/robot/
+                    export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
+                    cd $WORKSPACE/cord-tester/src/test/robot/
                     rm -rf Log/ || true
                     pybot -d Log -T SanityK8POD.robot || true
                     """
-                } catch(error) { currentBuild.result = 'FAILURE' }
+                }
             }
-        }
-        stage('Configuration and Tests') {
-            timeout(10) {
-                try {
+            stage('Configurations and Tests') {
+                timeout(10) {
                     sh """
-                    export KUBECONFIG=/home/cord/${deployment_config.pod_config}
-                    cd /home/cord/cord-tester/src/test/cord-api/Properties/
-                    sed -i \"s/^\\(SERVER_IP = \\).*/\\1\'${deployment_config.node1.ip}\'/\" RestApiProperties.py
+                    export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
+                    cd $WORKSPACE/cord-tester/src/test/cord-api/Properties/
+                    sed -i \"s/^\\(SERVER_IP = \\).*/\\1\'${deployment_config.nodes[0].ip}\'/\" RestApiProperties.py
                     sed -i \"s/^\\(SERVER_PORT = \\).*/\\1\'30006\'/\" RestApiProperties.py
                     sed -i \"s/^\\(XOS_USER = \\).*/\\1\'admin@opencord.org\'/\" RestApiProperties.py
                     sed -i \"s/^\\(XOS_PASSWD = \\).*/\\1\'letmein\'/\" RestApiProperties.py
-                    cd /home/cord/cord-tester/src/test/cord-api/Tests/
+                    cd $WORKSPACE/cord-tester/src/test/cord-api/Tests/
                     rm -rf Log/ || true
                     pybot -d Log -T FabricConfig.txt || true
                     pybot -d Log -T RealOLT_Test.txt || true
                     sleep 120
                     pybot -d Log -T Subscriber_TopDown.txt || true
                     """
-                } catch(error) { currentBuild.result = 'FAILURE' }
+                }
             }
-        }
-        stage('Publish') {
-            try {
-                sh """
+            stage('Subscriber Validation and Ping Tests') {
+                timeout(30) {
+                    sh """
+                    export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
+                    cd $WORKSPACE/cord-tester/src/test/cord-api/Tests/
+                    pybot -d Log -T -v src_ip:${deployment_config.srcHost.ip} -v src_user:${deployment_config.srcHost.user} -v src_pass:${deployment_config.srcHost.pass} -v dst_user:${deployment_config.dstHost.user} -v dst_pass:${deployment_config.dstHost.pass} -v dst_ip:${deployment_config.dstHost.ip} -v dst_host_ip:${deployment_config.dstHost.hostIp} -v src_gateway:${deployment_config.srcHost.gateway} -v dst_gateway:${deployment_config.dstHost.gateway} -v init_state:awaiting-auth -v INITIAL_STATUS:FAIL -v ENABLE_STATUS:FAIL -v MACIP_STATUS:PASS Subscriber_StatusChecks.txt || true
+                    """
+                }
+            }
+            stage('Publish test results') {
+                sh returnStdout: true, script: """
                 if [ -d RobotLogs ]; then rm -r RobotLogs; fi; mkdir RobotLogs;
                 mkdir RobotLogs/TestDoc || true
-                cp -r /home/cord/cord-tester/src/test/robot/Log/* ./RobotLogs || true
-                cp -r /home/cord/cord-tester/src/test/cord-api/Tests/Log/* ./RobotLogs || true
+                cp -r $WORKSPACE/cord-tester/src/test/robot/Log/* $WORKSPACE/RobotLogs || true
+                cp -r $WORKSPACE/cord-tester/src/test/cord-api/Tests/Log/* $WORKSPACE/RobotLogs || true
                 """
                 step([$class: 'RobotPublisher',
                     disableArchiveOutput: false,
@@ -233,9 +258,14 @@
                     outputPath: '.',
                     passThreshold: 100,
                     reportFileName: 'RobotLogs/report*.html',
-                    unstableThreshold: 0])
-            } catch(error) {}
+                    unstableThreshold: 0
+                ])
+            }
+            currentBuild.result = 'SUCCESS'
+        } catch (err) {
+            currentBuild.result = 'FAILURE'
+            step([$class: 'Mailer', notifyEveryUnstableBuild: true, recipients: "${notificationEmail}", sendToIndividuals: false])
         }
-        //step([$class: 'Mailer', notifyEveryUnstableBuild: true, recipients: "${notificationEmail}", sendToIndividuals: false])
+        echo "RESULT: ${currentBuild.result}"
     }
 }