Refactor jenkinsfile for both zero-touch and top-down provisioning
Change-Id: I04d0688bb42ac9b9eff56af481d48172f201f130
diff --git a/Jenkinsfile-rcordlite-topdown b/Jenkinsfile-rcordlite-topdown
index c7cc429..a264f5b 100644
--- a/Jenkinsfile-rcordlite-topdown
+++ b/Jenkinsfile-rcordlite-topdown
@@ -12,218 +12,243 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-pod_config = null
-
node ("${TestNodeName}") {
- timeout (time: 240) {
- stage ("Parse deployment configuration file") {
- sh returnStdout: true, script: 'rm -rf ${configRepoBaseDir}'
- sh returnStdout: true, script: 'git clone -b ${branch} ${configRepoUrl}'
- deployment_config = readYaml file: "${configRepoBaseDir}${configRepoFile}"
- }
-
- stage('Export pod config') {
- timeout(10) {
- try {
- sh """
- cd /home/cord
- rm -rf /home/cord/helm-charts
- git clone -b ${branch} https://gerrit.opencord.org/helm-charts
- export KUBECONFIG=/home/cord/${deployment_config.pod_config}
- kubectl get pods
- """
- } catch(error) { currentBuild.result = 'FAILURE' }
+ timeout (100) {
+ try {
+ stage ("Parse deployment configuration file") {
+ sh returnStdout: true, script: "rm -rf ${configBaseDir}"
+ sh returnStdout: true, script: "git clone -b ${branch} ${cordRepoUrl}/${configBaseDir}"
+ deployment_config = readYaml file: "${configBaseDir}/${configDeploymentDir}/${configFileName}.yml"
}
- }
- stage('Clean up') {
- timeout(10) {
- try {
- sh """
- cd /home/cord/helm-charts
- export KUBECONFIG=/home/cord/${deployment_config.pod_config}
- helm ls | grep onos-voltha && helm delete --purge onos-voltha || true
- helm ls | grep onos-fabric && helm delete --purge onos-fabric || true
- helm ls | grep voltha && helm delete --purge voltha || true
- helm ls | grep rcord-lite && helm delete --purge rcord-lite || true
- helm ls | grep xos-core && helm delete --purge xos-core || true
+ stage('Clean up') {
+ timeout(10) {
+ sh returnStdout: true, script: """
+ rm -rf helm-charts cord-tester
+ git clone -b ${branch} ${cordRepoUrl}/helm-charts
+ export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
+ for hchart in \$(helm list -q);
+ do
+ echo "Purging chart: \${hchart}"
+ helm delete --purge "\${hchart}"
+ done
"""
- } catch(error) { currentBuild.result = 'FAILURE' }
- // Wait for helm delete
- timeout(5) {
- waitUntil {
- try {
- sh """
- export KUBECONFIG=/home/cord/${deployment_config.pod_config}
- ! helm ls | grep -E 'onos-voltha|onos-fabric|voltha |rcord-lite|xos-core'
- ! kubectl get pods --all-namespaces | grep -E 'onos-voltha|onos-fabric|voltha |rcord-lite-|xos-'
+ timeout(5) {
+ waitUntil {
+ helm_deleted = sh returnStdout: true, script: """
+ export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf &&
+ helm ls | wc -l
"""
- return true
- } catch (exception) {
- return false
+ return helm_deleted.toInteger() == 0
+ }
+ }
+ timeout(5) {
+ waitUntil {
+ kubectl_deleted = sh returnStdout: true, script: """
+ export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf &&
+ kubectl get pods --all-namespaces --no-headers | grep -v kube-system | wc -l
+ """
+ return kubectl_deleted.toInteger() == 0
}
}
}
}
- }
- stage('Install xos-core and rcord-lite') {
- timeout(10) {
- try {
- sh """
- export KUBECONFIG=/home/cord/${deployment_config.pod_config}
- cd /home/cord/helm-charts
- helm dep update xos-core
- helm install -n xos-core xos-core
- helm dep update xos-profiles/rcord-lite
- helm install -n rcord-lite xos-profiles/rcord-lite
- """
- } catch(error) { currentBuild.result = 'FAILURE' }
- }
- }
- stage('Install voltha') {
- timeout(10) {
- try {
- sh """
- export KUBECONFIG=/home/cord/${deployment_config.pod_config}
- cd /home/cord/helm-charts
- helm repo add incubator https://kubernetes-charts-incubator.storage.googleapis.com/
- helm dep build voltha
- helm install -n voltha --set etcd-operator.customResources.createEtcdClusterCRD=false voltha
- helm upgrade --set etcd-operator.customResources.createEtcdClusterCRD=true voltha ./voltha
- """
- } catch(error) { currentBuild.result = 'FAILURE' }
- }
- }
- stage('Install ONOS-Voltha and ONOS-Fabric') {
- timeout(10) {
- try {
- sh """
- export KUBECONFIG=/home/cord/${deployment_config.pod_config}
- cd /home/cord/helm-charts
- helm install -n onos-fabric -f configs/onos-fabric.yaml onos
- helm install -n onos-voltha -f configs/onos-voltha.yaml onos
- sleep 60
- """
- } catch(error) { currentBuild.result = 'FAILURE' }
- }
- }
- /*
- stage('Download cord-tester repo') {
- timeout(10) {
- try {
- sh """
- rm -rf /home/cord/cord-tester
- cd /home/cord/
- git clone -b ${branch} https://gerrit.opencord.org/cord-tester
- """
- } catch(error) { currentBuild.result = 'FAILURE' }
- }
- }
- stage('Validate Helm Chart Install') {
- timeout(10) {
- try {
- sh """
- export KUBECONFIG=/home/cord/${deployment_config.pod_config}
- cd /home/cord/cord-tester/src/test/robot/
- rm -rf Log/ || true
- pybot -d Log -T SanityK8POD.robot || true
- """
- } catch(error) { currentBuild.result = 'FAILURE' }
- }
- }*/
- stage('Wait for PODs to complete') {
- /*
- timeout(5) {
- try {
- sh """
- export KUBECONFIG=/home/cord/${deployment_config.pod_config}
- chameleon=\$(kubectl get pods | grep chameleon | cut -d' ' -f1)
- echo \$chameleon
- kubectl delete pod \$chameleon
- """
- } catch(error) { currentBuild.result = 'FAILURE' }
- }*/
- timeout(5) {
- waitUntil {
- try {
- sh """
- export KUBECONFIG=/home/cord/${deployment_config.pod_config}
- kubectl get pods | grep -i tosca-loader | grep -i complete
- kubectl get pods | grep -i chameleon | grep -i running
- kubectl get pods | grep -i onos-fabric | grep -i running
- kubectl get pods --namespace=voltha | grep -i voltha- | grep -i running
- kubectl get pods --namespace=voltha | grep -i onos-voltha | grep -i running
+ dir ("helm-charts") {
+ stage('Install voltha') {
+ timeout(10) {
+ sh returnStdout: true, script: """
+ export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
+ helm repo add incubator https://kubernetes-charts-incubator.storage.googleapis.com/
+ helm dep build voltha
+ helm install -n voltha -f ../${configBaseDir}/${configKubernetesDir}/${configFileName}.yml --set etcd-operator.customResources.createEtcdClusterCRD=false voltha
+ helm upgrade --set etcd-operator.customResources.createEtcdClusterCRD=true voltha ./voltha
"""
- return true
- } catch (exception) {
- return false
+ }
+ timeout(10) {
+ waitUntil {
+ voltha_completed = sh returnStdout: true, script: """
+ export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf &&
+ kubectl get pods -n voltha | grep -i running | grep 1/1 | wc -l
+ """
+ return voltha_completed.toInteger() == 8
+ }
+ }
+ }
+ stage('Install ONOS-Voltha') {
+ timeout(10) {
+ sh returnStdout: true, script: """
+ export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
+ helm install -n onos-voltha -f ../${configBaseDir}/${configKubernetesDir}/${configFileName}.yml -f configs/onos-voltha.yaml onos
+ """
+ }
+ timeout(10) {
+ waitUntil {
+ onos_voltha_completed = sh returnStdout: true, script: """
+ export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf &&
+ kubectl get pods -n voltha | grep -i onos-voltha | grep -i running | grep 1/1 | wc -l
+ """
+ return onos_voltha_completed.toInteger() == 1
+ }
+ }
+ }
+ stage('Install ONOS-Fabric') {
+ timeout(10) {
+ sh returnStdout: true, script: """
+ export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
+ helm install -n onos-fabric -f ../${configBaseDir}/${configKubernetesDir}/${configFileName}.yml -f configs/onos-fabric.yaml onos
+ """
+ }
+ timeout(10) {
+ waitUntil {
+ onos_fabric_completed = sh returnStdout: true, script: """
+ export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf &&
+ kubectl get pods | grep -i onos-fabric | grep -i running | grep 1/1 | wc -l
+ """
+ return onos_fabric_completed.toInteger() == 1
+ }
+ }
+ }
+ stage('Install xos-core') {
+ timeout(10) {
+ sh returnStdout: true, script: """
+ export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
+ helm dep update xos-core
+ helm install -f ../${configBaseDir}/${configKubernetesDir}/${configFileName}.yml -n xos-core xos-core
+ """
+ }
+ timeout(10) {
+ waitUntil {
+ xos_core_completed = sh returnStdout: true, script: """
+ export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf &&
+ kubectl get pods | grep -i xos | grep -i running | grep 1/1 | wc -l
+ """
+ return xos_core_completed.toInteger() == 7
+ }
+ }
+ }
+ stage('Install rcord-lite') {
+ timeout(10) {
+ sh returnStdout: true, script: """
+ export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
+ helm dep update xos-profiles/rcord-lite
+ helm install -f ../${configBaseDir}/${configKubernetesDir}/${configFileName}.yml -n rcord-lite xos-profiles/rcord-lite
+ """
+ }
+ timeout(10) {
+ waitUntil {
+ rcord_tosca_completed = sh returnStdout: true, script: """
+ export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf &&
+ kubectl get pods | grep -i tosca-loader | grep -i completed | wc -l
+ """
+ return rcord_tosca_completed.toInteger() == 1
+ }
}
}
}
- }
- stage('Restart OLT processes') {
- timeout(5) {
- try {
- sh """
- export KUBECONFIG=/home/cord/${deployment_config.pod_config}
- sshpass -p onl ssh -l root 10.192.4.218 'pkill bal_core_dist' || true
- sshpass -p onl ssh -l root 10.192.4.218 'pkill openolt' || true
- sshpass -p onl ssh -l root 10.192.4.218 'cd /broadcom; ./bal_core_dist -C 127.0.0.1:40000 -A 127.0.0.1:50000 < /dev/tty1 > ./bal.log 2>&1 &'
- sshpass -p onl ssh -l root 10.192.4.218 'cd /broadcom; ./openolt -C 127.0.0.1:40000 -A 127.0.0.1:50000 < /dev/tty1 > ./openolt.log 2>&1 &'
- sleep 120
+ stage('Reinstall OLT software') {
+ for(int i=0; i < deployment_config.olts.size(); i++) {
+ sh returnStdout: true, script: """
+ sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'dpkg --remove asfvolt16 && dpkg --purge asfvolt16'
"""
- } catch(error) { currentBuild.result = 'FAILURE' }
- }
- }
- stage('Download cord-tester repo') {
- timeout(10) {
- try {
- sh """
- rm -rf /home/cord/cord-tester
- cd /home/cord/
- git clone -b ${branch} https://gerrit.opencord.org/cord-tester
+ timeout(5) {
+ waitUntil {
+ olt_sw_present = sh returnStdout: true, script: """
+ sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'dpkg --list | grep asfvolt16 | wc -l'
+ """
+ return olt_sw_present.toInteger() == 0
+ }
+ }
+ sh returnStdout: true, script: """
+ sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} "dpkg --install ${oltDebVersion}"
"""
- } catch(error) { currentBuild.result = 'FAILURE' }
+ timeout(5) {
+ waitUntil {
+ olt_sw_present = sh returnStdout: true, script: """
+ sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'dpkg --list | grep asfvolt16 | wc -l'
+ """
+ return olt_sw_present.toInteger() == 1
+ }
+ }
+ // If the OLT is connected to a 40G switch interface, set the NNI port to be downgraded
+ if ("${deployment_config.olts[i].fortygig}" != null && "${deployment_config.olts[i].fortygig}" == 'true') {
+ sh returnStdout: true, script: """
+ sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'echo "port ce128 sp=40000" >> /broadcom/qax.soc'
+ sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} '/opt/bcm68620/svk_init.sh'
+ """
+ }
+ }
}
- }
- stage('Validate Helm Chart Install') {
- timeout(10) {
- try {
+ stage('Restart OLT processes') {
+ for(int i=0; i < deployment_config.olts.size(); i++) {
+ timeout(5) {
+ sh returnStdout: true, script: """
+ ssh-keyscan -H ${deployment_config.olts[i].ip} >> ~/.ssh/known_hosts
+ sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'pkill bal_core_dist' || true
+ sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'pkill openolt' || true
+ sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} '> /broadcom/bal.log'
+ sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} '> /broadcom/openolt.log'
+ sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'cd /broadcom; ./bal_core_dist -C :55001 < /dev/tty1 > ./bal.log 2>&1 &'
+ sleep 5
+ sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'cd /broadcom; ./openolt -C 127.0.0.1:55001 < /dev/tty1 > ./openolt.log 2>&1 &'
+ """
+ }
+ timeout(15) {
+ waitUntil {
+ onu_discovered = sh returnStdout: true, script: "sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'cat /broadcom/openolt.log | grep \"oper_state:up\" | wc -l'"
+ return onu_discovered.toInteger() > 0
+ }
+ }
+ }
+ }
+ stage('Download cord-tester repo') {
+ timeout(2) {
+ sh returnStdout: true, script: """
+ git clone -b ${branch} ${cordRepoUrl}/cord-tester
+ """
+ }
+ }
+ stage('Validate installed PODs') {
+ timeout(10) {
sh """
- export KUBECONFIG=/home/cord/${deployment_config.pod_config}
- cd /home/cord/cord-tester/src/test/robot/
+ export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
+ cd $WORKSPACE/cord-tester/src/test/robot/
rm -rf Log/ || true
pybot -d Log -T SanityK8POD.robot || true
"""
- } catch(error) { currentBuild.result = 'FAILURE' }
+ }
}
- }
- stage('Configuration and Tests') {
- timeout(10) {
- try {
+ stage('Configurations and Tests') {
+ timeout(10) {
sh """
- export KUBECONFIG=/home/cord/${deployment_config.pod_config}
- cd /home/cord/cord-tester/src/test/cord-api/Properties/
- sed -i \"s/^\\(SERVER_IP = \\).*/\\1\'${deployment_config.node1.ip}\'/\" RestApiProperties.py
+ export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
+ cd $WORKSPACE/cord-tester/src/test/cord-api/Properties/
+ sed -i \"s/^\\(SERVER_IP = \\).*/\\1\'${deployment_config.nodes[0].ip}\'/\" RestApiProperties.py
sed -i \"s/^\\(SERVER_PORT = \\).*/\\1\'30006\'/\" RestApiProperties.py
sed -i \"s/^\\(XOS_USER = \\).*/\\1\'admin@opencord.org\'/\" RestApiProperties.py
sed -i \"s/^\\(XOS_PASSWD = \\).*/\\1\'letmein\'/\" RestApiProperties.py
- cd /home/cord/cord-tester/src/test/cord-api/Tests/
+ cd $WORKSPACE/cord-tester/src/test/cord-api/Tests/
rm -rf Log/ || true
pybot -d Log -T FabricConfig.txt || true
pybot -d Log -T RealOLT_Test.txt || true
sleep 120
pybot -d Log -T Subscriber_TopDown.txt || true
"""
- } catch(error) { currentBuild.result = 'FAILURE' }
+ }
}
- }
- stage('Publish') {
- try {
- sh """
+ stage('Subscriber Validation and Ping Tests') {
+ timeout(30) {
+ sh """
+ export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
+ cd $WORKSPACE/cord-tester/src/test/cord-api/Tests/
+ pybot -d Log -T -v src_ip:${deployment_config.srcHost.ip} -v src_user:${deployment_config.srcHost.user} -v src_pass:${deployment_config.srcHost.pass} -v dst_user:${deployment_config.dstHost.user} -v dst_pass:${deployment_config.dstHost.pass} -v dst_ip:${deployment_config.dstHost.ip} -v dst_host_ip:${deployment_config.dstHost.hostIp} -v src_gateway:${deployment_config.srcHost.gateway} -v dst_gateway:${deployment_config.dstHost.gateway} -v init_state:awaiting-auth -v INITIAL_STATUS:FAIL -v ENABLE_STATUS:FAIL -v MACIP_STATUS:PASS Subscriber_StatusChecks.txt || true
+ """
+ }
+ }
+ stage('Publish test results') {
+ sh returnStdout: true, script: """
if [ -d RobotLogs ]; then rm -r RobotLogs; fi; mkdir RobotLogs;
mkdir RobotLogs/TestDoc || true
- cp -r /home/cord/cord-tester/src/test/robot/Log/* ./RobotLogs || true
- cp -r /home/cord/cord-tester/src/test/cord-api/Tests/Log/* ./RobotLogs || true
+ cp -r $WORKSPACE/cord-tester/src/test/robot/Log/* $WORKSPACE/RobotLogs || true
+ cp -r $WORKSPACE/cord-tester/src/test/cord-api/Tests/Log/* $WORKSPACE/RobotLogs || true
"""
step([$class: 'RobotPublisher',
disableArchiveOutput: false,
@@ -233,9 +258,14 @@
outputPath: '.',
passThreshold: 100,
reportFileName: 'RobotLogs/report*.html',
- unstableThreshold: 0])
- } catch(error) {}
+ unstableThreshold: 0
+ ])
+ }
+ currentBuild.result = 'SUCCESS'
+ } catch (err) {
+ currentBuild.result = 'FAILURE'
+ step([$class: 'Mailer', notifyEveryUnstableBuild: true, recipients: "${notificationEmail}", sendToIndividuals: false])
}
- //step([$class: 'Mailer', notifyEveryUnstableBuild: true, recipients: "${notificationEmail}", sendToIndividuals: false])
+ echo "RESULT: ${currentBuild.result}"
}
}