Suchitra Vemuri | d9cffa8 | 2018-07-25 14:06:47 -0700 | [diff] [blame] | 1 | // Copyright 2017-present Open Networking Foundation |
| 2 | // |
| 3 | // Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | // you may not use this file except in compliance with the License. |
| 5 | // You may obtain a copy of the License at |
| 6 | // |
| 7 | // http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | // |
| 9 | // Unless required by applicable law or agreed to in writing, software |
| 10 | // distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | // See the License for the specific language governing permissions and |
| 13 | // limitations under the License. |
| 14 | |
Suchitra Vemuri | d9cffa8 | 2018-07-25 14:06:47 -0700 | [diff] [blame] | 15 | node ("${TestNodeName}") { |
You Wang | d6c5e8c | 2018-08-29 12:49:51 -0700 | [diff] [blame] | 16 | timeout (100) { |
| 17 | try { |
| 18 | stage ("Parse deployment configuration file") { |
| 19 | sh returnStdout: true, script: "rm -rf ${configBaseDir}" |
| 20 | sh returnStdout: true, script: "git clone -b ${branch} ${cordRepoUrl}/${configBaseDir}" |
| 21 | deployment_config = readYaml file: "${configBaseDir}/${configDeploymentDir}/${configFileName}.yml" |
Suchitra Vemuri | d9cffa8 | 2018-07-25 14:06:47 -0700 | [diff] [blame] | 22 | } |
You Wang | d6c5e8c | 2018-08-29 12:49:51 -0700 | [diff] [blame] | 23 | stage('Clean up') { |
| 24 | timeout(10) { |
| 25 | sh returnStdout: true, script: """ |
| 26 | rm -rf helm-charts cord-tester |
| 27 | git clone -b ${branch} ${cordRepoUrl}/helm-charts |
| 28 | export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf |
| 29 | for hchart in \$(helm list -q); |
| 30 | do |
| 31 | echo "Purging chart: \${hchart}" |
| 32 | helm delete --purge "\${hchart}" |
| 33 | done |
Suchitra Vemuri | d9cffa8 | 2018-07-25 14:06:47 -0700 | [diff] [blame] | 34 | """ |
You Wang | d6c5e8c | 2018-08-29 12:49:51 -0700 | [diff] [blame] | 35 | timeout(5) { |
| 36 | waitUntil { |
| 37 | helm_deleted = sh returnStdout: true, script: """ |
| 38 | export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf && |
| 39 | helm ls | wc -l |
Suchitra Vemuri | d9cffa8 | 2018-07-25 14:06:47 -0700 | [diff] [blame] | 40 | """ |
You Wang | d6c5e8c | 2018-08-29 12:49:51 -0700 | [diff] [blame] | 41 | return helm_deleted.toInteger() == 0 |
| 42 | } |
| 43 | } |
| 44 | timeout(5) { |
| 45 | waitUntil { |
| 46 | kubectl_deleted = sh returnStdout: true, script: """ |
| 47 | export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf && |
| 48 | kubectl get pods --all-namespaces --no-headers | grep -v kube-system | wc -l |
| 49 | """ |
| 50 | return kubectl_deleted.toInteger() == 0 |
Suchitra Vemuri | d9cffa8 | 2018-07-25 14:06:47 -0700 | [diff] [blame] | 51 | } |
| 52 | } |
| 53 | } |
| 54 | } |
You Wang | d6c5e8c | 2018-08-29 12:49:51 -0700 | [diff] [blame] | 55 | dir ("helm-charts") { |
Luca Prete | 08be18b | 2018-08-29 14:10:06 -0700 | [diff] [blame^] | 56 | stage('Install Voltha Kafka') { |
| 57 | timeout(10) { |
| 58 | sh returnStdout: true, script: """ |
| 59 | export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf |
| 60 | helm install --name voltha-kafka --set replicas=1 --set persistence.enabled=false --set zookeeper.replicaCount=1 --set zookeeper.persistence.enabled=false incubator/kafka |
| 61 | """ |
| 62 | } |
| 63 | timeout(10) { |
| 64 | waitUntil { |
| 65 | kafka_instances_running = sh returnStdout: true, script: """ |
| 66 | export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf && |
| 67 | kubectl get pods | grep voltha-kafka | grep -i running | grep 1/1 | wc -l |
| 68 | """ |
| 69 | return kafka_instances_running.toInteger() == 2 |
| 70 | } |
| 71 | } |
| 72 | } |
You Wang | d6c5e8c | 2018-08-29 12:49:51 -0700 | [diff] [blame] | 73 | stage('Install voltha') { |
| 74 | timeout(10) { |
| 75 | sh returnStdout: true, script: """ |
| 76 | export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf |
| 77 | helm repo add incubator https://kubernetes-charts-incubator.storage.googleapis.com/ |
| 78 | helm dep build voltha |
| 79 | helm install -n voltha -f ../${configBaseDir}/${configKubernetesDir}/${configFileName}.yml --set etcd-operator.customResources.createEtcdClusterCRD=false voltha |
| 80 | helm upgrade --set etcd-operator.customResources.createEtcdClusterCRD=true voltha ./voltha |
Suchitra Vemuri | d9cffa8 | 2018-07-25 14:06:47 -0700 | [diff] [blame] | 81 | """ |
You Wang | d6c5e8c | 2018-08-29 12:49:51 -0700 | [diff] [blame] | 82 | } |
| 83 | timeout(10) { |
| 84 | waitUntil { |
| 85 | voltha_completed = sh returnStdout: true, script: """ |
| 86 | export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf && |
| 87 | kubectl get pods -n voltha | grep -i running | grep 1/1 | wc -l |
| 88 | """ |
| 89 | return voltha_completed.toInteger() == 8 |
| 90 | } |
| 91 | } |
| 92 | } |
| 93 | stage('Install ONOS-Voltha') { |
| 94 | timeout(10) { |
| 95 | sh returnStdout: true, script: """ |
| 96 | export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf |
| 97 | helm install -n onos-voltha -f ../${configBaseDir}/${configKubernetesDir}/${configFileName}.yml -f configs/onos-voltha.yaml onos |
| 98 | """ |
| 99 | } |
| 100 | timeout(10) { |
| 101 | waitUntil { |
| 102 | onos_voltha_completed = sh returnStdout: true, script: """ |
| 103 | export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf && |
| 104 | kubectl get pods -n voltha | grep -i onos-voltha | grep -i running | grep 1/1 | wc -l |
| 105 | """ |
| 106 | return onos_voltha_completed.toInteger() == 1 |
| 107 | } |
| 108 | } |
| 109 | } |
| 110 | stage('Install ONOS-Fabric') { |
| 111 | timeout(10) { |
| 112 | sh returnStdout: true, script: """ |
| 113 | export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf |
| 114 | helm install -n onos-fabric -f ../${configBaseDir}/${configKubernetesDir}/${configFileName}.yml -f configs/onos-fabric.yaml onos |
| 115 | """ |
| 116 | } |
| 117 | timeout(10) { |
| 118 | waitUntil { |
| 119 | onos_fabric_completed = sh returnStdout: true, script: """ |
| 120 | export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf && |
| 121 | kubectl get pods | grep -i onos-fabric | grep -i running | grep 1/1 | wc -l |
| 122 | """ |
| 123 | return onos_fabric_completed.toInteger() == 1 |
| 124 | } |
| 125 | } |
| 126 | } |
| 127 | stage('Install xos-core') { |
| 128 | timeout(10) { |
| 129 | sh returnStdout: true, script: """ |
| 130 | export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf |
| 131 | helm dep update xos-core |
| 132 | helm install -f ../${configBaseDir}/${configKubernetesDir}/${configFileName}.yml -n xos-core xos-core |
| 133 | """ |
| 134 | } |
| 135 | timeout(10) { |
| 136 | waitUntil { |
| 137 | xos_core_completed = sh returnStdout: true, script: """ |
| 138 | export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf && |
| 139 | kubectl get pods | grep -i xos | grep -i running | grep 1/1 | wc -l |
| 140 | """ |
| 141 | return xos_core_completed.toInteger() == 7 |
| 142 | } |
| 143 | } |
| 144 | } |
| 145 | stage('Install rcord-lite') { |
| 146 | timeout(10) { |
| 147 | sh returnStdout: true, script: """ |
| 148 | export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf |
| 149 | helm dep update xos-profiles/rcord-lite |
| 150 | helm install -f ../${configBaseDir}/${configKubernetesDir}/${configFileName}.yml -n rcord-lite xos-profiles/rcord-lite |
| 151 | """ |
| 152 | } |
| 153 | timeout(10) { |
| 154 | waitUntil { |
| 155 | rcord_tosca_completed = sh returnStdout: true, script: """ |
| 156 | export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf && |
| 157 | kubectl get pods | grep -i tosca-loader | grep -i completed | wc -l |
| 158 | """ |
| 159 | return rcord_tosca_completed.toInteger() == 1 |
| 160 | } |
Suchitra Vemuri | d9cffa8 | 2018-07-25 14:06:47 -0700 | [diff] [blame] | 161 | } |
| 162 | } |
| 163 | } |
You Wang | d6c5e8c | 2018-08-29 12:49:51 -0700 | [diff] [blame] | 164 | stage('Reinstall OLT software') { |
| 165 | for(int i=0; i < deployment_config.olts.size(); i++) { |
| 166 | sh returnStdout: true, script: """ |
| 167 | sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'dpkg --remove asfvolt16 && dpkg --purge asfvolt16' |
Suchitra Vemuri | d9cffa8 | 2018-07-25 14:06:47 -0700 | [diff] [blame] | 168 | """ |
You Wang | d6c5e8c | 2018-08-29 12:49:51 -0700 | [diff] [blame] | 169 | timeout(5) { |
| 170 | waitUntil { |
| 171 | olt_sw_present = sh returnStdout: true, script: """ |
| 172 | sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'dpkg --list | grep asfvolt16 | wc -l' |
| 173 | """ |
| 174 | return olt_sw_present.toInteger() == 0 |
| 175 | } |
| 176 | } |
| 177 | sh returnStdout: true, script: """ |
| 178 | sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} "dpkg --install ${oltDebVersion}" |
Suchitra Vemuri | d9cffa8 | 2018-07-25 14:06:47 -0700 | [diff] [blame] | 179 | """ |
You Wang | d6c5e8c | 2018-08-29 12:49:51 -0700 | [diff] [blame] | 180 | timeout(5) { |
| 181 | waitUntil { |
| 182 | olt_sw_present = sh returnStdout: true, script: """ |
| 183 | sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'dpkg --list | grep asfvolt16 | wc -l' |
| 184 | """ |
| 185 | return olt_sw_present.toInteger() == 1 |
| 186 | } |
| 187 | } |
| 188 | // If the OLT is connected to a 40G switch interface, set the NNI port to be downgraded |
| 189 | if ("${deployment_config.olts[i].fortygig}" != null && "${deployment_config.olts[i].fortygig}" == 'true') { |
| 190 | sh returnStdout: true, script: """ |
| 191 | sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'echo "port ce128 sp=40000" >> /broadcom/qax.soc' |
| 192 | sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} '/opt/bcm68620/svk_init.sh' |
| 193 | """ |
| 194 | } |
| 195 | } |
Suchitra Vemuri | d9cffa8 | 2018-07-25 14:06:47 -0700 | [diff] [blame] | 196 | } |
You Wang | d6c5e8c | 2018-08-29 12:49:51 -0700 | [diff] [blame] | 197 | stage('Restart OLT processes') { |
| 198 | for(int i=0; i < deployment_config.olts.size(); i++) { |
| 199 | timeout(5) { |
| 200 | sh returnStdout: true, script: """ |
| 201 | ssh-keyscan -H ${deployment_config.olts[i].ip} >> ~/.ssh/known_hosts |
| 202 | sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'pkill bal_core_dist' || true |
| 203 | sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'pkill openolt' || true |
| 204 | sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} '> /broadcom/bal.log' |
| 205 | sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} '> /broadcom/openolt.log' |
| 206 | sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'cd /broadcom; ./bal_core_dist -C :55001 < /dev/tty1 > ./bal.log 2>&1 &' |
| 207 | sleep 5 |
| 208 | sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'cd /broadcom; ./openolt -C 127.0.0.1:55001 < /dev/tty1 > ./openolt.log 2>&1 &' |
| 209 | """ |
| 210 | } |
| 211 | timeout(15) { |
| 212 | waitUntil { |
| 213 | onu_discovered = sh returnStdout: true, script: "sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'cat /broadcom/openolt.log | grep \"oper_state:up\" | wc -l'" |
| 214 | return onu_discovered.toInteger() > 0 |
| 215 | } |
| 216 | } |
| 217 | } |
| 218 | } |
| 219 | stage('Download cord-tester repo') { |
| 220 | timeout(2) { |
| 221 | sh returnStdout: true, script: """ |
| 222 | git clone -b ${branch} ${cordRepoUrl}/cord-tester |
| 223 | """ |
| 224 | } |
| 225 | } |
| 226 | stage('Validate installed PODs') { |
| 227 | timeout(10) { |
Suchitra Vemuri | d9cffa8 | 2018-07-25 14:06:47 -0700 | [diff] [blame] | 228 | sh """ |
You Wang | d6c5e8c | 2018-08-29 12:49:51 -0700 | [diff] [blame] | 229 | export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf |
| 230 | cd $WORKSPACE/cord-tester/src/test/robot/ |
Suchitra Vemuri | d9cffa8 | 2018-07-25 14:06:47 -0700 | [diff] [blame] | 231 | rm -rf Log/ || true |
Kailash Khalasi | e9ad83c | 2018-08-07 11:22:36 -0700 | [diff] [blame] | 232 | pybot -d Log -T SanityK8POD.robot || true |
Suchitra Vemuri | d9cffa8 | 2018-07-25 14:06:47 -0700 | [diff] [blame] | 233 | """ |
You Wang | d6c5e8c | 2018-08-29 12:49:51 -0700 | [diff] [blame] | 234 | } |
Suchitra Vemuri | d9cffa8 | 2018-07-25 14:06:47 -0700 | [diff] [blame] | 235 | } |
You Wang | d6c5e8c | 2018-08-29 12:49:51 -0700 | [diff] [blame] | 236 | stage('Configurations and Tests') { |
| 237 | timeout(10) { |
Suchitra Vemuri | d9cffa8 | 2018-07-25 14:06:47 -0700 | [diff] [blame] | 238 | sh """ |
You Wang | d6c5e8c | 2018-08-29 12:49:51 -0700 | [diff] [blame] | 239 | export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf |
| 240 | cd $WORKSPACE/cord-tester/src/test/cord-api/Properties/ |
| 241 | sed -i \"s/^\\(SERVER_IP = \\).*/\\1\'${deployment_config.nodes[0].ip}\'/\" RestApiProperties.py |
Suchitra Vemuri | d9cffa8 | 2018-07-25 14:06:47 -0700 | [diff] [blame] | 242 | sed -i \"s/^\\(SERVER_PORT = \\).*/\\1\'30006\'/\" RestApiProperties.py |
| 243 | sed -i \"s/^\\(XOS_USER = \\).*/\\1\'admin@opencord.org\'/\" RestApiProperties.py |
| 244 | sed -i \"s/^\\(XOS_PASSWD = \\).*/\\1\'letmein\'/\" RestApiProperties.py |
You Wang | d6c5e8c | 2018-08-29 12:49:51 -0700 | [diff] [blame] | 245 | cd $WORKSPACE/cord-tester/src/test/cord-api/Tests/ |
Suchitra Vemuri | d9cffa8 | 2018-07-25 14:06:47 -0700 | [diff] [blame] | 246 | rm -rf Log/ || true |
Kailash Khalasi | e9ad83c | 2018-08-07 11:22:36 -0700 | [diff] [blame] | 247 | pybot -d Log -T FabricConfig.txt || true |
| 248 | pybot -d Log -T RealOLT_Test.txt || true |
Suchitra Vemuri | d9cffa8 | 2018-07-25 14:06:47 -0700 | [diff] [blame] | 249 | sleep 120 |
Kailash Khalasi | e9ad83c | 2018-08-07 11:22:36 -0700 | [diff] [blame] | 250 | pybot -d Log -T Subscriber_TopDown.txt || true |
Suchitra Vemuri | d9cffa8 | 2018-07-25 14:06:47 -0700 | [diff] [blame] | 251 | """ |
You Wang | d6c5e8c | 2018-08-29 12:49:51 -0700 | [diff] [blame] | 252 | } |
Suchitra Vemuri | d9cffa8 | 2018-07-25 14:06:47 -0700 | [diff] [blame] | 253 | } |
You Wang | d6c5e8c | 2018-08-29 12:49:51 -0700 | [diff] [blame] | 254 | stage('Subscriber Validation and Ping Tests') { |
| 255 | timeout(30) { |
| 256 | sh """ |
| 257 | export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf |
| 258 | cd $WORKSPACE/cord-tester/src/test/cord-api/Tests/ |
| 259 | pybot -d Log -T -v src_ip:${deployment_config.srcHost.ip} -v src_user:${deployment_config.srcHost.user} -v src_pass:${deployment_config.srcHost.pass} -v dst_user:${deployment_config.dstHost.user} -v dst_pass:${deployment_config.dstHost.pass} -v dst_ip:${deployment_config.dstHost.ip} -v dst_host_ip:${deployment_config.dstHost.hostIp} -v src_gateway:${deployment_config.srcHost.gateway} -v dst_gateway:${deployment_config.dstHost.gateway} -v init_state:awaiting-auth -v INITIAL_STATUS:FAIL -v ENABLE_STATUS:FAIL -v MACIP_STATUS:PASS Subscriber_StatusChecks.txt || true |
| 260 | """ |
| 261 | } |
| 262 | } |
| 263 | stage('Publish test results') { |
| 264 | sh returnStdout: true, script: """ |
Suchitra Vemuri | d9cffa8 | 2018-07-25 14:06:47 -0700 | [diff] [blame] | 265 | if [ -d RobotLogs ]; then rm -r RobotLogs; fi; mkdir RobotLogs; |
| 266 | mkdir RobotLogs/TestDoc || true |
You Wang | d6c5e8c | 2018-08-29 12:49:51 -0700 | [diff] [blame] | 267 | cp -r $WORKSPACE/cord-tester/src/test/robot/Log/* $WORKSPACE/RobotLogs || true |
| 268 | cp -r $WORKSPACE/cord-tester/src/test/cord-api/Tests/Log/* $WORKSPACE/RobotLogs || true |
Suchitra Vemuri | d9cffa8 | 2018-07-25 14:06:47 -0700 | [diff] [blame] | 269 | """ |
| 270 | step([$class: 'RobotPublisher', |
| 271 | disableArchiveOutput: false, |
| 272 | logFileName: 'RobotLogs/log*.html', |
| 273 | otherFiles: '', |
| 274 | outputFileName: 'RobotLogs/output*.xml', |
| 275 | outputPath: '.', |
| 276 | passThreshold: 100, |
| 277 | reportFileName: 'RobotLogs/report*.html', |
You Wang | d6c5e8c | 2018-08-29 12:49:51 -0700 | [diff] [blame] | 278 | unstableThreshold: 0 |
| 279 | ]) |
| 280 | } |
| 281 | currentBuild.result = 'SUCCESS' |
| 282 | } catch (err) { |
| 283 | currentBuild.result = 'FAILURE' |
| 284 | step([$class: 'Mailer', notifyEveryUnstableBuild: true, recipients: "${notificationEmail}", sendToIndividuals: false]) |
Suchitra Vemuri | d9cffa8 | 2018-07-25 14:06:47 -0700 | [diff] [blame] | 285 | } |
You Wang | d6c5e8c | 2018-08-29 12:49:51 -0700 | [diff] [blame] | 286 | echo "RESULT: ${currentBuild.result}" |
Suchitra Vemuri | d9cffa8 | 2018-07-25 14:06:47 -0700 | [diff] [blame] | 287 | } |
| 288 | } |