Suchitra Vemuri | a8e7bb2 | 2019-10-22 15:50:05 -0700 | [diff] [blame] | 1 | // Copyright 2017-present Open Networking Foundation |
| 2 | // |
| 3 | // Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | // you may not use this file except in compliance with the License. |
| 5 | // You may obtain a copy of the License at |
| 6 | // |
| 7 | // http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | // |
| 9 | // Unless required by applicable law or agreed to in writing, software |
| 10 | // distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | // See the License for the specific language governing permissions and |
| 13 | // limitations under the License. |
| 14 | |
| 15 | node ("${TestNodeName}") { |
Andrea Campanella | 80dcae0 | 2020-01-28 11:23:26 +0100 | [diff] [blame] | 16 | def withKind = false |
| 17 | if (params.withKind != null){ |
| 18 | withKind = params.withKind |
| 19 | } |
Suchitra Vemuri | a8e7bb2 | 2019-10-22 15:50:05 -0700 | [diff] [blame] | 20 | timeout (100) { |
| 21 | try { |
| 22 | stage ("Parse deployment configuration file") { |
| 23 | sh returnStdout: true, script: "rm -rf helm-repo-tools ${configBaseDir} voltha-system-tests kind-voltha" |
| 24 | sh returnStdout: true, script: "git clone -b master ${cordRepoUrl}/helm-repo-tools" |
| 25 | sh returnStdout: true, script: "git clone -b master ${cordRepoUrl}/${configBaseDir}" |
| 26 | sh returnStdout: true, script: "git clone -b master ${cordRepoUrl}/voltha-system-tests" |
Suchitra Vemuri | 6ab409e | 2020-03-19 18:08:33 -0700 | [diff] [blame^] | 27 | sh returnStdout: true, script: "git clone https://github.com/ciena/kind-voltha.git" |
Suchitra Vemuri | 2c6d081 | 2020-03-16 12:41:52 -0700 | [diff] [blame] | 28 | if ( params.workFlow == "DT" ) { |
| 29 | deployment_config = readYaml file: "${configBaseDir}/${configDeploymentDir}/${configFileName}-DT.yaml" |
| 30 | } |
| 31 | else |
| 32 | { |
| 33 | deployment_config = readYaml file: "${configBaseDir}/${configDeploymentDir}/${configFileName}.yaml" |
| 34 | } |
Suchitra Vemuri | a8e7bb2 | 2019-10-22 15:50:05 -0700 | [diff] [blame] | 35 | } |
| 36 | stage('Clean up') { |
| 37 | timeout(10) { |
| 38 | sh returnStdout: true, script: """ |
| 39 | export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf |
| 40 | for hchart in \$(helm list -q | grep -E -v 'docker-registry|mavenrepo|ponnet'); |
| 41 | do |
| 42 | echo "Purging chart: \${hchart}" |
| 43 | helm delete --purge "\${hchart}" |
| 44 | done |
| 45 | """ |
| 46 | timeout(5) { |
| 47 | waitUntil { |
| 48 | helm_deleted = sh returnStdout: true, script: """ |
| 49 | export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf && |
| 50 | helm ls -q | grep -E -v 'docker-registry|mavenrepo|ponnet' | wc -l |
| 51 | """ |
| 52 | return helm_deleted.toInteger() == 0 |
| 53 | } |
| 54 | } |
| 55 | timeout(5) { |
| 56 | waitUntil { |
| 57 | kubectl_deleted = sh returnStdout: true, script: """ |
| 58 | export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf && |
| 59 | kubectl get pods --all-namespaces --no-headers | grep -E -v 'kube-system|docker-registry|mavenrepo|ponnet' | wc -l |
| 60 | """ |
| 61 | return kubectl_deleted.toInteger() == 0 |
| 62 | } |
| 63 | } |
Andrea Campanella | 80dcae0 | 2020-01-28 11:23:26 +0100 | [diff] [blame] | 64 | timeout(1) { |
| 65 | sh returnStdout: true, script: """ |
| 66 | port_fwd_pid=`ps -ax | grep "port-forward -n default service/onos-openflow" | grep -v "grep" | awk '{print \$1}'` |
| 67 | if [[ "" != "\$port_fwd_pid" ]]; then |
| 68 | kill -9 \$port_fwd_pid > /dev/null 2>&1 |
| 69 | fi |
| 70 | """ |
| 71 | } |
Suchitra Vemuri | a8e7bb2 | 2019-10-22 15:50:05 -0700 | [diff] [blame] | 72 | } |
| 73 | } |
Suchitra Vemuri | 4ac4463 | 2019-11-12 13:41:08 -0800 | [diff] [blame] | 74 | stage('Install Voltha') { |
Suchitra Vemuri | a8e7bb2 | 2019-10-22 15:50:05 -0700 | [diff] [blame] | 75 | timeout(10) { |
| 76 | sh returnStdout: true, script: """ |
| 77 | export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf |
| 78 | cd kind-voltha/ |
Suchitra Vemuri | 4ac4463 | 2019-11-12 13:41:08 -0800 | [diff] [blame] | 79 | if ( ${released} ); then |
Suchitra Vemuri | 6ab409e | 2020-03-19 18:08:33 -0700 | [diff] [blame^] | 80 | export EXTRA_HELM_FLAGS="--set defaults.image_tag=null,images.onos.tag=4.0.0,images.onos.repository=voltha/voltha-onos,use_ofagent_go=false,images.ofagent.repository=voltha/voltha-ofagent,images.ofagent.tag=2.2.4" |
Suchitra Vemuri | 946da61 | 2020-02-25 16:36:30 -0800 | [diff] [blame] | 81 | cd $WORKSPACE/kind-voltha |
Suchitra Vemuri | 6ab409e | 2020-03-19 18:08:33 -0700 | [diff] [blame^] | 82 | source releases/voltha-2.3.0-rc1 |
Suchitra Vemuri | 225a2c8 | 2020-03-04 18:16:40 -0800 | [diff] [blame] | 83 | elif ( ${ofagentGo} ); then |
| 84 | export EXTRA_HELM_FLAGS='-f $WORKSPACE/${configBaseDir}/${configKubernetesDir}/voltha/${configFileName}-ofagent-go.yml' |
Suchitra Vemuri | 4ac4463 | 2019-11-12 13:41:08 -0800 | [diff] [blame] | 85 | else |
| 86 | export EXTRA_HELM_FLAGS='-f $WORKSPACE/${configBaseDir}/${configKubernetesDir}/voltha/${configFileName}.yml' |
| 87 | fi |
| 88 | |
| 89 | # VOL-2194 ONOS SSH and REST ports hardcoded to 30115/30120 in tests |
Suchitra Vemuri | ae9cecb | 2020-03-12 12:59:14 -0700 | [diff] [blame] | 90 | if [[ "${workFlow}" == "DT" ]]; then |
| 91 | ONOS_SSH_PORT=30115 ONOS_API_PORT=30120 VOLTHA_LOG_LEVEL=DEBUG WITH_SIM_ADAPTERS=n WITH_TP=yes DEPLOY_K8S=no INSTALL_KUBECTL=no INSTALL_HELM=no WITH_EAPOL=no WITH_DHCP=no WITH_IGMP=no WITH_RADIUS=no FANCY=0 ./voltha up |
| 92 | else |
| 93 | ONOS_SSH_PORT=30115 ONOS_API_PORT=30120 VOLTHA_LOG_LEVEL=DEBUG WITH_SIM_ADAPTERS=n WITH_RADIUS=y WITH_TP=yes DEPLOY_K8S=no INSTALL_KUBECTL=no INSTALL_HELM=no FANCY=0 ./voltha up |
| 94 | fi |
Suchitra Vemuri | 4ac4463 | 2019-11-12 13:41:08 -0800 | [diff] [blame] | 95 | |
| 96 | kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.image}{'\\t'}{.imageID}{'\\n'}" | sort | uniq -c |
| 97 | kubectl get nodes -o wide |
| 98 | kubectl get pods -n voltha -o wide |
Suchitra Vemuri | a8e7bb2 | 2019-10-22 15:50:05 -0700 | [diff] [blame] | 99 | """ |
Suchitra Vemuri | 4ac4463 | 2019-11-12 13:41:08 -0800 | [diff] [blame] | 100 | } |
Suchitra Vemuri | a8e7bb2 | 2019-10-22 15:50:05 -0700 | [diff] [blame] | 101 | } |
Andrea Campanella | 80dcae0 | 2020-01-28 11:23:26 +0100 | [diff] [blame] | 102 | if ( withKind && deployment_config.fabric_switches.size() > 0 ) { |
| 103 | stage('OpenFlow port forward for aggregation switch') { |
| 104 | timeout(1) { |
| 105 | sh returnStdout: true, script: """ |
| 106 | export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf |
| 107 | #This uses the default route out, porperly configured is the management |
| 108 | mgmt_address=`ip route get 1 | awk '{print \$NF;exit}'` |
Andrea Campanella | 8eeba9f | 2020-03-10 10:51:25 +0100 | [diff] [blame] | 109 | JENKINS_NODE_COOKIE=dontkill kubectl port-forward -n default service/onos-openflow --address=\$mgmt_address 6653:6653 >> /dev/null 2>&1 & |
Andrea Campanella | 80dcae0 | 2020-01-28 11:23:26 +0100 | [diff] [blame] | 110 | """ |
| 111 | } |
| 112 | } |
| 113 | } |
Suchitra Vemuri | 225a2c8 | 2020-03-04 18:16:40 -0800 | [diff] [blame] | 114 | if ( params.configurePod && params.profile != "Default" ) { |
Suchitra Vemuri | a8e7bb2 | 2019-10-22 15:50:05 -0700 | [diff] [blame] | 115 | stage('Push Tech-Profile') { |
| 116 | timeout(1) { |
| 117 | out_push_tp = sh returnStatus: true, script: """ |
| 118 | export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf |
| 119 | etcd_container=\$(kubectl get pods -n voltha | grep voltha-etcd-cluster | awk 'NR==1{print \$1}') |
Suchitra Vemuri | 4ac4463 | 2019-11-12 13:41:08 -0800 | [diff] [blame] | 120 | kubectl cp $WORKSPACE/voltha-system-tests/tests/data/TechProfile-${profile}.json voltha/\$etcd_container:/tmp/flexpod.json |
| 121 | put_result=\$(kubectl exec -it \$etcd_container -n voltha -- /bin/sh -c 'cat /tmp/flexpod.json | ETCDCTL_API=3 etcdctl put service/voltha/technology_profiles/XGS-PON/64') |
Suchitra Vemuri | a8e7bb2 | 2019-10-22 15:50:05 -0700 | [diff] [blame] | 122 | """ |
| 123 | return out_push_tp == 0 |
| 124 | } |
| 125 | timeout(1) { |
| 126 | out_get_tp = sh returnStatus: true, script: """ |
| 127 | etcd_container=\$(kubectl get pods -n voltha | grep voltha-etcd-cluster | awk 'NR==1{print \$1}') |
Suchitra Vemuri | 4ac4463 | 2019-11-12 13:41:08 -0800 | [diff] [blame] | 128 | get_result=\$(kubectl exec -it \$etcd_container -n voltha -- /bin/sh -c 'ETCDCTL_API=3 etcdctl get --prefix service/voltha/technology_profiles/XGS-PON/64') |
Suchitra Vemuri | a8e7bb2 | 2019-10-22 15:50:05 -0700 | [diff] [blame] | 129 | """ |
| 130 | return out_get_tp == 0 |
| 131 | } |
| 132 | } |
Suchitra Vemuri | 4ac4463 | 2019-11-12 13:41:08 -0800 | [diff] [blame] | 133 | } |
| 134 | stage('Push Sadis-config') { |
| 135 | timeout(1) { |
| 136 | sadis_out = sh returnStatus: true, script: """ |
Suchitra Vemuri | d3bf85c | 2020-02-14 14:15:39 -0800 | [diff] [blame] | 137 | if [[ "${onosVersion}" == "1.13.9" ]]; then |
Suchitra Vemuri | c735bab | 2020-02-05 14:55:00 -0800 | [diff] [blame] | 138 | curl -sSL --user karaf:karaf -X POST -H Content-Type:application/json http://${deployment_config.nodes[0].ip}:30120/onos/v1/network/configuration --data @$WORKSPACE/voltha-system-tests/tests/data/${configFileName}-${onosVersion}-sadis.json |
Suchitra Vemuri | ae9cecb | 2020-03-12 12:59:14 -0700 | [diff] [blame] | 139 | elif [[ "${workFlow}" == "DT" ]]; then |
| 140 | curl -sSL --user karaf:karaf -X POST -H Content-Type:application/json http://${deployment_config.nodes[0].ip}:30120/onos/v1/network/configuration --data @$WORKSPACE/voltha-system-tests/tests/data/${configFileName}-sadis-DT.json |
Suchitra Vemuri | c735bab | 2020-02-05 14:55:00 -0800 | [diff] [blame] | 141 | else |
Suchitra Vemuri | a8e7bb2 | 2019-10-22 15:50:05 -0700 | [diff] [blame] | 142 | curl -sSL --user karaf:karaf -X POST -H Content-Type:application/json http://${deployment_config.nodes[0].ip}:30120/onos/v1/network/configuration --data @$WORKSPACE/voltha-system-tests/tests/data/${configFileName}-sadis.json |
Suchitra Vemuri | c735bab | 2020-02-05 14:55:00 -0800 | [diff] [blame] | 143 | fi |
Suchitra Vemuri | ae9cecb | 2020-03-12 12:59:14 -0700 | [diff] [blame] | 144 | """ |
| 145 | return sadis_out == 0 |
Suchitra Vemuri | a8e7bb2 | 2019-10-22 15:50:05 -0700 | [diff] [blame] | 146 | } |
Suchitra Vemuri | 4ac4463 | 2019-11-12 13:41:08 -0800 | [diff] [blame] | 147 | } |
Suchitra Vemuri | a8e7bb2 | 2019-10-22 15:50:05 -0700 | [diff] [blame] | 148 | if ( params.reinstallOlt ) { |
| 149 | stage('Reinstall OLT software') { |
| 150 | for(int i=0; i < deployment_config.olts.size(); i++) { |
| 151 | sh returnStdout: true, script: """ |
| 152 | sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} "dpkg --install ${oltDebVersion}" |
| 153 | sleep 10 |
| 154 | """ |
| 155 | timeout(5) { |
| 156 | waitUntil { |
| 157 | olt_sw_present = sh returnStdout: true, script: """ |
| 158 | sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'dpkg --list | grep asfvolt16 | wc -l' |
Suchitra Vemuri | f574b94 | 2020-01-27 16:39:58 -0800 | [diff] [blame] | 159 | if ( ${deployment_config.olts[i].fortygig} ); then |
| 160 | sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'echo "port ce128 sp=40000" >> /broadcom/qax.soc' |
| 161 | fi |
Suchitra Vemuri | a8e7bb2 | 2019-10-22 15:50:05 -0700 | [diff] [blame] | 162 | """ |
| 163 | return olt_sw_present.toInteger() == 1 |
| 164 | } |
| 165 | } |
| 166 | } |
| 167 | } |
| 168 | stage('Restart OLT processes') { |
| 169 | for(int i=0; i < deployment_config.olts.size(); i++) { |
| 170 | timeout(5) { |
| 171 | sh returnStdout: true, script: """ |
| 172 | ssh-keyscan -H ${deployment_config.olts[i].ip} >> ~/.ssh/known_hosts |
| 173 | sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'reboot' || true |
| 174 | sleep 120 |
| 175 | """ |
| 176 | } |
| 177 | timeout(15) { |
| 178 | waitUntil { |
| 179 | devprocess = sh returnStdout: true, script: "sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'ps -ef | grep dev_mgmt_daemon | wc -l'" |
| 180 | return devprocess.toInteger() > 0 |
| 181 | } |
| 182 | } |
| 183 | timeout(15) { |
| 184 | waitUntil { |
| 185 | openoltprocess = sh returnStdout: true, script: "sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'ps -ef | grep openolt | wc -l'" |
| 186 | return openoltprocess.toInteger() > 0 |
| 187 | } |
| 188 | } |
| 189 | } |
| 190 | } |
| 191 | } |
Suchitra Vemuri | a8e7bb2 | 2019-10-22 15:50:05 -0700 | [diff] [blame] | 192 | if ( deployment_config.fabric_switches.size() > 0 ) { |
| 193 | stage('Switch Configurations in ONOS') { |
| 194 | timeout(1) { |
| 195 | netcfg_out = sh returnStatus: true, script: """ |
| 196 | curl -sSL --user karaf:karaf -X POST -H Content-Type:application/json http://${deployment_config.nodes[0].ip}:30120/onos/v1/network/configuration --data @$WORKSPACE/${configBaseDir}/${configToscaDir}/voltha/${configFileName}-onos-netcfg-switch.json |
| 197 | curl -sSL --user karaf:karaf -X POST http://${deployment_config.nodes[0].ip}:30120/onos/v1/applications/org.onosproject.segmentrouting/active |
Suchitra Vemuri | c735bab | 2020-02-05 14:55:00 -0800 | [diff] [blame] | 198 | |
Suchitra Vemuri | a8e7bb2 | 2019-10-22 15:50:05 -0700 | [diff] [blame] | 199 | """ |
| 200 | return netcfg_out == 0 |
| 201 | } |
| 202 | timeout(1) { |
| 203 | waitUntil { |
| 204 | sr_active_out = sh returnStatus: true, script: """ |
You Wang | eb76aeb | 2019-12-09 22:08:23 -0800 | [diff] [blame] | 205 | ssh-keygen -R [${deployment_config.nodes[0].ip}]:30115 |
Suchitra Vemuri | 6db64b2 | 2019-12-09 13:00:54 -0800 | [diff] [blame] | 206 | ssh-keyscan -p 30115 -H ${deployment_config.nodes[0].ip} >> ~/.ssh/known_hosts |
| 207 | sshpass -p karaf ssh -p 30115 karaf@${deployment_config.nodes[0].ip} "log:set TRACE org.opencord.dhcpl2relay" |
| 208 | sshpass -p karaf ssh -p 30115 karaf@${deployment_config.nodes[0].ip} "log:set TRACE org.opencord.aaa" |
| 209 | sshpass -p karaf ssh -p 30115 karaf@${deployment_config.nodes[0].ip} "log:set TRACE org.opencord.olt" |
Suchitra Vemuri | a8e7bb2 | 2019-10-22 15:50:05 -0700 | [diff] [blame] | 210 | curl -sSL --user karaf:karaf -X GET http://${deployment_config.nodes[0].ip}:30120/onos/v1/applications/org.onosproject.segmentrouting | jq '.state' | grep ACTIVE |
| 211 | """ |
| 212 | return sr_active_out == 0 |
| 213 | } |
| 214 | } |
| 215 | timeout(1) { |
| 216 | // FIXME support multiple OLTs |
| 217 | for(int i=0; i < deployment_config.hosts.src.size(); i++) { |
| 218 | xconnect_out = sh returnStatus: true, script: """ |
Suchitra Vemuri | c4d9293 | 2020-01-16 21:50:30 -0800 | [diff] [blame] | 219 | version=\$(sshpass -p karaf ssh -p 30115 karaf@${deployment_config.nodes[0].ip} "summary" | grep version) |
Suchitra Vemuri | d3bf85c | 2020-02-14 14:15:39 -0800 | [diff] [blame] | 220 | sleep 10 |
Suchitra Vemuri | c4d9293 | 2020-01-16 21:50:30 -0800 | [diff] [blame] | 221 | if [[ \$version == *"version=2.2"* ]]; then |
| 222 | curl -X POST --user karaf:karaf --header 'Content-Type: application/json' --header 'Accept: application/json' -d '{"deviceId": "${deployment_config.fabric_switches[0].device_id}", "vlanId": "${deployment_config.hosts.src[i].s_tag}", "endpoints": [${deployment_config.fabric_switches[0].bngPort},${deployment_config.fabric_switches[0].oltPort}]}' 'http://${deployment_config.nodes[0].ip}:30120/onos/segmentrouting/xconnect' |
| 223 | else |
| 224 | curl -X POST --user karaf:karaf --header 'Content-Type: application/json' --header 'Accept: application/json' -d '{"deviceId": "${deployment_config.fabric_switches[0].device_id}", "vlanId": "${deployment_config.hosts.src[i].s_tag}", "ports": [${deployment_config.fabric_switches[0].bngPort},${deployment_config.fabric_switches[0].oltPort}]}' 'http://${deployment_config.nodes[0].ip}:30120/onos/segmentrouting/xconnect' |
| 225 | fi |
Suchitra Vemuri | a8e7bb2 | 2019-10-22 15:50:05 -0700 | [diff] [blame] | 226 | """ |
| 227 | } |
| 228 | } |
| 229 | } |
| 230 | } |
| 231 | currentBuild.result = 'SUCCESS' |
| 232 | } catch (err) { |
| 233 | currentBuild.result = 'FAILURE' |
| 234 | step([$class: 'Mailer', notifyEveryUnstableBuild: true, recipients: "${notificationEmail}", sendToIndividuals: false]) |
| 235 | throw err |
| 236 | } |
| 237 | echo "RESULT: ${currentBuild.result}" |
| 238 | } |
| 239 | } |