Kailash | 6f5acb6 | 2019-08-28 14:38:45 -0700 | [diff] [blame] | 1 | // Copyright 2017-present Open Networking Foundation |
| 2 | // |
| 3 | // Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | // you may not use this file except in compliance with the License. |
| 5 | // You may obtain a copy of the License at |
| 6 | // |
| 7 | // http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | // |
| 9 | // Unless required by applicable law or agreed to in writing, software |
| 10 | // distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | // See the License for the specific language governing permissions and |
| 13 | // limitations under the License. |
| 14 | |
| 15 | node ("${TestNodeName}") { |
Suchitra Vemuri | b4f365f | 2020-03-27 15:09:36 -0700 | [diff] [blame] | 16 | def withKind = false |
| 17 | if (params.withKind != null){ |
| 18 | withKind = params.withKind |
| 19 | } |
Kailash | 6f5acb6 | 2019-08-28 14:38:45 -0700 | [diff] [blame] | 20 | timeout (100) { |
| 21 | try { |
| 22 | stage ("Parse deployment configuration file") { |
Suchitra Vemuri | b4f365f | 2020-03-27 15:09:36 -0700 | [diff] [blame] | 23 | sh returnStdout: true, script: "rm -rf ${configBaseDir} voltha-system-tests kind-voltha" |
Suchitra Vemuri | e514b8c | 2019-09-23 19:37:18 -0700 | [diff] [blame] | 24 | sh returnStdout: true, script: "git clone -b master ${cordRepoUrl}/${configBaseDir}" |
Suchitra Vemuri | b4f365f | 2020-03-27 15:09:36 -0700 | [diff] [blame] | 25 | sh returnStdout: true, script: "git clone -b ${branch} ${cordRepoUrl}/voltha-system-tests" |
Andrea Campanella | f60f883 | 2020-04-06 17:42:28 +0200 | [diff] [blame^] | 26 | if (params.branch == "voltha-2.3") { |
| 27 | sh returnStdout: true, script: "git clone -b 3.0.2 https://github.com/ciena/kind-voltha.git" |
| 28 | } else { |
| 29 | sh returnStdout: true, script: "git clone https://github.com/ciena/kind-voltha.git" |
| 30 | } |
Suchitra Vemuri | b4f365f | 2020-03-27 15:09:36 -0700 | [diff] [blame] | 31 | if ( params.workFlow == "DT" ) { |
| 32 | deployment_config = readYaml file: "${configBaseDir}/${configDeploymentDir}/${configFileName}-DT.yaml" |
| 33 | } |
| 34 | else |
| 35 | { |
| 36 | deployment_config = readYaml file: "${configBaseDir}/${configDeploymentDir}/${configFileName}.yaml" |
| 37 | } |
Kailash | 6f5acb6 | 2019-08-28 14:38:45 -0700 | [diff] [blame] | 38 | } |
| 39 | stage('Clean up') { |
You Wang | 25e5861 | 2019-10-01 16:30:28 -0700 | [diff] [blame] | 40 | timeout(10) { |
Kailash | 6f5acb6 | 2019-08-28 14:38:45 -0700 | [diff] [blame] | 41 | sh returnStdout: true, script: """ |
| 42 | export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf |
Matteo Scandolo | 1294aeb | 2019-09-24 16:20:32 -0700 | [diff] [blame] | 43 | for hchart in \$(helm list -q | grep -E -v 'docker-registry|mavenrepo|ponnet'); |
| 44 | do |
| 45 | echo "Purging chart: \${hchart}" |
| 46 | helm delete --purge "\${hchart}" |
| 47 | done |
Kailash | 6f5acb6 | 2019-08-28 14:38:45 -0700 | [diff] [blame] | 48 | """ |
| 49 | timeout(5) { |
| 50 | waitUntil { |
| 51 | helm_deleted = sh returnStdout: true, script: """ |
| 52 | export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf && |
| 53 | helm ls -q | grep -E -v 'docker-registry|mavenrepo|ponnet' | wc -l |
| 54 | """ |
| 55 | return helm_deleted.toInteger() == 0 |
| 56 | } |
| 57 | } |
| 58 | timeout(5) { |
| 59 | waitUntil { |
| 60 | kubectl_deleted = sh returnStdout: true, script: """ |
| 61 | export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf && |
| 62 | kubectl get pods --all-namespaces --no-headers | grep -E -v 'kube-system|docker-registry|mavenrepo|ponnet' | wc -l |
| 63 | """ |
| 64 | return kubectl_deleted.toInteger() == 0 |
| 65 | } |
| 66 | } |
Suchitra Vemuri | b4f365f | 2020-03-27 15:09:36 -0700 | [diff] [blame] | 67 | timeout(1) { |
| 68 | sh returnStdout: true, script: """ |
| 69 | port_fwd_pid=`ps -ax | grep "port-forward -n default service/onos-openflow" | grep -v "grep" | awk '{print \$1}'` |
| 70 | if [[ "" != "\$port_fwd_pid" ]]; then |
| 71 | kill -9 \$port_fwd_pid > /dev/null 2>&1 |
| 72 | fi |
| 73 | """ |
| 74 | } |
Kailash | 6f5acb6 | 2019-08-28 14:38:45 -0700 | [diff] [blame] | 75 | } |
| 76 | } |
Suchitra Vemuri | 1934abc | 2019-11-05 12:41:11 -0800 | [diff] [blame] | 77 | stage('Install Voltha') { |
Kailash | 6f5acb6 | 2019-08-28 14:38:45 -0700 | [diff] [blame] | 78 | timeout(10) { |
| 79 | sh returnStdout: true, script: """ |
| 80 | export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf |
| 81 | cd kind-voltha/ |
Suchitra Vemuri | b4f365f | 2020-03-27 15:09:36 -0700 | [diff] [blame] | 82 | if [ "${branch}" != "master" ]; then |
| 83 | cd $WORKSPACE/kind-voltha |
| 84 | source releases/${branch} |
Suchitra Vemuri | b948386 | 2019-11-05 21:37:29 -0800 | [diff] [blame] | 85 | else |
Andy Bavier | 7cee6f4 | 2019-11-05 16:49:47 -0700 | [diff] [blame] | 86 | export EXTRA_HELM_FLAGS='-f $WORKSPACE/${configBaseDir}/${configKubernetesDir}/voltha/${configFileName}.yml' |
Suchitra Vemuri | 1934abc | 2019-11-05 12:41:11 -0800 | [diff] [blame] | 87 | fi |
Andy Bavier | 7cee6f4 | 2019-11-05 16:49:47 -0700 | [diff] [blame] | 88 | |
| 89 | # VOL-2194 ONOS SSH and REST ports hardcoded to 30115/30120 in tests |
hwchiu | 0a39d51 | 2020-04-04 05:00:29 +0000 | [diff] [blame] | 90 | # JENKINS_NODE_COOKIE=dontKillMe ask Jenkins doesn't kill the proces after this job. we want the `while true; do kubectl port-forward` keep alive |
Suchitra Vemuri | b4f365f | 2020-03-27 15:09:36 -0700 | [diff] [blame] | 91 | if [[ "${workFlow}" == "DT" ]]; then |
hwchiu | 0a39d51 | 2020-04-04 05:00:29 +0000 | [diff] [blame] | 92 | JENKINS_NODE_COOKIE=dontKillMe ONOS_SSH_PORT=30115 ONOS_API_PORT=30120 VOLTHA_LOG_LEVEL=DEBUG WITH_SIM_ADAPTERS=n WITH_TP=yes DEPLOY_K8S=no INSTALL_KUBECTL=no INSTALL_HELM=no WITH_EAPOL=no WITH_DHCP=no WITH_IGMP=no WITH_RADIUS=no FANCY=0 ./voltha up |
Suchitra Vemuri | b4f365f | 2020-03-27 15:09:36 -0700 | [diff] [blame] | 93 | else |
hwchiu | 0a39d51 | 2020-04-04 05:00:29 +0000 | [diff] [blame] | 94 | JENKINS_NODE_COOKIE=dontKillMe ONOS_SSH_PORT=30115 ONOS_API_PORT=30120 VOLTHA_LOG_LEVEL=DEBUG WITH_SIM_ADAPTERS=n WITH_RADIUS=y WITH_TP=yes DEPLOY_K8S=no INSTALL_KUBECTL=no INSTALL_HELM=no FANCY=0 ./voltha up |
Suchitra Vemuri | b4f365f | 2020-03-27 15:09:36 -0700 | [diff] [blame] | 95 | fi |
Andy Bavier | 7cee6f4 | 2019-11-05 16:49:47 -0700 | [diff] [blame] | 96 | |
Suchitra Vemuri | 1934abc | 2019-11-05 12:41:11 -0800 | [diff] [blame] | 97 | kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.image}{'\\t'}{.imageID}{'\\n'}" | sort | uniq -c |
| 98 | kubectl get nodes -o wide |
| 99 | kubectl get pods -n voltha -o wide |
Kailash | 6f5acb6 | 2019-08-28 14:38:45 -0700 | [diff] [blame] | 100 | """ |
Suchitra Vemuri | 1934abc | 2019-11-05 12:41:11 -0800 | [diff] [blame] | 101 | } |
Kailash | 6f5acb6 | 2019-08-28 14:38:45 -0700 | [diff] [blame] | 102 | } |
Suchitra Vemuri | b4f365f | 2020-03-27 15:09:36 -0700 | [diff] [blame] | 103 | if ( withKind && deployment_config.fabric_switches.size() > 0 ) { |
| 104 | stage('OpenFlow port forward for aggregation switch') { |
| 105 | timeout(1) { |
| 106 | sh returnStdout: true, script: """ |
| 107 | export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf |
| 108 | #This uses the default route out, porperly configured is the management |
| 109 | mgmt_address=`ip route get 1 | awk '{print \$NF;exit}'` |
| 110 | JENKINS_NODE_COOKIE=dontkill kubectl port-forward -n default service/onos-openflow --address=\$mgmt_address 6653:6653 >> /dev/null 2>&1 & |
| 111 | """ |
| 112 | } |
| 113 | } |
| 114 | } |
| 115 | if ( params.configurePod && params.profile != "Default" ) { |
Suchitra Vemuri | 7268009 | 2019-10-03 12:39:20 -0700 | [diff] [blame] | 116 | stage('Push Tech-Profile') { |
| 117 | timeout(1) { |
| 118 | out_push_tp = sh returnStatus: true, script: """ |
| 119 | export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf |
| 120 | etcd_container=\$(kubectl get pods -n voltha | grep voltha-etcd-cluster | awk 'NR==1{print \$1}') |
hwchiu | c77c7d2 | 2019-10-18 13:58:35 -0700 | [diff] [blame] | 121 | kubectl cp $WORKSPACE/voltha-system-tests/tests/data/TechProfile-${profile}.json voltha/\$etcd_container:/tmp/flexpod.json |
Suchitra Vemuri | b4f365f | 2020-03-27 15:09:36 -0700 | [diff] [blame] | 122 | put_result=\$(kubectl exec -it \$etcd_container -n voltha -- /bin/sh -c 'cat /tmp/flexpod.json | ETCDCTL_API=3 etcdctl put service/voltha/technology_profiles/XGS-PON/64') |
Suchitra Vemuri | 7268009 | 2019-10-03 12:39:20 -0700 | [diff] [blame] | 123 | """ |
| 124 | return out_push_tp == 0 |
| 125 | } |
| 126 | timeout(1) { |
| 127 | out_get_tp = sh returnStatus: true, script: """ |
| 128 | etcd_container=\$(kubectl get pods -n voltha | grep voltha-etcd-cluster | awk 'NR==1{print \$1}') |
Suchitra Vemuri | b4f365f | 2020-03-27 15:09:36 -0700 | [diff] [blame] | 129 | get_result=\$(kubectl exec -it \$etcd_container -n voltha -- /bin/sh -c 'ETCDCTL_API=3 etcdctl get --prefix service/voltha/technology_profiles/XGS-PON/64') |
Suchitra Vemuri | 7268009 | 2019-10-03 12:39:20 -0700 | [diff] [blame] | 130 | """ |
| 131 | return out_get_tp == 0 |
| 132 | } |
hwchiu | c77c7d2 | 2019-10-18 13:58:35 -0700 | [diff] [blame] | 133 | } |
| 134 | } |
| 135 | stage('Push Sadis-config') { |
| 136 | timeout(1) { |
| 137 | sadis_out = sh returnStatus: true, script: """ |
Suchitra Vemuri | b4f365f | 2020-03-27 15:09:36 -0700 | [diff] [blame] | 138 | if [[ "${workFlow}" == "DT" ]]; then |
| 139 | curl -sSL --user karaf:karaf -X POST -H Content-Type:application/json http://${deployment_config.nodes[0].ip}:30120/onos/v1/network/configuration --data @$WORKSPACE/voltha-system-tests/tests/data/${configFileName}-sadis-DT.json |
| 140 | else |
Suchitra Vemuri | 7268009 | 2019-10-03 12:39:20 -0700 | [diff] [blame] | 141 | curl -sSL --user karaf:karaf -X POST -H Content-Type:application/json http://${deployment_config.nodes[0].ip}:30120/onos/v1/network/configuration --data @$WORKSPACE/voltha-system-tests/tests/data/${configFileName}-sadis.json |
Suchitra Vemuri | b4f365f | 2020-03-27 15:09:36 -0700 | [diff] [blame] | 142 | fi |
| 143 | """ |
| 144 | return sadis_out == 0 |
You Wang | b9526e6 | 2019-10-07 15:25:48 -0700 | [diff] [blame] | 145 | } |
hwchiu | c77c7d2 | 2019-10-18 13:58:35 -0700 | [diff] [blame] | 146 | } |
Kailash | 6f5acb6 | 2019-08-28 14:38:45 -0700 | [diff] [blame] | 147 | if ( params.reinstallOlt ) { |
| 148 | stage('Reinstall OLT software') { |
| 149 | for(int i=0; i < deployment_config.olts.size(); i++) { |
| 150 | sh returnStdout: true, script: """ |
Kailash | 6f5acb6 | 2019-08-28 14:38:45 -0700 | [diff] [blame] | 151 | sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} "dpkg --install ${oltDebVersion}" |
Suchitra Vemuri | b4f365f | 2020-03-27 15:09:36 -0700 | [diff] [blame] | 152 | sleep 10 |
Kailash | 6f5acb6 | 2019-08-28 14:38:45 -0700 | [diff] [blame] | 153 | """ |
| 154 | timeout(5) { |
| 155 | waitUntil { |
| 156 | olt_sw_present = sh returnStdout: true, script: """ |
| 157 | sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'dpkg --list | grep asfvolt16 | wc -l' |
Suchitra Vemuri | b4f365f | 2020-03-27 15:09:36 -0700 | [diff] [blame] | 158 | if ( ${deployment_config.olts[i].fortygig} ); then |
| 159 | sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'echo "port ce128 sp=40000" >> /broadcom/qax.soc' |
| 160 | fi |
Kailash | 6f5acb6 | 2019-08-28 14:38:45 -0700 | [diff] [blame] | 161 | """ |
| 162 | return olt_sw_present.toInteger() == 1 |
| 163 | } |
| 164 | } |
Kailash | 6f5acb6 | 2019-08-28 14:38:45 -0700 | [diff] [blame] | 165 | } |
| 166 | } |
Suchitra Vemuri | b4f365f | 2020-03-27 15:09:36 -0700 | [diff] [blame] | 167 | stage('Restart OLT processes') { |
Kailash | 6f5acb6 | 2019-08-28 14:38:45 -0700 | [diff] [blame] | 168 | for(int i=0; i < deployment_config.olts.size(); i++) { |
| 169 | timeout(5) { |
| 170 | sh returnStdout: true, script: """ |
| 171 | ssh-keyscan -H ${deployment_config.olts[i].ip} >> ~/.ssh/known_hosts |
Suchitra Vemuri | b4f365f | 2020-03-27 15:09:36 -0700 | [diff] [blame] | 172 | sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'reboot' || true |
| 173 | sleep 120 |
Kailash | 6f5acb6 | 2019-08-28 14:38:45 -0700 | [diff] [blame] | 174 | """ |
| 175 | } |
| 176 | timeout(15) { |
| 177 | waitUntil { |
Suchitra Vemuri | b4f365f | 2020-03-27 15:09:36 -0700 | [diff] [blame] | 178 | devprocess = sh returnStdout: true, script: "sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'ps -ef | grep dev_mgmt_daemon | wc -l'" |
| 179 | return devprocess.toInteger() > 0 |
| 180 | } |
| 181 | } |
| 182 | timeout(15) { |
| 183 | waitUntil { |
| 184 | openoltprocess = sh returnStdout: true, script: "sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].ip} 'ps -ef | grep openolt | wc -l'" |
| 185 | return openoltprocess.toInteger() > 0 |
Kailash | 6f5acb6 | 2019-08-28 14:38:45 -0700 | [diff] [blame] | 186 | } |
| 187 | } |
| 188 | } |
| 189 | } |
| 190 | } |
Matteo Scandolo | 1294aeb | 2019-09-24 16:20:32 -0700 | [diff] [blame] | 191 | if ( deployment_config.fabric_switches.size() > 0 ) { |
You Wang | 25e5861 | 2019-10-01 16:30:28 -0700 | [diff] [blame] | 192 | stage('Switch Configurations in ONOS') { |
| 193 | timeout(1) { |
| 194 | netcfg_out = sh returnStatus: true, script: """ |
| 195 | curl -sSL --user karaf:karaf -X POST -H Content-Type:application/json http://${deployment_config.nodes[0].ip}:30120/onos/v1/network/configuration --data @$WORKSPACE/${configBaseDir}/${configToscaDir}/voltha/${configFileName}-onos-netcfg-switch.json |
| 196 | curl -sSL --user karaf:karaf -X POST http://${deployment_config.nodes[0].ip}:30120/onos/v1/applications/org.onosproject.segmentrouting/active |
Suchitra Vemuri | b4f365f | 2020-03-27 15:09:36 -0700 | [diff] [blame] | 197 | |
You Wang | 25e5861 | 2019-10-01 16:30:28 -0700 | [diff] [blame] | 198 | """ |
| 199 | return netcfg_out == 0 |
| 200 | } |
| 201 | timeout(1) { |
| 202 | waitUntil { |
| 203 | sr_active_out = sh returnStatus: true, script: """ |
You Wang | eb76aeb | 2019-12-09 22:08:23 -0800 | [diff] [blame] | 204 | ssh-keygen -R [${deployment_config.nodes[0].ip}]:30115 |
Suchitra Vemuri | c325726 | 2019-12-05 19:06:42 -0800 | [diff] [blame] | 205 | ssh-keyscan -p 30115 -H ${deployment_config.nodes[0].ip} >> ~/.ssh/known_hosts |
| 206 | sshpass -p karaf ssh -p 30115 karaf@${deployment_config.nodes[0].ip} "log:set TRACE org.opencord.dhcpl2relay" |
| 207 | sshpass -p karaf ssh -p 30115 karaf@${deployment_config.nodes[0].ip} "log:set TRACE org.opencord.aaa" |
| 208 | sshpass -p karaf ssh -p 30115 karaf@${deployment_config.nodes[0].ip} "log:set TRACE org.opencord.olt" |
You Wang | 25e5861 | 2019-10-01 16:30:28 -0700 | [diff] [blame] | 209 | curl -sSL --user karaf:karaf -X GET http://${deployment_config.nodes[0].ip}:30120/onos/v1/applications/org.onosproject.segmentrouting | jq '.state' | grep ACTIVE |
Suchitra Vemuri | b4f365f | 2020-03-27 15:09:36 -0700 | [diff] [blame] | 210 | sshpass -p karaf ssh -p 30115 karaf@${deployment_config.nodes[0].ip} "cfg set org.onosproject.net.flow.impl.FlowRuleManager purgeOnDisconnection false" |
Andrea Campanella | 086bcbb | 2020-04-01 09:56:23 +0200 | [diff] [blame] | 211 | sshpass -p karaf ssh -p 30115 karaf@${deployment_config.nodes[0].ip} "cfg set org.onosproject.net.meter.impl.MeterManager purgeOnDisconnection false" |
You Wang | 25e5861 | 2019-10-01 16:30:28 -0700 | [diff] [blame] | 212 | """ |
| 213 | return sr_active_out == 0 |
| 214 | } |
| 215 | } |
| 216 | timeout(1) { |
| 217 | // FIXME support multiple OLTs |
| 218 | for(int i=0; i < deployment_config.hosts.src.size(); i++) { |
| 219 | xconnect_out = sh returnStatus: true, script: """ |
Suchitra Vemuri | f82086d | 2020-01-16 19:55:16 -0800 | [diff] [blame] | 220 | version=\$(sshpass -p karaf ssh -p 30115 karaf@${deployment_config.nodes[0].ip} "summary" | grep version) |
Suchitra Vemuri | b4f365f | 2020-03-27 15:09:36 -0700 | [diff] [blame] | 221 | sleep 10 |
Suchitra Vemuri | f82086d | 2020-01-16 19:55:16 -0800 | [diff] [blame] | 222 | if [[ \$version == *"version=2.2"* ]]; then |
| 223 | curl -X POST --user karaf:karaf --header 'Content-Type: application/json' --header 'Accept: application/json' -d '{"deviceId": "${deployment_config.fabric_switches[0].device_id}", "vlanId": "${deployment_config.hosts.src[i].s_tag}", "endpoints": [${deployment_config.fabric_switches[0].bngPort},${deployment_config.fabric_switches[0].oltPort}]}' 'http://${deployment_config.nodes[0].ip}:30120/onos/segmentrouting/xconnect' |
| 224 | else |
| 225 | curl -X POST --user karaf:karaf --header 'Content-Type: application/json' --header 'Accept: application/json' -d '{"deviceId": "${deployment_config.fabric_switches[0].device_id}", "vlanId": "${deployment_config.hosts.src[i].s_tag}", "ports": [${deployment_config.fabric_switches[0].bngPort},${deployment_config.fabric_switches[0].oltPort}]}' 'http://${deployment_config.nodes[0].ip}:30120/onos/segmentrouting/xconnect' |
| 226 | fi |
You Wang | 25e5861 | 2019-10-01 16:30:28 -0700 | [diff] [blame] | 227 | """ |
| 228 | } |
| 229 | } |
Kailash | 6f5acb6 | 2019-08-28 14:38:45 -0700 | [diff] [blame] | 230 | } |
Matteo Scandolo | 1294aeb | 2019-09-24 16:20:32 -0700 | [diff] [blame] | 231 | } |
Kailash | 6f5acb6 | 2019-08-28 14:38:45 -0700 | [diff] [blame] | 232 | currentBuild.result = 'SUCCESS' |
| 233 | } catch (err) { |
| 234 | currentBuild.result = 'FAILURE' |
| 235 | step([$class: 'Mailer', notifyEveryUnstableBuild: true, recipients: "${notificationEmail}", sendToIndividuals: false]) |
| 236 | throw err |
| 237 | } |
| 238 | echo "RESULT: ${currentBuild.result}" |
| 239 | } |
| 240 | } |