blob: f9f9e048d86f8a30db993c1e250e0ec0ec278c69 [file] [log] [blame]
Matteo Scandolo9b644ba2021-04-19 11:21:07 -07001
2// Copyright 2017-present Open Networking Foundation
3//
4// Licensed under the Apache License, Version 2.0 (the "License");
5// you may not use this file except in compliance with the License.
6// You may obtain a copy of the License at
7//
8// http://www.apache.org/licenses/LICENSE-2.0
9//
10// Unless required by applicable law or agreed to in writing, software
11// distributed under the License is distributed on an "AS IS" BASIS,
12// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13// See the License for the specific language governing permissions and
14// limitations under the License.
15// used to deploy VOLTHA and configure ONOS physical PODs
16// NOTE we are importing the library even if it's global so that it's
17// easier to change the keywords during a replay
18library identifier: 'cord-jenkins-libraries@master',
19 retriever: modernSCM([
20 $class: 'GitSCMSource',
21 remote: 'https://gerrit.opencord.org/ci-management.git'
22])
23def infraNamespace = "infra"
24def volthaNamespace = "voltha"
25def clusterName = "kind-ci"
26pipeline {
27 /* no label, executor is determined by JJB */
28 agent {
29 label "${params.buildNode}"
30 }
31 options {
32 timeout(time: 120, unit: 'MINUTES')
33 }
34 environment {
35 PATH="$PATH:$WORKSPACE/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin"
36 KUBECONFIG="$HOME/.kube/kind-${clusterName}"
37 VOLTCONFIG="$HOME/.volt/config"
Andrea Campanella1198cd52021-06-14 16:17:25 +020038 LOG_FOLDER="$WORKSPACE/${workflow}/"
39 APPS_TO_LOG="etcd kafka onos-classic adapter-open-onu adapter-open-olt rw-core ofagent bbsim radius bbsim-sadis-server onos-config-loader"
40
Matteo Scandolo9b644ba2021-04-19 11:21:07 -070041 }
42 stages{
43 stage('Download Code') {
44 steps {
45 getVolthaCode([
46 branch: "${branch}",
47 gerritProject: "${gerritProject}",
48 gerritRefspec: "${gerritRefspec}",
49 volthaSystemTestsChange: "${volthaSystemTestsChange}",
50 volthaHelmChartsChange: "${volthaHelmChartsChange}",
51 ])
52 }
53 }
54 stage ("Parse deployment configuration file") {
55 steps {
56 sh returnStdout: true, script: "rm -rf ${configBaseDir}"
Hardik Windlass6f854a12021-07-12 13:20:21 +000057 sh returnStdout: true, script: "git clone -b ${branch} ${cordRepoUrl}/${configBaseDir}"
Matteo Scandolo9b644ba2021-04-19 11:21:07 -070058 script {
Matteo Scandolob6d80732021-05-05 14:06:42 -070059
60 if (params.workflow.toUpperCase() == "TT") {
61 error("The Tucson POD does not support TT workflow at the moment")
62 }
63
Matteo Scandolo9b644ba2021-04-19 11:21:07 -070064 if ( params.workflow.toUpperCase() == "DT" ) {
65 deployment_config = readYaml file: "${configBaseDir}/${configDeploymentDir}/${configFileName}-DT.yaml"
66 }
67 else if ( params.workflow.toUpperCase() == "TT" ) {
68 deployment_config = readYaml file: "${configBaseDir}/${configDeploymentDir}/${configFileName}-TT.yaml"
69 }
70 else {
71 deployment_config = readYaml file: "${configBaseDir}/${configDeploymentDir}/${configFileName}.yaml"
72 }
73 }
74 }
75 }
76 stage('Clean up') {
77 steps {
78 timeout(15) {
79 script {
80 helmTeardown(["default", infraNamespace, volthaNamespace])
81 }
82 timeout(1) {
83 sh returnStdout: false, script: '''
84 # remove orphaned port-forward from different namespaces
Andrea Campanella4c8af942021-05-12 10:12:13 +020085 ps aux | grep port-forw | grep -v grep | awk '{print $2}' | xargs --no-run-if-empty kill -9 || true
Matteo Scandolo9b644ba2021-04-19 11:21:07 -070086 '''
87 }
88 }
89 }
90 }
91 stage('Build patch') {
92 steps {
93 // NOTE that the correct patch has already been checked out
94 // during the getVolthaCode step
95 buildVolthaComponent("${gerritProject}")
96 }
97 }
98 stage('Create K8s Cluster') {
99 steps {
100 script {
101 def clusterExists = sh returnStdout: true, script: """
102 kind get clusters | grep ${clusterName} | wc -l
103 """
104 if (clusterExists.trim() == "0") {
105 createKubernetesCluster([nodes: 3, name: clusterName])
106 }
107 }
108 }
109 }
110 stage('Load image in kind nodes') {
111 steps {
112 loadToKind()
113 }
114 }
115 stage('Install Voltha') {
116 steps {
117 timeout(20) {
118 script {
119 imageFlags = getVolthaImageFlags(gerritProject)
120 // if we're downloading a voltha-helm-charts patch, then install from a local copy of the charts
121 def localCharts = false
122 if (volthaHelmChartsChange != "" || gerritProject == "voltha-helm-charts") {
123 localCharts = true
124 }
Matteo Scandolobb7382d2021-05-05 08:32:35 -0700125 def flags = "-f $WORKSPACE/${configBaseDir}/${configKubernetesDir}/voltha/${configFileName}.yml ${imageFlags} "
Matteo Scandolo9b644ba2021-04-19 11:21:07 -0700126 // NOTE temporary workaround expose ONOS node ports (pod-config needs to be updated to contain these values)
Matteo Scandolobb7382d2021-05-05 08:32:35 -0700127 flags = flags + "--set onos-classic.onosSshPort=30115 " +
Matteo Scandolo9b644ba2021-04-19 11:21:07 -0700128 "--set onos-classic.onosApiPort=30120 " +
129 "--set onos-classic.onosOfPort=31653 " +
Matteo Scandolobb7382d2021-05-05 08:32:35 -0700130 "--set onos-classic.individualOpenFlowNodePorts=true " + extraHelmFlags
Matteo Scandolo9b644ba2021-04-19 11:21:07 -0700131 volthaDeploy([
132 workflow: workFlow.toLowerCase(),
Matteo Scandolobb7382d2021-05-05 08:32:35 -0700133 extraHelmFlags: flags,
Matteo Scandolo9b644ba2021-04-19 11:21:07 -0700134 localCharts: localCharts,
135 kubeconfig: "$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf",
136 onosReplica: 3,
137 atomixReplica: 3,
138 kafkaReplica: 3,
139 etcdReplica: 3,
140 ])
141 }
142 // start logging
143 sh """
Andrea Campanella1be85ad2021-06-14 13:01:15 +0200144 rm -rf $WORKSPACE/${workFlow}/
Matteo Scandolo9b644ba2021-04-19 11:21:07 -0700145 mkdir -p $WORKSPACE/${workFlow}
146 _TAG=kail-${workFlow} kail -n infra -n voltha > $WORKSPACE/${workFlow}/onos-voltha-combined.log &
147 """
Andrea Campanella1198cd52021-06-14 16:17:25 +0200148 sh returnStdout: false, script: '''
149 # start logging with kail
150
151 mkdir -p $LOG_FOLDER
152
153 list=($APPS_TO_LOG)
154 for app in "${list[@]}"
155 do
156 echo "Starting logs for: ${app}"
157 _TAG=kail-$app kail -l app=$app --since 1h > $LOG_FOLDER/$app.log&
158 done
159 '''
Matteo Scandolo9b644ba2021-04-19 11:21:07 -0700160 sh """
161 JENKINS_NODE_COOKIE="dontKillMe" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${volthaNamespace} svc/voltha-voltha-api 55555:55555; done"&
162 JENKINS_NODE_COOKIE="dontKillMe" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${infraNamespace} svc/voltha-infra-etcd 2379:2379; done"&
163 JENKINS_NODE_COOKIE="dontKillMe" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${infraNamespace} svc/voltha-infra-kafka 9092:9092; done"&
164 ps aux | grep port-forward
165 """
166 getPodsInfo("$WORKSPACE")
167 }
168 }
169 }
170 stage('Deploy Kafka Dump Chart') {
171 steps {
172 script {
173 sh returnStdout: false, script: """
174 helm repo add cord https://charts.opencord.org
175 helm repo update
176 if helm version -c --short|grep v2 -q; then
177 helm install -n voltha-kafka-dump cord/voltha-kafka-dump
178 else
179 helm install voltha-kafka-dump cord/voltha-kafka-dump
180 fi
181 """
182 }
183 }
184 }
185 stage('Push Tech-Profile') {
186 when {
187 expression { params.profile != "Default" }
188 }
189 steps {
190 sh returnStdout: false, script: """
191 etcd_container=\$(kubectl get pods -n voltha | grep voltha-etcd-cluster | awk 'NR==1{print \$1}')
192 kubectl cp $WORKSPACE/voltha-system-tests/tests/data/TechProfile-${profile}.json voltha/\$etcd_container:/tmp/flexpod.json
193 kubectl exec -it \$etcd_container -n voltha -- /bin/sh -c 'cat /tmp/flexpod.json | ETCDCTL_API=3 etcdctl put service/voltha/technology_profiles/XGS-PON/64'
194 """
195 }
196 }
197
198 stage('Push Sadis-config') {
199 steps {
200 sh returnStdout: false, script: """
201 ssh-keygen -R [${deployment_config.nodes[0].ip}]:30115
202 ssh-keyscan -p 30115 -H ${deployment_config.nodes[0].ip} >> ~/.ssh/known_hosts
203 sshpass -p karaf ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@${deployment_config.nodes[0].ip} "log:set TRACE org.opencord.dhcpl2relay"
204 sshpass -p karaf ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@${deployment_config.nodes[0].ip} "log:set TRACE org.opencord.aaa"
205 sshpass -p karaf ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@${deployment_config.nodes[0].ip} "log:set TRACE org.opencord.olt"
Andrea Campanella39b9b3d2021-09-09 16:54:20 +0200206 #TRACE in the pipeliner is too chatty, moving to DEBUG
207 sshpass -p karaf ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@${deployment_config.nodes[0].ip} "log:set DEBUG org.opencord.olt.driver"
Matteo Scandolo9b644ba2021-04-19 11:21:07 -0700208 sshpass -p karaf ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@${deployment_config.nodes[0].ip} "log:set DEBUG org.onosproject.net.flowobjective.impl.FlowObjectiveManager"
209 sshpass -p karaf ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@${deployment_config.nodes[0].ip} "log:set DEBUG org.onosproject.net.flowobjective.impl.InOrderFlowObjectiveManager"
210
211 if [[ "${workFlow.toUpperCase()}" == "DT" ]]; then
212 curl -sSL --user karaf:karaf -X POST -H Content-Type:application/json http://${deployment_config.nodes[0].ip}:30120/onos/v1/network/configuration --data @$WORKSPACE/voltha-system-tests/tests/data/${configFileName}-sadis-DT.json
213 elif [[ "${workFlow.toUpperCase()}" == "TT" ]]; then
214 curl -sSL --user karaf:karaf -X POST -H Content-Type:application/json http://${deployment_config.nodes[0].ip}:30120/onos/v1/network/configuration --data @$WORKSPACE/voltha-system-tests/tests/data/${configFileName}-sadis-TT.json
215 else
216 # this is the ATT case, rename the file in *-sadis-ATT.json so that we can avoid special cases and just load the file
217 curl -sSL --user karaf:karaf -X POST -H Content-Type:application/json http://${deployment_config.nodes[0].ip}:30120/onos/v1/network/configuration --data @$WORKSPACE/voltha-system-tests/tests/data/${configFileName}-sadis.json
218 fi
219 """
220 }
221 }
222 stage('Reinstall OLT software') {
223 when {
224 expression { params.reinstallOlt }
225 }
226 steps {
227 script {
228 deployment_config.olts.each { olt ->
229 sh returnStdout: false, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'dpkg --remove asfvolt16 && dpkg --purge asfvolt16'"
230 waitUntil {
231 olt_sw_present = sh returnStdout: true, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'dpkg --list | grep asfvolt16 | wc -l'"
232 return olt_sw_present.toInteger() == 0
233 }
234 if ( params.branch == 'voltha-2.3' ) {
235 oltDebVersion = oltDebVersionVoltha23
236 } else {
237 oltDebVersion = oltDebVersionMaster
238 }
239 sh returnStdout: false, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'dpkg --install ${oltDebVersion}'"
240 waitUntil {
241 olt_sw_present = sh returnStdout: true, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'dpkg --list | grep asfvolt16 | wc -l'"
242 return olt_sw_present.toInteger() == 1
243 }
244 if ( olt.fortygig ) {
245 // If the OLT is connected to a 40G switch interface, set the NNI port to be downgraded
246 sh returnStdout: false, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'echo port ce128 sp=40000 >> /broadcom/qax.soc ; /opt/bcm68620/svk_init.sh'"
247 }
248 }
249 }
250 }
251 }
252
253 stage('Restart OLT processes') {
254 steps {
255 script {
256 deployment_config.olts.each { olt ->
257 sh returnStdout: false, script: """
258 ssh-keyscan -H ${olt.ip} >> ~/.ssh/known_hosts
259 sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'rm -f /var/log/openolt.log; rm -f /var/log/dev_mgmt_daemon.log; reboot'
260 sleep 120
261 """
262 waitUntil {
263 onu_discovered = sh returnStdout: true, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'grep \"onu discover indication\" /var/log/openolt.log | wc -l'"
264 return onu_discovered.toInteger() > 0
265 }
266 }
267 }
268 }
269 }
270 stage('Run E2E Tests') {
271 steps {
272 script {
Matteo Scandolob6d80732021-05-05 14:06:42 -0700273 // different workflows need different make targets and different robot files
Matteo Scandolo9b644ba2021-04-19 11:21:07 -0700274 if ( params.workflow.toUpperCase() == "DT" ) {
275 robotConfigFile = "${configBaseDir}/${configDeploymentDir}/${configFileName}-DT.yaml"
Matteo Scandolob6d80732021-05-05 14:06:42 -0700276 robotFile = "Voltha_DT_PODTests.robot"
277 makeTarget = "voltha-dt-test"
278 robotFunctionalKeyword = "-i functionalDt"
279 robotDataplaneKeyword = "-i dataplaneDt"
Matteo Scandolo9b644ba2021-04-19 11:21:07 -0700280 }
281 else if ( params.workflow.toUpperCase() == "TT" ) {
Matteo Scandolob6d80732021-05-05 14:06:42 -0700282 // TODO the TT tests have diffent tags, address once/if TT is support on the Tucson POD
Matteo Scandolo9b644ba2021-04-19 11:21:07 -0700283 robotConfigFile = "${configBaseDir}/${configDeploymentDir}/${configFileName}-TT.yaml"
Matteo Scandolob6d80732021-05-05 14:06:42 -0700284 robotFile = "Voltha_TT_PODTests.robot"
285 makeTarget = "voltha-tt-test"
286 robotFunctionalKeyword = "-i functionalTt"
287 robotDataplaneKeyword = "-i dataplaneTt"
Matteo Scandolo9b644ba2021-04-19 11:21:07 -0700288 }
289 else {
290 robotConfigFile = "${configBaseDir}/${configDeploymentDir}/${configFileName}.yaml"
Matteo Scandolob6d80732021-05-05 14:06:42 -0700291 robotFile = "Voltha_PODTests.robot"
292 makeTarget = "voltha-test"
293 robotFunctionalKeyword = "-i functional"
294 robotDataplaneKeyword = "-i dataplane"
Matteo Scandolo9b644ba2021-04-19 11:21:07 -0700295 }
296 }
297 sh returnStdout: false, script: """
298 mkdir -p $WORKSPACE/RobotLogs
299
300 export ROBOT_CONFIG_FILE="$WORKSPACE/${robotConfigFile}"
301 export ROBOT_MISC_ARGS="${params.extraRobotArgs} --removekeywords wuks -d $WORKSPACE/RobotLogs -v container_log_dir:$WORKSPACE "
Matteo Scandolob6d80732021-05-05 14:06:42 -0700302 export ROBOT_FILE="${robotFile}"
Matteo Scandolo9b644ba2021-04-19 11:21:07 -0700303
304 # If the Gerrit comment contains a line with "functional tests" then run the full
305 # functional test suite. This covers tests tagged either 'sanity' or 'functional'.
306 # Note: Gerrit comment text will be prefixed by "Patch set n:" and a blank line
307 REGEX="functional tests"
308 if [[ "${gerritComment}" =~ \$REGEX ]]; then
Matteo Scandolob6d80732021-05-05 14:06:42 -0700309 ROBOT_MISC_ARGS+="${robotFunctionalKeyword} "
Matteo Scandolo9b644ba2021-04-19 11:21:07 -0700310 fi
311 # Likewise for dataplane tests
312 REGEX="dataplane tests"
313 if [[ "${gerritComment}" =~ \$REGEX ]]; then
Matteo Scandolob6d80732021-05-05 14:06:42 -0700314 ROBOT_MISC_ARGS+="${robotDataplaneKeyword}"
Matteo Scandolo9b644ba2021-04-19 11:21:07 -0700315 fi
316
Hardik Windlass7e4e3152021-09-29 06:42:05 +0000317 ROBOT_MISC_ARGS+=" -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace}"
Matteo Scandolob6d80732021-05-05 14:06:42 -0700318 make -C $WORKSPACE/voltha-system-tests ${makeTarget} || true
Matteo Scandolo9b644ba2021-04-19 11:21:07 -0700319 """
320 }
321 }
322 }
323 post {
324 always {
325 // stop logging
326 sh """
327 P_IDS="\$(ps e -ww -A | grep "_TAG=kail-${workFlow}" | grep -v grep | awk '{print \$1}')"
328 if [ -n "\$P_IDS" ]; then
329 echo \$P_IDS
330 for P_ID in \$P_IDS; do
331 kill -9 \$P_ID
332 done
333 fi
334 gzip $WORKSPACE/${workFlow}/onos-voltha-combined.log || true
335 """
Andrea Campanella1198cd52021-06-14 16:17:25 +0200336 sh '''
337 # stop the kail processes
338 list=($APPS_TO_LOG)
339 for app in "${list[@]}"
340 do
341 echo "Stopping logs for: ${app}"
342 _TAG="kail-$app"
343 P_IDS="$(ps e -ww -A | grep "_TAG=$_TAG" | grep -v grep | awk '{print $1}')"
344 if [ -n "$P_IDS" ]; then
345 echo $P_IDS
346 for P_ID in $P_IDS; do
347 kill -9 $P_ID
348 done
349 fi
350 done
351 '''
Matteo Scandolo9b644ba2021-04-19 11:21:07 -0700352 step([$class: 'RobotPublisher',
353 disableArchiveOutput: false,
354 logFileName: 'RobotLogs/log*.html',
355 otherFiles: '',
356 outputFileName: 'RobotLogs/output*.xml',
357 outputPath: '.',
358 passThreshold: 100,
359 reportFileName: 'RobotLogs/report*.html',
Andrea Campanellaabc09772021-06-16 12:08:57 +0200360 unstableThreshold: 0,
361 onlyCritical: true]);
Andrea Campanella1198cd52021-06-14 16:17:25 +0200362 archiveArtifacts artifacts: '**/*.txt,**/*.gz,*.gz,**/*.log'
Matteo Scandolo9b644ba2021-04-19 11:21:07 -0700363 }
364 }
365}
366
367// refs/changes/06/24206/5