blob: 501c7f53a05a57f48792fe3d37f3810498712a70 [file] [log] [blame]
Joey Armstrong16bd8e82023-01-12 15:20:06 -05001#!/usr/bin/env groovy
2// -----------------------------------------------------------------------
3// Copyright 2022-2023 Open Networking Foundation (ONF) and the ONF Contributors
Hardik Windlass36d5bdb2022-02-09 10:13:13 +00004//
5// Licensed under the Apache License, Version 2.0 (the "License");
6// you may not use this file except in compliance with the License.
7// You may obtain a copy of the License at
8//
9// http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing, software
12// distributed under the License is distributed on an "AS IS" BASIS,
13// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14// See the License for the specific language governing permissions and
15// limitations under the License.
Joey Armstrong16bd8e82023-01-12 15:20:06 -050016// -----------------------------------------------------------------------
17// Intent: used to deploy VOLTHA and configure ONOS physical PODs
18//
Hardik Windlass36d5bdb2022-02-09 10:13:13 +000019// NOTE we are importing the library even if it's global so that it's
20// easier to change the keywords during a replay
Joey Armstrong16bd8e82023-01-12 15:20:06 -050021// -----------------------------------------------------------------------
Joey Armstrong65405412022-11-22 10:43:06 -050022
Hardik Windlass36d5bdb2022-02-09 10:13:13 +000023library identifier: 'cord-jenkins-libraries@master',
24 retriever: modernSCM([
25 $class: 'GitSCMSource',
26 remote: 'https://gerrit.opencord.org/ci-management.git'
27])
28
29def infraNamespace = "infra"
30def volthaNamespace = "voltha"
31
32def deploy_custom_chart(namespace, name, chart, extraHelmFlags) {
33 sh """
34 helm install --create-namespace --set defaults.image_pullPolicy=Always --namespace ${namespace} ${extraHelmFlags} ${name} ${chart}
35 """
36}
37
38pipeline {
39
40 /* no label, executor is determined by JJB */
41 agent {
42 label "${params.buildNode}"
43 }
44 options {
45 timeout(time: 45, unit: 'MINUTES')
46 }
47 environment {
48 PATH="$PATH:$WORKSPACE/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin"
49 KUBECONFIG="$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf"
Jan Klarece1bc812023-09-28 13:15:52 +020050 VOLTCONFIG="$HOME/.volt/config"
Hardik Windlass36d5bdb2022-02-09 10:13:13 +000051 LOG_FOLDER="$WORKSPACE/dmi/"
52 APPS_TO_LOG="${OltDevMgr}"
53 }
54
55 stages{
56 stage('Download Code') {
57 steps {
58 getVolthaCode([
59 branch: "${branch}",
60 volthaSystemTestsChange: "${volthaSystemTestsChange}",
61 volthaHelmChartsChange: "${volthaHelmChartsChange}",
62 ])
63 }
64 }
65 stage ("Parse deployment configuration file") {
66 steps {
67 sh returnStdout: true, script: "rm -rf ${configBaseDir}"
68 sh returnStdout: true, script: "git clone -b ${branch} ${cordRepoUrl}/${configBaseDir}"
69 script {
70 if ( params.workFlow == "DT" ) {
71 deployment_config = readYaml file: "${configBaseDir}/${configDeploymentDir}/${configFileName}-DT.yaml"
72 }
73 else if ( params.workFlow == "TT" )
74 {
75 deployment_config = readYaml file: "${configBaseDir}/${configDeploymentDir}/${configFileName}-TT.yaml"
76 }
77 else
78 {
79 deployment_config = readYaml file: "${configBaseDir}/${configDeploymentDir}/${configFileName}.yaml"
80 }
81 }
82 }
83 }
84 stage('Clean up') {
85 steps {
86 timeout(15) {
87 script {
88 helmTeardown(["default", infraNamespace, volthaNamespace])
89 }
90 timeout(1) {
91 sh returnStdout: false, script: '''
92 # remove orphaned port-forward from different namespaces
93 ps aux | grep port-forw | grep -v grep | awk '{print $2}' | xargs --no-run-if-empty kill -9 || true
94 '''
95 }
96 }
97 }
98 }
99 stage('Install Voltha') {
100 steps {
101 timeout(20) {
102 installVoltctl("${branch}")
103 script {
104 // if we're downloading a voltha-helm-charts patch, then install from a local copy of the charts
105 def localCharts = false
106 if (volthaHelmChartsChange != "") {
107 localCharts = true
108 }
109
110 // should the config file be suffixed with the workflow? see "deployment_config"
111 def localHelmFlags = "-f $WORKSPACE/${configBaseDir}/${configKubernetesDir}/voltha/${configFileName}.yml --set global.log_level=${logLevel} "
112
113 if (workFlow.toLowerCase() == "dt") {
114 localHelmFlags += " --set radius.enabled=false "
115 }
116 if (workFlow.toLowerCase() == "tt") {
117 localHelmFlags += " --set radius.enabled=false --set global.incremental_evto_update=true "
118 if (enableMultiUni.toBoolean()) {
119 localHelmFlags += " --set voltha-adapter-openonu.adapter_open_onu.uni_port_mask=${uniPortMask} "
120 }
121 }
122
123 // NOTE temporary workaround expose ONOS node ports (pod-config needs to be updated to contain these values)
124 // and to connect the ofagent to all instances of ONOS
125 localHelmFlags = localHelmFlags + " --set onos-classic.onosSshPort=30115 " +
126 "--set onos-classic.onosApiPort=30120 " +
127 "--set onos-classic.onosOfPort=31653 " +
128 "--set onos-classic.individualOpenFlowNodePorts=true " +
129 "--set voltha.onos_classic.replicas=${params.NumOfOnos}"
130
131 if (bbsimReplicas.toInteger() != 0) {
132 localHelmFlags = localHelmFlags + " --set onu=${onuNumber},pon=${ponNumber} "
133 }
134
135 // adding user specified helm flags at the end so they'll have priority over everything else
136 localHelmFlags = localHelmFlags + " ${extraHelmFlags}"
137
138 def numberOfAdaptersToWait = 2
139
140 if(openoltAdapterChart != "onf/voltha-adapter-openolt") {
141 localHelmFlags = localHelmFlags + " --set voltha-adapter-openolt.enabled=false"
142 // We skip waiting for adapters in the volthaDeploy step because it's already waiting for
143 // both of them after the deployment of the custom olt adapter. See line 156.
144 numberOfAdaptersToWait = 0
145 }
146
147 volthaDeploy([
148 workflow: workFlow.toLowerCase(),
149 extraHelmFlags: localHelmFlags,
150 localCharts: localCharts,
151 kubeconfig: "$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf",
152 onosReplica: params.NumOfOnos,
153 atomixReplica: params.NumOfAtomix,
154 kafkaReplica: params.NumOfKafka,
155 etcdReplica: params.NumOfEtcd,
156 bbsimReplica: bbsimReplicas.toInteger(),
157 adaptersToWait: numberOfAdaptersToWait,
Hardik Windlass230fca62022-05-04 08:50:35 +0530158 withVolthaInfra: installVolthaInfra.toBoolean(),
159 withVolthaStack: installVolthaStack.toBoolean(),
Hardik Windlass36d5bdb2022-02-09 10:13:13 +0000160 ])
161
Hardik Windlass230fca62022-05-04 08:50:35 +0530162 if(installVolthaStack.toBoolean()) {
163 if(openoltAdapterChart != "onf/voltha-adapter-openolt"){
164 extraHelmFlags = extraHelmFlags + " --set global.log_level=${logLevel}"
165 deploy_custom_chart(volthaNamespace, oltAdapterReleaseName, openoltAdapterChart, extraHelmFlags)
166 waitForAdapters([
167 adaptersToWait: 2
168 ])
169 }
Hardik Windlass36d5bdb2022-02-09 10:13:13 +0000170 }
171 }
172 sh """
Hardik Windlass36d5bdb2022-02-09 10:13:13 +0000173 JENKINS_NODE_COOKIE="dontKillMe" _TAG="kafka" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${infraNamespace} svc/voltha-infra-kafka 9092:9092; done"&
174 ps aux | grep port-forward
175 """
176 getPodsInfo("$WORKSPACE")
177 }
178 }
179 }
180 stage('Deploy Device Manager Interface Chart') {
181 steps {
182 script {
183 deploy_custom_chart('default', 'olt-device-manager', dmiChart, extraHelmFlags)
184 }
185 println "Wait for olt-device-manager to start"
186 sh """
187 set +x
188 devmgr=\$(kubectl get pods -l app.kubernetes.io/name=${params.OltDevMgr} --no-headers | grep "0/" | wc -l)
189 while [[ \$devmgr != 0 ]]; do
190 sleep 5
191 devmgr=\$(kubectl get pods -l app.kubernetes.io/name=${params.OltDevMgr} --no-headers | grep "0/" | wc -l)
192 done
193 """
194 sh """
Hardik Windlassace19e52022-02-10 09:36:21 +0000195 JENKINS_NODE_COOKIE="dontKillMe" _TAG="${params.OltDevMgr}" bash -c "while true; do kubectl port-forward --address 0.0.0.0 svc/${params.OltDevMgr} 50051; done"&
Hardik Windlass36d5bdb2022-02-09 10:13:13 +0000196 ps aux | grep port-forward
197 """
198 }
199 }
Joey Armstrong16bd8e82023-01-12 15:20:06 -0500200 stage('Start logging')
201 {
202 steps
203 {
204 // Install kail
205 sh("""make -C "$WORKSPACE/voltha-system-tests" KAIL_PATH="$WORKSPACE/bin" kail""")
206
207 sh returnStdout: false, script: '''
Hardik Windlass36d5bdb2022-02-09 10:13:13 +0000208 # start logging with kail
209 cd $WORKSPACE
210 mkdir -p $LOG_FOLDER
211 list=($APPS_TO_LOG)
212 for app in "${list[@]}"
213 do
214 echo "Starting logs for: ${app}"
215 _TAG=kail-$app kail -l app.kubernetes.io/name=$app --since 1h > $LOG_FOLDER/$app.log&
216 done
217 '''
Joey Armstrong16bd8e82023-01-12 15:20:06 -0500218 }
219 }
220
221 stage('Reinstall OLT software') {
Hardik Windlass36d5bdb2022-02-09 10:13:13 +0000222 steps {
223 script {
224 if ( params.reinstallOlt ) {
225 for(int i=0; i < deployment_config.olts.size(); i++) {
226 sh returnStdout: true, script: """
227 ssh-keyscan -H ${deployment_config.olts[i].sship} >> ~/.ssh/known_hosts
228 if [ "${params.inBandManagement}" == "true" ]; then
229 sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].sship} 'kill -9 `pgrep -f "[b]ash /opt/openolt/openolt_dev_mgmt_daemon_process_watchdog"` || true'
230 fi
231 sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].sship} "dpkg --install ${deployment_config.olts[i].oltDebVersion}"
232 sleep 10
233 """
234 timeout(5) {
235 waitUntil {
236 olt_sw_present = sh returnStdout: true, script: """
237 if [[ "${deployment_config.olts[i].oltDebVersion}" == *"asfvolt16"* ]]; then
238 sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].sship} 'dpkg --list | grep asfvolt16 | wc -l'
239 elif [[ "${deployment_config.olts[i].oltDebVersion}" == *"asgvolt64"* ]]; then
240 sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].sship} 'dpkg --list | grep asgvolt64 | wc -l'
241 elif [[ "${deployment_config.olts[i].oltDebVersion}" == *"rlt-1600x-w"* ]]; then
242 sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].sship} 'dpkg --list | grep rlt-1600x-w | wc -l'
243 elif [[ "${deployment_config.olts[i].oltDebVersion}" == *"rlt-1600g-w"* ]]; then
244 sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].sship} 'dpkg --list | grep rlt-1600g-w | wc -l'
245 elif [[ "${deployment_config.olts[i].oltDebVersion}" == *"rlt-3200g-w"* ]]; then
246 sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].sship} 'dpkg --list | grep rlt-3200g-w | wc -l'
247 else
248 echo Unknown Debian package for openolt
249 fi
250 if (${deployment_config.olts[i].fortygig}); then
251 if [[ "${params.inBandManagement}" == "true" ]]; then
252 ssh-keyscan -H ${deployment_config.olts[i].sship} >> ~/.ssh/known_hosts
253 sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].sship} 'mkdir -p /opt/openolt/'
254 sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].sship} 'cp /root/watchdog-script/* /opt/openolt/'
255 sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].sship} 'cp /root/bal_cli_appl/example_user_appl /broadcom'
256 sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].sship} 'cp in-band-startup-script/* /etc/init.d/'
257 fi
258 fi
259 """
260 return olt_sw_present.toInteger() > 0
261 }
262 }
263 }
264 }
265 }
266 }
267 }
268 stage('Restart OLT processes') {
269 steps {
270 script {
271 if ( params.restartOlt ) {
272 //rebooting OLTs
273 for(int i=0; i < deployment_config.olts.size(); i++) {
274 timeout(15) {
275 sh returnStdout: true, script: """
276 ssh-keyscan -H ${deployment_config.olts[i].sship} >> ~/.ssh/known_hosts
277 sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].sship} 'rm -f /var/log/openolt.log; rm -f /var/log/dev_mgmt_daemon.log; rm -f /var/log/openolt_process_watchdog.log; reboot > /dev/null &' || true
278 """
279 }
280 }
281 sh returnStdout: true, script: """
282 sleep ${params.waitTimerForOltUp}
283 """
284 //Checking dev_management_deamon and openoltprocesses
285 for(int i=0; i < deployment_config.olts.size(); i++) {
286 if ( params.oltAdapterReleaseName != "open-olt" ) {
287 timeout(15) {
288 waitUntil {
289 devprocess = sh returnStdout: true, script: "sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].sship} 'ps -ef | grep dev_mgmt_daemon | wc -l'"
290 return devprocess.toInteger() > 0
291 }
292 }
293 timeout(15) {
294 waitUntil {
295 openoltprocess = sh returnStdout: true, script: "sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].sship} 'ps -ef | grep openolt | wc -l'"
296 return openoltprocess.toInteger() > 0
297 }
298 }
299 }
300 }
301 }
302 }
303 }
304 }
305 stage('Run Device Management Interface Tests') {
306 environment {
307 ROBOT_FILE="dmi-hw-management.robot"
308 ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs"
309 ROBOT_CONFIG_FILE="$WORKSPACE/voltha-system-tests/tests/data/dmi-components-adtran.yaml"
310 }
311 steps {
312 sh """
313 mkdir -p $ROBOT_LOGS_DIR
314 export ROBOT_MISC_ARGS="--removekeywords wuks -e notreadyDMI -i functionalDMI -d $ROBOT_LOGS_DIR"
315 make -C $WORKSPACE/voltha-system-tests voltha-dmi-test || true
316 """
317 }
318 }
319 }
320
321 post {
322 always {
Hardik Windlassace19e52022-02-10 09:36:21 +0000323 getPodsInfo("$WORKSPACE")
Hardik Windlass36d5bdb2022-02-09 10:13:13 +0000324 sh '''
325 # stop the kail processes
326 list=($APPS_TO_LOG)
327 for app in "${list[@]}"
328 do
329 echo "Stopping logs for: ${app}"
330 _TAG="kail-$app"
331 P_IDS="$(ps e -ww -A | grep "_TAG=$_TAG" | grep -v grep | awk '{print $1}')"
332 if [ -n "$P_IDS" ]; then
333 echo $P_IDS
334 for P_ID in $P_IDS; do
335 kill -9 $P_ID
336 done
337 fi
338 done
339 '''
340 step([$class: 'RobotPublisher',
341 disableArchiveOutput: false,
342 logFileName: 'RobotLogs/log*.html',
343 otherFiles: '',
344 outputFileName: 'RobotLogs/output*.xml',
345 outputPath: '.',
346 passThreshold: 100,
347 reportFileName: 'RobotLogs/report*.html',
348 unstableThreshold: 0,
349 onlyCritical: true]);
350 archiveArtifacts artifacts: '**/*.txt,**/*.gz,*.gz,**/*.log'
351 }
352 }
353}