blob: b91d2a9639bfbf06518693d31e2d3f0c11f80a46 [file] [log] [blame]
Joey Armstrong16bd8e82023-01-12 15:20:06 -05001#!/usr/bin/env groovy
2// -----------------------------------------------------------------------
3// Copyright 2022-2023 Open Networking Foundation (ONF) and the ONF Contributors
Hardik Windlass36d5bdb2022-02-09 10:13:13 +00004//
5// Licensed under the Apache License, Version 2.0 (the "License");
6// you may not use this file except in compliance with the License.
7// You may obtain a copy of the License at
8//
9// http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing, software
12// distributed under the License is distributed on an "AS IS" BASIS,
13// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14// See the License for the specific language governing permissions and
15// limitations under the License.
Joey Armstrong16bd8e82023-01-12 15:20:06 -050016// -----------------------------------------------------------------------
17// Intent: used to deploy VOLTHA and configure ONOS physical PODs
18//
Hardik Windlass36d5bdb2022-02-09 10:13:13 +000019// NOTE we are importing the library even if it's global so that it's
20// easier to change the keywords during a replay
Joey Armstrong16bd8e82023-01-12 15:20:06 -050021// -----------------------------------------------------------------------
Joey Armstrong65405412022-11-22 10:43:06 -050022
Hardik Windlass36d5bdb2022-02-09 10:13:13 +000023library identifier: 'cord-jenkins-libraries@master',
24 retriever: modernSCM([
25 $class: 'GitSCMSource',
26 remote: 'https://gerrit.opencord.org/ci-management.git'
27])
28
29def infraNamespace = "infra"
30def volthaNamespace = "voltha"
31
32def deploy_custom_chart(namespace, name, chart, extraHelmFlags) {
33 sh """
34 helm install --create-namespace --set defaults.image_pullPolicy=Always --namespace ${namespace} ${extraHelmFlags} ${name} ${chart}
35 """
36}
37
38pipeline {
39
40 /* no label, executor is determined by JJB */
41 agent {
42 label "${params.buildNode}"
43 }
44 options {
45 timeout(time: 45, unit: 'MINUTES')
46 }
47 environment {
48 PATH="$PATH:$WORKSPACE/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin"
49 KUBECONFIG="$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf"
Jan Klarece1bc812023-09-28 13:15:52 +020050 VOLTCONFIG="$HOME/.volt/config"
Hardik Windlass36d5bdb2022-02-09 10:13:13 +000051 LOG_FOLDER="$WORKSPACE/dmi/"
52 APPS_TO_LOG="${OltDevMgr}"
53 }
54
55 stages{
56 stage('Download Code') {
57 steps {
58 getVolthaCode([
59 branch: "${branch}",
60 volthaSystemTestsChange: "${volthaSystemTestsChange}",
61 volthaHelmChartsChange: "${volthaHelmChartsChange}",
62 ])
63 }
64 }
65 stage ("Parse deployment configuration file") {
66 steps {
67 sh returnStdout: true, script: "rm -rf ${configBaseDir}"
68 sh returnStdout: true, script: "git clone -b ${branch} ${cordRepoUrl}/${configBaseDir}"
69 script {
70 if ( params.workFlow == "DT" ) {
71 deployment_config = readYaml file: "${configBaseDir}/${configDeploymentDir}/${configFileName}-DT.yaml"
72 }
73 else if ( params.workFlow == "TT" )
74 {
75 deployment_config = readYaml file: "${configBaseDir}/${configDeploymentDir}/${configFileName}-TT.yaml"
76 }
77 else
78 {
79 deployment_config = readYaml file: "${configBaseDir}/${configDeploymentDir}/${configFileName}.yaml"
80 }
81 }
82 }
83 }
84 stage('Clean up') {
85 steps {
86 timeout(15) {
87 script {
88 helmTeardown(["default", infraNamespace, volthaNamespace])
89 }
90 timeout(1) {
91 sh returnStdout: false, script: '''
92 # remove orphaned port-forward from different namespaces
93 ps aux | grep port-forw | grep -v grep | awk '{print $2}' | xargs --no-run-if-empty kill -9 || true
94 '''
95 }
96 }
97 }
98 }
99 stage('Install Voltha') {
100 steps {
101 timeout(20) {
102 installVoltctl("${branch}")
103 script {
104 // if we're downloading a voltha-helm-charts patch, then install from a local copy of the charts
105 def localCharts = false
106 if (volthaHelmChartsChange != "") {
107 localCharts = true
108 }
109
110 // should the config file be suffixed with the workflow? see "deployment_config"
111 def localHelmFlags = "-f $WORKSPACE/${configBaseDir}/${configKubernetesDir}/voltha/${configFileName}.yml --set global.log_level=${logLevel} "
112
113 if (workFlow.toLowerCase() == "dt") {
114 localHelmFlags += " --set radius.enabled=false "
115 }
116 if (workFlow.toLowerCase() == "tt") {
117 localHelmFlags += " --set radius.enabled=false --set global.incremental_evto_update=true "
118 if (enableMultiUni.toBoolean()) {
119 localHelmFlags += " --set voltha-adapter-openonu.adapter_open_onu.uni_port_mask=${uniPortMask} "
120 }
121 }
122
123 // NOTE temporary workaround expose ONOS node ports (pod-config needs to be updated to contain these values)
124 // and to connect the ofagent to all instances of ONOS
125 localHelmFlags = localHelmFlags + " --set onos-classic.onosSshPort=30115 " +
126 "--set onos-classic.onosApiPort=30120 " +
127 "--set onos-classic.onosOfPort=31653 " +
128 "--set onos-classic.individualOpenFlowNodePorts=true " +
129 "--set voltha.onos_classic.replicas=${params.NumOfOnos}"
130
131 if (bbsimReplicas.toInteger() != 0) {
132 localHelmFlags = localHelmFlags + " --set onu=${onuNumber},pon=${ponNumber} "
133 }
134
135 // adding user specified helm flags at the end so they'll have priority over everything else
136 localHelmFlags = localHelmFlags + " ${extraHelmFlags}"
137
138 def numberOfAdaptersToWait = 2
139
140 if(openoltAdapterChart != "onf/voltha-adapter-openolt") {
141 localHelmFlags = localHelmFlags + " --set voltha-adapter-openolt.enabled=false"
142 // We skip waiting for adapters in the volthaDeploy step because it's already waiting for
143 // both of them after the deployment of the custom olt adapter. See line 156.
144 numberOfAdaptersToWait = 0
145 }
146
147 volthaDeploy([
148 workflow: workFlow.toLowerCase(),
149 extraHelmFlags: localHelmFlags,
150 localCharts: localCharts,
151 kubeconfig: "$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf",
152 onosReplica: params.NumOfOnos,
153 atomixReplica: params.NumOfAtomix,
154 kafkaReplica: params.NumOfKafka,
155 etcdReplica: params.NumOfEtcd,
156 bbsimReplica: bbsimReplicas.toInteger(),
157 adaptersToWait: numberOfAdaptersToWait,
Hardik Windlass230fca62022-05-04 08:50:35 +0530158 withVolthaInfra: installVolthaInfra.toBoolean(),
159 withVolthaStack: installVolthaStack.toBoolean(),
Hardik Windlass36d5bdb2022-02-09 10:13:13 +0000160 ])
161
Hardik Windlass230fca62022-05-04 08:50:35 +0530162 if(installVolthaStack.toBoolean()) {
163 if(openoltAdapterChart != "onf/voltha-adapter-openolt"){
164 extraHelmFlags = extraHelmFlags + " --set global.log_level=${logLevel}"
165 deploy_custom_chart(volthaNamespace, oltAdapterReleaseName, openoltAdapterChart, extraHelmFlags)
166 waitForAdapters([
167 adaptersToWait: 2
168 ])
169 }
Hardik Windlass36d5bdb2022-02-09 10:13:13 +0000170 }
171 }
172 sh """
173 JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-api" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${volthaNamespace} svc/voltha-voltha-api 55555:55555; done"&
174 JENKINS_NODE_COOKIE="dontKillMe" _TAG="etcd" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${infraNamespace} svc/voltha-infra-etcd ${params.VolthaEtcdPort}:2379; done"&
175 JENKINS_NODE_COOKIE="dontKillMe" _TAG="kafka" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${infraNamespace} svc/voltha-infra-kafka 9092:9092; done"&
176 ps aux | grep port-forward
177 """
178 getPodsInfo("$WORKSPACE")
179 }
180 }
181 }
182 stage('Deploy Device Manager Interface Chart') {
183 steps {
184 script {
185 deploy_custom_chart('default', 'olt-device-manager', dmiChart, extraHelmFlags)
186 }
187 println "Wait for olt-device-manager to start"
188 sh """
189 set +x
190 devmgr=\$(kubectl get pods -l app.kubernetes.io/name=${params.OltDevMgr} --no-headers | grep "0/" | wc -l)
191 while [[ \$devmgr != 0 ]]; do
192 sleep 5
193 devmgr=\$(kubectl get pods -l app.kubernetes.io/name=${params.OltDevMgr} --no-headers | grep "0/" | wc -l)
194 done
195 """
196 sh """
Hardik Windlassace19e52022-02-10 09:36:21 +0000197 JENKINS_NODE_COOKIE="dontKillMe" _TAG="${params.OltDevMgr}" bash -c "while true; do kubectl port-forward --address 0.0.0.0 svc/${params.OltDevMgr} 50051; done"&
Hardik Windlass36d5bdb2022-02-09 10:13:13 +0000198 ps aux | grep port-forward
199 """
200 }
201 }
Joey Armstrong16bd8e82023-01-12 15:20:06 -0500202 stage('Start logging')
203 {
204 steps
205 {
206 // Install kail
207 sh("""make -C "$WORKSPACE/voltha-system-tests" KAIL_PATH="$WORKSPACE/bin" kail""")
208
209 sh returnStdout: false, script: '''
Hardik Windlass36d5bdb2022-02-09 10:13:13 +0000210 # start logging with kail
211 cd $WORKSPACE
212 mkdir -p $LOG_FOLDER
213 list=($APPS_TO_LOG)
214 for app in "${list[@]}"
215 do
216 echo "Starting logs for: ${app}"
217 _TAG=kail-$app kail -l app.kubernetes.io/name=$app --since 1h > $LOG_FOLDER/$app.log&
218 done
219 '''
Joey Armstrong16bd8e82023-01-12 15:20:06 -0500220 }
221 }
222
223 stage('Reinstall OLT software') {
Hardik Windlass36d5bdb2022-02-09 10:13:13 +0000224 steps {
225 script {
226 if ( params.reinstallOlt ) {
227 for(int i=0; i < deployment_config.olts.size(); i++) {
228 sh returnStdout: true, script: """
229 ssh-keyscan -H ${deployment_config.olts[i].sship} >> ~/.ssh/known_hosts
230 if [ "${params.inBandManagement}" == "true" ]; then
231 sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].sship} 'kill -9 `pgrep -f "[b]ash /opt/openolt/openolt_dev_mgmt_daemon_process_watchdog"` || true'
232 fi
233 sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].sship} "dpkg --install ${deployment_config.olts[i].oltDebVersion}"
234 sleep 10
235 """
236 timeout(5) {
237 waitUntil {
238 olt_sw_present = sh returnStdout: true, script: """
239 if [[ "${deployment_config.olts[i].oltDebVersion}" == *"asfvolt16"* ]]; then
240 sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].sship} 'dpkg --list | grep asfvolt16 | wc -l'
241 elif [[ "${deployment_config.olts[i].oltDebVersion}" == *"asgvolt64"* ]]; then
242 sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].sship} 'dpkg --list | grep asgvolt64 | wc -l'
243 elif [[ "${deployment_config.olts[i].oltDebVersion}" == *"rlt-1600x-w"* ]]; then
244 sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].sship} 'dpkg --list | grep rlt-1600x-w | wc -l'
245 elif [[ "${deployment_config.olts[i].oltDebVersion}" == *"rlt-1600g-w"* ]]; then
246 sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].sship} 'dpkg --list | grep rlt-1600g-w | wc -l'
247 elif [[ "${deployment_config.olts[i].oltDebVersion}" == *"rlt-3200g-w"* ]]; then
248 sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].sship} 'dpkg --list | grep rlt-3200g-w | wc -l'
249 else
250 echo Unknown Debian package for openolt
251 fi
252 if (${deployment_config.olts[i].fortygig}); then
253 if [[ "${params.inBandManagement}" == "true" ]]; then
254 ssh-keyscan -H ${deployment_config.olts[i].sship} >> ~/.ssh/known_hosts
255 sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].sship} 'mkdir -p /opt/openolt/'
256 sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].sship} 'cp /root/watchdog-script/* /opt/openolt/'
257 sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].sship} 'cp /root/bal_cli_appl/example_user_appl /broadcom'
258 sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].sship} 'cp in-band-startup-script/* /etc/init.d/'
259 fi
260 fi
261 """
262 return olt_sw_present.toInteger() > 0
263 }
264 }
265 }
266 }
267 }
268 }
269 }
270 stage('Restart OLT processes') {
271 steps {
272 script {
273 if ( params.restartOlt ) {
274 //rebooting OLTs
275 for(int i=0; i < deployment_config.olts.size(); i++) {
276 timeout(15) {
277 sh returnStdout: true, script: """
278 ssh-keyscan -H ${deployment_config.olts[i].sship} >> ~/.ssh/known_hosts
279 sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].sship} 'rm -f /var/log/openolt.log; rm -f /var/log/dev_mgmt_daemon.log; rm -f /var/log/openolt_process_watchdog.log; reboot > /dev/null &' || true
280 """
281 }
282 }
283 sh returnStdout: true, script: """
284 sleep ${params.waitTimerForOltUp}
285 """
286 //Checking dev_management_deamon and openoltprocesses
287 for(int i=0; i < deployment_config.olts.size(); i++) {
288 if ( params.oltAdapterReleaseName != "open-olt" ) {
289 timeout(15) {
290 waitUntil {
291 devprocess = sh returnStdout: true, script: "sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].sship} 'ps -ef | grep dev_mgmt_daemon | wc -l'"
292 return devprocess.toInteger() > 0
293 }
294 }
295 timeout(15) {
296 waitUntil {
297 openoltprocess = sh returnStdout: true, script: "sshpass -p ${deployment_config.olts[i].pass} ssh -l ${deployment_config.olts[i].user} ${deployment_config.olts[i].sship} 'ps -ef | grep openolt | wc -l'"
298 return openoltprocess.toInteger() > 0
299 }
300 }
301 }
302 }
303 }
304 }
305 }
306 }
307 stage('Run Device Management Interface Tests') {
308 environment {
309 ROBOT_FILE="dmi-hw-management.robot"
310 ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs"
311 ROBOT_CONFIG_FILE="$WORKSPACE/voltha-system-tests/tests/data/dmi-components-adtran.yaml"
312 }
313 steps {
314 sh """
315 mkdir -p $ROBOT_LOGS_DIR
316 export ROBOT_MISC_ARGS="--removekeywords wuks -e notreadyDMI -i functionalDMI -d $ROBOT_LOGS_DIR"
317 make -C $WORKSPACE/voltha-system-tests voltha-dmi-test || true
318 """
319 }
320 }
321 }
322
323 post {
324 always {
Hardik Windlassace19e52022-02-10 09:36:21 +0000325 getPodsInfo("$WORKSPACE")
Hardik Windlass36d5bdb2022-02-09 10:13:13 +0000326 sh '''
327 # stop the kail processes
328 list=($APPS_TO_LOG)
329 for app in "${list[@]}"
330 do
331 echo "Stopping logs for: ${app}"
332 _TAG="kail-$app"
333 P_IDS="$(ps e -ww -A | grep "_TAG=$_TAG" | grep -v grep | awk '{print $1}')"
334 if [ -n "$P_IDS" ]; then
335 echo $P_IDS
336 for P_ID in $P_IDS; do
337 kill -9 $P_ID
338 done
339 fi
340 done
341 '''
342 step([$class: 'RobotPublisher',
343 disableArchiveOutput: false,
344 logFileName: 'RobotLogs/log*.html',
345 otherFiles: '',
346 outputFileName: 'RobotLogs/output*.xml',
347 outputPath: '.',
348 passThreshold: 100,
349 reportFileName: 'RobotLogs/report*.html',
350 unstableThreshold: 0,
351 onlyCritical: true]);
352 archiveArtifacts artifacts: '**/*.txt,**/*.gz,*.gz,**/*.log'
353 }
354 }
355}