blob: 3aa50e8bf9f6ca62b5ffdcb14f3c98613bbc7e93 [file] [log] [blame]
Joey Armstrong16bd8e82023-01-12 15:20:06 -05001#!/usr/bin/env groovy
2// -----------------------------------------------------------------------
Joey Armstrong56fdfec2024-03-01 13:43:36 -05003// Copyright 2017-2024 Open Networking Foundation Contributors
Andy Bavier358aa0b2019-11-22 11:55:39 -07004//
5// Licensed under the Apache License, Version 2.0 (the "License");
6// you may not use this file except in compliance with the License.
7// You may obtain a copy of the License at
8//
9// http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing, software
12// distributed under the License is distributed on an "AS IS" BASIS,
13// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14// See the License for the specific language governing permissions and
15// limitations under the License.
Joey Armstrong16bd8e82023-01-12 15:20:06 -050016// -----------------------------------------------------------------------
Joey Armstrong56fdfec2024-03-01 13:43:36 -050017// SPDX-FileCopyrightText: 2017-2024 Open Networking Foundation Contributors
18// SPDX-License-Identifier: Apache-2.0
19// -----------------------------------------------------------------------
20// Entropy: 0fcb5ffa-d1a4-11ee-be5e-9f44b7181764
21// -----------------------------------------------------------------------
22// Intent:
23// -----------------------------------------------------------------------
Andy Bavier358aa0b2019-11-22 11:55:39 -070024
Matteo Scandolo712514a2021-07-20 09:49:09 -070025library identifier: 'cord-jenkins-libraries@master',
26 retriever: modernSCM([
27 $class: 'GitSCMSource',
28 remote: 'https://gerrit.opencord.org/ci-management.git'
29])
30
Hardik Windlass7e4e3152021-09-29 06:42:05 +000031def infraNamespace = "infra"
32def volthaNamespace = "voltha"
33
Andy Bavier358aa0b2019-11-22 11:55:39 -070034node {
35 // Need this so that deployment_config has global scope when it's read later
36 deployment_config = null
37}
38
39pipeline {
40 /* no label, executor is determined by JJB */
41 agent {
42 label "${params.buildNode}"
43 }
44 options {
Hardik Windlassd0f01662021-09-03 08:07:15 +000045 timeout(time: "${timeout}", unit: 'MINUTES')
Andy Bavier358aa0b2019-11-22 11:55:39 -070046 }
47
48 environment {
49 KUBECONFIG="$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf"
Jan Klarece1bc812023-09-28 13:15:52 +020050 VOLTCONFIG="$HOME/.volt/config"
hwchiu942d1dc2020-03-03 12:03:45 -080051 PATH="$WORKSPACE/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
Andy Bavier358aa0b2019-11-22 11:55:39 -070052 }
Andy Bavier358aa0b2019-11-22 11:55:39 -070053 stages {
Suchitra Vemurie3bb90d2020-09-15 17:15:50 -070054 stage('Clone voltha-system-tests') {
55 steps {
Hardik Windlass13d8e952021-07-20 09:16:46 +000056 step([$class: 'WsCleanup'])
Suchitra Vemurie3bb90d2020-09-15 17:15:50 -070057 checkout([
58 $class: 'GitSCM',
59 userRemoteConfigs: [[
60 url: "https://gerrit.opencord.org/voltha-system-tests",
Matteo Scandoloa42c6f52020-09-19 01:35:12 +000061 refspec: "${volthaSystemTestsChange}"
Suchitra Vemurie3bb90d2020-09-15 17:15:50 -070062 ]],
Suchitra Vemuri1143ae32021-03-26 01:08:37 +000063 branches: [[ name: "${branch}", ]],
Suchitra Vemurie3bb90d2020-09-15 17:15:50 -070064 extensions: [
65 [$class: 'WipeWorkspace'],
66 [$class: 'RelativeTargetDirectory', relativeTargetDir: "voltha-system-tests"],
67 [$class: 'CloneOption', depth: 0, noTags: false, reference: '', shallow: false],
68 ],
69 ])
Hardik Windlassd4fc1792021-11-10 11:44:18 +000070 script {
71 sh(script:"""
72 if [ '${volthaSystemTestsChange}' != '' ] ; then
73 cd $WORKSPACE/voltha-system-tests;
74 git fetch https://gerrit.opencord.org/voltha-system-tests ${volthaSystemTestsChange} && git checkout FETCH_HEAD
75 fi
76 """)
77 }
Suchitra Vemurie3bb90d2020-09-15 17:15:50 -070078 }
79 }
Suchitra Vemurie3bb90d2020-09-15 17:15:50 -070080 stage('Download All the VOLTHA repos') {
81 when {
82 expression {
83 return "${branch}" == 'master';
84 }
85 }
86 steps {
87 checkout(changelog: true,
88 poll: false,
89 scm: [$class: 'RepoScm',
90 manifestRepositoryUrl: "${params.manifestUrl}",
91 manifestBranch: "${params.branch}",
92 currentBranch: true,
93 destinationDir: 'voltha',
94 forceSync: true,
95 resetFirst: true,
96 quiet: true,
97 jobs: 4,
98 showAllChanges: true]
99 )
100 }
101 }
102 stage ('Initialize') {
103 steps {
Hardik Windlass6f854a12021-07-12 13:20:21 +0000104 sh returnStdout: false, script: "git clone -b ${branch} ${cordRepoUrl}/${configBaseDir}"
Andy Bavier358aa0b2019-11-22 11:55:39 -0700105 script {
106 deployment_config = readYaml file: "${configBaseDir}/${configDeploymentDir}/${configFileName}.yaml"
107 }
Joey Armstrong16bd8e82023-01-12 15:20:06 -0500108 installVoltctl("${branch}")
Joey Armstrongdaa1f0a2024-04-03 18:07:59 -0400109
Joey Armstrong16bd8e82023-01-12 15:20:06 -0500110 sh(returnStdout: false, script: """
111
112 mkdir -p "$WORKSPACE/bin"
113
114 # install kail
115 make -C "$WORKSPACE/voltha-system-tests" KAIL_PATH="$WORKSPACE/bin" kail
Andy Bavier1f530f52020-04-14 15:57:33 -0700116
Suchitra Vemuri13421432020-06-05 17:34:33 -0700117 if [ "${params.branch}" == "master" ]; then
Andy Bavierd921f372020-04-20 17:03:31 -0700118 # Default kind-voltha config doesn't work on ONF demo pod for accessing kvstore.
119 # The issue is that the mgmt node is also one of the k8s nodes and so port forwarding doesn't work.
120 # We should change this. In the meantime here is a workaround.
Suchitra Vemuri13421432020-06-05 17:34:33 -0700121 set +e
Andy Bavierce83b172020-04-22 10:49:23 -0700122
Andy Bavier1f530f52020-04-14 15:57:33 -0700123 # Remove noise from voltha-core logs
Suchitra Vemuri13421432020-06-05 17:34:33 -0700124 voltctl log level set WARN read-write-core#github.com/opencord/voltha-go/db/model
125 voltctl log level set WARN read-write-core#github.com/opencord/voltha-lib-go/v3/pkg/kafka
Andy Bavier1f530f52020-04-14 15:57:33 -0700126 # Remove noise from openolt logs
Suchitra Vemuri13421432020-06-05 17:34:33 -0700127 voltctl log level set WARN adapter-open-olt#github.com/opencord/voltha-lib-go/v3/pkg/db
128 voltctl log level set WARN adapter-open-olt#github.com/opencord/voltha-lib-go/v3/pkg/probe
129 voltctl log level set WARN adapter-open-olt#github.com/opencord/voltha-lib-go/v3/pkg/kafka
130 fi
Joey Armstrong16bd8e82023-01-12 15:20:06 -0500131 """)
Andy Bavier358aa0b2019-11-22 11:55:39 -0700132 }
133 }
hwchiu942d1dc2020-03-03 12:03:45 -0800134
Suchitra Vemuriaf134ff2020-02-19 16:55:38 -0800135 stage('Functional Tests') {
Andy Bavier358aa0b2019-11-22 11:55:39 -0700136 environment {
137 ROBOT_CONFIG_FILE="$WORKSPACE/${configBaseDir}/${configDeploymentDir}/${configFileName}.yaml"
Andy Bavier358aa0b2019-11-22 11:55:39 -0700138 ROBOT_FILE="Voltha_PODTests.robot"
Andy Bavierfed57db2020-01-22 16:28:50 -0700139 ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/FunctionalTests"
Andy Bavier358aa0b2019-11-22 11:55:39 -0700140 }
141 steps {
142 sh """
Andy Bavierfed57db2020-01-22 16:28:50 -0700143 mkdir -p $ROBOT_LOGS_DIR
Suchitra Vemuri627786a2020-06-18 16:52:05 -0700144 if ( ${powerSwitch} ); then
Andrea Campanella0d3110c2021-01-20 12:25:45 +0100145 export ROBOT_MISC_ARGS="--removekeywords wuks -i PowerSwitch -i sanity -i functional -e bbsim -e notready -d $ROBOT_LOGS_DIR -v POD_NAME:${configFileName} -v KUBERNETES_CONFIGS_DIR:$WORKSPACE/${configBaseDir}/${configKubernetesDir} -v container_log_dir:$WORKSPACE -v OLT_ADAPTER_APP_LABEL:${oltAdapterAppLabel}"
Hardik Windlass28a7bd02022-04-12 13:14:54 +0000146 if ( ${powerCycleOlt} ); then
147 ROBOT_MISC_ARGS+=" -v power_cycle_olt:True"
148 fi
Suchitra Vemuri627786a2020-06-18 16:52:05 -0700149 else
Andrea Campanella0d3110c2021-01-20 12:25:45 +0100150 export ROBOT_MISC_ARGS="--removekeywords wuks -e PowerSwitch -i sanity -i functional -e bbsim -e notready -d $ROBOT_LOGS_DIR -v POD_NAME:${configFileName} -v KUBERNETES_CONFIGS_DIR:$WORKSPACE/${configBaseDir}/${configKubernetesDir} -v container_log_dir:$WORKSPACE -v OLT_ADAPTER_APP_LABEL:${oltAdapterAppLabel}"
Matteo Scandolo518c9e12020-07-21 11:32:23 -0700151 fi
Andrea Campanella79ced322021-11-03 17:24:26 +0000152 ROBOT_MISC_ARGS+=" -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace}"
Suchitra Vemurie3bb90d2020-09-15 17:15:50 -0700153 make -C $WORKSPACE/voltha-system-tests voltha-test || true
Andy Bavierfed57db2020-01-22 16:28:50 -0700154 """
155 }
156 }
157
Suchitra Vemuri5dae5392020-01-30 20:10:18 -0800158 stage('Failure/Recovery Tests') {
Andy Bavierfed57db2020-01-22 16:28:50 -0700159 environment {
160 ROBOT_CONFIG_FILE="$WORKSPACE/${configBaseDir}/${configDeploymentDir}/${configFileName}.yaml"
Suchitra Vemuri5dae5392020-01-30 20:10:18 -0800161 ROBOT_FILE="Voltha_FailureScenarios.robot"
162 ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/FailureScenarios"
Andy Bavierfed57db2020-01-22 16:28:50 -0700163 }
164 steps {
165 sh """
166 mkdir -p $ROBOT_LOGS_DIR
Suchitra Vemuri1560cb82020-03-27 15:34:57 -0700167 if ( ${powerSwitch} ); then
Andrea Campanella0d3110c2021-01-20 12:25:45 +0100168 export ROBOT_MISC_ARGS="--removekeywords wuks -L TRACE -i functional -i PowerSwitch -e bbsim -e notready -d $ROBOT_LOGS_DIR -v POD_NAME:${configFileName} -v KUBERNETES_CONFIGS_DIR:$WORKSPACE/${configBaseDir}/${configKubernetesDir} -v container_log_dir:$WORKSPACE -v OLT_ADAPTER_APP_LABEL:${oltAdapterAppLabel}"
Suchitra Vemuriaf134ff2020-02-19 16:55:38 -0800169 else
Andrea Campanella0d3110c2021-01-20 12:25:45 +0100170 export ROBOT_MISC_ARGS="--removekeywords wuks -L TRACE -i functional -e PowerSwitch -e bbsim -e notready -d $ROBOT_LOGS_DIR -v POD_NAME:${configFileName} -v KUBERNETES_CONFIGS_DIR:$WORKSPACE/${configBaseDir}/${configKubernetesDir} -v container_log_dir:$WORKSPACE -v OLT_ADAPTER_APP_LABEL:${oltAdapterAppLabel}"
Suchitra Vemuriaf134ff2020-02-19 16:55:38 -0800171 fi
Andrea Campanella79ced322021-11-03 17:24:26 +0000172 ROBOT_MISC_ARGS+=" -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace}"
Suchitra Vemurie3bb90d2020-09-15 17:15:50 -0700173 make -C $WORKSPACE/voltha-system-tests voltha-test || true
Andy Bavier358aa0b2019-11-22 11:55:39 -0700174 """
175 }
176 }
Andy Bavier358aa0b2019-11-22 11:55:39 -0700177
Andy Bavier5ad87c92020-05-11 16:31:35 -0700178 stage('Dataplane Tests') {
179 environment {
180 ROBOT_CONFIG_FILE="$WORKSPACE/${configBaseDir}/${configDeploymentDir}/${configFileName}.yaml"
181 ROBOT_FILE="Voltha_PODTests.robot"
182 ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/DataplaneTests"
183 }
184 steps {
185 sh """
186 mkdir -p $ROBOT_LOGS_DIR
Andrea Campanella0d3110c2021-01-20 12:25:45 +0100187 export ROBOT_MISC_ARGS="--removekeywords wuks -i dataplane -e bbsim -e notready -d $ROBOT_LOGS_DIR -v POD_NAME:${configFileName} -v KUBERNETES_CONFIGS_DIR:$WORKSPACE/${configBaseDir}/${configKubernetesDir} -v container_log_dir:$WORKSPACE -v OLT_ADAPTER_APP_LABEL:${oltAdapterAppLabel}"
Andrea Campanella79ced322021-11-03 17:24:26 +0000188 ROBOT_MISC_ARGS+=" -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace}"
Suchitra Vemurie3bb90d2020-09-15 17:15:50 -0700189 make -C $WORKSPACE/voltha-system-tests voltha-test || true
Andy Bavier5ad87c92020-05-11 16:31:35 -0700190 """
191 }
192 }
Suchitra Vemuri5e8e7892020-09-14 16:04:12 -0700193 stage('HA Tests') {
194 environment {
195 ROBOT_CONFIG_FILE="$WORKSPACE/${configBaseDir}/${configDeploymentDir}/${configFileName}.yaml"
196 ROBOT_FILE="Voltha_ONOSHATests.robot"
197 ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/ONOSHAScenarios"
198 }
199 steps {
200 sh """
201 mkdir -p $ROBOT_LOGS_DIR
Andrea Campanella0d3110c2021-01-20 12:25:45 +0100202 export ROBOT_MISC_ARGS="--removekeywords wuks -L TRACE -e bbsim -e notready -d $ROBOT_LOGS_DIR -v POD_NAME:${configFileName} -v workflow:${params.workFlow} -v KUBERNETES_CONFIGS_DIR:$WORKSPACE/${configBaseDir}/${configKubernetesDir} -v container_log_dir:$WORKSPACE -v OLT_ADAPTER_APP_LABEL:${oltAdapterAppLabel}"
Andrea Campanella79ced322021-11-03 17:24:26 +0000203 ROBOT_MISC_ARGS+=" -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace}"
Suchitra Vemurie3bb90d2020-09-15 17:15:50 -0700204 make -C $WORKSPACE/voltha-system-tests voltha-test || true
Suchitra Vemuri5e8e7892020-09-14 16:04:12 -0700205 """
206 }
207 }
Andy Bavier5ad87c92020-05-11 16:31:35 -0700208
Suchitra Vemuri5dae5392020-01-30 20:10:18 -0800209 stage('Error Scenario Tests') {
Suchitra Vemuri489c2ae2020-01-23 16:27:39 -0800210 environment {
211 ROBOT_CONFIG_FILE="$WORKSPACE/${configBaseDir}/${configDeploymentDir}/${configFileName}.yaml"
Suchitra Vemuri5dae5392020-01-30 20:10:18 -0800212 ROBOT_FILE="Voltha_ErrorScenarios.robot"
213 ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/ErrorScenarios"
Suchitra Vemuri489c2ae2020-01-23 16:27:39 -0800214 }
215 steps {
216 sh """
217 mkdir -p $ROBOT_LOGS_DIR
Andrea Campanella0d3110c2021-01-20 12:25:45 +0100218 export ROBOT_MISC_ARGS="--removekeywords wuks -L TRACE -i functional -e bbsim -e notready -d $ROBOT_LOGS_DIR -v POD_NAME:${configFileName} -v KUBERNETES_CONFIGS_DIR:$WORKSPACE/${configBaseDir}/${configKubernetesDir} -v container_log_dir:$WORKSPACE -v OLT_ADAPTER_APP_LABEL:${oltAdapterAppLabel}"
Andrea Campanella79ced322021-11-03 17:24:26 +0000219 ROBOT_MISC_ARGS+=" -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace}"
Suchitra Vemurie3bb90d2020-09-15 17:15:50 -0700220 make -C $WORKSPACE/voltha-system-tests voltha-test || true
Suchitra Vemuri489c2ae2020-01-23 16:27:39 -0800221 """
222 }
223 }
224 }
Andy Bavier358aa0b2019-11-22 11:55:39 -0700225 post {
226 always {
Matteo Scandolo5e7bd1d2021-07-16 13:29:42 -0700227 getPodsInfo("$WORKSPACE/pods")
Andy Bavier4af02722020-01-15 10:24:24 -0700228 sh returnStdout: false, script: '''
Andy Bavier358aa0b2019-11-22 11:55:39 -0700229 set +e
Andy Bavier4af02722020-01-15 10:24:24 -0700230
Matteo Scandolo5e7bd1d2021-07-16 13:29:42 -0700231 # collect logs collected in the Robot Framework StartLogging keyword
Andy Bavier3c1ccca2020-02-12 16:50:04 -0700232 cd $WORKSPACE
Andy Bavierb86ca422020-02-13 05:19:43 -0700233 gzip *-combined.log || true
Andrea Campanellac9c80682020-09-30 12:02:01 +0200234 rm *-combined.log || true
Matteo Scandolo518c9e12020-07-21 11:32:23 -0700235
Andrea Campanellaa20a9332021-03-09 09:18:42 +0100236 # store information on the running pods
Andrea Campanellae0c839f2021-03-16 17:53:40 +0100237 kubectl get pods --all-namespaces -o wide > $WORKSPACE/pods.txt || true
Andrea Campanellaa20a9332021-03-09 09:18:42 +0100238 kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.image}{'\\n'}" | sort | uniq | tee $WORKSPACE/pod-images.txt || true
239 kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.imageID}{'\\n'}" | sort | uniq | tee $WORKSPACE/pod-imagesId.txt || true
240
241
Matteo Scandolo518c9e12020-07-21 11:32:23 -0700242 # collect ETCD cluster logs
243 mkdir -p $WORKSPACE/etcd
Girish Gowdra3eb96f62020-10-15 12:48:34 -0700244 printf '%s\n' $(kubectl get pods -l app=etcd -o=jsonpath="{.items[*]['metadata.name']}") | xargs -I% bash -c "kubectl logs % > $WORKSPACE/etcd/%.log"
Andy Bavier4af02722020-01-15 10:24:24 -0700245 '''
Andy Bavier358aa0b2019-11-22 11:55:39 -0700246 script {
247 deployment_config.olts.each { olt ->
Andrea Campanella0d3110c2021-01-20 12:25:45 +0100248 if (olt.type == null || olt.type == "" || olt.type == "openolt") {
249 sh returnStdout: false, script: """
250 sshpass -p ${olt.pass} scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ${olt.user}@${olt.sship}:/var/log/openolt.log $WORKSPACE/openolt-${olt.sship}.log || true
251 sed -i 's/\\x1b\\[[0-9;]*[a-zA-Z]//g' $WORKSPACE/openolt-${olt.sship}.log # Remove escape sequences
252 sshpass -p ${olt.pass} scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ${olt.user}@${olt.sship}:/var/log/dev_mgmt_daemon.log $WORKSPACE/dev_mgmt_daemon-${olt.sship}.log || true
253 sed -i 's/\\x1b\\[[0-9;]*[a-zA-Z]//g' $WORKSPACE/dev_mgmt_daemon-${olt.sship}.log # Remove escape sequences
254 sshpass -p ${olt.pass} scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ${olt.user}@${olt.sship}:/var/log/startup.log $WORKSPACE/startup-${olt.sship}.log || true
255 sed -i 's/\\x1b\\[[0-9;]*[a-zA-Z]//g' $WORKSPACE/startup-${olt.sship}.log || true # Remove escape sequences
256 sshpass -p ${olt.pass} scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ${olt.user}@${olt.sship}:/var/log/openolt_process_watchdog.log $WORKSPACE/openolt_process_watchdog-${olt.sship}.log || true
257 sed -i 's/\\x1b\\[[0-9;]*[a-zA-Z]//g' $WORKSPACE/openolt_process_watchdog-${olt.sship}.log || true # Remove escape sequences
258 """
259 }
Andy Bavier358aa0b2019-11-22 11:55:39 -0700260 }
261 }
262 step([$class: 'RobotPublisher',
263 disableArchiveOutput: false,
Andy Bavierfed57db2020-01-22 16:28:50 -0700264 logFileName: '**/log*.html',
Andy Bavier358aa0b2019-11-22 11:55:39 -0700265 otherFiles: '',
Andy Bavierfed57db2020-01-22 16:28:50 -0700266 outputFileName: '**/output*.xml',
267 outputPath: 'RobotLogs',
Andy Bavier358aa0b2019-11-22 11:55:39 -0700268 passThreshold: 100,
Andy Bavierfed57db2020-01-22 16:28:50 -0700269 reportFileName: '**/report*.html',
Andrea Campanellaabc09772021-06-16 12:08:57 +0200270 unstableThreshold: 0,
271 onlyCritical: true
Andy Bavier358aa0b2019-11-22 11:55:39 -0700272 ]);
Matteo Scandolo5e7bd1d2021-07-16 13:29:42 -0700273 archiveArtifacts artifacts: '**/*.log,**/*.gz,**/*.tgz,*.txt,pods/*.txt'
Andy Bavier358aa0b2019-11-22 11:55:39 -0700274 }
Andy Bavier358aa0b2019-11-22 11:55:39 -0700275 }
276}
Joey Armstrong56fdfec2024-03-01 13:43:36 -0500277
278// [EOF]