blob: 5a14eaba5edfed8e68e7ada464d8e818c07def8c [file] [log] [blame]
Joey Armstrong980e37f2023-02-28 18:57:41 -05001#!/usr/bin/env groovy
2// -----------------------------------------------------------------------
3// Copyright 2017-2023 Open Networking Foundation (ONF) and the ONF Contributors
4//
5// Licensed under the Apache License, Version 2.0 (the "License");
6// you may not use this file except in compliance with the License.
7// You may obtain a copy of the License at
8//
9// http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing, software
12// distributed under the License is distributed on an "AS IS" BASIS,
13// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14// See the License for the specific language governing permissions and
15// limitations under the License.
16// -----------------------------------------------------------------------
17
18library identifier: 'cord-jenkins-libraries@master',
19 retriever: modernSCM([
20 $class: 'GitSCMSource',
21 remote: 'https://gerrit.opencord.org/ci-management.git'
22])
23
24node {
25 // Need this so that deployment_config has global scope when it's read later
26 deployment_config = null
27}
28
29def infraNamespace = "infra"
30def volthaNamespace = "voltha"
31
32pipeline {
33 /* no label, executor is determined by JJB */
34 agent {
35 label "${params.buildNode}"
36 }
37 options {
38 timeout(time: "${timeout}", unit: 'MINUTES')
39 }
40
41 environment {
42 KUBECONFIG="$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf"
43 VOLTCONFIG="$HOME/.volt/config-minimal"
44 PATH="$WORKSPACE/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
45 }
46
47 stages {
48 stage('Clone voltha-system-tests') {
49 steps {
50 step([$class: 'WsCleanup'])
51 checkout([
52 $class: 'GitSCM',
53 userRemoteConfigs: [[
54 url: "https://gerrit.opencord.org/voltha-system-tests",
55 refspec: "${volthaSystemTestsChange}"
56 ]],
57 branches: [[ name: "${branch}", ]],
58 extensions: [
59 [$class: 'WipeWorkspace'],
60 [$class: 'RelativeTargetDirectory', relativeTargetDir: "voltha-system-tests"],
61 [$class: 'CloneOption', depth: 0, noTags: false, reference: '', shallow: false],
62 ],
63 ])
64 script {
65 sh(script:"""
66 if [ '${volthaSystemTestsChange}' != '' ] ; then
67 cd $WORKSPACE/voltha-system-tests;
68 git fetch https://gerrit.opencord.org/voltha-system-tests ${volthaSystemTestsChange} && git checkout FETCH_HEAD
69 fi
70 """)
71 }
72 }
73 }
74 // This checkout allows us to show changes in Jenkins
75 // we only do this on master as we don't branch all the repos for all the releases
76 // (we should compute the difference by tracking the container version, not the code)
77 stage('Download All the VOLTHA repos') {
78 when {
79 expression {
80 return "${branch}" == 'master';
81 }
82 }
83 steps {
84 checkout(changelog: true,
85 poll: false,
86 scm: [$class: 'RepoScm',
87 manifestRepositoryUrl: "${params.manifestUrl}",
88 manifestBranch: "${params.branch}",
89 currentBranch: true,
90 destinationDir: 'voltha',
91 forceSync: true,
92 resetFirst: true,
93 quiet: true,
94 jobs: 4,
95 showAllChanges: true]
96 )
97 }
98 }
99 stage ('Initialize') {
100 steps {
101 sh returnStdout: false, script: "git clone -b ${branch} ${cordRepoUrl}/${configBaseDir}"
102 script {
103 deployment_config = readYaml file: "${configBaseDir}/${configDeploymentDir}/${configFileName}-DT.yaml"
104 }
105 installVoltctl("${branch}")
106 sh """
107 ps -ef | grep port-forward
108 """
109
110 sh returnStdout: false, script: '''
111 # remove orphaned port-forward from different namespaces
112 ps aux | grep port-forw | grep -v grep | awk '{print $2}' | xargs --no-run-if-empty kill -9 || true
113 '''
114 sh """
115 JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-api" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${volthaNamespace} svc/voltha-voltha-api 55555:55555; done"&
116 JENKINS_NODE_COOKIE="dontKillMe" _TAG="etcd" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${infraNamespace} svc/voltha-infra-etcd ${params.VolthaEtcdPort}:2379; done"&
117 JENKINS_NODE_COOKIE="dontKillMe" _TAG="kafka" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${infraNamespace} svc/voltha-infra-kafka 9092:9092; done"&
118 ps aux | grep port-forward
119 """
120
121 sh("""ps -ef | grep port-forward""")
122
123 sh(returnStdout: false, script: """
124 mkdir -p "$WORKSPACE/bin"
125
126 # install kail
127 make -C "$WORKSPACE/voltha-system-tests" KAIL_PATH="$WORKSPACE/bin" kail
128
129 # Default kind-voltha config doesn't work on ONF demo pod for accessing kvstore.
130 # The issue is that the mgmt node is also one of the k8s nodes and so port forwarding doesn't work.
131 # We should change this. In the meantime here is a workaround.
132 set +e
133
134 # Remove noise from voltha-core logs
135 voltctl log level set WARN read-write-core#github.com/opencord/voltha-go/db/model
136 voltctl log level set WARN read-write-core#github.com/opencord/voltha-lib-go/v3/pkg/kafka
137 # Remove noise from openolt logs
138 voltctl log level set WARN adapter-open-olt#github.com/opencord/voltha-lib-go/v3/pkg/db
139 voltctl log level set WARN adapter-open-olt#github.com/opencord/voltha-lib-go/v3/pkg/probe
140 voltctl log level set WARN adapter-open-olt#github.com/opencord/voltha-lib-go/v3/pkg/kafka
141 """)
142 }
143 }
144
145 stage('Functional Tests') {
146 environment {
147 ROBOT_CONFIG_FILE="$WORKSPACE/${configBaseDir}/${configDeploymentDir}/${configFileName}-DT.yaml"
148 ROBOT_FILE="Voltha_DT_PODTests.robot"
149 ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/dt-workflow/FunctionalTests"
150 }
151 steps {
152 sh """
153 ps -ef | grep port-forward
154 mkdir -p $ROBOT_LOGS_DIR
155 if [ ${params.withFttb} = false ]; then
156 if ( ${powerSwitch} ); then
157 export ROBOT_MISC_ARGS="--removekeywords wuks -i PowerSwitch -i sanityDt -i functionalDt -e bbsim -e notready -d $ROBOT_LOGS_DIR -v POD_NAME:${configFileName} -v KUBERNETES_CONFIGS_DIR:$WORKSPACE/${configBaseDir}/${configKubernetesDir} -v container_log_dir:$WORKSPACE -v OLT_ADAPTER_APP_LABEL:${oltAdapterAppLabel}"
158 if ( ${powerCycleOlt} ); then
159 ROBOT_MISC_ARGS+=" -v power_cycle_olt:True"
160 fi
161 else
162 export ROBOT_MISC_ARGS="--removekeywords wuks -e PowerSwitch -i sanityDt -i functionalDt -e bbsim -e notready -d $ROBOT_LOGS_DIR -v POD_NAME:${configFileName} -v KUBERNETES_CONFIGS_DIR:$WORKSPACE/${configBaseDir}/${configKubernetesDir} -v container_log_dir:$WORKSPACE -v OLT_ADAPTER_APP_LABEL:${oltAdapterAppLabel}"
163 fi
164 ROBOT_MISC_ARGS+=" -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace}"
165 make -C $WORKSPACE/voltha-system-tests voltha-dt-test || true
166 fi
167 """
168 }
169 }
170
171 stage('FTTB Functional Tests') {
172 environment {
173 ROBOT_CONFIG_FILE="$WORKSPACE/${configBaseDir}/${configDeploymentDir}/${configFileName}-DT.yaml"
174 ROBOT_FILE="Voltha_DT_FTTB_Tests.robot"
175 ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/dt-workflow/FunctionalTests"
176 }
177 steps {
178 sh """
179 mkdir -p $ROBOT_LOGS_DIR
180 if [ ${params.withFttb} = true ]; then
181 export ROBOT_MISC_ARGS="--removekeywords wuks -L TRACE -i sanityDtFttb -e bbsim -e notready -d $ROBOT_LOGS_DIR -v POD_NAME:${configFileName} -v KUBERNETES_CONFIGS_DIR:$WORKSPACE/${configBaseDir}/${configKubernetesDir} -v container_log_dir:$WORKSPACE"
182 ROBOT_MISC_ARGS+=" -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace} -v has_dataplane:False"
183 make -C $WORKSPACE/voltha-system-tests voltha-dt-test || true
184 fi
185 """
186 }
187 }
188
189 stage('Failure/Recovery Tests') {
190 environment {
191 ROBOT_CONFIG_FILE="$WORKSPACE/${configBaseDir}/${configDeploymentDir}/${configFileName}-DT.yaml"
192 ROBOT_FILE="Voltha_DT_FailureScenarios.robot"
193 ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/dt-workflow/FailureScenarios"
194 }
195 steps {
196 sh """
197 ps -ef | grep port-forward
198 mkdir -p $ROBOT_LOGS_DIR
199 if [ ${params.withFttb} = false ]; then
200 if ( ${powerSwitch} ); then
201 export ROBOT_MISC_ARGS="--removekeywords wuks -L TRACE -i functionalDt -i PowerSwitch -e bbsim -e notready -d $ROBOT_LOGS_DIR -v POD_NAME:${configFileName} -v KUBERNETES_CONFIGS_DIR:$WORKSPACE/${configBaseDir}/${configKubernetesDir} -v container_log_dir:$WORKSPACE -v OLT_ADAPTER_APP_LABEL:${oltAdapterAppLabel}"
202 else
203 export ROBOT_MISC_ARGS="--removekeywords wuks -L TRACE -i functionalDt -e PowerSwitch -e bbsim -e notready -d $ROBOT_LOGS_DIR -v POD_NAME:${configFileName} -v KUBERNETES_CONFIGS_DIR:$WORKSPACE/${configBaseDir}/${configKubernetesDir} -v container_log_dir:$WORKSPACE -v OLT_ADAPTER_APP_LABEL:${oltAdapterAppLabel}"
204 fi
205 ROBOT_MISC_ARGS+=" -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace}"
206 make -C $WORKSPACE/voltha-system-tests voltha-dt-test || true
207 fi
208 """
209 }
210 }
211
212 stage('Dataplane Tests') {
213 environment {
214 ROBOT_CONFIG_FILE="$WORKSPACE/${configBaseDir}/${configDeploymentDir}/${configFileName}-DT.yaml"
215 ROBOT_FILE="Voltha_DT_PODTests.robot"
216 ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/dt-workflow/DataplaneTests"
217 }
218 steps {
219 sh """
220 ps -ef | grep port-forward
221 mkdir -p $ROBOT_LOGS_DIR
222 if [ ${params.withFttb} = false ]; then
223 export ROBOT_MISC_ARGS="--removekeywords wuks -i dataplaneDt -e bbsim -e notready -d $ROBOT_LOGS_DIR -v POD_NAME:${configFileName} -v KUBERNETES_CONFIGS_DIR:$WORKSPACE/${configBaseDir}/${configKubernetesDir} -v container_log_dir:$WORKSPACE -v OLT_ADAPTER_APP_LABEL:${oltAdapterAppLabel}"
224 ROBOT_MISC_ARGS+=" -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace}"
225 make -C $WORKSPACE/voltha-system-tests voltha-dt-test || true
226 fi
227 """
228 }
229 }
230 stage('HA Tests') {
231 environment {
232 ROBOT_CONFIG_FILE="$WORKSPACE/${configBaseDir}/${configDeploymentDir}/${configFileName}-DT.yaml"
233 ROBOT_FILE="Voltha_ONOSHATests.robot"
234 ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/ONOSHAScenarios"
235 }
236 steps {
237 sh """
238 ps -ef | grep port-forward
239 mkdir -p $ROBOT_LOGS_DIR
240 if [ ${params.withFttb} = false ]; then
241 export ROBOT_MISC_ARGS="--removekeywords wuks -L TRACE -e bbsim -e notready -d $ROBOT_LOGS_DIR -v POD_NAME:${configFileName} -v workflow:${params.workFlow} -v KUBERNETES_CONFIGS_DIR:$WORKSPACE/${configBaseDir}/${configKubernetesDir} -v container_log_dir:$WORKSPACE -v OLT_ADAPTER_APP_LABEL:${oltAdapterAppLabel}"
242 ROBOT_MISC_ARGS+=" -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace}"
243 make -C $WORKSPACE/voltha-system-tests voltha-test || true
244 fi
245 """
246 }
247 }
248
249 stage('Multiple OLT Tests') {
250 environment {
251 ROBOT_CONFIG_FILE="$WORKSPACE/${configBaseDir}/${configDeploymentDir}/${configFileName}-DT.yaml"
252 ROBOT_FILE="Voltha_DT_MultiOLT_Tests.robot"
253 ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/dt-workflow/MultipleOLTScenarios"
254 }
255 steps {
256 sh """
257 ps -ef | grep port-forward
258 mkdir -p $ROBOT_LOGS_DIR
259 if [ ${params.withFttb} = false ]; then
260 if ( ${powerSwitch} ); then
261 export ROBOT_MISC_ARGS="--removekeywords wuks -L TRACE -i functionalDt -i PowerSwitch -e bbsim -e notready -d $ROBOT_LOGS_DIR -v POD_NAME:${configFileName} -v KUBERNETES_CONFIGS_DIR:$WORKSPACE/${configBaseDir}/${configKubernetesDir} -v container_log_dir:$WORKSPACE -v OLT_ADAPTER_APP_LABEL:${oltAdapterAppLabel}"
262 else
263 export ROBOT_MISC_ARGS="--removekeywords wuks -L TRACE -i functionalDt -e PowerSwitch -e bbsim -e notready -d $ROBOT_LOGS_DIR -v POD_NAME:${configFileName} -v KUBERNETES_CONFIGS_DIR:$WORKSPACE/${configBaseDir}/${configKubernetesDir} -v container_log_dir:$WORKSPACE -v OLT_ADAPTER_APP_LABEL:${oltAdapterAppLabel}"
264 fi
265 ROBOT_MISC_ARGS+=" -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace}"
266 make -C $WORKSPACE/voltha-system-tests voltha-dt-test || true
267 fi
268 """
269 }
270 }
271
272
273 stage('Error Scenario Tests') {
274 environment {
275 ROBOT_CONFIG_FILE="$WORKSPACE/${configBaseDir}/${configDeploymentDir}/${configFileName}-DT.yaml"
276 ROBOT_FILE="Voltha_ErrorScenarios.robot"
277 ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/dt-workflow/ErrorScenarios"
278 }
279 steps {
280 sh """
281 mkdir -p $ROBOT_LOGS_DIR
282 if [ ${params.withFttb} = false ]; then
283 export ROBOT_MISC_ARGS="--removekeywords wuks -L TRACE -i functional -e bbsim -e notready -d $ROBOT_LOGS_DIR -v POD_NAME:${configFileName} -v workflow:${params.workFlow} -v KUBERNETES_CONFIGS_DIR:$WORKSPACE/${configBaseDir}/${configKubernetesDir} -v container_log_dir:$WORKSPACE -v OLT_ADAPTER_APP_LABEL:${oltAdapterAppLabel}"
284 ROBOT_MISC_ARGS+=" -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace}"
285 make -C $WORKSPACE/voltha-system-tests voltha-test || true
286 fi
287 """
288 }
289 }
290 }
291 post {
292 always {
293 getPodsInfo("$WORKSPACE/pods")
294 sh returnStdout: false, script: '''
295 set +e
296
297 # collect logs collected in the Robot Framework StartLogging keyword
298 cd $WORKSPACE
299 gzip *-combined.log || true
300 rm *-combined.log || true
301 '''
302 script {
303 deployment_config.olts.each { olt ->
304 if (olt.type == null || olt.type == "" || olt.type == "openolt") {
305 sh returnStdout: false, script: """
306 sshpass -p ${olt.pass} scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ${olt.user}@${olt.sship}:/var/log/openolt.log $WORKSPACE/openolt-${olt.sship}.log || true
307 sed -i 's/\\x1b\\[[0-9;]*[a-zA-Z]//g' $WORKSPACE/openolt-${olt.sship}.log # Remove escape sequences
308 sshpass -p ${olt.pass} scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ${olt.user}@${olt.sship}:/var/log/dev_mgmt_daemon.log $WORKSPACE/dev_mgmt_daemon-${olt.sship}.log || true
309 sed -i 's/\\x1b\\[[0-9;]*[a-zA-Z]//g' $WORKSPACE/dev_mgmt_daemon-${olt.sship}.log # Remove escape sequences
310 sshpass -p ${olt.pass} scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ${olt.user}@${olt.sship}:/var/log/startup.log $WORKSPACE/startup-${olt.sship}.log || true
311 sed -i 's/\\x1b\\[[0-9;]*[a-zA-Z]//g' $WORKSPACE/startup-${olt.sship}.log || true # Remove escape sequences
312 sshpass -p ${olt.pass} scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ${olt.user}@${olt.sship}:/var/log/openolt_process_watchdog.log $WORKSPACE/openolt_process_watchdog-${olt.sship}.log || true
313 sed -i 's/\\x1b\\[[0-9;]*[a-zA-Z]//g' $WORKSPACE/openolt_process_watchdog-${olt.sship}.log || true # Remove escape sequences
314 """
315 }
316 }
317 }
318 step([$class: 'RobotPublisher',
319 disableArchiveOutput: false,
320 logFileName: '**/log*.html',
321 otherFiles: '',
322 outputFileName: '**/output*.xml',
323 outputPath: 'RobotLogs',
324 passThreshold: 100,
325 reportFileName: '**/report*.html',
326 unstableThreshold: 0,
327 onlyCritical: true
328 ]);
329 archiveArtifacts artifacts: '**/*.log,**/*.gz,**/*.tgz,*.txt,pods/*.txt'
330 }
331 }
332}