blob: 856514889287901ab950beae3f8573cd34be867b [file] [log] [blame]
Joey Armstrong7bcb5782023-06-07 12:25:57 -04001#!/usr/bin/env groovy
2// -----------------------------------------------------------------------
3// Copyright 2017-2023 Open Networking Foundation (ONF) and the ONF Contributors
4//
5// Licensed under the Apache License, Version 2.0 (the "License");
6// you may not use this file except in compliance with the License.
7// You may obtain a copy of the License at
8//
9// http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing, software
12// distributed under the License is distributed on an "AS IS" BASIS,
13// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14// See the License for the specific language governing permissions and
15// limitations under the License.
16// -----------------------------------------------------------------------
17
18library identifier: 'cord-jenkins-libraries@master',
19 retriever: modernSCM([
20 $class: 'GitSCMSource',
21 remote: 'https://gerrit.opencord.org/ci-management.git'
22])
23
24def infraNamespace = "infra"
25def volthaNamespace = "voltha"
26
27node {
28 // Need this so that deployment_config has global scope when it's read later
29 deployment_config = null
30}
31
32pipeline {
33 /* no label, executor is determined by JJB */
34 agent {
35 label "${params.buildNode}"
36 }
37 options {
38 timeout(time: "${timeout}", unit: 'MINUTES')
39 }
40
41 environment {
42 KUBECONFIG="$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf"
43 VOLTCONFIG="$HOME/.volt/config-minimal"
44 PATH="$WORKSPACE/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
45 }
46 stages {
47 stage('Clone voltha-system-tests') {
48 steps {
49 step([$class: 'WsCleanup'])
50 checkout([
51 $class: 'GitSCM',
52 userRemoteConfigs: [[
53 url: "https://gerrit.opencord.org/voltha-system-tests",
54 refspec: "${volthaSystemTestsChange}"
55 ]],
56 branches: [[ name: "${branch}", ]],
57 extensions: [
58 [$class: 'WipeWorkspace'],
59 [$class: 'RelativeTargetDirectory', relativeTargetDir: "voltha-system-tests"],
60 [$class: 'CloneOption', depth: 0, noTags: false, reference: '', shallow: false],
61 ],
62 ])
63 script {
64 sh(script:"""
65 if [ '${volthaSystemTestsChange}' != '' ] ; then
66 cd $WORKSPACE/voltha-system-tests;
67 git fetch https://gerrit.opencord.org/voltha-system-tests ${volthaSystemTestsChange} && git checkout FETCH_HEAD
68 fi
69 """)
70 }
71 }
72 }
73 stage('Download All the VOLTHA repos') {
74 when {
75 expression {
76 return "${branch}" == 'master';
77 }
78 }
79 steps {
80 checkout(changelog: true,
81 poll: false,
82 scm: [$class: 'RepoScm',
83 manifestRepositoryUrl: "${params.manifestUrl}",
84 manifestBranch: "${params.branch}",
85 currentBranch: true,
86 destinationDir: 'voltha',
87 forceSync: true,
88 resetFirst: true,
89 quiet: true,
90 jobs: 4,
91 showAllChanges: true]
92 )
93 }
94 }
95 stage ('Initialize') {
96 steps {
97 sh returnStdout: false, script: "git clone -b ${branch} ${cordRepoUrl}/${configBaseDir}"
98 script {
99 deployment_config = readYaml file: "${configBaseDir}/${configDeploymentDir}/${configFileName}.yaml"
100 }
101 installVoltctl("${branch}")
102
103 sh(returnStdout: false, script: """
104
105 mkdir -p "$WORKSPACE/bin"
106
107 # install kail
108 make -C "$WORKSPACE/voltha-system-tests" KAIL_PATH="$WORKSPACE/bin" kail
109
110 if [ "${params.branch}" == "master" ]; then
111 # Default kind-voltha config doesn't work on ONF demo pod for accessing kvstore.
112 # The issue is that the mgmt node is also one of the k8s nodes and so port forwarding doesn't work.
113 # We should change this. In the meantime here is a workaround.
114 set +e
115
116 # Remove noise from voltha-core logs
117 voltctl log level set WARN read-write-core#github.com/opencord/voltha-go/db/model
118 voltctl log level set WARN read-write-core#github.com/opencord/voltha-lib-go/v3/pkg/kafka
119 # Remove noise from openolt logs
120 voltctl log level set WARN adapter-open-olt#github.com/opencord/voltha-lib-go/v3/pkg/db
121 voltctl log level set WARN adapter-open-olt#github.com/opencord/voltha-lib-go/v3/pkg/probe
122 voltctl log level set WARN adapter-open-olt#github.com/opencord/voltha-lib-go/v3/pkg/kafka
123 fi
124 """)
125 }
126 }
127
128 stage('Functional Tests') {
129 environment {
130 ROBOT_CONFIG_FILE="$WORKSPACE/${configBaseDir}/${configDeploymentDir}/${configFileName}.yaml"
131 ROBOT_FILE="Voltha_PODTests.robot"
132 ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/FunctionalTests"
133 }
134 steps {
135 sh """
136 mkdir -p $ROBOT_LOGS_DIR
137 if ( ${powerSwitch} ); then
138 export ROBOT_MISC_ARGS="--removekeywords wuks -i PowerSwitch -i sanity -i functional -e bbsim -e notready -d $ROBOT_LOGS_DIR -v POD_NAME:${configFileName} -v KUBERNETES_CONFIGS_DIR:$WORKSPACE/${configBaseDir}/${configKubernetesDir} -v container_log_dir:$WORKSPACE -v OLT_ADAPTER_APP_LABEL:${oltAdapterAppLabel}"
139 if ( ${powerCycleOlt} ); then
140 ROBOT_MISC_ARGS+=" -v power_cycle_olt:True"
141 fi
142 else
143 export ROBOT_MISC_ARGS="--removekeywords wuks -e PowerSwitch -i sanity -i functional -e bbsim -e notready -d $ROBOT_LOGS_DIR -v POD_NAME:${configFileName} -v KUBERNETES_CONFIGS_DIR:$WORKSPACE/${configBaseDir}/${configKubernetesDir} -v container_log_dir:$WORKSPACE -v OLT_ADAPTER_APP_LABEL:${oltAdapterAppLabel}"
144 fi
145 ROBOT_MISC_ARGS+=" -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace}"
146 make -C $WORKSPACE/voltha-system-tests voltha-test || true
147 """
148 }
149 }
150
151 stage('Failure/Recovery Tests') {
152 environment {
153 ROBOT_CONFIG_FILE="$WORKSPACE/${configBaseDir}/${configDeploymentDir}/${configFileName}.yaml"
154 ROBOT_FILE="Voltha_FailureScenarios.robot"
155 ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/FailureScenarios"
156 }
157 steps {
158 sh """
159 mkdir -p $ROBOT_LOGS_DIR
160 if ( ${powerSwitch} ); then
161 export ROBOT_MISC_ARGS="--removekeywords wuks -L TRACE -i functional -i PowerSwitch -e bbsim -e notready -d $ROBOT_LOGS_DIR -v POD_NAME:${configFileName} -v KUBERNETES_CONFIGS_DIR:$WORKSPACE/${configBaseDir}/${configKubernetesDir} -v container_log_dir:$WORKSPACE -v OLT_ADAPTER_APP_LABEL:${oltAdapterAppLabel}"
162 else
163 export ROBOT_MISC_ARGS="--removekeywords wuks -L TRACE -i functional -e PowerSwitch -e bbsim -e notready -d $ROBOT_LOGS_DIR -v POD_NAME:${configFileName} -v KUBERNETES_CONFIGS_DIR:$WORKSPACE/${configBaseDir}/${configKubernetesDir} -v container_log_dir:$WORKSPACE -v OLT_ADAPTER_APP_LABEL:${oltAdapterAppLabel}"
164 fi
165 ROBOT_MISC_ARGS+=" -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace}"
166 make -C $WORKSPACE/voltha-system-tests voltha-test || true
167 """
168 }
169 }
170
171 stage('Dataplane Tests') {
172 environment {
173 ROBOT_CONFIG_FILE="$WORKSPACE/${configBaseDir}/${configDeploymentDir}/${configFileName}.yaml"
174 ROBOT_FILE="Voltha_PODTests.robot"
175 ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/DataplaneTests"
176 }
177 steps {
178 sh """
179 mkdir -p $ROBOT_LOGS_DIR
180 export ROBOT_MISC_ARGS="--removekeywords wuks -i dataplane -e bbsim -e notready -d $ROBOT_LOGS_DIR -v POD_NAME:${configFileName} -v KUBERNETES_CONFIGS_DIR:$WORKSPACE/${configBaseDir}/${configKubernetesDir} -v container_log_dir:$WORKSPACE -v OLT_ADAPTER_APP_LABEL:${oltAdapterAppLabel}"
181 ROBOT_MISC_ARGS+=" -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace}"
182 make -C $WORKSPACE/voltha-system-tests voltha-test || true
183 """
184 }
185 }
186 stage('HA Tests') {
187 environment {
188 ROBOT_CONFIG_FILE="$WORKSPACE/${configBaseDir}/${configDeploymentDir}/${configFileName}.yaml"
189 ROBOT_FILE="Voltha_ONOSHATests.robot"
190 ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/ONOSHAScenarios"
191 }
192 steps {
193 sh """
194 mkdir -p $ROBOT_LOGS_DIR
195 export ROBOT_MISC_ARGS="--removekeywords wuks -L TRACE -e bbsim -e notready -d $ROBOT_LOGS_DIR -v POD_NAME:${configFileName} -v workflow:${params.workFlow} -v KUBERNETES_CONFIGS_DIR:$WORKSPACE/${configBaseDir}/${configKubernetesDir} -v container_log_dir:$WORKSPACE -v OLT_ADAPTER_APP_LABEL:${oltAdapterAppLabel}"
196 ROBOT_MISC_ARGS+=" -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace}"
197 make -C $WORKSPACE/voltha-system-tests voltha-test || true
198 """
199 }
200 }
201
202 stage('Error Scenario Tests') {
203 environment {
204 ROBOT_CONFIG_FILE="$WORKSPACE/${configBaseDir}/${configDeploymentDir}/${configFileName}.yaml"
205 ROBOT_FILE="Voltha_ErrorScenarios.robot"
206 ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/ErrorScenarios"
207 }
208 steps {
209 sh """
210 mkdir -p $ROBOT_LOGS_DIR
211 export ROBOT_MISC_ARGS="--removekeywords wuks -L TRACE -i functional -e bbsim -e notready -d $ROBOT_LOGS_DIR -v POD_NAME:${configFileName} -v KUBERNETES_CONFIGS_DIR:$WORKSPACE/${configBaseDir}/${configKubernetesDir} -v container_log_dir:$WORKSPACE -v OLT_ADAPTER_APP_LABEL:${oltAdapterAppLabel}"
212 ROBOT_MISC_ARGS+=" -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace}"
213 make -C $WORKSPACE/voltha-system-tests voltha-test || true
214 """
215 }
216 }
217 }
218 post {
219 always {
220 getPodsInfo("$WORKSPACE/pods")
221 sh returnStdout: false, script: '''
222 set +e
223
224 # collect logs collected in the Robot Framework StartLogging keyword
225 cd $WORKSPACE
226 gzip *-combined.log || true
227 rm *-combined.log || true
228
229 # store information on the running pods
230 kubectl get pods --all-namespaces -o wide > $WORKSPACE/pods.txt || true
231 kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.image}{'\\n'}" | sort | uniq | tee $WORKSPACE/pod-images.txt || true
232 kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.imageID}{'\\n'}" | sort | uniq | tee $WORKSPACE/pod-imagesId.txt || true
233
234
235 # collect ETCD cluster logs
236 mkdir -p $WORKSPACE/etcd
237 printf '%s\n' $(kubectl get pods -l app=etcd -o=jsonpath="{.items[*]['metadata.name']}") | xargs -I% bash -c "kubectl logs % > $WORKSPACE/etcd/%.log"
238 '''
239 script {
240 deployment_config.olts.each { olt ->
241 if (olt.type == null || olt.type == "" || olt.type == "openolt") {
242 sh returnStdout: false, script: """
243 sshpass -p ${olt.pass} scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ${olt.user}@${olt.sship}:/var/log/openolt.log $WORKSPACE/openolt-${olt.sship}.log || true
244 sed -i 's/\\x1b\\[[0-9;]*[a-zA-Z]//g' $WORKSPACE/openolt-${olt.sship}.log # Remove escape sequences
245 sshpass -p ${olt.pass} scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ${olt.user}@${olt.sship}:/var/log/dev_mgmt_daemon.log $WORKSPACE/dev_mgmt_daemon-${olt.sship}.log || true
246 sed -i 's/\\x1b\\[[0-9;]*[a-zA-Z]//g' $WORKSPACE/dev_mgmt_daemon-${olt.sship}.log # Remove escape sequences
247 sshpass -p ${olt.pass} scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ${olt.user}@${olt.sship}:/var/log/startup.log $WORKSPACE/startup-${olt.sship}.log || true
248 sed -i 's/\\x1b\\[[0-9;]*[a-zA-Z]//g' $WORKSPACE/startup-${olt.sship}.log || true # Remove escape sequences
249 sshpass -p ${olt.pass} scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ${olt.user}@${olt.sship}:/var/log/openolt_process_watchdog.log $WORKSPACE/openolt_process_watchdog-${olt.sship}.log || true
250 sed -i 's/\\x1b\\[[0-9;]*[a-zA-Z]//g' $WORKSPACE/openolt_process_watchdog-${olt.sship}.log || true # Remove escape sequences
251 """
252 }
253 }
254 }
255 step([$class: 'RobotPublisher',
256 disableArchiveOutput: false,
257 logFileName: '**/log*.html',
258 otherFiles: '',
259 outputFileName: '**/output*.xml',
260 outputPath: 'RobotLogs',
261 passThreshold: 100,
262 reportFileName: '**/report*.html',
263 unstableThreshold: 0,
264 onlyCritical: true
265 ]);
266 archiveArtifacts artifacts: '**/*.log,**/*.gz,**/*.tgz,*.txt,pods/*.txt'
267 }
268 }
269}