blob: 8726f753875aeca02660015ffaacd58d9a1ec3ed [file] [log] [blame]
Hardik Windlassec9341b2021-06-07 11:58:29 +00001// Copyright 2021-present Open Networking Foundation
Matteo Scandolo42f6e572021-01-25 15:11:34 -08002//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
Hardik Windlassec9341b2021-06-07 11:58:29 +000015// voltha-2.x e2e tests for openonu-go
Matteo Scandolo42f6e572021-01-25 15:11:34 -080016// uses bbsim to simulate OLT/ONUs
17
Matteo Scandoloa156b572021-02-04 11:52:18 -080018library identifier: 'cord-jenkins-libraries@master',
19 retriever: modernSCM([
20 $class: 'GitSCMSource',
21 remote: 'https://gerrit.opencord.org/ci-management.git'
22])
23
Hardik Windlassec9341b2021-06-07 11:58:29 +000024def clusterName = "kind-ci"
25
26def execute_test(testTarget, workflow, teardown, testSpecificHelmFlags = "") {
27 def infraNamespace = "default"
28 def volthaNamespace = "voltha"
29 def robotLogsDir = "RobotLogs"
30 stage('Cleanup') {
31 if (teardown) {
32 timeout(15) {
33 script {
34 helmTeardown(["default", infraNamespace, volthaNamespace])
35 }
36 timeout(1) {
37 sh returnStdout: false, script: '''
38 # remove orphaned port-forward from different namespaces
39 ps aux | grep port-forw | grep -v grep | awk '{print $2}' | xargs --no-run-if-empty kill -9 || true
40 '''
41 }
42 }
43 }
44 }
45 stage('Deploy Voltha') {
46 if (teardown) {
Matteo Scandoloea6348e2021-06-10 18:44:58 +020047 timeout(10) {
Hardik Windlassec9341b2021-06-07 11:58:29 +000048 script {
49
50 sh """
51 mkdir -p $WORKSPACE/${testTarget}-components
Matteo Scandolof2460e8ff2021-06-21 17:53:20 +020052 _TAG=kail-startup kail -n ${infraNamespace} -n ${volthaNamespace} > $WORKSPACE/${testTarget}-components/onos-voltha-startup-combined.log &
Hardik Windlassec9341b2021-06-07 11:58:29 +000053 """
54
55 // if we're downloading a voltha-helm-charts patch, then install from a local copy of the charts
56 def localCharts = false
Matteo Scandoloea6348e2021-06-10 18:44:58 +020057 if (volthaHelmChartsChange != "" || gerritProject == "voltha-helm-charts") {
Hardik Windlassec9341b2021-06-07 11:58:29 +000058 localCharts = true
59 }
60
61 // NOTE temporary workaround expose ONOS node ports
62 def localHelmFlags = extraHelmFlags + " --set global.log_level=${logLevel.toUpperCase()} " +
Matteo Scandolo9114c892021-05-20 11:41:55 -070063 " --set onos-classic.onosSshPort=30115 " +
Hardik Windlassec9341b2021-06-07 11:58:29 +000064 " --set onos-classic.onosApiPort=30120 " +
65 " --set onos-classic.onosOfPort=31653 " +
66 " --set onos-classic.individualOpenFlowNodePorts=true " + testSpecificHelmFlags
Matteo Scandolo42f6e572021-01-25 15:11:34 -080067
Hardik Windlassec9341b2021-06-07 11:58:29 +000068 if (gerritProject != "") {
69 localHelmFlags = "${localHelmFlags} " + getVolthaImageFlags("${gerritProject}")
70 }
71
72 volthaDeploy([
73 infraNamespace: infraNamespace,
74 volthaNamespace: volthaNamespace,
75 workflow: workflow.toLowerCase(),
76 extraHelmFlags: localHelmFlags,
77 localCharts: localCharts,
78 bbsimReplica: olts.toInteger(),
79 dockerRegistry: registry,
80 ])
Matteo Scandolod17de3a2021-04-09 15:47:52 -070081 }
Matteo Scandolofcfc60d2021-02-24 09:06:48 -080082
Hardik Windlassec9341b2021-06-07 11:58:29 +000083 // stop logging
Matteo Scandolod17de3a2021-04-09 15:47:52 -070084 sh """
Hardik Windlassec9341b2021-06-07 11:58:29 +000085 P_IDS="\$(ps e -ww -A | grep "_TAG=kail-startup" | grep -v grep | awk '{print \$1}')"
86 if [ -n "\$P_IDS" ]; then
87 echo \$P_IDS
88 for P_ID in \$P_IDS; do
89 kill -9 \$P_ID
90 done
91 fi
92 cd $WORKSPACE/${testTarget}-components/
93 gzip -k onos-voltha-startup-combined.log
94 rm onos-voltha-startup-combined.log
Matteo Scandolod17de3a2021-04-09 15:47:52 -070095 """
Hardik Windlassec9341b2021-06-07 11:58:29 +000096 }
97 sh """
98 JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-voltha-api" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${volthaNamespace} svc/voltha-voltha-api 55555:55555; done"&
99 JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-infra-etcd" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${infraNamespace} svc/voltha-infra-etcd 2379:2379; done"&
100 JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-infra-kafka" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${infraNamespace} svc/voltha-infra-kafka 9092:9092; done"&
101 bbsimDmiPortFwd=50075
102 for i in {0..${olts.toInteger() - 1}}; do
103 JENKINS_NODE_COOKIE="dontKillMe" _TAG="bbsim\${i}" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${volthaNamespace} svc/bbsim\${i} \${bbsimDmiPortFwd}:50075; done"&
104 ((bbsimDmiPortFwd++))
105 done
106 ps aux | grep port-forward
107 """
Matteo Scandolod17de3a2021-04-09 15:47:52 -0700108 }
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800109 }
Hardik Windlassec9341b2021-06-07 11:58:29 +0000110 stage('Run test ' + testTarget + ' on ' + workflow + ' workFlow') {
Hardik Windlass72947302021-06-14 10:36:57 +0000111 // start logging
112 sh """
113 mkdir -p $WORKSPACE/${testTarget}-components
114 _TAG=kail-${workflow} kail -n infra -n voltha > $WORKSPACE/${testTarget}-components/onos-voltha-combined.log &
115 """
116 sh """
117 mkdir -p $WORKSPACE/${robotLogsDir}/${testTarget}-robot
118 export ROBOT_MISC_ARGS="-d $WORKSPACE/${robotLogsDir}/${testTarget}-robot "
119 ROBOT_MISC_ARGS+="-v ONOS_SSH_PORT:30115 -v ONOS_REST_PORT:30120 -v INFRA_NAMESPACE:${infraNamespace}"
120 export KVSTOREPREFIX=voltha/voltha_voltha
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800121
Hardik Windlass72947302021-06-14 10:36:57 +0000122 make -C $WORKSPACE/voltha-system-tests ${testTarget} || true
123 """
124 // stop logging
125 sh """
126 P_IDS="\$(ps e -ww -A | grep "_TAG=kail-${workflow}" | grep -v grep | awk '{print \$1}')"
127 if [ -n "\$P_IDS" ]; then
128 echo \$P_IDS
129 for P_ID in \$P_IDS; do
130 kill -9 \$P_ID
131 done
132 fi
133 cd $WORKSPACE/${testTarget}-components/
Hardik Windlass3853dc32021-06-15 15:35:47 +0000134 rm onos-voltha-combined.log.gz || true
Hardik Windlass72947302021-06-14 10:36:57 +0000135 gzip -k onos-voltha-combined.log
136 rm onos-voltha-combined.log
137 """
138 getPodsInfo("$WORKSPACE/${testTarget}-components")
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800139 }
140}
141
Hardik Windlassec9341b2021-06-07 11:58:29 +0000142def collectArtifacts(exitStatus) {
143 getPodsInfo("$WORKSPACE/${exitStatus}")
144 sh """
145 kubectl logs -n voltha -l app.kubernetes.io/part-of=voltha > $WORKSPACE/${exitStatus}/voltha.log || true
146 """
147 archiveArtifacts artifacts: '**/*.log,**/*.gz,**/*.txt,**/*.html'
148 sh '''
149 sync
150 pkill kail || true
151 which voltctl
152 md5sum $(which voltctl)
153 '''
154 step([$class: 'RobotPublisher',
155 disableArchiveOutput: false,
156 logFileName: "RobotLogs/*/log*.html",
157 otherFiles: '',
158 outputFileName: "RobotLogs/*/output*.xml",
159 outputPath: '.',
160 passThreshold: 100,
161 reportFileName: "RobotLogs/*/report*.html",
Andrea Campanellaabc09772021-06-16 12:08:57 +0200162 unstableThreshold: 0,
163 onlyCritical: true]);
Hardik Windlassec9341b2021-06-07 11:58:29 +0000164}
165
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800166pipeline {
167
168 /* no label, executor is determined by JJB */
169 agent {
170 label "${params.buildNode}"
171 }
172 options {
Hardik Windlassec9341b2021-06-07 11:58:29 +0000173 timeout(time: "${timeout}", unit: 'MINUTES')
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800174 }
175 environment {
Hardik Windlassec9341b2021-06-07 11:58:29 +0000176 KUBECONFIG="$HOME/.kube/kind-${clusterName}"
177 VOLTCONFIG="$HOME/.volt/config"
178 PATH="$PATH:$WORKSPACE/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
179 ROBOT_MISC_ARGS="-e PowerSwitch ${params.extraRobotArgs}"
180 DIAGS_PROFILE="VOLTHA_PROFILE"
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800181 }
Hardik Windlassec9341b2021-06-07 11:58:29 +0000182 stages {
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800183 stage('Download Code') {
184 steps {
185 getVolthaCode([
186 branch: "${branch}",
187 gerritProject: "${gerritProject}",
188 gerritRefspec: "${gerritRefspec}",
189 volthaSystemTestsChange: "${volthaSystemTestsChange}",
190 volthaHelmChartsChange: "${volthaHelmChartsChange}",
191 ])
192 }
193 }
194 stage('Build patch') {
Hardik Windlassec9341b2021-06-07 11:58:29 +0000195 // build the patch only if gerritProject is specified
196 when {
197 expression {
198 return !gerritProject.isEmpty()
199 }
200 }
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800201 steps {
202 // NOTE that the correct patch has already been checked out
203 // during the getVolthaCode step
204 buildVolthaComponent("${gerritProject}")
205 }
206 }
207 stage('Create K8s Cluster') {
208 steps {
Hardik Windlassec9341b2021-06-07 11:58:29 +0000209 script {
210 def clusterExists = sh returnStdout: true, script: """
211 kind get clusters | grep ${clusterName} | wc -l
212 """
213 if (clusterExists.trim() == "0") {
214 createKubernetesCluster([nodes: 3, name: clusterName])
215 }
216 }
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800217 }
218 }
Hardik Windlassc6ba23e2021-06-08 08:19:52 +0000219 stage('Replace voltctl') {
220 // if the project is voltctl override the downloaded one with the built one
221 when {
222 expression {
223 return gerritProject == "voltctl"
224 }
225 }
226 steps{
227 sh """
228 mv `ls $WORKSPACE/voltctl/release/voltctl-*-linux-amd*` $WORKSPACE/bin/voltctl
229 chmod +x $WORKSPACE/bin/voltctl
230 """
231 }
232 }
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800233 stage('Load image in kind nodes') {
Hardik Windlassec9341b2021-06-07 11:58:29 +0000234 when {
235 expression {
236 return !gerritProject.isEmpty()
237 }
238 }
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800239 steps {
240 loadToKind()
241 }
242 }
Hardik Windlassec9341b2021-06-07 11:58:29 +0000243 stage('Parse and execute tests') {
244 steps {
245 script {
246 def tests = readYaml text: testTargets
247
248 for(int i = 0;i<tests.size();i++) {
249 def test = tests[i]
250 def target = test["target"]
251 def workflow = test["workflow"]
252 def flags = test["flags"]
253 def teardown = test["teardown"].toBoolean()
254 println "Executing test ${target} on workflow ${workflow} with extra flags ${flags}"
255 execute_test(target, workflow, teardown, flags)
256 }
257 }
Matteo Scandolo29597f02021-02-12 10:53:57 -0800258 }
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800259 }
260 }
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800261 post {
Matteo Scandoloe543c4c2021-03-09 07:34:29 -0800262 aborted {
Hardik Windlassec9341b2021-06-07 11:58:29 +0000263 collectArtifacts("aborted")
Matteo Scandoloe543c4c2021-03-09 07:34:29 -0800264 }
265 failure {
Hardik Windlassec9341b2021-06-07 11:58:29 +0000266 collectArtifacts("failed")
Matteo Scandoloe543c4c2021-03-09 07:34:29 -0800267 }
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800268 always {
Hardik Windlassec9341b2021-06-07 11:58:29 +0000269 collectArtifacts("always")
Matteo Scandolo42f6e572021-01-25 15:11:34 -0800270 }
271 }
272}