blob: 5e34c488cf8588c8ce6640ddfabf62e88eca1f6a [file] [log] [blame]
Kailash66c389c2019-04-24 14:56:11 -07001// Copyright 2017-present Open Networking Foundation
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15// chart-api-test-helm.groovy
16// Checks functionality of the helm-chart, without overriding the version/tag used
17
Matteo Scandolo5c6a4802019-05-14 10:44:21 -070018def serviceName = "${gerritProject}"
19
Kailash66c389c2019-04-24 14:56:11 -070020pipeline {
21
22 /* no label, executor is determined by JJB */
23 agent {
Zack Williamsb3292082019-10-11 17:15:18 -070024 label "${params.buildNode}"
Kailash66c389c2019-04-24 14:56:11 -070025 }
26
27 stages {
28
29 stage('repo') {
30 steps {
31 checkout(changelog: false, \
32 poll: false,
33 scm: [$class: 'RepoScm', \
34 manifestRepositoryUrl: "${params.manifestUrl}", \
35 manifestBranch: "${params.manifestBranch}", \
36 currentBranch: true, \
37 destinationDir: 'cord', \
38 forceSync: true,
39 resetFirst: true, \
40 quiet: true, \
41 jobs: 4, \
42 showAllChanges: true] \
43 )
44 }
45 }
46
47 stage('patch') {
48 steps {
49 sh '''
50 pushd cord
51 PROJECT_PATH=\$(xmllint --xpath "string(//project[@name=\\\"${gerritProject}\\\"]/@path)" .repo/manifest.xml)
52 repo download "\$PROJECT_PATH" "${gerritChangeNumber}/${gerritPatchsetNumber}"
53 popd
54 '''
55 }
56 }
57
58 stage('minikube') {
59 steps {
60 /* see https://github.com/kubernetes/minikube/#linux-continuous-integration-without-vm-support */
61 sh '''
62 export MINIKUBE_WANTUPDATENOTIFICATION=false
63 export MINIKUBE_WANTREPORTERRORPROMPT=false
64 export CHANGE_MINIKUBE_NONE_USER=true
65 export MINIKUBE_HOME=$HOME
66 mkdir -p $HOME/.kube || true
67 touch $HOME/.kube/config
68 export KUBECONFIG=$HOME/.kube/config
69 sudo -E /usr/bin/minikube start --vm-driver=none
70 '''
71 script {
72 timeout(3) {
73 waitUntil {
74 sleep 5
75 def kc_ret = sh script: "kubectl get po", returnStatus: true
76 return (kc_ret == 0);
77 }
78 }
79 }
80 }
81 }
82
83 stage('helm') {
84 steps {
85 sh '''
86 helm init
87 sleep 60
88 helm repo add incubator https://kubernetes-charts-incubator.storage.googleapis.com/
Carmelo Cascone319b34c2020-01-14 16:48:07 -080089 helm repo add cord https://charts.opencord.org
Kailash66c389c2019-04-24 14:56:11 -070090 '''
91 }
92 }
93
94 stage('Install XOS w/Service') {
95 steps {
Kailash5f300b92019-05-21 14:04:19 -070096 script {
97 if (serviceName == "olt-service") {
98 serviceName = "volt"
99 }
100 else if (serviceName == "onos-service") {
101 serviceName = "onos"
102 }
103 else if (serviceName == "kubernetes-service") {
104 serviceName = "kubernetes"
105 }
106 }
Kailash66c389c2019-04-24 14:56:11 -0700107 sh """
108 #!/usr/bin/env bash
109 set -eu -o pipefail
110
111 pushd cord/helm-charts
112 helm dep update xos-core
113 helm install xos-core -n xos-core
114
Kailash66c389c2019-04-24 14:56:11 -0700115 helm-repo-tools/wait_for_pods.sh
116
117 #install service
Kailash5f300b92019-05-21 14:04:19 -0700118 helm dep update xos-services/${serviceName}
119 helm install xos-services/${serviceName} -n ${serviceName}
Kailash66c389c2019-04-24 14:56:11 -0700120 popd
121 """
122 }
123 }
124 stage('Verify') {
125 steps {
Matteo Scandolo5c6a4802019-05-14 10:44:21 -0700126 echo "serviceName: ${serviceName}"
Kailash66c389c2019-04-24 14:56:11 -0700127 sh """
128 #!/usr/bin/env bash
129 set -ex -o pipefail
130 export DOCKER_TAG=\$(cat $WORKSPACE/cord/orchestration/xos/VERSION)
131
132 #wait for xos-core and models to be loaded
133 timeout 300 bash -c "until http -a admin@opencord.org:letmein GET http://127.0.0.1:30001/xosapi/v1/dynamicload/load_status | jq '.services[] | select(.name==\\"core\\").state'| grep -q present; do echo 'Waiting for Core to be loaded'; sleep 5; done"
Matteo Scandolo5c6a4802019-05-14 10:44:21 -0700134 timeout 300 bash -c "until http -a admin@opencord.org:letmein GET http://127.0.0.1:30001/xosapi/v1/dynamicload/load_status | jq '.services[] | select(.name==\\"${serviceName}\\").state'| grep -q present; do echo 'Waiting for Core to be loaded'; sleep 5; done"
Kailash66c389c2019-04-24 14:56:11 -0700135
136 ## get pod logs
137 for pod in \$(kubectl get pods --no-headers | awk '{print \$1}');
138 do
Daniele Moroa5f00782020-02-24 11:23:46 -0800139 kubectl logs \${pod} > $WORKSPACE/\${pod}_pre.log;
Kailash66c389c2019-04-24 14:56:11 -0700140 done
141 """
142 }
143 }
Kailashf5c3f2b2019-04-30 08:58:49 -0700144 stage('Generate Model API Tests') {
145 steps {
146 sh """
147 CORE_CONTAINER=\$(docker ps | grep k8s_xos-core | awk '{print \$1}')
148 cd $WORKSPACE/cord/test/cord-tester
Kailashf5c3f2b2019-04-30 08:58:49 -0700149 docker cp $WORKSPACE/cord/test/cord-tester/src/test/cord-api/Tests/targets/xosserviceupgradetest.xtarget \$CORE_CONTAINER:/opt/xos/lib/xos-genx/xosgenx/targets/xosserviceupgradetest.xtarget
150 docker cp $WORKSPACE/cord/test/cord-tester/src/test/cord-api/Tests/targets/xosstaticlibrary.xtarget \$CORE_CONTAINER:/opt/xos/lib/xos-genx/xosgenx/targets/xosstaticlibrary.xtarget
151
152 export testname=_service_api.robot
153 export library=_library.robot
154
155 SERVICES=\$(docker exec -i \$CORE_CONTAINER /bin/bash -c "cd /opt/xos/dynamic_services/;find -name '*.xproto'" | awk -F[//] '{print \$2}')
156 echo \$SERVICES
157
158 for i in \$SERVICES; do bash -c "docker exec -i \$CORE_CONTAINER /bin/bash -c 'xosgenx --target /opt/xos/lib/xos-genx/xosgenx/targets/./xosserviceupgradetest.xtarget /opt/xos/dynamic_services/\$i/\$i.xproto /opt/xos/core/models/core.xproto'" > $WORKSPACE/cord/test/cord-tester/src/test/cord-api/Tests/\$i\$testname; done
159
160 for i in \$SERVICES; do bash -c "docker exec -i \$CORE_CONTAINER /bin/bash -c 'xosgenx --target /opt/xos/lib/xos-genx/xosgenx/targets/./xosstaticlibrary.xtarget /opt/xos/dynamic_services/\$i/\$i.xproto /opt/xos/core/models/core.xproto'" > $WORKSPACE/cord/test/cord-tester/src/test/cord-api/Tests/\$i\$library; done
161 ls -al $WORKSPACE/cord/test/cord-tester/src/test/cord-api/Tests/
162 """
163 }
164 }
165
166 stage('Test Pre-Upgrade') {
167 steps {
168 sh """
169 pushd cord/test/cord-tester/src/test/cord-api/Tests
170
171 CORE_CONTAINER=\$(docker ps | grep k8s_xos-core | awk '{print \$1}')
172 CHAM_CONTAINER=\$(docker ps | grep k8s_xos-chameleon | awk '{print \$1}')
173 XOS_CHAMELEON=\$(docker exec \$CHAM_CONTAINER ip a | grep -oE "([0-9]{1,3}\\.){3}[0-9]{1,3}\\b" | grep 172)
174
175 cd $WORKSPACE/cord/test/cord-tester/src/test/cord-api/Properties/
176 sed -i \"s/^\\(SERVER_IP = \\).*/\\1\'\$XOS_CHAMELEON\'/\" RestApiProperties.py
177 sed -i \"s/^\\(SERVER_PORT = \\).*/\\1\'9101\'/\" RestApiProperties.py
178 sed -i \"s/^\\(XOS_USER = \\).*/\\1\'admin@opencord.org\'/\" RestApiProperties.py
179 sed -i \"s/^\\(XOS_PASSWD = \\).*/\\1\'letmein\'/\" RestApiProperties.py
180 sed -i \"s/^\\(PASSWD = \\).*/\\1\'letmein\'/\" RestApiProperties.py
181
182 export testname=_service_api.robot
183 export library=_library.robot
184 SERVICES=\$(docker exec -i \$CORE_CONTAINER /bin/bash -c "cd /opt/xos/dynamic_services/;find -name '*.xproto'" | awk -F[//] '{print \$2}')
185 echo \$SERVICES
186
187 cd $WORKSPACE/cord/test/cord-tester/src/test/cord-api/Tests
Matteo Scandolo5c6a4802019-05-14 10:44:21 -0700188 for i in \$SERVICES; do bash -c "robot -v SETUP_FLAG:Setup -i create -d Log -T -v TESTLIBRARY:${serviceName}_library.robot \$i\$testname"; sleep 2; done || true
Kailashf5c3f2b2019-04-30 08:58:49 -0700189
190 popd
191 """
192 }
193 }
Kailash66c389c2019-04-24 14:56:11 -0700194
195 stage('Build/Install New Service') {
196 steps {
197 sh """
198 #!/usr/bin/env bash
199 set -eu -o pipefail
200 pushd $WORKSPACE/cord/orchestration/xos-services/${gerritProject}
201 export DOCKER_TAG=\$(cat VERSION)-test
202 export DOCKER_REPOSITORY=xosproject/
203 export DOCKER_BUILD_ARGS=--no-cache
Kailash66c389c2019-04-24 14:56:11 -0700204 echo "\$(cat VERSION)-test" > VERSION
205 make docker-build
206
207 #install newservice
208 cd $WORKSPACE/cord/helm-charts
209 helm upgrade --set image.tag=\$DOCKER_TAG \
210 --set image.pullPolicy=Never \
Kailash5f300b92019-05-21 14:04:19 -0700211 --recreate-pods ${serviceName} xos-services/${serviceName}
Kailash66c389c2019-04-24 14:56:11 -0700212 popd
213 """
214 }
215 }
Kailash44fd62c2019-04-30 10:26:31 -0700216
Kailash66c389c2019-04-24 14:56:11 -0700217 stage('Verify Service Upgrade') {
218 steps {
219 sh """
220 #!/usr/bin/env bash
221 set -ex -o pipefail
222 DOCKER_TAG=\$(cat $WORKSPACE/cord/orchestration/xos-services/${gerritProject}/VERSION)
223
224 #wait for xos-core and models to be loaded
225 timeout 300 bash -c "until http -a admin@opencord.org:letmein GET http://127.0.0.1:30001/xosapi/v1/dynamicload/load_status | jq '.services[] | select(.name==\\"core\\").state'| grep -q present; do echo 'Waiting for Core to be loaded'; sleep 5; done"
Matteo Scandolo5c6a4802019-05-14 10:44:21 -0700226 timeout 300 bash -c "until http -a admin@opencord.org:letmein GET http://127.0.0.1:30001/xosapi/v1/dynamicload/load_status | jq '.services[] | select(.name==\\"${serviceName}\\").state'| grep -q present; do echo 'Waiting for New Service to be loaded'; sleep 5; done"
227 timeout 300 bash -c "until http -a admin@opencord.org:letmein GET http://127.0.0.1:30001/xosapi/v1/dynamicload/load_status | jq '.services[] | select(.name==\\"${serviceName}\\").version'| grep -q \$DOCKER_TAG; do echo 'Waiting for New Service Version Check'; sleep 5; done"
Kailash4590ca12019-04-30 14:49:22 -0700228 sleep 120
Kailash66c389c2019-04-24 14:56:11 -0700229 """
230 }
231 }
Kailash44fd62c2019-04-30 10:26:31 -0700232
Kailashf5c3f2b2019-04-30 08:58:49 -0700233 stage('Test Post-Upgrade') {
234 steps {
235 sh """
236 CORE_CONTAINER=\$(docker ps | grep k8s_xos-core | awk '{print \$1}')
237
238 export testname=_service_api.robot
239 export library=_library.robot
240 SERVICES=\$(docker exec -i \$CORE_CONTAINER /bin/bash -c "cd /opt/xos/dynamic_services/;find -name '*.xproto'" | awk -F[//] '{print \$2}')
241 echo \$SERVICES
242 cd $WORKSPACE/cord/test/cord-tester/src/test/cord-api/Tests
Matteo Scandolo5c6a4802019-05-14 10:44:21 -0700243 for i in \$SERVICES; do bash -c "robot -v SETUP_FLAG:Setup -i get -d Log -T -v TESTLIBRARY:${serviceName}_library.robot \$i\$testname"; sleep 2; done || true
Kailash44fd62c2019-04-30 10:26:31 -0700244 """
245 }
246 }
247
Andy Baviere053ad32019-05-09 13:58:34 -0700248 /* Disable the downgrade step because the core doesn't support reverse migrations
Kailash44fd62c2019-04-30 10:26:31 -0700249 stage('Downgrade Service') {
250 steps {
251 sh """
252 #!/usr/bin/env bash
253 set -eu -o pipefail
254
255 pushd cord/helm-charts
256 #delete service
257 helm delete --purge ${gerritProject}
258
259 #re-install service with previous version
260 helm dep update xos-services/${gerritProject}
261 helm install xos-services/${gerritProject} -n ${gerritProject}
262
263 #wait for xos-core and models to be loaded
264 timeout 300 bash -c "until http -a admin@opencord.org:letmein GET http://127.0.0.1:30001/xosapi/v1/dynamicload/load_status | jq '.services[] | select(.name==\\"core\\").state'| grep -q present; do echo 'Waiting for Core to be loaded'; sleep 5; done"
265 timeout 300 bash -c "until http -a admin@opencord.org:letmein GET http://127.0.0.1:30001/xosapi/v1/dynamicload/load_status | jq '.services[] | select(.name==\\"${gerritProject}\\").state'| grep -q present; do echo 'Waiting for Service to be loaded'; sleep 5; done"
Kailash4590ca12019-04-30 14:49:22 -0700266 sleep 120
Kailash44fd62c2019-04-30 10:26:31 -0700267 cd $WORKSPACE/cord/test/cord-tester/src/test/cord-api/Tests
268
269 CORE_CONTAINER=\$(docker ps | grep k8s_xos-core | awk '{print \$1}')
270
271 export testname=_service_api.robot
272 export library=_library.robot
273 SERVICES=\$(docker exec -i \$CORE_CONTAINER /bin/bash -c "cd /opt/xos/dynamic_services/;find -name '*.xproto'" | awk -F[//] '{print \$2}')
274 echo \$SERVICES
275 for i in \$SERVICES; do bash -c "robot -v SETUP_FLAG:Setup -i get -d Log -T -v TESTLIBRARY:${gerritProject}_library.robot \$i\$testname"; sleep 2; done || true
276
277 ## get pod logs
278 for pod in \$(kubectl get pods --no-headers | awk '{print \$1}');
279 do
280 kubectl logs \$pod> $WORKSPACE/\$pod.log;
281 done || true
282
283 popd
Kailashf5c3f2b2019-04-30 08:58:49 -0700284 """
285 }
286 }
Andy Baviere053ad32019-05-09 13:58:34 -0700287 */
Kailashf5c3f2b2019-04-30 08:58:49 -0700288
Kailash66c389c2019-04-24 14:56:11 -0700289 }
Kailash44fd62c2019-04-30 10:26:31 -0700290
Kailash66c389c2019-04-24 14:56:11 -0700291 post {
292 always {
293 sh """
Daniele Moroa5f00782020-02-24 11:23:46 -0800294 ## get pod logs
295 for pod in \$(kubectl get pods --no-headers | awk '{print \$1}');
296 do
297 kubectl logs \${pod} > $WORKSPACE/\${pod}_post.log;
298 done
299
Kailashf5c3f2b2019-04-30 08:58:49 -0700300 # copy robot logs
301 if [ -d RobotLogs ]; then rm -r RobotLogs; fi; mkdir RobotLogs
302 cp -r $WORKSPACE/cord/test/cord-tester/src/test/cord-api/Tests/Log/*ml ./RobotLogs
Kailash66c389c2019-04-24 14:56:11 -0700303 kubectl get pods --all-namespaces
304 kubectl describe pods
305 http -a admin@opencord.org:letmein GET http://127.0.0.1:30001/xosapi/v1/dynamicload/load_status
306 echo "# removing helm deployments"
307 kubectl get pods
308 helm list
309
310 for hchart in \$(helm list -q);
311 do
312 echo "## Purging chart: \${hchart} ##"
313 helm delete --purge "\${hchart}"
314 done
315
316 sudo minikube delete
317 """
Kailashf5c3f2b2019-04-30 08:58:49 -0700318 step([$class: 'RobotPublisher',
319 disableArchiveOutput: false,
320 logFileName: 'RobotLogs/log*.html',
321 otherFiles: '',
322 outputFileName: 'RobotLogs/output*.xml',
323 outputPath: '.',
324 passThreshold: 100,
325 reportFileName: 'RobotLogs/report*.html',
326 unstableThreshold: 0]);
Kailash66c389c2019-04-24 14:56:11 -0700327 archiveArtifacts artifacts: '*.log'
328 step([$class: 'Mailer', notifyEveryUnstableBuild: true, recipients: "kailash@opennetworking.org, scottb@opennetworking.org", sendToIndividuals: false])
Kailash66c389c2019-04-24 14:56:11 -0700329 }
330 }
331}