blob: 6d4d07e9aff43128b0edad06a1aaebd2515669f1 [file] [log] [blame]
Kailash66c389c2019-04-24 14:56:11 -07001// Copyright 2017-present Open Networking Foundation
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15// chart-api-test-helm.groovy
16// Checks functionality of the helm-chart, without overriding the version/tag used
17
Matteo Scandolo5c6a4802019-05-14 10:44:21 -070018def serviceName = "${gerritProject}"
19
Kailash66c389c2019-04-24 14:56:11 -070020pipeline {
21
22 /* no label, executor is determined by JJB */
23 agent {
24 label "${params.executorNode}"
25 }
26
27 stages {
28
29 stage('repo') {
30 steps {
31 checkout(changelog: false, \
32 poll: false,
33 scm: [$class: 'RepoScm', \
34 manifestRepositoryUrl: "${params.manifestUrl}", \
35 manifestBranch: "${params.manifestBranch}", \
36 currentBranch: true, \
37 destinationDir: 'cord', \
38 forceSync: true,
39 resetFirst: true, \
40 quiet: true, \
41 jobs: 4, \
42 showAllChanges: true] \
43 )
44 }
45 }
46
47 stage('patch') {
48 steps {
49 sh '''
50 pushd cord
51 PROJECT_PATH=\$(xmllint --xpath "string(//project[@name=\\\"${gerritProject}\\\"]/@path)" .repo/manifest.xml)
52 repo download "\$PROJECT_PATH" "${gerritChangeNumber}/${gerritPatchsetNumber}"
53 popd
54 '''
55 }
56 }
57
58 stage('minikube') {
59 steps {
60 /* see https://github.com/kubernetes/minikube/#linux-continuous-integration-without-vm-support */
61 sh '''
62 export MINIKUBE_WANTUPDATENOTIFICATION=false
63 export MINIKUBE_WANTREPORTERRORPROMPT=false
64 export CHANGE_MINIKUBE_NONE_USER=true
65 export MINIKUBE_HOME=$HOME
66 mkdir -p $HOME/.kube || true
67 touch $HOME/.kube/config
68 export KUBECONFIG=$HOME/.kube/config
69 sudo -E /usr/bin/minikube start --vm-driver=none
70 '''
71 script {
72 timeout(3) {
73 waitUntil {
74 sleep 5
75 def kc_ret = sh script: "kubectl get po", returnStatus: true
76 return (kc_ret == 0);
77 }
78 }
79 }
80 }
81 }
82
83 stage('helm') {
84 steps {
85 sh '''
86 helm init
87 sleep 60
88 helm repo add incubator https://kubernetes-charts-incubator.storage.googleapis.com/
89 '''
90 }
91 }
92
93 stage('Install XOS w/Service') {
94 steps {
Kailash5f300b92019-05-21 14:04:19 -070095 script {
96 if (serviceName == "olt-service") {
97 serviceName = "volt"
98 }
99 else if (serviceName == "onos-service") {
100 serviceName = "onos"
101 }
102 else if (serviceName == "kubernetes-service") {
103 serviceName = "kubernetes"
104 }
105 }
Kailash66c389c2019-04-24 14:56:11 -0700106 sh """
107 #!/usr/bin/env bash
108 set -eu -o pipefail
109
110 pushd cord/helm-charts
111 helm dep update xos-core
112 helm install xos-core -n xos-core
113
Kailash66c389c2019-04-24 14:56:11 -0700114 helm-repo-tools/wait_for_pods.sh
115
116 #install service
Kailash5f300b92019-05-21 14:04:19 -0700117 helm dep update xos-services/${serviceName}
118 helm install xos-services/${serviceName} -n ${serviceName}
Kailash66c389c2019-04-24 14:56:11 -0700119 popd
120 """
121 }
122 }
123 stage('Verify') {
124 steps {
Matteo Scandolo5c6a4802019-05-14 10:44:21 -0700125 echo "serviceName: ${serviceName}"
Kailash66c389c2019-04-24 14:56:11 -0700126 sh """
127 #!/usr/bin/env bash
128 set -ex -o pipefail
129 export DOCKER_TAG=\$(cat $WORKSPACE/cord/orchestration/xos/VERSION)
130
131 #wait for xos-core and models to be loaded
132 timeout 300 bash -c "until http -a admin@opencord.org:letmein GET http://127.0.0.1:30001/xosapi/v1/dynamicload/load_status | jq '.services[] | select(.name==\\"core\\").state'| grep -q present; do echo 'Waiting for Core to be loaded'; sleep 5; done"
Matteo Scandolo5c6a4802019-05-14 10:44:21 -0700133 timeout 300 bash -c "until http -a admin@opencord.org:letmein GET http://127.0.0.1:30001/xosapi/v1/dynamicload/load_status | jq '.services[] | select(.name==\\"${serviceName}\\").state'| grep -q present; do echo 'Waiting for Core to be loaded'; sleep 5; done"
Kailash66c389c2019-04-24 14:56:11 -0700134
135 ## get pod logs
136 for pod in \$(kubectl get pods --no-headers | awk '{print \$1}');
137 do
138 kubectl logs \$pod> $WORKSPACE/\$pod.log;
139 done
140 """
141 }
142 }
Kailashf5c3f2b2019-04-30 08:58:49 -0700143 stage('Generate Model API Tests') {
144 steps {
145 sh """
146 CORE_CONTAINER=\$(docker ps | grep k8s_xos-core | awk '{print \$1}')
147 cd $WORKSPACE/cord/test/cord-tester
Kailashf5c3f2b2019-04-30 08:58:49 -0700148 docker cp $WORKSPACE/cord/test/cord-tester/src/test/cord-api/Tests/targets/xosserviceupgradetest.xtarget \$CORE_CONTAINER:/opt/xos/lib/xos-genx/xosgenx/targets/xosserviceupgradetest.xtarget
149 docker cp $WORKSPACE/cord/test/cord-tester/src/test/cord-api/Tests/targets/xosstaticlibrary.xtarget \$CORE_CONTAINER:/opt/xos/lib/xos-genx/xosgenx/targets/xosstaticlibrary.xtarget
150
151 export testname=_service_api.robot
152 export library=_library.robot
153
154 SERVICES=\$(docker exec -i \$CORE_CONTAINER /bin/bash -c "cd /opt/xos/dynamic_services/;find -name '*.xproto'" | awk -F[//] '{print \$2}')
155 echo \$SERVICES
156
157 for i in \$SERVICES; do bash -c "docker exec -i \$CORE_CONTAINER /bin/bash -c 'xosgenx --target /opt/xos/lib/xos-genx/xosgenx/targets/./xosserviceupgradetest.xtarget /opt/xos/dynamic_services/\$i/\$i.xproto /opt/xos/core/models/core.xproto'" > $WORKSPACE/cord/test/cord-tester/src/test/cord-api/Tests/\$i\$testname; done
158
159 for i in \$SERVICES; do bash -c "docker exec -i \$CORE_CONTAINER /bin/bash -c 'xosgenx --target /opt/xos/lib/xos-genx/xosgenx/targets/./xosstaticlibrary.xtarget /opt/xos/dynamic_services/\$i/\$i.xproto /opt/xos/core/models/core.xproto'" > $WORKSPACE/cord/test/cord-tester/src/test/cord-api/Tests/\$i\$library; done
160 ls -al $WORKSPACE/cord/test/cord-tester/src/test/cord-api/Tests/
161 """
162 }
163 }
164
165 stage('Test Pre-Upgrade') {
166 steps {
167 sh """
168 pushd cord/test/cord-tester/src/test/cord-api/Tests
169
170 CORE_CONTAINER=\$(docker ps | grep k8s_xos-core | awk '{print \$1}')
171 CHAM_CONTAINER=\$(docker ps | grep k8s_xos-chameleon | awk '{print \$1}')
172 XOS_CHAMELEON=\$(docker exec \$CHAM_CONTAINER ip a | grep -oE "([0-9]{1,3}\\.){3}[0-9]{1,3}\\b" | grep 172)
173
174 cd $WORKSPACE/cord/test/cord-tester/src/test/cord-api/Properties/
175 sed -i \"s/^\\(SERVER_IP = \\).*/\\1\'\$XOS_CHAMELEON\'/\" RestApiProperties.py
176 sed -i \"s/^\\(SERVER_PORT = \\).*/\\1\'9101\'/\" RestApiProperties.py
177 sed -i \"s/^\\(XOS_USER = \\).*/\\1\'admin@opencord.org\'/\" RestApiProperties.py
178 sed -i \"s/^\\(XOS_PASSWD = \\).*/\\1\'letmein\'/\" RestApiProperties.py
179 sed -i \"s/^\\(PASSWD = \\).*/\\1\'letmein\'/\" RestApiProperties.py
180
181 export testname=_service_api.robot
182 export library=_library.robot
183 SERVICES=\$(docker exec -i \$CORE_CONTAINER /bin/bash -c "cd /opt/xos/dynamic_services/;find -name '*.xproto'" | awk -F[//] '{print \$2}')
184 echo \$SERVICES
185
186 cd $WORKSPACE/cord/test/cord-tester/src/test/cord-api/Tests
Matteo Scandolo5c6a4802019-05-14 10:44:21 -0700187 for i in \$SERVICES; do bash -c "robot -v SETUP_FLAG:Setup -i create -d Log -T -v TESTLIBRARY:${serviceName}_library.robot \$i\$testname"; sleep 2; done || true
Kailashf5c3f2b2019-04-30 08:58:49 -0700188
189 popd
190 """
191 }
192 }
Kailash66c389c2019-04-24 14:56:11 -0700193
194 stage('Build/Install New Service') {
195 steps {
196 sh """
197 #!/usr/bin/env bash
198 set -eu -o pipefail
199 pushd $WORKSPACE/cord/orchestration/xos-services/${gerritProject}
200 export DOCKER_TAG=\$(cat VERSION)-test
201 export DOCKER_REPOSITORY=xosproject/
202 export DOCKER_BUILD_ARGS=--no-cache
Kailash66c389c2019-04-24 14:56:11 -0700203 echo "\$(cat VERSION)-test" > VERSION
204 make docker-build
205
206 #install newservice
207 cd $WORKSPACE/cord/helm-charts
208 helm upgrade --set image.tag=\$DOCKER_TAG \
209 --set image.pullPolicy=Never \
Kailash5f300b92019-05-21 14:04:19 -0700210 --recreate-pods ${serviceName} xos-services/${serviceName}
Kailash66c389c2019-04-24 14:56:11 -0700211 popd
212 """
213 }
214 }
Kailash44fd62c2019-04-30 10:26:31 -0700215
Kailash66c389c2019-04-24 14:56:11 -0700216 stage('Verify Service Upgrade') {
217 steps {
218 sh """
219 #!/usr/bin/env bash
220 set -ex -o pipefail
221 DOCKER_TAG=\$(cat $WORKSPACE/cord/orchestration/xos-services/${gerritProject}/VERSION)
222
223 #wait for xos-core and models to be loaded
224 timeout 300 bash -c "until http -a admin@opencord.org:letmein GET http://127.0.0.1:30001/xosapi/v1/dynamicload/load_status | jq '.services[] | select(.name==\\"core\\").state'| grep -q present; do echo 'Waiting for Core to be loaded'; sleep 5; done"
Matteo Scandolo5c6a4802019-05-14 10:44:21 -0700225 timeout 300 bash -c "until http -a admin@opencord.org:letmein GET http://127.0.0.1:30001/xosapi/v1/dynamicload/load_status | jq '.services[] | select(.name==\\"${serviceName}\\").state'| grep -q present; do echo 'Waiting for New Service to be loaded'; sleep 5; done"
226 timeout 300 bash -c "until http -a admin@opencord.org:letmein GET http://127.0.0.1:30001/xosapi/v1/dynamicload/load_status | jq '.services[] | select(.name==\\"${serviceName}\\").version'| grep -q \$DOCKER_TAG; do echo 'Waiting for New Service Version Check'; sleep 5; done"
Kailash4590ca12019-04-30 14:49:22 -0700227 sleep 120
Kailash66c389c2019-04-24 14:56:11 -0700228 """
229 }
230 }
Kailash44fd62c2019-04-30 10:26:31 -0700231
Kailashf5c3f2b2019-04-30 08:58:49 -0700232 stage('Test Post-Upgrade') {
233 steps {
234 sh """
235 CORE_CONTAINER=\$(docker ps | grep k8s_xos-core | awk '{print \$1}')
236
237 export testname=_service_api.robot
238 export library=_library.robot
239 SERVICES=\$(docker exec -i \$CORE_CONTAINER /bin/bash -c "cd /opt/xos/dynamic_services/;find -name '*.xproto'" | awk -F[//] '{print \$2}')
240 echo \$SERVICES
241 cd $WORKSPACE/cord/test/cord-tester/src/test/cord-api/Tests
Matteo Scandolo5c6a4802019-05-14 10:44:21 -0700242 for i in \$SERVICES; do bash -c "robot -v SETUP_FLAG:Setup -i get -d Log -T -v TESTLIBRARY:${serviceName}_library.robot \$i\$testname"; sleep 2; done || true
Kailash44fd62c2019-04-30 10:26:31 -0700243
244 ## get pod logs
245 for pod in \$(kubectl get pods --no-headers | awk '{print \$1}');
246 do
247 kubectl logs \$pod> $WORKSPACE/\$pod.log;
248 done || true
249 """
250 }
251 }
252
Andy Baviere053ad32019-05-09 13:58:34 -0700253 /* Disable the downgrade step because the core doesn't support reverse migrations
Kailash44fd62c2019-04-30 10:26:31 -0700254 stage('Downgrade Service') {
255 steps {
256 sh """
257 #!/usr/bin/env bash
258 set -eu -o pipefail
259
260 pushd cord/helm-charts
261 #delete service
262 helm delete --purge ${gerritProject}
263
264 #re-install service with previous version
265 helm dep update xos-services/${gerritProject}
266 helm install xos-services/${gerritProject} -n ${gerritProject}
267
268 #wait for xos-core and models to be loaded
269 timeout 300 bash -c "until http -a admin@opencord.org:letmein GET http://127.0.0.1:30001/xosapi/v1/dynamicload/load_status | jq '.services[] | select(.name==\\"core\\").state'| grep -q present; do echo 'Waiting for Core to be loaded'; sleep 5; done"
270 timeout 300 bash -c "until http -a admin@opencord.org:letmein GET http://127.0.0.1:30001/xosapi/v1/dynamicload/load_status | jq '.services[] | select(.name==\\"${gerritProject}\\").state'| grep -q present; do echo 'Waiting for Service to be loaded'; sleep 5; done"
Kailash4590ca12019-04-30 14:49:22 -0700271 sleep 120
Kailash44fd62c2019-04-30 10:26:31 -0700272 cd $WORKSPACE/cord/test/cord-tester/src/test/cord-api/Tests
273
274 CORE_CONTAINER=\$(docker ps | grep k8s_xos-core | awk '{print \$1}')
275
276 export testname=_service_api.robot
277 export library=_library.robot
278 SERVICES=\$(docker exec -i \$CORE_CONTAINER /bin/bash -c "cd /opt/xos/dynamic_services/;find -name '*.xproto'" | awk -F[//] '{print \$2}')
279 echo \$SERVICES
280 for i in \$SERVICES; do bash -c "robot -v SETUP_FLAG:Setup -i get -d Log -T -v TESTLIBRARY:${gerritProject}_library.robot \$i\$testname"; sleep 2; done || true
281
282 ## get pod logs
283 for pod in \$(kubectl get pods --no-headers | awk '{print \$1}');
284 do
285 kubectl logs \$pod> $WORKSPACE/\$pod.log;
286 done || true
287
288 popd
Kailashf5c3f2b2019-04-30 08:58:49 -0700289 """
290 }
291 }
Andy Baviere053ad32019-05-09 13:58:34 -0700292 */
Kailashf5c3f2b2019-04-30 08:58:49 -0700293
Kailash66c389c2019-04-24 14:56:11 -0700294 }
Kailash44fd62c2019-04-30 10:26:31 -0700295
Kailash66c389c2019-04-24 14:56:11 -0700296 post {
297 always {
298 sh """
Kailashf5c3f2b2019-04-30 08:58:49 -0700299 # copy robot logs
300 if [ -d RobotLogs ]; then rm -r RobotLogs; fi; mkdir RobotLogs
301 cp -r $WORKSPACE/cord/test/cord-tester/src/test/cord-api/Tests/Log/*ml ./RobotLogs
Kailash66c389c2019-04-24 14:56:11 -0700302 kubectl get pods --all-namespaces
303 kubectl describe pods
304 http -a admin@opencord.org:letmein GET http://127.0.0.1:30001/xosapi/v1/dynamicload/load_status
305 echo "# removing helm deployments"
306 kubectl get pods
307 helm list
308
309 for hchart in \$(helm list -q);
310 do
311 echo "## Purging chart: \${hchart} ##"
312 helm delete --purge "\${hchart}"
313 done
314
315 sudo minikube delete
316 """
Kailashf5c3f2b2019-04-30 08:58:49 -0700317 step([$class: 'RobotPublisher',
318 disableArchiveOutput: false,
319 logFileName: 'RobotLogs/log*.html',
320 otherFiles: '',
321 outputFileName: 'RobotLogs/output*.xml',
322 outputPath: '.',
323 passThreshold: 100,
324 reportFileName: 'RobotLogs/report*.html',
325 unstableThreshold: 0]);
Kailash66c389c2019-04-24 14:56:11 -0700326 archiveArtifacts artifacts: '*.log'
327 step([$class: 'Mailer', notifyEveryUnstableBuild: true, recipients: "kailash@opennetworking.org, scottb@opennetworking.org", sendToIndividuals: false])
Kailash66c389c2019-04-24 14:56:11 -0700328 }
329 }
330}