blob: 10d2a144f853f115411d6b3c75eb91923cade7e5 [file] [log] [blame]
// Copyright 2017-present Open Networking Foundation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// chart-api-test-helm.groovy
// Checks functionality of the helm-chart, without overriding the version/tag used
def serviceName = "${gerritProject}"
def doTest = true
pipeline {
/* no label, executor is determined by JJB */
agent {
label "${params.buildNode}"
}
stages {
stage('repo') {
steps {
checkout(changelog: false, \
poll: false,
scm: [$class: 'RepoScm', \
manifestRepositoryUrl: "${params.manifestUrl}", \
manifestBranch: "${params.manifestBranch}", \
currentBranch: true, \
destinationDir: 'cord', \
forceSync: true,
resetFirst: true, \
quiet: true, \
jobs: 4, \
showAllChanges: true] \
)
}
}
stage('minikube') {
steps {
/* see https://github.com/kubernetes/minikube/#linux-continuous-integration-without-vm-support */
sh '''
export MINIKUBE_WANTUPDATENOTIFICATION=false
export MINIKUBE_WANTREPORTERRORPROMPT=false
export CHANGE_MINIKUBE_NONE_USER=true
export MINIKUBE_HOME=$HOME
mkdir -p $HOME/.kube || true
touch $HOME/.kube/config
export KUBECONFIG=$HOME/.kube/config
sudo -E /usr/bin/minikube start --vm-driver=none
'''
script {
timeout(3) {
waitUntil {
sleep 5
def kc_ret = sh script: "kubectl get pods", returnStatus: true
return (kc_ret == 0);
}
}
}
}
}
stage('helm') {
steps {
sh '''
helm init
sleep 60
helm repo add incubator https://kubernetes-charts-incubator.storage.googleapis.com/
helm repo add cord https://charts.opencord.org
'''
}
}
stage('Verify xos-core version requirements') {
steps {
script {
if (serviceName == "olt-service") {
serviceName = "volt"
}
else if (serviceName == "kubernetes-service") {
serviceName = "kubernetes"
}
def result = sh returnStdout: true, script: """
#!/usr/bin/env bash
set -eu -o pipefail
# Obtain git tag of the service corresponding to the the docker image
# used in the latest released version of the helm chart (i.e., HEAD
# of cord/helm-charts master branch, which should be already checked
# out by repo).
pushd cord/helm-charts
export RELEASED_GIT_TAG=\$(echo -e "import yaml\\nwith open('xos-services/${serviceName}/Chart.yaml', 'r') as f: print yaml.safe_load(f)['appVersion']" | python)
popd
# Obtain the xos-core version requirement from the config.yaml of the
# released service.
pushd cord
PROJECT_PATH=\$(xmllint --xpath "string(//project[@name=\\\"${gerritProject}\\\"]/@path)" .repo/manifests/default.xml)
pushd \${PROJECT_PATH}
git fetch --all --tags
git checkout tags/\${RELEASED_GIT_TAG} -b foobar
export RELEASED_CORE_VER_REQ=\$(echo -e "import yaml\\nwith open('xos/synchronizer/config.yaml', 'r') as f: print yaml.safe_load(f)['core_version']" | python)
popd
popd
# Do the same for the patchset we want to verify.
pushd cord
repo download "\$PROJECT_PATH" "${gerritChangeNumber}/${gerritPatchsetNumber}"
pushd \${PROJECT_PATH}
export PATCHSET_CORE_VER_REQ=\$(echo -e "import yaml\\nwith open('xos/synchronizer/config.yaml', 'r') as f: print yaml.safe_load(f)['core_version']" | python)
popd
popd
# We need to produce at least one log file so the archiveArtifacts
# step later won't complain.
echo "RELEASED_CORE_VER_REQ: \${RELEASED_CORE_VER_REQ}" >> $WORKSPACE/version_requirements.log
echo "PATCHSET_CORE_VER_REQ: \${PATCHSET_CORE_VER_REQ}" >> $WORKSPACE/version_requirements.log
if [ "\${PATCHSET_CORE_VER_REQ}" == "\${RELEASED_CORE_VER_REQ}" ]; then
echo 0
else
# versionMismatch is true
echo 1
fi
"""
def versionMismatch = result.readLines().last().toBoolean()
if (versionMismatch) {
echo "Detected xos-core version requirements mismatch. Will skip the rest of the pipeline and return SUCCESS"
}
doTest = !versionMismatch
}
}
}
// The patchset should be already checked out, but for consistency with
// other pipeline jobs, we re-do the same here.
stage('patch') {
when {
expression { doTest }
}
steps {
sh '''
pushd cord
PROJECT_PATH=\$(xmllint --xpath "string(//project[@name=\\\"${gerritProject}\\\"]/@path)" .repo/manifests/default.xml)
repo download "\$PROJECT_PATH" "${gerritChangeNumber}/${gerritPatchsetNumber}"
popd
'''
}
}
stage('Install XOS w/Service') {
when {
expression { doTest }
}
steps {
script {
if (serviceName == "olt-service") {
serviceName = "volt"
}
else if (serviceName == "kubernetes-service") {
serviceName = "kubernetes"
}
}
sh """
#!/usr/bin/env bash
set -eu -o pipefail
pushd cord/helm-charts
helm dep update xos-core
helm install xos-core -n xos-core
helm-repo-tools/wait_for_pods.sh
#install service
helm dep update xos-services/${serviceName}
helm install xos-services/${serviceName} -n ${serviceName}
popd
"""
}
}
stage('Verify') {
when {
expression { doTest }
}
steps {
echo "serviceName: ${serviceName}"
sh """
#!/usr/bin/env bash
set -ex -o pipefail
export DOCKER_TAG=\$(cat $WORKSPACE/cord/orchestration/xos/VERSION)
#wait for xos-core and models to be loaded
timeout 300 bash -c "until http -a admin@opencord.org:letmein GET http://127.0.0.1:30001/xosapi/v1/dynamicload/load_status | jq '.services[] | select(.name==\\"core\\").state'| grep -q present; do echo 'Waiting for Core to be loaded'; sleep 5; done"
timeout 300 bash -c "until http -a admin@opencord.org:letmein GET http://127.0.0.1:30001/xosapi/v1/dynamicload/load_status | jq '.services[] | select(.name==\\"${serviceName}\\").state'| grep -q present; do echo 'Waiting for Core to be loaded'; sleep 5; done"
## get pod logs
for pod in \$(kubectl get pods --no-headers | awk '{print \$1}');
do
kubectl logs \${pod} > $WORKSPACE/\${pod}_pre.log;
done
"""
}
}
stage('Generate Model API Tests') {
when {
expression { doTest }
}
steps {
sh """
CORE_CONTAINER=\$(docker ps | grep k8s_xos-core | awk '{print \$1}')
cd $WORKSPACE/cord/test/cord-tester
docker cp $WORKSPACE/cord/test/cord-tester/src/test/cord-api/Tests/targets/xosserviceupgradetest.xtarget \$CORE_CONTAINER:/opt/xos/lib/xos-genx/xosgenx/targets/xosserviceupgradetest.xtarget
docker cp $WORKSPACE/cord/test/cord-tester/src/test/cord-api/Tests/targets/xosstaticlibrary.xtarget \$CORE_CONTAINER:/opt/xos/lib/xos-genx/xosgenx/targets/xosstaticlibrary.xtarget
export testname=_service_api.robot
export library=_library.robot
SERVICES=\$(docker exec -i \$CORE_CONTAINER /bin/bash -c "cd /opt/xos/dynamic_services/;find -name '*.xproto'" | awk -F[//] '{print \$2}')
echo \$SERVICES
for i in \$SERVICES; do bash -c "docker exec -i \$CORE_CONTAINER /bin/bash -c 'xosgenx --target /opt/xos/lib/xos-genx/xosgenx/targets/./xosserviceupgradetest.xtarget /opt/xos/dynamic_services/\$i/\$i.xproto /opt/xos/core/models/core.xproto'" > $WORKSPACE/cord/test/cord-tester/src/test/cord-api/Tests/\$i\$testname; done
for i in \$SERVICES; do bash -c "docker exec -i \$CORE_CONTAINER /bin/bash -c 'xosgenx --target /opt/xos/lib/xos-genx/xosgenx/targets/./xosstaticlibrary.xtarget /opt/xos/dynamic_services/\$i/\$i.xproto /opt/xos/core/models/core.xproto'" > $WORKSPACE/cord/test/cord-tester/src/test/cord-api/Tests/\$i\$library; done
ls -al $WORKSPACE/cord/test/cord-tester/src/test/cord-api/Tests/
"""
}
}
stage('Test Pre-Upgrade') {
when {
expression { doTest }
}
steps {
sh """
#!/usr/bin/env bash
set -ex -o pipefail
pushd cord/test/cord-tester
make venv_cord
source venv_cord/bin/activate
cd src/test/cord-api/Tests
CORE_CONTAINER=\$(docker ps | grep k8s_xos-core | awk '{print \$1}')
CHAM_CONTAINER=\$(docker ps | grep k8s_xos-chameleon | awk '{print \$1}')
XOS_CHAMELEON=\$(docker exec \$CHAM_CONTAINER ip a | grep -oE "([0-9]{1,3}\\.){3}[0-9]{1,3}\\b" | grep 172)
cd $WORKSPACE/cord/test/cord-tester/src/test/cord-api/Properties/
sed -i \"s/^\\(SERVER_IP = \\).*/\\1\'\$XOS_CHAMELEON\'/\" RestApiProperties.py
sed -i \"s/^\\(SERVER_PORT = \\).*/\\1\'9101\'/\" RestApiProperties.py
sed -i \"s/^\\(XOS_USER = \\).*/\\1\'admin@opencord.org\'/\" RestApiProperties.py
sed -i \"s/^\\(XOS_PASSWD = \\).*/\\1\'letmein\'/\" RestApiProperties.py
sed -i \"s/^\\(PASSWD = \\).*/\\1\'letmein\'/\" RestApiProperties.py
export testname=_service_api.robot
export library=_library.robot
SERVICES=\$(docker exec -i \$CORE_CONTAINER /bin/bash -c "cd /opt/xos/dynamic_services/;find -name '*.xproto'" | awk -F[//] '{print \$2}')
echo \$SERVICES
cd $WORKSPACE/cord/test/cord-tester/src/test/cord-api/Tests
for i in \$SERVICES; do bash -c "robot -v SETUP_FLAG:Setup -i create -d Log -T -v TESTLIBRARY:${serviceName}_library.robot \$i\$testname"; sleep 2; done || true
popd
"""
}
}
stage('Build/Install New Service') {
when {
expression { doTest }
}
steps {
sh """
#!/usr/bin/env bash
set -eu -o pipefail
pushd $WORKSPACE/cord/orchestration/xos-services/${gerritProject}
export DOCKER_TAG=\$(cat VERSION)-test
export DOCKER_REPOSITORY=xosproject/
export DOCKER_BUILD_ARGS=--no-cache
echo "\$(cat VERSION)-test" > VERSION
make docker-build
#install newservice
cd $WORKSPACE/cord/helm-charts
helm upgrade --set image.tag=\$DOCKER_TAG \
--set image.pullPolicy=Never \
--recreate-pods ${serviceName} xos-services/${serviceName}
popd
"""
}
}
stage('Verify Service Upgrade') {
when {
expression { doTest }
}
steps {
sh """
#!/usr/bin/env bash
set -ex -o pipefail
DOCKER_TAG=\$(cat $WORKSPACE/cord/orchestration/xos-services/${gerritProject}/VERSION)
#wait for xos-core and models to be loaded
timeout 300 bash -c "until http -a admin@opencord.org:letmein GET http://127.0.0.1:30001/xosapi/v1/dynamicload/load_status | jq '.services[] | select(.name==\\"core\\").state'| grep -q present; do echo 'Waiting for Core to be loaded'; sleep 5; done"
timeout 300 bash -c "until http -a admin@opencord.org:letmein GET http://127.0.0.1:30001/xosapi/v1/dynamicload/load_status | jq '.services[] | select(.name==\\"${serviceName}\\").state'| grep -q present; do echo 'Waiting for New Service to be loaded'; sleep 5; done"
timeout 300 bash -c "until http -a admin@opencord.org:letmein GET http://127.0.0.1:30001/xosapi/v1/dynamicload/load_status | jq '.services[] | select(.name==\\"${serviceName}\\").version'| grep -q \$DOCKER_TAG; do echo 'Waiting for New Service Version Check'; sleep 5; done"
sleep 120
"""
}
}
stage('Test Post-Upgrade') {
when {
expression { doTest }
}
steps {
sh """
#!/usr/bin/env bash
set -ex -o pipefail
cd cord/test/cord-tester
make venv_cord
source venv_cord/bin/activate
cd src/test/cord-api/Tests
CORE_CONTAINER=\$(docker ps | grep k8s_xos-core | awk '{print \$1}')
export testname=_service_api.robot
export library=_library.robot
SERVICES=\$(docker exec -i \$CORE_CONTAINER /bin/bash -c "cd /opt/xos/dynamic_services/;find -name '*.xproto'" | awk -F[//] '{print \$2}')
echo \$SERVICES
for i in \$SERVICES; do bash -c "robot -v SETUP_FLAG:Setup -i get -d Log -T -v TESTLIBRARY:${serviceName}_library.robot \$i\$testname"; sleep 2; done || true
"""
}
}
/* Disable the downgrade step because the core doesn't support reverse migrations
stage('Downgrade Service') {
when {
expression { doTest }
}
steps {
sh """
#!/usr/bin/env bash
set -eu -o pipefail
pushd cord/helm-charts
#delete service
helm delete --purge ${gerritProject}
#re-install service with previous version
helm dep update xos-services/${gerritProject}
helm install xos-services/${gerritProject} -n ${gerritProject}
#wait for xos-core and models to be loaded
timeout 300 bash -c "until http -a admin@opencord.org:letmein GET http://127.0.0.1:30001/xosapi/v1/dynamicload/load_status | jq '.services[] | select(.name==\\"core\\").state'| grep -q present; do echo 'Waiting for Core to be loaded'; sleep 5; done"
timeout 300 bash -c "until http -a admin@opencord.org:letmein GET http://127.0.0.1:30001/xosapi/v1/dynamicload/load_status | jq '.services[] | select(.name==\\"${gerritProject}\\").state'| grep -q present; do echo 'Waiting for Service to be loaded'; sleep 5; done"
sleep 120
cd $WORKSPACE/cord/test/cord-tester
make venv_cord
source venv_cord/bin/activate
cd src/test/cord-api/Tests
CORE_CONTAINER=\$(docker ps | grep k8s_xos-core | awk '{print \$1}')
export testname=_service_api.robot
export library=_library.robot
SERVICES=\$(docker exec -i \$CORE_CONTAINER /bin/bash -c "cd /opt/xos/dynamic_services/;find -name '*.xproto'" | awk -F[//] '{print \$2}')
echo \$SERVICES
for i in \$SERVICES; do bash -c "robot -v SETUP_FLAG:Setup -i get -d Log -T -v TESTLIBRARY:${gerritProject}_library.robot \$i\$testname"; sleep 2; done || true
## get pod logs
for pod in \$(kubectl get pods --no-headers | awk '{print \$1}');
do
kubectl logs \$pod> $WORKSPACE/\$pod.log;
done || true
popd
"""
}
}
*/
}
post {
always {
sh """
## get pod logs
for pod in \$(kubectl get pods --no-headers | awk '{print \$1}');
do
kubectl logs \${pod} > $WORKSPACE/\${pod}_post.log;
done
# copy robot logs
if [ -d RobotLogs ]; then rm -r RobotLogs; fi; mkdir RobotLogs
cp -r $WORKSPACE/cord/test/cord-tester/src/test/cord-api/Tests/Log/*ml ./RobotLogs || true
# Workaorund for when doTest is false and there's no Robot output to
# process. Without this file, RobotPublisher below will fail marking
# the build as FAILED.
if [ ! -f RobotLogs/output.xml ]; then echo '<xml></xml>' > RobotLogs/output.xml; fi;
kubectl get pods --all-namespaces
kubectl describe pods
http -a admin@opencord.org:letmein GET http://127.0.0.1:30001/xosapi/v1/dynamicload/load_status || true
echo "# removing helm deployments"
kubectl get pods
helm list
for hchart in \$(helm list -q);
do
echo "## Purging chart: \${hchart} ##"
helm delete --purge "\${hchart}"
done
sudo minikube delete
"""
step([$class: 'RobotPublisher',
disableArchiveOutput: false,
logFileName: 'RobotLogs/log*.html',
otherFiles: '',
outputFileName: 'RobotLogs/output*.xml',
outputPath: '.',
passThreshold: 100,
reportFileName: 'RobotLogs/report*.html',
unstableThreshold: 0]);
archiveArtifacts artifacts: '*.log'
step([$class: 'Mailer', notifyEveryUnstableBuild: true, recipients: "scottb@opennetworking.org", sendToIndividuals: false])
}
}
}