[VOL-3780] Migrating pipelines to use voltha-infra and voltha-stack charts through shared keywords
Change-Id: I55a9847126b4ecfa326fa882d57e6381e4cef425
diff --git a/jjb/pipeline/bbsim-scale.groovy b/jjb/pipeline/bbsim-scale.groovy
deleted file mode 100644
index 6a05cfd..0000000
--- a/jjb/pipeline/bbsim-scale.groovy
+++ /dev/null
@@ -1,63 +0,0 @@
-/* bbsim-scale test */
-
-pipeline {
-
- /* no label, executor is determined by JJB */
- agent {
- label "${params.buildNode}"
- }
-
- options {
- timeout(time: 90, unit: 'MINUTES')
- }
-
- environment {
- VOLTHA_LOG_LEVEL="DEBUG"
- NAME="minimal"
- WITH_RADIUS="y"
- WITH_BBSIM="y"
- INSTALL_ONOS_APPS="y"
- CONFIG_SADIS="y"
- FANCY=0
- WITH_SIM_ADAPTERS="n"
- }
-
- stages {
-
- stage('Create K8s Cluster') {
- steps {
- sh """
- git clone https:/gerrit.opencord.org/kind-voltha
- cd kind-voltha/
- DEPLOY_K8S=y JUST_K8S=y ./voltha up
- """
- }
- }
-
- stage('Deploy Voltha') {
- steps {
- sh '''
- cd $WORKSPACE/kind-voltha/
- echo \$HELM_FLAG
- ./voltha up
- '''
- }
- }
-
-
- }
-
- post {
- always {
- sh '''
- WAIT_ON_DOWN=y ./voltha down
- cd $WORKSPACE/
- rm -rf kind-voltha/ voltha/ || true
- '''
- }
- failure {
- sh '''
- '''
- }
- }
-}
diff --git a/jjb/pipeline/device-management-mock-tests.groovy b/jjb/pipeline/device-management-mock-tests.groovy
deleted file mode 100644
index ac575a5..0000000
--- a/jjb/pipeline/device-management-mock-tests.groovy
+++ /dev/null
@@ -1,198 +0,0 @@
-// Copyright 2017-present Open Networking Foundation
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// voltha-2.x e2e tests
-// uses kind-voltha to deploy voltha-2.X
-// uses bbsim to simulate OLT/ONUs
-
-pipeline {
-
- /* no label, executor is determined by JJB */
- agent {
- label "${params.buildNode}"
- }
- options {
- timeout(time: 90, unit: 'MINUTES')
- }
- environment {
- KUBECONFIG="$HOME/.kube/kind-config-voltha-minimal"
- VOLTCONFIG="$HOME/.volt/config-minimal"
- PATH="$WORKSPACE/voltha/kind-voltha/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
- NAME="minimal"
- FANCY=0
- WITH_SIM_ADAPTERS="n"
- WITH_RADIUS="y"
- WITH_BBSIM="y"
- DEPLOY_K8S="y"
- VOLTHA_LOG_LEVEL="DEBUG"
- CONFIG_SADIS="n"
- ROBOT_MISC_ARGS="-d $WORKSPACE/RobotLogs"
- }
-
- stages {
-
- stage('Repo') {
- steps {
- step([$class: 'WsCleanup'])
- checkout(changelog: false, \
- poll: false,
- scm: [$class: 'RepoScm', \
- manifestRepositoryUrl: "${params.manifestUrl}", \
- manifestBranch: "${params.manifestBranch}", \
- currentBranch: true, \
- destinationDir: 'voltha', \
- forceSync: true,
- resetFirst: true, \
- quiet: true, \
- jobs: 4, \
- showAllChanges: true] \
- )
- }
- }
- stage('Patch') {
- steps {
- sh """
- pushd $WORKSPACE/
- echo "${gerritProject}" "${gerritChangeNumber}" "${gerritPatchsetNumber}"
- echo "${GERRIT_REFSPEC}"
- git clone https://gerrit.opencord.org/${gerritProject}
- cd "${gerritProject}"
- git fetch https://gerrit.opencord.org/${gerritProject} "${GERRIT_REFSPEC}" && git checkout FETCH_HEAD
- popd
- """
- }
- }
- stage('Create K8s Cluster') {
- steps {
- sh """
- cd $WORKSPACE/voltha/kind-voltha/
- JUST_K8S=y ./voltha up
- bash <( curl -sfL https://raw.githubusercontent.com/boz/kail/master/godownloader.sh) -b "$WORKSPACE/voltha/kind-voltha/bin"
- """
- }
- }
-
- stage('Build Redfish Importer Image') {
- steps {
- sh """
- make -C $WORKSPACE/device-management/\$1 DOCKER_REPOSITORY=opencord/ DOCKER_TAG=citest docker-build-importer
- """
- }
- }
-
- stage('Build demo_test Image') {
- steps {
- sh """
- make -C $WORKSPACE/device-management/\$1/demo_test DOCKER_REPOSITORY=opencord/ DOCKER_TAG=citest docker-build
- """
- }
- }
-
- stage('Build mock-redfish-server Image') {
- steps {
- sh """
- make -C $WORKSPACE/device-management/\$1/mock-redfish-server DOCKER_REPOSITORY=opencord/ DOCKER_TAG=citest docker-build
- """
- }
- }
-
- stage('Push Images') {
- steps {
- sh '''
- docker images | grep citest
- for image in \$(docker images -f "reference=*/*citest" --format "{{.Repository}}"); do echo "Pushing \$image to nodes"; kind load docker-image \$image:citest --name voltha-\$NAME --nodes voltha-\$NAME-worker,voltha-\$NAME-worker2; done
- '''
- }
- }
- stage('Deploy Voltha') {
- steps {
- sh '''
- export EXTRA_HELM_FLAGS="--set log_agent.enabled=False ${extraHelmFlags} "
-
- cd $WORKSPACE/voltha/kind-voltha/
- echo \$EXTRA_HELM_FLAGS
- kail -n voltha -n default > $WORKSPACE/onos-voltha-combined.log &
- ./voltha up
- '''
- }
- }
-
- stage('Run E2E Tests') {
- steps {
- sh '''
- mkdir -p $WORKSPACE/RobotLogs
-
- # tell the kubernetes script to use images tagged citest and pullPolicy:Never
- sed -i 's/master/citest/g' $WORKSPACE/device-management/kubernetes/deploy*.yaml
- sed -i 's/imagePullPolicy: Always/imagePullPolicy: Never/g' $WORKSPACE/device-management/kubernetes/deploy*.yaml
- make -C $WORKSPACE/device-management functional-mock-test || true
- '''
- }
- }
- }
-
- post {
- always {
- sh '''
- set +e
- cp $WORKSPACE/voltha/kind-voltha/install-minimal.log $WORKSPACE/
- kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.image}{'\\n'}" | sort | uniq
- kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.imageID}{'\\n'}" | sort | uniq
- kubectl get nodes -o wide
- kubectl get pods -o wide
- kubectl get pods -n voltha -o wide
-
- sync
- pkill kail || true
- md5sum $WORKSPACE/voltha/kind-voltha/bin/voltctl
-
- ## Pull out errors from log files
- extract_errors_go() {
- echo
- echo "Error summary for $1:"
- grep $1 $WORKSPACE/onos-voltha-combined.log | grep '"level":"error"' | cut -d ' ' -f 2- | jq -r '.msg'
- echo
- }
-
- extract_errors_python() {
- echo
- echo "Error summary for $1:"
- grep $1 $WORKSPACE/onos-voltha-combined.log | grep 'ERROR' | cut -d ' ' -f 2-
- echo
- }
-
- extract_errors_go voltha-rw-core > $WORKSPACE/error-report.log
- extract_errors_go adapter-open-olt >> $WORKSPACE/error-report.log
- extract_errors_python adapter-open-onu >> $WORKSPACE/error-report.log
- extract_errors_python voltha-ofagent >> $WORKSPACE/error-report.log
-
- gzip $WORKSPACE/onos-voltha-combined.log
-
- ## shut down kind-voltha
- cd $WORKSPACE/voltha/kind-voltha
- WAIT_ON_DOWN=y ./voltha down
- '''
- step([$class: 'RobotPublisher',
- disableArchiveOutput: false,
- logFileName: 'RobotLogs/log*.html',
- otherFiles: '',
- outputFileName: 'RobotLogs/output*.xml',
- outputPath: '.',
- passThreshold: 80,
- reportFileName: 'RobotLogs/report*.html',
- unstableThreshold: 0]);
- archiveArtifacts artifacts: '*.log,*.gz'
- }
- }
-}
diff --git a/jjb/pipeline/voltha/master/bbsim-tests.groovy b/jjb/pipeline/voltha/master/bbsim-tests.groovy
index 779c506..8b02a88 100644
--- a/jjb/pipeline/voltha/master/bbsim-tests.groovy
+++ b/jjb/pipeline/voltha/master/bbsim-tests.groovy
@@ -66,15 +66,6 @@
stage('Deploy - '+ name + ' workflow') {
def extraHelmFlags = "${extraHelmFlags} --set global.log_level=DEBUG,onu=1,pon=1 "
- // use harbor as passthrough cache for docker containers
- extraHelmFlags += " --set global.image_registry=mirror.registry.opennetworking.org/ "
- extraHelmFlags += " --set etcd.image.registry=mirror.registry.opennetworking.org "
- extraHelmFlags += " --set kafka.image.registry=mirror.registry.opennetworking.org "
- extraHelmFlags += " --set kafka.zookeper.image.registry=mirror.registry.opennetworking.org "
- extraHelmFlags += " --set onos-classic.image.repository=mirror.registry.opennetworking.org/voltha/voltha-onos "
- extraHelmFlags += " --set onos-classic.atomix.image.repository=mirror.registry.opennetworking.org/atomix/atomix "
- extraHelmFlags += " --set freeradius.images.radius.registry=mirror.registry.opennetworking.org/ "
-
if (gerritProject != "") {
extraHelmFlags = extraHelmFlags + customImageFlags("${gerritProject}")
}
@@ -84,7 +75,12 @@
localCharts = true
}
- volthaDeploy([workflow: name, extraHelmFlags: extraHelmFlags, localCharts: localCharts])
+ volthaDeploy([
+ workflow: name,
+ extraHelmFlags:extraHelmFlags,
+ localCharts: localCharts,
+ dockerRegistry: "mirror.registry.opennetworking.org"
+ ])
// start logging
sh """
mkdir -p $WORKSPACE/${name}
diff --git a/jjb/pipeline/voltha/master/device-management-mock-tests.groovy b/jjb/pipeline/voltha/master/device-management-mock-tests.groovy
new file mode 100644
index 0000000..8a15ac6
--- /dev/null
+++ b/jjb/pipeline/voltha/master/device-management-mock-tests.groovy
@@ -0,0 +1,172 @@
+// Copyright 2017-present Open Networking Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// NOTE we are importing the library even if it's global so that it's
+// easier to change the keywords during a replay
+library identifier: 'cord-jenkins-libraries@master',
+ retriever: modernSCM([
+ $class: 'GitSCMSource',
+ remote: 'https://gerrit.opencord.org/ci-management.git'
+])
+
+def localCharts = false
+
+pipeline {
+
+ /* no label, executor is determined by JJB */
+ agent {
+ label "${params.buildNode}"
+ }
+ options {
+ timeout(time: 90, unit: 'MINUTES')
+ }
+ environment {
+ KUBECONFIG="$HOME/.kube/kind-config-voltha-minimal"
+ PATH="$PATH:$WORKSPACE/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+ ROBOT_MISC_ARGS="-d $WORKSPACE/RobotLogs"
+ }
+
+ stages {
+
+ stage('Download Code') {
+ steps {
+ getVolthaCode([
+ branch: "${branch}",
+ gerritProject: "${gerritProject}",
+ gerritRefspec: "${gerritRefspec}",
+ volthaSystemTestsChange: "${volthaSystemTestsChange}",
+ volthaHelmChartsChange: "${volthaHelmChartsChange}",
+ ])
+ }
+ }
+ stage('Build Redfish Importer Image') {
+ steps {
+ sh """
+ make -C $WORKSPACE/device-management/\$1 DOCKER_REPOSITORY=opencord/ DOCKER_TAG=citest docker-build-importer
+ """
+ }
+ }
+ stage('Build demo_test Image') {
+ steps {
+ sh """
+ make -C $WORKSPACE/device-management/\$1/demo_test DOCKER_REPOSITORY=opencord/ DOCKER_TAG=citest docker-build
+ """
+ }
+ }
+ stage('Build mock-redfish-server Image') {
+ steps {
+ sh """
+ make -C $WORKSPACE/device-management/\$1/mock-redfish-server DOCKER_REPOSITORY=opencord/ DOCKER_TAG=citest docker-build
+ """
+ }
+ }
+ stage('Create K8s Cluster') {
+ steps {
+ createKubernetesCluster([nodes: 3])
+ }
+ }
+ stage('Load image in kind nodes') {
+ steps {
+ loadToKind()
+ }
+ }
+ stage('Deploy Voltha') {
+ steps {
+ script {
+ if (branch != "master" || volthaHelmChartsChange != "") {
+ // if we're using a release or testing changes in the charts, then use the local clone
+ localCharts = true
+ }
+ }
+ volthaDeploy([
+ workflow: "att",
+ extraHelmFlags: extraHelmFlags,
+ dockerRegistry: "mirror.registry.opennetworking.org",
+ localCharts: localCharts,
+ ])
+ // start logging
+ sh """
+ mkdir -p $WORKSPACE/att
+ _TAG=kail-att kail -n infra -n voltha -n default > $WORKSPACE/att/onos-voltha-combined.log &
+ """
+ // forward ONOS and VOLTHA ports
+ sh """
+ _TAG=onos-port-forward kubectl port-forward --address 0.0.0.0 -n infra svc/voltha-infra-onos-classic-hs 8101:8101&
+ _TAG=onos-port-forward kubectl port-forward --address 0.0.0.0 -n infra svc/voltha-infra-onos-classic-hs 8181:8181&
+ _TAG=voltha-port-forward kubectl port-forward --address 0.0.0.0 -n voltha svc/voltha-voltha-api 55555:55555&
+ """
+ }
+ }
+
+ stage('Run E2E Tests') {
+ steps {
+ sh '''
+ mkdir -p $WORKSPACE/RobotLogs
+
+ # tell the kubernetes script to use images tagged citest and pullPolicy:Never
+ sed -i 's/master/citest/g' $WORKSPACE/device-management/kubernetes/deploy*.yaml
+ sed -i 's/imagePullPolicy: Always/imagePullPolicy: Never/g' $WORKSPACE/device-management/kubernetes/deploy*.yaml
+ make -C $WORKSPACE/device-management functional-mock-test || true
+ '''
+ }
+ }
+ }
+
+ post {
+ always {
+ sh '''
+ set +e
+ kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.image}{'\\n'}" | sort | uniq
+ kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.imageID}{'\\n'}" | sort | uniq
+ kubectl get nodes -o wide
+ kubectl get pods -o wide --all-namespaces
+
+ sync
+ pkill kail || true
+
+ ## Pull out errors from log files
+ extract_errors_go() {
+ echo
+ echo "Error summary for $1:"
+ grep $1 $WORKSPACE/att/onos-voltha-combined.log | grep '"level":"error"' | cut -d ' ' -f 2- | jq -r '.msg'
+ echo
+ }
+
+ extract_errors_python() {
+ echo
+ echo "Error summary for $1:"
+ grep $1 $WORKSPACE/att/onos-voltha-combined.log | grep 'ERROR' | cut -d ' ' -f 2-
+ echo
+ }
+
+ extract_errors_go voltha-rw-core > $WORKSPACE/error-report.log
+ extract_errors_go adapter-open-olt >> $WORKSPACE/error-report.log
+ extract_errors_python adapter-open-onu >> $WORKSPACE/error-report.log
+ extract_errors_python voltha-ofagent >> $WORKSPACE/error-report.log
+
+ gzip $WORKSPACE/att/onos-voltha-combined.log
+ '''
+ step([$class: 'RobotPublisher',
+ disableArchiveOutput: false,
+ logFileName: 'RobotLogs/log*.html',
+ otherFiles: '',
+ outputFileName: 'RobotLogs/output*.xml',
+ outputPath: '.',
+ passThreshold: 80,
+ reportFileName: 'RobotLogs/report*.html',
+ unstableThreshold: 0]);
+ archiveArtifacts artifacts: '**/*.log,**/*.gz'
+ }
+ }
+}