SEBA-971 jobs for device-management
Change-Id: If5816b9a0f08afde875a4ea512de109c791c5532
diff --git a/jjb/device-management.yaml b/jjb/device-management.yaml
new file mode 100644
index 0000000..cd8b9f6
--- /dev/null
+++ b/jjb/device-management.yaml
@@ -0,0 +1,252 @@
+---
+# device-management tests
+
+- project:
+ name: device-management-e2e
+
+ project-name: '{name}'
+
+ jobs:
+ # Per-patchset Pod builds on Tucson pod
+ - 'verify_physical_device-management_patchset_manual':
+ name: 'verify_physical_device-management_patchset_manual'
+ testvm: 'tucson-pod'
+ config-pod: 'tucson-pod'
+ branch: 'master'
+ oltDebVersion: 'openolt_asfvolt16-2.3.0-bc6e0853e0e8bf6bd7e4223d4a7ee0dd35ce634d.deb'
+ profile: 'Default'
+ withPatchset: true
+
+- job-template:
+ id: 'device-management-patch-test'
+ name: 'verify_{project}_sanity-test{name-extension}'
+ extra-helm-flags: ''
+ skip-vote: false
+
+ description: |
+ <!-- Managed by Jenkins Job Builder -->
+ Created by {id} job-template from ci-management/jjb/device-management.yaml <br /><br />
+ Validation for device-management using mock redfish servers
+
+ properties:
+ - cord-infra-properties:
+ build-days-to-keep: '{build-days-to-keep}'
+ artifact-num-to-keep: '{artifact-num-to-keep}'
+
+ wrappers:
+ - lf-infra-wrappers:
+ build-timeout: '{build-timeout}'
+ jenkins-ssh-credential: '{jenkins-ssh-credential}'
+
+ parameters:
+ - string:
+ name: buildNode
+ default: 'ubuntu16.04-basebuild-4c-8g'
+ description: 'Name of the Jenkins node to run the job on'
+
+ - string:
+ name: manifestUrl
+ default: '{gerrit-server-url}/{voltha-test-manifest-repo}'
+ description: 'URL to the repo manifest'
+
+ - string:
+ name: manifestBranch
+ default: 'master'
+ description: 'Name of the repo branch to use'
+
+ - string:
+ name: gerritProject
+ default: '$GERRIT_PROJECT'
+ description: 'Name of the Gerrit project'
+
+ - string:
+ name: gerritChangeNumber
+ default: '$GERRIT_CHANGE_NUMBER'
+ description: 'Changeset number in Gerrit'
+
+ - string:
+ name: gerritPatchsetNumber
+ default: '$GERRIT_PATCHSET_NUMBER'
+ description: 'PatchSet number in Gerrit'
+
+ - string:
+ name: extraHelmFlags
+ default: '{extra-helm-flags}'
+ description: 'Helm flags to pass to ./voltha up'
+
+ project-type: pipeline
+ concurrent: true
+
+ dsl: !include-raw-escape: pipeline/{pipeline-script}
+
+ triggers:
+ - gerrit:
+ server-name: '{gerrit-server-name}'
+ dependency-jobs: '{dependency-jobs}'
+ silent-start: true
+ trigger-on:
+ - patchset-created-event:
+ exclude-drafts: true
+ exclude-trivial-rebase: false
+ exclude-no-code-change: true
+ - draft-published-event
+ - comment-added-contains-event:
+ comment-contains-value: '(?i)^.*recheck$'
+ projects:
+ - project-compare-type: REG_EXP
+ project-pattern: '^device-management$'
+ branches:
+ - branch-compare-type: REG_EXP
+ branch-pattern: '{branch-regexp}'
+ file-paths:
+ - compare-type: REG_EXP
+ pattern: '{all-files-regexp}'
+ skip-vote:
+ successful: '{skip-vote}'
+ failed: '{skip-vote}'
+ unstable: '{skip-vote}'
+ notbuilt: '{skip-vote}'
+
+# POD Per Patchset Pipeline Jobs
+
+- job-template:
+ name: '{name}'
+ id: verify_physical_device-management_patchset_manual
+ description: |
+ <!-- Managed by Jenkins Job Builder -->
+ Automated build on POD {config-pod} using {pipeline-script} <br /><br />
+ Created from job-template {id} from ci-management/jjb/device-management.yaml <br />
+ Created by Andy Bavier, andy@opennetworking.org <br />
+ Copyright (c) 2019 Open Networking Foundation (ONF)
+ sandbox: true
+ pipeline-script: 'device-management-physical-build-and-tests.groovy'
+
+ properties:
+ - cord-infra-properties:
+ build-days-to-keep: '{build-days-to-keep}'
+ artifact-num-to-keep: '{artifact-num-to-keep}'
+
+ parameters:
+ - string:
+ name: buildNode
+ default: '{testvm}'
+ description: 'Pod management node'
+
+ - string:
+ name: manifestUrl
+ default: '{gerrit-server-url}/{voltha-test-manifest-repo}'
+ description: 'URL to the repo manifest'
+
+ - string:
+ name: manifestBranch
+ default: master
+ description: 'Name of the repo branch to use'
+
+ - string:
+ name: gerritProject
+ default: '$GERRIT_PROJECT'
+ description: 'Name of the Gerrit project'
+
+ - string:
+ name: gerritChangeNumber
+ default: '$GERRIT_CHANGE_NUMBER'
+ description: 'Changeset number in Gerrit'
+
+ - string:
+ name: gerritPatchsetNumber
+ default: '$GERRIT_PATCHSET_NUMBER'
+ description: 'PatchSet number in Gerrit'
+
+ - string:
+ name: cordRepoUrl
+ default: '{gerrit-server-url}'
+ description: 'The URL of the CORD Project repository'
+
+ - string:
+ name: podName
+ default: '{config-pod}'
+
+ - string:
+ name: deploymentConfigFile
+ default: 'pod-configs/deployment-configs/{config-pod}.yaml'
+ description: 'Path of deployment config file'
+
+ - string:
+ name: kindVolthaValuesFile
+ default: 'pod-configs/kubernetes-configs/voltha/{config-pod}.yml'
+ description: 'Path of kind-voltha values override file'
+
+ - string:
+ name: sadisConfigFile
+ default: 'voltha/voltha-system-tests/tests/data/{config-pod}-sadis.json'
+ description: 'Path of SADIS config to load'
+
+ - string:
+ name: localConfigDir
+ default: null
+ description: 'If specified, config file paths are relative to this dir; otherwise $WORKSPACE'
+
+ - string:
+ name: configRepo
+ default: 'pod-configs'
+ description: 'A repository containing the config files, will be checked out if specified'
+
+ - string:
+ name: oltDebVersion
+ default: '{oltDebVersion}'
+ description: 'OLT Software version to install'
+
+ - string:
+ name: branch
+ default: '{branch}'
+
+ - string:
+ name: profile
+ default: '{profile}'
+ description: 'Technology Profile pushed to the ETCD'
+
+ - string:
+ name: notificationEmail
+ default: 'andy@opennetworking.org'
+ description: ''
+
+ - bool:
+ name: reinstallOlt
+ default: true
+ description: "Re-install OLT software"
+
+ - bool:
+ name: withPatchset
+ default: true
+ description: "Build with Gerrit patchset"
+
+ - string:
+ name: extraRobotArgs
+ default: '-i sanity'
+ description: 'Arguments to pass to robot'
+
+ project-type: pipeline
+ concurrent: true
+
+ dsl: !include-raw-escape: pipeline/{pipeline-script}
+
+ triggers:
+ - gerrit:
+ server-name: '{gerrit-server-name}'
+ dependency-jobs: '{dependency-jobs}'
+ silent-start: false
+ successful-message: "PASSED hardware test"
+ failure-message: "FAILED hardware test"
+ unstable-message: "UNSTABLE hardware test"
+ trigger-on:
+ - comment-added-contains-event:
+ comment-contains-value: '^hardware test$'
+ - comment-added-contains-event:
+ comment-contains-value: '^hardware test with delay$'
+ projects:
+ - project-compare-type: REG_EXP
+ project-pattern: '^device-management$'
+ branches:
+ - branch-compare-type: PLAIN
+ branch-pattern: 'master'
+
diff --git a/jjb/pipeline/device-management-mock-tests.groovy b/jjb/pipeline/device-management-mock-tests.groovy
new file mode 100644
index 0000000..b3bd20d
--- /dev/null
+++ b/jjb/pipeline/device-management-mock-tests.groovy
@@ -0,0 +1,181 @@
+// Copyright 2017-present Open Networking Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// voltha-2.x e2e tests
+// uses kind-voltha to deploy voltha-2.X
+// uses bbsim to simulate OLT/ONUs
+
+pipeline {
+
+ /* no label, executor is determined by JJB */
+ agent {
+ label "${params.buildNode}"
+ }
+ options {
+ timeout(time: 90, unit: 'MINUTES')
+ }
+ environment {
+ KUBECONFIG="$HOME/.kube/kind-config-voltha-minimal"
+ VOLTCONFIG="$HOME/.volt/config-minimal"
+ PATH="$WORKSPACE/kind-voltha/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+ TYPE="minimal"
+ FANCY=0
+ WITH_SIM_ADAPTERS="n"
+ WITH_RADIUS="y"
+ WITH_BBSIM="y"
+ DEPLOY_K8S="y"
+ VOLTHA_LOG_LEVEL="DEBUG"
+ CONFIG_SADIS="n"
+ ROBOT_MISC_ARGS="-d $WORKSPACE/RobotLogs"
+ }
+
+ stages {
+
+ stage('Repo') {
+ steps {
+ step([$class: 'WsCleanup'])
+ checkout(changelog: false, \
+ poll: false,
+ scm: [$class: 'RepoScm', \
+ manifestRepositoryUrl: "${params.manifestUrl}", \
+ manifestBranch: "${params.manifestBranch}", \
+ currentBranch: true, \
+ destinationDir: 'voltha', \
+ forceSync: true,
+ resetFirst: true, \
+ quiet: true, \
+ jobs: 4, \
+ showAllChanges: true] \
+ )
+ }
+ }
+ stage('Patch') {
+ steps {
+ sh """
+ pushd voltha
+ repo download "${gerritProject}" "${gerritChangeNumber}/${gerritPatchsetNumber}"
+ popd
+ """
+ }
+ }
+ stage('Create K8s Cluster') {
+ steps {
+ sh """
+ git clone https://github.com/ciena/kind-voltha.git
+ cd kind-voltha/
+ JUST_K8S=y ./voltha up
+ bash <( curl -sfL https://raw.githubusercontent.com/boz/kail/master/godownloader.sh) -b "$WORKSPACE/kind-voltha/bin"
+ """
+ }
+ }
+
+ stage('Build Images') {
+ steps {
+ sh """
+ make-local () {
+ make -C $WORKSPACE/voltha/\$1 DOCKER_REPOSITORY=voltha/ DOCKER_TAG=citest docker-build
+ }
+ """
+ }
+ }
+
+ stage('Push Images') {
+ steps {
+ sh '''
+ export GOROOT=/usr/local/go
+ export GOPATH=\$(pwd)
+ docker images | grep citest
+ for image in \$(docker images -f "reference=*/*citest" --format "{{.Repository}}"); do echo "Pushing \$image to nodes"; kind load docker-image \$image:citest --name voltha-\$TYPE --nodes voltha-\$TYPE-worker,voltha-\$TYPE-worker2; done
+ '''
+ }
+ }
+ stage('Deploy Voltha') {
+ steps {
+ sh '''
+ export EXTRA_HELM_FLAGS="--set log_agent.enabled=False ${extraHelmFlags} "
+
+ cd $WORKSPACE/kind-voltha/
+ echo \$EXTRA_HELM_FLAGS
+ kail -n voltha -n default > $WORKSPACE/onos-voltha-combined.log &
+ ./voltha up
+ '''
+ }
+ }
+
+ stage('Run E2E Tests') {
+ steps {
+ sh '''
+ mkdir -p $WORKSPACE/RobotLogs
+
+ # tell the kubernetes script to use images tagged citest and pullPolicy:Never
+ sed -i 's/master/citest/g' $WORKSPACE/voltha/device-management/kubernetes/deploy-redfish-importer.yaml
+ sed -i 's/imagePullPolicy: Always/imagePullPolicy: Never/g' $WORKSPACE/voltha/device-management/kubernetes/deploy-redfish-importer.yaml
+ make -C $WORKSPACE/voltha/device-management functional-mock-test || true
+ '''
+ }
+ }
+ }
+
+ post {
+ always {
+ sh '''
+ set +e
+ cp $WORKSPACE/kind-voltha/install-minimal.log $WORKSPACE/
+ kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.image}{'\\t'}{.imageID}{'\\n'}" | sort | uniq -c
+ kubectl get nodes -o wide
+ kubectl get pods -o wide
+ kubectl get pods -n voltha -o wide
+
+ sync
+ pkill kail || true
+
+ ## Pull out errors from log files
+ extract_errors_go() {
+ echo
+ echo "Error summary for $1:"
+ grep $1 $WORKSPACE/onos-voltha-combined.log | grep '"level":"error"' | cut -d ' ' -f 2- | jq -r '.msg'
+ echo
+ }
+
+ extract_errors_python() {
+ echo
+ echo "Error summary for $1:"
+ grep $1 $WORKSPACE/onos-voltha-combined.log | grep 'ERROR' | cut -d ' ' -f 2-
+ echo
+ }
+
+ extract_errors_go voltha-rw-core > $WORKSPACE/error-report.log
+ extract_errors_go adapter-open-olt >> $WORKSPACE/error-report.log
+ extract_errors_python adapter-open-onu >> $WORKSPACE/error-report.log
+ extract_errors_python voltha-ofagent >> $WORKSPACE/error-report.log
+
+ gzip $WORKSPACE/onos-voltha-combined.log
+
+ ## shut down kind-voltha
+ cd $WORKSPACE/kind-voltha
+ WAIT_ON_DOWN=y ./voltha down
+ '''
+ step([$class: 'RobotPublisher',
+ disableArchiveOutput: false,
+ logFileName: 'RobotLogs/log*.html',
+ otherFiles: '',
+ outputFileName: 'RobotLogs/output*.xml',
+ outputPath: '.',
+ passThreshold: 80,
+ reportFileName: 'RobotLogs/report*.html',
+ unstableThreshold: 0]);
+ archiveArtifacts artifacts: '*.log,*.gz'
+ }
+ }
+}
diff --git a/jjb/pipeline/device-management-physical-build-and-tests.groovy b/jjb/pipeline/device-management-physical-build-and-tests.groovy
new file mode 100644
index 0000000..1a50e9e
--- /dev/null
+++ b/jjb/pipeline/device-management-physical-build-and-tests.groovy
@@ -0,0 +1,345 @@
+// Copyright 2019-present Open Networking Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// deploy VOLTHA built from patchset on a physical pod and run e2e test
+// uses kind-voltha to deploy voltha-2.X
+
+// Need this so that deployment_config has global scope when it's read later
+deployment_config = null
+localDeploymentConfigFile = null
+localKindVolthaValuesFile = null
+localSadisConfigFile = null
+
+// The pipeline assumes these variables are always defined
+if ( ! params.withPatchset ) {
+ GERRIT_EVENT_COMMENT_TEXT = ""
+ GERRIT_PROJECT = ""
+ GERRIT_CHANGE_NUMBER = ""
+ GERRIT_PATCHSET_NUMBER = ""
+}
+
+pipeline {
+
+ /* no label, executor is determined by JJB */
+ agent {
+ label "${params.buildNode}"
+ }
+ options {
+ timeout(time: 90, unit: 'MINUTES')
+ }
+
+ environment {
+ KUBECONFIG="$HOME/.kube/kind-config-voltha-minimal"
+ VOLTCONFIG="$HOME/.volt/config-minimal"
+ PATH="$WORKSPACE/kind-voltha/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+ TYPE="minimal"
+ FANCY=0
+ //VOL-2194 ONOS SSH and REST ports hardcoded to 30115/30120 in tests
+ ONOS_SSH_PORT=30115
+ ONOS_API_PORT=30120
+ }
+
+ stages {
+ stage ('Initialize') {
+ steps {
+ sh returnStdout: false, script: """
+ test -e $WORKSPACE/kind-voltha/voltha && cd $WORKSPACE/kind-voltha && ./voltha down
+ cd $WORKSPACE
+ rm -rf $WORKSPACE/*
+ """
+ script {
+ if (env.configRepo && ! env.localConfigDir) {
+ env.localConfigDir = "$WORKSPACE"
+ sh returnStdout: false, script: "git clone -b master ${cordRepoUrl}/${configRepo}"
+ }
+ localDeploymentConfigFile = "${env.localConfigDir}/${params.deploymentConfigFile}"
+ localKindVolthaValuesFile = "${env.localConfigDir}/${params.kindVolthaValuesFile}"
+ localSadisConfigFile = "${env.localConfigDir}/${params.sadisConfigFile}"
+ }
+ }
+ }
+
+ stage('Repo') {
+ steps {
+ checkout(changelog: true,
+ poll: false,
+ scm: [$class: 'RepoScm',
+ manifestRepositoryUrl: "${params.manifestUrl}",
+ manifestBranch: "${params.manifestBranch}",
+ currentBranch: true,
+ destinationDir: 'voltha',
+ forceSync: true,
+ resetFirst: true,
+ quiet: true,
+ jobs: 4,
+ showAllChanges: true]
+ )
+ }
+ }
+
+ stage('Get Patch') {
+ when {
+ expression { params.withPatchset }
+ }
+ steps {
+ sh returnStdout: false, script: """
+ cd voltha
+ repo download "${gerritProject}" "${gerritChangeNumber}/${gerritPatchsetNumber}"
+ """
+ }
+ }
+
+ stage('Check config files') {
+ steps {
+ script {
+ try {
+ deployment_config = readYaml file: "${localDeploymentConfigFile}"
+ } catch (err) {
+ echo "Error reading ${localDeploymentConfigFile}"
+ throw err
+ }
+ sh returnStdout: false, script: """
+ if [ ! -e ${localKindVolthaValuesFile} ]; then echo "${localKindVolthaValuesFile} not found"; exit 1; fi
+ if [ ! -e ${localSadisConfigFile} ]; then echo "${localSadisConfigFile} not found"; exit 1; fi
+ """
+ }
+ }
+ }
+
+ stage('Create KinD Cluster') {
+ steps {
+ sh returnStdout: false, script: """
+ git clone https://github.com/ciena/kind-voltha.git
+ cd kind-voltha/
+ JUST_K8S=y ./voltha up
+ """
+ }
+ }
+
+ stage('Build and Push Images') {
+ when {
+ expression { params.withPatchset }
+ }
+ steps {
+ sh returnStdout: false, script: """
+ make -C $WORKSPACE/voltha/\$1 DOCKER_REPOSITORY=voltha/ DOCKER_TAG=citest docker-build
+ docker images | grep citest
+ for image in \$(docker images -f "reference=*/*citest" --format "{{.Repository}}"); do echo "Pushing \$image to nodes"; kind load docker-image \$image:citest --name voltha-\$TYPE --nodes voltha-\$TYPE-worker,voltha-\$TYPE-worker2; done
+ """
+ }
+ }
+
+ stage('Deploy Voltha') {
+ environment {
+ WITH_SIM_ADAPTERS="n"
+ WITH_RADIUS="y"
+ DEPLOY_K8S="n"
+ VOLTHA_LOG_LEVEL="debug"
+ }
+ steps {
+ script {
+ sh returnStdout: false, script: """
+ export EXTRA_HELM_FLAGS='--set log_agent.enabled=False -f ${localKindVolthaValuesFile} '
+
+ cd $WORKSPACE/kind-voltha/
+ echo \$EXTRA_HELM_FLAGS
+ kail -n voltha -n default > $WORKSPACE/onos-voltha-combined.log &
+ ./voltha up
+ """
+ }
+ }
+ }
+
+ stage('Deploy Kafka Dump Chart') {
+ steps {
+ script {
+ sh returnStdout: false, script: """
+ helm repo add cord https://charts.opencord.org
+ helm repo update
+ helm install -n voltha-kafka-dump cord/voltha-kafka-dump
+ """
+ }
+ }
+ }
+
+ stage('Push Tech-Profile') {
+ when {
+ expression { params.profile != "Default" }
+ }
+ steps {
+ sh returnStdout: false, script: """
+ etcd_container=\$(kubectl get pods -n voltha | grep voltha-etcd-cluster | awk 'NR==1{print \$1}')
+ kubectl cp $WORKSPACE/voltha/voltha-system-tests/tests/data/TechProfile-${profile}.json voltha/\$etcd_container:/tmp/flexpod.json
+ kubectl exec -it \$etcd_container -n voltha -- /bin/sh -c 'cat /tmp/flexpod.json | ETCDCTL_API=3 etcdctl put service/voltha/technology_profiles/XGS-PON/64'
+ """
+ }
+ }
+
+ stage('Push Sadis-config') {
+ steps {
+ sh returnStdout: false, script: """
+ curl -sSL --user karaf:karaf -X POST -H Content-Type:application/json http://${deployment_config.nodes[0].ip}:$ONOS_API_PORT/onos/v1/network/configuration --data @${localSadisConfigFile}
+ """
+ }
+ }
+
+ stage('Reinstall OLT software') {
+ when {
+ expression { params.reinstallOlt }
+ }
+ steps {
+ script {
+ deployment_config.olts.each { olt ->
+ sh returnStdout: false, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'service openolt stop' || true"
+ sh returnStdout: false, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'killall dev_mgmt_daemon' || true"
+ sh returnStdout: false, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'dpkg --remove asfvolt16 && dpkg --purge asfvolt16'"
+ waitUntil {
+ olt_sw_present = sh returnStdout: true, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'dpkg --list | grep asfvolt16 | wc -l'"
+ return olt_sw_present.toInteger() == 0
+ }
+ sh returnStdout: false, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'dpkg --install ${oltDebVersion}'"
+ waitUntil {
+ olt_sw_present = sh returnStdout: true, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'dpkg --list | grep asfvolt16 | wc -l'"
+ return olt_sw_present.toInteger() == 1
+ }
+ if ( olt.fortygig ) {
+ // If the OLT is connected to a 40G switch interface, set the NNI port to be downgraded
+ sh returnStdout: false, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'echo port ce128 sp=40000 >> /broadcom/qax.soc ; /opt/bcm68620/svk_init.sh'"
+ }
+ }
+ }
+ }
+ }
+
+ stage('Restart OLT processes') {
+ steps {
+ script {
+ deployment_config.olts.each { olt ->
+ sh returnStdout: false, script: """
+ ssh-keyscan -H ${olt.ip} >> ~/.ssh/known_hosts
+ sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'service openolt stop' || true
+ sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'killall dev_mgmt_daemon' || true
+ sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'rm -f /var/log/openolt.log'
+ sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'rm -f /var/log/dev_mgmt_daemon.log'
+ sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'service dev_mgmt_daemon start &'
+ sleep 5
+ sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'service openolt start &'
+ """
+ waitUntil {
+ onu_discovered = sh returnStdout: true, script: "sshpass -p ${olt.pass} ssh -l ${olt.user} ${olt.ip} 'grep \"onu discover indication\" /var/log/openolt.log | wc -l'"
+ return onu_discovered.toInteger() > 0
+ }
+ }
+ }
+ }
+ }
+
+ stage('Run E2E Tests') {
+ environment {
+ ROBOT_CONFIG_FILE="${localDeploymentConfigFile}"
+ ROBOT_MISC_ARGS="${params.extraRobotArgs} --removekeywords wuks -d $WORKSPACE/RobotLogs -v container_log_dir:$WORKSPACE "
+ ROBOT_FILE="Voltha_PODTests.robot"
+ }
+ steps {
+ sh returnStdout: false, script: """
+ cd voltha
+ git clone -b ${branch} ${cordRepoUrl}/cord-tester
+ mkdir -p $WORKSPACE/RobotLogs
+
+ # tell the kubernetes script to use images tagged citest and pullPolicy:Never
+ sed -i 's/master/citest/g' $WORKSPACE/voltha/device-management/kubernetes/deploy-redfish-importer.yaml
+ sed -i 's/imagePullPolicy: Always/imagePullPolicy: Never/g' $WORKSPACE/voltha/device-management/kubernetes/deploy-redfish-importer.yaml
+ make -C $WORKSPACE/voltha/device-management functional-mock-test || true
+ """
+ }
+ }
+
+ stage('After-Test Delay') {
+ when {
+ expression { params.withPatchset }
+ }
+ steps {
+ sh returnStdout: false, script: """
+ # Note: Gerrit comment text will be prefixed by "Patch set n:" and a blank line
+ REGEX="hardware test with delay\$"
+ [[ "$GERRIT_EVENT_COMMENT_TEXT" =~ \$REGEX ]] && sleep 10m || true
+ """
+ }
+ }
+ }
+
+ post {
+ always {
+ sh returnStdout: false, script: '''
+ set +e
+ cp kind-voltha/install-minimal.log $WORKSPACE/
+ kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.image}{'\\t'}{.imageID}{'\\n'}" | sort | uniq -c
+ kubectl get nodes -o wide
+ kubectl get pods -o wide
+ kubectl get pods -n voltha -o wide
+
+ sync
+ pkill kail || true
+
+ ## Pull out errors from log files
+ extract_errors_go() {
+ echo
+ echo "Error summary for $1:"
+ grep $1 $WORKSPACE/onos-voltha-combined.log | grep '"level":"error"' | cut -d ' ' -f 2- | jq -r '.msg'
+ echo
+ }
+
+ extract_errors_python() {
+ echo
+ echo "Error summary for $1:"
+ grep $1 $WORKSPACE/onos-voltha-combined.log | grep 'ERROR' | cut -d ' ' -f 2-
+ echo
+ }
+
+ extract_errors_go voltha-rw-core > $WORKSPACE/error-report.log
+ extract_errors_go adapter-open-olt >> $WORKSPACE/error-report.log
+ extract_errors_python adapter-open-onu >> $WORKSPACE/error-report.log
+ extract_errors_python voltha-ofagent >> $WORKSPACE/error-report.log
+
+ gzip $WORKSPACE/onos-voltha-combined.log
+
+ ## collect events, the chart should be running by now
+ kubectl get pods | grep -i voltha-kafka-dump | grep -i running
+ if [[ $? == 0 ]]; then
+ kubectl exec -it `kubectl get pods | grep -i voltha-kafka-dump | grep -i running | cut -f1 -d " "` ./voltha-dump-events.sh > $WORKSPACE/voltha-events.log
+ fi
+ '''
+ script {
+ deployment_config.olts.each { olt ->
+ sh returnStdout: false, script: """
+ sshpass -p ${olt.pass} scp ${olt.user}@${olt.ip}:/var/log/openolt.log $WORKSPACE/openolt-${olt.ip}.log || true
+ sed -i 's/\\x1b\\[[0-9;]*[a-zA-Z]//g' $WORKSPACE/openolt-${olt.ip}.log # Remove escape sequences
+ sshpass -p ${olt.pass} scp ${olt.user}@${olt.ip}:/var/log/dev_mgmt_daemon.log $WORKSPACE/dev_mgmt_daemon-${olt.ip}.log || true
+ sed -i 's/\\x1b\\[[0-9;]*[a-zA-Z]//g' $WORKSPACE/dev_mgmt_daemon-${olt.ip}.log # Remove escape sequences
+ """
+ }
+ }
+ step([$class: 'RobotPublisher',
+ disableArchiveOutput: false,
+ logFileName: 'RobotLogs/log*.html',
+ otherFiles: '',
+ outputFileName: 'RobotLogs/output*.xml',
+ outputPath: '.',
+ passThreshold: 100,
+ reportFileName: 'RobotLogs/report*.html',
+ unstableThreshold: 0]);
+ archiveArtifacts artifacts: '*.log,*.gz'
+ }
+ }
+}
diff --git a/jjb/verify/device-management.yaml b/jjb/verify/device-management.yaml
index 193c78c..10cfec2 100644
--- a/jjb/verify/device-management.yaml
+++ b/jjb/verify/device-management.yaml
@@ -19,6 +19,8 @@
dependency-jobs: 'verify_device-management_licensed'
- 'make-unit-test':
junit-allow-empty-results: true
+ - 'device-management-patch-test':
+ pipeline-script: 'device-management-mock-tests.groovy'
- job-group:
name: 'publish-device-management-jobs'