Kailash | 8b8de74 | 2019-07-30 08:54:51 -0700 | [diff] [blame] | 1 | // Copyright 2017-present Open Networking Foundation |
| 2 | // |
| 3 | // Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | // you may not use this file except in compliance with the License. |
| 5 | // You may obtain a copy of the License at |
| 6 | // |
| 7 | // http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | // |
| 9 | // Unless required by applicable law or agreed to in writing, software |
| 10 | // distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | // See the License for the specific language governing permissions and |
| 13 | // limitations under the License. |
| 14 | |
| 15 | // voltha-2.x e2e tests |
| 16 | // uses kind-voltha to deploy voltha-2.X |
| 17 | // uses bbsim to simulate OLT/ONUs |
| 18 | |
| 19 | pipeline { |
| 20 | |
| 21 | /* no label, executor is determined by JJB */ |
| 22 | agent { |
Zack Williams | b329208 | 2019-10-11 17:15:18 -0700 | [diff] [blame] | 23 | label "${params.buildNode}" |
Kailash | 8b8de74 | 2019-07-30 08:54:51 -0700 | [diff] [blame] | 24 | } |
Kailash | 4b323ff | 2019-08-02 17:52:49 -0700 | [diff] [blame] | 25 | options { |
Andrea Campanella | 2f21f98 | 2020-09-11 13:26:40 +0200 | [diff] [blame] | 26 | timeout(time: 60, unit: 'MINUTES') |
Kailash | 4b323ff | 2019-08-02 17:52:49 -0700 | [diff] [blame] | 27 | } |
Andy Bavier | 218385e | 2019-11-13 09:51:38 -0700 | [diff] [blame] | 28 | environment { |
| 29 | KUBECONFIG="$HOME/.kube/kind-config-voltha-minimal" |
| 30 | VOLTCONFIG="$HOME/.volt/config-minimal" |
Andy Bavier | 292121d | 2020-04-09 12:50:35 -0700 | [diff] [blame] | 31 | PATH="$HOME/kind-voltha/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" |
Andy Bavier | 218385e | 2019-11-13 09:51:38 -0700 | [diff] [blame] | 32 | TYPE="minimal" |
| 33 | FANCY=0 |
Hung-Wei Chiu | f284d3f | 2020-04-08 11:38:13 -0700 | [diff] [blame] | 34 | WITH_SIM_ADAPTERS="no" |
| 35 | WITH_RADIUS="yes" |
| 36 | WITH_BBSIM="yes" |
| 37 | DEPLOY_K8S="yes" |
Andy Bavier | 218385e | 2019-11-13 09:51:38 -0700 | [diff] [blame] | 38 | VOLTHA_LOG_LEVEL="DEBUG" |
Andrea Campanella | fe95038 | 2020-09-10 18:12:11 +0200 | [diff] [blame] | 39 | CONFIG_SADIS="external" |
| 40 | BBSIM_CFG="configs/bbsim-sadis-att.yaml" |
Andy Bavier | d987004 | 2020-06-23 13:16:47 -0700 | [diff] [blame] | 41 | ROBOT_MISC_ARGS="${params.extraRobotArgs} -d $WORKSPACE/RobotLogs -e PowerSwitch" |
pierventre | f48ebc1 | 2020-07-17 16:01:32 +0200 | [diff] [blame] | 42 | KARAF_HOME="${params.karafHome}" |
| 43 | DIAGS_PROFILE="VOLTHA_PROFILE" |
Andy Bavier | 218385e | 2019-11-13 09:51:38 -0700 | [diff] [blame] | 44 | } |
Kailash | 8b8de74 | 2019-07-30 08:54:51 -0700 | [diff] [blame] | 45 | stages { |
| 46 | |
Andy Bavier | 4c8b323 | 2019-11-22 16:17:47 -0700 | [diff] [blame] | 47 | stage('Repo') { |
| 48 | steps { |
Andy Bavier | 2c9ce04 | 2019-11-23 07:36:31 -0700 | [diff] [blame] | 49 | step([$class: 'WsCleanup']) |
Andy Bavier | 4c8b323 | 2019-11-22 16:17:47 -0700 | [diff] [blame] | 50 | checkout(changelog: true, |
| 51 | poll: false, |
| 52 | scm: [$class: 'RepoScm', |
| 53 | manifestRepositoryUrl: "${params.manifestUrl}", |
Andy Bavier | 6dfaf7e | 2020-03-30 14:52:59 -0700 | [diff] [blame] | 54 | manifestBranch: "${params.branch}", |
Andy Bavier | 4c8b323 | 2019-11-22 16:17:47 -0700 | [diff] [blame] | 55 | currentBranch: true, |
| 56 | destinationDir: 'voltha', |
| 57 | forceSync: true, |
| 58 | resetFirst: true, |
| 59 | quiet: true, |
| 60 | jobs: 4, |
| 61 | showAllChanges: true] |
| 62 | ) |
| 63 | } |
| 64 | } |
| 65 | |
Kailash | 8b8de74 | 2019-07-30 08:54:51 -0700 | [diff] [blame] | 66 | stage('Download kind-voltha') { |
| 67 | steps { |
| 68 | sh """ |
Andy Bavier | 36c688c | 2020-03-17 07:32:30 -0700 | [diff] [blame] | 69 | cd $HOME |
Andy Bavier | b5c8caf | 2020-04-06 14:12:07 -0700 | [diff] [blame] | 70 | [ -d kind-voltha ] || git clone https://gerrit.opencord.org/kind-voltha |
Andy Bavier | 2e00930 | 2020-03-19 08:46:13 -0700 | [diff] [blame] | 71 | rm -rf $HOME/kind-voltha/scripts/logger |
Andy Bavier | 36c688c | 2020-03-17 07:32:30 -0700 | [diff] [blame] | 72 | cd $HOME/kind-voltha |
| 73 | git pull |
Kailash | 8b8de74 | 2019-07-30 08:54:51 -0700 | [diff] [blame] | 74 | """ |
| 75 | } |
| 76 | } |
| 77 | |
| 78 | stage('Deploy Voltha') { |
| 79 | steps { |
| 80 | sh """ |
Andy Bavier | dc8bc2c | 2020-03-29 16:25:02 -0700 | [diff] [blame] | 81 | export EXTRA_HELM_FLAGS="" |
Andy Bavier | 6dfaf7e | 2020-03-30 14:52:59 -0700 | [diff] [blame] | 82 | if [ "${branch}" != "master" ]; then |
| 83 | echo "on branch: ${branch}, sourcing kind-voltha/releases/${branch}" |
| 84 | source "$HOME/kind-voltha/releases/${branch}" |
Andy Bavier | 10926f1 | 2020-03-27 16:53:43 -0700 | [diff] [blame] | 85 | else |
| 86 | echo "on master, using default settings for kind-voltha" |
| 87 | fi |
| 88 | |
Hung-Wei Chiu | f284d3f | 2020-04-08 11:38:13 -0700 | [diff] [blame] | 89 | if [ "${workFlow}" == "DT" ]; then |
| 90 | export WITH_DHCP=no |
| 91 | export WITH_IGMP=no |
| 92 | export WITH_EAPOL=no |
| 93 | export WITH_RADIUS=no |
Andrea Campanella | fe95038 | 2020-09-10 18:12:11 +0200 | [diff] [blame] | 94 | export BBSIM_CFG="configs/bbsim-sadis-dt.yaml" |
Hung-Wei Chiu | f284d3f | 2020-04-08 11:38:13 -0700 | [diff] [blame] | 95 | fi |
| 96 | |
Andy Bavier | 10926f1 | 2020-03-27 16:53:43 -0700 | [diff] [blame] | 97 | EXTRA_HELM_FLAGS+="--set log_agent.enabled=False ${params.extraHelmFlags} " |
| 98 | |
Andy Bavier | 36c688c | 2020-03-17 07:32:30 -0700 | [diff] [blame] | 99 | cd $HOME/kind-voltha/ |
| 100 | WAIT_ON_DOWN=y DEPLOY_K8S=n ./voltha down || ./voltha down |
Andy Bavier | 218385e | 2019-11-13 09:51:38 -0700 | [diff] [blame] | 101 | ./voltha up |
Kailash | 8b8de74 | 2019-07-30 08:54:51 -0700 | [diff] [blame] | 102 | """ |
| 103 | } |
| 104 | } |
| 105 | |
Kailash | 8b8de74 | 2019-07-30 08:54:51 -0700 | [diff] [blame] | 106 | stage('Run E2E Tests') { |
| 107 | steps { |
| 108 | sh ''' |
Andy Bavier | 4af0272 | 2020-01-15 10:24:24 -0700 | [diff] [blame] | 109 | set +e |
Andy Bavier | 2c9ce04 | 2019-11-23 07:36:31 -0700 | [diff] [blame] | 110 | mkdir -p $WORKSPACE/RobotLogs |
Andy Bavier | 3c1ccca | 2020-02-12 16:50:04 -0700 | [diff] [blame] | 111 | |
Andy Bavier | 36c688c | 2020-03-17 07:32:30 -0700 | [diff] [blame] | 112 | cd $HOME/kind-voltha/scripts |
Andy Bavier | 3c1ccca | 2020-02-12 16:50:04 -0700 | [diff] [blame] | 113 | ./log-collector.sh > /dev/null & |
Andy Bavier | 7afb11c | 2020-02-13 11:25:20 -0700 | [diff] [blame] | 114 | ./log-combine.sh > /dev/null & |
Andy Bavier | 3c1ccca | 2020-02-12 16:50:04 -0700 | [diff] [blame] | 115 | |
Andy Bavier | 6612f07 | 2020-04-16 10:22:19 -0700 | [diff] [blame] | 116 | make -C $WORKSPACE/voltha/voltha-system-tests ${makeTarget} || true |
Kailash | 8b8de74 | 2019-07-30 08:54:51 -0700 | [diff] [blame] | 117 | ''' |
| 118 | } |
| 119 | } |
| 120 | } |
| 121 | |
| 122 | post { |
| 123 | always { |
| 124 | sh ''' |
Andy Bavier | 218385e | 2019-11-13 09:51:38 -0700 | [diff] [blame] | 125 | set +e |
Andy Bavier | 36c688c | 2020-03-17 07:32:30 -0700 | [diff] [blame] | 126 | cp $HOME/kind-voltha/install-minimal.log $WORKSPACE/ |
Matteo Scandolo | 97b1257 | 2020-04-13 12:44:46 -0700 | [diff] [blame] | 127 | kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.image}{'\\n'}" | sort | uniq |
| 128 | kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.imageID}{'\\n'}" | sort | uniq |
Kailash | 887a296 | 2019-08-14 13:26:33 -0700 | [diff] [blame] | 129 | kubectl get nodes -o wide |
| 130 | kubectl get pods -o wide |
| 131 | kubectl get pods -n voltha -o wide |
Andy Bavier | 4af0272 | 2020-01-15 10:24:24 -0700 | [diff] [blame] | 132 | |
Andy Bavier | 7afb11c | 2020-02-13 11:25:20 -0700 | [diff] [blame] | 133 | sleep 60 # Wait for log-collector and log-combine to complete |
| 134 | |
Andy Bavier | b30c0a0 | 2020-03-03 17:21:23 -0700 | [diff] [blame] | 135 | ## Pull out errors from log files |
| 136 | extract_errors_go() { |
| 137 | echo |
| 138 | echo "Error summary for $1:" |
Andy Bavier | 36c688c | 2020-03-17 07:32:30 -0700 | [diff] [blame] | 139 | grep '"level":"error"' $HOME/kind-voltha/scripts/logger/combined/$1* |
Andy Bavier | b30c0a0 | 2020-03-03 17:21:23 -0700 | [diff] [blame] | 140 | echo |
| 141 | } |
| 142 | |
| 143 | extract_errors_python() { |
| 144 | echo |
| 145 | echo "Error summary for $1:" |
Andy Bavier | 36c688c | 2020-03-17 07:32:30 -0700 | [diff] [blame] | 146 | grep 'ERROR' $HOME/kind-voltha/scripts/logger/combined/$1* |
Andy Bavier | b30c0a0 | 2020-03-03 17:21:23 -0700 | [diff] [blame] | 147 | echo |
| 148 | } |
| 149 | |
| 150 | extract_errors_go voltha-rw-core > $WORKSPACE/error-report.log |
| 151 | extract_errors_go adapter-open-olt >> $WORKSPACE/error-report.log |
| 152 | extract_errors_python adapter-open-onu >> $WORKSPACE/error-report.log |
| 153 | extract_errors_go voltha-ofagent >> $WORKSPACE/error-report.log |
| 154 | extract_errors_python onos >> $WORKSPACE/error-report.log |
| 155 | |
Andrea Campanella | 9a95cb7 | 2020-09-03 11:28:35 +0200 | [diff] [blame] | 156 | gzip error-report.log || true |
| 157 | |
Andy Bavier | 36c688c | 2020-03-17 07:32:30 -0700 | [diff] [blame] | 158 | cd $HOME/kind-voltha/scripts/logger/combined/ |
Andy Bavier | 7afb11c | 2020-02-13 11:25:20 -0700 | [diff] [blame] | 159 | tar czf $WORKSPACE/container-logs.tgz * |
Andy Bavier | 4af0272 | 2020-01-15 10:24:24 -0700 | [diff] [blame] | 160 | |
Andy Bavier | 3c1ccca | 2020-02-12 16:50:04 -0700 | [diff] [blame] | 161 | cd $WORKSPACE |
Andy Bavier | b86ca42 | 2020-02-13 05:19:43 -0700 | [diff] [blame] | 162 | gzip *-combined.log || true |
Andy Bavier | 2178d10 | 2020-02-06 16:22:11 -0700 | [diff] [blame] | 163 | |
Hung-Wei Chiu | 9669b7d | 2020-05-14 12:12:05 -0700 | [diff] [blame] | 164 | ## Collect ONOS log |
| 165 | wget https://raw.githubusercontent.com/opennetworkinglab/onos/master/tools/package/runtime/bin/onos-diagnostics-k8s |
| 166 | wget https://raw.githubusercontent.com/opennetworkinglab/onos/master/tools/package/runtime/bin/onos-diagnostics-profile |
| 167 | chmod 755 onos-diagnostics-k8s |
pierventre | f48ebc1 | 2020-07-17 16:01:32 +0200 | [diff] [blame] | 168 | pod_names=$(kubectl get pods -lapp=onos-onos-classic -o name | cut -d '/' -f2 | tr '\n' ' ') |
| 169 | ./onos-diagnostics-k8s ${pod_names} |
Hung-Wei Chiu | 9669b7d | 2020-05-14 12:12:05 -0700 | [diff] [blame] | 170 | cp /tmp/onos-diags.tar.gz $WORKSPACE/onos-diags.tar.gz |
| 171 | |
Andy Bavier | 36c688c | 2020-03-17 07:32:30 -0700 | [diff] [blame] | 172 | ## shut down voltha but leave kind-voltha cluster |
| 173 | cd $HOME/kind-voltha/ |
| 174 | DEPLOY_K8S=n WAIT_ON_DOWN=y ./voltha down |
Andy Bavier | fcae383 | 2020-06-04 08:51:10 -0700 | [diff] [blame] | 175 | kubectl delete deployment voltctl || true |
Kailash | 8b8de74 | 2019-07-30 08:54:51 -0700 | [diff] [blame] | 176 | ''' |
| 177 | step([$class: 'RobotPublisher', |
| 178 | disableArchiveOutput: false, |
| 179 | logFileName: 'RobotLogs/log*.html', |
| 180 | otherFiles: '', |
| 181 | outputFileName: 'RobotLogs/output*.xml', |
| 182 | outputPath: '.', |
Matteo Scandolo | 26bc17e | 2019-09-24 09:03:01 -0700 | [diff] [blame] | 183 | passThreshold: 100, |
Kailash | 8b8de74 | 2019-07-30 08:54:51 -0700 | [diff] [blame] | 184 | reportFileName: 'RobotLogs/report*.html', |
| 185 | unstableThreshold: 0]); |
Andy Bavier | 3c1ccca | 2020-02-12 16:50:04 -0700 | [diff] [blame] | 186 | archiveArtifacts artifacts: '*.log,*.gz,*.tgz' |
Kailash | 8b8de74 | 2019-07-30 08:54:51 -0700 | [diff] [blame] | 187 | |
| 188 | } |
| 189 | } |
| 190 | } |