Matteo Scandolo | fbcbdb8 | 2020-05-06 15:41:32 -0700 | [diff] [blame] | 1 | // Copyright 2019-present Open Networking Foundation |
| 2 | // |
| 3 | // Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | // you may not use this file except in compliance with the License. |
| 5 | // You may obtain a copy of the License at |
| 6 | // |
| 7 | // http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | // |
| 9 | // Unless required by applicable law or agreed to in writing, software |
| 10 | // distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | // See the License for the specific language governing permissions and |
| 13 | // limitations under the License. |
| 14 | |
Matteo Scandolo | 39c531a | 2021-01-22 16:05:32 -0800 | [diff] [blame^] | 15 | // deploy VOLTHA and performs a scale test |
Matteo Scandolo | fbcbdb8 | 2020-05-06 15:41:32 -0700 | [diff] [blame] | 16 | |
| 17 | pipeline { |
| 18 | |
| 19 | /* no label, executor is determined by JJB */ |
| 20 | agent { |
| 21 | label "${params.buildNode}" |
| 22 | } |
| 23 | options { |
Andrea Campanella | c68b8d7 | 2020-09-24 11:25:47 +0200 | [diff] [blame] | 24 | timeout(time: 60, unit: 'MINUTES') |
Matteo Scandolo | fbcbdb8 | 2020-05-06 15:41:32 -0700 | [diff] [blame] | 25 | } |
| 26 | environment { |
Matteo Scandolo | c3dea53 | 2020-06-04 10:46:49 -0700 | [diff] [blame] | 27 | JENKINS_NODE_COOKIE="dontKillMe" // do not kill processes after the build is done |
Matteo Scandolo | fbcbdb8 | 2020-05-06 15:41:32 -0700 | [diff] [blame] | 28 | KUBECONFIG="$HOME/.kube/config" |
| 29 | VOLTCONFIG="$HOME/.volt/config" |
Matteo Scandolo | b70b3e0 | 2020-05-07 11:50:26 -0700 | [diff] [blame] | 30 | SSHPASS="karaf" |
Matteo Scandolo | 39c531a | 2021-01-22 16:05:32 -0800 | [diff] [blame^] | 31 | // PATH="$PATH:$WORKSPACE/kind-voltha/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" |
| 32 | // SCHEDULE_ON_CONTROL_NODES="yes" |
| 33 | // FANCY=0 |
| 34 | // WITH_SIM_ADAPTERS="no" |
| 35 | // WITH_RADIUS="${withRadius}" |
| 36 | // WITH_BBSIM="yes" |
| 37 | // LEGACY_BBSIM_INDEX="no" |
| 38 | // DEPLOY_K8S="no" |
| 39 | // CONFIG_SADIS="external" |
| 40 | // WITH_KAFKA="kafka.default.svc.cluster.local" |
| 41 | // WITH_ETCD="etcd.default.svc.cluster.local" |
| 42 | // VOLTHA_ETCD_PORT=9999 |
Matteo Scandolo | fbcbdb8 | 2020-05-06 15:41:32 -0700 | [diff] [blame] | 43 | |
Matteo Scandolo | 0430f67 | 2020-05-07 11:50:26 -0700 | [diff] [blame] | 44 | // configurable options |
Matteo Scandolo | 39c531a | 2021-01-22 16:05:32 -0800 | [diff] [blame^] | 45 | // WITH_EAPOL="${withEapol}" |
| 46 | // WITH_DHCP="${withDhcp}" |
| 47 | // WITH_IGMP="${withIgmp}" |
Matteo Scandolo | fbcbdb8 | 2020-05-06 15:41:32 -0700 | [diff] [blame] | 48 | VOLTHA_LOG_LEVEL="${logLevel}" |
| 49 | NUM_OF_BBSIM="${olts}" |
Matteo Scandolo | 0430f67 | 2020-05-07 11:50:26 -0700 | [diff] [blame] | 50 | NUM_OF_OPENONU="${openonuAdapterReplicas}" |
| 51 | NUM_OF_ONOS="${onosReplicas}" |
| 52 | NUM_OF_ATOMIX="${atomixReplicas}" |
Matteo Scandolo | 39c531a | 2021-01-22 16:05:32 -0800 | [diff] [blame^] | 53 | // WITH_PPROF="${withProfiling}" |
Matteo Scandolo | c4efe8a | 2020-07-23 20:02:18 +0000 | [diff] [blame] | 54 | EXTRA_HELM_FLAGS="${extraHelmFlags} " // note that the trailing space is required to separate the parameters from appends done later |
Matteo Scandolo | 39c531a | 2021-01-22 16:05:32 -0800 | [diff] [blame^] | 55 | // VOLTHA_CHART="${volthaChart}" |
| 56 | // VOLTHA_BBSIM_CHART="${bbsimChart}" |
| 57 | // VOLTHA_ADAPTER_OPEN_OLT_CHART="${openoltAdapterChart}" |
| 58 | // VOLTHA_ADAPTER_OPEN_ONU_CHART="${openonuAdapterChart}" |
| 59 | // ONOS_CLASSIC_CHART="${onosChart}" |
| 60 | // RADIUS_CHART="${radiusChart}" |
Matteo Scandolo | 7d6695c | 2020-08-25 13:13:16 -0700 | [diff] [blame] | 61 | |
Matteo Scandolo | 7c803cb | 2020-12-15 11:33:32 -1000 | [diff] [blame] | 62 | APPS_TO_LOG="etcd kafka onos-onos-classic adapter-open-onu adapter-open-olt rw-core ofagent bbsim radius bbsim-sadis-server" |
Matteo Scandolo | f54c9b3 | 2020-08-27 07:58:27 -0700 | [diff] [blame] | 63 | LOG_FOLDER="$WORKSPACE/logs" |
Matteo Scandolo | 5f72f52 | 2020-11-02 17:59:44 -0800 | [diff] [blame] | 64 | |
| 65 | GERRIT_PROJECT="${GERRIT_PROJECT}" |
Matteo Scandolo | fbcbdb8 | 2020-05-06 15:41:32 -0700 | [diff] [blame] | 66 | } |
| 67 | |
| 68 | stages { |
| 69 | stage ('Cleanup') { |
Matteo Scandolo | fbcbdb8 | 2020-05-06 15:41:32 -0700 | [diff] [blame] | 70 | steps { |
Matteo Scandolo | a731f8f | 2020-05-11 10:27:26 -0700 | [diff] [blame] | 71 | timeout(time: 11, unit: 'MINUTES') { |
Matteo Scandolo | 39c531a | 2021-01-22 16:05:32 -0800 | [diff] [blame^] | 72 | sh returnStdout: false, script: ''' |
Matteo Scandolo | d2e2a0d | 2020-06-10 18:02:03 -0700 | [diff] [blame] | 73 | helm repo add onf https://charts.opencord.org |
Matteo Scandolo | d2e2a0d | 2020-06-10 18:02:03 -0700 | [diff] [blame] | 74 | helm repo update |
| 75 | |
Matteo Scandolo | 9fe8cad | 2020-11-13 12:54:19 -0800 | [diff] [blame] | 76 | NAMESPACES="voltha1 voltha2 infra default" |
Matteo Scandolo | 39c531a | 2021-01-22 16:05:32 -0800 | [diff] [blame^] | 77 | for NS in $NAMESPACES |
Matteo Scandolo | a731f8f | 2020-05-11 10:27:26 -0700 | [diff] [blame] | 78 | do |
Matteo Scandolo | 39c531a | 2021-01-22 16:05:32 -0800 | [diff] [blame^] | 79 | for hchart in $(helm list -n $NS -q | grep -E -v 'docker-registry|kafkacat'); |
Matteo Scandolo | 9fe8cad | 2020-11-13 12:54:19 -0800 | [diff] [blame] | 80 | do |
Matteo Scandolo | 39c531a | 2021-01-22 16:05:32 -0800 | [diff] [blame^] | 81 | echo "Purging chart: ${hchart}" |
| 82 | helm delete -n $NS "${hchart}" |
Matteo Scandolo | 9fe8cad | 2020-11-13 12:54:19 -0800 | [diff] [blame] | 83 | done |
Matteo Scandolo | a731f8f | 2020-05-11 10:27:26 -0700 | [diff] [blame] | 84 | done |
Matteo Scandolo | 9fe8cad | 2020-11-13 12:54:19 -0800 | [diff] [blame] | 85 | |
Matteo Scandolo | 39c531a | 2021-01-22 16:05:32 -0800 | [diff] [blame^] | 86 | # wait for pods to be removed |
| 87 | echo -ne "\nWaiting for PODs to be removed..." |
| 88 | PODS=$(kubectl get pods --all-namespaces --no-headers | grep -v -E "kube|cattle|registry" | wc -l) |
| 89 | while [[ $PODS != 0 ]]; do |
| 90 | sleep 5 |
| 91 | echo -ne "." |
| 92 | PODS=$(kubectl get pods --all-namespaces --no-headers | grep -v -E "kube|cattle|registry" | wc -l) |
| 93 | done |
Matteo Scandolo | ed41d8a | 2020-06-03 13:13:49 -0700 | [diff] [blame] | 94 | |
Matteo Scandolo | d406449 | 2020-11-13 10:48:30 -0800 | [diff] [blame] | 95 | # remove orphaned port-forward from different namespaces |
Matteo Scandolo | 39c531a | 2021-01-22 16:05:32 -0800 | [diff] [blame^] | 96 | ps aux | grep port-forw | grep -v grep | awk '{print $2}' | xargs --no-run-if-empty kill -9 |
Matteo Scandolo | d406449 | 2020-11-13 10:48:30 -0800 | [diff] [blame] | 97 | |
Matteo Scandolo | a731f8f | 2020-05-11 10:27:26 -0700 | [diff] [blame] | 98 | cd $WORKSPACE |
| 99 | rm -rf $WORKSPACE/* |
Matteo Scandolo | 39c531a | 2021-01-22 16:05:32 -0800 | [diff] [blame^] | 100 | ''' |
Matteo Scandolo | 3f2e415 | 2020-09-22 15:55:27 -0700 | [diff] [blame] | 101 | } |
Matteo Scandolo | fbcbdb8 | 2020-05-06 15:41:32 -0700 | [diff] [blame] | 102 | } |
| 103 | } |
| 104 | stage('Clone voltha-system-tests') { |
| 105 | steps { |
| 106 | checkout([ |
| 107 | $class: 'GitSCM', |
Matteo Scandolo | 9aae495 | 2020-09-14 13:05:53 -0700 | [diff] [blame] | 108 | userRemoteConfigs: [[ |
| 109 | url: "https://gerrit.opencord.org/voltha-system-tests", |
Matteo Scandolo | a42c6f5 | 2020-09-19 01:35:12 +0000 | [diff] [blame] | 110 | refspec: "${volthaSystemTestsChange}" |
Matteo Scandolo | 9aae495 | 2020-09-14 13:05:53 -0700 | [diff] [blame] | 111 | ]], |
Matteo Scandolo | d43064b | 2020-07-13 17:18:45 -0700 | [diff] [blame] | 112 | branches: [[ name: "${release}", ]], |
Matteo Scandolo | fbcbdb8 | 2020-05-06 15:41:32 -0700 | [diff] [blame] | 113 | extensions: [ |
| 114 | [$class: 'WipeWorkspace'], |
| 115 | [$class: 'RelativeTargetDirectory', relativeTargetDir: "voltha-system-tests"], |
| 116 | [$class: 'CloneOption', depth: 0, noTags: false, reference: '', shallow: false], |
| 117 | ], |
| 118 | ]) |
Matteo Scandolo | 3f2e415 | 2020-09-22 15:55:27 -0700 | [diff] [blame] | 119 | script { |
| 120 | sh(script:""" |
| 121 | if [ '${volthaSystemTestsChange}' != '' ] ; then |
| 122 | cd $WORKSPACE/voltha-system-tests; |
| 123 | git fetch https://gerrit.opencord.org/voltha-system-tests ${volthaSystemTestsChange} && git checkout FETCH_HEAD |
| 124 | fi |
| 125 | """) |
| 126 | } |
Matteo Scandolo | fbcbdb8 | 2020-05-06 15:41:32 -0700 | [diff] [blame] | 127 | } |
| 128 | } |
Matteo Scandolo | 9cdacc5 | 2020-10-21 14:54:06 -0700 | [diff] [blame] | 129 | stage('Build patch') { |
| 130 | when { |
| 131 | expression { |
Matteo Scandolo | 65f80bf | 2020-11-03 10:57:03 -0800 | [diff] [blame] | 132 | return params.GERRIT_PROJECT |
Matteo Scandolo | 9cdacc5 | 2020-10-21 14:54:06 -0700 | [diff] [blame] | 133 | } |
| 134 | } |
| 135 | steps { |
| 136 | sh """ |
Matteo Scandolo | 5f72f52 | 2020-11-02 17:59:44 -0800 | [diff] [blame] | 137 | git clone https://\$GERRIT_HOST/\$GERRIT_PROJECT |
| 138 | cd \$GERRIT_PROJECT |
| 139 | git fetch https://\$GERRIT_HOST/\$GERRIT_PROJECT \$GERRIT_REFSPEC && git checkout FETCH_HEAD |
Matteo Scandolo | 9cdacc5 | 2020-10-21 14:54:06 -0700 | [diff] [blame] | 140 | |
| 141 | DOCKER_REGISTRY=${dockerRegistry}/ DOCKER_REPOSITORY=voltha/ DOCKER_TAG=voltha-scale make docker-build |
| 142 | DOCKER_REGISTRY=${dockerRegistry}/ DOCKER_REPOSITORY=voltha/ DOCKER_TAG=voltha-scale make docker-push |
| 143 | """ |
| 144 | } |
| 145 | } |
Matteo Scandolo | 0430f67 | 2020-05-07 11:50:26 -0700 | [diff] [blame] | 146 | stage('Deploy common infrastructure') { |
Matteo Scandolo | a8afe12 | 2020-05-11 10:45:56 -0700 | [diff] [blame] | 147 | // includes monitoring, kafka, etcd |
Matteo Scandolo | 0430f67 | 2020-05-07 11:50:26 -0700 | [diff] [blame] | 148 | steps { |
| 149 | sh ''' |
Matteo Scandolo | 39c531a | 2021-01-22 16:05:32 -0800 | [diff] [blame^] | 150 | helm install kafka $HOME/teone/helm-charts/kafka --set replicaCount=${kafkaReplicas},replicas=${kafkaReplicas} --set persistence.enabled=false \ |
Matteo Scandolo | 070ded8 | 2020-11-02 12:54:05 -0800 | [diff] [blame] | 151 | --set zookeeper.replicaCount=${kafkaReplicas} --set zookeeper.persistence.enabled=false \ |
| 152 | --set prometheus.kafka.enabled=true,prometheus.operator.enabled=true,prometheus.jmx.enabled=true,prometheus.operator.serviceMonitor.namespace=default |
Matteo Scandolo | 0430f67 | 2020-05-07 11:50:26 -0700 | [diff] [blame] | 153 | |
Matteo Scandolo | cb580f6 | 2020-06-12 12:55:07 -0700 | [diff] [blame] | 154 | # the ETCD chart use "auth" for resons different than BBsim, so strip that away |
| 155 | ETCD_FLAGS=$(echo ${extraHelmFlags} | sed -e 's/--set auth=false / /g') | sed -e 's/--set auth=true / /g' |
Matteo Scandolo | 70a1aa8 | 2020-11-10 14:04:59 -0800 | [diff] [blame] | 156 | ETCD_FLAGS+=" --set auth.rbac.enabled=false,persistence.enabled=false,statefulset.replicaCount=${etcdReplicas}" |
Matteo Scandolo | ccc0619 | 2020-06-22 10:16:17 -0700 | [diff] [blame] | 157 | ETCD_FLAGS+=" --set memoryMode=${inMemoryEtcdStorage} " |
Matteo Scandolo | 39c531a | 2021-01-22 16:05:32 -0800 | [diff] [blame^] | 158 | helm install --set replicas=${etcdReplicas} etcd $HOME/teone/helm-charts/etcd $ETCD_FLAGS |
Matteo Scandolo | a8afe12 | 2020-05-11 10:45:56 -0700 | [diff] [blame] | 159 | |
Matteo Scandolo | 0430f67 | 2020-05-07 11:50:26 -0700 | [diff] [blame] | 160 | if [ ${withMonitoring} = true ] ; then |
Matteo Scandolo | 39c531a | 2021-01-22 16:05:32 -0800 | [diff] [blame^] | 161 | helm install nem-monitoring onf/nem-monitoring \ |
Matteo Scandolo | 8f185c3 | 2020-07-16 16:30:04 -0700 | [diff] [blame] | 162 | -f $HOME/voltha-scale/grafana.yaml \ |
Matteo Scandolo | b70b3e0 | 2020-05-07 11:50:26 -0700 | [diff] [blame] | 163 | --set prometheus.alertmanager.enabled=false,prometheus.pushgateway.enabled=false \ |
| 164 | --set kpi_exporter.enabled=false,dashboards.xos=false,dashboards.onos=false,dashboards.aaa=false,dashboards.voltha=false |
Matteo Scandolo | 0430f67 | 2020-05-07 11:50:26 -0700 | [diff] [blame] | 165 | fi |
Matteo Scandolo | 0430f67 | 2020-05-07 11:50:26 -0700 | [diff] [blame] | 166 | ''' |
| 167 | } |
| 168 | } |
Matteo Scandolo | fbcbdb8 | 2020-05-06 15:41:32 -0700 | [diff] [blame] | 169 | stage('Deploy Voltha') { |
| 170 | steps { |
| 171 | script { |
Matteo Scandolo | fbcbdb8 | 2020-05-06 15:41:32 -0700 | [diff] [blame] | 172 | sh returnStdout: false, script: """ |
Matteo Scandolo | d43064b | 2020-07-13 17:18:45 -0700 | [diff] [blame] | 173 | |
Matteo Scandolo | c4efe8a | 2020-07-23 20:02:18 +0000 | [diff] [blame] | 174 | export EXTRA_HELM_FLAGS+=' ' |
Matteo Scandolo | c9dd44f | 2020-06-18 15:06:04 -0700 | [diff] [blame] | 175 | |
Matteo Scandolo | 39c531a | 2021-01-22 16:05:32 -0800 | [diff] [blame^] | 176 | # TODO add support for older releases, how do we do that? |
Matteo Scandolo | b10be9a | 2020-08-04 13:55:59 -0700 | [diff] [blame] | 177 | # Load the release defaults |
Matteo Scandolo | 39c531a | 2021-01-22 16:05:32 -0800 | [diff] [blame^] | 178 | # if [ '${release.trim()}' != 'master' ]; then |
| 179 | # source $WORKSPACE/kind-voltha/releases/${release} |
| 180 | # EXTRA_HELM_FLAGS+=" ${extraHelmFlags} " |
| 181 | # fi |
Matteo Scandolo | fbcbdb8 | 2020-05-06 15:41:32 -0700 | [diff] [blame] | 182 | |
Matteo Scandolo | 7e8a696 | 2020-08-04 12:19:59 -0700 | [diff] [blame] | 183 | # BBSim custom image handling |
Matteo Scandolo | 90aacab | 2020-11-19 18:20:49 +0000 | [diff] [blame] | 184 | if [ '${bbsimImg.trim()}' != '' ] && [ '\$GERRIT_PROJECT' != 'bbsim' ]; then |
Matteo Scandolo | b10be9a | 2020-08-04 13:55:59 -0700 | [diff] [blame] | 185 | IFS=: read -r bbsimRepo bbsimTag <<< '${bbsimImg.trim()}' |
| 186 | EXTRA_HELM_FLAGS+="--set images.bbsim.repository=\$bbsimRepo,images.bbsim.tag=\$bbsimTag " |
| 187 | fi |
Matteo Scandolo | 7e8a696 | 2020-08-04 12:19:59 -0700 | [diff] [blame] | 188 | |
Matteo Scandolo | 39c531a | 2021-01-22 16:05:32 -0800 | [diff] [blame^] | 189 | # VOLTHA custom image handling |
| 190 | if [ '${rwCoreImg.trim()}' != '' ] && [ '\$GERRIT_PROJECT' != 'voltha-go' ]; then |
Matteo Scandolo | b10be9a | 2020-08-04 13:55:59 -0700 | [diff] [blame] | 191 | IFS=: read -r rwCoreRepo rwCoreTag <<< '${rwCoreImg.trim()}' |
Matteo Scandolo | 39c531a | 2021-01-22 16:05:32 -0800 | [diff] [blame^] | 192 | EXTRA_HELM_FLAGS+="--set voltha.images.rw_core.repository=\$rwCoreRepo,voltha.images.rw_core.tag=\$rwCoreTag " |
| 193 | fi |
| 194 | |
| 195 | # ofAgent custom image handling |
| 196 | if [ '${ofAgentImg.trim()}' != '' ] && [ '\$GERRIT_PROJECT' != 'of-agent' ]; then |
Matteo Scandolo | b10be9a | 2020-08-04 13:55:59 -0700 | [diff] [blame] | 197 | IFS=: read -r ofAgentRepo ofAgentTag <<< '${ofAgentImg.trim()}' |
Matteo Scandolo | 39c531a | 2021-01-22 16:05:32 -0800 | [diff] [blame^] | 198 | EXTRA_HELM_FLAGS+="--set voltha.images.ofagent.repository=\$ofAgentRepo,voltha.images.ofagent.tag=\$ofAgentTag " |
Matteo Scandolo | b10be9a | 2020-08-04 13:55:59 -0700 | [diff] [blame] | 199 | fi |
Matteo Scandolo | 7e8a696 | 2020-08-04 12:19:59 -0700 | [diff] [blame] | 200 | |
| 201 | # OpenOLT custom image handling |
Matteo Scandolo | 90aacab | 2020-11-19 18:20:49 +0000 | [diff] [blame] | 202 | if [ '${openoltAdapterImg.trim()}' != '' ] && [ '\$GERRIT_PROJECT' != 'voltha-openolt-adapter' ]; then |
Matteo Scandolo | b10be9a | 2020-08-04 13:55:59 -0700 | [diff] [blame] | 203 | IFS=: read -r openoltAdapterRepo openoltAdapterTag <<< '${openoltAdapterImg.trim()}' |
Matteo Scandolo | 39c531a | 2021-01-22 16:05:32 -0800 | [diff] [blame^] | 204 | EXTRA_HELM_FLAGS+="--set voltha-adapter-openolt.images.adapter_open_olt.repository=\$openoltAdapterRepo,voltha-adapter-openolt.images.adapter_open_olt.tag=\$openoltAdapterTag " |
Matteo Scandolo | b10be9a | 2020-08-04 13:55:59 -0700 | [diff] [blame] | 205 | fi |
Matteo Scandolo | 7e8a696 | 2020-08-04 12:19:59 -0700 | [diff] [blame] | 206 | |
| 207 | # OpenONU custom image handling |
Matteo Scandolo | 90aacab | 2020-11-19 18:20:49 +0000 | [diff] [blame] | 208 | if [ '${openonuAdapterImg.trim()}' != '' ] && [ '\$GERRIT_PROJECT' != 'voltha-openonu-adapter' ]; then |
Matteo Scandolo | b10be9a | 2020-08-04 13:55:59 -0700 | [diff] [blame] | 209 | IFS=: read -r openonuAdapterRepo openonuAdapterTag <<< '${openonuAdapterImg.trim()}' |
Matteo Scandolo | 39c531a | 2021-01-22 16:05:32 -0800 | [diff] [blame^] | 210 | EXTRA_HELM_FLAGS+="--set voltha-adapter-openonu.images.adapter_open_onu.repository=\$openonuAdapterRepo,voltha-adapter-openonu.images.adapter_open_onu.tag=\$openonuAdapterTag " |
Matteo Scandolo | b10be9a | 2020-08-04 13:55:59 -0700 | [diff] [blame] | 211 | fi |
Matteo Scandolo | 7e8a696 | 2020-08-04 12:19:59 -0700 | [diff] [blame] | 212 | |
Andrea Campanella | 2876308 | 2020-11-18 10:55:31 +0100 | [diff] [blame] | 213 | # OpenONU GO custom image handling |
Matteo Scandolo | 90aacab | 2020-11-19 18:20:49 +0000 | [diff] [blame] | 214 | if [ '${openonuAdapterGoImg.trim()}' != '' ] && [ '\$GERRIT_PROJECT' != 'voltha-openonu-adapter-go' ]; then |
Andrea Campanella | 2876308 | 2020-11-18 10:55:31 +0100 | [diff] [blame] | 215 | IFS=: read -r openonuAdapterGoRepo openonuAdapterGoTag <<< '${openonuAdapterGoImg.trim()}' |
Matteo Scandolo | 39c531a | 2021-01-22 16:05:32 -0800 | [diff] [blame^] | 216 | EXTRA_HELM_FLAGS+="--set voltha-adapter-openonu.images.adapter_open_onu_go.repository=\$openonuAdapterGoRepo,voltha-adapter-openonu.images.adapter_open_onu_go.tag=\$openonuAdapterGoTag " |
Andrea Campanella | 2876308 | 2020-11-18 10:55:31 +0100 | [diff] [blame] | 217 | fi |
| 218 | |
Matteo Scandolo | 7e8a696 | 2020-08-04 12:19:59 -0700 | [diff] [blame] | 219 | # ONOS custom image handling |
Matteo Scandolo | 90aacab | 2020-11-19 18:20:49 +0000 | [diff] [blame] | 220 | if [ '${onosImg.trim()}' != '' ] && [ '\$GERRIT_PROJECT' != 'voltha-onos' ]; then |
Matteo Scandolo | b10be9a | 2020-08-04 13:55:59 -0700 | [diff] [blame] | 221 | IFS=: read -r onosRepo onosTag <<< '${onosImg.trim()}' |
Matteo Scandolo | 39c531a | 2021-01-22 16:05:32 -0800 | [diff] [blame^] | 222 | EXTRA_HELM_FLAGS+="--set onos-classic.images.onos.repository=\$onosRepo,onos-classic.images.onos.tag=\$onosTag " |
Matteo Scandolo | b10be9a | 2020-08-04 13:55:59 -0700 | [diff] [blame] | 223 | fi |
Matteo Scandolo | 7e8a696 | 2020-08-04 12:19:59 -0700 | [diff] [blame] | 224 | |
Matteo Scandolo | 1c5ffab | 2020-07-17 08:53:01 -0700 | [diff] [blame] | 225 | # set BBSim parameters |
Matteo Scandolo | c4efe8a | 2020-07-23 20:02:18 +0000 | [diff] [blame] | 226 | EXTRA_HELM_FLAGS+='--set enablePerf=true,pon=${pons},onu=${onus} ' |
Matteo Scandolo | 1c5ffab | 2020-07-17 08:53:01 -0700 | [diff] [blame] | 227 | |
| 228 | # disable the securityContext, this is a development cluster |
Matteo Scandolo | c4efe8a | 2020-07-23 20:02:18 +0000 | [diff] [blame] | 229 | EXTRA_HELM_FLAGS+='--set securityContext.enabled=false ' |
Matteo Scandolo | 1c5ffab | 2020-07-17 08:53:01 -0700 | [diff] [blame] | 230 | |
Matteo Scandolo | d2e2a0d | 2020-06-10 18:02:03 -0700 | [diff] [blame] | 231 | # No persistent-volume-claims in Atomix |
Matteo Scandolo | 39c531a | 2021-01-22 16:05:32 -0800 | [diff] [blame^] | 232 | EXTRA_HELM_FLAGS+="--set onos-classic.atomix.persistence.enabled=false " |
Matteo Scandolo | d2e2a0d | 2020-06-10 18:02:03 -0700 | [diff] [blame] | 233 | |
Matteo Scandolo | 0e3fc2b | 2020-08-06 14:01:42 -0700 | [diff] [blame] | 234 | echo "Installing with the following extra arguments:" |
| 235 | echo $EXTRA_HELM_FLAGS |
| 236 | |
Matteo Scandolo | c415805 | 2020-09-17 11:25:07 -0700 | [diff] [blame] | 237 | # if it's newer than voltha-2.4 set the correct BBSIM_CFG |
Matteo Scandolo | 39c531a | 2021-01-22 16:05:32 -0800 | [diff] [blame^] | 238 | # TODO customize the workflow |
| 239 | #if [ '${release.trim()}' != 'voltha-2.4' ]; then |
| 240 | # export BBSIM_CFG="$WORKSPACE/kind-voltha/configs/bbsim-sadis-${workflow}.yaml" |
| 241 | #fi |
Matteo Scandolo | c05da5c | 2020-09-01 15:05:50 -0700 | [diff] [blame] | 242 | |
Matteo Scandolo | 9cdacc5 | 2020-10-21 14:54:06 -0700 | [diff] [blame] | 243 | # Use custom built images |
| 244 | |
Matteo Scandolo | 5f72f52 | 2020-11-02 17:59:44 -0800 | [diff] [blame] | 245 | if [ '\$GERRIT_PROJECT' == 'voltha-go' ]; then |
Matteo Scandolo | 39c531a | 2021-01-22 16:05:32 -0800 | [diff] [blame^] | 246 | EXTRA_HELM_FLAGS+="--set voltha.images.rw_core.repository=${dockerRegistry}/voltha/voltha-rw-core,voltha.images.rw_core.tag=voltha-scale " |
Matteo Scandolo | 9cdacc5 | 2020-10-21 14:54:06 -0700 | [diff] [blame] | 247 | fi |
| 248 | |
Matteo Scandolo | 5f72f52 | 2020-11-02 17:59:44 -0800 | [diff] [blame] | 249 | if [ '\$GERRIT_PROJECT' == 'voltha-openolt-adapter' ]; then |
Matteo Scandolo | 39c531a | 2021-01-22 16:05:32 -0800 | [diff] [blame^] | 250 | EXTRA_HELM_FLAGS+="--set voltha-openolt-adapter.images.adapter_open_olt.repository=${dockerRegistry}/voltha/voltha-openolt-adapter,voltha-openolt-adapter.images.adapter_open_olt.tag=voltha-scale " |
Matteo Scandolo | 9cdacc5 | 2020-10-21 14:54:06 -0700 | [diff] [blame] | 251 | fi |
| 252 | |
Matteo Scandolo | 5f72f52 | 2020-11-02 17:59:44 -0800 | [diff] [blame] | 253 | if [ '\$GERRIT_PROJECT' == 'voltha-openonu-adapter' ]; then |
Matteo Scandolo | 39c531a | 2021-01-22 16:05:32 -0800 | [diff] [blame^] | 254 | EXTRA_HELM_FLAGS+="--set voltha-openonu-adapter.images.adapter_open_onu.repository=${dockerRegistry}/voltha/voltha-openonu-adapter,voltha-openonu-adapter.images.adapter_open_onu.tag=voltha-scale " |
Matteo Scandolo | 9cdacc5 | 2020-10-21 14:54:06 -0700 | [diff] [blame] | 255 | fi |
| 256 | |
Andrea Campanella | ed02698 | 2020-11-20 10:06:42 +0100 | [diff] [blame] | 257 | if [ '\$GERRIT_PROJECT' == 'voltha-openonu-adapter-go' ]; then |
Matteo Scandolo | 39c531a | 2021-01-22 16:05:32 -0800 | [diff] [blame^] | 258 | EXTRA_HELM_FLAGS+="--set voltha-openonu-adapter-go.images.adapter_open_onu_go.repository=${dockerRegistry}/voltha/voltha-openonu-adapter-go,voltha-openonu-adapter-go.images.adapter_open_onu_go.tag=voltha-scale " |
Andrea Campanella | ed02698 | 2020-11-20 10:06:42 +0100 | [diff] [blame] | 259 | fi |
| 260 | |
Matteo Scandolo | 5f72f52 | 2020-11-02 17:59:44 -0800 | [diff] [blame] | 261 | if [ '\$GERRIT_PROJECT' == 'ofagent-go' ]; then |
Matteo Scandolo | 39c531a | 2021-01-22 16:05:32 -0800 | [diff] [blame^] | 262 | EXTRA_HELM_FLAGS+="--set voltha.images.ofagent.repository=${dockerRegistry}/voltha/voltha-ofagent-go,ofagent-go.images.ofagent.tag=voltha-scale " |
Matteo Scandolo | 9cdacc5 | 2020-10-21 14:54:06 -0700 | [diff] [blame] | 263 | fi |
| 264 | |
Matteo Scandolo | 5f72f52 | 2020-11-02 17:59:44 -0800 | [diff] [blame] | 265 | if [ '\$GERRIT_PROJECT' == 'voltha-onos' ]; then |
Matteo Scandolo | 39c531a | 2021-01-22 16:05:32 -0800 | [diff] [blame^] | 266 | EXTRA_HELM_FLAGS+="--set onos-classic.images.onos.repository=${dockerRegistry}/voltha/voltha-onos,onos-classic.images.onos.tag=voltha-scale " |
Matteo Scandolo | 9cdacc5 | 2020-10-21 14:54:06 -0700 | [diff] [blame] | 267 | fi |
| 268 | |
Matteo Scandolo | 5f72f52 | 2020-11-02 17:59:44 -0800 | [diff] [blame] | 269 | if [ '\$GERRIT_PROJECT' == 'bbsim' ]; then |
Matteo Scandolo | 9cdacc5 | 2020-10-21 14:54:06 -0700 | [diff] [blame] | 270 | EXTRA_HELM_FLAGS+="--set images.bbsim.repository=${dockerRegistry}/voltha/bbsim,images.bbsim.tag=voltha-scale " |
| 271 | fi |
| 272 | |
Matteo Scandolo | 39c531a | 2021-01-22 16:05:32 -0800 | [diff] [blame^] | 273 | helm upgrade --install voltha-infra onf/voltha-infra \ |
| 274 | --set onos-classic.replicas=${onosReplicas},onos-classic.atomix.replicas=${atomixReplicas} \ |
| 275 | --set etcd.enabled=false,kafka.enabled=false \ |
| 276 | --set global.log_level=${logLevel} |
Matteo Scandolo | cb580f6 | 2020-06-12 12:55:07 -0700 | [diff] [blame] | 277 | |
Matteo Scandolo | 39c531a | 2021-01-22 16:05:32 -0800 | [diff] [blame^] | 278 | helm upgrade --install voltha1 onf/voltha-stack \ |
| 279 | --set global.stack_name=voltha1 \ |
| 280 | --set global.voltha_infra_name=voltha-infra \ |
| 281 | --set global.voltha_infra_namespace=default \ |
| 282 | --set global.log_level=${logLevel} \ |
| 283 | --set voltha.services.kafka.adapter.address=kafka.default.svc:9092 \ |
| 284 | --set voltha.services.kafka.cluster.address=kafka.default.svc:9092 \ |
| 285 | --set voltha.services.etcd.address=etcd.default.svc:2379 \ |
| 286 | --set voltha-adapter-openolt.services.kafka.adapter.address=kafka.default.svc:9092 \ |
| 287 | --set voltha-adapter-openolt.services.kafka.cluster.address=kafka.default.svc:9092 \ |
| 288 | --set voltha-adapter-openolt.services.etcd.address=etcd.default.svc:2379 \ |
| 289 | --set voltha-adapter-openonu.services.kafka.adapter.service=kafka.default.svc \ |
| 290 | --set voltha-adapter-openonu.services.kafka.cluster.service=kafka.default.svc \ |
| 291 | --set voltha-adapter-openonu.services.etcd.service=etcd.default.svc |
| 292 | # TODO having to set all of these values is annoying, is there a better solution? |
| 293 | |
| 294 | |
| 295 | for i in {0..${olts.toInteger() - 1}}; do |
| 296 | stackId=1 |
| 297 | helm upgrade --install bbsim\$i onf/bbsim --set olt_id="\${stackId}\${i}" \ |
| 298 | --set onu=${onus},pon=${pons} \ |
| 299 | --set defaults.log_level=${logLevel.toLowerCase()} |
| 300 | done |
| 301 | |
| 302 | echo -ne "\nWaiting for VOLTHA to start..." |
| 303 | voltha=\$(kubectl get pods --all-namespaces -l app.kubernetes.io/part-of=voltha --no-headers | grep "0/" | wc -l) |
| 304 | while [[ \$voltha != 0 ]]; do |
| 305 | sleep 5 |
| 306 | echo -ne "." |
| 307 | voltha=\$(kubectl get pods --all-namespaces -l app.kubernetes.io/part-of=voltha --no-headers | grep "0/" | wc -l) |
| 308 | done |
| 309 | |
| 310 | # forward ONOS and VOLTHA ports |
| 311 | _TAG=onos-port-forward kubectl port-forward --address 0.0.0.0 -n default svc/voltha-infra-onos-classic-hs 8101:8101& |
| 312 | _TAG=onos-port-forward kubectl port-forward --address 0.0.0.0 -n default svc/voltha-infra-onos-classic-hs 8181:8181& |
| 313 | _TAG=voltha-port-forward kubectl port-forward --address 0.0.0.0 -n default svc/voltha1-voltha-api 55555:55555& |
Matteo Scandolo | fbcbdb8 | 2020-05-06 15:41:32 -0700 | [diff] [blame] | 314 | """ |
| 315 | } |
Matteo Scandolo | 7d6695c | 2020-08-25 13:13:16 -0700 | [diff] [blame] | 316 | sh returnStdout: false, script: ''' |
| 317 | # start logging with kail |
| 318 | |
Matteo Scandolo | 7d6695c | 2020-08-25 13:13:16 -0700 | [diff] [blame] | 319 | mkdir -p $LOG_FOLDER |
| 320 | |
| 321 | list=($APPS_TO_LOG) |
| 322 | for app in "${list[@]}" |
| 323 | do |
| 324 | echo "Starting logs for: ${app}" |
| 325 | _TAG=kail-$app kail -l app=$app --since 1h > $LOG_FOLDER/$app.log& |
| 326 | done |
| 327 | ''' |
Matteo Scandolo | fbcbdb8 | 2020-05-06 15:41:32 -0700 | [diff] [blame] | 328 | } |
| 329 | } |
Matteo Scandolo | 0430f67 | 2020-05-07 11:50:26 -0700 | [diff] [blame] | 330 | stage('Configuration') { |
Matteo Scandolo | fbcbdb8 | 2020-05-06 15:41:32 -0700 | [diff] [blame] | 331 | steps { |
Matteo Scandolo | 37cba47 | 2020-10-26 14:15:34 -0700 | [diff] [blame] | 332 | script { |
| 333 | sh returnStdout: false, script: """ |
Matteo Scandolo | 0430f67 | 2020-05-07 11:50:26 -0700 | [diff] [blame] | 334 | #Setting link discovery |
| 335 | sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 cfg set org.onosproject.provider.lldp.impl.LldpLinkProvider enabled ${withLLDP} |
| 336 | |
Matteo Scandolo | c348aea | 2020-10-07 13:36:04 -0700 | [diff] [blame] | 337 | sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 cfg set org.onosproject.net.flow.impl.FlowRuleManager allowExtraneousRules true |
| 338 | sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 cfg set org.onosproject.net.flow.impl.FlowRuleManager importExtraneousRules true |
Matteo Scandolo | 2264d9c | 2020-10-16 14:25:32 -0700 | [diff] [blame] | 339 | |
Matteo Scandolo | 9cdacc5 | 2020-10-21 14:54:06 -0700 | [diff] [blame] | 340 | |
Andrea Campanella | 816b466 | 2020-10-28 11:35:23 +0100 | [diff] [blame] | 341 | sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 cfg set org.onosproject.net.flowobjective.impl.InOrderFlowObjectiveManager accumulatorMaxBatchMillis 1000 |
Andrea Campanella | deec158 | 2020-10-28 09:37:58 +0100 | [diff] [blame] | 342 | |
Andrea Campanella | 816b466 | 2020-10-28 11:35:23 +0100 | [diff] [blame] | 343 | sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 cfg set org.onosproject.net.flowobjective.impl.InOrderFlowObjectiveManager accumulatorMaxIdleMillis 500 |
Andrea Campanella | deec158 | 2020-10-28 09:37:58 +0100 | [diff] [blame] | 344 | |
Matteo Scandolo | afc3c2d | 2020-12-02 17:25:53 -0800 | [diff] [blame] | 345 | sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 cfg log:set ${logLevel} org.onosproject |
| 346 | sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 cfg log:set ${logLevel} org.opencord |
Andrea Campanella | 33b4872 | 2020-10-23 12:31:47 +0200 | [diff] [blame] | 347 | |
Matteo Scandolo | c348aea | 2020-10-07 13:36:04 -0700 | [diff] [blame] | 348 | |
Matteo Scandolo | 37cba47 | 2020-10-26 14:15:34 -0700 | [diff] [blame] | 349 | kubectl exec \$(kubectl get pods | grep -E "bbsim[0-9]" | awk 'NR==1{print \$1}') -- bbsimctl log ${logLevel.toLowerCase()} false |
Matteo Scandolo | 0430f67 | 2020-05-07 11:50:26 -0700 | [diff] [blame] | 350 | |
Matteo Scandolo | 3136cca | 2020-05-15 14:14:27 -0700 | [diff] [blame] | 351 | # Set Flows/Ports/Meters poll frequency |
| 352 | sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 cfg set org.onosproject.provider.of.flow.impl.OpenFlowRuleProvider flowPollFrequency ${onosStatInterval} |
| 353 | sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 cfg set org.onosproject.provider.of.device.impl.OpenFlowDeviceProvider portStatsPollFrequency ${onosStatInterval} |
| 354 | |
Matteo Scandolo | 0430f67 | 2020-05-07 11:50:26 -0700 | [diff] [blame] | 355 | if [ ${withFlows} = false ]; then |
Matteo Scandolo | b70b3e0 | 2020-05-07 11:50:26 -0700 | [diff] [blame] | 356 | sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 app deactivate org.opencord.olt |
Matteo Scandolo | 0430f67 | 2020-05-07 11:50:26 -0700 | [diff] [blame] | 357 | fi |
| 358 | |
Matteo Scandolo | c9dd44f | 2020-06-18 15:06:04 -0700 | [diff] [blame] | 359 | if [ ${withPcap} = true ] ; then |
Matteo Scandolo | 34d6a7a | 2020-10-20 09:29:58 -0700 | [diff] [blame] | 360 | # Start the tcp-dump in ofagent |
Matteo Scandolo | 37cba47 | 2020-10-26 14:15:34 -0700 | [diff] [blame] | 361 | export OF_AGENT=\$(kubectl get pods -l app=ofagent -o name) |
| 362 | kubectl exec \$OF_AGENT -- apk update |
| 363 | kubectl exec \$OF_AGENT -- apk add tcpdump |
| 364 | kubectl exec \$OF_AGENT -- mv /usr/sbin/tcpdump /usr/bin/tcpdump |
| 365 | _TAG=ofagent-tcpdump kubectl exec \$OF_AGENT -- tcpdump -nei eth0 -w out.pcap& |
Matteo Scandolo | 77fa78d | 2020-09-16 16:53:50 -0700 | [diff] [blame] | 366 | |
Matteo Scandolo | 34d6a7a | 2020-10-20 09:29:58 -0700 | [diff] [blame] | 367 | # Start the tcp-dump in radius |
Matteo Scandolo | 37cba47 | 2020-10-26 14:15:34 -0700 | [diff] [blame] | 368 | export RADIUS=\$(kubectl get pods -l app=radius -o name) |
| 369 | kubectl exec \$RADIUS -- apt-get update |
| 370 | kubectl exec \$RADIUS -- apt-get install -y tcpdump |
| 371 | _TAG=radius-tcpdump kubectl exec \$RADIUS -- tcpdump -w out.pcap& |
Matteo Scandolo | 34d6a7a | 2020-10-20 09:29:58 -0700 | [diff] [blame] | 372 | |
| 373 | # Start the tcp-dump in ONOS |
Matteo Scandolo | 37cba47 | 2020-10-26 14:15:34 -0700 | [diff] [blame] | 374 | for i in \$(seq 0 \$ONOSES); do |
| 375 | INSTANCE="onos-onos-classic-\$i" |
| 376 | kubectl exec \$INSTANCE -- apt-get update |
| 377 | kubectl exec \$INSTANCE -- apt-get install -y tcpdump |
| 378 | kubectl exec \$INSTANCE -- mv /usr/sbin/tcpdump /usr/bin/tcpdump |
| 379 | _TAG=\$INSTANCE kubectl exec \$INSTANCE -- /usr/bin/tcpdump -nei eth0 port 1812 -w out.pcap& |
Matteo Scandolo | 34d6a7a | 2020-10-20 09:29:58 -0700 | [diff] [blame] | 380 | done |
Matteo Scandolo | c9dd44f | 2020-06-18 15:06:04 -0700 | [diff] [blame] | 381 | fi |
Matteo Scandolo | 37cba47 | 2020-10-26 14:15:34 -0700 | [diff] [blame] | 382 | """ |
| 383 | } |
Matteo Scandolo | fbcbdb8 | 2020-05-06 15:41:32 -0700 | [diff] [blame] | 384 | } |
| 385 | } |
| 386 | stage('Run Test') { |
| 387 | steps { |
Matteo Scandolo | a731f8f | 2020-05-11 10:27:26 -0700 | [diff] [blame] | 388 | sh ''' |
| 389 | mkdir -p $WORKSPACE/RobotLogs |
| 390 | cd $WORKSPACE/voltha-system-tests |
| 391 | make vst_venv |
| 392 | ''' |
| 393 | sh ''' |
| 394 | if [ ${withProfiling} = true ] ; then |
Matteo Scandolo | f54c9b3 | 2020-08-27 07:58:27 -0700 | [diff] [blame] | 395 | mkdir -p $LOG_FOLDER/pprof |
Matteo Scandolo | a731f8f | 2020-05-11 10:27:26 -0700 | [diff] [blame] | 396 | echo $PATH |
| 397 | #Creating Python script for ONU Detection |
| 398 | cat << EOF > $WORKSPACE/pprof.sh |
Matteo Scandolo | 3ddd3fb | 2020-05-12 14:31:14 -0700 | [diff] [blame] | 399 | timestamp() { |
| 400 | date +"%T" |
| 401 | } |
| 402 | |
Matteo Scandolo | a731f8f | 2020-05-11 10:27:26 -0700 | [diff] [blame] | 403 | i=0 |
| 404 | while [[ true ]]; do |
| 405 | ((i++)) |
Matteo Scandolo | 3ddd3fb | 2020-05-12 14:31:14 -0700 | [diff] [blame] | 406 | ts=$(timestamp) |
Matteo Scandolo | f54c9b3 | 2020-08-27 07:58:27 -0700 | [diff] [blame] | 407 | go tool pprof -png http://127.0.0.1:6060/debug/pprof/heap > $LOG_FOLDER/pprof/rw-core-heap-\\$i-\\$ts.png |
| 408 | go tool pprof -png http://127.0.0.1:6060/debug/pprof/goroutine > $LOG_FOLDER/pprof/rw-core-goroutine-\\$i-\\$ts.png |
| 409 | curl -o $LOG_FOLDER/pprof/rw-core-profile-\\$i-\\$ts.pprof http://127.0.0.1:6060/debug/pprof/profile?seconds=10 |
| 410 | go tool pprof -png $LOG_FOLDER/pprof/rw-core-profile-\\$i-\\$ts.pprof > $LOG_FOLDER/pprof/rw-core-profile-\\$i-\\$ts.png |
Matteo Scandolo | a731f8f | 2020-05-11 10:27:26 -0700 | [diff] [blame] | 411 | |
Matteo Scandolo | f54c9b3 | 2020-08-27 07:58:27 -0700 | [diff] [blame] | 412 | go tool pprof -png http://127.0.0.1:6061/debug/pprof/heap > $LOG_FOLDER/pprof/openolt-heap-\\$i-\\$ts.png |
| 413 | go tool pprof -png http://127.0.0.1:6061/debug/pprof/goroutine > $LOG_FOLDER/pprof/openolt-goroutine-\\$i-\\$ts.png |
| 414 | curl -o $LOG_FOLDER/pprof/openolt-profile-\\$i-\\$ts.pprof http://127.0.0.1:6061/debug/pprof/profile?seconds=10 |
| 415 | go tool pprof -png $LOG_FOLDER/pprof/openolt-profile-\\$i-\\$ts.pprof > $LOG_FOLDER/pprof/openolt-profile-\\$i-\\$ts.png |
Matteo Scandolo | a731f8f | 2020-05-11 10:27:26 -0700 | [diff] [blame] | 416 | |
Matteo Scandolo | f54c9b3 | 2020-08-27 07:58:27 -0700 | [diff] [blame] | 417 | go tool pprof -png http://127.0.0.1:6062/debug/pprof/heap > $LOG_FOLDER/pprof/ofagent-heap-\\$i-\\$ts.png |
| 418 | go tool pprof -png http://127.0.0.1:6062/debug/pprof/goroutine > $LOG_FOLDER/pprof/ofagent-goroutine-\\$i-\\$ts.png |
| 419 | curl -o $LOG_FOLDER/pprof/ofagent-profile-\\$i-\\$ts.pprof http://127.0.0.1:6062/debug/pprof/profile?seconds=10 |
| 420 | go tool pprof -png $LOG_FOLDER/pprof/ofagent-profile-\\$i-\\$ts.pprof > $LOG_FOLDER/pprof/ofagent-profile-\\$i-\\$ts.png |
Matteo Scandolo | 731d12c | 2020-05-13 14:33:26 -0700 | [diff] [blame] | 421 | |
Matteo Scandolo | a731f8f | 2020-05-11 10:27:26 -0700 | [diff] [blame] | 422 | sleep 10 |
| 423 | done |
| 424 | EOF |
| 425 | |
| 426 | _TAG="pprof" |
| 427 | _TAG=$_TAG bash $WORKSPACE/pprof.sh & |
| 428 | fi |
| 429 | ''' |
Matteo Scandolo | b7d60ba | 2020-11-06 09:46:11 -0800 | [diff] [blame] | 430 | timeout(time: 15, unit: 'MINUTES') { |
Matteo Scandolo | 9b0728b | 2020-05-11 08:25:57 -0700 | [diff] [blame] | 431 | sh ''' |
| 432 | ROBOT_PARAMS="-v olt:${olts} \ |
| 433 | -v pon:${pons} \ |
| 434 | -v onu:${onus} \ |
| 435 | -v workflow:${workflow} \ |
| 436 | -v withEapol:${withEapol} \ |
| 437 | -v withDhcp:${withDhcp} \ |
| 438 | -v withIgmp:${withIgmp} \ |
| 439 | --noncritical non-critical \ |
| 440 | -e teardown " |
Matteo Scandolo | 0430f67 | 2020-05-07 11:50:26 -0700 | [diff] [blame] | 441 | |
Matteo Scandolo | 9b0728b | 2020-05-11 08:25:57 -0700 | [diff] [blame] | 442 | if [ ${withEapol} = false ] ; then |
| 443 | ROBOT_PARAMS+="-e authentication " |
| 444 | fi |
Matteo Scandolo | 0430f67 | 2020-05-07 11:50:26 -0700 | [diff] [blame] | 445 | |
Matteo Scandolo | 9b0728b | 2020-05-11 08:25:57 -0700 | [diff] [blame] | 446 | if [ ${withDhcp} = false ] ; then |
| 447 | ROBOT_PARAMS+="-e dhcp " |
| 448 | fi |
Matteo Scandolo | 0430f67 | 2020-05-07 11:50:26 -0700 | [diff] [blame] | 449 | |
Matteo Scandolo | 9b0728b | 2020-05-11 08:25:57 -0700 | [diff] [blame] | 450 | if [ ${provisionSubscribers} = false ] ; then |
Matteo Scandolo | cd59fd0 | 2020-05-12 15:00:35 -0700 | [diff] [blame] | 451 | # if we're not considering subscribers then we don't care about authentication and dhcp |
Matteo Scandolo | 111c893 | 2020-05-12 18:28:05 -0700 | [diff] [blame] | 452 | ROBOT_PARAMS+="-e authentication -e provision -e flow-after -e dhcp " |
Matteo Scandolo | 9b0728b | 2020-05-11 08:25:57 -0700 | [diff] [blame] | 453 | fi |
Matteo Scandolo | 0430f67 | 2020-05-07 11:50:26 -0700 | [diff] [blame] | 454 | |
Matteo Scandolo | 9b0728b | 2020-05-11 08:25:57 -0700 | [diff] [blame] | 455 | if [ ${withFlows} = false ] ; then |
| 456 | ROBOT_PARAMS+="-i setup -i activation " |
| 457 | fi |
Matteo Scandolo | 0430f67 | 2020-05-07 11:50:26 -0700 | [diff] [blame] | 458 | |
Matteo Scandolo | a731f8f | 2020-05-11 10:27:26 -0700 | [diff] [blame] | 459 | cd $WORKSPACE/voltha-system-tests |
Matteo Scandolo | 9b0728b | 2020-05-11 08:25:57 -0700 | [diff] [blame] | 460 | source ./vst_venv/bin/activate |
| 461 | robot -d $WORKSPACE/RobotLogs \ |
| 462 | $ROBOT_PARAMS tests/scale/Voltha_Scale_Tests.robot |
| 463 | ''' |
| 464 | } |
Matteo Scandolo | fbcbdb8 | 2020-05-06 15:41:32 -0700 | [diff] [blame] | 465 | } |
| 466 | } |
Matteo Scandolo | fbcbdb8 | 2020-05-06 15:41:32 -0700 | [diff] [blame] | 467 | } |
| 468 | post { |
| 469 | always { |
Matteo Scandolo | 157f66c | 2020-05-12 09:26:19 -0700 | [diff] [blame] | 470 | // collect result, done in the "post" step so it's executed even in the |
| 471 | // event of a timeout in the tests |
| 472 | sh ''' |
Matteo Scandolo | c9dd44f | 2020-06-18 15:06:04 -0700 | [diff] [blame] | 473 | |
Matteo Scandolo | 7d6695c | 2020-08-25 13:13:16 -0700 | [diff] [blame] | 474 | # stop the kail processes |
| 475 | list=($APPS_TO_LOG) |
| 476 | for app in "${list[@]}" |
| 477 | do |
| 478 | echo "Stopping logs for: ${app}" |
| 479 | _TAG="kail-$app" |
| 480 | P_IDS="$(ps e -ww -A | grep "_TAG=$_TAG" | grep -v grep | awk '{print $1}')" |
| 481 | if [ -n "$P_IDS" ]; then |
| 482 | echo $P_IDS |
| 483 | for P_ID in $P_IDS; do |
| 484 | kill -9 $P_ID |
| 485 | done |
| 486 | fi |
| 487 | done |
| 488 | |
Matteo Scandolo | c9dd44f | 2020-06-18 15:06:04 -0700 | [diff] [blame] | 489 | if [ ${withPcap} = true ] ; then |
| 490 | # stop ofAgent tcpdump |
| 491 | P_ID="\$(ps e -ww -A | grep "_TAG=ofagent-tcpdump" | grep -v grep | awk '{print \$1}')" |
| 492 | if [ -n "\$P_ID" ]; then |
| 493 | kill -9 \$P_ID |
| 494 | fi |
| 495 | |
Matteo Scandolo | 34d6a7a | 2020-10-20 09:29:58 -0700 | [diff] [blame] | 496 | # stop radius tcpdump |
| 497 | P_ID="\$(ps e -ww -A | grep "_TAG=radius-tcpdump" | grep -v grep | awk '{print \$1}')" |
| 498 | if [ -n "\$P_ID" ]; then |
| 499 | kill -9 \$P_ID |
| 500 | fi |
| 501 | |
| 502 | # stop onos tcpdump |
| 503 | LIMIT=$(($NUM_OF_ONOS - 1)) |
| 504 | for i in $(seq 0 $LIMIT); do |
| 505 | INSTANCE="onos-onos-classic-$i" |
| 506 | P_ID="\$(ps e -ww -A | grep "_TAG=$INSTANCE" | grep -v grep | awk '{print \$1}')" |
| 507 | if [ -n "\$P_ID" ]; then |
| 508 | kill -9 \$P_ID |
| 509 | fi |
| 510 | done |
| 511 | |
Matteo Scandolo | c9dd44f | 2020-06-18 15:06:04 -0700 | [diff] [blame] | 512 | # copy the file |
Matteo Scandolo | a7600f1 | 2020-09-21 16:09:27 -0700 | [diff] [blame] | 513 | export OF_AGENT=$(kubectl get pods -l app=ofagent | awk 'NR==2{print $1}') || true |
| 514 | kubectl cp $OF_AGENT:out.pcap $LOG_FOLDER/ofagent.pcap || true |
Matteo Scandolo | 77fa78d | 2020-09-16 16:53:50 -0700 | [diff] [blame] | 515 | |
Andrea Campanella | 190854a | 2020-10-20 14:30:42 +0200 | [diff] [blame] | 516 | export RADIUS=$(kubectl get pods -l app=radius | awk 'NR==2{print $1}') || true |
| 517 | kubectl cp $RADIUS:out.pcap $LOG_FOLDER/radius.pcap || true |
Matteo Scandolo | 34d6a7a | 2020-10-20 09:29:58 -0700 | [diff] [blame] | 518 | |
| 519 | LIMIT=$(($NUM_OF_ONOS - 1)) |
| 520 | for i in $(seq 0 $LIMIT); do |
| 521 | INSTANCE="onos-onos-classic-$i" |
| 522 | kubectl cp $INSTANCE:out.pcap $LOG_FOLDER/$INSTANCE.pcap || true |
| 523 | done |
Matteo Scandolo | c9dd44f | 2020-06-18 15:06:04 -0700 | [diff] [blame] | 524 | fi |
| 525 | |
Matteo Scandolo | 157f66c | 2020-05-12 09:26:19 -0700 | [diff] [blame] | 526 | cd voltha-system-tests |
| 527 | source ./vst_venv/bin/activate |
Matteo Scandolo | 7d6695c | 2020-08-25 13:13:16 -0700 | [diff] [blame] | 528 | python tests/scale/collect-result.py -r $WORKSPACE/RobotLogs/output.xml -p $WORKSPACE/plots > $WORKSPACE/execution-time.txt || true |
Matteo Scandolo | 157f66c | 2020-05-12 09:26:19 -0700 | [diff] [blame] | 529 | cat $WORKSPACE/execution-time.txt |
| 530 | ''' |
Matteo Scandolo | a731f8f | 2020-05-11 10:27:26 -0700 | [diff] [blame] | 531 | sh ''' |
| 532 | if [ ${withProfiling} = true ] ; then |
| 533 | _TAG="pprof" |
| 534 | P_IDS="$(ps e -ww -A | grep "_TAG=$_TAG" | grep -v grep | awk '{print $1}')" |
| 535 | if [ -n "$P_IDS" ]; then |
| 536 | echo $P_IDS |
| 537 | for P_ID in $P_IDS; do |
| 538 | kill -9 $P_ID |
| 539 | done |
| 540 | fi |
| 541 | fi |
| 542 | ''' |
Matteo Scandolo | fbcbdb8 | 2020-05-06 15:41:32 -0700 | [diff] [blame] | 543 | plot([ |
| 544 | csvFileName: 'scale-test.csv', |
| 545 | csvSeries: [ |
Matteo Scandolo | 272aed4 | 2020-05-08 15:05:57 -0700 | [diff] [blame] | 546 | [file: 'plots/plot-voltha-onus.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''], |
| 547 | [file: 'plots/plot-onos-ports.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''], |
| 548 | [file: 'plots/plot-voltha-flows-before.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''], |
Matteo Scandolo | 6496676 | 2020-05-13 11:51:38 -0700 | [diff] [blame] | 549 | [file: 'plots/plot-voltha-openolt-flows-before.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''], |
Matteo Scandolo | 272aed4 | 2020-05-08 15:05:57 -0700 | [diff] [blame] | 550 | [file: 'plots/plot-onos-flows-before.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''], |
| 551 | [file: 'plots/plot-onos-auth.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''], |
| 552 | [file: 'plots/plot-voltha-flows-after.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''], |
Matteo Scandolo | 6496676 | 2020-05-13 11:51:38 -0700 | [diff] [blame] | 553 | [file: 'plots/plot-voltha-openolt-flows-after.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''], |
Matteo Scandolo | 272aed4 | 2020-05-08 15:05:57 -0700 | [diff] [blame] | 554 | [file: 'plots/plot-onos-flows-after.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''], |
| 555 | [file: 'plots/plot-onos-dhcp.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''], |
Matteo Scandolo | fbcbdb8 | 2020-05-06 15:41:32 -0700 | [diff] [blame] | 556 | ], |
| 557 | group: 'Voltha-Scale-Numbers', numBuilds: '20', style: 'line', title: "Scale Test (OLTs: ${olts}, PONs: ${pons}, ONUs: ${onus})", yaxis: 'Time (s)', useDescr: true |
| 558 | ]) |
| 559 | step([$class: 'RobotPublisher', |
| 560 | disableArchiveOutput: false, |
Matteo Scandolo | b70b3e0 | 2020-05-07 11:50:26 -0700 | [diff] [blame] | 561 | logFileName: 'RobotLogs/log.html', |
Matteo Scandolo | fbcbdb8 | 2020-05-06 15:41:32 -0700 | [diff] [blame] | 562 | otherFiles: '', |
Matteo Scandolo | b70b3e0 | 2020-05-07 11:50:26 -0700 | [diff] [blame] | 563 | outputFileName: 'RobotLogs/output.xml', |
Matteo Scandolo | fbcbdb8 | 2020-05-06 15:41:32 -0700 | [diff] [blame] | 564 | outputPath: '.', |
| 565 | passThreshold: 100, |
Matteo Scandolo | b70b3e0 | 2020-05-07 11:50:26 -0700 | [diff] [blame] | 566 | reportFileName: 'RobotLogs/report.html', |
Matteo Scandolo | fbcbdb8 | 2020-05-06 15:41:32 -0700 | [diff] [blame] | 567 | unstableThreshold: 0]); |
Matteo Scandolo | 502644b | 2020-05-18 08:56:18 -0700 | [diff] [blame] | 568 | // get all the logs from kubernetes PODs |
Matteo Scandolo | ede5abd | 2020-05-22 10:57:10 -0700 | [diff] [blame] | 569 | sh returnStdout: false, script: ''' |
Matteo Scandolo | ede5abd | 2020-05-22 10:57:10 -0700 | [diff] [blame] | 570 | |
Matteo Scandolo | c495b36 | 2020-07-17 10:44:12 -0700 | [diff] [blame] | 571 | # store information on running charts |
Matteo Scandolo | 9214ebd | 2020-07-30 17:57:10 -0700 | [diff] [blame] | 572 | helm ls > $LOG_FOLDER/helm-list.txt || true |
Matteo Scandolo | c495b36 | 2020-07-17 10:44:12 -0700 | [diff] [blame] | 573 | |
Matteo Scandolo | ede5abd | 2020-05-22 10:57:10 -0700 | [diff] [blame] | 574 | # store information on the running pods |
Matteo Scandolo | 9214ebd | 2020-07-30 17:57:10 -0700 | [diff] [blame] | 575 | kubectl get pods -o wide > $LOG_FOLDER/pods.txt || true |
| 576 | kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.image}{'\\n'}" | sort | uniq | tee $LOG_FOLDER/pod-images.txt || true |
| 577 | kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.imageID}{'\\n'}" | sort | uniq | tee $LOG_FOLDER/pod-imagesId.txt || true |
Matteo Scandolo | ede5abd | 2020-05-22 10:57:10 -0700 | [diff] [blame] | 578 | |
Matteo Scandolo | 32da27d | 2020-05-29 10:38:31 -0700 | [diff] [blame] | 579 | # copy the ONOS logs directly from the container to avoid the color codes |
Matteo Scandolo | be82324 | 2020-11-20 13:48:13 -0800 | [diff] [blame] | 580 | printf '%s\n' $(kubectl get pods -l app=onos-onos-classic -o=jsonpath="{.items[*]['metadata.name']}") | xargs --no-run-if-empty -I# bash -c "kubectl cp #:${karafHome}/data/log/karaf.log $LOG_FOLDER/#.log" || true |
Matteo Scandolo | a2f07fc | 2020-10-08 18:30:54 -0700 | [diff] [blame] | 581 | |
| 582 | # get radius logs out of the container |
Andrea Campanella | eb360f4 | 2020-10-09 12:31:23 +0200 | [diff] [blame] | 583 | kubectl cp $(kubectl get pods -l app=radius --no-headers | awk '{print $1}'):/var/log/freeradius/radius.log $LOG_FOLDER//radius.log || true |
Matteo Scandolo | b70b3e0 | 2020-05-07 11:50:26 -0700 | [diff] [blame] | 584 | ''' |
Matteo Scandolo | 27553c0 | 2020-07-16 17:21:57 -0700 | [diff] [blame] | 585 | // dump all the BBSim(s) ONU information |
Matteo Scandolo | b70b3e0 | 2020-05-07 11:50:26 -0700 | [diff] [blame] | 586 | sh ''' |
| 587 | BBSIM_IDS=$(kubectl get pods | grep bbsim | grep -v server | awk '{print $1}') |
| 588 | IDS=($BBSIM_IDS) |
| 589 | |
| 590 | for bbsim in "${IDS[@]}" |
| 591 | do |
Matteo Scandolo | 900e511 | 2020-09-08 10:16:31 -0700 | [diff] [blame] | 592 | kubectl exec -t $bbsim -- bbsimctl onu list > $LOG_FOLDER/$bbsim-device-list.txt || true |
| 593 | kubectl exec -t $bbsim -- bbsimctl service list > $LOG_FOLDER/$bbsim-service-list.txt || true |
Matteo Scandolo | b70b3e0 | 2020-05-07 11:50:26 -0700 | [diff] [blame] | 594 | done |
| 595 | ''' |
Matteo Scandolo | 40b9e8e | 2020-09-28 13:31:26 -0700 | [diff] [blame] | 596 | // get DHCP server stats |
| 597 | sh ''' |
| 598 | BBSIM_IDS=$(kubectl get pods | grep bbsim | grep -v server | awk '{print $1}') |
| 599 | IDS=($BBSIM_IDS) |
| 600 | |
| 601 | for bbsim in "${IDS[@]}" |
| 602 | do |
| 603 | kubectl exec -t $bbsim -- dhcpd -lf /var/lib/dhcp/dhcpd.leases -play /tmp/dhcplog 2>&1 | tee $LOG_FOLDER/$bbsim-dhcp-replay.txt || true |
| 604 | kubectl cp $bbsim:/tmp/dhcplog $LOG_FOLDER/$bbsim-dhcp-logs || true |
| 605 | kubectl cp $bbsim:/var/lib/dhcp/dhcpd.leases $LOG_FOLDER/$bbsim-dhcp-leases || true |
| 606 | done |
| 607 | ''' |
Matteo Scandolo | 32da27d | 2020-05-29 10:38:31 -0700 | [diff] [blame] | 608 | // get ONOS debug infos |
Matteo Scandolo | b70b3e0 | 2020-05-07 11:50:26 -0700 | [diff] [blame] | 609 | sh ''' |
Matteo Scandolo | bbd8f90 | 2020-09-24 08:40:58 -0700 | [diff] [blame] | 610 | |
| 611 | sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 apps -a -s > $LOG_FOLDER/onos-apps.txt || true |
| 612 | sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 nodes > $LOG_FOLDER/onos-nodes.txt || true |
| 613 | sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 masters > $LOG_FOLDER/onos-masters.txt || true |
| 614 | sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 roles > $LOG_FOLDER/onos-roles.txt || true |
| 615 | |
Matteo Scandolo | f54c9b3 | 2020-08-27 07:58:27 -0700 | [diff] [blame] | 616 | sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 ports > $LOG_FOLDER/onos-ports-list.txt || true |
Matteo Scandolo | a7600f1 | 2020-09-21 16:09:27 -0700 | [diff] [blame] | 617 | sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 hosts > $LOG_FOLDER/onos-hosts-list.txt || true |
Matteo Scandolo | 32da27d | 2020-05-29 10:38:31 -0700 | [diff] [blame] | 618 | |
| 619 | if [ ${withFlows} = true ] ; then |
Matteo Scandolo | f54c9b3 | 2020-08-27 07:58:27 -0700 | [diff] [blame] | 620 | sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 volt-olts > $LOG_FOLDER/onos-olt-list.txt || true |
| 621 | sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 flows -s > $LOG_FOLDER/onos-flows-list.txt || true |
| 622 | sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 meters > $LOG_FOLDER/onos-meters-list.txt || true |
Matteo Scandolo | 32da27d | 2020-05-29 10:38:31 -0700 | [diff] [blame] | 623 | fi |
| 624 | |
Matteo Scandolo | 0bf7e9e | 2020-05-29 18:17:50 -0700 | [diff] [blame] | 625 | if [ ${provisionSubscribers} = true ]; then |
Matteo Scandolo | f54c9b3 | 2020-08-27 07:58:27 -0700 | [diff] [blame] | 626 | sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 volt-programmed-subscribers > $LOG_FOLDER/onos-programmed-subscribers.txt || true |
| 627 | sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 volt-programmed-meters > $LOG_FOLDER/onos-programmed-meters.txt || true |
Matteo Scandolo | 7e46c96 | 2020-10-30 17:10:43 -0700 | [diff] [blame] | 628 | sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 volt-bpmeter-mappings > $LOG_FOLDER/onos-bpmeter-mappings.txt || true |
| 629 | sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 volt-failed-subscribers > $LOG_FOLDER/onos-failed-subscribers.txt || true |
Matteo Scandolo | 32da27d | 2020-05-29 10:38:31 -0700 | [diff] [blame] | 630 | fi |
| 631 | |
Matteo Scandolo | 172c9ca | 2020-06-10 12:23:48 -0700 | [diff] [blame] | 632 | if [ ${withEapol} = true ] ; then |
Matteo Scandolo | f54c9b3 | 2020-08-27 07:58:27 -0700 | [diff] [blame] | 633 | sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 aaa-users > $LOG_FOLDER/onos-aaa-users.txt || true |
| 634 | sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 aaa-statistics > $LOG_FOLDER/onos-aaa-statistics.txt || true |
Matteo Scandolo | 32da27d | 2020-05-29 10:38:31 -0700 | [diff] [blame] | 635 | fi |
| 636 | |
Matteo Scandolo | 172c9ca | 2020-06-10 12:23:48 -0700 | [diff] [blame] | 637 | if [ ${withDhcp} = true ] ; then |
Matteo Scandolo | f54c9b3 | 2020-08-27 07:58:27 -0700 | [diff] [blame] | 638 | sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 dhcpl2relay-allocations > $LOG_FOLDER/onos-dhcp-allocations.txt || true |
Matteo Scandolo | 32da27d | 2020-05-29 10:38:31 -0700 | [diff] [blame] | 639 | fi |
Matteo Scandolo | b70b3e0 | 2020-05-07 11:50:26 -0700 | [diff] [blame] | 640 | ''' |
Shrey Baid | cf652c8 | 2020-05-21 13:35:04 +0530 | [diff] [blame] | 641 | // collect etcd metrics |
| 642 | sh ''' |
Matteo Scandolo | 77fa78d | 2020-09-16 16:53:50 -0700 | [diff] [blame] | 643 | mkdir -p $WORKSPACE/etcd-metrics |
Matteo Scandolo | 9214ebd | 2020-07-30 17:57:10 -0700 | [diff] [blame] | 644 | curl -s -X GET -G http://10.90.0.101:31301/api/v1/query --data-urlencode 'query=etcd_debugging_mvcc_keys_total' | jq '.data' > $WORKSPACE/etcd-metrics/etcd-key-count.json || true |
| 645 | curl -s -X GET -G http://10.90.0.101:31301/api/v1/query --data-urlencode 'query=grpc_server_handled_total{grpc_service="etcdserverpb.KV"}' | jq '.data' > $WORKSPACE/etcd-metrics/etcd-rpc-count.json || true |
| 646 | curl -s -X GET -G http://10.90.0.101:31301/api/v1/query --data-urlencode 'query=etcd_debugging_mvcc_db_total_size_in_bytes' | jq '.data' > $WORKSPACE/etcd-metrics/etcd-db-size.json || true |
| 647 | curl -s -X GET -G http://10.90.0.101:31301/api/v1/query --data-urlencode 'query=etcd_disk_backend_commit_duration_seconds_sum' | jq '.data' > $WORKSPACE/etcd-metrics/etcd-backend-write-time.json || true |
Andrea Campanella | 31fd31f | 2020-11-20 19:02:58 +0100 | [diff] [blame] | 648 | curl -s -X GET -G http://10.90.0.101:31301/api/v1/query --data-urlencode 'query=etcd_disk_backend_commit_duration_seconds_sum' | jq '.data' > $WORKSPACE/etcd-metrics/etcd-backend-write-time-sum.json || true |
| 649 | curl -s -X GET -G http://10.90.0.101:31301/api/v1/query --data-urlencode 'query=etcd_disk_backend_commit_duration_seconds_bucket' | jq '.data' > $WORKSPACE/etcd-metrics/etcd-backend-write-time-bucket.json || true |
| 650 | curl -s -X GET -G http://10.90.0.101:31301/api/v1/query --data-urlencode 'query=etcd_disk_wal_fsync_duration_seconds_bucket' | jq '.data' > $WORKSPACE/etcd-metrics/etcd-wal-fsync-time-bucket.json || true |
Andrea Campanella | 23a1c32 | 2020-11-25 17:11:55 +0100 | [diff] [blame] | 651 | curl -s -X GET -G http://10.90.0.101:31301/api/v1/query --data-urlencode 'query=etcd_network_peer_round_trip_time_seconds_bucket' | jq '.data' > $WORKSPACE/etcd-metrics/etcd-network-peer-round-trip-time-seconds.json || true |
| 652 | |
Shrey Baid | cf652c8 | 2020-05-21 13:35:04 +0530 | [diff] [blame] | 653 | ''' |
Matteo Scandolo | 32da27d | 2020-05-29 10:38:31 -0700 | [diff] [blame] | 654 | // get VOLTHA debug infos |
Matteo Scandolo | 502644b | 2020-05-18 08:56:18 -0700 | [diff] [blame] | 655 | script { |
| 656 | try { |
| 657 | sh ''' |
Matteo Scandolo | f54c9b3 | 2020-08-27 07:58:27 -0700 | [diff] [blame] | 658 | voltctl -m 8MB device list -o json > $LOG_FOLDER/device-list.json || true |
| 659 | python -m json.tool $LOG_FOLDER/device-list.json > $LOG_FOLDER/voltha-devices-list.json || true |
| 660 | rm $LOG_FOLDER/device-list.json || true |
| 661 | voltctl -m 8MB device list > $LOG_FOLDER/voltha-devices-list.txt || true |
Matteo Scandolo | 32da27d | 2020-05-29 10:38:31 -0700 | [diff] [blame] | 662 | |
Matteo Scandolo | be82324 | 2020-11-20 13:48:13 -0800 | [diff] [blame] | 663 | printf '%s\n' $(voltctl -m 8MB device list | grep olt | awk '{print $1}') | xargs --no-run-if-empty -I# bash -c "voltctl -m 8MB device flows # > $LOG_FOLDER/voltha-device-flows-#.txt" || true |
| 664 | printf '%s\n' $(voltctl -m 8MB device list | grep olt | awk '{print $1}') | xargs --no-run-if-empty -I# bash -c "voltctl -m 8MB device port list --format 'table{{.PortNo}}\t{{.Label}}\t{{.Type}}\t{{.AdminState}}\t{{.OperStatus}}' # > $LOG_FOLDER/voltha-device-ports-#.txt" || true |
Matteo Scandolo | 32da27d | 2020-05-29 10:38:31 -0700 | [diff] [blame] | 665 | |
Matteo Scandolo | be82324 | 2020-11-20 13:48:13 -0800 | [diff] [blame] | 666 | printf '%s\n' $(voltctl -m 8MB logicaldevice list -q) | xargs --no-run-if-empty -I# bash -c "voltctl -m 8MB logicaldevice flows # > $LOG_FOLDER/voltha-logicaldevice-flows-#.txt" || true |
| 667 | printf '%s\n' $(voltctl -m 8MB logicaldevice list -q) | xargs --no-run-if-empty -I# bash -c "voltctl -m 8MB logicaldevice port list # > $LOG_FOLDER/voltha-logicaldevice-ports-#.txt" || true |
Matteo Scandolo | 502644b | 2020-05-18 08:56:18 -0700 | [diff] [blame] | 668 | ''' |
| 669 | } catch(e) { |
| 670 | sh ''' |
| 671 | echo "Can't get device list from voltclt" |
| 672 | ''' |
| 673 | } |
| 674 | } |
Matteo Scandolo | 27553c0 | 2020-07-16 17:21:57 -0700 | [diff] [blame] | 675 | // get cpu usage by container |
| 676 | sh ''' |
Matteo Scandolo | 7d6695c | 2020-08-25 13:13:16 -0700 | [diff] [blame] | 677 | if [ ${withMonitoring} = true ] ; then |
| 678 | cd $WORKSPACE/voltha-system-tests |
| 679 | source ./vst_venv/bin/activate |
| 680 | sleep 60 # we have to wait for prometheus to collect all the information |
| 681 | python tests/scale/sizing.py -o $WORKSPACE/plots || true |
| 682 | fi |
Matteo Scandolo | 27553c0 | 2020-07-16 17:21:57 -0700 | [diff] [blame] | 683 | ''' |
Matteo Scandolo | 39c531a | 2021-01-22 16:05:32 -0800 | [diff] [blame^] | 684 | archiveArtifacts artifacts: 'execution-time.txt,logs/*,logs/pprof/*,RobotLogs/*,plots/*,etcd-metrics/*' |
Matteo Scandolo | fbcbdb8 | 2020-05-06 15:41:32 -0700 | [diff] [blame] | 685 | } |
| 686 | } |
| 687 | } |