Blame - jjb/pipeline/voltha/voltha-2.7/voltha-scale-test.groovy - ci-management

EXTRA_HELM_FLAGS+="--set voltha-adapter-openolt.images.adapter_open_olt.repository=\$openoltAdapterRepo,voltha-adapter-openolt.images.adapter_open_olt.tag=\$openoltAdapterTag "

222

fi

223

224

# OpenONU custom image handling

225

if [ '${openonuAdapterImg.trim()}' != '' ] && [ '\$GERRIT_PROJECT' != 'voltha-openonu-adapter' ]; then

226

IFS=: read -r openonuAdapterRepo openonuAdapterTag <<< '${openonuAdapterImg.trim()}'

227

EXTRA_HELM_FLAGS+="--set voltha-adapter-openonu.images.adapter_open_onu.repository=\$openonuAdapterRepo,voltha-adapter-openonu.images.adapter_open_onu.tag=\$openonuAdapterTag "

228

fi

229

230

# OpenONU GO custom image handling

231

if [ '${openonuAdapterGoImg.trim()}' != '' ] && [ '\$GERRIT_PROJECT' != 'voltha-openonu-adapter-go' ]; then

232

IFS=: read -r openonuAdapterGoRepo openonuAdapterGoTag <<< '${openonuAdapterGoImg.trim()}'

233

EXTRA_HELM_FLAGS+="--set voltha-adapter-openonu.images.adapter_open_onu_go.repository=\$openonuAdapterGoRepo,voltha-adapter-openonu.images.adapter_open_onu_go.tag=\$openonuAdapterGoTag "

234

fi

235

236

# ONOS custom image handling

237

if [ '${onosImg.trim()}' != '' ] && [ '\$GERRIT_PROJECT' != 'voltha-onos' ]; then

238

IFS=: read -r onosRepo onosTag <<< '${onosImg.trim()}'

239

EXTRA_HELM_FLAGS+="--set onos-classic.image.repository=\$onosRepo,onos-classic.image.tag=\$onosTag "

240

fi

241

242

# set BBSim parameters

243

EXTRA_HELM_FLAGS+='--set enablePerf=true,pon=${pons},onu=${onus} '

244

245

# disable the securityContext, this is a development cluster

246

EXTRA_HELM_FLAGS+='--set securityContext.enabled=false '

247

248

# No persistent-volume-claims in Atomix

249

EXTRA_HELM_FLAGS+="--set onos-classic.atomix.persistence.enabled=false "

250

251

echo "Installing with the following extra arguments:"

252

echo $EXTRA_HELM_FLAGS

# Use custom built images

257

258

if [ '\$GERRIT_PROJECT' == 'voltha-go' ]; then

259

EXTRA_HELM_FLAGS+="--set voltha.images.rw_core.repository=${dockerRegistry}/voltha/voltha-rw-core,voltha.images.rw_core.tag=voltha-scale "

260

fi

261

262

if [ '\$GERRIT_PROJECT' == 'voltha-openolt-adapter' ]; then

263

EXTRA_HELM_FLAGS+="--set voltha-openolt-adapter.images.adapter_open_olt.repository=${dockerRegistry}/voltha/voltha-openolt-adapter,voltha-openolt-adapter.images.adapter_open_olt.tag=voltha-scale "

264

fi

265

266

if [ '\$GERRIT_PROJECT' == 'voltha-openonu-adapter' ]; then

267

EXTRA_HELM_FLAGS+="--set voltha-openonu-adapter.images.adapter_open_onu.repository=${dockerRegistry}/voltha/voltha-openonu-adapter,voltha-openonu-adapter.images.adapter_open_onu.tag=voltha-scale "

268

fi

269

270

if [ '\$GERRIT_PROJECT' == 'voltha-openonu-adapter-go' ]; then

271

EXTRA_HELM_FLAGS+="--set voltha-openonu-adapter-go.images.adapter_open_onu_go.repository=${dockerRegistry}/voltha/voltha-openonu-adapter-go,voltha-openonu-adapter-go.images.adapter_open_onu_go.tag=voltha-scale "

272

fi

273

274

if [ '\$GERRIT_PROJECT' == 'ofagent-go' ]; then

275

EXTRA_HELM_FLAGS+="--set voltha.images.ofagent.repository=${dockerRegistry}/voltha/voltha-ofagent-go,ofagent-go.images.ofagent.tag=voltha-scale "

276

fi

277

278

if [ '\$GERRIT_PROJECT' == 'voltha-onos' ]; then

279

EXTRA_HELM_FLAGS+="--set onos-classic.image.repository=${dockerRegistry}/voltha/voltha-onos,onos-classic.image.tag=voltha-scale "

280

fi

281

282

if [ '\$GERRIT_PROJECT' == 'bbsim' ]; then

283

EXTRA_HELM_FLAGS+="--set images.bbsim.repository=${dockerRegistry}/voltha/bbsim,images.bbsim.tag=voltha-scale "

284

fi

285

286

helm upgrade --install voltha-infra onf/voltha-infra \$EXTRA_HELM_FLAGS \

287

--set onos-classic.replicas=${onosReplicas},onos-classic.atomix.replicas=${atomixReplicas} \

288

--set etcd.enabled=false,kafka.enabled=false \

289

--set global.log_level=${logLevel} \

290

-f $WORKSPACE/voltha-helm-charts/examples/${workflow}-values.yaml \

291

--version 0.1.13

292

293

helm upgrade --install voltha1 onf/voltha-stack \$EXTRA_HELM_FLAGS \

294

--set global.stack_name=voltha1 \

295

--set global.voltha_infra_name=voltha-infra \

296

--set global.voltha_infra_namespace=default \

297

--set global.log_level=${logLevel} \

298

${ofAgentConnections(onosReplicas.toInteger(), "voltha-infra", "default")} \

299

--set voltha.services.kafka.adapter.address=kafka.default.svc:9092 \

300

--set voltha.services.kafka.cluster.address=kafka.default.svc:9092 \

301

--set voltha.services.etcd.address=etcd.default.svc:2379 \

302

--set voltha-adapter-openolt.services.kafka.adapter.address=kafka.default.svc:9092 \

303

--set voltha-adapter-openolt.services.kafka.cluster.address=kafka.default.svc:9092 \

304

--set voltha-adapter-openolt.services.etcd.address=etcd.default.svc:2379 \

305

--set voltha-adapter-openonu.services.kafka.adapter.address=kafka.default.svc:9092 \

306

--set voltha-adapter-openonu.services.kafka.cluster.address=kafka.default.svc:9092 \

307

--set voltha-adapter-openonu.services.etcd.address=etcd.default.svc:2379 \

--version 0.1.17

for i in {0..${olts.toInteger() - 1}}; do

312

stackId=1

313

helm upgrade --install bbsim\$i onf/bbsim \$EXTRA_HELM_FLAGS \

314

--set olt_id="\${stackId}\${i}" \

315

--set onu=${onus},pon=${pons} \

316

--set global.log_level=${logLevel.toLowerCase()} \

317

-f $WORKSPACE/voltha-helm-charts/examples/${workflow}-values.yaml \

--version 4.2.0

done

"""

sh """

set +x

echo -ne "\nWaiting for VOLTHA and ONOS to start..."

325

voltha=\$(kubectl get pods --all-namespaces -l app.kubernetes.io/part-of=voltha --no-headers | grep "0/" | wc -l)

326

onos=\$(kubectl get pods --all-namespaces -l app=onos-classic --no-headers | grep "0/" | wc -l)

327

while [[ \$voltha != 0 || \$onos != 0 ]]; do

328

sleep 5

329

echo -ne "."

330

voltha=\$(kubectl get pods --all-namespaces -l app.kubernetes.io/part-of=voltha --no-headers | grep "0/" | wc -l)

331

onos=\$(kubectl get pods --all-namespaces -l app=onos-classic --no-headers | grep "0/" | wc -l)

332

done

333

echo -ne "\nVOLTHA and ONOS pods ready\n"

334

kubectl get pods --all-namespaces -l app.kubernetes.io/part-of=voltha --no-headers | grep "0/" | wc -l

335

kubectl get pods --all-namespaces -l app=onos-classic --no-headers | grep "0/" | wc -l

336

"""

337

start_port_forward(olts)

}

}

}

}

stage('Configuration') {

343

steps {

344

script {

345

def tech_prof_directory = "XGS-PON"

346

sh returnStdout: false, script: """

347

#Setting link discovery

348

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 cfg set org.onosproject.provider.lldp.impl.LldpLinkProvider enabled ${withLLDP}

349

350

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 cfg set org.onosproject.net.flow.impl.FlowRuleManager allowExtraneousRules true

351

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 cfg set org.onosproject.net.flow.impl.FlowRuleManager importExtraneousRules true

352

353

354

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 cfg set org.onosproject.net.flowobjective.impl.InOrderFlowObjectiveManager accumulatorMaxBatchMillis 900

355

356

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 cfg set org.onosproject.net.flowobjective.impl.InOrderFlowObjectiveManager accumulatorMaxIdleMillis 500

357

358

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 cfg set org.opencord.olt.impl.Olt provisionDelay 1000

359

360

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 log:set ${logLevel} org.onosproject

361

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 log:set ${logLevel} org.opencord

362

363

# sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 log:set DEBUG org.opencord.cordmcast

364

# sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 log:set DEBUG org.onosproject.mcast

365

# sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 log:set DEBUG org.opencord.igmpproxy

366

# sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 log:set DEBUG org.opencord.olt

367

368

kubectl exec \$(kubectl get pods | grep -E "bbsim[0-9]" | awk 'NR==1{print \$1}') -- bbsimctl log ${logLevel.toLowerCase()} false

369

370

# Set Flows/Ports/Meters poll frequency

371

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 cfg set org.onosproject.provider.of.flow.impl.OpenFlowRuleProvider flowPollFrequency ${onosStatInterval}

372

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 cfg set org.onosproject.provider.of.device.impl.OpenFlowDeviceProvider portStatsPollFrequency ${onosStatInterval}

373

374

if [ ${withFlows} = false ]; then

375

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 app deactivate org.opencord.olt

376

fi

377

378

if [ '${workflow}' = 'tt' ]; then

379

etcd_container=\$(kubectl get pods --all-namespaces | grep etcd | awk 'NR==1{print \$2}')

380

kubectl cp $WORKSPACE/voltha-system-tests/tests/data/TechProfile-TT-HSIA.json \$etcd_container:/tmp/hsia.json

381

put_result=\$(kubectl exec -it \$etcd_container -- /bin/sh -c 'cat /tmp/hsia.json | ETCDCTL_API=3 etcdctl put service/voltha/technology_profiles/${tech_prof_directory}/64')

382

kubectl cp $WORKSPACE/voltha-system-tests/tests/data/TechProfile-TT-VoIP.json \$etcd_container:/tmp/voip.json

383

put_result=\$(kubectl exec -it \$etcd_container -- /bin/sh -c 'cat /tmp/voip.json | ETCDCTL_API=3 etcdctl put service/voltha/technology_profiles/${tech_prof_directory}/65')

384

kubectl cp $WORKSPACE/voltha-system-tests/tests/data/TechProfile-TT-MCAST.json \$etcd_container:/tmp/mcast.json

385

put_result=\$(kubectl exec -it \$etcd_container -- /bin/sh -c 'cat /tmp/mcast.json | ETCDCTL_API=3 etcdctl put service/voltha/technology_profiles/${tech_prof_directory}/66')

386

fi

387

388

if [ ${withPcap} = true ] ; then

389

# Start the tcp-dump in ofagent

390

export OF_AGENT=\$(kubectl get pods -l app=ofagent -o name)

391

kubectl exec \$OF_AGENT -- apk update

392

kubectl exec \$OF_AGENT -- apk add tcpdump

393

kubectl exec \$OF_AGENT -- mv /usr/sbin/tcpdump /usr/bin/tcpdump

394

_TAG=ofagent-tcpdump kubectl exec \$OF_AGENT -- tcpdump -nei eth0 -w out.pcap&

395

396

# Start the tcp-dump in radius

397

export RADIUS=\$(kubectl get pods -l app=radius -o name)

398

kubectl exec \$RADIUS -- apt-get update

399

kubectl exec \$RADIUS -- apt-get install -y tcpdump

400

_TAG=radius-tcpdump kubectl exec \$RADIUS -- tcpdump -w out.pcap&

401

402

# Start the tcp-dump in ONOS

403

for i in \$(seq 0 \$ONOSES); do

404

INSTANCE="onos-onos-classic-\$i"

405

kubectl exec \$INSTANCE -- apt-get update

406

kubectl exec \$INSTANCE -- apt-get install -y tcpdump

407

kubectl exec \$INSTANCE -- mv /usr/sbin/tcpdump /usr/bin/tcpdump

408

_TAG=\$INSTANCE kubectl exec \$INSTANCE -- /usr/bin/tcpdump -nei eth0 port 1812 -w out.pcap&

done

fi

"""

}

}

}

stage('Run Test') {

steps {

sh '''

mkdir -p $WORKSPACE/RobotLogs

419

cd $WORKSPACE/voltha-system-tests

make vst_venv

'''

sh '''

if [ ${withProfiling} = true ] ; then

424

mkdir -p $LOG_FOLDER/pprof

425

echo $PATH

426

#Creating Python script for ONU Detection

427

cat << EOF > $WORKSPACE/pprof.sh

timestamp() {

date +"%T"

}

i=0

while [[ true ]]; do

((i++))

ts=$(timestamp)

go tool pprof -png http://127.0.0.1:6060/debug/pprof/heap > $LOG_FOLDER/pprof/rw-core-heap-\\$i-\\$ts.png

437

go tool pprof -png http://127.0.0.1:6060/debug/pprof/goroutine > $LOG_FOLDER/pprof/rw-core-goroutine-\\$i-\\$ts.png

438

curl -o $LOG_FOLDER/pprof/rw-core-profile-\\$i-\\$ts.pprof http://127.0.0.1:6060/debug/pprof/profile?seconds=10

439

go tool pprof -png $LOG_FOLDER/pprof/rw-core-profile-\\$i-\\$ts.pprof > $LOG_FOLDER/pprof/rw-core-profile-\\$i-\\$ts.png

440

441

go tool pprof -png http://127.0.0.1:6061/debug/pprof/heap > $LOG_FOLDER/pprof/openolt-heap-\\$i-\\$ts.png

442

go tool pprof -png http://127.0.0.1:6061/debug/pprof/goroutine > $LOG_FOLDER/pprof/openolt-goroutine-\\$i-\\$ts.png

443

curl -o $LOG_FOLDER/pprof/openolt-profile-\\$i-\\$ts.pprof http://127.0.0.1:6061/debug/pprof/profile?seconds=10

444

go tool pprof -png $LOG_FOLDER/pprof/openolt-profile-\\$i-\\$ts.pprof > $LOG_FOLDER/pprof/openolt-profile-\\$i-\\$ts.png

445

446

go tool pprof -png http://127.0.0.1:6062/debug/pprof/heap > $LOG_FOLDER/pprof/ofagent-heap-\\$i-\\$ts.png

447

go tool pprof -png http://127.0.0.1:6062/debug/pprof/goroutine > $LOG_FOLDER/pprof/ofagent-goroutine-\\$i-\\$ts.png

448

curl -o $LOG_FOLDER/pprof/ofagent-profile-\\$i-\\$ts.pprof http://127.0.0.1:6062/debug/pprof/profile?seconds=10

449

go tool pprof -png $LOG_FOLDER/pprof/ofagent-profile-\\$i-\\$ts.pprof > $LOG_FOLDER/pprof/ofagent-profile-\\$i-\\$ts.png

sleep 10

done

EOF

_TAG="pprof"

_TAG=$_TAG bash $WORKSPACE/pprof.sh &

457

fi

458

'''

459

timeout(time: 15, unit: 'MINUTES') {

460

sh '''

461

ROBOT_PARAMS="--exitonfailure \

-v olt:${olts} \

-v pon:${pons} \

-v onu:${onus} \

-v workflow:${workflow} \

466

-v withEapol:${withEapol} \

467

-v withDhcp:${withDhcp} \

468

-v withIgmp:${withIgmp} \

469

--noncritical non-critical \

470

-e igmp -e teardown "

471

472

if [ ${withEapol} = false ] ; then

473

ROBOT_PARAMS+="-e authentication "

474

fi

475

476

if [ ${withDhcp} = false ] ; then

477

ROBOT_PARAMS+="-e dhcp "

478

fi

479

480

if [ ${provisionSubscribers} = false ] ; then

481

# if we're not considering subscribers then we don't care about authentication and dhcp

482

ROBOT_PARAMS+="-e authentication -e provision -e flow-after -e dhcp "

483

fi

484

485

if [ ${withFlows} = false ] ; then

486

ROBOT_PARAMS+="-i setup -i activation "

487

fi

488

489

cd $WORKSPACE/voltha-system-tests

490

source ./vst_venv/bin/activate

491

robot -d $WORKSPACE/RobotLogs \

492

$ROBOT_PARAMS tests/scale/Voltha_Scale_Tests.robot

'''

}

}

}

stage('Run Igmp Tests') {

498

environment {

499

ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/IgmpTests"

}

when {

expression {

return params.withIgmp

}

}

steps {

sh '''

set +e

mkdir -p $ROBOT_LOGS_DIR

510

cd $WORKSPACE/voltha-system-tests

511

make vst_venv

512

'''

513

timeout(time: 11, unit: 'MINUTES') {

514

sh '''

515

ROBOT_PARAMS="--exitonfailure \

-v olt:${olts} \

-v pon:${pons} \

-v onu:${onus} \

-v workflow:${workflow} \

520

-v withEapol:${withEapol} \

521

-v withDhcp:${withDhcp} \

522

-v withIgmp:${withIgmp} \

523

--noncritical non-critical \

524

-i igmp \

525

-e setup -e activation -e flow-before \

526

-e authentication -e provision -e flow-after \

527

-e dhcp -e teardown "

528

cd $WORKSPACE/voltha-system-tests

529

source ./vst_venv/bin/activate

530

robot -d $ROBOT_LOGS_DIR \

531

$ROBOT_PARAMS tests/scale/Voltha_Scale_Tests.robot

'''

}

}

}

}

post {

always {

// collect result, done in the "post" step so it's executed even in the

540

// event of a timeout in the tests

541

sh '''

542

543

# stop the kail processes

544

list=($APPS_TO_LOG)

545

for app in "${list[@]}"

546

do

547

echo "Stopping logs for: ${app}"

548

_TAG="kail-$app"

549

P_IDS="$(ps e -ww -A | grep "_TAG=$_TAG" | grep -v grep | awk '{print $1}')"

550

if [ -n "$P_IDS" ]; then

551

echo $P_IDS

552

for P_ID in $P_IDS; do

kill -9 $P_ID

done

fi

done

if [ ${withPcap} = true ] ; then

559

# stop ofAgent tcpdump

560

P_ID="\$(ps e -ww -A | grep "_TAG=ofagent-tcpdump" | grep -v grep | awk '{print \$1}')"

561

if [ -n "\$P_ID" ]; then

kill -9 \$P_ID

fi

# stop radius tcpdump

566

P_ID="\$(ps e -ww -A | grep "_TAG=radius-tcpdump" | grep -v grep | awk '{print \$1}')"

567

if [ -n "\$P_ID" ]; then

kill -9 \$P_ID

fi

# stop onos tcpdump

LIMIT=$(($NUM_OF_ONOS - 1))

573

for i in $(seq 0 $LIMIT); do

574

INSTANCE="onos-onos-classic-$i"

575

P_ID="\$(ps e -ww -A | grep "_TAG=$INSTANCE" | grep -v grep | awk '{print \$1}')"

576

if [ -n "\$P_ID" ]; then

kill -9 \$P_ID

fi

done

# copy the file

export OF_AGENT=$(kubectl get pods -l app=ofagent | awk 'NR==2{print $1}') || true

583

kubectl cp $OF_AGENT:out.pcap $LOG_FOLDER/ofagent.pcap || true

584

585

export RADIUS=$(kubectl get pods -l app=radius | awk 'NR==2{print $1}') || true

586

kubectl cp $RADIUS:out.pcap $LOG_FOLDER/radius.pcap || true

587

588

LIMIT=$(($NUM_OF_ONOS - 1))

589

for i in $(seq 0 $LIMIT); do

590

INSTANCE="onos-onos-classic-$i"

591

kubectl cp $INSTANCE:out.pcap $LOG_FOLDER/$INSTANCE.pcap || true

done

fi

cd voltha-system-tests

596

source ./vst_venv/bin/activate

597

python tests/scale/collect-result.py -r $WORKSPACE/RobotLogs/output.xml -p $WORKSPACE/plots > $WORKSPACE/execution-time.txt || true

598

cat $WORKSPACE/execution-time.txt

599

'''

600

sh '''

601

if [ ${withProfiling} = true ] ; then

602

_TAG="pprof"

603

P_IDS="$(ps e -ww -A | grep "_TAG=$_TAG" | grep -v grep | awk '{print $1}')"

604

if [ -n "$P_IDS" ]; then

605

echo $P_IDS

606

for P_ID in $P_IDS; do

kill -9 $P_ID

done

fi

fi

'''

plot([

csvFileName: 'scale-test.csv',

614

csvSeries: [

615

[file: 'plots/plot-voltha-onus.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''],

616

[file: 'plots/plot-onos-ports.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''],

617

[file: 'plots/plot-voltha-flows-before.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''],

618

[file: 'plots/plot-voltha-openolt-flows-before.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''],

619

[file: 'plots/plot-onos-flows-before.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''],

620

[file: 'plots/plot-onos-auth.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''],

621

[file: 'plots/plot-voltha-flows-after.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''],

622

[file: 'plots/plot-voltha-openolt-flows-after.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''],

623

[file: 'plots/plot-onos-flows-after.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''],

624

[file: 'plots/plot-onos-dhcp.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''],

625

],

626

group: 'Voltha-Scale-Numbers', numBuilds: '20', style: 'line', title: "Scale Test (OLTs: ${olts}, PONs: ${pons}, ONUs: ${onus})", yaxis: 'Time (s)', useDescr: true

627

])

628

step([$class: 'RobotPublisher',

629

disableArchiveOutput: false,

630

logFileName: '**/log*.html',

631

otherFiles: '',

632

outputFileName: '**/output*.xml',

633

outputPath: 'RobotLogs',

634

passThreshold: 100,

635

reportFileName: '**/report*.html',

636

unstableThreshold: 0]);

637

// get all the logs from kubernetes PODs

638

sh returnStdout: false, script: '''

639

640

# store information on running charts

641

helm ls > $LOG_FOLDER/helm-list.txt || true

642

643

# store information on the running pods

644

kubectl get pods -o wide > $LOG_FOLDER/pods.txt || true

645

kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.image}{'\\n'}" | sort | uniq | tee $LOG_FOLDER/pod-images.txt || true

646

kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.imageID}{'\\n'}" | sort | uniq | tee $LOG_FOLDER/pod-imagesId.txt || true

647

648

# copy the ONOS logs directly from the container to avoid the color codes

649

printf '%s\n' $(kubectl get pods -l app=onos-classic -o=jsonpath="{.items[*]['metadata.name']}") | xargs --no-run-if-empty -I# bash -c "kubectl cp #:${karafHome}/data/log/karaf.log $LOG_FOLDER/#.log" || true

650

651

# get ONOS cfg from the 3 nodes

652

printf '%s\n' $(kubectl get pods -l app=onos-classic -o=jsonpath="{.items[*]['metadata.name']}") | xargs --no-run-if-empty -I# bash -c "kubectl exec -it # -- ${karafHome}/bin/client cfg get > $LOG_FOLDER/#.cfg" || true

653

654

655

# get radius logs out of the container

656

kubectl cp $(kubectl get pods -l app=radius --no-headers | awk '{print $1}'):/var/log/freeradius/radius.log $LOG_FOLDER/radius.log || true

657

'''

658

// dump all the BBSim(s) ONU information

659

sh '''

660

BBSIM_IDS=$(kubectl get pods | grep bbsim | grep -v server | awk '{print $1}')

661

IDS=($BBSIM_IDS)

662

663

for bbsim in "${IDS[@]}"

664

do

665

kubectl exec -t $bbsim -- bbsimctl onu list > $LOG_FOLDER/$bbsim-device-list.txt || true

666

kubectl exec -t $bbsim -- bbsimctl service list > $LOG_FOLDER/$bbsim-service-list.txt || true

667

kubectl exec -t $bbsim -- bbsimctl olt resources GEM_PORT > $LOG_FOLDER/$bbsim-flows-gem-ports.txt || true

668

kubectl exec -t $bbsim -- bbsimctl olt resources ALLOC_ID > $LOG_FOLDER/$bbsim-flows-alloc-ids.txt || true

669

kubectl exec -t $bbsim -- bbsimctl olt pons > $LOG_FOLDER/$bbsim-pon-resources.txt || true

done

'''

script {

// first make sure the port-forward is still running,

674

// sometimes Jenkins kills it relardless of the JENKINS_NODE_COOKIE=dontKillMe

def running = sh (

returnStdout: true

).trim()

// if any of the voltha-api, onos-rest, onos-ssh port-forwards are not there

680

// kill all and restart

681

if (running != "3") {

682

start_port_forward(olts)

683

}

684

}

685

// get ONOS debug infos

686

sh '''

687

688

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 apps -a -s > $LOG_FOLDER/onos-apps.txt

689

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 nodes > $LOG_FOLDER/onos-nodes.txt

690

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 masters > $LOG_FOLDER/onos-masters.txt

691

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 roles > $LOG_FOLDER/onos-roles.txt

692

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 cfg get > $LOG_FOLDER/onos-cfg.txt

693

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 netcfg > $LOG_FOLDER/onos-netcfg.txt

694

695

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 ports > $LOG_FOLDER/onos-ports-list.txt

696

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 hosts > $LOG_FOLDER/onos-hosts-list.txt

697

698

if [ ${withFlows} = true ] ; then

699

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 volt-olts > $LOG_FOLDER/onos-olt-list.txt

700

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 flows -s > $LOG_FOLDER/onos-flows-list.txt

701

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 meters > $LOG_FOLDER/onos-meters-list.txt

702

fi

703

704

if [ ${provisionSubscribers} = true ]; then

705

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 volt-programmed-subscribers > $LOG_FOLDER/onos-programmed-subscribers.txt

706

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 volt-programmed-meters > $LOG_FOLDER/onos-programmed-meters.txt

707

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 volt-bpmeter-mappings > $LOG_FOLDER/onos-bpmeter-mappings.txt

708

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 volt-failed-subscribers > $LOG_FOLDER/onos-failed-subscribers.txt

709

fi

710

711

if [ ${withEapol} = true ] ; then

712

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 aaa-users > $LOG_FOLDER/onos-aaa-users.txt

713

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 aaa-statistics > $LOG_FOLDER/onos-aaa-statistics.txt

714

fi

715

716

if [ ${withDhcp} = true ] ; then

717

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 dhcpl2relay-allocations > $LOG_FOLDER/onos-dhcp-allocations.txt

718

fi

719

720

if [ ${withIgmp} = true ] ; then

721

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 mcast-host-routes > $LOG_FOLDER/onos-mcast-host-routes.txt

722

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 mcast-host-show > $LOG_FOLDER/onos-mcast-host-show.txt

723

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 groups > $LOG_FOLDER/onos-groups.txt

724

fi

725

'''

726

// collect etcd metrics

727

sh '''

728

mkdir -p $WORKSPACE/etcd-metrics

729

curl -s -X GET -G http://10.90.0.101:31301/api/v1/query --data-urlencode 'query=etcd_debugging_mvcc_keys_total' | jq '.data' > $WORKSPACE/etcd-metrics/etcd-key-count.json || true

730

curl -s -X GET -G http://10.90.0.101:31301/api/v1/query --data-urlencode 'query=grpc_server_handled_total{grpc_service="etcdserverpb.KV"}' | jq '.data' > $WORKSPACE/etcd-metrics/etcd-rpc-count.json || true

731

curl -s -X GET -G http://10.90.0.101:31301/api/v1/query --data-urlencode 'query=etcd_debugging_mvcc_db_total_size_in_bytes' | jq '.data' > $WORKSPACE/etcd-metrics/etcd-db-size.json || true

732

curl -s -X GET -G http://10.90.0.101:31301/api/v1/query --data-urlencode 'query=etcd_disk_backend_commit_duration_seconds_sum' | jq '.data' > $WORKSPACE/etcd-metrics/etcd-backend-write-time.json || true

733

curl -s -X GET -G http://10.90.0.101:31301/api/v1/query --data-urlencode 'query=etcd_disk_backend_commit_duration_seconds_sum' | jq '.data' > $WORKSPACE/etcd-metrics/etcd-backend-write-time-sum.json || true

734

curl -s -X GET -G http://10.90.0.101:31301/api/v1/query --data-urlencode 'query=etcd_disk_backend_commit_duration_seconds_bucket' | jq '.data' > $WORKSPACE/etcd-metrics/etcd-backend-write-time-bucket.json || true

735

curl -s -X GET -G http://10.90.0.101:31301/api/v1/query --data-urlencode 'query=etcd_disk_wal_fsync_duration_seconds_bucket' | jq '.data' > $WORKSPACE/etcd-metrics/etcd-wal-fsync-time-bucket.json || true

736

curl -s -X GET -G http://10.90.0.101:31301/api/v1/query --data-urlencode 'query=etcd_network_peer_round_trip_time_seconds_bucket' | jq '.data' > $WORKSPACE/etcd-metrics/etcd-network-peer-round-trip-time-seconds.json || true

737

738

'''

739

// get VOLTHA debug infos

script {

try {

sh '''

voltctl -m 8MB device list -o json > $LOG_FOLDER/device-list.json || true

744

python -m json.tool $LOG_FOLDER/device-list.json > $LOG_FOLDER/voltha-devices-list.json || true

745

rm $LOG_FOLDER/device-list.json || true

746

voltctl -m 8MB device list > $LOG_FOLDER/voltha-devices-list.txt || true

747

748

printf '%s\n' $(voltctl -m 8MB device list | grep olt | awk '{print $1}') | xargs --no-run-if-empty -I# bash -c "voltctl -m 8MB device flows # > $LOG_FOLDER/voltha-device-flows-#.txt" || true

749

printf '%s\n' $(voltctl -m 8MB device list | grep olt | awk '{print $1}') | xargs --no-run-if-empty -I# bash -c "voltctl -m 8MB device port list --format 'table{{.PortNo}}\t{{.Label}}\t{{.Type}}\t{{.AdminState}}\t{{.OperStatus}}' # > $LOG_FOLDER/voltha-device-ports-#.txt" || true

750

751

printf '%s\n' $(voltctl -m 8MB logicaldevice list -q) | xargs --no-run-if-empty -I# bash -c "voltctl -m 8MB logicaldevice flows # > $LOG_FOLDER/voltha-logicaldevice-flows-#.txt" || true

752

printf '%s\n' $(voltctl -m 8MB logicaldevice list -q) | xargs --no-run-if-empty -I# bash -c "voltctl -m 8MB logicaldevice port list # > $LOG_FOLDER/voltha-logicaldevice-ports-#.txt" || true

'''

} catch(e) {

sh '''

echo "Can't get device list from voltclt"

'''

}

}

// get cpu usage by container

761

sh '''

762

if [ ${withMonitoring} = true ] ; then

763

cd $WORKSPACE/voltha-system-tests

764

source ./vst_venv/bin/activate

765

sleep 60 # we have to wait for prometheus to collect all the information

766

python tests/scale/sizing.py -o $WORKSPACE/plots || true

767

fi

768

'''

769

archiveArtifacts artifacts: 'execution-time.txt,logs/*,logs/pprof/*,RobotLogs/**/*,plots/*,etcd-metrics/*'

}

}

}

def start_port_forward(olts) {

775

sh """

776

daemonize -E JENKINS_NODE_COOKIE="dontKillMe" /usr/local/bin/kubectl port-forward --address 0.0.0.0 -n default svc/voltha-infra-onos-classic-hs 8101:8101

777

daemonize -E JENKINS_NODE_COOKIE="dontKillMe" /usr/local/bin/kubectl port-forward --address 0.0.0.0 -n default svc/voltha-infra-onos-classic-hs 8181:8181

778

daemonize -E JENKINS_NODE_COOKIE="dontKillMe" /usr/local/bin/kubectl port-forward --address 0.0.0.0 -n default svc/voltha1-voltha-api 55555:55555

779

780

bbsimRestPortFwd=50071

781

for i in {0..${olts.toInteger() - 1}}; do

782

daemonize -E JENKINS_NODE_COOKIE="dontKillMe" /usr/local/bin/kubectl port-forward --address 0.0.0.0 -n default svc/bbsim\${i} \${bbsimRestPortFwd}:50071

783

((bbsimRestPortFwd++))

784

done

785

"""

786

}