Blame - jjb/pipeline/voltha/voltha-2.7/voltha-scale-test.groovy - ci-management

EXTRA_HELM_FLAGS+="--set voltha-adapter-openolt.images.adapter_open_olt.repository=\$openoltAdapterRepo,voltha-adapter-openolt.images.adapter_open_olt.tag=\$openoltAdapterTag "

222

fi

223

224

# OpenONU custom image handling

225

if [ '${openonuAdapterImg.trim()}' != '' ] && [ '\$GERRIT_PROJECT' != 'voltha-openonu-adapter' ]; then

226

IFS=: read -r openonuAdapterRepo openonuAdapterTag <<< '${openonuAdapterImg.trim()}'

227

EXTRA_HELM_FLAGS+="--set voltha-adapter-openonu.images.adapter_open_onu.repository=\$openonuAdapterRepo,voltha-adapter-openonu.images.adapter_open_onu.tag=\$openonuAdapterTag "

228

fi

229

230

# OpenONU GO custom image handling

231

if [ '${openonuAdapterGoImg.trim()}' != '' ] && [ '\$GERRIT_PROJECT' != 'voltha-openonu-adapter-go' ]; then

232

IFS=: read -r openonuAdapterGoRepo openonuAdapterGoTag <<< '${openonuAdapterGoImg.trim()}'

233

EXTRA_HELM_FLAGS+="--set voltha-adapter-openonu.images.adapter_open_onu_go.repository=\$openonuAdapterGoRepo,voltha-adapter-openonu.images.adapter_open_onu_go.tag=\$openonuAdapterGoTag "

234

fi

235

236

# ONOS custom image handling

237

if [ '${onosImg.trim()}' != '' ] && [ '\$GERRIT_PROJECT' != 'voltha-onos' ]; then

238

IFS=: read -r onosRepo onosTag <<< '${onosImg.trim()}'

239

EXTRA_HELM_FLAGS+="--set onos-classic.image.repository=\$onosRepo,onos-classic.image.tag=\$onosTag "

240

fi

241

242

# set BBSim parameters

243

EXTRA_HELM_FLAGS+='--set enablePerf=true,pon=${pons},onu=${onus} '

244

245

# disable the securityContext, this is a development cluster

246

EXTRA_HELM_FLAGS+='--set securityContext.enabled=false '

247

248

# No persistent-volume-claims in Atomix

249

EXTRA_HELM_FLAGS+="--set onos-classic.atomix.persistence.enabled=false "

250

251

echo "Installing with the following extra arguments:"

252

echo $EXTRA_HELM_FLAGS

# Use custom built images

257

258

if [ '\$GERRIT_PROJECT' == 'voltha-go' ]; then

259

EXTRA_HELM_FLAGS+="--set voltha.images.rw_core.repository=${dockerRegistry}/voltha/voltha-rw-core,voltha.images.rw_core.tag=voltha-scale "

260

fi

261

262

if [ '\$GERRIT_PROJECT' == 'voltha-openolt-adapter' ]; then

263

EXTRA_HELM_FLAGS+="--set voltha-openolt-adapter.images.adapter_open_olt.repository=${dockerRegistry}/voltha/voltha-openolt-adapter,voltha-openolt-adapter.images.adapter_open_olt.tag=voltha-scale "

264

fi

265

266

if [ '\$GERRIT_PROJECT' == 'voltha-openonu-adapter' ]; then

267

EXTRA_HELM_FLAGS+="--set voltha-openonu-adapter.images.adapter_open_onu.repository=${dockerRegistry}/voltha/voltha-openonu-adapter,voltha-openonu-adapter.images.adapter_open_onu.tag=voltha-scale "

268

fi

269

270

if [ '\$GERRIT_PROJECT' == 'voltha-openonu-adapter-go' ]; then

271

EXTRA_HELM_FLAGS+="--set voltha-openonu-adapter-go.images.adapter_open_onu_go.repository=${dockerRegistry}/voltha/voltha-openonu-adapter-go,voltha-openonu-adapter-go.images.adapter_open_onu_go.tag=voltha-scale "

272

fi

273

274

if [ '\$GERRIT_PROJECT' == 'ofagent-go' ]; then

275

EXTRA_HELM_FLAGS+="--set voltha.images.ofagent.repository=${dockerRegistry}/voltha/voltha-ofagent-go,ofagent-go.images.ofagent.tag=voltha-scale "

276

fi

277

278

if [ '\$GERRIT_PROJECT' == 'voltha-onos' ]; then

279

EXTRA_HELM_FLAGS+="--set onos-classic.image.repository=${dockerRegistry}/voltha/voltha-onos,onos-classic.image.tag=voltha-scale "

280

fi

281

282

if [ '\$GERRIT_PROJECT' == 'bbsim' ]; then

283

EXTRA_HELM_FLAGS+="--set images.bbsim.repository=${dockerRegistry}/voltha/bbsim,images.bbsim.tag=voltha-scale "

284

fi

285

286

helm upgrade --install voltha-infra onf/voltha-infra \$EXTRA_HELM_FLAGS \

287

--set onos-classic.replicas=${onosReplicas},onos-classic.atomix.replicas=${atomixReplicas} \

288

--set etcd.enabled=false,kafka.enabled=false \

289

--set global.log_level=${logLevel} \

290

-f $WORKSPACE/voltha-helm-charts/examples/${workflow}-values.yaml \

291

--version 0.1.13

292

293

helm upgrade --install voltha1 onf/voltha-stack \$EXTRA_HELM_FLAGS \

294

--set global.stack_name=voltha1 \

295

--set global.voltha_infra_name=voltha-infra \

296

--set global.voltha_infra_namespace=default \

297

--set global.log_level=${logLevel} \

298

${ofAgentConnections(onosReplicas.toInteger(), "voltha-infra", "default")} \

299

--set voltha.services.kafka.adapter.address=kafka.default.svc:9092 \

300

--set voltha.services.kafka.cluster.address=kafka.default.svc:9092 \

301

--set voltha.services.etcd.address=etcd.default.svc:2379 \

302

--set voltha-adapter-openolt.services.kafka.adapter.address=kafka.default.svc:9092 \

303

--set voltha-adapter-openolt.services.kafka.cluster.address=kafka.default.svc:9092 \

304

--set voltha-adapter-openolt.services.etcd.address=etcd.default.svc:2379 \

305

--set voltha-adapter-openonu.services.kafka.adapter.address=kafka.default.svc:9092 \

306

--set voltha-adapter-openonu.services.kafka.cluster.address=kafka.default.svc:9092 \

307

--set voltha-adapter-openonu.services.etcd.address=etcd.default.svc:2379 \

--version 0.1.17

for i in {0..${olts.toInteger() - 1}}; do

312

stackId=1

313

helm upgrade --install bbsim\$i onf/bbsim \$EXTRA_HELM_FLAGS \

314

--set olt_id="\${stackId}\${i}" \

315

--set onu=${onus},pon=${pons} \

316

--set global.log_level=${logLevel.toLowerCase()} \

317

-f $WORKSPACE/voltha-helm-charts/examples/${workflow}-values.yaml \

--version 4.2.0

done

"""

sh """

set +x

echo -ne "\nWaiting for VOLTHA and ONOS to start..."

325

voltha=\$(kubectl get pods --all-namespaces -l app.kubernetes.io/part-of=voltha --no-headers | grep "0/" | wc -l)

326

onos=\$(kubectl get pods --all-namespaces -l app=onos-classic --no-headers | grep "0/" | wc -l)

327

while [[ \$voltha != 0 || \$onos != 0 ]]; do

328

sleep 5

329

echo -ne "."

330

voltha=\$(kubectl get pods --all-namespaces -l app.kubernetes.io/part-of=voltha --no-headers | grep "0/" | wc -l)

331

onos=\$(kubectl get pods --all-namespaces -l app=onos-classic --no-headers | grep "0/" | wc -l)

332

done

333

echo -ne "\nVOLTHA and ONOS pods ready\n"

334

kubectl get pods --all-namespaces -l app.kubernetes.io/part-of=voltha --no-headers | grep "0/" | wc -l

335

kubectl get pods --all-namespaces -l app=onos-classic --no-headers | grep "0/" | wc -l

336

"""

337

start_port_forward(olts)

}

}

}

}

stage('Configuration') {

343

steps {

344

script {

345

def tech_prof_directory = "XGS-PON"

346

sh returnStdout: false, script: """

347

#Setting link discovery

348

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 cfg set org.onosproject.provider.lldp.impl.LldpLinkProvider enabled ${withLLDP}

349

350

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 cfg set org.onosproject.net.flow.impl.FlowRuleManager allowExtraneousRules true

351

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 cfg set org.onosproject.net.flow.impl.FlowRuleManager importExtraneousRules true

352

353

354

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 cfg set org.onosproject.net.flowobjective.impl.InOrderFlowObjectiveManager accumulatorMaxBatchMillis 900

355

356

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 cfg set org.onosproject.net.flowobjective.impl.InOrderFlowObjectiveManager accumulatorMaxIdleMillis 500

357

358

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 cfg set org.opencord.olt.impl.Olt provisionDelay 1000

359

360

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 log:set ${logLevel} org.onosproject

361

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 log:set ${logLevel} org.opencord

362

363

# sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 log:set DEBUG org.opencord.cordmcast

364

# sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 log:set DEBUG org.onosproject.mcast

365

# sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 log:set DEBUG org.opencord.igmpproxy

366

# sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 log:set DEBUG org.opencord.olt

367

368

kubectl exec \$(kubectl get pods | grep -E "bbsim[0-9]" | awk 'NR==1{print \$1}') -- bbsimctl log ${logLevel.toLowerCase()} false

369

370

# Set Flows/Ports/Meters poll frequency

371

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 cfg set org.onosproject.provider.of.flow.impl.OpenFlowRuleProvider flowPollFrequency ${onosStatInterval}

372

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 cfg set org.onosproject.provider.of.device.impl.OpenFlowDeviceProvider portStatsPollFrequency ${onosStatInterval}

373

374

if [ ${withFlows} = false ]; then

375

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 app deactivate org.opencord.olt

376

fi

377

378

if [ '${workflow}' = 'tt' ]; then

379

etcd_container=\$(kubectl get pods --all-namespaces | grep etcd | awk 'NR==1{print \$2}')

380

kubectl cp $WORKSPACE/voltha-system-tests/tests/data/TechProfile-TT-HSIA.json \$etcd_container:/tmp/hsia.json

381

put_result=\$(kubectl exec -it \$etcd_container -- /bin/sh -c 'cat /tmp/hsia.json | ETCDCTL_API=3 etcdctl put service/voltha/technology_profiles/${tech_prof_directory}/64')

382

kubectl cp $WORKSPACE/voltha-system-tests/tests/data/TechProfile-TT-VoIP.json \$etcd_container:/tmp/voip.json

383

put_result=\$(kubectl exec -it \$etcd_container -- /bin/sh -c 'cat /tmp/voip.json | ETCDCTL_API=3 etcdctl put service/voltha/technology_profiles/${tech_prof_directory}/65')

384

kubectl cp $WORKSPACE/voltha-system-tests/tests/data/TechProfile-TT-MCAST.json \$etcd_container:/tmp/mcast.json

385

put_result=\$(kubectl exec -it \$etcd_container -- /bin/sh -c 'cat /tmp/mcast.json | ETCDCTL_API=3 etcdctl put service/voltha/technology_profiles/${tech_prof_directory}/66')

386

fi

387

388

if [ ${withPcap} = true ] ; then

389

# Start the tcp-dump in ofagent

390

export OF_AGENT=\$(kubectl get pods -l app=ofagent -o name)

391

kubectl exec \$OF_AGENT -- apk update

392

kubectl exec \$OF_AGENT -- apk add tcpdump

393

kubectl exec \$OF_AGENT -- mv /usr/sbin/tcpdump /usr/bin/tcpdump

394

_TAG=ofagent-tcpdump kubectl exec \$OF_AGENT -- tcpdump -nei eth0 -w out.pcap&

395

396

# Start the tcp-dump in radius

397

export RADIUS=\$(kubectl get pods -l app=radius -o name)

398

kubectl exec \$RADIUS -- apt-get update

399

kubectl exec \$RADIUS -- apt-get install -y tcpdump

400

_TAG=radius-tcpdump kubectl exec \$RADIUS -- tcpdump -w out.pcap&

401

402

# Start the tcp-dump in ONOS

403

for i in \$(seq 0 \$ONOSES); do

404

INSTANCE="onos-onos-classic-\$i"

405

kubectl exec \$INSTANCE -- apt-get update

406

kubectl exec \$INSTANCE -- apt-get install -y tcpdump

407

kubectl exec \$INSTANCE -- mv /usr/sbin/tcpdump /usr/bin/tcpdump

408

_TAG=\$INSTANCE kubectl exec \$INSTANCE -- /usr/bin/tcpdump -nei eth0 port 1812 -w out.pcap&

done

fi

"""

}

}

}

Matteo Scandolo

fcb7563

2021-04-06 16:25:19 -0700

[diff] [blame]

415

stage('Load MIB Template') {

416

when {

417

expression {

418

return params.withMibTemplate

}

}

steps {

sh """

# load MIB template

wget https://raw.githubusercontent.com/opencord/voltha-openonu-adapter-go/master/templates/BBSM-12345123451234512345-00000000000001-v1.json

425

cat BBSM-12345123451234512345-00000000000001-v1.json | kubectl exec -it \$(kubectl get pods |grep etcd | awk 'NR==1{print \$1}') -- etcdctl put service/voltha/omci_mibs/go_templates/BBSM/12345123451234512345/00000000000001

426

"""

427

}

428

}

Matteo Scandolo

fed6bab

2021-03-31 14:19:57 -0700

[diff] [blame]

stage('Run Test') {

steps {

sh '''

mkdir -p $WORKSPACE/RobotLogs

433

cd $WORKSPACE/voltha-system-tests

make vst_venv

'''

sh '''

if [ ${withProfiling} = true ] ; then

438

mkdir -p $LOG_FOLDER/pprof

439

echo $PATH

440

#Creating Python script for ONU Detection

441

cat << EOF > $WORKSPACE/pprof.sh

timestamp() {

date +"%T"

}

i=0

while [[ true ]]; do

((i++))

ts=$(timestamp)

go tool pprof -png http://127.0.0.1:6060/debug/pprof/heap > $LOG_FOLDER/pprof/rw-core-heap-\\$i-\\$ts.png

451

go tool pprof -png http://127.0.0.1:6060/debug/pprof/goroutine > $LOG_FOLDER/pprof/rw-core-goroutine-\\$i-\\$ts.png

452

curl -o $LOG_FOLDER/pprof/rw-core-profile-\\$i-\\$ts.pprof http://127.0.0.1:6060/debug/pprof/profile?seconds=10

453

go tool pprof -png $LOG_FOLDER/pprof/rw-core-profile-\\$i-\\$ts.pprof > $LOG_FOLDER/pprof/rw-core-profile-\\$i-\\$ts.png

454

455

go tool pprof -png http://127.0.0.1:6061/debug/pprof/heap > $LOG_FOLDER/pprof/openolt-heap-\\$i-\\$ts.png

456

go tool pprof -png http://127.0.0.1:6061/debug/pprof/goroutine > $LOG_FOLDER/pprof/openolt-goroutine-\\$i-\\$ts.png

457

curl -o $LOG_FOLDER/pprof/openolt-profile-\\$i-\\$ts.pprof http://127.0.0.1:6061/debug/pprof/profile?seconds=10

458

go tool pprof -png $LOG_FOLDER/pprof/openolt-profile-\\$i-\\$ts.pprof > $LOG_FOLDER/pprof/openolt-profile-\\$i-\\$ts.png

459

460

go tool pprof -png http://127.0.0.1:6062/debug/pprof/heap > $LOG_FOLDER/pprof/ofagent-heap-\\$i-\\$ts.png

461

go tool pprof -png http://127.0.0.1:6062/debug/pprof/goroutine > $LOG_FOLDER/pprof/ofagent-goroutine-\\$i-\\$ts.png

462

curl -o $LOG_FOLDER/pprof/ofagent-profile-\\$i-\\$ts.pprof http://127.0.0.1:6062/debug/pprof/profile?seconds=10

463

go tool pprof -png $LOG_FOLDER/pprof/ofagent-profile-\\$i-\\$ts.pprof > $LOG_FOLDER/pprof/ofagent-profile-\\$i-\\$ts.png

sleep 10

done

EOF

_TAG="pprof"

_TAG=$_TAG bash $WORKSPACE/pprof.sh &

471

fi

472

'''

473

timeout(time: 15, unit: 'MINUTES') {

474

sh '''

475

ROBOT_PARAMS="--exitonfailure \

-v olt:${olts} \

-v pon:${pons} \

-v onu:${onus} \

-v workflow:${workflow} \

480

-v withEapol:${withEapol} \

481

-v withDhcp:${withDhcp} \

482

-v withIgmp:${withIgmp} \

483

--noncritical non-critical \

484

-e igmp -e teardown "

485

486

if [ ${withEapol} = false ] ; then

487

ROBOT_PARAMS+="-e authentication "

488

fi

489

490

if [ ${withDhcp} = false ] ; then

491

ROBOT_PARAMS+="-e dhcp "

492

fi

493

494

if [ ${provisionSubscribers} = false ] ; then

495

# if we're not considering subscribers then we don't care about authentication and dhcp

496

ROBOT_PARAMS+="-e authentication -e provision -e flow-after -e dhcp "

497

fi

498

499

if [ ${withFlows} = false ] ; then

500

ROBOT_PARAMS+="-i setup -i activation "

501

fi

502

503

cd $WORKSPACE/voltha-system-tests

504

source ./vst_venv/bin/activate

505

robot -d $WORKSPACE/RobotLogs \

506

$ROBOT_PARAMS tests/scale/Voltha_Scale_Tests.robot

'''

}

}

}

stage('Run Igmp Tests') {

512

environment {

513

ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/IgmpTests"

}

when {

expression {

return params.withIgmp

}

}

steps {

sh '''

set +e

mkdir -p $ROBOT_LOGS_DIR

524

cd $WORKSPACE/voltha-system-tests

525

make vst_venv

526

'''

527

timeout(time: 11, unit: 'MINUTES') {

528

sh '''

529

ROBOT_PARAMS="--exitonfailure \

-v olt:${olts} \

-v pon:${pons} \

-v onu:${onus} \

-v workflow:${workflow} \

534

-v withEapol:${withEapol} \

535

-v withDhcp:${withDhcp} \

536

-v withIgmp:${withIgmp} \

537

--noncritical non-critical \

538

-i igmp \

539

-e setup -e activation -e flow-before \

540

-e authentication -e provision -e flow-after \

541

-e dhcp -e teardown "

542

cd $WORKSPACE/voltha-system-tests

543

source ./vst_venv/bin/activate

544

robot -d $ROBOT_LOGS_DIR \

545

$ROBOT_PARAMS tests/scale/Voltha_Scale_Tests.robot

'''

}

}

}

}

post {

always {

// collect result, done in the "post" step so it's executed even in the

554

// event of a timeout in the tests

555

sh '''

556

557

# stop the kail processes

558

list=($APPS_TO_LOG)

559

for app in "${list[@]}"

560

do

561

echo "Stopping logs for: ${app}"

562

_TAG="kail-$app"

563

P_IDS="$(ps e -ww -A | grep "_TAG=$_TAG" | grep -v grep | awk '{print $1}')"

564

if [ -n "$P_IDS" ]; then

565

echo $P_IDS

566

for P_ID in $P_IDS; do

kill -9 $P_ID

done

fi

done

if [ ${withPcap} = true ] ; then

573

# stop ofAgent tcpdump

574

P_ID="\$(ps e -ww -A | grep "_TAG=ofagent-tcpdump" | grep -v grep | awk '{print \$1}')"

575

if [ -n "\$P_ID" ]; then

kill -9 \$P_ID

fi

# stop radius tcpdump

580

P_ID="\$(ps e -ww -A | grep "_TAG=radius-tcpdump" | grep -v grep | awk '{print \$1}')"

581

if [ -n "\$P_ID" ]; then

kill -9 \$P_ID

fi

# stop onos tcpdump

LIMIT=$(($NUM_OF_ONOS - 1))

587

for i in $(seq 0 $LIMIT); do

588

INSTANCE="onos-onos-classic-$i"

589

P_ID="\$(ps e -ww -A | grep "_TAG=$INSTANCE" | grep -v grep | awk '{print \$1}')"

590

if [ -n "\$P_ID" ]; then

kill -9 \$P_ID

fi

done

# copy the file

export OF_AGENT=$(kubectl get pods -l app=ofagent | awk 'NR==2{print $1}') || true

597

kubectl cp $OF_AGENT:out.pcap $LOG_FOLDER/ofagent.pcap || true

598

599

export RADIUS=$(kubectl get pods -l app=radius | awk 'NR==2{print $1}') || true

600

kubectl cp $RADIUS:out.pcap $LOG_FOLDER/radius.pcap || true

601

602

LIMIT=$(($NUM_OF_ONOS - 1))

603

for i in $(seq 0 $LIMIT); do

604

INSTANCE="onos-onos-classic-$i"

605

kubectl cp $INSTANCE:out.pcap $LOG_FOLDER/$INSTANCE.pcap || true

done

fi

cd voltha-system-tests

610

source ./vst_venv/bin/activate

611

python tests/scale/collect-result.py -r $WORKSPACE/RobotLogs/output.xml -p $WORKSPACE/plots > $WORKSPACE/execution-time.txt || true

612

cat $WORKSPACE/execution-time.txt

613

'''

614

sh '''

615

if [ ${withProfiling} = true ] ; then

616

_TAG="pprof"

617

P_IDS="$(ps e -ww -A | grep "_TAG=$_TAG" | grep -v grep | awk '{print $1}')"

618

if [ -n "$P_IDS" ]; then

619

echo $P_IDS

620

for P_ID in $P_IDS; do

kill -9 $P_ID

done

fi

fi

'''

plot([

csvFileName: 'scale-test.csv',

628

csvSeries: [

629

[file: 'plots/plot-voltha-onus.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''],

630

[file: 'plots/plot-onos-ports.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''],

631

[file: 'plots/plot-voltha-flows-before.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''],

632

[file: 'plots/plot-voltha-openolt-flows-before.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''],

633

[file: 'plots/plot-onos-flows-before.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''],

634

[file: 'plots/plot-onos-auth.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''],

635

[file: 'plots/plot-voltha-flows-after.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''],

636

[file: 'plots/plot-voltha-openolt-flows-after.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''],

637

[file: 'plots/plot-onos-flows-after.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''],

638

[file: 'plots/plot-onos-dhcp.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''],

639

],

640

group: 'Voltha-Scale-Numbers', numBuilds: '20', style: 'line', title: "Scale Test (OLTs: ${olts}, PONs: ${pons}, ONUs: ${onus})", yaxis: 'Time (s)', useDescr: true

641

])

642

step([$class: 'RobotPublisher',

643

disableArchiveOutput: false,

644

logFileName: '**/log*.html',

645

otherFiles: '',

646

outputFileName: '**/output*.xml',

647

outputPath: 'RobotLogs',

648

passThreshold: 100,

649

reportFileName: '**/report*.html',

650

unstableThreshold: 0]);

651

// get all the logs from kubernetes PODs

652

sh returnStdout: false, script: '''

653

654

# store information on running charts

655

helm ls > $LOG_FOLDER/helm-list.txt || true

656

657

# store information on the running pods

658

kubectl get pods -o wide > $LOG_FOLDER/pods.txt || true

659

kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.image}{'\\n'}" | sort | uniq | tee $LOG_FOLDER/pod-images.txt || true

660

kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.imageID}{'\\n'}" | sort | uniq | tee $LOG_FOLDER/pod-imagesId.txt || true

661

662

# copy the ONOS logs directly from the container to avoid the color codes

663

printf '%s\n' $(kubectl get pods -l app=onos-classic -o=jsonpath="{.items[*]['metadata.name']}") | xargs --no-run-if-empty -I# bash -c "kubectl cp #:${karafHome}/data/log/karaf.log $LOG_FOLDER/#.log" || true

664

665

# get ONOS cfg from the 3 nodes

666

printf '%s\n' $(kubectl get pods -l app=onos-classic -o=jsonpath="{.items[*]['metadata.name']}") | xargs --no-run-if-empty -I# bash -c "kubectl exec -it # -- ${karafHome}/bin/client cfg get > $LOG_FOLDER/#.cfg" || true

667

668

669

# get radius logs out of the container

670

kubectl cp $(kubectl get pods -l app=radius --no-headers | awk '{print $1}'):/var/log/freeradius/radius.log $LOG_FOLDER/radius.log || true

671

'''

672

// dump all the BBSim(s) ONU information

673

sh '''

674

BBSIM_IDS=$(kubectl get pods | grep bbsim | grep -v server | awk '{print $1}')

675

IDS=($BBSIM_IDS)

676

677

for bbsim in "${IDS[@]}"

678

do

679

kubectl exec -t $bbsim -- bbsimctl onu list > $LOG_FOLDER/$bbsim-device-list.txt || true

680

kubectl exec -t $bbsim -- bbsimctl service list > $LOG_FOLDER/$bbsim-service-list.txt || true

681

kubectl exec -t $bbsim -- bbsimctl olt resources GEM_PORT > $LOG_FOLDER/$bbsim-flows-gem-ports.txt || true

682

kubectl exec -t $bbsim -- bbsimctl olt resources ALLOC_ID > $LOG_FOLDER/$bbsim-flows-alloc-ids.txt || true

683

kubectl exec -t $bbsim -- bbsimctl olt pons > $LOG_FOLDER/$bbsim-pon-resources.txt || true

done

'''

script {

// first make sure the port-forward is still running,

688

// sometimes Jenkins kills it relardless of the JENKINS_NODE_COOKIE=dontKillMe

def running = sh (

returnStdout: true

).trim()

// if any of the voltha-api, onos-rest, onos-ssh port-forwards are not there

694

// kill all and restart

695

if (running != "3") {

696

start_port_forward(olts)

697

}

698

}

699

// get ONOS debug infos

700

sh '''

701

702

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 apps -a -s > $LOG_FOLDER/onos-apps.txt

703

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 nodes > $LOG_FOLDER/onos-nodes.txt

704

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 masters > $LOG_FOLDER/onos-masters.txt

705

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 roles > $LOG_FOLDER/onos-roles.txt

706

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 cfg get > $LOG_FOLDER/onos-cfg.txt

707

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 netcfg > $LOG_FOLDER/onos-netcfg.txt

708

709

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 ports > $LOG_FOLDER/onos-ports-list.txt

710

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 hosts > $LOG_FOLDER/onos-hosts-list.txt

711

712

if [ ${withFlows} = true ] ; then

713

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 volt-olts > $LOG_FOLDER/onos-olt-list.txt

714

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 flows -s > $LOG_FOLDER/onos-flows-list.txt

715

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 meters > $LOG_FOLDER/onos-meters-list.txt

716

fi

717

718

if [ ${provisionSubscribers} = true ]; then

719

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 volt-programmed-subscribers > $LOG_FOLDER/onos-programmed-subscribers.txt

720

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 volt-programmed-meters > $LOG_FOLDER/onos-programmed-meters.txt

721

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 volt-bpmeter-mappings > $LOG_FOLDER/onos-bpmeter-mappings.txt

722

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 volt-failed-subscribers > $LOG_FOLDER/onos-failed-subscribers.txt

723

fi

724

725

if [ ${withEapol} = true ] ; then

726

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 aaa-users > $LOG_FOLDER/onos-aaa-users.txt

727

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 aaa-statistics > $LOG_FOLDER/onos-aaa-statistics.txt

728

fi

729

730

if [ ${withDhcp} = true ] ; then

731

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 dhcpl2relay-allocations > $LOG_FOLDER/onos-dhcp-allocations.txt

732

fi

733

734

if [ ${withIgmp} = true ] ; then

735

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 mcast-host-routes > $LOG_FOLDER/onos-mcast-host-routes.txt

736

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 mcast-host-show > $LOG_FOLDER/onos-mcast-host-show.txt

737

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 groups > $LOG_FOLDER/onos-groups.txt

738

fi

739

'''

740

// collect etcd metrics

741

sh '''

742

mkdir -p $WORKSPACE/etcd-metrics

743

curl -s -X GET -G http://10.90.0.101:31301/api/v1/query --data-urlencode 'query=etcd_debugging_mvcc_keys_total' | jq '.data' > $WORKSPACE/etcd-metrics/etcd-key-count.json || true

744

curl -s -X GET -G http://10.90.0.101:31301/api/v1/query --data-urlencode 'query=grpc_server_handled_total{grpc_service="etcdserverpb.KV"}' | jq '.data' > $WORKSPACE/etcd-metrics/etcd-rpc-count.json || true

745

curl -s -X GET -G http://10.90.0.101:31301/api/v1/query --data-urlencode 'query=etcd_debugging_mvcc_db_total_size_in_bytes' | jq '.data' > $WORKSPACE/etcd-metrics/etcd-db-size.json || true

746

curl -s -X GET -G http://10.90.0.101:31301/api/v1/query --data-urlencode 'query=etcd_disk_backend_commit_duration_seconds_sum' | jq '.data' > $WORKSPACE/etcd-metrics/etcd-backend-write-time.json || true

747

curl -s -X GET -G http://10.90.0.101:31301/api/v1/query --data-urlencode 'query=etcd_disk_backend_commit_duration_seconds_sum' | jq '.data' > $WORKSPACE/etcd-metrics/etcd-backend-write-time-sum.json || true

748

curl -s -X GET -G http://10.90.0.101:31301/api/v1/query --data-urlencode 'query=etcd_disk_backend_commit_duration_seconds_bucket' | jq '.data' > $WORKSPACE/etcd-metrics/etcd-backend-write-time-bucket.json || true

749

curl -s -X GET -G http://10.90.0.101:31301/api/v1/query --data-urlencode 'query=etcd_disk_wal_fsync_duration_seconds_bucket' | jq '.data' > $WORKSPACE/etcd-metrics/etcd-wal-fsync-time-bucket.json || true

750

curl -s -X GET -G http://10.90.0.101:31301/api/v1/query --data-urlencode 'query=etcd_network_peer_round_trip_time_seconds_bucket' | jq '.data' > $WORKSPACE/etcd-metrics/etcd-network-peer-round-trip-time-seconds.json || true

751

752

'''

753

// get VOLTHA debug infos

script {

try {

sh '''

voltctl -m 8MB device list -o json > $LOG_FOLDER/device-list.json || true

758

python -m json.tool $LOG_FOLDER/device-list.json > $LOG_FOLDER/voltha-devices-list.json || true

759

rm $LOG_FOLDER/device-list.json || true

760

voltctl -m 8MB device list > $LOG_FOLDER/voltha-devices-list.txt || true

761

762

printf '%s\n' $(voltctl -m 8MB device list | grep olt | awk '{print $1}') | xargs --no-run-if-empty -I# bash -c "voltctl -m 8MB device flows # > $LOG_FOLDER/voltha-device-flows-#.txt" || true

763

printf '%s\n' $(voltctl -m 8MB device list | grep olt | awk '{print $1}') | xargs --no-run-if-empty -I# bash -c "voltctl -m 8MB device port list --format 'table{{.PortNo}}\t{{.Label}}\t{{.Type}}\t{{.AdminState}}\t{{.OperStatus}}' # > $LOG_FOLDER/voltha-device-ports-#.txt" || true

764

765

printf '%s\n' $(voltctl -m 8MB logicaldevice list -q) | xargs --no-run-if-empty -I# bash -c "voltctl -m 8MB logicaldevice flows # > $LOG_FOLDER/voltha-logicaldevice-flows-#.txt" || true

766

printf '%s\n' $(voltctl -m 8MB logicaldevice list -q) | xargs --no-run-if-empty -I# bash -c "voltctl -m 8MB logicaldevice port list # > $LOG_FOLDER/voltha-logicaldevice-ports-#.txt" || true

'''

} catch(e) {

sh '''

echo "Can't get device list from voltclt"

'''

}

}

// get cpu usage by container

775

sh '''

776

if [ ${withMonitoring} = true ] ; then

777

cd $WORKSPACE/voltha-system-tests

778

source ./vst_venv/bin/activate

779

sleep 60 # we have to wait for prometheus to collect all the information

780

python tests/scale/sizing.py -o $WORKSPACE/plots || true

781

fi

782

'''

783

archiveArtifacts artifacts: 'execution-time.txt,logs/*,logs/pprof/*,RobotLogs/**/*,plots/*,etcd-metrics/*'

}

}

}

def start_port_forward(olts) {

789

sh """

790

daemonize -E JENKINS_NODE_COOKIE="dontKillMe" /usr/local/bin/kubectl port-forward --address 0.0.0.0 -n default svc/voltha-infra-onos-classic-hs 8101:8101

791

daemonize -E JENKINS_NODE_COOKIE="dontKillMe" /usr/local/bin/kubectl port-forward --address 0.0.0.0 -n default svc/voltha-infra-onos-classic-hs 8181:8181

792

daemonize -E JENKINS_NODE_COOKIE="dontKillMe" /usr/local/bin/kubectl port-forward --address 0.0.0.0 -n default svc/voltha1-voltha-api 55555:55555

793

794

bbsimRestPortFwd=50071

795

for i in {0..${olts.toInteger() - 1}}; do

796

daemonize -E JENKINS_NODE_COOKIE="dontKillMe" /usr/local/bin/kubectl port-forward --address 0.0.0.0 -n default svc/bbsim\${i} \${bbsimRestPortFwd}:50071

797

((bbsimRestPortFwd++))

798

done

799

"""

800

}