Blame - jjb/pipeline/voltha/playground/voltha-scale-test.groovy - ci-management

EXTRA_HELM_FLAGS+="--set voltha-adapter-openolt.images.adapter_open_olt.repository=\$openoltAdapterRepo,voltha-adapter-openolt.images.adapter_open_olt.tag=\$openoltAdapterTag "

196

fi

197

198

# OpenONU custom image handling

199

if [ '${openonuAdapterImg.trim()}' != '' ] && [ '\$GERRIT_PROJECT' != 'voltha-openonu-adapter' ]; then

200

IFS=: read -r openonuAdapterRepo openonuAdapterTag <<< '${openonuAdapterImg.trim()}'

201

EXTRA_HELM_FLAGS+="--set voltha-adapter-openonu.images.adapter_open_onu.repository=\$openonuAdapterRepo,voltha-adapter-openonu.images.adapter_open_onu.tag=\$openonuAdapterTag "

202

fi

203

204

# OpenONU GO custom image handling

205

if [ '${openonuAdapterGoImg.trim()}' != '' ] && [ '\$GERRIT_PROJECT' != 'voltha-openonu-adapter-go' ]; then

206

IFS=: read -r openonuAdapterGoRepo openonuAdapterGoTag <<< '${openonuAdapterGoImg.trim()}'

207

EXTRA_HELM_FLAGS+="--set voltha-adapter-openonu.images.adapter_open_onu_go.repository=\$openonuAdapterGoRepo,voltha-adapter-openonu.images.adapter_open_onu_go.tag=\$openonuAdapterGoTag "

208

fi

209

210

# ONOS custom image handling

211

if [ '${onosImg.trim()}' != '' ] && [ '\$GERRIT_PROJECT' != 'voltha-onos' ]; then

212

IFS=: read -r onosRepo onosTag <<< '${onosImg.trim()}'

213

EXTRA_HELM_FLAGS+="--set onos-classic.image.repository=\$onosRepo,onos-classic.image.tag=\$onosTag "

214

fi

215

216

# set BBSim parameters

217

EXTRA_HELM_FLAGS+='--set enablePerf=true,pon=${pons},onu=${onus},uni=${unis} '

218

219

# disable the securityContext, this is a development cluster

220

EXTRA_HELM_FLAGS+='--set securityContext.enabled=false '

221

222

# No persistent-volume-claims in Atomix

223

EXTRA_HELM_FLAGS+="--set onos-classic.atomix.persistence.enabled=false "

224

225

# Use custom built images

226

227

if [ '\$GERRIT_PROJECT' == 'voltha-go' ]; then

228

EXTRA_HELM_FLAGS+="--set voltha.images.rw_core.repository=${dockerRegistry}/voltha/voltha-rw-core,voltha.images.rw_core.tag=voltha-scale "

229

fi

230

231

if [ '\$GERRIT_PROJECT' == 'voltha-openolt-adapter' ]; then

232

EXTRA_HELM_FLAGS+="--set voltha-openolt-adapter.images.adapter_open_olt.repository=${dockerRegistry}/voltha/voltha-openolt-adapter,voltha-openolt-adapter.images.adapter_open_olt.tag=voltha-scale "

233

fi

234

235

if [ '\$GERRIT_PROJECT' == 'voltha-openonu-adapter' ]; then

236

EXTRA_HELM_FLAGS+="--set voltha-openonu-adapter.images.adapter_open_onu.repository=${dockerRegistry}/voltha/voltha-openonu-adapter,voltha-openonu-adapter.images.adapter_open_onu.tag=voltha-scale "

237

fi

238

239

if [ '\$GERRIT_PROJECT' == 'voltha-openonu-adapter-go' ]; then

240

EXTRA_HELM_FLAGS+="--set voltha-openonu-adapter-go.images.adapter_open_onu_go.repository=${dockerRegistry}/voltha/voltha-openonu-adapter-go,voltha-openonu-adapter-go.images.adapter_open_onu_go.tag=voltha-scale "

241

fi

242

243

if [ '\$GERRIT_PROJECT' == 'ofagent-go' ]; then

244

EXTRA_HELM_FLAGS+="--set voltha.images.ofagent.repository=${dockerRegistry}/voltha/voltha-ofagent-go,ofagent-go.images.ofagent.tag=voltha-scale "

245

fi

246

247

if [ '\$GERRIT_PROJECT' == 'voltha-onos' ]; then

248

EXTRA_HELM_FLAGS+="--set onos-classic.image.repository=${dockerRegistry}/voltha/voltha-onos,onos-classic.image.tag=voltha-scale "

249

fi

250

251

if [ '\$GERRIT_PROJECT' == 'bbsim' ]; then

252

EXTRA_HELM_FLAGS+="--set images.bbsim.repository=${dockerRegistry}/voltha/bbsim,images.bbsim.tag=voltha-scale "

253

fi

254

echo \$EXTRA_HELM_FLAGS

""").trim()

def extraHelmFlags = returned_flags

259

// The added space before params.extraHelmFlags is required due to the .trim() above

260

def infraHelmFlags =

261

"--set global.log_level=${logLevel} " +

262

"--set radius.enabled=${withEapol} " +

263

"--set onos-classic.onosSshPort=30115 " +

264

"--set onos-classic.onosApiPort=30120 " +

265

extraHelmFlags + " " + params.extraHelmFlags

266

267

println "Passing the following parameters to the VOLTHA infra deploy: ${infraHelmFlags}."

268

269

def localCharts = false

270

if (volthaHelmChartsChange != "") {

localCharts = true

}

volthaInfraDeploy([

workflow: workflow,

infraNamespace: "default",

277

extraHelmFlags: infraHelmFlags,

278

localCharts: localCharts,

279

onosReplica: onosReplicas,

280

atomixReplica: atomixReplicas,

281

kafkaReplica: kafkaReplicas,

282

etcdReplica: etcdReplicas,

283

])

284

285

stackHelmFlags = " --set onu=${onus},pon=${pons},uni=${unis} --set global.log_level=${logLevel.toLowerCase()} "

286

stackHelmFlags += " --set voltha.ingress.enabled=true --set voltha.ingress.enableVirtualHosts=true --set voltha.fullHostnameOverride=voltha.scale1.dev "

287

stackHelmFlags += extraHelmFlags + " " + params.extraHelmFlags

288

289

volthaStackDeploy([

290

bbsimReplica: olts.toInteger(),

291

infraNamespace: "default",

292

volthaNamespace: "default",

293

stackName: "voltha1", // TODO support custom charts

294

workflow: workflow,

295

extraHelmFlags: stackHelmFlags,

296

localCharts: localCharts,

297

onosReplica: onosReplicas,

])

sh """

set +x

echo -ne "\nWaiting for VOLTHA and ONOS to start..."

303

voltha=\$(kubectl get pods --all-namespaces -l app.kubernetes.io/part-of=voltha --no-headers | grep "0/" | wc -l)

304

onos=\$(kubectl get pods --all-namespaces -l app=onos-classic --no-headers | grep "0/" | wc -l)

305

while [[ \$voltha != 0 || \$onos != 0 ]]; do

306

sleep 5

307

echo -ne "."

308

voltha=\$(kubectl get pods --all-namespaces -l app.kubernetes.io/part-of=voltha --no-headers | grep "0/" | wc -l)

309

onos=\$(kubectl get pods --all-namespaces -l app=onos-classic --no-headers | grep "0/" | wc -l)

310

done

311

echo -ne "\nVOLTHA and ONOS pods ready\n"

312

kubectl get pods --all-namespaces -l app.kubernetes.io/part-of=voltha --no-headers | grep "0/" | wc -l

313

kubectl get pods --all-namespaces -l app=onos-classic --no-headers | grep "0/" | wc -l

314

"""

315

start_port_forward(olts)

}

}

}

}

stage('Configuration') {

steps {

script {

setOnosLogLevels([

onosNamespace: "default",

325

apps: [

326

'org.opencord.dhcpl2relay',

327

'org.opencord.olt',

328

'org.opencord.aaa',

329

'org.opencord.maclearner',

330

'org.onosproject.net.flowobjective.impl.FlowObjectiveManager',

331

'org.onosproject.net.flowobjective.impl.InOrderFlowObjectiveManager'

],

logLevel: logLevel

])

def tech_prof_directory = "XGS-PON"

336

sh returnStdout: false, script: """

337

#Setting link discovery

338

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 cfg set org.onosproject.provider.lldp.impl.LldpLinkProvider enabled ${withLLDP}

339

340

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 cfg set org.onosproject.net.flow.impl.FlowRuleManager allowExtraneousRules true

341

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 cfg set org.onosproject.net.flow.impl.FlowRuleManager importExtraneousRules true

342

343

344

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 cfg set org.onosproject.net.flowobjective.impl.InOrderFlowObjectiveManager accumulatorMaxBatchMillis 900

345

346

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 cfg set org.onosproject.net.flowobjective.impl.InOrderFlowObjectiveManager accumulatorMaxIdleMillis 500

347

348

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 cfg set org.opencord.olt.impl.Olt provisionDelay 1000

349

350

# BBSim logs at debug level don't slow down the system much and are very helpful while troubleshooting

351

BBSIM_IDS=\$(kubectl get pods | grep bbsim | grep -v server | awk '{print \$1}')

352

IDS=(\$BBSIM_IDS)

353

354

for bbsim in "\${IDS[@]}"

355

do

356

kubectl exec -t \$bbsim -- bbsimctl log debug false

357

done

358

359

# Set Flows/Ports/Meters/Groups poll frequency

360

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 cfg set org.onosproject.provider.of.flow.impl.OpenFlowRuleProvider flowPollFrequency ${onosStatInterval}

361

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 cfg set org.onosproject.provider.of.device.impl.OpenFlowDeviceProvider portStatsPollFrequency ${onosStatInterval}

362

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 cfg set org.onosproject.provider.of.group.impl.OpenFlowGroupProvider groupPollInterval ${onosGroupInterval}

363

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 cfg set org.onosproject.net.flowobjective.impl.FlowObjectiveManager numThreads ${flowObjWorkerThreads}

364

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 cfg set org.onosproject.net.flowobjective.impl.InOrderFlowObjectiveManager objectiveTimeoutMs 300000

365

366

if [ ${withFlows} = false ]; then

367

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 app deactivate org.opencord.olt

368

fi

369

370

if [ '${workflow}' = 'tt' ]; then

371

etcd_container=\$(kubectl get pods --all-namespaces | grep etcd-0 | awk 'NR==1{print \$2}')

372

kubectl cp $WORKSPACE/voltha-system-tests/tests/data/TechProfile-TT-HSIA.json \$etcd_container:/tmp/hsia.json

373

put_result=\$(kubectl exec -it \$etcd_container -- /bin/sh -c 'cat /tmp/hsia.json | ETCDCTL_API=3 etcdctl put service/voltha/technology_profiles/${tech_prof_directory}/64')

374

kubectl cp $WORKSPACE/voltha-system-tests/tests/data/TechProfile-TT-VoIP.json \$etcd_container:/tmp/voip.json

375

put_result=\$(kubectl exec -it \$etcd_container -- /bin/sh -c 'cat /tmp/voip.json | ETCDCTL_API=3 etcdctl put service/voltha/technology_profiles/${tech_prof_directory}/65')

376

kubectl cp $WORKSPACE/voltha-system-tests/tests/data/TechProfile-TT-MCAST-AdditionalBW-NA.json \$etcd_container:/tmp/mcast.json

377

put_result=\$(kubectl exec -it \$etcd_container -- /bin/sh -c 'cat /tmp/mcast.json | ETCDCTL_API=3 etcdctl put service/voltha/technology_profiles/${tech_prof_directory}/66')

378

fi

379

380

if [ ${withPcap} = true ] ; then

381

# Start the tcp-dump in ofagent

382

export OF_AGENT=\$(kubectl get pods -l app=ofagent -o name)

383

kubectl exec \$OF_AGENT -- apk update

384

kubectl exec \$OF_AGENT -- apk add tcpdump

385

_TAG=ofagent-tcpdump kubectl exec \$OF_AGENT -- tcpdump -nei eth0 -w out.pcap&

386

387

# Start the tcp-dump in radius

388

export RADIUS=\$(kubectl get pods -l app=radius -o name)

389

kubectl exec \$RADIUS -- apt-get update

390

kubectl exec \$RADIUS -- apt-get install -y tcpdump

391

_TAG=radius-tcpdump kubectl exec \$RADIUS -- tcpdump -w out.pcap&

392

393

# Start the tcp-dump in ONOS

394

for i in \$(seq 0 \$ONOSES); do

395

INSTANCE="onos-onos-classic-\$i"

396

kubectl exec \$INSTANCE -- apt-get update

397

kubectl exec \$INSTANCE -- apt-get install -y tcpdump

398

kubectl exec \$INSTANCE -- mv /usr/sbin/tcpdump /usr/bin/tcpdump

399

_TAG=\$INSTANCE kubectl exec \$INSTANCE -- /usr/bin/tcpdump -nei eth0 port 1812 -w out.pcap&

done

fi

"""

}

}

}

stage('Load MIB Template') {

407

when {

408

expression {

409

return params.withMibTemplate

}

}

steps {

sh """

# load MIB template

wget ${mibTemplateUrl} -O mibTemplate.json

416

cat mibTemplate.json | kubectl exec -it \$(kubectl get pods |grep etcd-0 | awk 'NR==1{print \$1}') -- etcdctl put service/voltha/omci_mibs/go_templates/BBSM/v0.0.1/BBSM_IMG_00001

"""

}

}

stage('Run Test') {

steps {

sh '''

mkdir -p $WORKSPACE/RobotLogs

424

cd $WORKSPACE/voltha-system-tests

make vst_venv

'''

sh '''

if [ ${withProfiling} = true ] ; then

429

mkdir -p $LOG_FOLDER/pprof

430

echo $PATH

431

#Creating Python script for ONU Detection

432

cat << EOF > $WORKSPACE/pprof.sh

timestamp() {

date +"%T"

}

i=0

while [[ true ]]; do

((i++))

ts=$(timestamp)

go tool pprof -png http://127.0.0.1:6060/debug/pprof/heap > $LOG_FOLDER/pprof/rw-core-heap-\\$i-\\$ts.png

442

go tool pprof -png http://127.0.0.1:6060/debug/pprof/goroutine > $LOG_FOLDER/pprof/rw-core-goroutine-\\$i-\\$ts.png

443

curl -o $LOG_FOLDER/pprof/rw-core-profile-\\$i-\\$ts.pprof http://127.0.0.1:6060/debug/pprof/profile?seconds=10

444

go tool pprof -png $LOG_FOLDER/pprof/rw-core-profile-\\$i-\\$ts.pprof > $LOG_FOLDER/pprof/rw-core-profile-\\$i-\\$ts.png

445

446

go tool pprof -png http://127.0.0.1:6061/debug/pprof/heap > $LOG_FOLDER/pprof/openolt-heap-\\$i-\\$ts.png

447

go tool pprof -png http://127.0.0.1:6061/debug/pprof/goroutine > $LOG_FOLDER/pprof/openolt-goroutine-\\$i-\\$ts.png

448

curl -o $LOG_FOLDER/pprof/openolt-profile-\\$i-\\$ts.pprof http://127.0.0.1:6061/debug/pprof/profile?seconds=10

449

go tool pprof -png $LOG_FOLDER/pprof/openolt-profile-\\$i-\\$ts.pprof > $LOG_FOLDER/pprof/openolt-profile-\\$i-\\$ts.png

450

451

go tool pprof -png http://127.0.0.1:6062/debug/pprof/heap > $LOG_FOLDER/pprof/ofagent-heap-\\$i-\\$ts.png

452

go tool pprof -png http://127.0.0.1:6062/debug/pprof/goroutine > $LOG_FOLDER/pprof/ofagent-goroutine-\\$i-\\$ts.png

453

curl -o $LOG_FOLDER/pprof/ofagent-profile-\\$i-\\$ts.pprof http://127.0.0.1:6062/debug/pprof/profile?seconds=10

454

go tool pprof -png $LOG_FOLDER/pprof/ofagent-profile-\\$i-\\$ts.pprof > $LOG_FOLDER/pprof/ofagent-profile-\\$i-\\$ts.png

sleep 10

done

EOF

_TAG="pprof"

_TAG=$_TAG bash $WORKSPACE/pprof.sh &

462

fi

463

'''

464

timeout(time: "${testTimeout.toInteger() + 5}", unit: 'MINUTES') {

465

sh '''

466

ROBOT_PARAMS="--exitonfailure \

-v olt:${olts} \

-v pon:${pons} \

-v onu:${onus} \

-v ONOS_SSH_PORT:30115 \

471

-v ONOS_REST_PORT:30120 \

472

-v workflow:${workflow} \

473

-v withEapol:${withEapol} \

474

-v withDhcp:${withDhcp} \

475

-v withIgmp:${withIgmp} \

476

-v timeout:${testTimeout}m \

477

-v withMaclearning:${withMaclearning} \

478

--noncritical non-critical \

479

-e onu-upgrade -e igmp -e teardown "

480

481

if [ ${withEapol} = false ] ; then

482

ROBOT_PARAMS+="-e authentication "

483

fi

484

485

if [ ${withDhcp} = false ] ; then

486

ROBOT_PARAMS+="-e dhcp "

487

fi

488

489

if [ ${provisionSubscribers} = false ] ; then

490

# if we're not considering subscribers then we don't care about authentication and dhcp

491

ROBOT_PARAMS+="-e authentication -e provision -e flow-after -e dhcp "

492

fi

493

494

if [ ${withFlows} = false ] ; then

495

ROBOT_PARAMS+="-i setup -i activation "

496

fi

497

498

if [ ${withOnuUpgrade} = true ] ; then

499

ROBOT_PARAMS+="-e flow-before "

500

fi

501

502

cd $WORKSPACE/voltha-system-tests

503

source ./vst_venv/bin/activate

504

robot -d $WORKSPACE/RobotLogs \

505

$ROBOT_PARAMS tests/scale/Voltha_Scale_Tests.robot

506

507

python tests/scale/collect-result.py -r $WORKSPACE/RobotLogs/output.xml -p $WORKSPACE/plots > $WORKSPACE/execution-time.txt || true

508

cat $WORKSPACE/execution-time.txt

'''

}

}

}

stage('Run ONU Upgrade Tests') {

514

environment {

515

ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/OnuUpgradeTests"

}

when {

expression {

return params.withOnuUpgrade

}

}

options {

timeout(time: "${testTimeout.toInteger() + 1}", unit: 'MINUTES')

}

steps {

sh '''

set +e

mkdir -p $ROBOT_LOGS_DIR

529

cd $WORKSPACE/voltha-system-tests

make vst_venv

'''

script {

Exception caughtException = null

534

535

catchError(buildResult: 'SUCCESS', stageResult: 'ABORTED') {

536

try {

537

sh '''

538

ROBOT_PARAMS="--exitonfailure \

-v olt:${olts} \

-v pon:${pons} \

-v onu:${onus} \

-v workflow:${workflow} \

543

-v withEapol:${withEapol} \

544

-v withDhcp:${withDhcp} \

545

-v withIgmp:${withIgmp} \

546

-v timeout:${testTimeout}m \

547

-v withMaclearning:${withMaclearning}

548

-v image_version:BBSM_IMG_00002 \

549

-v image_url:http://bbsim0:50074/images/software-image.img \

550

-v image_vendor:BBSM \

551

-v image_activate_on_success:false \

552

-v image_commit_on_success:false \

553

-v image_crc:0 \

554

-v ONOS_SSH_PORT:30115 \

555

-v ONOS_REST_PORT:30120 \

556

--noncritical non-critical \

557

-i onu-upgrade \

558

-e setup -e activation -e flow-before \

559

-e authentication -e provision -e flow-after \

560

-e dhcp -e igmp -e teardown "

561

cd $WORKSPACE/voltha-system-tests

562

source ./vst_venv/bin/activate

563

robot -d $ROBOT_LOGS_DIR \

564

$ROBOT_PARAMS tests/scale/Voltha_Scale_Tests.robot

565

'''

566

} catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e) {

567

// if the error is a timeout don't mark the build as failed

568

println "ONU Upgrade test timed out"

569

} catch (Throwable e) {

caughtException = e

}

}

if (caughtException) {

575

error caughtException.message

}

}

}

}

stage('Run Igmp Tests') {

581

environment {

582

ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/IgmpTests"

}

when {

expression {

return params.withIgmp

}

}

options {

timeout(time: "${testTimeout.toInteger() + 1}", unit: 'MINUTES')

591

}

592

steps {

593

sh returnStdout: false, script: """

594

# sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 log:set DEBUG org.onosproject.store.group.impl

"""

sh '''

set +e

mkdir -p $ROBOT_LOGS_DIR

599

cd $WORKSPACE/voltha-system-tests

make vst_venv

'''

script {

Exception caughtException = null

604

605

catchError(buildResult: 'SUCCESS', stageResult: 'ABORTED') {

606

try {

607

sh '''

608

ROBOT_PARAMS="--exitonfailure \

-v olt:${olts} \

-v pon:${pons} \

-v onu:${onus} \

-v workflow:${workflow} \

613

-v withEapol:${withEapol} \

614

-v withDhcp:${withDhcp} \

615

-v withIgmp:${withIgmp} \

616

-v timeout:${testTimeout}m \

617

-v withMaclearning:${withMaclearning}

618

-v ONOS_SSH_PORT:30115 \

619

-v ONOS_REST_PORT:30120 \

620

--noncritical non-critical \

621

-i igmp \

622

-e setup -e activation -e flow-before \

623

-e authentication -e provision -e flow-after \

624

-e dhcp -e onu-upgrade -e teardown "

625

cd $WORKSPACE/voltha-system-tests

626

source ./vst_venv/bin/activate

627

robot -d $ROBOT_LOGS_DIR \

628

$ROBOT_PARAMS tests/scale/Voltha_Scale_Tests.robot

629

'''

630

} catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e) {

631

// if the error is a timeout don't mark the build as failed

632

println "IGMP test timed out"

633

} catch (Throwable e) {

caughtException = e

}

}

if (caughtException) {

639

error caughtException.message

}

}

}

}

stage("Device removal") {

645

options {

646

timeout(time: "${testTimeout.toInteger() + 5}", unit: 'MINUTES')

}

steps {

sh '''

set +e

mkdir -p $ROBOT_LOGS_DIR

652

cd $WORKSPACE/voltha-system-tests

make vst_venv

'''

script {

Exception caughtException = null

657

658

catchError(buildResult: 'SUCCESS', stageResult: 'ABORTED') {

659

try {

660

sh '''

661

ROBOT_PARAMS="--exitonfailure \

-v olt:${olts} \

-v pon:${pons} \

-v onu:${onus} \

-v ONOS_SSH_PORT:30115 \

666

-v ONOS_REST_PORT:30120 \

667

-v workflow:${workflow} \

668

-v withEapol:${withEapol} \

669

-v withDhcp:${withDhcp} \

670

-v withIgmp:${withIgmp} \

671

-v timeout:${testTimeout}m \

672

-v withMaclearning:${withMaclearning} \

673

--noncritical non-critical \

674

-i teardown"

675

676

cd $WORKSPACE/voltha-system-tests

677

source ./vst_venv/bin/activate

678

robot -d $WORKSPACE/RobotLogs \

679

$ROBOT_PARAMS tests/scale/Voltha_Scale_Tests.robot

680

'''

681

} catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e) {

682

// if the error is a timeout don't mark the build as failed

683

println "Cleanup test timed out"

684

} catch (Throwable e) {

caughtException = e

}

}

}

}

}

}

post {

always {

stopComponentsLogs()

// collect result, done in the "post" step so it's executed even in the

696

// event of a timeout in the tests

697

sh '''

698

if [ ${withPcap} = true ] ; then

699

# stop ofAgent tcpdump

700

P_ID="\$(ps e -ww -A | grep "_TAG=ofagent-tcpdump" | grep -v grep | awk '{print \$1}')"

701

if [ -n "\$P_ID" ]; then

kill -9 \$P_ID

fi

# stop radius tcpdump

706

P_ID="\$(ps e -ww -A | grep "_TAG=radius-tcpdump" | grep -v grep | awk '{print \$1}')"

707

if [ -n "\$P_ID" ]; then

kill -9 \$P_ID

fi

# stop onos tcpdump

LIMIT=$(($NUM_OF_ONOS - 1))

713

for i in $(seq 0 $LIMIT); do

714

INSTANCE="onos-onos-classic-$i"

715

P_ID="\$(ps e -ww -A | grep "_TAG=$INSTANCE" | grep -v grep | awk '{print \$1}')"

716

if [ -n "\$P_ID" ]; then

kill -9 \$P_ID

fi

done

# copy the file

export OF_AGENT=$(kubectl get pods -l app=ofagent | awk 'NR==2{print $1}') || true

723

kubectl cp $OF_AGENT:out.pcap $LOG_FOLDER/ofagent.pcap || true

724

725

export RADIUS=$(kubectl get pods -l app=radius | awk 'NR==2{print $1}') || true

726

kubectl cp $RADIUS:out.pcap $LOG_FOLDER/radius.pcap || true

727

728

LIMIT=$(($NUM_OF_ONOS - 1))

729

for i in $(seq 0 $LIMIT); do

730

INSTANCE="onos-onos-classic-$i"

731

kubectl cp $INSTANCE:out.pcap $LOG_FOLDER/$INSTANCE.pcap || true

done

fi

'''

sh '''

if [ ${withProfiling} = true ] ; then

737

_TAG="pprof"

738

P_IDS="$(ps e -ww -A | grep "_TAG=$_TAG" | grep -v grep | awk '{print $1}')"

739

if [ -n "$P_IDS" ]; then

740

echo $P_IDS

741

for P_ID in $P_IDS; do

kill -9 $P_ID

done

fi

fi

'''

plot([

csvFileName: 'scale-test.csv',

749

csvSeries: [

750

[file: 'plots/plot-voltha-onus.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''],

751

[file: 'plots/plot-onos-ports.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''],

752

[file: 'plots/plot-voltha-flows-before.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''],

753

[file: 'plots/plot-voltha-openolt-flows-before.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''],

754

[file: 'plots/plot-onos-flows-before.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''],

755

[file: 'plots/plot-onos-auth.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''],

756

[file: 'plots/plot-voltha-flows-after.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''],

757

[file: 'plots/plot-voltha-openolt-flows-after.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''],

758

[file: 'plots/plot-onos-flows-after.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''],

759

[file: 'plots/plot-onos-dhcp.txt', displayTableFlag: false, exclusionValues: '', inclusionFlag: 'OFF', url: ''],

760

],

761

group: 'Voltha-Scale-Numbers', numBuilds: '20', style: 'line', title: "Scale Test (OLTs: ${olts}, PONs: ${pons}, ONUs: ${onus}, UNIs: ${unis})", yaxis: 'Time (s)', useDescr: true

])

script {

try {

step([$class: 'RobotPublisher',

766

disableArchiveOutput: false,

767

logFileName: '**/log*.html',

768

otherFiles: '',

769

outputFileName: '**/output*.xml',

770

outputPath: 'RobotLogs',

771

passThreshold: 100,

772

reportFileName: '**/report*.html',

773

onlyCritical: true,

774

unstableThreshold: 0]);

775

} catch (Exception e) {

776

println "Cannot archive Robot Logs: ${e.toString()}"

}

}

getPodsInfo("$LOG_FOLDER")

781

// get all the logs from kubernetes PODs

782

sh returnStdout: false, script: '''

783

# copy the ONOS logs directly from the container to avoid the color codes

784

printf '%s\n' $(kubectl get pods -l app=onos-classic -o=jsonpath="{.items[*]['metadata.name']}") | xargs --no-run-if-empty -I# bash -c "kubectl cp #:${karafHome}/data/log/karaf.log $LOG_FOLDER/#.log" || true

785

786

# get ONOS cfg from the 3 nodes

787

# kubectl exec -t voltha-infra-onos-classic-0 -- sh /root/onos/${karafHome}/bin/client cfg get > ~/voltha-infra-onos-classic-0-cfg.txt || true

788

# kubectl exec -t voltha-infra-onos-classic-1 -- sh /root/onos/${karafHome}/bin/client cfg get > ~/voltha-infra-onos-classic-1-cfg.txt || true

789

# kubectl exec -t voltha-infra-onos-classic-2 -- sh /root/onos/${karafHome}/bin/client cfg get > ~/voltha-infra-onos-classic-2-cfg.txt || true

790

791

# kubectl exec -t voltha-infra-onos-classic-0 -- sh /root/onos/${karafHome}/bin/client obj-next-ids > ~/voltha-infra-onos-classic-0-next-objs.txt || true

792

# kubectl exec -t voltha-infra-onos-classic-1 -- sh /root/onos/${karafHome}/bin/client obj-next-ids > ~/voltha-infra-onos-classic-1-next-objs.txt || true

793

# kubectl exec -t voltha-infra-onos-classic-2 -- sh /root/onos/${karafHome}/bin/client obj-next-ids > ~/voltha-infra-onos-classic-2-next-objs.txt || true

794

795

# get radius logs out of the container

796

kubectl cp $(kubectl get pods -l app=radius --no-headers | awk '{print $1}'):/var/log/freeradius/radius.log $LOG_FOLDER/radius.log || true

797

'''

798

// dump all the BBSim(s) ONU information

799

sh '''

800

BBSIM_IDS=$(kubectl get pods | grep bbsim | grep -v server | awk '{print $1}')

801

IDS=($BBSIM_IDS)

802

803

for bbsim in "${IDS[@]}"

804

do

805

kubectl exec -t $bbsim -- bbsimctl onu list > $LOG_FOLDER/$bbsim-device-list.txt || true

806

kubectl exec -t $bbsim -- bbsimctl uni list > $LOG_FOLDER/$bbsim-uni-list.txt || true

807

kubectl exec -t $bbsim -- bbsimctl service list > $LOG_FOLDER/$bbsim-service-list.txt || true

808

kubectl exec -t $bbsim -- bbsimctl olt resources GEM_PORT > $LOG_FOLDER/$bbsim-flows-gem-ports.txt || true

809

kubectl exec -t $bbsim -- bbsimctl olt resources ALLOC_ID > $LOG_FOLDER/$bbsim-flows-alloc-ids.txt || true

810

kubectl exec -t $bbsim -- bbsimctl olt pons > $LOG_FOLDER/$bbsim-pon-resources.txt || true

done

'''

script {

// first make sure the port-forward is still running,

815

// sometimes Jenkins kills it regardless of the JENKINS_NODE_COOKIE=dontKillMe

def running = sh (

returnStdout: true

).trim()

// if any of the voltha-api, onos-rest, onos-ssh port-forwards are not there

821

// kill all and restart

822

if (running != "3") {

823

start_port_forward(olts)

824

}

825

}

826

// get ONOS debug infos

827

sh '''

828

829

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 apps -a -s > $LOG_FOLDER/onos-apps.txt

830

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 nodes > $LOG_FOLDER/onos-nodes.txt

831

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 masters > $LOG_FOLDER/onos-masters.txt

832

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 roles > $LOG_FOLDER/onos-roles.txt

833

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 cfg get > $LOG_FOLDER/onos-cfg.txt

834

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 netcfg > $LOG_FOLDER/onos-netcfg.txt

835

836

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 ports > $LOG_FOLDER/onos-ports-list.txt

837

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 hosts > $LOG_FOLDER/onos-hosts-list.txt

838

839

if [ ${withFlows} = true ] ; then

840

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 volt-olts > $LOG_FOLDER/onos-olt-list.txt

841

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 flows -s > $LOG_FOLDER/onos-flows-list.txt

842

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 meters > $LOG_FOLDER/onos-meters-list.txt

843

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 volt-port-status > $LOG_FOLDER/onos-volt-port-status.txt

844

fi

845

846

if [ ${provisionSubscribers} = true ]; then

847

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 volt-programmed-subscribers > $LOG_FOLDER/onos-programmed-subscribers.txt

848

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 volt-programmed-meters > $LOG_FOLDER/onos-programmed-meters.txt

849

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 volt-bpmeter-mappings > $LOG_FOLDER/onos-bpmeter-mappings.txt

850

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 volt-failed-subscribers > $LOG_FOLDER/onos-failed-subscribers.txt

851

fi

852

853

if [ ${withEapol} = true ] ; then

854

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 aaa-users > $LOG_FOLDER/onos-aaa-users.txt

855

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 aaa-statistics > $LOG_FOLDER/onos-aaa-statistics.txt

856

fi

857

858

if [ ${withDhcp} = true ] ; then

859

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 dhcpl2relay-allocations > $LOG_FOLDER/onos-dhcp-allocations.txt

860

fi

861

862

if [ ${withIgmp} = true ] ; then

863

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 mcast-host-routes > $LOG_FOLDER/onos-mcast-host-routes.txt

864

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 mcast-host-show > $LOG_FOLDER/onos-mcast-host-show.txt

865

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 groups > $LOG_FOLDER/onos-groups.txt

866

fi

867

868

if [ ${withMaclearning} = true ] ; then

869

sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 mac-learner-get-mapping > $LOG_FOLDER/onos-maclearning-host-mappings.txt

870

fi

871

'''

872

// collect etcd metrics

873

sh '''

874

mkdir -p $WORKSPACE/etcd-metrics

875

curl -s -X GET -G http://10.90.0.101:31301/api/v1/query --data-urlencode 'query=etcd_debugging_mvcc_keys_total' | jq '.data' > $WORKSPACE/etcd-metrics/etcd-key-count.json || true

876

curl -s -X GET -G http://10.90.0.101:31301/api/v1/query --data-urlencode 'query=grpc_server_handled_total{grpc_service="etcdserverpb.KV"}' | jq '.data' > $WORKSPACE/etcd-metrics/etcd-rpc-count.json || true

877

curl -s -X GET -G http://10.90.0.101:31301/api/v1/query --data-urlencode 'query=etcd_debugging_mvcc_db_total_size_in_bytes' | jq '.data' > $WORKSPACE/etcd-metrics/etcd-db-size.json || true

878

curl -s -X GET -G http://10.90.0.101:31301/api/v1/query --data-urlencode 'query=etcd_disk_backend_commit_duration_seconds_sum' | jq '.data' > $WORKSPACE/etcd-metrics/etcd-backend-write-time.json || true

879

curl -s -X GET -G http://10.90.0.101:31301/api/v1/query --data-urlencode 'query=etcd_disk_backend_commit_duration_seconds_sum' | jq '.data' > $WORKSPACE/etcd-metrics/etcd-backend-write-time-sum.json || true

880

curl -s -X GET -G http://10.90.0.101:31301/api/v1/query --data-urlencode 'query=etcd_disk_backend_commit_duration_seconds_bucket' | jq '.data' > $WORKSPACE/etcd-metrics/etcd-backend-write-time-bucket.json || true

881

curl -s -X GET -G http://10.90.0.101:31301/api/v1/query --data-urlencode 'query=etcd_disk_wal_fsync_duration_seconds_bucket' | jq '.data' > $WORKSPACE/etcd-metrics/etcd-wal-fsync-time-bucket.json || true

882

curl -s -X GET -G http://10.90.0.101:31301/api/v1/query --data-urlencode 'query=etcd_network_peer_round_trip_time_seconds_bucket' | jq '.data' > $WORKSPACE/etcd-metrics/etcd-network-peer-round-trip-time-seconds.json || true

883

etcd_namespace=\$(kubectl get pods --all-namespaces | grep etcd-0 | awk 'NR==1{print \$1}')

884

etcd_container=\$(kubectl get pods --all-namespaces | grep etcd-0 | awk 'NR==1{print \$2}')

885

kubectl exec -it -n \$etcd_namespace \$etcd_container -- etcdctl defrag --cluster || true

886

kubectl exec -it -n \$etcd_namespace \$etcd_container -- etcdctl endpoint status -w table > $WORKSPACE/etcd-metrics/etcd-status-table.txt || true

887

888

'''

889

// get VOLTHA debug infos

script {

try {

sh '''

voltctl -m 32MB device list -o json > $LOG_FOLDER/device-list.json || true

894

python -m json.tool $LOG_FOLDER/device-list.json > $LOG_FOLDER/voltha-devices-list.json || true

895

rm $LOG_FOLDER/device-list.json || true

896

voltctl -m 32MB device list > $LOG_FOLDER/voltha-devices-list.txt || true

897

898

printf '%s\n' $(voltctl -m 32MB device list | grep olt | awk '{print $1}') | xargs --no-run-if-empty -I# bash -c "voltctl -m 32MB device flows # > $LOG_FOLDER/voltha-device-flows-#.txt" || true

899

printf '%s\n' $(voltctl -m 32MB device list | grep olt | awk '{print $1}') | xargs --no-run-if-empty -I# bash -c "voltctl -m 32MB device port list --format 'table{{.PortNo}}\t{{.Label}}\t{{.Type}}\t{{.AdminState}}\t{{.OperStatus}}' # > $LOG_FOLDER/voltha-device-ports-#.txt" || true

900

901

printf '%s\n' $(voltctl -m 32MB logicaldevice list -q) | xargs --no-run-if-empty -I# bash -c "voltctl -m 32MB logicaldevice flows # > $LOG_FOLDER/voltha-logicaldevice-flows-#.txt" || true

902

printf '%s\n' $(voltctl -m 32MB logicaldevice list -q) | xargs --no-run-if-empty -I# bash -c "voltctl -m 32MB logicaldevice port list # > $LOG_FOLDER/voltha-logicaldevice-ports-#.txt" || true

'''

} catch(e) {

sh '''

echo "Can't get device list from voltclt"

'''

}

}

// get cpu usage by container

911

sh '''

912

if [ ${withMonitoring} = true ] ; then

913

cd $WORKSPACE/voltha-system-tests

914

make vst_venv

915

source ./vst_venv/bin/activate || true

916

sleep 60 # we have to wait for prometheus to collect all the information

917

python scripts/sizing.py -o $WORKSPACE/plots || true

918

fi

919

'''

920

archiveArtifacts artifacts: 'execution-time.txt,logs/*,logs/pprof/*,RobotLogs/**/*,plots/*,etcd-metrics/*'

}

}

}

def start_port_forward(olts) {

926

sh """

927

bbsimRestPortFwd=50071

928

for i in {0..${olts.toInteger() - 1}}; do

929

daemonize -E JENKINS_NODE_COOKIE="dontKillMe" /usr/local/bin/kubectl port-forward --address 0.0.0.0 -n default svc/bbsim\${i} \${bbsimRestPortFwd}:50071

930

((bbsimRestPortFwd++))

931

done

932

"""

933

}