Updating voltha-scale-multi-stack pipeline to latest keywords
Change-Id: I744107a1762ba1fe1b0361417b5e0af7834a4179
diff --git a/jjb/pipeline/voltha/master/voltha-scale-multi-stack.groovy b/jjb/pipeline/voltha/master/voltha-scale-multi-stack.groovy
index 8c2f72e..a4584ab 100644
--- a/jjb/pipeline/voltha/master/voltha-scale-multi-stack.groovy
+++ b/jjb/pipeline/voltha/master/voltha-scale-multi-stack.groovy
@@ -22,14 +22,6 @@
remote: 'https://gerrit.opencord.org/ci-management.git'
])
-def ofAgentConnections(numOfOnos, releaseName, namespace) {
- def params = " "
- numOfOnos.times {
- params += "--set voltha.services.controller[${it}].address=${releaseName}-onos-classic-${it}.${releaseName}-onos-classic-hs.${namespace}.svc:6653 "
- }
- return params
-}
-
pipeline {
/* no label, executor is determined by JJB */
@@ -45,7 +37,6 @@
SSHPASS="karaf"
PATH="$PATH:$WORKSPACE/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin"
- APPS_TO_LOG="etcd kafka onos-onos-classic adapter-open-onu adapter-open-olt rw-core ofagent bbsim radius bbsim-sadis-server"
LOG_FOLDER="$WORKSPACE/logs"
}
@@ -98,32 +89,55 @@
'''
}
}
+ stage('Start logging') {
+ steps {
+ script {
+ startComponentsLogs([
+ appsToLog: [
+ 'app.kubernetes.io/name=etcd',
+ 'app.kubernetes.io/name=kafka',
+ 'app=onos-classic',
+ 'app=adapter-open-onu',
+ 'app=adapter-open-olt',
+ 'app=rw-core',
+ 'app=ofagent',
+ 'app=bbsim',
+ 'app=radius',
+ 'app=bbsim-sadis-server',
+ 'app=onos-config-loader',
+ ]
+ ])
+ }
+ }
+ }
stage('Deploy VOLTHA infrastructure') {
steps {
- sh returnStdout: false, script: '''
+ timeout(time: 5, unit: 'MINUTES') {
+ script {
+ def localCharts = false
+ if (volthaHelmChartsChange != "" || release != "master") {
+ localCharts = true
+ }
- kubectl create configmap -n infra kube-config "--from-file=kube_config=\$KUBECONFIG" || true
+ def infraHelmFlags =
+ "--set global.log_level=${logLevel} " +
+ "--set radius.enabled=${withEapol} " +
+ "--set onos-classic.onosSshPort=30115 " +
+ "--set onos-classic.onosApiPort=30120 " +
+ params.extraHelmFlags
- export EXTRA_HELM_FLAGS+=' '
-
- # No persistent-volume-claims in Atomix
- EXTRA_HELM_FLAGS+="--set onos-classic.atomix.persistence.enabled=false "
- # disable the securityContext, this is a development cluster
- EXTRA_HELM_FLAGS+='--set securityContext.enabled=false '
-
- echo \$EXTRA_HELM_FLAGS
-
- helm upgrade --install -n infra voltha-infra onf/voltha-infra \
- -f $WORKSPACE/voltha-helm-charts/examples/${workflow}-values.yaml \
- -f /home/jenkins/voltha-scale/voltha-values.yaml \
- --set onos-classic.replicas=${onosReplicas},onos-classic.atomix.replicas=${atomixReplicas} \
- --set radius.enabled=${withEapol} \
- --set global.log_level=${logLevel} \
- --set onos-classic.onosSshPort=30115 \
- --set onos-classic.onosApiPort=30120 \
- --set kafka.replicaCount=3,kafka.zookeeper.replicaCount=3 \
- --set etcd.statefulset.replicaCount=3 \$EXTRA_HELM_FLAGS
- '''
+ volthaInfraDeploy([
+ workflow: workflow,
+ infraNamespace: "infra",
+ extraHelmFlags: infraHelmFlags,
+ localCharts: localCharts,
+ onosReplica: onosReplicas,
+ atomixReplica: atomixReplicas,
+ kafkaReplica: kafkaReplicas,
+ etcdReplica: etcdReplicas,
+ ])
+ }
+ }
}
}
stage('Deploy Voltha') {
@@ -131,23 +145,6 @@
deploy_voltha_stacks(params.volthaStacks)
}
}
- stage('Start logging') {
- //FIXME this collects the logs all in one file for all the 10 stacks
- steps {
- sh returnStdout: false, script: '''
- # start logging with kail
-
- mkdir -p $LOG_FOLDER
-
- list=($APPS_TO_LOG)
- for app in "${list[@]}"
- do
- echo "Starting logs for: ${app}"
- _TAG=kail-$app kail -l app=$app --since 1h > $LOG_FOLDER/$app.log&
- done
- '''
- }
- }
stage('Configuration') {
steps {
script {
@@ -210,31 +207,9 @@
}
post {
always {
+ stopComponentsLogs([compress: true])
// collect result, done in the "post" step so it's executed even in the
// event of a timeout in the tests
- sh '''
-
- # stop the kail processes
- list=($APPS_TO_LOG)
- for app in "${list[@]}"
- do
- echo "Stopping logs for: ${app}"
- _TAG="kail-$app"
- P_IDS="$(ps e -ww -A | grep "_TAG=$_TAG" | grep -v grep | awk '{print $1}')"
- if [ -n "$P_IDS" ]; then
- echo $P_IDS
- for P_ID in $P_IDS; do
- kill -9 $P_ID
- done
- fi
- done
- '''
- // compressing the logs to save space on Jenkins
- sh '''
- cd $LOG_FOLDER
- tar -czf logs.tar.gz *.log
- rm *.log
- '''
plot([
csvFileName: 'scale-test.csv',
csvSeries: [
@@ -304,6 +279,9 @@
sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 masters > $LOG_FOLDER/onos-masters.txt || true
sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 roles > $LOG_FOLDER/onos-roles.txt || true
+ sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 netcfg > $LOG_FOLDER/onos-netcfg.txt || true
+ sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 cfg get > $LOG_FOLDER/onos-cfg.txt || true
+
sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 ports > $LOG_FOLDER/onos-ports-list.txt || true
sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 hosts > $LOG_FOLDER/onos-hosts-list.txt || true
@@ -387,30 +365,32 @@
def deploy_voltha_stacks(numberOfStacks) {
for (int i = 1; i <= numberOfStacks.toInteger(); i++) {
- stage("Deploy VOLTHA stack " + i) {
- // ${logLevel}
- def extraHelmFlags = "${extraHelmFlags} --set global.log_level=${logLevel},enablePerf=true,onu=${onus},pon=${pons} "
- extraHelmFlags += " --set securityContext.enabled=false,atomix.persistence.enabled=false "
+ timeout(time: 5, unit: 'MINUTES') {
+ stage("Deploy VOLTHA stack " + i) {
- // FIXME having to set all of these values is annoying, is there a better solution?
- def volthaHelmFlags = extraHelmFlags +
- ofAgentConnections(onosReplicas.toInteger(), "voltha-infra", "infra")
+ def localCharts = false
+ if (volthaHelmChartsChange != "" || release != "master") {
+ localCharts = true
+ }
- def localCharts = false
- if (volthaHelmChartsChange != "") {
- localCharts = true
+ def volthaHelmFlags =
+ "--set global.log_level=${logLevel} " +
+ "--set enablePerf=true,onu=${onus},pon=${pons} " +
+ "--set securityContext.enabled=false " +
+ params.extraHelmFlags
+
+ volthaStackDeploy([
+ bbsimReplica: olts.toInteger(),
+ infraNamespace: "infra",
+ volthaNamespace: "voltha${i}",
+ stackName: "voltha${i}",
+ stackId: i,
+ workflow: workflow,
+ extraHelmFlags: volthaHelmFlags,
+ localCharts: localCharts,
+ onosReplica: onosReplicas,
+ ])
}
-
- volthaStackDeploy([
- bbsimReplica: olts.toInteger(),
- infraNamespace: "infra",
- volthaNamespace: "voltha${i}",
- stackName: "voltha${i}",
- stackId: i,
- workflow: workflow,
- extraHelmFlags: volthaHelmFlags,
- localCharts: localCharts
- ])
}
}
}
diff --git a/jjb/pipeline/voltha/master/voltha-scale-test.groovy b/jjb/pipeline/voltha/master/voltha-scale-test.groovy
index 8248f36..92844c8 100644
--- a/jjb/pipeline/voltha/master/voltha-scale-test.groovy
+++ b/jjb/pipeline/voltha/master/voltha-scale-test.groovy
@@ -230,7 +230,8 @@
def extraHelmFlags = returned_flags
// The added space before params.extraHelmFlags is required due to the .trim() above
def infraHelmFlags =
- " --set global.log_level=${logLevel} " +
+ "--set global.log_level=${logLevel} " +
+ "--set radius.enabled=${withEapol} " +
"--set onos-classic.onosSshPort=30115 " +
"--set onos-classic.onosApiPort=30120 " +
extraHelmFlags + " " + params.extraHelmFlags
@@ -265,6 +266,7 @@
workflow: workflow,
extraHelmFlags: stackHelmFlags,
localCharts: localCharts,
+ onosReplica: onosReplicas,
])
sh """
set +x
diff --git a/jjb/pipeline/voltha/voltha-2.8/voltha-scale-multi-stack.groovy b/jjb/pipeline/voltha/voltha-2.8/voltha-scale-multi-stack.groovy
index 8c2f72e..a4584ab 100644
--- a/jjb/pipeline/voltha/voltha-2.8/voltha-scale-multi-stack.groovy
+++ b/jjb/pipeline/voltha/voltha-2.8/voltha-scale-multi-stack.groovy
@@ -22,14 +22,6 @@
remote: 'https://gerrit.opencord.org/ci-management.git'
])
-def ofAgentConnections(numOfOnos, releaseName, namespace) {
- def params = " "
- numOfOnos.times {
- params += "--set voltha.services.controller[${it}].address=${releaseName}-onos-classic-${it}.${releaseName}-onos-classic-hs.${namespace}.svc:6653 "
- }
- return params
-}
-
pipeline {
/* no label, executor is determined by JJB */
@@ -45,7 +37,6 @@
SSHPASS="karaf"
PATH="$PATH:$WORKSPACE/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin"
- APPS_TO_LOG="etcd kafka onos-onos-classic adapter-open-onu adapter-open-olt rw-core ofagent bbsim radius bbsim-sadis-server"
LOG_FOLDER="$WORKSPACE/logs"
}
@@ -98,32 +89,55 @@
'''
}
}
+ stage('Start logging') {
+ steps {
+ script {
+ startComponentsLogs([
+ appsToLog: [
+ 'app.kubernetes.io/name=etcd',
+ 'app.kubernetes.io/name=kafka',
+ 'app=onos-classic',
+ 'app=adapter-open-onu',
+ 'app=adapter-open-olt',
+ 'app=rw-core',
+ 'app=ofagent',
+ 'app=bbsim',
+ 'app=radius',
+ 'app=bbsim-sadis-server',
+ 'app=onos-config-loader',
+ ]
+ ])
+ }
+ }
+ }
stage('Deploy VOLTHA infrastructure') {
steps {
- sh returnStdout: false, script: '''
+ timeout(time: 5, unit: 'MINUTES') {
+ script {
+ def localCharts = false
+ if (volthaHelmChartsChange != "" || release != "master") {
+ localCharts = true
+ }
- kubectl create configmap -n infra kube-config "--from-file=kube_config=\$KUBECONFIG" || true
+ def infraHelmFlags =
+ "--set global.log_level=${logLevel} " +
+ "--set radius.enabled=${withEapol} " +
+ "--set onos-classic.onosSshPort=30115 " +
+ "--set onos-classic.onosApiPort=30120 " +
+ params.extraHelmFlags
- export EXTRA_HELM_FLAGS+=' '
-
- # No persistent-volume-claims in Atomix
- EXTRA_HELM_FLAGS+="--set onos-classic.atomix.persistence.enabled=false "
- # disable the securityContext, this is a development cluster
- EXTRA_HELM_FLAGS+='--set securityContext.enabled=false '
-
- echo \$EXTRA_HELM_FLAGS
-
- helm upgrade --install -n infra voltha-infra onf/voltha-infra \
- -f $WORKSPACE/voltha-helm-charts/examples/${workflow}-values.yaml \
- -f /home/jenkins/voltha-scale/voltha-values.yaml \
- --set onos-classic.replicas=${onosReplicas},onos-classic.atomix.replicas=${atomixReplicas} \
- --set radius.enabled=${withEapol} \
- --set global.log_level=${logLevel} \
- --set onos-classic.onosSshPort=30115 \
- --set onos-classic.onosApiPort=30120 \
- --set kafka.replicaCount=3,kafka.zookeeper.replicaCount=3 \
- --set etcd.statefulset.replicaCount=3 \$EXTRA_HELM_FLAGS
- '''
+ volthaInfraDeploy([
+ workflow: workflow,
+ infraNamespace: "infra",
+ extraHelmFlags: infraHelmFlags,
+ localCharts: localCharts,
+ onosReplica: onosReplicas,
+ atomixReplica: atomixReplicas,
+ kafkaReplica: kafkaReplicas,
+ etcdReplica: etcdReplicas,
+ ])
+ }
+ }
}
}
stage('Deploy Voltha') {
@@ -131,23 +145,6 @@
deploy_voltha_stacks(params.volthaStacks)
}
}
- stage('Start logging') {
- //FIXME this collects the logs all in one file for all the 10 stacks
- steps {
- sh returnStdout: false, script: '''
- # start logging with kail
-
- mkdir -p $LOG_FOLDER
-
- list=($APPS_TO_LOG)
- for app in "${list[@]}"
- do
- echo "Starting logs for: ${app}"
- _TAG=kail-$app kail -l app=$app --since 1h > $LOG_FOLDER/$app.log&
- done
- '''
- }
- }
stage('Configuration') {
steps {
script {
@@ -210,31 +207,9 @@
}
post {
always {
+ stopComponentsLogs([compress: true])
// collect result, done in the "post" step so it's executed even in the
// event of a timeout in the tests
- sh '''
-
- # stop the kail processes
- list=($APPS_TO_LOG)
- for app in "${list[@]}"
- do
- echo "Stopping logs for: ${app}"
- _TAG="kail-$app"
- P_IDS="$(ps e -ww -A | grep "_TAG=$_TAG" | grep -v grep | awk '{print $1}')"
- if [ -n "$P_IDS" ]; then
- echo $P_IDS
- for P_ID in $P_IDS; do
- kill -9 $P_ID
- done
- fi
- done
- '''
- // compressing the logs to save space on Jenkins
- sh '''
- cd $LOG_FOLDER
- tar -czf logs.tar.gz *.log
- rm *.log
- '''
plot([
csvFileName: 'scale-test.csv',
csvSeries: [
@@ -304,6 +279,9 @@
sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 masters > $LOG_FOLDER/onos-masters.txt || true
sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 roles > $LOG_FOLDER/onos-roles.txt || true
+ sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 netcfg > $LOG_FOLDER/onos-netcfg.txt || true
+ sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 30115 karaf@127.0.0.1 cfg get > $LOG_FOLDER/onos-cfg.txt || true
+
sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 ports > $LOG_FOLDER/onos-ports-list.txt || true
sshpass -e ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 hosts > $LOG_FOLDER/onos-hosts-list.txt || true
@@ -387,30 +365,32 @@
def deploy_voltha_stacks(numberOfStacks) {
for (int i = 1; i <= numberOfStacks.toInteger(); i++) {
- stage("Deploy VOLTHA stack " + i) {
- // ${logLevel}
- def extraHelmFlags = "${extraHelmFlags} --set global.log_level=${logLevel},enablePerf=true,onu=${onus},pon=${pons} "
- extraHelmFlags += " --set securityContext.enabled=false,atomix.persistence.enabled=false "
+ timeout(time: 5, unit: 'MINUTES') {
+ stage("Deploy VOLTHA stack " + i) {
- // FIXME having to set all of these values is annoying, is there a better solution?
- def volthaHelmFlags = extraHelmFlags +
- ofAgentConnections(onosReplicas.toInteger(), "voltha-infra", "infra")
+ def localCharts = false
+ if (volthaHelmChartsChange != "" || release != "master") {
+ localCharts = true
+ }
- def localCharts = false
- if (volthaHelmChartsChange != "") {
- localCharts = true
+ def volthaHelmFlags =
+ "--set global.log_level=${logLevel} " +
+ "--set enablePerf=true,onu=${onus},pon=${pons} " +
+ "--set securityContext.enabled=false " +
+ params.extraHelmFlags
+
+ volthaStackDeploy([
+ bbsimReplica: olts.toInteger(),
+ infraNamespace: "infra",
+ volthaNamespace: "voltha${i}",
+ stackName: "voltha${i}",
+ stackId: i,
+ workflow: workflow,
+ extraHelmFlags: volthaHelmFlags,
+ localCharts: localCharts,
+ onosReplica: onosReplicas,
+ ])
}
-
- volthaStackDeploy([
- bbsimReplica: olts.toInteger(),
- infraNamespace: "infra",
- volthaNamespace: "voltha${i}",
- stackName: "voltha${i}",
- stackId: i,
- workflow: workflow,
- extraHelmFlags: volthaHelmFlags,
- localCharts: localCharts
- ])
}
}
}
diff --git a/jjb/pipeline/voltha/voltha-2.8/voltha-scale-test.groovy b/jjb/pipeline/voltha/voltha-2.8/voltha-scale-test.groovy
index 82fbf27..195da4a 100644
--- a/jjb/pipeline/voltha/voltha-2.8/voltha-scale-test.groovy
+++ b/jjb/pipeline/voltha/voltha-2.8/voltha-scale-test.groovy
@@ -230,7 +230,8 @@
def extraHelmFlags = returned_flags
// The added space before params.extraHelmFlags is required due to the .trim() above
def infraHelmFlags =
- " --set global.log_level=${logLevel} " +
+ "--set global.log_level=${logLevel} " +
+ "--set radius.enabled=${withEapol} " +
"--set onos-classic.onosSshPort=30115 " +
"--set onos-classic.onosApiPort=30120 " +
extraHelmFlags + " " + params.extraHelmFlags
@@ -249,6 +250,7 @@
atomixReplica: atomixReplicas,
kafkaReplica: kafkaReplicas,
etcdReplica: etcdReplicas,
+ onosReplica: onosReplicas,
])
stackHelmFlags = " --set onu=${onus},pon=${pons} --set global.log_level=${logLevel.toLowerCase()} "
diff --git a/jjb/voltha-scale.yaml b/jjb/voltha-scale.yaml
index b3ccb65..35e3848 100644
--- a/jjb/voltha-scale.yaml
+++ b/jjb/voltha-scale.yaml
@@ -47,7 +47,7 @@
withEapol: false
withDhcp: false
withIgmp: false
- extraHelmFlags: " -f /home/jenkins/voltha-scale/voltha-values.yaml --set etcd.persistence.enabled=true,etcd.persistence.storageClass=longhorn "
+ # extraHelmFlags: " -f /home/jenkins/voltha-scale/voltha-values.yaml --set etcd.persistence.enabled=true,etcd.persistence.storageClass=longhorn "
- 'voltha-scale-measurements':
name: 'voltha-scale-measurements-master-2-16-32-att-subscribers-ssd'
@@ -192,7 +192,6 @@
withEapol: true
withDhcp: true
withIgmp: false
- extraHelmFlags: '--set defaults.rw_core.timeout=30s '
release: voltha-2.8
bbsimImg: ''
rwCoreImg: ''
@@ -217,7 +216,6 @@
withEapol: false
withDhcp: false
withIgmp: false
- extraHelmFlags: '--set defaults.rw_core.timeout=30s '
release: voltha-2.8
bbsimImg: ''
rwCoreImg: ''
@@ -242,7 +240,6 @@
withEapol: false
withDhcp: true
withIgmp: true
- extraHelmFlags: '--set defaults.rw_core.timeout=30s '
release: voltha-2.8
bbsimImg: ''
rwCoreImg: ''
@@ -279,6 +276,7 @@
openonuAdapterImg: ''
openonuAdapterGoImg: ''
onosImg: ''
+ extraHelmFlags: "-f /home/jenkins/voltha-scale/voltha-values.yaml "
- 'voltha-scale-measurements':
name: 'voltha-scale-measurements-2.8-10-stacks-2-16-32-dt-subscribers'
@@ -306,6 +304,7 @@
openonuAdapterImg: ''
openonuAdapterGoImg: ''
onosImg: ''
+ extraHelmFlags: "-f /home/jenkins/voltha-scale/voltha-values.yaml "
- 'voltha-scale-measurements':
name: 'voltha-scale-measurements-2.8-10-stacks-2-16-32-tt-subscribers'
@@ -333,6 +332,7 @@
openonuAdapterImg: ''
openonuAdapterGoImg: ''
onosImg: ''
+ extraHelmFlags: "-f /home/jenkins/voltha-scale/voltha-values.yaml "
# per patchset job
- 'voltha-scale-measurements':
@@ -383,7 +383,6 @@
- 'voltha-scale-measurements-dev':
name: 'voltha-scale-measurements-master-dev'
build-node: 'voltha-scale'
- extraHelmFlags: '--set defaults.rw_core.timeout=30s '
# list of parameters for the VOLTHA Jobs,