Moving all apps log to DEBUG
Cleaning up logs and adding per instance ONOS ones
Adding custom onos image to TT scale jobs and SW upgrade

Change-Id: I726e4706c8c90a181d30fdff09a2c3a52f6201f3
diff --git a/jjb/pipeline/voltha/master/software-upgrades.groovy b/jjb/pipeline/voltha/master/software-upgrades.groovy
index 8cb4a8a..95f238d 100644
--- a/jjb/pipeline/voltha/master/software-upgrades.groovy
+++ b/jjb/pipeline/voltha/master/software-upgrades.groovy
@@ -57,6 +57,9 @@
       _TAG=onos-port-forward kubectl port-forward --address 0.0.0.0 -n infra svc/voltha-infra-onos-classic-hs 8181:8181&
       _TAG=voltha-port-forward kubectl port-forward --address 0.0.0.0 -n voltha svc/voltha-voltha-api 55555:55555&
       """
+      sh """
+      sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 log:set DEBUG org.opencord
+      """
   }
   stage('Test - '+ name) {
       sh """
@@ -133,6 +136,7 @@
 def get_pods_info(dest) {
   // collect pod details, this is here in case of failure
   sh """
+  rm -rf ${dest} || true
   mkdir -p ${dest}
   kubectl get pods --all-namespaces -o wide > ${dest}/pods.txt || true
   kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.image}{'\\n'}" | sort | uniq | tee ${dest}/pod-images.txt || true
@@ -142,7 +146,7 @@
   """
   sh '''
   # copy the ONOS logs directly from the container to avoid the color codes
-  printf '%s\n' $(kubectl get pods -l app=onos-classic -o=jsonpath="{.items[*]['metadata.name']}") | xargs --no-run-if-empty -I# bash -c "kubectl cp #:${karafHome}/data/log/karaf.log ${dest}/#.log" || true
+  printf '%s\\n' $(kubectl get pods -n infra -l app=onos-classic -o=jsonpath="{.items[*]['metadata.name']}") | xargs --no-run-if-empty -I# bash -c 'kubectl -n infra cp #:apache-karaf-4.2.9/data/log/karaf.log ''' + dest + '''/#.log' || true
   '''
 }
 pipeline {
@@ -156,6 +160,7 @@
   environment {
     PATH="$PATH:$WORKSPACE/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin"
     KUBECONFIG="$HOME/.kube/kind-config-voltha-minimal"
+    SSHPASS="karaf"
   }
   stages{
     stage('Download Code') {
diff --git a/jjb/pipeline/voltha/master/voltha-scale-test.groovy b/jjb/pipeline/voltha/master/voltha-scale-test.groovy
index 74d0af6..2360f40 100644
--- a/jjb/pipeline/voltha/master/voltha-scale-test.groovy
+++ b/jjb/pipeline/voltha/master/voltha-scale-test.groovy
@@ -695,6 +695,7 @@
         sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 masters > $LOG_FOLDER/onos-masters.txt
         sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 roles > $LOG_FOLDER/onos-roles.txt
         sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 cfg get > $LOG_FOLDER/onos-cfg.txt
+        sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 netcfg > $LOG_FOLDER/onos-netcfg.txt
 
         sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 ports > $LOG_FOLDER/onos-ports-list.txt
         sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 hosts > $LOG_FOLDER/onos-hosts-list.txt
diff --git a/jjb/software-upgrades.yaml b/jjb/software-upgrades.yaml
index 945749d..2a0e48c 100644
--- a/jjb/software-upgrades.yaml
+++ b/jjb/software-upgrades.yaml
@@ -9,6 +9,7 @@
       - 'software-upgrades-test':
           name: 'periodic-software-upgrade-test-bbsim'
           pipeline-script: 'voltha/master/software-upgrades.groovy'
+          onosImg: 'andreacampanella/voltha-onos:olt-group-pkt-req-mcast-fix'
           build-node: 'qct-pod4-node2'
           code-branch: 'master'
           aaa-version: '2.3.0.SNAPSHOT'
diff --git a/jjb/voltha-scale.yaml b/jjb/voltha-scale.yaml
index ccace7e..6ddc308 100644
--- a/jjb/voltha-scale.yaml
+++ b/jjb/voltha-scale.yaml
@@ -193,7 +193,7 @@
           withDhcp: true
           withIgmp: false
           extraHelmFlags: '--set authRetry=true,dhcpRetry=true'
-          onosImg: andreacampanella/voltha-onos:olt-mac-integration
+#          onosImg: andreacampanella/voltha-onos:olt-mac-integration
           openoltAdapterImg: matteoscandolo/voltha-openolt-adapter:id-validation
           openonuAdapterGoImg: matteoscandolo/voltha-openonu-adapter-go:id-validation
           bbsimImg: matteoscandolo/bbsim:id-check
@@ -213,7 +213,7 @@
           withDhcp: false
           withIgmp: false
           extraHelmFlags: '--set authRetry=true,dhcpRetry=true'
-          onosImg: andreacampanella/voltha-onos:olt-mac-integration
+#          onosImg: andreacampanella/voltha-onos:olt-mac-integration
 #          bbsimImg: matteoscandolo/bbsim:id-check
 
       - 'voltha-scale-measurements':
@@ -233,7 +233,7 @@
           onosReplicas: 3
           atomixReplicas: 3
           extraHelmFlags: '--set authRetry=true,dhcpRetry=true'
-          onosImg: andreacampanella/voltha-onos:olt-mac-integration
+          onosImg: andreacampanella/voltha-onos:olt-group-pkt-req-mcast-fix
 #          bbsimImg: matteoscandolo/bbsim:id-check
 
       # multi-stack jobs
@@ -255,7 +255,7 @@
           withEapol: true
           withDhcp: true
           withIgmp: false
-          onosImg: andreacampanella/voltha-onos:olt-mac-integration
+          #onosImg: andreacampanella/voltha-onos:olt-mac-integration
 
       - 'voltha-scale-measurements':
           name: 'voltha-scale-measurements-master-10-stacks-2-16-32-dt-subscribers'
@@ -275,7 +275,7 @@
           withEapol: false
           withDhcp: false
           withIgmp: false
-          onosImg: andreacampanella/voltha-onos:olt-mac-integration
+          #onosImg: andreacampanella/voltha-onos:olt-mac-integration
 
       - 'voltha-scale-measurements':
           name: 'voltha-scale-measurements-master-10-stacks-2-16-32-tt-subscribers'
@@ -295,7 +295,7 @@
           withEapol: false
           withDhcp: true
           withIgmp: true
-          onosImg: andreacampanella/voltha-onos:olt-mac-integration
+          onosImg: andreacampanella/voltha-onos:olt-group-pkt-req-mcast-fix
 
       # voltha-2.6 Jobs
       - 'voltha-scale-measurements':