jjb/pipeline/voltha/software-upgrades.groovy

#!/usr/bin/env groovy
// -----------------------------------------------------------------------
// Copyright 2021-2024 Open Networking Foundation Contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// -----------------------------------------------------------------------
// SPDX-FileCopyrightText: 2021-2024 Open Networking Foundation Contributors
// SPDX-License-Identifier: Apache-2.0
// -----------------------------------------------------------------------
// Entropy: 0fcb5ffa-d1a4-11ee-be5e-9f44b7181764
// -----------------------------------------------------------------------
// Intent: voltha-2.x e2e tests, uses bbsim to simulate OLT/ONUs
// -----------------------------------------------------------------------
// NOTE: we are importing the library even if it's global so that it's
// easier to change the keywords during a replay
// -----------------------------------------------------------------------

library identifier: 'cord-jenkins-libraries@master',
    retriever: modernSCM([
      $class: 'GitSCMSource',
      remote: 'https://gerrit.opencord.org/ci-management.git'
])

// -----------------------------------------------------------------------
// Intent:
// -----------------------------------------------------------------------
String branchName() {
    String name = 'master'

    // [TODO] Sanity check the target branch
    // if (name != jenkins.branch) { fatal }
    return(name)
}

// -----------------------------------------------------------------------
// Intent: Due to lack of a reliable stack trace, construct a literal.
//         Jenkins will re-write the call stack for serialization.
// -----------------------------------------------------------------------
String getIam(String func) {
    String branchName = branchName()
    String src = [
        'ci-management',
        'jjb',
        'pipeline',
        'voltha',
        branchName,
        'software-upgrades.groovy'
    ].join('/')

    String name = [src, func].join('::')
    return(name)
}

// -----------------------------------------------------------------------
// fetches the versions/tags of the voltha component
// returns the deployment version which is one less than the latest available tag of the repo, first voltha stack gets deployed using this;
// returns the test version which is the latest tag of the repo, the component upgrade gets tested on this.
// Note: if there is a major version change between deployment and test tags, then deployment tag will be same as test tag, i.e. both as latest.
// -----------------------------------------------------------------------
// git ls repo: gerrit (source of truth) VS github (mirror)
// Branch deletions are not mirrored out to github.
// Always perform checkouts/queries using the gerrit server as added
// insulation to guard against repository deltas.
// -----------------------------------------------------------------------
// [TODO] Revisit logic:
//   o Refactor logic into a standalone shell script.
//   o Redundancy can be removed and a script will be testable.
// -----------------------------------------------------------------------
def get_voltha_comp_versions(component, base_deploy_tag) {
    def comp_test_tag = sh(
        // script: "git ls-remote --refs --tags https://github.com/opencord/${component} | cut --delimiter='/' --fields=3 | tr '-' '~' | sort --version-sort | tail --lines=1 | sed 's/v//'",
        script: "git ls-remote --refs --tags https://gerrit.opencord.org/${component} | cut --delimiter='/' --fields=3 | tr '-' '~' | sort --version-sort | tail --lines=1 | sed 's/v//'",
        returnStdout: true
    ).trim()
    def comp_deploy_tag = sh(
        // script: "git ls-remote --refs --tags https://github.com/opencord/${component} | cut --delimiter='/' --fields=3 | tr '-' '~' | sort --version-sort | tail --lines=2 | head -n 1 | sed 's/v//'",
        script: "git ls-remote --refs --tags https://gerrit.opencord.org/${component} | cut --delimiter='/' --fields=3 | tr '-' '~' | sort --version-sort | tail --lines=2 | head -n 1 | sed 's/v//'",
        returnStdout: true
    ).trim()
    def comp_deploy_major = comp_deploy_tag.substring(0, comp_deploy_tag.indexOf('.'))
    def comp_test_major = comp_test_tag.substring(0, comp_test_tag.indexOf('.'))
    if ("${comp_deploy_major.trim()}" != "${comp_test_major.trim()}") {
        comp_deploy_tag = comp_test_tag
    }
    if ("${comp_test_tag.trim()}" == "${base_deploy_tag.trim()}") {
        comp_deploy_tag = comp_test_tag
        comp_test_tag = "master"
    }
    println "${component}: deploy_tag: ${comp_deploy_tag}, test_tag: ${comp_test_tag}"
    return [comp_deploy_tag, comp_test_tag]
}

// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
def test_software_upgrade(name) {
    def infraNamespace = "infra"
    def volthaNamespace = "voltha"
    def openolt_adapter_deploy_tag = ''
    def openolt_adapter_test_tag = ''
    def openonu_adapter_deploy_tag = ''
    def openonu_adapter_test_tag = ''
    def rw_core_deploy_tag = ''
    def rw_core_test_tag = ''
    def ofagent_deploy_tag = ''
    def ofagent_test_tag = ''
    def logsDir = "$WORKSPACE/${name}"
    stage('Deploy Voltha - ' + name) {
        timeout(10) {
            // start logging
            sh """
      rm -rf ${logsDir} || true
      mkdir -p ${logsDir}
      _TAG=kail-${name} kail -n ${infraNamespace} -n ${volthaNamespace} > ${logsDir}/onos-voltha-startup-combined.log &
      """
            def extraHelmFlags = extraHelmFlags.trim()
            if ("${name}" == "onos-app-upgrade" || "${name}" == "onu-software-upgrade" || "${name}" == "onu-software-upgrade-omci-extended-msg" || "${name}" == "voltha-component-upgrade" || "${name}" == "voltha-component-rolling-upgrade") {
                extraHelmFlags = " --set global.log_level=${logLevel.toUpperCase()},onu=1,pon=1 --set onos-classic.replicas=3,onos-classic.atomix.replicas=3 " + extraHelmFlags
            }
            if ("${name}" == "onu-software-upgrade" || "${name}" == "onu-software-upgrade-omci-extended-msg") {
                extraHelmFlags = " --set global.extended_omci_support.enabled=true " + extraHelmFlags
            }
            if ("${name}" == "onu-software-upgrade-omci-extended-msg") {
                extraHelmFlags = " --set omccVersion=180 " + extraHelmFlags
            }
            if ("${name}" == "onu-image-dwl-simultaneously") {
                extraHelmFlags = " --set global.log_level=${logLevel.toUpperCase()},onu=2,pon=2 --set onos-classic.replicas=3,onos-classic.atomix.replicas=3 " + extraHelmFlags
            }
            if ("${name}" == "onos-app-upgrade" || "${name}" == "onu-software-upgrade" || "${name}" == "onu-software-upgrade-omci-extended-msg" || "${name}" == "onu-image-dwl-simultaneously") {
                extraHelmFlags = " --set global.image_tag=master --set onos-classic.image.tag=master " + extraHelmFlags
            }
            if ("${name}" == "voltha-component-upgrade" || "${name}" == "voltha-component-rolling-upgrade") {
                extraHelmFlags = " --set images.onos_config_loader.tag=master-onos-config-loader --set onos-classic.image.tag=master " + extraHelmFlags
            }
            extraHelmFlags += " --set onos-classic.onosSshPort=30115 --set onos-classic.onosApiPort=30120 "
            extraHelmFlags += " --set voltha.onos_classic.replicas=3"
            //ONOS custom image handling
            if ( onosImg.trim() != '' ) {
                String[] split;
                onosImg = onosImg.trim()
                split = onosImg.split(':')
                extraHelmFlags += " --set onos-classic.image.repository=" + split[0] +",onos-classic.image.tag=" + split[1] + " "
            }
            Integer olts = 1
            if ("${name}" == 'onu-image-dwl-simultaneously') {
                olts = 2
            }
            if ("${name}" == 'voltha-component-upgrade' || "${name}" == 'voltha-component-rolling-upgrade') {
                // fetch voltha components versions/tags
                (openolt_adapter_deploy_tag, openolt_adapter_test_tag) = get_voltha_comp_versions('voltha-openolt-adapter', openoltAdapterDeployBaseTag.trim())
                extraHelmFlags += " --set voltha-adapter-openolt.images.adapter_open_olt.tag=${openolt_adapter_deploy_tag} "
                (openonu_adapter_deploy_tag, openonu_adapter_test_tag) = get_voltha_comp_versions('voltha-openonu-adapter-go', openonuAdapterDeployBaseTag.trim())
                extraHelmFlags += " --set voltha-adapter-openonu.images.adapter_open_onu_go.tag=${openonu_adapter_deploy_tag} "
                (rw_core_deploy_tag, rw_core_test_tag) = get_voltha_comp_versions('voltha-go', rwCoreDeployBaseTag.trim())
                extraHelmFlags += " --set voltha.images.rw_core.tag=${rw_core_deploy_tag} "
                (ofagent_deploy_tag, ofagent_test_tag) = get_voltha_comp_versions('ofagent-go', ofagentDeployBaseTag.trim())
                extraHelmFlags += " --set voltha.images.ofagent.tag=${ofagent_deploy_tag} "
            }
            def localCharts = false
            // Currently only testing with ATT workflow
            // TODO: Support for other workflows
            volthaDeploy([bbsimReplica: olts.toInteger(), workflow: 'att', extraHelmFlags: extraHelmFlags, localCharts: localCharts])
            // stop logging
            sh """
        P_IDS="\$(ps e -ww -A | grep "_TAG=kail-${name}" | grep -v grep | awk '{print \$1}')"
        if [ -n "\$P_IDS" ]; then
          echo \$P_IDS
          for P_ID in \$P_IDS; do
            kill -9 \$P_ID
          done
        fi
        cd ${logsDir}
        gzip -k onos-voltha-startup-combined.log
        rm onos-voltha-startup-combined.log
      """
            // forward ONOS and VOLTHA ports
            sh('''
      JENKINS_NODE_COOKIE="dontKillMe" _TAG=onos-port-forward /bin/bash -c "while true; do kubectl -n infra port-forward --address 0.0.0.0 service/voltha-infra-onos-classic-hs 8101:8101; done 2>&1 " &
      JENKINS_NODE_COOKIE="dontKillMe" _TAG=onos-port-forward /bin/bash -c "while true; do kubectl -n infra port-forward --address 0.0.0.0 service/voltha-infra-onos-classic-hs 8181:8181; done 2>&1 " &
      ''')
            sh('''
      sshpass -e ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 8101 karaf@127.0.0.1 log:set DEBUG org.opencord
      ''')
        }
    }

    stage('Test - ' + name) {
        timeout(75) {
            sh """
        ROBOT_LOGS_DIR="$WORKSPACE/RobotLogs/${name}"
        mkdir -p \$ROBOT_LOGS_DIR
        if [[ ${name} == 'onos-app-upgrade' ]]; then
          export ONOS_APPS_UNDER_TEST+=''
          if [ ${aaaVer.trim()} != '' ] && [ ${aaaOarUrl.trim()} != '' ]; then
            ONOS_APPS_UNDER_TEST+="org.opencord.aaa,${aaaVer.trim()},${aaaOarUrl.trim()}*"
          fi
          if [ ${oltVer.trim()} != '' ] && [ ${oltOarUrl.trim()} != '' ]; then
            ONOS_APPS_UNDER_TEST+="org.opencord.olt,${oltVer.trim()},${oltOarUrl.trim()}*"
          fi
          if [ ${dhcpl2relayVer.trim()} != '' ] && [ ${dhcpl2relayOarUrl.trim()} != '' ]; then
            ONOS_APPS_UNDER_TEST+="org.opencord.dhcpl2relay,${dhcpl2relayVer.trim()},${dhcpl2relayOarUrl.trim()}*"
          fi
          if [ ${igmpproxyVer.trim()} != '' ] && [ ${igmpproxyOarUrl.trim()} != '' ]; then
            ONOS_APPS_UNDER_TEST+="org.opencord.igmpproxy,${igmpproxyVer.trim()},${igmpproxyOarUrl.trim()}*"
          fi
          if [ ${sadisVer.trim()} != '' ] && [ ${sadisOarUrl.trim()} != '' ]; then
            ONOS_APPS_UNDER_TEST+="org.opencord.sadis,${sadisVer.trim()},${sadisOarUrl.trim()}*"
          fi
          if [ ${mcastVer.trim()} != '' ] && [ ${mcastOarUrl.trim()} != '' ]; then
            ONOS_APPS_UNDER_TEST+="org.opencord.mcast,${mcastVer.trim()},${mcastOarUrl.trim()}*"
          fi
          if [ ${kafkaVer.trim()} != '' ] && [ ${kafkaOarUrl.trim()} != '' ]; then
            ONOS_APPS_UNDER_TEST+="org.opencord.kafka,${kafkaVer.trim()},${kafkaOarUrl.trim()}*"
          fi
          export ROBOT_MISC_ARGS="-d \$ROBOT_LOGS_DIR -v onos_apps_under_test:\$ONOS_APPS_UNDER_TEST -e PowerSwitch"
          export TARGET=onos-app-upgrade-test
        fi
        if [ ${name} == 'voltha-component-upgrade' ] || [ ${name} == 'voltha-component-rolling-upgrade' ]; then
          export VOLTHA_COMPS_UNDER_TEST+=''
          VOLTHA_COMPS_UNDER_TEST+="adapter-open-olt,adapter-open-olt,voltha/voltha-openolt-adapter:${openolt_adapter_test_tag}*"
          VOLTHA_COMPS_UNDER_TEST+="adapter-open-onu,adapter-open-onu,voltha/voltha-openonu-adapter-go:${openonu_adapter_test_tag}*"
          VOLTHA_COMPS_UNDER_TEST+="rw-core,voltha,voltha/voltha-rw-core:${rw_core_test_tag}*"
          VOLTHA_COMPS_UNDER_TEST+="ofagent,ofagent,voltha/voltha-ofagent-go:${ofagent_test_tag}*"
          export ROBOT_MISC_ARGS="-d \$ROBOT_LOGS_DIR -v voltha_comps_under_test:\$VOLTHA_COMPS_UNDER_TEST -e PowerSwitch"
        fi
        if [[ ${name} == 'voltha-component-upgrade' ]]; then
          export TARGET=voltha-comp-upgrade-test
        fi
        if [[ ${name} == 'voltha-component-rolling-upgrade' ]]; then
          export TARGET=voltha-comp-rolling-upgrade-test
        fi
        if [ ${name} == 'onu-software-upgrade' ] || [ ${name} == 'onu-software-upgrade-omci-extended-msg' ]; then
          export ROBOT_MISC_ARGS="-d \$ROBOT_LOGS_DIR -v image_version:${onuImageVersion.trim()} -v image_url:${onuImageUrl.trim()} -v image_vendor:${onuImageVendor.trim()} -v image_activate_on_success:${onuImageActivateOnSuccess.trim()} -v image_commit_on_success:${onuImageCommitOnSuccess.trim()} -v image_crc:${onuImageCrc.trim()} -e PowerSwitch"
          export TARGET=onu-upgrade-test
        fi
        if [[ ${name} == 'onu-image-dwl-simultaneously' ]]; then
          export ROBOT_MISC_ARGS="-d \$ROBOT_LOGS_DIR -v image_version:${onuImageVersion.trim()} -v image_url:${onuImageUrl.trim()} -v image_vendor:${onuImageVendor.trim()} -v image_activate_on_success:${onuImageActivateOnSuccess.trim()} -v image_commit_on_success:${onuImageCommitOnSuccess.trim()} -v image_crc:${onuImageCrc.trim()} -e PowerSwitch"
          export TARGET=onu-upgrade-test-multiolt-kind-att
        fi
        testLogging='False'
        if [ ${logging} = true ]; then
          testLogging='True'
        fi
        export VOLTCONFIG=$HOME/.volt/config
        export KUBECONFIG=$HOME/.kube/kind-config-voltha-minimal
        ROBOT_MISC_ARGS+=" -v ONOS_SSH_PORT:30115 -v ONOS_REST_PORT:30120 -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace} -v container_log_dir:${logsDir} -v logging:\$testLogging"
        # Run the specified tests
        make -C $WORKSPACE/voltha-system-tests \$TARGET || true
      """
            // remove port-forwarding
            sh """
        # remove orphaned port-forward from different namespaces
        ps aux | grep port-forw | grep -v grep | awk '{print \$2}' | xargs --no-run-if-empty kill -9 || true
      """
            // collect pod details
            get_pods_info("$WORKSPACE/${name}")
            sh """
        set +e
        # collect logs collected in the Robot Framework StartLogging keyword
        cd ${logsDir}
        gzip *-combined.log || true
        rm *-combined.log || true
      """
            helmTeardown(['infra', 'voltha'])
        }
    }
}

// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
void get_pods_info(dest) {
    // collect pod details, this is here in case of failure
    sh """
  mkdir -p ${dest} || true
  kubectl get pods --all-namespaces -o wide > ${dest}/pods.txt || true
  kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.image}{'\\n'}" | sort | uniq | tee ${dest}/pod-images.txt || true
  kubectl get pods --all-namespaces -o jsonpath="{range .items[*].status.containerStatuses[*]}{.imageID}{'\\n'}" | sort | uniq | tee ${dest}/pod-imagesId.txt || true
  kubectl describe pods --all-namespaces -l app.kubernetes.io/part-of=voltha > ${dest}/voltha-pods-describe.txt
  kubectl describe pods -n infra -l app=onos-classic > ${dest}/onos-pods-describe.txt
  helm ls --all-namespaces > ${dest}/helm-charts.txt
  """
    sh '''
  # copy the ONOS logs directly from the container to avoid the color codes
  printf '%s\\n' $(kubectl get pods -n infra -l app=onos-classic -o=jsonpath="{.items[*]['metadata.name']}") | xargs --no-run-if-empty -I# bash -c 'kubectl -n infra cp #:apache-karaf-4.2.14/data/log/karaf.log ''' + dest + '''/#.log' || true
  '''
    return
}

// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
pipeline {
    /* no label, executor is determined by JJB */
    agent {
        label "${params.buildNode}"
    }

    options {
        timeout(time: 220, unit: 'MINUTES')
    }

    environment {
        PATH="$PATH:$WORKSPACE/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin"
        KUBECONFIG="$HOME/.kube/kind-config-voltha-minimal"
        SSHPASS="karaf"
    }

    stages {
        stage('Download Code') {
            steps {
                getVolthaCode([
                    branch: "${branch}",
                    volthaSystemTestsChange: "${volthaSystemTestsChange}",
                    volthaHelmChartsChange: "${volthaHelmChartsChange}",
                ])
            }
        }

        // -----------------------------------------------------------------------
        // -----------------------------------------------------------------------
        stage('Install Tools')
        {
            steps
            {
                script
                {
                    String iam = getIam('Install Kind')
                    println("${iam}: ENTER")
                    installKind("$branch")   // needed early by stage(Cleanup)
                    println("${iam}: LEAVE")
                } // script
	        } // steps
        } // stage

        // -----------------------------------------------------------------------
        // -----------------------------------------------------------------------
        stage('Cleanup') {
            steps {
                // remove port-forwarding
                sh(label  : 'Remove port forwarding',
                   script : """
if [[ \$(pgrep --count 'port-forw') -gt 0 ]]; then
    pkill --uid "\$(id -u)" --echo --full 'port-forw'
fi
""")
                helmTeardown(['infra', 'voltha'])
            }
        }

        // -----------------------------------------------------------------------
        // -----------------------------------------------------------------------
        stage('Create K8s Cluster') {
            steps {
                createKubernetesCluster([nodes: 3])
            }
        }

        // -----------------------------------------------------------------------
        // -----------------------------------------------------------------------
        stage('Run Test') {
            steps {
                test_software_upgrade('onos-app-upgrade')
                test_software_upgrade('voltha-component-upgrade')
                test_software_upgrade('voltha-component-rolling-upgrade')
                test_software_upgrade('onu-software-upgrade')
                test_software_upgrade('onu-software-upgrade-omci-extended-msg')
                test_software_upgrade('onu-image-dwl-simultaneously')
            }
        }
    }

    // -----------------------------------------------------------------------
    // -----------------------------------------------------------------------
    post {
        aborted {
            get_pods_info("$WORKSPACE/failed")
        }
        failure {
            get_pods_info("$WORKSPACE/failed")
        }
        always {
            step([$class: 'RobotPublisher',
                  disableArchiveOutput: false,
                  logFileName: 'RobotLogs/*/log*.html',
                  otherFiles: '',
                  outputFileName: 'RobotLogs/*/output*.xml',
                  outputPath: '.',
                  passThreshold: 100,
                  reportFileName: 'RobotLogs/*/report*.html',
                  unstableThreshold: 0,
                  onlyCritical: true])
            archiveArtifacts artifacts: '*.log,**/*.log,**/*.gz,*.gz,*.txt,**/*.txt'
        } // always
    } // post
} // pipeline

// [EOF]