Update M-CORD POD Jenkins to deploy multi-cluster demo scenario

Change-Id: I2527c1157e26c85c724b040baf058dbdcfa65931
diff --git a/Jenkinsfile-mcord-local-build b/Jenkinsfile-mcord-local-build
index 65e4e6b..48e8320 100644
--- a/Jenkinsfile-mcord-local-build
+++ b/Jenkinsfile-mcord-local-build
@@ -20,6 +20,7 @@
                 sh returnStdout: true, script: "git clone -b ${branch} ${cordRepoUrl}/helm-repo-tools"
                 sh returnStdout: true, script: "git clone -b ${branch} ${cordRepoUrl}/helm-charts"
                 sh returnStdout: true, script: "git clone -b ${branch} ${cordRepoUrl}/${configBaseDir}"
+                //sh returnStdout: true, script: "cd ${configBaseDir}; git fetch https://gerrit.opencord.org/pod-configs refs/changes/02/12902/1 && git checkout FETCH_HEAD; cd -"
                 deployment_config = readYaml file: "${configBaseDir}/${configDeploymentDir}/${configFileName}.yaml"
             }
             stage('Clean up') {
@@ -41,20 +42,25 @@
                             return helm_deleted.toInteger() == 0
                         }
                     }
-                    timeout(5) {
-                        dir ("helm-charts") {
-                            stage("Cleanup SR-IOV CNI and SR-IOV Network Device Plugin") {
-                                sh returnStdout: true, script: """
-                                export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
-                                kubectl delete -f mcord/cni-config/05-sriov-device-plugin.yaml || true
-                                kubectl delete -f mcord/cni-config/04-sriov-device-plugin-configmap.yaml || true
-                                kubectl delete -f mcord/cni-config/03-network-definition.yaml || true
-                                kubectl delete -f mcord/cni-config/02-network-crd.yaml || true
-                                kubectl delete -f mcord/cni-config/01-cni-service-account.yaml || true
-                                """
+
+                    // In Multi-cluster scenario, we only need to install SR-IOV on data plane cluster
+                    if ( ! params.installEpcControlPlane ) {
+                        timeout(5) {
+                            dir ("helm-charts") {
+                                stage("Cleanup SR-IOV Network Device Plugin Daemonset") {
+                                    sh returnStdout: true, script: """
+                                    export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
+                                    kubectl delete -f mcord/cni-config/05-sriov-device-plugin.yaml || true
+                                    kubectl delete -f mcord/cni-config/04-sriov-device-plugin-configmap.yaml || true
+                                    kubectl delete -f mcord/cni-config/03-network-definition.yaml || true
+                                    kubectl delete -f mcord/cni-config/02-network-crd.yaml || true
+                                    kubectl delete -f mcord/cni-config/01-cni-service-account.yaml || true
+                                    """
+                                }
                             }
                         }
                     }
+
                     timeout(5) {
                         waitUntil {
                             kubectl_deleted = sh returnStdout: true, script: """
@@ -66,6 +72,7 @@
                     }
                 }
             }
+
             // OLT Software START
             if ( params.reinstallOlt ) {
                 stage('Reinstall OLT software') {
@@ -101,6 +108,7 @@
                         }
                     }
                 }
+
                 stage('Restart OLT processes') {
                     for(int i=0; i < deployment_config.olts.size(); i++) {
                         timeout(5) {
@@ -124,7 +132,8 @@
                     }
                 }
             }
-            // OLT Software END
+
+            // Start to deploy etcd clusters and M-CORD Services
             stage('Add Helm repositories') {
                 sh returnStdout: true, script: """
                 export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
@@ -152,63 +161,148 @@
                     }
                 }
             }
+            stage('Install etcd-cluster') {
+                timeout(10) {
+                    sh returnStdout: true, script: """
+                    export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
+                    helm install -f $WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.yml --version 0.8.3 -n etcd-operator stable/etcd-operator
+                    """
+                }
+                timeout(10) {
+                    waitUntil {
+                        etcd_operator_running = sh returnStdout: true, script: """
+                        export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf &&
+                        kubectl get pods | grep etcd-operator | grep -i running | grep 1/1 | wc -l
+                        """
+                        return etcd_operator_running.toInteger() == 3
+                    }
+                }
+                timeout(1) {
+                    waitUntil {
+                        etcd_operator_definition = sh returnStdout: true, script: """
+                        export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
+                        kubectl get crd -o json | jq -r '.items[].spec | select(.group=="etcd.database.coreos.com").names.kind' | grep -E 'EtcdBackup|EtcdCluster|EtcdRestore' | wc -l
+                        """
+                        return etcd_operator_definition.toInteger() == 3
+                    }
 
+                }
+                timeout(10) {
+                    sh returnStdout: true, script: """
+                    export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
+                    helm install -f $WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.yml --set etcdNodePort=32379 --set clusterSize=1 -n etcd cord/etcd-cluster
+                    """
+                }
+                timeout(10) {
+                    waitUntil {
+                        etcd_cluster_running = sh returnStdout: true, script: """
+                        export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf &&
+                        kubectl get pods | grep etcd-cluster | grep -i running | grep 1/1 | wc -l
+                        """
+                        return etcd_cluster_running.toInteger() == 1
+                    }
+                }
+            }
 
             dir ("helm-charts") {
                 stage('Install SR-IOV CNI and SR-IOV Network Device Plugin') {
                     sh returnStdout: true, script: """
                     export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
                     kubectl apply -f mcord/cni-config/02-network-crd.yaml
-                    kubectl apply -f mcord/cni-config/
-                    sleep 5
                     """
-                }
-            }
 
-            stage('Install etcd-cluster') {
-                timeout(10) {
+                    timeout(1) {
+                        waitUntil {
+                            network_crd_exist = sh returnStdout: true, script: """
+                            export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
+                            kubectl get crd -o json | jq -r '.items[].spec | select(.group=="k8s.cni.cncf.io").names.kind' | grep -E 'NetworkAttachmentDefinition' | wc -l
+                            """
+                            return network_crd_exist.toInteger() == 1
+                        }
+                    }
+
                     sh returnStdout: true, script: """
                     export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
-                    helm install -f $WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.yml --version 0.8.3 -n etcd-operator stable/etcd-operator
-                    helm install -f $WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.yml --set etcdNodePort=32379 -n etcd cord/etcd-cluster
+                    kubectl apply -f mcord/cni-config/
                     """
-                }
-                timeout(10) {
-                    waitUntil {
-                        etcd_running = sh returnStdout: true, script: """
-                        export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf &&
-                        kubectl get pods | grep etcd | grep -i running | grep 1/1 | wc -l
-                        """
-                        return etcd_running.toInteger() == 6
+
+                    // Because of SR-IOV daemonset will restart the kubelet and containerd, so we need to wait tiller up.
+                    timeout(1) {
+                        waitUntil {
+                            tillerpod_running = sh returnStdout: true, script: """
+                            export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
+                            kubectl -n kube-system get pods | grep tiller-deploy | grep Running | wc -l
+                            """
+                            return tillerpod_running.toInteger() == 1
+                        }
+                    }
+
+                    // Sometimes tillerpod is up, but it's not ready to accept deployment yet, let's wait.
+                    timeout(1) {
+                        waitUntil {
+                            helm_client_working = sh returnStatus: true, script: """
+                            export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
+                            helm ls
+                            """
+                            return helm_client_working == 0
+                        }
                     }
                 }
             }
 
-            // In current development progress, we prefered to keep this for testing eNodeB's functionality
+            // Deploy Sequence: EPC Data plane > vBBU > EPC Control plane
             stage("Install M-CORD Data Plane Services") {
-                sh returnStdout: true, script: """
+                mcord_dataplane_working = sh returnStdout: true, script: """
                 export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
                 helm install -n mcord-data-plane --namespace epc -f $WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.yml cord/mcord-data-plane
                 """
             }
 
-            if ( params.installEpcControlPlane ) {
-                stage("Install M-CORD Control Plane Services") {
-                    sh returnStdout: true, script: """
-                    export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
-                    ngic_sriov=\$(kubectl exec -n epc ngic-dp-0 ifconfig s1u-net | grep 'inet addr' | cut -d: -f2 | awk '{print \$1}')
-                    helm install -n mcord-control-plane --namespace epc -f $WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.yml cord/mcord-control-plane --set spgwu_s1u_ip=$ngic_sriov
-                    """
-                }
-            }
-
             stage("Install M-CORD BBU Services") {
-                sh returnStdout: true, script: """
+                mcord_bbu_working = sh returnStdout: true, script: """
                 export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
                 helm install -n mcord-bbu --namespace epc -f $WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.yml cord/mcord-bbu
                 """
             }
 
+            // if ( params.installEpcControlPlane ) {
+            //     timeout(2) {
+            //         waitUntil {
+            //             mcord_bbu_running = sh returnStdout: true, script: """
+            //             export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf &&
+            //             kubectl -n epc get pods | grep accelleran | grep -i running | grep 3/3 | wc -l
+            //             """
+            //             return mcord_bbu_running.toInteger() == 1
+            //         }
+            //     }
+            //     stage("Install M-CORD Control Plane Services") {
+            //         mcord_controlplane_working = sh returnStdout: true, script: """
+            //         export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
+            //         ngic_s1u_net_ip=\$(kubectl exec -n epc ngic-dp-0 ifconfig s1u-net | grep 'inet addr' | cut -d: -f2 | awk '{print \$1}')
+            //         bbu_eth0=\$(kubectl -n epc get pods accelleran-0 --template={{.status.podIP}})
+            //         helm install -n mcord-control-plane --namespace epc -f $WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.yml cord/mcord-control-plane --set accelleran_hostname=\$bbu_eth0 --set spgwu_s1u_ip=\$ngic_s1u_net_ip
+            //         """
+            //     }
+            // }
+
+            stage("Install CDN Local Services") {
+                timeout(2) {
+                    waitUntil {
+                        ngic_dataplane_running = sh returnStdout: true, script: """
+                        export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf &&
+                        kubectl -n epc get pods | grep ngic-dp | grep -i running | grep 1/1 | wc -l
+                        """
+                        return ngic_dataplane_running.toInteger() == 1
+                    }
+                }
+
+                mcord_local_cdn_working =  sh returnStdout: true, script: """
+                export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
+                ngic_sgi_net_ip=\$(kubectl exec -n epc ngic-dp-0 ifconfig sgi-net | grep 'inet addr' | cut -d: -f2 | awk '{print \$1}')
+                helm install -n cdn-local --set remote_ip="10.90.0.152" --set spgwu_sgiip=\$ngic_sgi_net_ip --namespace epc cord/mcord-cdn-local
+                """
+            }
+
             stage('Install CORD Kafka') {
                 timeout(10) {
                     sh returnStdout: true, script: """
@@ -255,12 +349,32 @@
                     waitUntil {
                         onos_completed = sh returnStdout: true, script: """
                         export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf &&
-                        kubectl get pods | grep -i onos | grep -i running | grep 2/2 | wc -l
+                        kubectl get pods | grep -i onos | grep -i running | grep 1/1 | wc -l
                         """
                         return onos_completed.toInteger() == 1
                     }
                 }
             }
+
+            stage('Install ONOS progRAN') {
+                timeout(10) {
+                    sh returnStdout: true, script: """
+                    export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
+                    echo \$(pwd)
+                    helm install -n onos-progran -f $WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.yml cord/onos-progran
+                    """
+                }
+                timeout(10) {
+                    waitUntil {
+                        onos_progran_completed = sh returnStdout: true, script: """
+                        export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf &&
+                        kubectl get pods | grep -i onos-progran | grep -i running | grep 1/1 | wc -l
+                        """
+                        return onos_progran_completed.toInteger() == 1
+                    }
+                }
+            }
+
             stage('Install xos-core') {
                 timeout(10) {
                     sh returnStdout: true, script: """
@@ -418,6 +532,7 @@
             }
             currentBuild.result = 'SUCCESS'
         } catch (err) {
+            println err.message
             currentBuild.result = 'FAILURE'
             step([$class: 'Mailer', notifyEveryUnstableBuild: true, recipients: "${notificationEmail}", sendToIndividuals: false])
         }
diff --git a/Jenkinsfile-mcord-remote-build b/Jenkinsfile-mcord-remote-build
index 7582e76..ef37d1e 100644
--- a/Jenkinsfile-mcord-remote-build
+++ b/Jenkinsfile-mcord-remote-build
@@ -18,6 +18,7 @@
             stage ("Parse deployment configuration file") {
                 sh returnStdout: true, script: "rm -rf helm-charts helm-repo-tools ${configBaseDir}"
                 sh returnStdout: true, script: "git clone -b ${branch} ${cordRepoUrl}/helm-repo-tools"
+                sh returnStdout: true, script: "git clone -b ${branch} ${cordRepoUrl}/helm-charts"
                 sh returnStdout: true, script: "git clone -b ${branch} ${cordRepoUrl}/${configBaseDir}"
                 deployment_config = readYaml file: "${configBaseDir}/${configDeploymentDir}/${configFileName}.yaml"
             }
@@ -79,22 +80,42 @@
                 }
             }
 
+            
+
             stage("Install M-CORD Control Plane Services") {
                 sh returnStdout: true, script: """
                 export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
-                helm install -n mcord-control-plane --namespace epc -f $WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.yml cord/mcord-control-plane
+                helm install -n mcord-control-plane --namespace epc -f $WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.yml --set spgwu_s1u_ip=${dataplane_s1u_ip} --set accelleran_hostname=${accelleran_bbu_ip} cord/mcord-control-plane
                 """
 
                 timeout(20) {
                     control_plane = sh returnStdout: true, script: """
                     export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
-                    kubectl -n epc get pods | grep Running | grep 1/1 | wc -l
+                    kubectl -n epc get pods | grep -i running | grep 1/1 | wc -l
                     """
                     // We have 4 pods: cassandra, hss, mme and ngic-cp
                     return control_plane.toInteger() == 4
                 }
             }
 
+            // TODO: Use --namespace wowz to create the namespace
+            stage("Install CDN Remote Services") {
+                sh returnStdout: true, script: """
+                export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf
+                helm install -n cdn-remote cord/mcord-cdn-remote
+                """
+
+                timeout(3) {
+                    waitUntil {
+                        mcord_remote_cdn_working = sh returnStdout: true, script: """
+                        export KUBECONFIG=$WORKSPACE/${configBaseDir}/${configKubernetesDir}/${configFileName}.conf &&
+                        kubectl -n wowz get pods -o wide | grep -i running | grep 1/1 | wc -l
+                        """
+                        return mcord_remote_cdn_working.toInteger() == 1
+                    }
+                }
+            }
+
             currentBuild.result = 'SUCCESS'
         } catch (err) {
             currentBuild.result = 'FAILURE'