[AETHER]-615 Update the Jenkins job

1. Replace the Rancher CLI with Terraform in TOST related jobs.
2. Still the Rancher CLI in FluentBit but will fix it soon
3. Add new jenkins job to install shared resources

Change-Id: I6e4d5ecf696e7a17a1d954df19093005bf14bf8c
diff --git a/jjb/pipeline/tost-deploy-fluentbit.groovy b/jjb/pipeline/tost-deploy-fluentbit.groovy
index 17dbf04..2925236 100644
--- a/jjb/pipeline/tost-deploy-fluentbit.groovy
+++ b/jjb/pipeline/tost-deploy-fluentbit.groovy
@@ -7,8 +7,7 @@
     }
     environment {
         KUBECONFIG = credentials("${params.k8s_config}")
-        git_password = credentials("${params.git_password_env}")
-        rancher_token = credentials("${params.rancher_api_env}")
+        rancher_token = credentials("${params.rancher_cli_env}")
     }
     stages {
         stage('Install tools') {
@@ -36,23 +35,33 @@
                 '''
             }
         }
-        stage('Clone Config Repo') {
-            options {
-                timeout(time: 10, unit: "SECONDS")
-            }
-            steps {
-                sh '''
-                git clone https://${git_user}:${git_password}@${git_server}/${git_repo}
+    stage('Init git') {
+        steps {
+                withCredentials([sshUserPrivateKey(credentialsId: "aether_jenkins", keyFileVariable: 'keyfile')]) {
+
+                sh """#!/bin/bash
+                set -x
+                mkdir -p ~/.ssh
+                ssh-keyscan -t rsa -p 29418 ${git_server} >> ~/.ssh/known_hosts
+cat <<EOF > ~/.ssh/config
+Host ${git_server}
+  User ${git_user}
+  Hostname ${git_server}
+  Port 29418
+  IdentityFile ${keyfile}
+EOF
+
+                git clone "ssh://${git_server}:29418/${git_repo}"
                 if [ ! -z ${config_review} ] && [ ! -z ${config_patchset} ]; then
-                    cd ${git_repo}
-                    CFG_LAST2=$(echo ${config_review} | tail -c 3)
-                    git fetch "https://${git_user}:${git_password}@${git_server}/a/${git_repo}" refs/changes/${CFG_LAST2}/${config_review}/${config_patchset} && git checkout FETCH_HEAD
-                    git checkout FETCH_HEAD
-                    cd ..
+                    CFG_LAST2=\$(echo ${config_review} | tail -c 3)
+                    git fetch "ssh://@${git_server}:29418/${git_repo}" refs/changes/\${CFG_LAST2}/${config_review}/${config_patchset} && git checkout FETCH_HEAD
                 fi
-                '''
-             }
+
+                """
+                }
+            }
         }
+
         stage('Login Rancher') {
             steps {
                 sh '''
@@ -78,7 +87,7 @@
             }
             steps {
                 sh '''
-                cd ${git_repo}/deployment-configs/aether/apps/${config_env}/
+                cd ${workspace}/${git_repo}/deployment-configs/aether/apps/${config_env}/
                 until rancher apps install --answers fluentbit-ans.yml --namespace ${fluentbit_ns} cattle-global-data:fluent-fluent-bit fluentbit; do :; done
                 apps=$(rancher apps -q | grep fluentbit)
                 for app in $apps; do until rancher wait $app --timeout 20; do :; done; rancher apps ls; done
@@ -89,6 +98,9 @@
     }
     post {
         always {
+            sh """
+            rm -rf ${workspace}/${git_repo}
+            """
             cleanWs()
       }
     }
diff --git a/jjb/pipeline/tost-deploy-onos.groovy b/jjb/pipeline/tost-deploy-onos.groovy
index d2c062a..24a7914 100644
--- a/jjb/pipeline/tost-deploy-onos.groovy
+++ b/jjb/pipeline/tost-deploy-onos.groovy
@@ -7,80 +7,71 @@
     }
     environment {
         KUBECONFIG = credentials("${params.k8s_config}")
-        registry_password = credentials("${params.registry_password_env}")
-        git_password = credentials("${params.git_password_env}")
         onos_password = credentials("${params.onos_password}")
-        rancher_token = credentials("${params.rancher_api_env}")
+        git_password = credentials("${params.git_password_env}")
+        gcp = credentials("${params.gcp_credential}")
+        rancher_dev = credentials("${params.rancher_api_env}")
     }
     stages {
         stage('Install tools') {
             steps {
-                sh '''
+                sh """
                 set -x
                 apt-get update -y
-                apt-get install -y curl wget jq git
+                apt-get install -y curl wget jq git unzip
+
+                # Install yq
+                wget https://github.com/mikefarah/yq/releases/download/3.4.0/yq_linux_amd64 -O /usr/bin/yq &&\
+                chmod +x /usr/bin/yq
+                yq --help
 
                 # Install kubectl
                 curl -LO "https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/linux/amd64/kubectl"
                 chmod +x ./kubectl
                 mv ./kubectl /usr/local/bin/kubectl
 
-                # Install rancher
-                wget https://github.com/rancher/cli/releases/download/v2.4.5/rancher-linux-amd64-v2.4.5.tar.gz
-                tar -xvf rancher-linux-amd64-v2.4.5.tar.gz
-                mv rancher-v2.4.5/rancher /usr/local/bin
-
-                rm rancher-linux-amd64-v2.4.5.tar.gz
-                rm -rf rancher-v2.4.5
-
-                # Test Kubectl & Rancher
-                KUBE_CONFIG=$KUBECONFIG kubectl get nodes
-                rancher --version
-                '''
+                # Install terraform
+                wget https://releases.hashicorp.com/terraform/0.13.2/terraform_0.13.2_linux_amd64.zip
+                unzip terraform_0.13.2_linux_amd64.zip
+                mv terraform /usr/local/bin
+                terraform version
+                """
             }
         }
-        stage('Clone Config Repo') {
-            options {
-                timeout(time: 10, unit: "SECONDS")
-            }
-            steps {
-                sh '''
-                git clone https://${git_user}:${git_password}@${git_server}/${git_repo}
+    stage('Init Terraform') {
+        steps {
+                withCredentials([sshUserPrivateKey(credentialsId: "aether_jenkins", keyFileVariable: 'keyfile')]) {
+
+                sh """#!/bin/bash
+                set -x
+                mkdir -p ~/.ssh
+                ssh-keyscan -t rsa -p 29418 ${git_server} >> ~/.ssh/known_hosts
+cat <<EOF > ~/.ssh/config
+Host ${git_server}
+  User ${git_user}
+  Hostname ${git_server}
+  Port 29418
+  IdentityFile ${keyfile}
+EOF
+
+                git clone "ssh://${git_server}:29418/${git_repo}"
+                cd ${workspace}/${git_repo}/${terraform_dir}/tost/onos
                 if [ ! -z ${config_review} ] && [ ! -z ${config_patchset} ]; then
-                    cd ${git_repo}
-                    CFG_LAST2=$(echo ${config_review} | tail -c 3)
-                    git fetch "https://${git_user}:${git_password}@${git_server}/a/${git_repo}" refs/changes/${CFG_LAST2}/${config_review}/${config_patchset} && git checkout FETCH_HEAD
-                    git checkout FETCH_HEAD
-                    echo "config.review: ${config_review}" >> deployment-configs/aether/apps/${config_env}/onos-ans.yml
-                    echo "config.patchset: ${config_patchset}" >> deployment-configs/aether/apps/${config_env}/onos-ans.yml
-                    cd ..
+                    CFG_LAST2=\$(echo ${config_review} | tail -c 3)
+                    git fetch "ssh://${git_server}:29418/${git_repo}" refs/changes/\${CFG_LAST2}/${config_review}/${config_patchset} && git checkout FETCH_HEAD
+                    cp onos.yaml tmp.yaml
+
+cat <<EOF >> config.yaml
+config:
+  review: ${config_review}
+  patchset: ${config_patchset}
+EOF
+
+                    yq merge tmp.yaml config.yaml > onos.yaml
                 fi
-                '''
-             }
-        }
-        stage('Login Rancher') {
-            steps {
-                sh '''
-                rancher login ${rancher_server} --token ${rancher_token} --context ${rancher_context}:${rancher_project}
-                '''
-             }
-        }
-        stage('Push Secrets') {
-            steps {
-                sh '''
-
-                rancher namespaces ls | grep ${onos_ns} || rancher namespaces create ${onos_ns}
-
-                kubectl -n ${onos_ns} delete secret git-secret --ignore-not-found=true
-                kubectl -n ${onos_ns} create secret generic git-secret --from-literal=username=${git_user} --from-literal=password=${git_password}
-                kubectl -n ${onos_ns} delete secret aether-registry-credential --ignore-not-found=true
-                kubectl -n ${onos_ns} create secret docker-registry aether-registry-credential  --docker-server=${registry_server} --docker-username=${registry_user} --docker-password=${registry_password}
-
-
-
-                kubectl -n ${onos_ns} delete secret onos-secret --ignore-not-found=true
-                kubectl -n ${onos_ns} create secret generic onos-secret --from-literal=username=${onos_user} --from-literal=password=${onos_password}
-                '''
+                GOOGLE_BACKEND_CREDENTIALS=${gcp} terraform init
+                """
+                }
             }
         }
 
@@ -89,22 +80,24 @@
                 timeout(time: 90, unit: "SECONDS")
             }
             steps {
-                sh '''
-                for app in $(rancher apps ls -q | grep -E '(onos-tost)'); do rancher apps delete $app; done
-
-                until [ "$(rancher apps ls -q | grep -E '(onos-tost)')" = "" ]; do echo "wait deleted apps"; rancher apps ls ; sleep 1; done
-                '''
+                sh """
+                cd ${workspace}/${git_repo}/${terraform_dir}/tost/onos
+                GOOGLE_BACKEND_CREDENTIALS=${gcp} terraform destroy -var-file=${rancher_dev} -var 'cluster_name=${rancher_cluster}' -var 'project_name=tost' -var-file=app_map.tfvars -auto-approve
+                """
              }
         }
-        stage('Remove PVC') {
+        stage('Remove resources') {
             options {
                 timeout(time: 300, unit: "SECONDS")
             }
             steps {
-                sh '''
-                pvcs=$(kubectl -n onos-tost get pvc -lapp=onos-tost-atomix -o name)
-                for pv in $pvcs; do kubectl -n onos-tost delete $pv; done
-                '''
+                sh """
+                pvcs=\$(kubectl -n onos-tost get pvc -lapp=onos-tost-atomix -o name)
+                for pv in \${pvcs}; do kubectl -n onos-tost delete \${pv}; done
+
+                kubectl -n ${onos_ns} delete secret onos-git-secret || true
+                kubectl -n ${onos_ns} delete secret onos-secret || true
+                """
              }
         }
         stage('Install apps') {
@@ -112,19 +105,29 @@
                 timeout(time: 600, unit: "SECONDS")
             }
             steps {
-                sh '''
-                cd ${git_repo}/deployment-configs/aether/apps/${config_env}/
-
-                until rancher apps install --answers onos-ans.yml --namespace ${onos_ns} cattle-global-data:${onos_catalog_name}-onos-tost onos-tost; do :; done
-                apps=$(rancher apps -q | grep onos-tost)
-                for app in $apps; do until rancher wait $app --timeout 20; do :; done; rancher apps ls; done
-                '''
+                sh """
+                cd ${workspace}/${git_repo}/${terraform_dir}/tost/onos
+                GOOGLE_BACKEND_CREDENTIALS=${gcp} terraform apply -var-file=${rancher_dev} -var 'cluster_name=${rancher_cluster}' -var 'project_name=tost' -var-file=app_map.tfvars -auto-approve
+                """
              }
         }
+        stage('Push Secrets') {
+            steps {
+                sh """
+
+                kubectl -n ${onos_ns} create secret generic onos-git-secret --from-literal=username=${git_user} --from-literal=password=${git_password}
+                kubectl -n ${onos_ns} create secret generic onos-secret --from-literal=username=${onos_user} --from-literal=password=${onos_password}
+                """
+            }
+        }
 
     }
     post {
         always {
+
+            sh """
+            rm -rf ${workspace}/${git_repo}
+            """
             cleanWs()
         }
     }
diff --git a/jjb/pipeline/tost-deploy-shared.groovy b/jjb/pipeline/tost-deploy-shared.groovy
new file mode 100644
index 0000000..2869696
--- /dev/null
+++ b/jjb/pipeline/tost-deploy-shared.groovy
@@ -0,0 +1,43 @@
+pipeline {
+    agent {
+        docker {
+            image 'ubuntu:18.04'
+            args '-u root:sudo'
+        }
+    }
+    environment {
+        KUBECONFIG = credentials("${params.k8s_config}")
+        registry_password = credentials("${params.registry_password_env}")
+    }
+    stages {
+        stage('Install tools') {
+            steps {
+                sh '''
+                set -x
+                apt-get update -y
+                apt-get install -y curl
+
+                # Install kubectl
+                curl -LO "https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/linux/amd64/kubectl"
+                chmod +x ./kubectl
+                mv ./kubectl /usr/local/bin/kubectl
+
+                # Test Kubectl & Rancher
+                kubectl get nodes
+                '''
+            }
+        }
+        stage('Perform Terraform') {
+             steps {
+                sh """
+                kubectl -n ${target_namespace} create secret docker-registry aether-registry-credential  --docker-server=${registry_server} --docker-username=${registry_user} --docker-password=${registry_password}
+                """
+             }
+        }
+    }
+    post {
+        always {
+            cleanWs()
+        }
+    }
+}
diff --git a/jjb/pipeline/tost-deploy-stratum.groovy b/jjb/pipeline/tost-deploy-stratum.groovy
index 24a777b..c662f37 100644
--- a/jjb/pipeline/tost-deploy-stratum.groovy
+++ b/jjb/pipeline/tost-deploy-stratum.groovy
@@ -7,89 +7,82 @@
     }
     environment {
         KUBECONFIG = credentials("${params.k8s_config}")
-        registry_password = credentials("${params.registry_password_env}")
+        gcp = credentials("${params.gcp_credential}")
         git_password = credentials("${params.git_password_env}")
-        rancher_token = credentials("${params.rancher_api_env}")
+        rancher_dev = credentials("${params.rancher_api_env}")
     }
     stages {
         stage('Install tools') {
             steps {
-                sh '''
+                sh """
                 set -x
                 apt-get update -y
-                apt-get install -y curl wget jq git
+                apt-get install -y curl wget jq git unzip
+
+                # Install yq
+                wget https://github.com/mikefarah/yq/releases/download/3.4.0/yq_linux_amd64 -O /usr/bin/yq &&\
+                chmod +x /usr/bin/yq
+                yq --help
 
                 # Install kubectl
                 curl -LO "https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/linux/amd64/kubectl"
                 chmod +x ./kubectl
                 mv ./kubectl /usr/local/bin/kubectl
 
-                # Install rancher
-                wget https://github.com/rancher/cli/releases/download/v2.4.5/rancher-linux-amd64-v2.4.5.tar.gz
-                tar -xvf rancher-linux-amd64-v2.4.5.tar.gz
-                mv rancher-v2.4.5/rancher /usr/local/bin
-
-                rm rancher-linux-amd64-v2.4.5.tar.gz
-                rm -rf rancher-v2.4.5
-                # Test Kubectl & Rancher
-                KUBE_CONFIG=$KUBECONFIG kubectl get nodes
-                rancher --version
-                '''
+                # Install terraform
+                wget https://releases.hashicorp.com/terraform/0.13.2/terraform_0.13.2_linux_amd64.zip
+                unzip terraform_0.13.2_linux_amd64.zip
+                mv terraform /usr/local/bin
+                terraform version
+                """
             }
         }
-        stage('Clone Config Repo') {
-            options {
-                timeout(time: 10, unit: "SECONDS")
-            }
-            steps {
-                sh '''
-                git clone https://${git_user}:${git_password}@${git_server}/${git_repo}
+    stage('Init Terraform') {
+        steps {
+                withCredentials([sshUserPrivateKey(credentialsId: "aether_jenkins", keyFileVariable: 'keyfile')]) {
+
+                sh """#!/bin/bash
+                set -x
+                mkdir -p ~/.ssh
+                ssh-keyscan -t rsa -p 29418 ${git_server} >> ~/.ssh/known_hosts
+cat <<EOF >> ~/.ssh/config
+Host ${git_server}
+  User ${git_user}
+  Hostname ${git_server}
+  Port 29418
+  IdentityFile ${keyfile}
+EOF
+
+                git clone "ssh://${git_server}:29418/${git_repo}"
+                cd ${workspace}/${git_repo}/${terraform_dir}/tost/stratum
                 if [ ! -z ${config_review} ] && [ ! -z ${config_patchset} ]; then
-                    cd ${git_repo}
-                    CFG_LAST2=$(echo ${config_review} | tail -c 3)
-                    git fetch "https://${git_user}:${git_password}@${git_server}/a/${git_repo}" refs/changes/${CFG_LAST2}/${config_review}/${config_patchset} && git checkout FETCH_HEAD
-                    git checkout FETCH_HEAD
-                    echo "config.review: ${config_review}" >> deployment-configs/aether/apps/${config_env}/stratum-ans.yml
-                    echo "config.patchset: ${config_patchset}" >> deployment-configs/aether/apps/${config_env}/stratum-ans.yml
-                    cd ..
+                    CFG_LAST2=\$(echo ${config_review} | tail -c 3)
+                    git fetch "ssh://${git_server}:29418/${git_repo}" refs/changes/\${CFG_LAST2}/${config_review}/${config_patchset} && git checkout FETCH_HEAD
+                    cp stratum.yaml tmp.yaml
+
+cat <<EOF >> config.yaml
+config:
+  review: ${config_review}
+  patchset: ${config_patchset}
+EOF
+
+                    yq merge tmp.yaml config.yaml > stratum.yaml
                 fi
-
-                '''
-             }
-        }
-        stage('Login Rancher') {
-            steps {
-                sh '''
-                rancher login ${rancher_server} --token ${rancher_token} --context ${rancher_context}:${rancher_project}
-                '''
-             }
-        }
-        stage('Push Secrets') {
-            steps {
-                sh '''
-
-                rancher namespaces ls | grep ${stratum_ns} || rancher namespaces create ${stratum_ns}
-
-                kubectl -n ${stratum_ns} delete secret git-secret --ignore-not-found=true
-                kubectl -n ${stratum_ns} create secret generic git-secret --from-literal=username=${git_user} --from-literal=password=${git_password}
-                kubectl -n ${stratum_ns} delete secret aether-registry-credential --ignore-not-found=true
-                kubectl -n ${stratum_ns} create secret docker-registry aether-registry-credential  --docker-server=${registry_server} --docker-username=${registry_user} --docker-password=${registry_password}
-
-
-                '''
+                GOOGLE_BACKEND_CREDENTIALS=${gcp} terraform init
+                """
+                }
             }
         }
-
         stage('Uninstall Apps') {
             options {
                 timeout(time: 90, unit: "SECONDS")
             }
             steps {
-                sh '''
-                for app in $(rancher apps ls -q | grep -E '(stratum)'); do rancher apps delete $app; done
-
-                until [ "$(rancher apps ls -q | grep -E '(stratum)')" = "" ]; do echo "wait deleted apps"; rancher apps ls ; sleep 1; done
-                '''
+                sh """
+                cd ${workspace}/${git_repo}/${terraform_dir}/tost/stratum
+                GOOGLE_BACKEND_CREDENTIALS=${gcp} terraform destroy -var-file=${rancher_dev} -var 'cluster_name=${rancher_cluster}' -var 'project_name=tost' -var-file=app_map.tfvars -auto-approve
+                kubectl -n ${stratum_ns} delete secret stratum-git-secret || true
+                """
              }
         }
         stage('Install apps') {
@@ -97,19 +90,29 @@
                 timeout(time: 600, unit: "SECONDS")
             }
             steps {
-                sh '''
-                cd ${git_repo}/deployment-configs/aether/apps/${config_env}/
-                until rancher apps install --answers stratum-ans.yml --namespace ${stratum_ns} cattle-global-data:${stratum_catalog_name}-stratum stratum; do :; done
-
-                apps=$(rancher apps -q | grep stratum)
-                for app in $apps; do until rancher wait $app --timeout 20; do :; done; rancher apps ls; done
-                '''
+                sh """
+                cd ${workspace}/${git_repo}/${terraform_dir}/tost/stratum
+                GOOGLE_BACKEND_CREDENTIALS=${gcp} terraform apply -var-file=${rancher_dev} -var 'cluster_name=${rancher_cluster}' -var 'project_name=tost' -var-file=app_map.tfvars -auto-approve
+                """
              }
         }
+        stage('Push Secrets') {
+            steps {
+                sh """
+
+                kubectl -n ${stratum_ns} create secret generic stratum-git-secret --from-literal=username=${git_user} --from-literal=password=${git_password}
+
+                """
+            }
+        }
 
     }
     post {
         always {
+
+            sh """
+            rm -rf ${workspace}/${git_repo}
+            """
             cleanWs()
         }
     }
diff --git a/jjb/pipeline/tost-deploy-telegraf.groovy b/jjb/pipeline/tost-deploy-telegraf.groovy
index aa5daa1..ac61496 100644
--- a/jjb/pipeline/tost-deploy-telegraf.groovy
+++ b/jjb/pipeline/tost-deploy-telegraf.groovy
@@ -7,69 +7,64 @@
     }
     environment {
         KUBECONFIG = credentials("${params.k8s_config}")
-        git_password = credentials("${params.git_password_env}")
-        rancher_token = credentials("${params.rancher_api_env}")
+        gcp = credentials("${params.gcp_credential}")
+        rancher_dev = credentials("${params.rancher_api_env}")
     }
     stages {
         stage('Install tools') {
             steps {
-                sh '''
+                sh """
                 set -x
                 apt-get update -y
-                apt-get install -y curl wget jq git
+                apt-get install -y curl wget jq git unzip
 
                 # Install kubectl
                 curl -LO "https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/linux/amd64/kubectl"
                 chmod +x ./kubectl
                 mv ./kubectl /usr/local/bin/kubectl
 
-                # Install rancher
-                wget https://github.com/rancher/cli/releases/download/v2.4.5/rancher-linux-amd64-v2.4.5.tar.gz
-                tar -xvf rancher-linux-amd64-v2.4.5.tar.gz
-                mv rancher-v2.4.5/rancher /usr/local/bin
-
-                rm rancher-linux-amd64-v2.4.5.tar.gz
-                rm -rf rancher-v2.4.5
-                # Test Kubectl & Rancher
-                KUBE_CONFIG=$KUBECONFIG kubectl get nodes
-                rancher --version
-                '''
+                wget https://releases.hashicorp.com/terraform/0.13.2/terraform_0.13.2_linux_amd64.zip
+                unzip terraform_0.13.2_linux_amd64.zip
+                mv terraform /usr/local/bin
+                terraform version
+                """
             }
         }
-        stage('Clone Config Repo') {
-            options {
-                timeout(time: 10, unit: "SECONDS")
-            }
+        stage('Init Terraform') {
             steps {
-                sh '''
-                git clone https://${git_user}:${git_password}@${git_server}/${git_repo}
+                withCredentials([sshUserPrivateKey(credentialsId: "aether_jenkins", keyFileVariable: 'keyfile')]) {
+                sh """#!/bin/bash
+                set -x
+                mkdir -p ~/.ssh
+                ssh-keyscan -t rsa -p 29418 ${git_server} >> ~/.ssh/known_hosts
+cat <<EOF > ~/.ssh/config
+Host ${git_server}
+  User ${git_user}
+  Hostname ${git_server}
+  Port 29418
+  IdentityFile ${keyfile}
+EOF
+                git clone "ssh://${git_server}:29418/${git_repo}"
+                cd ${workspace}/${git_repo}/${terraform_dir}/tost/telegraf
                 if [ ! -z ${config_review} ] && [ ! -z ${config_patchset} ]; then
-                    cd ${git_repo}
-                    CFG_LAST2=$(echo ${config_review} | tail -c 3)
-                    git fetch "https://${git_user}:${git_password}@${git_server}/a/${git_repo}" refs/changes/${CFG_LAST2}/${config_review}/${config_patchset} && git checkout FETCH_HEAD
+                    CFG_LAST2=\$(echo ${config_review} | tail -c 3)
+                    git fetch "ssh://@${git_server}:29418/${git_repo}" refs/changes/\${CFG_LAST2}/${config_review}/${config_patchset} && git checkout FETCH_HEAD
                     git checkout FETCH_HEAD
-                    cd ..
                 fi
-                '''
-             }
-        }
-        stage('Login Rancher') {
-            steps {
-                sh '''
-                rancher login ${rancher_server} --token ${rancher_token} --context ${rancher_context}:${rancher_project}
-                '''
-             }
+                GOOGLE_BACKEND_CREDENTIALS=${gcp} terraform init
+                """
+                }
+            }
         }
         stage('Uninstall Apps') {
             options {
                 timeout(time: 90, unit: "SECONDS")
             }
             steps {
-                sh '''
-                for app in $(rancher apps ls -q | grep -E '(telegraf)'); do rancher apps delete $app; done
-
-                until [ "$(rancher apps ls -q | grep -E '(telegraf)')" = "" ]; do echo "wait deleted apps"; rancher apps ls ; sleep 1; done
-                '''
+                sh """
+                cd ${workspace}/${git_repo}/${terraform_dir}/tost/telegraf
+                GOOGLE_BACKEND_CREDENTIALS=${gcp} terraform destroy -var-file=${rancher_dev} -var 'cluster_name=${rancher_cluster}' -var 'project_name=tost' -var-file=app_map.tfvars -auto-approve
+                """
              }
         }
        stage('Install apps') {
@@ -77,18 +72,19 @@
                 timeout(time: 600, unit: "SECONDS")
             }
             steps {
-                sh '''
-                cd ${git_repo}/deployment-configs/aether/apps/${config_env}/
-                until rancher apps install --answers telegraf-ans.yml --namespace ${telegraf_ns} cattle-global-data:influxdata-telegraf telegraf; do :; done
-                apps=$(rancher apps -q | grep telegraf)
-                for app in $apps; do until rancher wait $app --timeout 20; do :; done; rancher apps ls; done
-                '''
+                sh """
+                cd ${workspace}/${git_repo}/${terraform_dir}/tost/telegraf
+                GOOGLE_BACKEND_CREDENTIALS=${gcp} terraform apply -var-file=${rancher_dev} -var 'cluster_name=${rancher_cluster}'  -var 'project_name=tost'  -var-file=app_map.tfvars -auto-approve
+                """
              }
         }
 
     }
     post {
         always {
+            sh """
+            rm -rf ${workspace}/${git_repo}
+            """
             cleanWs()
       }
     }
diff --git a/jjb/pipeline/tost-deploy.groovy b/jjb/pipeline/tost-deploy.groovy
index 42961ff..2eba5ad 100644
--- a/jjb/pipeline/tost-deploy.groovy
+++ b/jjb/pipeline/tost-deploy.groovy
@@ -7,6 +7,8 @@
     }
     environment {
         KUBECONFIG = credentials("${params.k8s_config}")
+        gcp = credentials("${params.gcp_credential}")
+        rancher_dev = credentials("${params.rancher_api_env}")
     }
     stages {
         stage('Install tools') {
@@ -14,7 +16,7 @@
                 sh '''
                 set -x
                 apt-get update -y
-                apt-get install -y curl wget jq git
+                apt-get install -y curl wget jq git unzip
 
                 # Install kubectl
                 curl -LO "https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/linux/amd64/kubectl"
@@ -23,9 +25,61 @@
 
                 # Test Kubectl & Rancher
                 kubectl get nodes
+
+                # Install
+                wget https://releases.hashicorp.com/terraform/0.13.2/terraform_0.13.2_linux_amd64.zip
+                unzip terraform_0.13.2_linux_amd64.zip
+                mv terraform /usr/local/bin
+                terraform version
                 '''
             }
         }
+
+        stage('Init Terraform') {
+            steps {
+                withCredentials([sshUserPrivateKey(credentialsId: "aether_jenkins", keyFileVariable: 'keyfile')]) {
+
+                sh """#!/bin/bash
+                set -x
+                mkdir -p ~/.ssh
+                ssh-keyscan -t rsa -p 29418 ${git_server} >> ~/.ssh/known_hosts
+cat <<EOF > ~/.ssh/config
+Host ${git_server}
+  User ${git_user}
+  Hostname ${git_server}
+  Port 29418
+  IdentityFile ${keyfile}
+EOF
+
+                git clone "ssh://${git_server}:29418/${git_repo}"
+                cd ${workspace}/${git_repo}/${terraform_dir}/tost
+                if [ ! -z ${config_review} ] && [ ! -z ${config_patchset} ]; then
+                    CFG_LAST2=\$(echo ${config_review} | tail -c 3)
+                    git fetch "ssh://${git_server}:29418/${git_repo}" refs/changes/\${CFG_LAST2}/${config_review}/${config_patchset} && git checkout FETCH_HEAD
+                fi
+                GOOGLE_BACKEND_CREDENTIALS=${gcp} terraform init
+
+                """
+                }
+            }
+        }
+        stage('Perform Terraform') {
+             steps {
+                sh """
+                cd ${workspace}/${git_repo}/${terraform_dir}/tost
+                GOOGLE_BACKEND_CREDENTIALS=${gcp} terraform destroy -var-file=${rancher_dev} -var 'cluster_name=${rancher_cluster}' -var-file=app_map.tfvars -auto-approve
+                GOOGLE_BACKEND_CREDENTIALS=${gcp} terraform apply -var-file=${rancher_dev} -var 'cluster_name=${rancher_cluster}' -var-file=app_map.tfvars -auto-approve
+
+                """
+             }
+        }
+
+        stage('Install shared resources') {
+            steps {
+                sh(script: "date -u")
+                    build(job: "${params.job_name}-shared")
+            }
+        }
         stage('Parallel Stage') {
             parallel {
                 stage('onos') {
@@ -48,7 +102,7 @@
                 }
             }
         }
-        stage('E2E Testing') {
+    stage('E2E Testing') {
             options {
                 timeout(time: 120, unit: "SECONDS")
             }
@@ -101,6 +155,7 @@
             if [ ! -z "${params.target_server}" ]; then
                 kubectl delete -f test.yaml
             fi
+            rm -rf ${workspace}/${git_repo}
             """
             cleanWs()
         }
diff --git a/jjb/tost.yaml b/jjb/tost.yaml
index cf2947c..76125a0 100644
--- a/jjb/tost.yaml
+++ b/jjb/tost.yaml
@@ -1,294 +1,373 @@
----
-# TOST deployment tasks
+- project:
+    name: deploy-menlo-tost-dev
+    rancher_cluster: "tost-dev"
+    k8s_config: "tost-dev-k8s"
+    config_env: "menlo-tost-dev"
+    terraform_dir: "developing/ace-menlo"
+    #remove after migrating to terraform
+    rancher_context: "c-lggdr"
+    rancher_project: "p-npzjd"
+    rancher_server: "https://testing.aether.onlab.us/"
+    rancher_cli_api: "rancher_dev_token"
+    jobs:
+      - "deploy"
+      - "deploy-shared"
+      - "deploy-onos"
+      - "deploy-stratum"
+      - "deploy-telegraf"
+      - "deploy-fluentbit"
 
 - project:
-      name: deploy-menlo-tost-dev
-      rancher_context: "c-lggdr"
-      rancher_project: "p-2nd5q"
-      k8s_config: "tost-dev-k8s"
-      config_env: "menlo-tost-dev"
-      jobs:
-          - "deploy"
-          - "deploy-onos"
-          - "deploy-stratum"
-          - "deploy-telegraf"
-          - "deploy-fluentbit":
-                rancher_project: "p -npzjd"
-
-- project:
-      name: deploy-menlo-tost-prod2
-      rancher_context: "c-5cfd8"
-      rancher_project: "p-54glz"
-      k8s_config: "tost-prod-k8s"
-      config_env: "menlo-prd"
-      rancher_server: "https://aether.onlab.us/"
-      rancher_api: "rancher_prod_token"
-      jobs:
-          - "deploy-onos"
-          - "deploy-stratum"
-          - "deploy-telegraf"
-          - "deploy-fluentbit":
-                rancher_project: "p-7kx2z"
+    name: deploy-menlo-production
+    rancher_cluster: "production-edge-onf-menlo"
+    k8s_config: "tost-prod-k8s"
+    config_env: "menlo-prd"
+    terraform_dir: "production/ace-menlo"
+    rancher_api: "rancher_prod_terraform"
+    #remove after migrating to terraform
+    rancher_context: "c-5cfd8"
+    rancher_project: "p-htfwq"
+    rancher_server: "https://aether.onlab.us/"
+    rancher_cli_api: "rancher_prod_token"
+    scm_type: "prod-scm"
+    jobs:
+      - "deploy"
+      - "deploy-onos"
+      - "deploy-shared"
+      - "deploy-stratum"
+      - "deploy-telegraf"
+      - "deploy-fluentbit"
 
 - job-template:
-      name: "{name}-onos"
-      id: "deploy-onos"
-
-      rancher_server: "https://testing.aether.onlab.us/"
-      rancher_api: "rancher_dev_token"
-      git_server: "gerrit.opencord.org"
-      git_user: "aether_jenkins"
-      git_repo: "aether-pod-configs"
-      registry_server: "registry.aetherproject.org"
-      registry_user: "admin"
-
-      disabled: true
-      pipeline-script: "tost-deploy-onos.groovy"
-      project-type: pipeline
-      dsl: !include-raw-escape: pipeline/{pipeline-script}
-      parameters:
-          - string:
-                name: rancher_server
-                default: "{rancher_server}"
-          - string:
-                name: rancher_context
-                default: "{rancher_context}"
-          - string:
-                name: rancher_project
-                default: "{rancher_project}"
-          - string:
-                name: rancher_api_env
-                default: "{rancher_api}"
-          - string:
-                name: git_repo
-                default: "{git_repo}"
-          - string:
-                name: git_server
-                default: "{git_server}"
-          - string:
-                name: git_user
-                default: "{git_user}"
-          - string:
-                name: git_password_env
-                default: "gerrit_password"
-          - string:
-                name: onos_catalog_name
-                default: "onos"
-          - string:
-                name: onos_ns
-                default: "onos-tost"
-          - string:
-                name: onos_user
-                default: "onos"
-          - string:
-                name: onos_password
-                default: "onos_password"
-          - string:
-                name: registry_server
-                default: "{registry_server}"
-          - string:
-                name: registry_user
-                default: "{registry_user}"
-          - string:
-                name: registry_password_env
-                default: "aether_registry_password"
-          - string:
-                name: config_review
-          - string:
-                name: config_patchset
-          - string:
-                name: config_env
-                default: "{config_env}"
-          - string:
-                name: k8s_config
-                default: "{k8s_config}"
+    name: "{name}-onos"
+    id: "deploy-onos"
+    rancher_api: "Rancher_Dev_Terraform"
+    git_server: "gerrit.opencord.org"
+    git_user: "aether_jenkins"
+    git_repo: "aether-pod-configs"
+    gcp_credential: "gcp_bucket_terraform"
+    project-type: pipeline
+    disabled: true
+    pipeline-script: "tost-deploy-onos.groovy"
+    dsl: !include-raw-escape: pipeline/{pipeline-script}
+    logrotate:
+      daysToKeep: 7
+      numToKeep: 10
+      artifactDaysToKeep: 7
+      artifactNumToKeep: 10
+    parameters:
+      - string:
+          name: gcp_credential
+          default: "{gcp_credential}"
+      - string:
+          name: rancher_cluster
+          default: "{rancher_cluster}"
+      - string:
+          name: rancher_api_env
+          default: "{rancher_api}"
+      - string:
+          name: git_repo
+          default: "{git_repo}"
+      - string:
+          name: git_server
+          default: "{git_server}"
+      - string:
+          name: git_user
+          default: "{git_user}"
+      - string:
+          name: git_password_env
+          default: "gerrit_password"
+      - string:
+          name: terraform_dir
+          default: "{terraform_dir}"
+      - string:
+          name: onos_catalog_name
+          default: "onos"
+      - string:
+          name: onos_ns
+          default: "tost"
+      - string:
+          name: onos_user
+          default: "onos"
+      - string:
+          name: onos_password
+          default: "onos_password"
+      - string:
+          name: config_review
+      - string:
+          name: config_patchset
+      - string:
+          name: config_env
+          default: "{config_env}"
+      - string:
+          name: k8s_config
+          default: "{k8s_config}"
 
 - job-template:
-      name: "{name}-stratum"
-      id: "deploy-stratum"
-
-      rancher_server: "https://testing.aether.onlab.us/"
-      rancher_api: "rancher_dev_token"
-      git_server: "gerrit.opencord.org"
-      git_user: "aether_jenkins"
-      git_repo: "aether-pod-configs"
-      registry_server: "registry.aetherproject.org"
-      registry_user: "admin"
-
-      disabled: true
-      pipeline-script: "tost-deploy-stratum.groovy"
-      project-type: pipeline
-      dsl: !include-raw-escape: pipeline/{pipeline-script}
-      parameters:
-          - string:
-                name: rancher_server
-                default: "{rancher_server}"
-          - string:
-                name: rancher_context
-                default: "{rancher_context}"
-          - string:
-                name: rancher_project
-                default: "{rancher_project}"
-          - string:
-                name: rancher_api_env
-                default: "{rancher_api}"
-          - string:
-                name: git_repo
-                default: "{git_repo}"
-          - string:
-                name: git_server
-                default: "{git_server}"
-          - string:
-                name: git_user
-                default: "{git_user}"
-          - string:
-                name: git_password_env
-                default: "gerrit_password"
-          - string:
-                name: stratum_ns
-                default: "stratum"
-          - string:
-                name: stratum_catalog_name
-                default: "stratum"
-          - string:
-                name: registry_server
-                default: "{registry_server}"
-          - string:
-                name: registry_user
-                default: "{registry_user}"
-          - string:
-                name: registry_password_env
-                default: "aether_registry_password"
-          - string:
-                name: config_review
-          - string:
-                name: config_patchset
-          - string:
-                name: config_env
-                default: "{config_env}"
-          - string:
-                name: k8s_config
-                default: "{k8s_config}"
+    name: "{name}-stratum"
+    id: "deploy-stratum"
+    rancher_api: "Rancher_Dev_Terraform"
+    git_server: "gerrit.opencord.org"
+    git_user: "aether_jenkins"
+    git_repo: "aether-pod-configs"
+    gcp_credential: "gcp_bucket_terraform"
+    project-type: pipeline
+    disabled: true
+    pipeline-script: "tost-deploy-stratum.groovy"
+    dsl: !include-raw-escape: pipeline/{pipeline-script}
+    logrotate:
+      daysToKeep: 7
+      numToKeep: 10
+      artifactDaysToKeep: 7
+      artifactNumToKeep: 10
+    parameters:
+      - string:
+          name: gcp_credential
+          default: "{gcp_credential}"
+      - string:
+          name: rancher_cluster
+          default: "{rancher_cluster}"
+      - string:
+          name: rancher_api_env
+          default: "{rancher_api}"
+      - string:
+          name: git_repo
+          default: "{git_repo}"
+      - string:
+          name: git_server
+          default: "{git_server}"
+      - string:
+          name: git_user
+          default: "{git_user}"
+      - string:
+          name: git_password_env
+          default: "gerrit_password"
+      - string:
+          name: terraform_dir
+          default: "{terraform_dir}"
+      - string:
+          name: stratum_ns
+          default: "tost"
+      - string:
+          name: stratum_catalog_name
+          default: "stratum"
+      - string:
+          name: config_review
+      - string:
+          name: config_patchset
+      - string:
+          name: config_env
+          default: "{config_env}"
+      - string:
+          name: k8s_config
+          default: "{k8s_config}"
 
 - job-template:
-      name: "{name}-telegraf"
-      id: "deploy-telegraf"
-
-      rancher_server: "https://testing.aether.onlab.us/"
-      rancher_api: "rancher_dev_token"
-      git_server: "gerrit.opencord.org"
-      git_user: "aether_jenkins"
-      git_repo: "aether-pod-configs"
-
-      disabled: true
-      pipeline-script: "tost-deploy-telegraf.groovy"
-      project-type: pipeline
-      dsl: !include-raw-escape: pipeline/{pipeline-script}
-      parameters:
-          - string:
-                name: rancher_server
-                default: "{rancher_server}"
-          - string:
-                name: rancher_context
-                default: "{rancher_context}"
-          - string:
-                name: rancher_project
-                default: "{rancher_project}"
-          - string:
-                name: rancher_api_env
-                default: "{rancher_api}"
-          - string:
-                name: git_repo
-                default: "{git_repo}"
-          - string:
-                name: git_server
-                default: "{git_server}"
-          - string:
-                name: git_user
-                default: "{git_user}"
-          - string:
-                name: git_password_env
-                default: "gerrit_password"
-          - string:
-                name: telegraf_ns
-                default: "telegraf"
-          - string:
-                name: config_review
-          - string:
-                name: config_patchset
-          - string:
-                name: config_env
-                default: "{config_env}"
-          - string:
-                name: k8s_config
-                default: "{k8s_config}"
+    name: "{name}-telegraf"
+    id: "deploy-telegraf"
+    rancher_api: "Rancher_Dev_Terraform"
+    git_server: "gerrit.opencord.org"
+    git_user: "aether_jenkins"
+    git_repo: "aether-pod-configs"
+    gcp_credential: "gcp_bucket_terraform"
+    project-type: pipeline
+    disabled: true
+    pipeline-script: "tost-deploy-telegraf.groovy"
+    dsl: !include-raw-escape: pipeline/{pipeline-script}
+    logrotate:
+      daysToKeep: 7
+      numToKeep: 10
+      artifactDaysToKeep: 7
+      artifactNumToKeep: 10
+    parameters:
+      - string:
+          name: gcp_credential
+          default: "{gcp_credential}"
+      - string:
+          name: rancher_cluster
+          default: "{rancher_cluster}"
+      - string:
+          name: rancher_api_env
+          default: "{rancher_api}"
+      - string:
+          name: git_repo
+          default: "{git_repo}"
+      - string:
+          name: git_server
+          default: "{git_server}"
+      - string:
+          name: git_user
+          default: "{git_user}"
+      - string:
+          name: terraform_dir
+          default: "{terraform_dir}"
+      - string:
+          name: telegraf_ns
+          default: "tost"
+      - string:
+          name: config_review
+      - string:
+          name: config_patchset
+      - string:
+          name: config_env
+          default: "{config_env}"
+      - string:
+          name: k8s_config
+          default: "{k8s_config}"
 
 - job-template:
-      name: "{name}-fluentbit"
-      id: "deploy-fluentbit"
-
-      rancher_server: "https://testing.aether.onlab.us/"
-      rancher_api: "rancher_dev_token"
-      git_server: "gerrit.opencord.org"
-      git_user: "aether_jenkins"
-      git_repo: "aether-pod-configs"
-
-      disabled: true
-      pipeline-script: "tost-deploy-fluentbit.groovy"
-      project-type: pipeline
-      dsl: !include-raw-escape: pipeline/{pipeline-script}
-      parameters:
-          - string:
-                name: rancher_server
-                default: "{rancher_server}"
-          - string:
-                name: rancher_context
-                default: "{rancher_context}"
-          - string:
-                name: rancher_project
-                default: "{rancher_project}"
-          - string:
-                name: rancher_api_env
-                default: "{rancher_api}"
-          - string:
-                name: git_repo
-                default: "{git_repo}"
-          - string:
-                name: git_server
-                default: "{git_server}"
-          - string:
-                name: git_user
-                default: "{git_user}"
-          - string:
-                name: git_password_env
-                default: "gerrit_password"
-          - string:
-                name: fluentbit_ns
-                default: "logging"
-          - string:
-                name: config_review
-          - string:
-                name: config_patchset
-          - string:
-                name: config_env
-                default: "{config_env}"
-          - string:
-                name: k8s_config
-                default: "{k8s_config}"
+    name: "{name}-fluentbit"
+    id: "deploy-fluentbit"
+    # Remove after migrating to Terraform
+    rancher_cli_api: "rancher_dev_token"
+    # End of parametes
+    rancher_api: "Rancher_Dev_Terraform"
+    git_server: "gerrit.opencord.org"
+    git_user: "aether_jenkins"
+    git_repo: "aether-pod-configs"
+    gcp_credential: "gcp_bucket_terraform"
+    project-type: pipeline
+    disabled: true
+    pipeline-script: "tost-deploy-fluentbit.groovy"
+    dsl: !include-raw-escape: pipeline/{pipeline-script}
+    logrotate:
+      daysToKeep: 7
+      numToKeep: 10
+      artifactDaysToKeep: 7
+      artifactNumToKeep: 10
+    parameters:
+      # Remove after migrating to Terraform
+      - string:
+          name: rancher_cli_env
+          default: "{rancher_cli_api}"
+      - string:
+          name: rancher_server
+          default: "{rancher_server}"
+      - string:
+          name: rancher_context
+          default: "{rancher_context}"
+      - string:
+          name: rancher_project
+          default: "{rancher_project}"
+      # End of parametes
+      - string:
+          name: gcp_credential
+          default: "{gcp_credential}"
+      - string:
+          name: rancher_cluster
+          default: "{rancher_cluster}"
+      - string:
+          name: rancher_api_env
+          default: "{rancher_api}"
+      - string:
+          name: git_repo
+          default: "{git_repo}"
+      - string:
+          name: git_server
+          default: "{git_server}"
+      - string:
+          name: git_user
+          default: "{git_user}"
+      - string:
+          name: terraform_dir
+          default: "{terraform_dir}"
+      - string:
+          name: fluentbit_ns
+          default: "logging"
+      - string:
+          name: config_review
+      - string:
+          name: config_patchset
+      - string:
+          name: config_env
+          default: "{config_env}"
+      - string:
+          name: k8s_config
+          default: "{k8s_config}"
 
 - job-template:
-      name: "{name}"
-      id: "deploy"
-      disabled: true
-      pipeline-script: "tost-deploy.groovy"
-      project-type: pipeline
-      dsl: !include-raw-escape: pipeline/{pipeline-script}
-      parameters:
-          - string:
-                name: job_name
-                default: "{name}"
-          - string:
-                name: target_server
-          - string:
-                name: k8s_config
-                default: "{k8s_config}"
+    name: "{name}"
+    id: "deploy"
+    git_server: "gerrit.opencord.org"
+    git_user: "aether_jenkins"
+    git_repo: "aether-pod-configs"
+    rancher_api: "Rancher_Dev_Terraform"
+    gcp_credential: "gcp_bucket_terraform"
+    project-type: pipeline
+    disabled: true
+    pipeline-script: "tost-deploy.groovy"
+    dsl: !include-raw-escape: pipeline/{pipeline-script}
+    logrotate:
+      daysToKeep: 7
+      numToKeep: 10
+      artifactDaysToKeep: 7
+      artifactNumToKeep: 10
+    parameters:
+      - string:
+          name: job_name
+          default: "{name}"
+      - string:
+          name: gcp_credential
+          default: "{gcp_credential}"
+      - string:
+          name: target_server
+      - string:
+          name: k8s_config
+          default: "{k8s_config}"
+      - string:
+          name: rancher_api_env
+          default: "{rancher_api}"
+      - string:
+          name: git_repo
+          default: "{git_repo}"
+      - string:
+          name: git_server
+          default: "{git_server}"
+      - string:
+          name: git_user
+          default: "{git_user}"
+      - string:
+          name: rancher_cluster
+          default: "{rancher_cluster}"
+      - string:
+          name: terraform_dir
+          default: "{terraform_dir}"
+      - string:
+          name: config_review
+      - string:
+          name: config_patchset
+
+- job-template:
+    name: "{name}-shared"
+    id: "deploy-shared"
+    registry_server: "registry.aetherproject.org"
+    registry_user: "admin"
+    target_namespace: "tost"
+    project-type: pipeline
+    disabled: true
+    pipeline-script: "tost-deploy-shared.groovy"
+    dsl: !include-raw-escape: pipeline/{pipeline-script}
+    logrotate:
+      daysToKeep: 7
+      numToKeep: 10
+      artifactDaysToKeep: 7
+      artifactNumToKeep: 10
+    parameters:
+      - string:
+          name: job_name
+          default: "{name}"
+      - string:
+          name: registry_server
+          default: "{registry_server}"
+      - string:
+          name: registry_user
+          default: "{registry_user}"
+      - string:
+          name: registry_password_env
+          default: "aether_registry_password"
+      - string:
+          name: target_namespace
+          default: "{target_namespace}"
+      - string:
+          name: k8s_config
+          default: "{k8s_config}"