[COMAC-128] Add SR-IOV verification/redeployment steps in jenkins file

Change-Id: I11b44bb01ea03be16ca5d159c7028a01681cf497
diff --git a/Jenkinsfile-mcord-release-build b/Jenkinsfile-mcord-release-build
index 2111456..636e6dd 100644
--- a/Jenkinsfile-mcord-release-build
+++ b/Jenkinsfile-mcord-release-build
@@ -153,6 +153,41 @@
             }
           }
 
+          // verification: sriov pods will be deployed for each machine
+          timeout (1) {
+              waitUntil {
+                  num_sriov_pods = sh returnStdout: true, script: """
+                  kubectl get pods -n kube-system | grep sriov | wc -l
+                  """
+                  return num_sriov_pods.toInteger() == deployment_config.nodes.size()
+              }
+          }
+
+          // redeployment procedure for sriov pod
+          // Sometimes a SR-IOV pod encounters "MountVolume.Setup failed". To resolve it, the SR-IOV pod need to be removed and redployed.
+          max_retry_index = 100;
+          for (int i = 0; i<max_retry_index; i++) {
+              sh returnStdout: true, script: "sleep 10"
+
+              sh script: """
+              init_num_sriov=\$(kubectl get pods -n kube-system | grep sriov | grep Init | wc -l);
+              for pod in \$(kubectl get pods -n kube-system | grep sriov | grep Init | awk '{print \$1}');
+              do
+                echo \$pod is initializing
+                num_err_msgs=\$(kubectl describe pods \$pod -n kube-system | grep -E 'MountVolume.SetUp failed' | wc -l)
+                if [ \$num_err_msgs > 0 ]; then
+                    kubectl delete pod \$pod -n kube-system --force --grace-period=0
+                fi
+              done
+              """
+              sriov_all_ready = sh returnStdout: true, script: """
+              kubectl get pods -n kube-system | grep sriov | grep Init | wc -l;
+              """
+              if (sriov_all_ready.toInteger() == 0) {
+                  break;
+              }
+          }
+
           // Sometimes tillerpod is up, but it's not ready to accept deployment yet
           // use helm ls to make sure it's ready.
           timeout(1) {