Adding delay between ONU operations

Change-Id: I3442325e3b4ad9d0fe77b0182f16d3db42e5f0a1
diff --git a/bbsim.go b/bbsim.go
index 1950668..0374700 100644
--- a/bbsim.go
+++ b/bbsim.go
@@ -17,8 +17,6 @@
 package main
 
 import (
-	"gerrit.opencord.org/voltha-bbsim/protos"
-	"gerrit.opencord.org/voltha-bbsim/core"
 	"flag"
 	"fmt"
 	"log"
@@ -28,6 +26,9 @@
 	"strings"
 	"sync"
 	"time"
+
+	"gerrit.opencord.org/voltha-bbsim/core"
+	"gerrit.opencord.org/voltha-bbsim/protos"
 )
 
 func printBanner() {
@@ -39,7 +40,7 @@
 	log.Println("/________/ /________/ /________/ /_/ /_/ /_/ /_/  ")
 }
 
-func getOptions() (uint32, string, uint32, uint32, uint32, int, int, string, core.Mode) {
+func getOptions() (uint32, string, uint32, uint32, uint32, int, int, string, int, core.Mode) {
 	addressport := flag.String("H", ":50060", "IP address:port")
 	oltid := flag.Int("id", 0, "OLT-ID")
 	nintfs := flag.Int("i", 1, "Number of PON-IF ports")
@@ -48,6 +49,7 @@
 	aaawait := flag.Int("a", 30, "Wait time (sec) for activation WPA supplicants")
 	dhcpwait := flag.Int("d", 10, "Wait time (sec) for activation DHCP clients")
 	dhcpservip := flag.String("s", "182.21.0.1", "DHCP Server IP Address")
+	delay := flag.Int("delay", 1, "Delay between ONU events")
 	mode := core.DEFAULT
 	flag.Parse()
 	if *modeopt == "aaa" {
@@ -58,13 +60,13 @@
 	address := strings.Split(*addressport, ":")[0]
 	tmp, _ := strconv.Atoi(strings.Split(*addressport, ":")[1])
 	port := uint32(tmp)
-	return uint32(*oltid), address, port, uint32(*nintfs), uint32(*nonus), *aaawait, *dhcpwait, *dhcpservip, mode
+	return uint32(*oltid), address, port, uint32(*nintfs), uint32(*nonus), *aaawait, *dhcpwait, *dhcpservip, *delay, mode
 }
 
 func main() {
 	// CLI Shows up
 	printBanner()
-	oltid, ip, port, npon, nonus, aaawait, dhcpwait, dhcpservip, mode := getOptions()
+	oltid, ip, port, npon, nonus, aaawait, dhcpwait, dhcpservip, delay, mode := getOptions()
 	log.Printf("ip:%s, baseport:%d, npon:%d, nonus:%d, mode:%d\n", ip, port, npon, nonus, mode)
 
 	// Set up gRPC Server
@@ -73,7 +75,7 @@
 	addressport := ip + ":" + strconv.Itoa(int(port))
 	endchan := make(chan int, 1)
 	listener, gserver, err := core.CreateGrpcServer(oltid, npon, nonus, addressport)
-	server := core.Create(oltid, npon, nonus, aaawait, dhcpwait, dhcpservip, gserver, mode, endchan)
+	server := core.Create(oltid, npon, nonus, aaawait, dhcpwait, dhcpservip, delay, gserver, mode, endchan)
 	if err != nil {
 		log.Println(err)
 	}
diff --git a/core/core_server.go b/core/core_server.go
index 8b52e49..76c219f 100644
--- a/core/core_server.go
+++ b/core/core_server.go
@@ -17,6 +17,12 @@
 package core
 
 import (
+	"errors"
+	"log"
+	"strconv"
+	"sync"
+	"time"
+
 	"gerrit.opencord.org/voltha-bbsim/device"
 	"gerrit.opencord.org/voltha-bbsim/protos"
 	"gerrit.opencord.org/voltha-bbsim/setup"
@@ -24,11 +30,6 @@
 	"github.com/google/gopacket/layers"
 	"github.com/google/gopacket/pcap"
 	"google.golang.org/grpc"
-	"errors"
-	"log"
-	"strconv"
-	"sync"
-	"time"
 )
 
 type Mode int
@@ -50,6 +51,7 @@
 	AAAWait      int
 	DhcpWait     int
 	DhcpServerIP string
+	Delay        int
 	gRPCserver   *grpc.Server
 	VethEnv      []string
 	TestFlag     bool
@@ -63,7 +65,7 @@
 	Pkt  gopacket.Packet
 }
 
-func (s *Server)Initialize(){
+func (s *Server) Initialize() {
 	s.VethEnv = []string{}
 	s.Endchan = make(chan int)
 	s.TestFlag = false
@@ -71,7 +73,7 @@
 	s.Ioinfos = []*Ioinfo{}
 }
 
-func Create(oltid uint32, npon uint32, nonus uint32, aaawait int, dhcpwait int, ip string, g *grpc.Server, mode Mode, e chan int) *Server {
+func Create(oltid uint32, npon uint32, nonus uint32, aaawait int, dhcpwait int, ip string, delay int, g *grpc.Server, mode Mode, e chan int) *Server {
 	s := new(Server)
 	s.Olt = device.CreateOlt(oltid, npon, 1)
 	nnni := s.Olt.NumNniIntf
@@ -81,6 +83,7 @@
 	s.DhcpWait = dhcpwait
 	s.DhcpServerIP = ip
 	s.gRPCserver = g
+	s.Delay = delay
 	s.Mode = mode
 	s.Endchan = e
 	s.VethEnv = []string{}
@@ -92,11 +95,11 @@
 
 	//TODO: To be fixed
 	s.CtagMap = make(map[string]uint32)
-	for i := 0; i < MAX_ONUS_PER_PON; i ++ {
+	for i := 0; i < MAX_ONUS_PER_PON; i++ {
 		oltid := s.Olt.ID
 		intfid := uint32(1)
 		sn := convB2S(device.CreateSN(oltid, intfid, uint32(i)))
-		s.CtagMap[sn] = uint32(900 + i)	// This is hard coded for BBWF
+		s.CtagMap[sn] = uint32(900 + i) // This is hard coded for BBWF
 	}
 	return s
 }
@@ -148,7 +151,7 @@
 	}
 
 	for intfid, _ := range s.Onumap {
-		sendOnuInd(stream, s.Onumap[intfid])
+		sendOnuInd(stream, s.Onumap[intfid], s.Delay)
 		log.Printf("OLT id:%d sent ONUInd.\n", olt.ID)
 	}
 
@@ -168,7 +171,7 @@
 
 		errchan := make(chan error)
 		go func() {
-			<- errchan
+			<-errchan
 			close(s.Endchan)
 		}()
 
@@ -222,7 +225,7 @@
 	return nil
 }
 
-func createIoinfos(oltid uint32, vethenv []string, onumap map[uint32][]*device.Onu)([]*Ioinfo, []string, error){
+func createIoinfos(oltid uint32, vethenv []string, onumap map[uint32][]*device.Onu) ([]*Ioinfo, []string, error) {
 	ioinfos := []*Ioinfo{}
 	var err error
 	for intfid, _ := range onumap {
@@ -309,7 +312,7 @@
 				//C-TAG
 				onu, _ := s.getOnuByID(onuid)
 				sn := convB2S(onu.SerialNumber.VendorSpecific)
-				if ctag, ok := s.CtagMap[sn]; ok == true{
+				if ctag, ok := s.CtagMap[sn]; ok == true {
 					tagpkt, err := PushVLAN(pkt, uint16(ctag))
 					if err != nil {
 						log.Println("Error happend in C-tag tagging")
@@ -373,14 +376,14 @@
 		univeths = append(univeths, info.name)
 	}
 
-	for  {
+	for {
 		select {
 		case <-s.Endchan:
 			log.Println("exeAAATest thread receives close !")
 			return nil
-		case <- time.After(time.Second * time.Duration(s.AAAWait)):
+		case <-time.After(time.Second * time.Duration(s.AAAWait)):
 			log.Println("exeAAATest Start")
-			err = setup.ActivateWPASups(univeths)
+			err = setup.ActivateWPASups(univeths, s.Delay)
 			if err != nil {
 				return err
 			}
@@ -416,14 +419,14 @@
 		univeths = append(univeths, info.name)
 	}
 
-	for  {
+	for {
 		select {
 		case <-s.Endchan:
 			log.Println("exeDHCPTest thread receives close !")
 			return nil
-		case <- time.After(time.Second * time.Duration(s.DhcpWait)):
+		case <-time.After(time.Second * time.Duration(s.DhcpWait)):
 			log.Println("exeDHCPTest Start")
-			err = setup.ActivateDHCPClients(univeths)
+			err = setup.ActivateDHCPClients(univeths, s.Delay)
 			if err != nil {
 				return err
 			}
@@ -479,7 +482,7 @@
 	return nil
 }
 
-func (s *Server)IsAllOnuActive(regonus map[uint32][]*device.Onu) bool {
+func (s *Server) IsAllOnuActive(regonus map[uint32][]*device.Onu) bool {
 	for _, onus := range regonus {
 		for _, onu := range onus {
 			if onu.GetIntStatus() != device.ONU_ACTIVATED {
@@ -543,7 +546,7 @@
 }
 
 func killProcesses(pnames []string) error {
-	for _, pname := range  pnames {
+	for _, pname := range pnames {
 		setup.KillProcess(pname)
 	}
 	return nil
@@ -584,7 +587,7 @@
 func convB2S(b []byte) string {
 	s := ""
 	for _, i := range b {
-		s = s + strconv. FormatInt(int64(i/16), 16) + strconv. FormatInt(int64(i%16), 16)
+		s = s + strconv.FormatInt(int64(i/16), 16) + strconv.FormatInt(int64(i%16), 16)
 	}
 	return s
-}
\ No newline at end of file
+}
diff --git a/core/openolt_service.go b/core/openolt_service.go
index dbb7644..a0d00bf 100644
--- a/core/openolt_service.go
+++ b/core/openolt_service.go
@@ -17,9 +17,11 @@
 package core
 
 import (
-	"gerrit.opencord.org/voltha-bbsim/protos"
-	"gerrit.opencord.org/voltha-bbsim/device"
 	"log"
+	"time"
+
+	"gerrit.opencord.org/voltha-bbsim/device"
+	"gerrit.opencord.org/voltha-bbsim/protos"
 )
 
 func sendOltIndUp(stream openolt.Openolt_EnableIndicationServer, olt *device.Olt) error {
@@ -43,7 +45,7 @@
 func sendIntfInd(stream openolt.Openolt_EnableIndicationServer, olt *device.Olt) error {
 	for i := uint32(0); i < olt.NumPonIntf+olt.NumNniIntf; i++ {
 		intf := olt.Intfs[i]
-		if intf.Type == "pon"{	// There is no need to send IntfInd for NNI
+		if intf.Type == "pon" { // There is no need to send IntfInd for NNI
 			data := &openolt.Indication_IntfInd{&openolt.IntfIndication{IntfId: intf.IntfID, OperState: intf.OperState}}
 			if err := stream.Send(&openolt.Indication{Data: data}); err != nil {
 				log.Printf("Failed to send Intf [id: %d] indication : %v\n", i, err)
@@ -80,8 +82,9 @@
 	return nil
 }
 
-func sendOnuInd(stream openolt.Openolt_EnableIndicationServer, onus []*device.Onu) error {
+func sendOnuInd(stream openolt.Openolt_EnableIndicationServer, onus []*device.Onu, delay int) error {
 	for i, onu := range onus {
+		time.Sleep(time.Duration(delay) * time.Second)
 		data := &openolt.Indication_OnuInd{&openolt.OnuIndication{IntfId: onu.IntfID, OnuId: onu.OnuID, OperState: "up", AdminState: "up", SerialNumber: onu.SerialNumber}}
 		log.Printf("sendONUInd Onuid: %d\n", i)
 		if err := stream.Send(&openolt.Indication{Data: data}); err != nil {
diff --git a/setup/setup_env.go b/setup/setup_env.go
index 6d9981d..82dd6cf 100644
--- a/setup/setup_env.go
+++ b/setup/setup_env.go
@@ -19,6 +19,7 @@
 import (
 	"log"
 	"os/exec"
+	"time"
 )
 
 const (
@@ -28,8 +29,10 @@
 	NNI_VETH_DW_PFX = "sim_nd"
 )
 
-func ActivateWPASups(vethnames []string) error {
+func ActivateWPASups(vethnames []string, delay int) error {
 	for _, vethname := range vethnames {
+		time.Sleep(time.Duration(delay) * time.Second)
+		log.Printf("ActivateWPASups for interface %v\n", vethname)
 		if err := activateWPASupplicant(vethname); err != nil {
 			return err
 		}
@@ -37,8 +40,10 @@
 	return nil
 }
 
-func ActivateDHCPClients(vethnames []string) error {
+func ActivateDHCPClients(vethnames []string, delay int) error {
 	for _, vethname := range vethnames {
+		time.Sleep(time.Duration(delay) * time.Second)
+		log.Printf("activateDHCPClient for interface %v\n", vethname)
 		if err := activateDHCPClient(vethname); err != nil {
 			return err
 		}
@@ -103,21 +108,29 @@
 	conf := "/etc/wpa_supplicant/wpa_supplicant.conf"
 	err = exec.Command(cmd, "-D", "wired", "-i", vethname, "-c", conf).Start()
 	if err != nil {
-		log.Printf("[ERROR] Fail to activateWPASupplicant() for :%s %v\n", vethname, err)
+		log.Printf("[ERROR] Fail to activateWPASupplicant() for: %s %v\n", vethname, err)
 		return
 	}
-	log.Printf("activateWPASupplicant() for :%s\n", vethname)
+	log.Printf("activateWPASupplicant() for: %s\n", vethname)
 	return
 }
 
 func activateDHCPClient(vethname string) (err error) {
-	cmd := "/usr/local/bin/dhclient"
-	err = exec.Command(cmd, vethname).Start()
-	if err != nil {
-		log.Printf("[ERROR] Faile to activateWPASupplicant() for :%s %v\n", vethname, err)
-		return
+	log.Printf("activateDHCPClient() start for: %s\n", vethname)
+	cmd := exec.Command("/usr/local/bin/dhclient", vethname)
+	// if err := cmd.Run(); err != nil {
+	if err := cmd.Start(); err != nil {
+		log.Printf("[ERROR] Failed to activateDHCPClient() for: %s", vethname)
+		log.Panic(err)
 	}
-	log.Printf("activateDHCPClient() for: %s\n", vethname)
+
+	// if the dhclient commands return we got an IP,
+	// so kill it
+	// if err := KillProcess("/usr/local/bin/dhclient"); err != nil {
+	// 	log.Printf("[ERROR] Failed to kill activateDHCPClient() for: %s", vethname, err)
+	// }
+
+	log.Printf("activateDHCPClient() done for: %s\n", vethname)
 	return
 }