core/core_server.go - voltha-bbsim - Gitiles

 /*
  * Copyright 2018-present Open Networking Foundation

  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at

  * http://www.apache.org/licenses/LICENSE-2.0

  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */

 package core

 import (
 	"gerrit.opencord.org/voltha-bbsim/device"
 	"gerrit.opencord.org/voltha-bbsim/protos"
 	"gerrit.opencord.org/voltha-bbsim/setup"
 	"errors"
 	"github.com/google/gopacket"
 	"github.com/google/gopacket/layers"
 	"github.com/google/gopacket/pcap"
 	"google.golang.org/grpc"
 	"log"
 	"strconv"
 	"sync"
 	"time"
 )

 type Mode int

 const MAX_ONUS_PER_PON = 64 // This value should be the same with the value in AdapterPlatrorm class

 const (
 	DEFAULT Mode = iota
 	AAA
 	BOTH
 )

 type Server struct {
 	Olt          *device.Olt
 	Onumap       map[uint32][]*device.Onu
 	Ioinfos      []*Ioinfo
 	Endchan      chan int
 	Mode         Mode
 	AAAWait      int
 	DhcpWait     int
 	DhcpServerIP string
 	gRPCserver   *grpc.Server
 }

 type Packet struct {
 	Info *Ioinfo
 	Pkt  gopacket.Packet
 }

 func CreateServer(oltid uint32, npon uint32, nonus uint32, aaawait int, dhcpwait int, ip string, g *grpc.Server, mode Mode, e chan int) *Server {
 	s := new(Server)
 	s.Olt = device.CreateOlt(oltid, npon, 1)
 	nnni := s.Olt.NumNniIntf
 	log.Printf("OLT ID: %d was retrieved.\n", s.Olt.ID)
 	s.Onumap = make(map[uint32][]*device.Onu)
 	s.AAAWait = aaawait
 	s.DhcpWait = dhcpwait
 	s.DhcpServerIP = ip
 	s.gRPCserver = g
 	s.Mode = mode
 	s.Endchan = e
 	for intfid := nnni; intfid < npon+nnni; intfid++ {
 		s.Onumap[intfid] = device.CreateOnus(oltid, intfid, nonus, nnni)
 	}
 	return s
 }

 func (s *Server) activateOLT(stream openolt.Openolt_EnableIndicationServer) error {
 	// Activate OLT
 	olt := s.Olt
 	oltid := olt.ID
 	vethenv := []string{}
 	wg := &sync.WaitGroup{}

 	if err := sendOltInd(stream, olt); err != nil {
 		return err
 	}
 	olt.OperState = "up"
 	olt.InternalState = device.OLT_UP
 	log.Printf("OLT %s sent OltInd.\n", olt.Name)

 	// OLT sends Interface Indication to Adapter
 	if err := sendIntfInd(stream, olt); err != nil {
 		log.Printf("[ERROR] Fail to sendIntfInd: %v\n", err)
 		return err
 	}
 	log.Printf("OLT %s sent IntfInd.\n", olt.Name)

 	// OLT sends Operation Indication to Adapter after activating each interface
 	//time.Sleep(IF_UP_TIME * time.Second)
 	olt.InternalState = device.PONIF_UP
 	if err := sendOperInd(stream, olt); err != nil {
 		log.Printf("[ERROR] Fail to sendOperInd: %v\n", err)
 		return err
 	}
 	log.Printf("OLT %s sent OperInd.\n", olt.Name)

 	// OLT sends ONU Discover Indication to Adapter after ONU discovery
 	for intfid, _ := range s.Onumap {
 		device.UpdateOnusOpStatus(intfid, s.Onumap[intfid], "up")
 	}

 	for intfid, _ := range s.Onumap {
 		sendOnuDiscInd(stream, s.Onumap[intfid])
 		log.Printf("OLT id:%d sent ONUDiscInd.\n", olt.ID)
 	}

 	// OLT Sends OnuInd after waiting all of those ONUs up
 	for {
 		if s.IsAllONUActive() {
 			break
 		}
 	}
 	for intfid, _ := range s.Onumap {
 		sendOnuInd(stream, s.Onumap[intfid])
 		log.Printf("OLT id:%d sent ONUInd.\n", olt.ID)
 	}

 	if s.Mode == DEFAULT {
 		//EnableIndication's stream should be kept even after activateOLT() is finished.
 		//Otherwise, OpenOLT adapter sends EnableIndication again.
 		<-s.Endchan
 		log.Println("core server thread receives close !")
 	} else if s.Mode == AAA || s.Mode == BOTH {
 		var err error
 		s.Ioinfos = []*Ioinfo{}
 		for intfid, _ := range s.Onumap {
 			for i := 0; i < len(s.Onumap[intfid]); i++ {
 				var handler *pcap.Handle
 				onuid := s.Onumap[intfid][i].OnuID
 				uniup, unidw := makeUniName(oltid, intfid, onuid)
 				if handler, vethenv, err = setupVethHandler(uniup, unidw, vethenv); err != nil {
 					return err
 				}
 				iinfo := Ioinfo{name: uniup, iotype: "uni", ioloc: "inside", intfid: intfid, onuid: onuid, handler: handler}
 				s.Ioinfos = append(s.Ioinfos, &iinfo)
 				oinfo := Ioinfo{name: unidw, iotype: "uni", ioloc: "outside", intfid: intfid, onuid: onuid, handler: nil}
 				s.Ioinfos = append(s.Ioinfos, &oinfo)
 			}
 		}
 		var handler *pcap.Handle
 		nniup, nnidw := makeNniName(oltid)
 		if handler, vethenv, err = setupVethHandler(nniup, nnidw, vethenv); err != nil {
 			return err
 		}
 		iinfo := Ioinfo{name: nnidw, iotype: "nni", ioloc: "inside", intfid: 1, handler: handler}
 		s.Ioinfos = append(s.Ioinfos, &iinfo)
 		oinfo := Ioinfo{name: nnidw, iotype: "nni", ioloc: "outside", intfid: 1, handler: nil}
 		s.Ioinfos = append(s.Ioinfos, &oinfo)

 		errchan := make(chan error)
 		go func() {
 			<-errchan
 			close(s.Endchan)
 		}()

 		wg.Add(1)
 		go func() {
 			defer func() {
 				log.Println("runPacketInDaemon Done")
 				wg.Done()
 			}()
 			err := s.runPacketInDaemon(stream)
 			if err != nil {
 				errchan <- err
 				return
 			}
 		}()

 		wg.Add(1)
 		go func() {
 			defer func() {
 				log.Println("exeAAATest Done")
 				wg.Done()
 			}()
 			infos, err := s.getUniIoinfos("outside")
 			if err != nil {
 				errchan <- err
 				return
 			}
 			univeths := []string{}
 			for _, info := range infos {
 				univeths = append(univeths, info.name)
 			}
 			err = s.exeAAATest(univeths)
 			if err != nil {
 				errchan <- err
 				return
 			}
 		}()
 		wg.Wait()
 		tearDown(vethenv) // Grace teardown
 		log.Println("Grace shutdown down")
 	}
 	return nil
 }

 func (s *Server) runPacketInDaemon(stream openolt.Openolt_EnableIndicationServer) error {
 	log.Println("runPacketInDaemon Start")
 	unichannel := make(chan Packet, 2048)
 	flag := false

 	for intfid, _ := range s.Onumap {
 		for _, onu := range s.Onumap[intfid] { //TODO: should be updated for multiple-Interface
 			onuid := onu.OnuID
 			ioinfo, err := s.identifyUniIoinfo("inside", intfid, onuid)
 			if err != nil {
 				log.Printf("[ERROR] Fail to identifyUniIoinfo (onuid: %d): %v\n", onuid, err)
 				return err
 			}
 			uhandler := ioinfo.handler
 			defer uhandler.Close()
 			go RecvWorker(ioinfo, uhandler, unichannel)
 		}
 	}

 	ioinfo, err := s.identifyNniIoinfo("inside")
 	if err != nil {
 		return err
 	}
 	nhandler := ioinfo.handler
 	defer nhandler.Close()
 	nnichannel := make(chan Packet, 32)
 	go RecvWorker(ioinfo, nhandler, nnichannel)

 	data := &openolt.Indication_PktInd{}
 	for {
 		select {
 		case unipkt := <-unichannel:
 			log.Println("Received packet in grpc Server from UNI.")
 			if unipkt.Info == nil || unipkt.Info.iotype != "uni" {
 				log.Println("[WARNING] This packet does not come from UNI !")
 				continue
 			}
 			intfid := unipkt.Info.intfid
 			onuid := unipkt.Info.onuid
 			gemid, _ := getGemPortID(intfid, onuid)
 			pkt := unipkt.Pkt
 			layerEth := pkt.Layer(layers.LayerTypeEthernet)
 			le, _ := layerEth.(*layers.Ethernet)
 			ethtype := le.EthernetType
 			if ethtype == 0x888e {
 				log.Printf("Received upstream packet is EAPOL.")
 				log.Println(unipkt.Pkt.Dump())
 				log.Println(pkt.Dump())
 			} else if layerDHCP := pkt.Layer(layers.LayerTypeDHCPv4); layerDHCP != nil {
 				log.Printf("Received upstream packet is DHCP.")
 				log.Println(unipkt.Pkt.Dump())
 				log.Println(pkt.Dump())
 			} else {
 				continue
 			}
 			log.Printf("sendPktInd intfid:%d (onuid: %d) gemid:%d\n", intfid, onuid, gemid)
 			data = &openolt.Indication_PktInd{PktInd: &openolt.PacketIndication{IntfType: "pon", IntfId: intfid, GemportId: gemid, Pkt: pkt.Data()}}
 			if err := stream.Send(&openolt.Indication{Data: data}); err != nil {
 				log.Printf("[ERROR] Failed to send PktInd indication. %v\n", err)
 				return err
 			}
 		case nnipkt := <-nnichannel:
 			if nnipkt.Info == nil || nnipkt.Info.iotype != "nni" {
 				log.Println("[WARNING] This packet does not come from NNI !")
 				continue
 			}
 			log.Println("Received packet in grpc Server from NNI.")
 			intfid := nnipkt.Info.intfid
 			pkt := nnipkt.Pkt
 			log.Printf("sendPktInd intfid:%d\n", intfid)
 			data = &openolt.Indication_PktInd{PktInd: &openolt.PacketIndication{IntfType: "nni", IntfId: intfid, Pkt: pkt.Data()}}
 			if err := stream.Send(&openolt.Indication{Data: data}); err != nil {
 				log.Printf("[ERROR] Failed to send PktInd indication. %v\n", err)
 				return err
 			}
 		case <-s.Endchan:
 			if flag == false {
 				log.Println("PacketInDaemon thread receives close !")
 				close(unichannel)
 				log.Println("Closed unichannel !")
 				close(nnichannel)
 				log.Println("Closed nnichannel !")
 				flag = true
 				return nil
 			}
 		}
 	}
 	return nil
 }

 func (s *Server) exeAAATest(vethenv []string) error {
 	log.Println("exeAAATest Start")
 	for i := 0; i < s.AAAWait; i++ {
 		select {
 		case <-s.Endchan:
 			log.Println("exeAAATest thread receives close !")
 			return nil
 		default:
 			log.Println("exeAAATest is now sleeping....")
 			time.Sleep(time.Second)
 		}
 	}
 	err := setup.ActivateWPASups(vethenv)
 	if err != nil {
 		return err
 	}
 	return nil
 }

 func (s *Server) onuPacketOut(intfid uint32, onuid uint32, rawpkt gopacket.Packet) error {
 	layerEth := rawpkt.Layer(layers.LayerTypeEthernet)
 	if layerEth != nil {
 		pkt, _ := layerEth.(*layers.Ethernet)
 		ethtype := pkt.EthernetType
 		if ethtype == 0x888e {
 			log.Printf("Received downstream packet is EAPOL.")
 			log.Println(rawpkt.Dump())
 		} else if layerDHCP := rawpkt.Layer(layers.LayerTypeDHCPv4); layerDHCP != nil {
 			log.Printf("Received downstream packet is DHCP.")
 			log.Println(rawpkt.Dump())
 			rawpkt, _, _ = PopVLAN(rawpkt)
 			rawpkt, _, _ = PopVLAN(rawpkt)
 		} else {
 			return nil
 		}
 		ioinfo, err := s.identifyUniIoinfo("inside", intfid, onuid)
 		if err != nil {
 			return err
 		}
 		handle := ioinfo.handler
 		SendUni(handle, rawpkt)
 		return nil
 	}
 	log.Printf("[WARNING] Received packet is not supported")
 	return nil
 }

 func (s *Server) uplinkPacketOut(rawpkt gopacket.Packet) error {
 	log.Println("")
 	poppkt, _, err := PopVLAN(rawpkt)
 	poppkt, _, err = PopVLAN(poppkt)
 	if err != nil {
 		log.Println(err)
 		return err
 	}
 	ioinfo, err := s.identifyNniIoinfo("inside")
 	if err != nil {
 		return err
 	}
 	handle := ioinfo.handler
 	SendNni(handle, poppkt)
 	return nil
 }

 func (s *Server) IsAllONUActive() bool {
 	for _, onus := range s.Onumap {
 		for _, onu := range onus {
 			if onu.InternalState != device.ONU_ACTIVATED {
 				return false
 			}
 		}
 	}
 	return true
 }

 func getVID(onuid uint32) (uint16, error) {
 	return uint16(onuid), nil
 }

 func getGemPortID(intfid uint32, onuid uint32) (uint32, error) {
 	idx := uint32(0)
 	return 1024 + (((MAX_ONUS_PER_PON*intfid + onuid - 1) * 7) + idx), nil
 	//return uint32(1032 + 8 * (vid - 1)), nil
 }

 func (s *Server) getOnuBySN(sn *openolt.SerialNumber) (*device.Onu, error) {
 	for _, onus := range s.Onumap {
 		for _, onu := range onus {
 			if device.ValidateSN(*sn, *onu.SerialNumber) {
 				return onu, nil
 			}
 		}
 	}
 	err := errors.New("No mathced SN is found !")
 	log.Println(err)
 	return nil, err
 }

 func (s *Server) getOnuByID(onuid uint32) (*device.Onu, error) {
 	for _, onus := range s.Onumap {
 		for _, onu := range onus {
 			if onu.OnuID == onuid {
 				return onu, nil
 			}
 		}
 	}
 	err := errors.New("No matched OnuID is found !")
 	log.Println(err)
 	return nil, err
 }

 func makeUniName(oltid uint32, intfid uint32, onuid uint32) (upif string, dwif string) {
 	upif = setup.UNI_VETH_UP_PFX + strconv.Itoa(int(oltid)) + "_" + strconv.Itoa(int(intfid)) + "_" + strconv.Itoa(int(onuid))
 	dwif = setup.UNI_VETH_DW_PFX + strconv.Itoa(int(oltid)) + "_" + strconv.Itoa(int(intfid)) + "_" + strconv.Itoa(int(onuid))
 	return
 }

 func makeNniName(oltid uint32) (upif string, dwif string) {
 	upif = setup.NNI_VETH_UP_PFX + strconv.Itoa(int(oltid))
 	dwif = setup.NNI_VETH_DW_PFX + strconv.Itoa(int(oltid))
 	return
 }

 func tearDown(vethenv []string) error {
 	log.Println("tearDown()")
 	setup.KillAllWPASups()
 	setup.KillAllDHCPClients()
 	setup.TearVethDown(vethenv)
 	return nil
 }

 func setupVethHandler(inveth string, outveth string, vethenv []string) (*pcap.Handle, []string, error) {
 	log.Printf("setupVethHandler: %s and %s\n", inveth, outveth)
 	err1 := setup.CreateVethPairs(inveth, outveth)
 	vethenv = append(vethenv, inveth)
 	if err1 != nil {
 		setup.RemoveVeths(vethenv)
 		return nil, vethenv, err1
 	}
 	handler, err2 := getVethHandler(inveth)
 	if err2 != nil {
 		setup.RemoveVeths(vethenv)
 		return nil, vethenv, err2
 	}
 	return handler, vethenv, nil
 }

 func getVethHandler(vethname string) (*pcap.Handle, error) {
 	var (
 		device       string = vethname
 		snapshot_len int32  = 1518
 		promiscuous  bool   = false
 		err          error
 		timeout      time.Duration = pcap.BlockForever
 	)
 	handle, err := pcap.OpenLive(device, snapshot_len, promiscuous, timeout)
 	if err != nil {
 		return nil, err
 	}
 	log.Printf("Server handle created for %s\n", vethname)
 	return handle, nil
 }
	/*
	* Copyright 2018-present Open Networking Foundation

	* Licensed under the Apache License, Version 2.0 (the "License");
	* you may not use this file except in compliance with the License.
	* You may obtain a copy of the License at

	* http://www.apache.org/licenses/LICENSE-2.0

	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/

	package core

	import (
	"gerrit.opencord.org/voltha-bbsim/device"
	"gerrit.opencord.org/voltha-bbsim/protos"
	"gerrit.opencord.org/voltha-bbsim/setup"
	"errors"
	"github.com/google/gopacket"
	"github.com/google/gopacket/layers"
	"github.com/google/gopacket/pcap"
	"google.golang.org/grpc"
	"log"
	"strconv"
	"sync"
	"time"
	)

	type Mode int

	const MAX_ONUS_PER_PON = 64 // This value should be the same with the value in AdapterPlatrorm class

	const (
	DEFAULT Mode = iota
	AAA
	BOTH
	)

	type Server struct {
	Olt *device.Olt
	Onumap map[uint32][]*device.Onu
	Ioinfos []*Ioinfo
	Endchan chan int
	Mode Mode
	AAAWait int
	DhcpWait int
	DhcpServerIP string
	gRPCserver *grpc.Server
	}

	type Packet struct {
	Info *Ioinfo
	Pkt gopacket.Packet
	}

	func CreateServer(oltid uint32, npon uint32, nonus uint32, aaawait int, dhcpwait int, ip string, g grpc.Server, mode Mode, e chan int) Server {
	s := new(Server)
	s.Olt = device.CreateOlt(oltid, npon, 1)
	nnni := s.Olt.NumNniIntf
	log.Printf("OLT ID: %d was retrieved.\n", s.Olt.ID)
	s.Onumap = make(map[uint32][]*device.Onu)
	s.AAAWait = aaawait
	s.DhcpWait = dhcpwait
	s.DhcpServerIP = ip
	s.gRPCserver = g
	s.Mode = mode
	s.Endchan = e
	for intfid := nnni; intfid < npon+nnni; intfid++ {
	s.Onumap[intfid] = device.CreateOnus(oltid, intfid, nonus, nnni)
	}
	return s
	}

	func (s *Server) activateOLT(stream openolt.Openolt_EnableIndicationServer) error {
	// Activate OLT
	olt := s.Olt
	oltid := olt.ID
	vethenv := []string{}
	wg := &sync.WaitGroup{}

	if err := sendOltInd(stream, olt); err != nil {
	return err
	}
	olt.OperState = "up"
	olt.InternalState = device.OLT_UP
	log.Printf("OLT %s sent OltInd.\n", olt.Name)

	// OLT sends Interface Indication to Adapter
	if err := sendIntfInd(stream, olt); err != nil {
	log.Printf("[ERROR] Fail to sendIntfInd: %v\n", err)
	return err
	}
	log.Printf("OLT %s sent IntfInd.\n", olt.Name)

	// OLT sends Operation Indication to Adapter after activating each interface
	//time.Sleep(IF_UP_TIME * time.Second)
	olt.InternalState = device.PONIF_UP
	if err := sendOperInd(stream, olt); err != nil {
	log.Printf("[ERROR] Fail to sendOperInd: %v\n", err)
	return err
	}
	log.Printf("OLT %s sent OperInd.\n", olt.Name)

	// OLT sends ONU Discover Indication to Adapter after ONU discovery
	for intfid, _ := range s.Onumap {
	device.UpdateOnusOpStatus(intfid, s.Onumap[intfid], "up")
	}

	for intfid, _ := range s.Onumap {
	sendOnuDiscInd(stream, s.Onumap[intfid])
	log.Printf("OLT id:%d sent ONUDiscInd.\n", olt.ID)
	}

	// OLT Sends OnuInd after waiting all of those ONUs up
	for {
	if s.IsAllONUActive() {
	break
	}
	}
	for intfid, _ := range s.Onumap {
	sendOnuInd(stream, s.Onumap[intfid])
	log.Printf("OLT id:%d sent ONUInd.\n", olt.ID)
	}

	if s.Mode == DEFAULT {
	//EnableIndication's stream should be kept even after activateOLT() is finished.
	//Otherwise, OpenOLT adapter sends EnableIndication again.
	<-s.Endchan
	log.Println("core server thread receives close !")
	} else if s.Mode == AAA \|\| s.Mode == BOTH {
	var err error
	s.Ioinfos = []*Ioinfo{}
	for intfid, _ := range s.Onumap {
	for i := 0; i < len(s.Onumap[intfid]); i++ {
	var handler *pcap.Handle
	onuid := s.Onumap[intfid][i].OnuID
	uniup, unidw := makeUniName(oltid, intfid, onuid)
	if handler, vethenv, err = setupVethHandler(uniup, unidw, vethenv); err != nil {
	return err
	}
	iinfo := Ioinfo{name: uniup, iotype: "uni", ioloc: "inside", intfid: intfid, onuid: onuid, handler: handler}
	s.Ioinfos = append(s.Ioinfos, &iinfo)
	oinfo := Ioinfo{name: unidw, iotype: "uni", ioloc: "outside", intfid: intfid, onuid: onuid, handler: nil}
	s.Ioinfos = append(s.Ioinfos, &oinfo)
	}
	}
	var handler *pcap.Handle
	nniup, nnidw := makeNniName(oltid)
	if handler, vethenv, err = setupVethHandler(nniup, nnidw, vethenv); err != nil {
	return err
	}
	iinfo := Ioinfo{name: nnidw, iotype: "nni", ioloc: "inside", intfid: 1, handler: handler}
	s.Ioinfos = append(s.Ioinfos, &iinfo)
	oinfo := Ioinfo{name: nnidw, iotype: "nni", ioloc: "outside", intfid: 1, handler: nil}
	s.Ioinfos = append(s.Ioinfos, &oinfo)

	errchan := make(chan error)
	go func() {
	<-errchan
	close(s.Endchan)
	}()

	wg.Add(1)
	go func() {
	defer func() {
	log.Println("runPacketInDaemon Done")
	wg.Done()
	}()
	err := s.runPacketInDaemon(stream)
	if err != nil {
	errchan <- err
	return
	}
	}()

	wg.Add(1)
	go func() {
	defer func() {
	log.Println("exeAAATest Done")
	wg.Done()
	}()
	infos, err := s.getUniIoinfos("outside")
	if err != nil {
	errchan <- err
	return
	}
	univeths := []string{}
	for _, info := range infos {
	univeths = append(univeths, info.name)
	}
	err = s.exeAAATest(univeths)
	if err != nil {
	errchan <- err
	return
	}
	}()
	wg.Wait()
	tearDown(vethenv) // Grace teardown
	log.Println("Grace shutdown down")
	}
	return nil
	}

	func (s *Server) runPacketInDaemon(stream openolt.Openolt_EnableIndicationServer) error {
	log.Println("runPacketInDaemon Start")
	unichannel := make(chan Packet, 2048)
	flag := false

	for intfid, _ := range s.Onumap {
	for _, onu := range s.Onumap[intfid] { //TODO: should be updated for multiple-Interface
	onuid := onu.OnuID
	ioinfo, err := s.identifyUniIoinfo("inside", intfid, onuid)
	if err != nil {
	log.Printf("[ERROR] Fail to identifyUniIoinfo (onuid: %d): %v\n", onuid, err)
	return err
	}
	uhandler := ioinfo.handler
	defer uhandler.Close()
	go RecvWorker(ioinfo, uhandler, unichannel)
	}
	}

	ioinfo, err := s.identifyNniIoinfo("inside")
	if err != nil {
	return err
	}
	nhandler := ioinfo.handler
	defer nhandler.Close()
	nnichannel := make(chan Packet, 32)
	go RecvWorker(ioinfo, nhandler, nnichannel)

	data := &openolt.Indication_PktInd{}
	for {
	select {
	case unipkt := <-unichannel:
	log.Println("Received packet in grpc Server from UNI.")
	if unipkt.Info == nil \|\| unipkt.Info.iotype != "uni" {
	log.Println("[WARNING] This packet does not come from UNI !")
	continue
	}
	intfid := unipkt.Info.intfid
	onuid := unipkt.Info.onuid
	gemid, _ := getGemPortID(intfid, onuid)
	pkt := unipkt.Pkt
	layerEth := pkt.Layer(layers.LayerTypeEthernet)
	le, _ := layerEth.(*layers.Ethernet)
	ethtype := le.EthernetType
	if ethtype == 0x888e {
	log.Printf("Received upstream packet is EAPOL.")
	log.Println(unipkt.Pkt.Dump())
	log.Println(pkt.Dump())
	} else if layerDHCP := pkt.Layer(layers.LayerTypeDHCPv4); layerDHCP != nil {
	log.Printf("Received upstream packet is DHCP.")
	log.Println(unipkt.Pkt.Dump())
	log.Println(pkt.Dump())
	} else {
	continue
	}
	log.Printf("sendPktInd intfid:%d (onuid: %d) gemid:%d\n", intfid, onuid, gemid)
	data = &openolt.Indication_PktInd{PktInd: &openolt.PacketIndication{IntfType: "pon", IntfId: intfid, GemportId: gemid, Pkt: pkt.Data()}}
	if err := stream.Send(&openolt.Indication{Data: data}); err != nil {
	log.Printf("[ERROR] Failed to send PktInd indication. %v\n", err)
	return err
	}
	case nnipkt := <-nnichannel:
	if nnipkt.Info == nil \|\| nnipkt.Info.iotype != "nni" {
	log.Println("[WARNING] This packet does not come from NNI !")
	continue
	}
	log.Println("Received packet in grpc Server from NNI.")
	intfid := nnipkt.Info.intfid
	pkt := nnipkt.Pkt
	log.Printf("sendPktInd intfid:%d\n", intfid)
	data = &openolt.Indication_PktInd{PktInd: &openolt.PacketIndication{IntfType: "nni", IntfId: intfid, Pkt: pkt.Data()}}
	if err := stream.Send(&openolt.Indication{Data: data}); err != nil {
	log.Printf("[ERROR] Failed to send PktInd indication. %v\n", err)
	return err
	}
	case <-s.Endchan:
	if flag == false {
	log.Println("PacketInDaemon thread receives close !")
	close(unichannel)
	log.Println("Closed unichannel !")
	close(nnichannel)
	log.Println("Closed nnichannel !")
	flag = true
	return nil
	}
	}
	}
	return nil
	}

	func (s *Server) exeAAATest(vethenv []string) error {
	log.Println("exeAAATest Start")
	for i := 0; i < s.AAAWait; i++ {
	select {
	case <-s.Endchan:
	log.Println("exeAAATest thread receives close !")
	return nil
	default:
	log.Println("exeAAATest is now sleeping....")
	time.Sleep(time.Second)
	}
	}
	err := setup.ActivateWPASups(vethenv)
	if err != nil {
	return err
	}
	return nil
	}

	func (s *Server) onuPacketOut(intfid uint32, onuid uint32, rawpkt gopacket.Packet) error {
	layerEth := rawpkt.Layer(layers.LayerTypeEthernet)
	if layerEth != nil {
	pkt, _ := layerEth.(*layers.Ethernet)
	ethtype := pkt.EthernetType
	if ethtype == 0x888e {
	log.Printf("Received downstream packet is EAPOL.")
	log.Println(rawpkt.Dump())
	} else if layerDHCP := rawpkt.Layer(layers.LayerTypeDHCPv4); layerDHCP != nil {
	log.Printf("Received downstream packet is DHCP.")
	log.Println(rawpkt.Dump())
	rawpkt, _, _ = PopVLAN(rawpkt)
	rawpkt, _, _ = PopVLAN(rawpkt)
	} else {
	return nil
	}
	ioinfo, err := s.identifyUniIoinfo("inside", intfid, onuid)
	if err != nil {
	return err
	}
	handle := ioinfo.handler
	SendUni(handle, rawpkt)
	return nil
	}
	log.Printf("[WARNING] Received packet is not supported")
	return nil
	}

	func (s *Server) uplinkPacketOut(rawpkt gopacket.Packet) error {
	log.Println("")
	poppkt, _, err := PopVLAN(rawpkt)
	poppkt, _, err = PopVLAN(poppkt)
	if err != nil {
	log.Println(err)
	return err
	}
	ioinfo, err := s.identifyNniIoinfo("inside")
	if err != nil {
	return err
	}
	handle := ioinfo.handler
	SendNni(handle, poppkt)
	return nil
	}

	func (s *Server) IsAllONUActive() bool {
	for _, onus := range s.Onumap {
	for _, onu := range onus {
	if onu.InternalState != device.ONU_ACTIVATED {
	return false
	}
	}
	}
	return true
	}

	func getVID(onuid uint32) (uint16, error) {
	return uint16(onuid), nil
	}

	func getGemPortID(intfid uint32, onuid uint32) (uint32, error) {
	idx := uint32(0)
	return 1024 + (((MAX_ONUS_PER_PONintfid + onuid - 1) 7) + idx), nil
	//return uint32(1032 + 8 * (vid - 1)), nil
	}

	func (s Server) getOnuBySN(sn openolt.SerialNumber) (*device.Onu, error) {
	for _, onus := range s.Onumap {
	for _, onu := range onus {
	if device.ValidateSN(sn, onu.SerialNumber) {
	return onu, nil
	}
	}
	}
	err := errors.New("No mathced SN is found !")
	log.Println(err)
	return nil, err
	}

	func (s Server) getOnuByID(onuid uint32) (device.Onu, error) {
	for _, onus := range s.Onumap {
	for _, onu := range onus {
	if onu.OnuID == onuid {
	return onu, nil
	}
	}
	}
	err := errors.New("No matched OnuID is found !")
	log.Println(err)
	return nil, err
	}

	func makeUniName(oltid uint32, intfid uint32, onuid uint32) (upif string, dwif string) {
	upif = setup.UNI_VETH_UP_PFX + strconv.Itoa(int(oltid)) + "_" + strconv.Itoa(int(intfid)) + "_" + strconv.Itoa(int(onuid))
	dwif = setup.UNI_VETH_DW_PFX + strconv.Itoa(int(oltid)) + "_" + strconv.Itoa(int(intfid)) + "_" + strconv.Itoa(int(onuid))
	return
	}

	func makeNniName(oltid uint32) (upif string, dwif string) {
	upif = setup.NNI_VETH_UP_PFX + strconv.Itoa(int(oltid))
	dwif = setup.NNI_VETH_DW_PFX + strconv.Itoa(int(oltid))
	return
	}

	func tearDown(vethenv []string) error {
	log.Println("tearDown()")
	setup.KillAllWPASups()
	setup.KillAllDHCPClients()
	setup.TearVethDown(vethenv)
	return nil
	}

	func setupVethHandler(inveth string, outveth string, vethenv []string) (*pcap.Handle, []string, error) {
	log.Printf("setupVethHandler: %s and %s\n", inveth, outveth)
	err1 := setup.CreateVethPairs(inveth, outveth)
	vethenv = append(vethenv, inveth)
	if err1 != nil {
	setup.RemoveVeths(vethenv)
	return nil, vethenv, err1
	}
	handler, err2 := getVethHandler(inveth)
	if err2 != nil {
	setup.RemoveVeths(vethenv)
	return nil, vethenv, err2
	}
	return handler, vethenv, nil
	}

	func getVethHandler(vethname string) (*pcap.Handle, error) {
	var (
	device string = vethname
	snapshot_len int32 = 1518
	promiscuous bool = false
	err error
	timeout time.Duration = pcap.BlockForever
	)
	handle, err := pcap.OpenLive(device, snapshot_len, promiscuous, timeout)
	if err != nil {
	return nil, err
	}
	log.Printf("Server handle created for %s\n", vethname)
	return handle, nil
	}