VOL-2518 - reconnect to voltha on disconnect
Change-Id: Ia497bb6a83312f15e54de0d7556753e1d9ef58b0
diff --git a/internal/pkg/ofagent/changeEvent.go b/internal/pkg/ofagent/changeEvent.go
index 69b14ff..20c55de 100644
--- a/internal/pkg/ofagent/changeEvent.go
+++ b/internal/pkg/ofagent/changeEvent.go
@@ -29,38 +29,51 @@
func (ofa *OFAgent) receiveChangeEvents(ctx context.Context) {
logger.Debug("receive-change-events-started")
+ // If we exit, assume disconnected
+ defer func() {
+ ofa.events <- ofaEventVolthaDisconnected
+ logger.Debug("receive-change-events-finished")
+ }()
+ if ofa.volthaClient == nil {
+ logger.Error("no-voltha-connection")
+ return
+ }
opt := grpc.EmptyCallOption{}
streamCtx, streamDone := context.WithCancel(context.Background())
+ defer streamDone()
stream, err := ofa.volthaClient.ReceiveChangeEvents(streamCtx, &empty.Empty{}, opt)
if err != nil {
logger.Errorw("Unable to establish Receive Change Event Stream",
log.Fields{"error": err})
- ofa.events <- ofaEventVolthaDisconnected
+ return
}
- defer streamDone()
+top:
for {
select {
case <-ctx.Done():
- return
+ break top
default:
- if ce, err := stream.Recv(); err != nil {
+ ce, err := stream.Recv()
+ if err != nil {
logger.Errorw("error receiving change event",
log.Fields{"error": err})
- ofa.events <- ofaEventVolthaDisconnected
- } else {
- ofa.changeEventChannel <- ce
+ break top
}
+ ofa.changeEventChannel <- ce
+ logger.Debug("receive-change-event-queued")
}
}
}
func (ofa *OFAgent) handleChangeEvents(ctx context.Context) {
- logger.Debugln("handle-change-event-started")
+ logger.Debug("handle-change-event-started")
+
+top:
for {
select {
case <-ctx.Done():
- return
+ break top
case changeEvent := <-ofa.changeEventChannel:
deviceID := changeEvent.GetId()
portStatus := changeEvent.GetPortStatus()
@@ -107,4 +120,6 @@
ofa.getOFClient(deviceID).SendMessage(ofPortStatus)
}
}
+
+ logger.Debug("handle-change-event-finsihed")
}
diff --git a/internal/pkg/ofagent/ofagent.go b/internal/pkg/ofagent/ofagent.go
index ea3b091..25041ab 100644
--- a/internal/pkg/ofagent/ofagent.go
+++ b/internal/pkg/ofagent/ofagent.go
@@ -18,6 +18,7 @@
import (
"context"
+ "fmt"
"github.com/opencord/ofagent-go/internal/pkg/openflow"
"github.com/opencord/voltha-lib-go/v3/pkg/log"
"github.com/opencord/voltha-lib-go/v3/pkg/probe"
@@ -39,6 +40,7 @@
ofaEventError
ofaStateConnected = ofaState(iota)
+ ofaStateConnecting
ofaStateDisconnected
)
@@ -147,6 +149,7 @@
// Kick off process to attempt to establish
// connection to voltha
+ state = ofaStateConnecting
go ofa.establishConnectionToVoltha(p)
case ofaEventVolthaConnected:
@@ -156,6 +159,16 @@
if state != ofaStateConnected {
state = ofaStateConnected
volthaCtx, volthaDone = context.WithCancel(context.Background())
+ // Reconnect clients
+ for _, client := range ofa.clientMap {
+ if logger.V(log.DebugLevel) {
+ logger.Debugw("reset-client-voltha-connection",
+ log.Fields{
+ "from": fmt.Sprintf("0x%p", &client.VolthaClient),
+ "to": fmt.Sprintf("0x%p", &ofa.volthaClient)})
+ }
+ client.VolthaClient = ofa.volthaClient
+ }
go ofa.receiveChangeEvents(volthaCtx)
go ofa.receivePacketsIn(volthaCtx)
go ofa.streamPacketOut(volthaCtx)
@@ -163,13 +176,31 @@
}
case ofaEventVolthaDisconnected:
+ if p != nil {
+ p.UpdateStatus("voltha", probe.ServiceStatusNotReady)
+ }
logger.Debug("ofagent-voltha-disconnect-event")
if state == ofaStateConnected {
state = ofaStateDisconnected
+ ofa.volthaClient = nil
+ for _, client := range ofa.clientMap {
+ client.VolthaClient = nil
+ if logger.V(log.DebugLevel) {
+ logger.Debugw("reset-client-voltha-connection",
+ log.Fields{
+ "from": fmt.Sprintf("0x%p", &client.VolthaClient),
+ "to": "nil"})
+ }
+ }
volthaDone()
volthaDone = nil
volthaCtx = nil
}
+ if state != ofaStateConnecting {
+ state = ofaStateConnecting
+ go ofa.establishConnectionToVoltha(p)
+ }
+
case ofaEventError:
logger.Debug("ofagent-error-event")
default:
diff --git a/internal/pkg/ofagent/packetIn.go b/internal/pkg/ofagent/packetIn.go
index 324d568..175a9c8 100644
--- a/internal/pkg/ofagent/packetIn.go
+++ b/internal/pkg/ofagent/packetIn.go
@@ -31,37 +31,50 @@
func (ofa *OFAgent) receivePacketsIn(ctx context.Context) {
logger.Debug("receive-packets-in-started")
+ // If we exit, assume disconnected
+ defer func() {
+ ofa.events <- ofaEventVolthaDisconnected
+ logger.Debug("receive-packets-in-finished")
+ }()
+ if ofa.volthaClient == nil {
+ logger.Error("no-voltha-connection")
+ return
+ }
opt := grpc.EmptyCallOption{}
streamCtx, streamDone := context.WithCancel(context.Background())
+ defer streamDone()
stream, err := ofa.volthaClient.ReceivePacketsIn(streamCtx, &empty.Empty{}, opt)
if err != nil {
logger.Errorw("Unable to establish Receive PacketIn Stream",
log.Fields{"error": err})
+ return
}
- defer streamDone()
+
+top:
for {
select {
case <-ctx.Done():
- return
+ break top
default:
- if pkt, err := stream.Recv(); err != nil {
+ pkt, err := stream.Recv()
+ if err != nil {
logger.Errorw("error receiving packet",
log.Fields{"error": err})
- ofa.events <- ofaEventVolthaDisconnected
- } else {
- ofa.packetInChannel <- pkt
+ break top
}
+ ofa.packetInChannel <- pkt
}
}
}
func (ofa *OFAgent) handlePacketsIn(ctx context.Context) {
logger.Debug("handle-packets-in-started")
+top:
for {
select {
case <-ctx.Done():
- return
+ break top
case packet := <-ofa.packetInChannel:
packetIn := packet.GetPacketIn()
@@ -157,4 +170,5 @@
}
}
+ logger.Debug("handle-packets-in-finished")
}
diff --git a/internal/pkg/ofagent/packetOut.go b/internal/pkg/ofagent/packetOut.go
index 30466a6..928d19c 100644
--- a/internal/pkg/ofagent/packetOut.go
+++ b/internal/pkg/ofagent/packetOut.go
@@ -24,8 +24,15 @@
)
func (ofa *OFAgent) streamPacketOut(ctx context.Context) {
- if logger.V(log.DebugLevel) {
- logger.Debug("GrpcClient streamPacketOut called")
+ logger.Debug("packet-out-started")
+ // If we exit, assume disconnected
+ defer func() {
+ ofa.events <- ofaEventVolthaDisconnected
+ logger.Debug("packet-out-finished")
+ }()
+ if ofa.volthaClient == nil {
+ logger.Error("no-voltha-connection")
+ return
}
opt := grpc.EmptyCallOption{}
streamCtx, streamDone := context.WithCancel(context.Background())
@@ -33,18 +40,24 @@
defer streamDone()
if err != nil {
logger.Errorw("streamPacketOut Error creating packetout stream ", log.Fields{"error": err})
- ofa.events <- ofaEventVolthaDisconnected
+ return
}
+top:
for {
select {
case <-ctx.Done():
- return
+ break top
case ofPacketOut := <-ofa.packetOutChannel:
if logger.V(log.DebugLevel) {
js, _ := json.Marshal(ofPacketOut)
logger.Debugw("streamPacketOut Receive PacketOut from Channel", log.Fields{"PacketOut": js})
}
- outClient.Send(ofPacketOut)
+ if err := outClient.Send(ofPacketOut); err != nil {
+ logger.Errorw("packet-out-send-error",
+ log.Fields{"error": err.Error()})
+ break top
+ }
+ logger.Debug("packet-out-send")
}
}
}
diff --git a/internal/pkg/ofagent/refresh.go b/internal/pkg/ofagent/refresh.go
index 275ba2e..d50b3ec 100644
--- a/internal/pkg/ofagent/refresh.go
+++ b/internal/pkg/ofagent/refresh.go
@@ -42,10 +42,17 @@
}
func (ofa *OFAgent) refreshDeviceList() {
+ // If we exit, assume disconnected
+ if ofa.volthaClient == nil {
+ logger.Error("no-voltha-connection")
+ ofa.events <- ofaEventVolthaDisconnected
+ return
+ }
deviceList, err := ofa.volthaClient.ListLogicalDevices(context.Background(), &empty.Empty{})
if err != nil {
logger.Errorw("ofagent failed to query device list from voltha",
log.Fields{"error": err})
+ ofa.events <- ofaEventVolthaDisconnected
return
}
devices := deviceList.GetItems()
diff --git a/internal/pkg/openflow/client.go b/internal/pkg/openflow/client.go
index eded9e8..97df5d2 100644
--- a/internal/pkg/openflow/client.go
+++ b/internal/pkg/openflow/client.go
@@ -32,6 +32,7 @@
)
var logger, _ = log.AddPackage(log.JSON, log.DebugLevel, nil)
+var NoVolthaConnectionError = errors.New("no-voltha-connection")
type ofcEvent byte
type ofcState byte
@@ -492,15 +493,27 @@
if err := ofc.doSend(msg); err != nil {
ofc.lastUnsentMessage = msg
ofc.events <- ofcEventDisconnect
- return
+ logger.Debugw("message-sender-error",
+ log.Fields{
+ "device-id": ofc.DeviceID,
+ "error": err.Error()})
+ break top
}
+ logger.Debugw("message-sender-send",
+ log.Fields{
+ "device-id": ofc.DeviceID})
ofc.lastUnsentMessage = nil
}
}
+
+ logger.Debugw("message-sender-finished",
+ log.Fields{
+ "device-id": ofc.DeviceID})
}
// SendMessage queues a message to be sent to the openflow controller
func (ofc *OFClient) SendMessage(message Message) error {
+ logger.Debug("queuing-message")
ofc.sendChannel <- message
return nil
}
diff --git a/internal/pkg/openflow/feature.go b/internal/pkg/openflow/feature.go
index 54de0d0..f456175 100644
--- a/internal/pkg/openflow/feature.go
+++ b/internal/pkg/openflow/feature.go
@@ -32,6 +32,9 @@
"device-id": ofc.DeviceID,
"request": js})
}
+ if ofc.VolthaClient == nil {
+ return NoVolthaConnectionError
+ }
var id = common.ID{Id: ofc.DeviceID}
logicalDevice, err := ofc.VolthaClient.GetLogicalDevice(context.Background(), &id)
reply := ofp.NewFeaturesReply()
diff --git a/internal/pkg/openflow/flowMod.go b/internal/pkg/openflow/flowMod.go
index 18390d1..4ab1afc 100644
--- a/internal/pkg/openflow/flowMod.go
+++ b/internal/pkg/openflow/flowMod.go
@@ -77,6 +77,12 @@
"params": js})
}
+ if ofc.VolthaClient == nil {
+ logger.Errorw("no-voltha-connection",
+ log.Fields{"device-id": ofc.DeviceID})
+ return
+ }
+
// Construct the match
var oxmList []*voltha.OfpOxmField
for _, oxmField := range flowAdd.Match.GetOxmList() {
@@ -257,6 +263,12 @@
"flow-delete-strict": js})
}
+ if ofc.VolthaClient == nil {
+ logger.Errorw("no-voltha-connection",
+ log.Fields{"device-id": ofc.DeviceID})
+ return
+ }
+
// Construct match
var oxmList []*voltha.OfpOxmField
for _, oxmField := range flowDeleteStrict.Match.GetOxmList() {
diff --git a/internal/pkg/openflow/meter.go b/internal/pkg/openflow/meter.go
index d4299ba..61ded75 100644
--- a/internal/pkg/openflow/meter.go
+++ b/internal/pkg/openflow/meter.go
@@ -32,6 +32,12 @@
"request": js})
}
+ if ofc.VolthaClient == nil {
+ logger.Errorw("no-voltha-connection",
+ log.Fields{"device-id": ofc.DeviceID})
+ return
+ }
+
meterModUpdate := openflow_13.MeterModUpdate{Id: ofc.DeviceID}
meterMod := openflow_13.OfpMeterMod{
MeterId: request.MeterId,
diff --git a/internal/pkg/openflow/stats.go b/internal/pkg/openflow/stats.go
index 4617598..97d389b 100644
--- a/internal/pkg/openflow/stats.go
+++ b/internal/pkg/openflow/stats.go
@@ -288,6 +288,9 @@
}
func (ofc *OFClient) handleDescStatsRequest(request *ofp.DescStatsRequest) (*ofp.DescStatsReply, error) {
+ if ofc.VolthaClient == nil {
+ return nil, NoVolthaConnectionError
+ }
response := ofp.NewDescStatsReply()
response.SetXid(request.GetXid())
response.SetVersion(request.GetVersion())
@@ -309,6 +312,9 @@
}
func (ofc *OFClient) handleFlowStatsRequest(request *ofp.FlowStatsRequest) (*ofp.FlowStatsReply, error) {
+ if ofc.VolthaClient == nil {
+ return nil, NoVolthaConnectionError
+ }
response := ofp.NewFlowStatsReply()
response.SetXid(request.GetXid())
response.SetVersion(4)
@@ -382,6 +388,9 @@
}
func (ofc *OFClient) handleGroupStatsRequest(request *ofp.GroupStatsRequest) (*ofp.GroupStatsReply, error) {
+ if ofc.VolthaClient == nil {
+ return nil, NoVolthaConnectionError
+ }
response := ofp.NewGroupStatsReply()
response.SetVersion(request.GetVersion())
response.SetXid(request.GetXid())
@@ -417,6 +426,9 @@
}
func (ofc *OFClient) handleGroupStatsDescRequest(request *ofp.GroupDescStatsRequest) (*ofp.GroupDescStatsReply, error) {
+ if ofc.VolthaClient == nil {
+ return nil, NoVolthaConnectionError
+ }
response := ofp.NewGroupDescStatsReply()
response.SetVersion(request.GetVersion())
response.SetXid(request.GetXid())
@@ -456,6 +468,9 @@
}
func (ofc *OFClient) handleMeterStatsRequest(request *ofp.MeterStatsRequest) (*ofp.MeterStatsReply, error) {
+ if ofc.VolthaClient == nil {
+ return nil, NoVolthaConnectionError
+ }
response := ofp.NewMeterStatsReply()
response.SetVersion(request.GetVersion())
response.SetXid(request.GetXid())
@@ -531,6 +546,9 @@
}
func (ofc *OFClient) handlePortStatsRequest(request *ofp.PortStatsRequest) (*ofp.PortStatsReply, error) {
+ if ofc.VolthaClient == nil {
+ return nil, NoVolthaConnectionError
+ }
response := ofp.NewPortStatsReply()
response.SetXid(request.GetXid())
response.SetVersion(request.GetVersion())
@@ -557,6 +575,9 @@
}
func (ofc *OFClient) handlePortDescStatsRequest(request *ofp.PortDescStatsRequest) (*ofp.PortDescStatsReply, error) {
+ if ofc.VolthaClient == nil {
+ return nil, NoVolthaConnectionError
+ }
response := ofp.NewPortDescStatsReply()
response.SetVersion(request.GetVersion())
response.SetXid(request.GetXid())