Offload of incoming EAPOL packet to executor
Re-initializing the secret value when changed for RADIUS server check status

Change-Id: I6854d74cd7ca20854293139d5acf179700284b74
diff --git a/app/src/main/java/org/opencord/aaa/impl/AaaManager.java b/app/src/main/java/org/opencord/aaa/impl/AaaManager.java
index 82828fa..cebf6aa 100644
--- a/app/src/main/java/org/opencord/aaa/impl/AaaManager.java
+++ b/app/src/main/java/org/opencord/aaa/impl/AaaManager.java
@@ -18,7 +18,7 @@
 import com.google.common.base.Strings;
 import com.google.common.collect.Maps;
 
-import static java.util.concurrent.Executors.newFixedThreadPool;
+import static java.util.concurrent.Executors.newSingleThreadExecutor;
 import static org.onlab.util.Tools.groupedThreads;
 import static org.onosproject.net.config.basics.SubjectFactories.APP_SUBJECT_FACTORY;
 import static org.opencord.aaa.impl.OsgiPropertyConstants.*;
@@ -388,8 +388,7 @@
             if (packetProcessorExecutor != null) {
                 packetProcessorExecutor.shutdown();
             }
-            packetProcessorExecutor = newFixedThreadPool(packetProcessorThreads,
-                                                         groupedThreads("onos/aaa", "aaa-packet-%d", log));
+            packetProcessorExecutor = newSingleThreadExecutor(groupedThreads("onos/aaa", "aaa-packet-%d", log));
 
         }
     }
@@ -491,169 +490,175 @@
      * @param radiusPacket RADIUS packet coming from the RADIUS server.
      */
     public void handleRadiusPacket(RADIUS radiusPacket) {
-        packetProcessorExecutor.execute(() -> {
-            if (log.isTraceEnabled()) {
-                log.trace("Received RADIUS packet {}", radiusPacket);
-            }
-            if (radiusOperationalStatusService.isRadiusResponseForOperationalStatus(radiusPacket.getIdentifier())) {
-                radiusOperationalStatusService.handleRadiusPacketForOperationalStatus(radiusPacket);
-                return;
-            }
+        if (log.isTraceEnabled()) {
+            log.trace("Received RADIUS packet {}", radiusPacket);
+        }
+        if (radiusOperationalStatusService.isRadiusResponseForOperationalStatus(radiusPacket.getIdentifier())) {
+            radiusOperationalStatusService.handleRadiusPacketForOperationalStatus(radiusPacket);
+            return;
+        }
 
-            RequestIdentifier identifier = RequestIdentifier.of(radiusPacket.getIdentifier());
-            String sessionId = idManager.getSessionId(identifier);
+        RequestIdentifier identifier = RequestIdentifier.of(radiusPacket.getIdentifier());
+        String sessionId = idManager.getSessionId(identifier);
 
-            if (sessionId == null) {
-                log.error("Invalid packet identifier {}, could not find corresponding "
-                                  + "state machine ... exiting", radiusPacket.getIdentifier());
-                aaaStatisticsManager.getAaaStats().incrementNumberOfSessionsExpired();
-                aaaStatisticsManager.getAaaStats().countDroppedResponsesRx();
-                return;
-            }
+        if (sessionId == null) {
+            log.error("Invalid packet identifier {}, could not find corresponding "
+                              + "state machine ... exiting", radiusPacket.getIdentifier());
+            aaaStatisticsManager.getAaaStats().incrementNumberOfSessionsExpired();
+            aaaStatisticsManager.getAaaStats().countDroppedResponsesRx();
+            return;
+        }
 
-            idManager.releaseIdentifier(identifier);
-            StateMachine stateMachine = stateMachines.get(sessionId);
-            if (stateMachine == null) {
-                log.error("Invalid packet identifier {}, could not find corresponding "
-                                  + "state machine ... exiting", radiusPacket.getIdentifier());
-                aaaStatisticsManager.getAaaStats().incrementNumberOfSessionsExpired();
-                aaaStatisticsManager.getAaaStats().countDroppedResponsesRx();
-                return;
-            }
+        idManager.releaseIdentifier(identifier);
+        StateMachine stateMachine = stateMachines.get(sessionId);
+        if (stateMachine == null) {
+            log.error("Invalid packet identifier {}, could not find corresponding "
+                              + "state machine ... exiting", radiusPacket.getIdentifier());
+            aaaStatisticsManager.getAaaStats().incrementNumberOfSessionsExpired();
+            aaaStatisticsManager.getAaaStats().countDroppedResponsesRx();
+            return;
+        }
 
-            //instance of StateMachine using the sessionId for updating machine stats
-            StateMachine machineStats = stateMachines.get(stateMachine.sessionId());
+        //instance of StateMachine using the sessionId for updating machine stats
+        StateMachine machineStats = stateMachines.get(stateMachine.sessionId());
 
-            EAP eapPayload;
-            Ethernet eth;
-            checkReceivedPacketForValidValidator(radiusPacket, stateMachine.requestAuthenticator());
+        EAP eapPayload;
+        Ethernet eth;
+        checkReceivedPacketForValidValidator(radiusPacket, stateMachine.requestAuthenticator());
 
-            //increasing packets and octets received from server
-            machineStats.incrementTotalPacketsReceived();
-            try {
-                machineStats.incrementTotalOctetReceived(radiusPacket.decapsulateMessage().getLength());
-            } catch (DeserializationException e) {
-                log.error(e.getMessage());
-                return;
-            }
+        //increasing packets and octets received from server
+        machineStats.incrementTotalPacketsReceived();
+        try {
+            machineStats.incrementTotalOctetReceived(radiusPacket.decapsulateMessage().getLength());
+        } catch (DeserializationException e) {
+            log.error(e.getMessage());
+            return;
+        }
 
-            if (outPacketSet.contains(radiusPacket.getIdentifier())) {
-                aaaStatisticsManager.getAaaStats().increaseOrDecreasePendingRequests(false);
-                outPacketSet.remove(new Byte(radiusPacket.getIdentifier()));
-            }
+        if (outPacketSet.contains(radiusPacket.getIdentifier())) {
+            aaaStatisticsManager.getAaaStats().increaseOrDecreasePendingRequests(false);
+            outPacketSet.remove(new Byte(radiusPacket.getIdentifier()));
+        }
 
-            switch (radiusPacket.getCode()) {
-                case RADIUS.RADIUS_CODE_ACCESS_CHALLENGE:
-                    log.debug("RADIUS packet: RADIUS_CODE_ACCESS_CHALLENGE");
-                    RADIUSAttribute radiusAttrState = radiusPacket.getAttribute(RADIUSAttribute.RADIUS_ATTR_STATE);
-                    byte[] challengeState = null;
-                    if (radiusAttrState != null) {
-                        challengeState = radiusAttrState.getValue();
-                    }
-                    try {
-                        eapPayload = radiusPacket.decapsulateMessage();
-                        eth = buildEapolResponse(stateMachine.supplicantAddress(),
-                                                 MacAddress.valueOf(nasMacAddress),
-                                                 stateMachine.vlanId(),
-                                                 EAPOL.EAPOL_PACKET,
-                                                 eapPayload, stateMachine.priorityCode());
-                        stateMachine.setChallengeInfo(eapPayload.getIdentifier(), challengeState);
-                    } catch (DeserializationException e) {
-                        log.error(e.getMessage());
-                        break;
-                    }
-                    log.debug("Send EAP challenge response to supplicant {}",
-                              stateMachine.supplicantAddress().toString());
-                    sendPacketToSupplicant(eth, stateMachine.supplicantConnectpoint(), true);
-                    aaaStatisticsManager.getAaaStats().increaseChallengeResponsesRx();
-                    outPacketSupp.add(eapPayload.getIdentifier());
-                    aaaStatisticsManager.getAaaStats().incrementPendingResSupp();
-                    //increasing packets send to server
-                    machineStats.incrementTotalPacketsSent();
-                    machineStats.incrementTotalOctetSent(eapPayload.getLength());
+        MacAddress dstMac = stateMachine.supplicantAddress();
+        ConnectPoint supplicantCp = stateMachine.supplicantConnectpoint();
+        switch (radiusPacket.getCode()) {
+            case RADIUS.RADIUS_CODE_ACCESS_CHALLENGE:
+                log.debug("RADIUS packet: RADIUS_CODE_ACCESS_CHALLENGE for dev/port: {}/{} with MacAddress {}",
+                          supplicantCp.deviceId(), supplicantCp.port(), dstMac);
+                RADIUSAttribute radiusAttrState = radiusPacket.getAttribute(RADIUSAttribute.RADIUS_ATTR_STATE);
+                byte[] challengeState = null;
+                if (radiusAttrState != null) {
+                    challengeState = radiusAttrState.getValue();
+                }
+                try {
+                    eapPayload = radiusPacket.decapsulateMessage();
+                    eth = buildEapolResponse(stateMachine.supplicantAddress(),
+                                             MacAddress.valueOf(nasMacAddress),
+                                             stateMachine.vlanId(),
+                                             EAPOL.EAPOL_PACKET,
+                                             eapPayload, stateMachine.priorityCode());
+                    stateMachine.setChallengeInfo(eapPayload.getIdentifier(), challengeState);
+                } catch (DeserializationException e) {
+                    log.error(e.getMessage());
                     break;
-                case RADIUS.RADIUS_CODE_ACCESS_ACCEPT:
-                    log.debug("RADIUS packet: RADIUS_CODE_ACCESS_ACCEPT");
-                    //send an EAPOL - Success to the supplicant.
-                    byte[] eapMessageSuccess =
-                            radiusPacket.getAttribute(RADIUSAttribute.RADIUS_ATTR_EAP_MESSAGE).getValue();
-                    try {
+                }
+                log.debug("Send EAP challenge response to supplicant {} on dev/port: {}/{} with MacAddress {}",
+                          supplicantCp.deviceId(), supplicantCp.port(), dstMac);
+                sendPacketToSupplicant(eth, stateMachine.supplicantConnectpoint(), true);
+                aaaStatisticsManager.getAaaStats().increaseChallengeResponsesRx();
+                outPacketSupp.add(eapPayload.getIdentifier());
+                aaaStatisticsManager.getAaaStats().incrementPendingResSupp();
+                //increasing packets send to server
+                machineStats.incrementTotalPacketsSent();
+                machineStats.incrementTotalOctetSent(eapPayload.getLength());
+                break;
+            case RADIUS.RADIUS_CODE_ACCESS_ACCEPT:
+                log.debug("RADIUS packet: RADIUS_CODE_ACCESS_ACCEPT for dev/port: {}/{} with MacAddress {}",
+                          supplicantCp.deviceId(), supplicantCp.port(), dstMac);
+                //send an EAPOL - Success to the supplicant.
+                byte[] eapMessageSuccess =
+                        radiusPacket.getAttribute(RADIUSAttribute.RADIUS_ATTR_EAP_MESSAGE).getValue();
+                try {
                     eapPayload = EAP.deserializer().deserialize(
                             eapMessageSuccess, 0, eapMessageSuccess.length);
-                    } catch (DeserializationException e) {
-                        log.error(e.getMessage());
-                        break;
-                    }
-
-                    eth = buildEapolResponse(stateMachine.supplicantAddress(),
-                                             MacAddress.valueOf(nasMacAddress),
-                                             stateMachine.vlanId(),
-                                             EAPOL.EAPOL_PACKET,
-                                             eapPayload, stateMachine.priorityCode());
-                    log.info("Send EAP success message to supplicant {}", stateMachine.supplicantAddress().toString());
-                    sendPacketToSupplicant(eth, stateMachine.supplicantConnectpoint(), false);
-                    aaaStatisticsManager.getAaaStats().incrementEapolAuthSuccessTrans();
-
-                    stateMachine.authorizeAccess();
-                    aaaStatisticsManager.getAaaStats().increaseAcceptResponsesRx();
-                    //increasing packets send to server
-                    machineStats.incrementTotalPacketsSent();
-                    machineStats.incrementTotalOctetSent(eapPayload.getLength());
+                } catch (DeserializationException e) {
+                    log.error(e.getMessage());
                     break;
-                case RADIUS.RADIUS_CODE_ACCESS_REJECT:
-                    log.debug("RADIUS packet: RADIUS_CODE_ACCESS_REJECT");
-                    //send an EAPOL - Failure to the supplicant.
-                    byte[] eapMessageFailure;
-                    eapPayload = new EAP();
-                    RADIUSAttribute radiusAttrEap = radiusPacket.getAttribute(RADIUSAttribute.RADIUS_ATTR_EAP_MESSAGE);
-                    if (radiusAttrEap == null) {
-                        eapPayload.setCode(EAP.FAILURE);
-                        eapPayload.setIdentifier(stateMachine.challengeIdentifier());
-                        eapPayload.setLength(EAP.EAP_HDR_LEN_SUC_FAIL);
-                    } else {
-                        eapMessageFailure = radiusAttrEap.getValue();
-                        try {
-                            eapPayload = EAP.deserializer().deserialize(
-                                    eapMessageFailure, 0, eapMessageFailure.length);
-                        } catch (DeserializationException e) {
-                            log.error(e.getMessage());
-                            break;
-                        }
-                    }
-                    eth = buildEapolResponse(stateMachine.supplicantAddress(),
-                                             MacAddress.valueOf(nasMacAddress),
-                                             stateMachine.vlanId(),
-                                             EAPOL.EAPOL_PACKET,
-                                             eapPayload, stateMachine.priorityCode());
-                    log.warn("Send EAP failure message to supplicant {}", stateMachine.supplicantAddress().toString());
-                    sendPacketToSupplicant(eth, stateMachine.supplicantConnectpoint(), false);
-                    aaaStatisticsManager.getAaaStats().incrementEapolauthFailureTrans();
+                }
 
-                    stateMachine.denyAccess();
-                    aaaStatisticsManager.getAaaStats().increaseRejectResponsesRx();
-                    //increasing packets send to server
-                    machineStats.incrementTotalPacketsSent();
-                    machineStats.incrementTotalOctetSent(eapPayload.getLength());
-                    //pushing machine stats to kafka
-                    AaaSupplicantMachineStats machineObj = aaaSupplicantStatsManager.getSupplicantStats(machineStats);
-                    aaaSupplicantStatsManager.getMachineStatsDelegate()
-                            .notify(new AaaMachineStatisticsEvent(AaaMachineStatisticsEvent.Type.STATS_UPDATE,
-                                                                  machineObj));
-                    break;
-                default:
-                    log.warn("Unknown RADIUS message received with code: {}", radiusPacket.getCode());
-                    aaaStatisticsManager.getAaaStats().increaseUnknownTypeRx();
-                    //increasing packets received to server
-                    machineStats.incrementTotalPacketsReceived();
+                eth = buildEapolResponse(stateMachine.supplicantAddress(),
+                                         MacAddress.valueOf(nasMacAddress),
+                                         stateMachine.vlanId(),
+                                         EAPOL.EAPOL_PACKET,
+                                         eapPayload, stateMachine.priorityCode());
+                log.info("Send EAP success message to supplicant {} on dev/port: {}/{} with MacAddress {}",
+                         supplicantCp.deviceId(), supplicantCp.port(), dstMac);
+                sendPacketToSupplicant(eth, stateMachine.supplicantConnectpoint(), false);
+                aaaStatisticsManager.getAaaStats().incrementEapolAuthSuccessTrans();
+
+                stateMachine.authorizeAccess();
+                aaaStatisticsManager.getAaaStats().increaseAcceptResponsesRx();
+                //increasing packets send to server
+                machineStats.incrementTotalPacketsSent();
+                machineStats.incrementTotalOctetSent(eapPayload.getLength());
+                break;
+            case RADIUS.RADIUS_CODE_ACCESS_REJECT:
+                log.debug("RADIUS packet: RADIUS_CODE_ACCESS_REJECT for dev/port: {}/{} with MacAddress {}",
+                          supplicantCp.deviceId(), supplicantCp.port(), dstMac);
+                //send an EAPOL - Failure to the supplicant.
+                byte[] eapMessageFailure;
+                eapPayload = new EAP();
+                RADIUSAttribute radiusAttrEap = radiusPacket.getAttribute(RADIUSAttribute.RADIUS_ATTR_EAP_MESSAGE);
+                if (radiusAttrEap == null) {
+                    eapPayload.setCode(EAP.FAILURE);
+                    eapPayload.setIdentifier(stateMachine.challengeIdentifier());
+                    eapPayload.setLength(EAP.EAP_HDR_LEN_SUC_FAIL);
+                } else {
+                    eapMessageFailure = radiusAttrEap.getValue();
                     try {
-                        machineStats.incrementTotalOctetReceived(radiusPacket.decapsulateMessage().getLength());
+                        eapPayload = EAP.deserializer().deserialize(
+                                eapMessageFailure, 0, eapMessageFailure.length);
                     } catch (DeserializationException e) {
                         log.error(e.getMessage());
                         break;
                     }
-            }
-            aaaStatisticsManager.getAaaStats().countDroppedResponsesRx();
-        });
+                }
+                eth = buildEapolResponse(stateMachine.supplicantAddress(),
+                                         MacAddress.valueOf(nasMacAddress),
+                                         stateMachine.vlanId(),
+                                         EAPOL.EAPOL_PACKET,
+                                         eapPayload, stateMachine.priorityCode());
+                log.warn("Send EAP failure message to supplicant {} on dev/port: {}/{} with MacAddress {}",
+                         supplicantCp.deviceId(), supplicantCp.port(), dstMac);
+                sendPacketToSupplicant(eth, stateMachine.supplicantConnectpoint(), false);
+                aaaStatisticsManager.getAaaStats().incrementEapolauthFailureTrans();
+
+                stateMachine.denyAccess();
+                aaaStatisticsManager.getAaaStats().increaseRejectResponsesRx();
+                //increasing packets send to server
+                machineStats.incrementTotalPacketsSent();
+                machineStats.incrementTotalOctetSent(eapPayload.getLength());
+                //pushing machine stats to kafka
+                AaaSupplicantMachineStats machineObj = aaaSupplicantStatsManager.getSupplicantStats(machineStats);
+                aaaSupplicantStatsManager.getMachineStatsDelegate()
+                        .notify(new AaaMachineStatisticsEvent(AaaMachineStatisticsEvent.Type.STATS_UPDATE,
+                                                              machineObj));
+                break;
+            default:
+                log.warn("Unknown RADIUS message received with code: {} for dev/port: {}/{} with MacAddress {}",
+                         radiusPacket.getCode(), supplicantCp.deviceId(), supplicantCp.port(), dstMac);
+                aaaStatisticsManager.getAaaStats().increaseUnknownTypeRx();
+                //increasing packets received to server
+                machineStats.incrementTotalPacketsReceived();
+                try {
+                    machineStats.incrementTotalOctetReceived(radiusPacket.decapsulateMessage().getLength());
+                } catch (DeserializationException e) {
+                    log.error(e.getMessage());
+                    break;
+                }
+        }
+        aaaStatisticsManager.getAaaStats().countDroppedResponsesRx();
     }
 
     /**
@@ -668,8 +673,8 @@
                                                           treatment, ByteBuffer.wrap(ethernetPkt.serialize()));
         EAPOL eap = ((EAPOL) ethernetPkt.getPayload());
         if (log.isTraceEnabled()) {
-            log.trace("Sending eapol payload {} enclosed in {} to supplicant at {}",
-                      eap, ethernetPkt, connectPoint);
+            log.trace("Sending eapol payload {} enclosed in {} to supplicant at {} with MacAddress {}",
+                      eap, ethernetPkt, connectPoint, ethernetPkt.getDestinationMAC());
         }
         packetService.emit(packet);
         if (isChallengeResponse) {
@@ -722,23 +727,24 @@
     private class ReactivePacketProcessor implements PacketProcessor {
         @Override
         public void process(PacketContext context) {
+            packetProcessorExecutor.execute(() -> {
+                // Extract the original Ethernet frame from the packet information
+                InboundPacket pkt = context.inPacket();
+                Ethernet ethPkt = pkt.parsed();
+                if (ethPkt == null) {
+                    return;
+                }
 
-            // Extract the original Ethernet frame from the packet information
-            InboundPacket pkt = context.inPacket();
-            Ethernet ethPkt = pkt.parsed();
-            if (ethPkt == null) {
-                return;
-            }
-
-            // identify if incoming packet comes from supplicant (EAP) or RADIUS
-            switch (EthType.EtherType.lookup(ethPkt.getEtherType())) {
-                case EAPOL:
-                    handleSupplicantPacket(context.inPacket());
-                    break;
-                default:
-                    // any other packets let the specific implementation handle
-                    impl.handlePacketFromServer(context);
-            }
+                // identify if incoming packet comes from supplicant (EAP) or RADIUS
+                switch (EthType.EtherType.lookup(ethPkt.getEtherType())) {
+                    case EAPOL:
+                        handleSupplicantPacket(context.inPacket());
+                        break;
+                    default:
+                        // any other packets let the specific implementation handle
+                        impl.handlePacketFromServer(context);
+                }
+            });
         }
 
         /**
@@ -784,8 +790,9 @@
             EAPOL eapol = (EAPOL) ethPkt.getPayload();
             if (log.isTraceEnabled()) {
                 log.trace("Received EAPOL packet {} in enclosing packet {} from "
-                        + "dev/port: {}/{}", eapol, ethPkt, deviceId,
-                          portNumber);
+                        + "dev/port: {}/{} with MacAddress {}",
+                          eapol, ethPkt, deviceId,
+                          portNumber, srcMac);
             }
 
             short pktlen = eapol.getPacketLength();
@@ -809,7 +816,8 @@
 
             switch (eapol.getEapolType()) {
                 case EAPOL.EAPOL_START:
-                    log.debug("EAP packet: EAPOL_START");
+                    log.debug("EAP packet: EAPOL_START from dev/port: {}/{} with MacAddress {}",
+                              deviceId, portNumber, srcMac);
                     stateMachine.setSupplicantConnectpoint(inPacket.receivedFrom());
                     stateMachine.setSupplicantAddress(srcMac);
                     stateMachine.start();
@@ -831,7 +839,8 @@
 
                     break;
                 case EAPOL.EAPOL_LOGOFF:
-                    log.debug("EAP packet: EAPOL_LOGOFF");
+                    log.debug("EAP packet: EAPOL_LOGOFF from dev/port: {}/{} with MacAddress {}",
+                              deviceId, portNumber, srcMac);
                     //posting the machine stat data for current supplicant device.
                     if (stateMachine.getSessionTerminateReason() == null ||
                             stateMachine.getSessionTerminateReason().equals("")) {
@@ -863,7 +872,8 @@
                     switch (dataType) {
 
                         case EAP.ATTR_IDENTITY:
-                            log.debug("EAP packet: EAPOL_PACKET ATTR_IDENTITY");
+                            log.debug("EAP packet: EAPOL_PACKET ATTR_IDENTITY from dev/port: {}/{} with MacAddress {}",
+                                      deviceId, portNumber, srcMac);
                             //Setting the time of this response from RG, only when its not a re-transmission.
                             if (stateMachine.getLastPacketReceivedTime() == 0) {
                                stateMachine.setLastPacketReceivedTime(System.currentTimeMillis());
@@ -873,7 +883,12 @@
 
                             radiusPayload = getRadiusPayload(stateMachine, radiusIdentifier.identifier(), eapPacket);
                             radiusPayload = pktCustomizer.customizePacket(radiusPayload, inPacket);
-                            radiusPayload.addMessageAuthenticator(AaaManager.this.radiusSecret);
+                            radiusPayload.addMessageAuthenticator(radiusSecret);
+
+                            if (log.isTraceEnabled()) {
+                                log.trace("Sending ATTR_IDENTITY packet to RADIUS for supplicant at dev/port: " +
+                                                  "{}/{} with MacAddress {}", deviceId, portNumber, srcMac);
+                            }
 
                             sendRadiusPacket(radiusPayload, inPacket);
                             stateMachine.setWaitingForRadiusResponse(true);
@@ -888,7 +903,8 @@
                             break;
                         case EAP.ATTR_MD5:
                             stateMachine.setLastPacketReceivedTime(System.currentTimeMillis());
-                            log.debug("EAP packet: EAPOL_PACKET ATTR_MD5");
+                            log.debug("EAP packet: EAPOL_PACKET ATTR_MD5 from dev/port: {}/{} with MacAddress {}",
+                                      deviceId, portNumber, srcMac);
                             // verify if the EAP identifier corresponds to the
                             // challenge identifier from the client state
                             // machine.
@@ -902,18 +918,23 @@
                                     radiusPayload.setAttribute(RADIUSAttribute.RADIUS_ATTR_STATE,
                                             stateMachine.challengeState());
                                 }
-                                radiusPayload.addMessageAuthenticator(AaaManager.this.radiusSecret);
+                                radiusPayload.addMessageAuthenticator(radiusSecret);
                                 if (outPacketSupp.contains(eapPacket.getIdentifier())) {
                                     aaaStatisticsManager.getAaaStats().decrementPendingResSupp();
                                     outPacketSupp.remove(identifier);
                                 }
+                                if (log.isTraceEnabled()) {
+                                    log.trace("Sending ATTR_MD5 packet to RADIUS for supplicant at dev/port: " +
+                                                      "{}/{} with MacAddress {}", deviceId, portNumber, srcMac);
+                                }
                                 sendRadiusPacket(radiusPayload, inPacket);
                                 stateMachine.setWaitingForRadiusResponse(true);
                                 aaaStatisticsManager.getAaaStats().incrementEapolMd5RspChall();
                             }
                             break;
                         case EAP.ATTR_TLS:
-                            log.debug("EAP packet: EAPOL_PACKET ATTR_TLS");
+                            log.debug("EAP packet: EAPOL_PACKET ATTR_TLS from dev/port: {}/{} with MacAddress {}",
+                                      deviceId, portNumber, srcMac);
                             // request id access to RADIUS
                             radiusPayload = getRadiusPayload(stateMachine, radiusIdentifier.identifier(), eapPacket);
                             radiusPayload = pktCustomizer.customizePacket(radiusPayload, inPacket);
@@ -923,12 +944,15 @@
                                         stateMachine.challengeState());
                             }
                             stateMachine.setRequestAuthenticator(radiusPayload.generateAuthCode());
-
-                            radiusPayload.addMessageAuthenticator(AaaManager.this.radiusSecret);
+                            radiusPayload.addMessageAuthenticator(radiusSecret);
                             if (outPacketSupp.contains(eapPacket.getIdentifier())) {
                                 aaaStatisticsManager.getAaaStats().decrementPendingResSupp();
                                 outPacketSupp.remove(identifier);
                             }
+                            if (log.isTraceEnabled()) {
+                                log.trace("Sending ATTR_TLS packet to RADIUS for supplicant at dev/port: " +
+                                                  "{}/{} with MacAddress {}", deviceId, portNumber, srcMac);
+                            }
                             sendRadiusPacket(radiusPayload, inPacket);
                             stateMachine.setWaitingForRadiusResponse(true);
                             aaaStatisticsManager.getAaaStats().incrementEapolTlsRespChall();
@@ -939,12 +963,14 @@
 
                             break;
                         default:
-                            log.warn("Unknown EAP packet type");
+                            log.warn("Unknown EAP packet type from dev/port: {}/{} with MacAddress {}",
+                                     deviceId, portNumber, srcMac);
                             return;
                     }
                     break;
                 default:
-                    log.debug("Skipping EAPOL message {}", eapol.getEapolType());
+                    log.debug("Skipping EAPOL message {} from dev/port: {}/{} with MacAddress {}",
+                              eapol.getEapolType(), deviceId, portNumber, srcMac);
             }
             aaaStatisticsManager.getAaaStats().countTransRespNotNak();
             aaaStatisticsManager.getAaaStats().countEapolResIdentityMsgTrans();
@@ -1024,8 +1050,9 @@
             if (newCfg.nasMac() != null) {
                 nasMacAddress = newCfg.nasMac();
             }
-            if (newCfg.radiusSecret() != null) {
+            if (newCfg.radiusSecret() != null && !newCfg.radiusSecret().equals(radiusSecret)) {
                 radiusSecret = newCfg.radiusSecret();
+                radiusOperationalStatusService.initialize(nasIpAddress.getAddress(), radiusSecret, impl);
             }
 
             boolean reconfigureCustomizer = false;
diff --git a/app/src/main/java/org/opencord/aaa/impl/RadiusOperationalStatusManager.java b/app/src/main/java/org/opencord/aaa/impl/RadiusOperationalStatusManager.java
index 2fe5755..77e58c0 100644
--- a/app/src/main/java/org/opencord/aaa/impl/RadiusOperationalStatusManager.java
+++ b/app/src/main/java/org/opencord/aaa/impl/RadiusOperationalStatusManager.java
@@ -56,7 +56,8 @@
 
     private RadiusOperationalStatusEvaluationMode radiusOperationalStatusEvaluationMode;
 
-    private static final String DUMMY_USER = new String("dummy-user");
+    // FIXME the DUMMY_USER should be read from config (or netcfg)
+    private static final String DUMMY_USER = new String("user");
     private static final byte RADIUS_CODE_STATUS_REQUEST = (byte) 12;
     private long lastRadiusPacketInTimeInMillis;
 
diff --git a/app/src/test/java/org/opencord/aaa/impl/AaaManagerTest.java b/app/src/test/java/org/opencord/aaa/impl/AaaManagerTest.java
index 110ebec..83d5c42 100644
--- a/app/src/test/java/org/opencord/aaa/impl/AaaManagerTest.java
+++ b/app/src/test/java/org/opencord/aaa/impl/AaaManagerTest.java
@@ -204,45 +204,58 @@
 
         Ethernet startPacket = constructSupplicantStartPacket();
         sendPacket(startPacket);
-
-        Ethernet responsePacket = (Ethernet) fetchPacket(0);
-        checkRadiusPacket(aaaManager, responsePacket, EAP.ATTR_IDENTITY);
-
-        //  (2) Supplicant identify
-
-        Ethernet identifyPacket = constructSupplicantIdentifyPacket(null,
-                EAP.ATTR_IDENTITY, (byte) 3, null);
-        sendPacket(identifyPacket);
-
-        RADIUS radiusIdentifyPacket = (RADIUS) fetchPacket(1);
-        AtomicReference<Byte> reqId = new AtomicReference<>(radiusIdentifyPacket.getIdentifier());
-
-        checkRadiusPacketFromSupplicant(radiusIdentifyPacket);
-
-        assertThat(radiusIdentifyPacket.getCode(), is(RADIUS.RADIUS_CODE_ACCESS_REQUEST));
-        assertThat(new String(radiusIdentifyPacket.getAttribute(RADIUSAttribute.RADIUS_ATTR_USERNAME).getValue()),
-                   is("testuser"));
-
-        IpAddress nasIp =
-                IpAddress.valueOf(IpAddress.Version.INET,
-                                  radiusIdentifyPacket.getAttribute(RADIUSAttribute.RADIUS_ATTR_NAS_IP)
-                                          .getValue());
-        assertThat(nasIp.toString(), is(aaaManager.nasIpAddress.getHostAddress()));
-
-        //  State machine should have been created by now
-
-        StateMachine stateMachine = aaaManager.getStateMachine(SESSION_ID);
-        assertThat(stateMachine, notNullValue());
-        assertThat(stateMachine.state(), is(StateMachine.STATE_PENDING));
-
-        // (3) RADIUS MD5 challenge
-
-        RADIUS radiusCodeAccessChallengePacket =
-                constructRadiusCodeAccessChallengePacket(RADIUS.RADIUS_CODE_ACCESS_CHALLENGE, EAP.ATTR_MD5,
-                                                         reqId.get(), aaaManager.radiusSecret.getBytes());
-        aaaManager.handleRadiusPacket(radiusCodeAccessChallengePacket);
-
         assertAfter(ASSERTION_DELAY, ASSERTION_LENGTH, () -> {
+            Ethernet responsePacket = (Ethernet) fetchPacket(0);
+            checkRadiusPacket(aaaManager, responsePacket, EAP.ATTR_IDENTITY);
+
+            //  (2) Supplicant identify
+
+            Ethernet identifyPacket = null;
+            try {
+                identifyPacket = constructSupplicantIdentifyPacket(null,
+                                                                   EAP.ATTR_IDENTITY, (byte) 3, null);
+            } catch (Exception e) {
+                log.error(e.getMessage());
+                fail();
+            }
+            sendPacket(identifyPacket);
+        });
+        assertAfter(ASSERTION_DELAY, ASSERTION_LENGTH, () -> {
+            RADIUS radiusIdentifyPacket = (RADIUS) fetchPacket(1);
+            AtomicReference<Byte> reqId = new AtomicReference<>(radiusIdentifyPacket.getIdentifier());
+
+            try {
+                checkRadiusPacketFromSupplicant(radiusIdentifyPacket);
+            } catch (DeserializationException e) {
+                log.error(e.getMessage());
+                fail();
+            }
+
+            assertThat(radiusIdentifyPacket.getCode(), is(RADIUS.RADIUS_CODE_ACCESS_REQUEST));
+            assertThat(new String(radiusIdentifyPacket.getAttribute(RADIUSAttribute.RADIUS_ATTR_USERNAME).getValue()),
+                       is("testuser"));
+
+            IpAddress nasIp =
+                    IpAddress.valueOf(IpAddress.Version.INET,
+                                      radiusIdentifyPacket.getAttribute(RADIUSAttribute.RADIUS_ATTR_NAS_IP)
+                                              .getValue());
+            assertThat(nasIp.toString(), is(aaaManager.nasIpAddress.getHostAddress()));
+
+            // (3) RADIUS MD5 challenge
+
+            RADIUS radiusCodeAccessChallengePacket =
+                    constructRadiusCodeAccessChallengePacket(RADIUS.RADIUS_CODE_ACCESS_CHALLENGE, EAP.ATTR_MD5,
+                                                             reqId.get(), aaaManager.radiusSecret.getBytes());
+            aaaManager.handleRadiusPacket(radiusCodeAccessChallengePacket);
+        });
+        assertAfter(ASSERTION_DELAY, ASSERTION_LENGTH, () -> {
+
+            //  State machine should have been created by now
+
+            StateMachine stateMachine = aaaManager.getStateMachine(SESSION_ID);
+            assertThat(stateMachine, notNullValue());
+            assertThat(stateMachine.state(), is(StateMachine.STATE_PENDING));
+
             Ethernet radiusChallengeMD5Packet = (Ethernet) fetchPacket(2);
             checkRadiusPacket(aaaManager, radiusChallengeMD5Packet, EAP.ATTR_MD5);
 
@@ -259,7 +272,14 @@
                 fail();
             }
         });
+
+        //  State machine should have been created by now
+
+        StateMachine stateMachine = aaaManager.getStateMachine(SESSION_ID);
+
         assertAfter(ASSERTION_DELAY, ASSERTION_LENGTH, () -> {
+
+
             RADIUS responseMd5RadiusPacket = (RADIUS) fetchPacket(3);
             try {
                 checkRadiusPacketFromSupplicant(responseMd5RadiusPacket);
@@ -268,7 +288,7 @@
                 fail();
             }
             //assertThat(responseMd5RadiusPacket.getIdentifier(), is((byte) 9));
-            reqId.set(responseMd5RadiusPacket.getIdentifier());
+            AtomicReference<Byte> reqId = new AtomicReference<>(responseMd5RadiusPacket.getIdentifier());
             assertThat(responseMd5RadiusPacket.getCode(), is(RADIUS.RADIUS_CODE_ACCESS_REQUEST));
 
             //  State machine should be in pending state
@@ -300,15 +320,16 @@
     public void testRemoveAuthentication() {
         Ethernet startPacket = constructSupplicantStartPacket();
         sendPacket(startPacket);
+        assertAfter(ASSERTION_DELAY, ASSERTION_LENGTH, () -> {
+            StateMachine stateMachine = aaaManager.getStateMachine(SESSION_ID);
 
-        StateMachine stateMachine = aaaManager.getStateMachine(SESSION_ID);
+            assertThat(stateMachine, notNullValue());
+            assertThat(stateMachine.state(), is(StateMachine.STATE_STARTED));
 
-        assertThat(stateMachine, notNullValue());
-        assertThat(stateMachine.state(), is(StateMachine.STATE_STARTED));
+            aaaManager.removeAuthenticationStateByMac(stateMachine.supplicantAddress());
 
-        aaaManager.removeAuthenticationStateByMac(stateMachine.supplicantAddress());
-
-        assertThat(aaaManager.getStateMachine(SESSION_ID), nullValue());
+            assertThat(aaaManager.getStateMachine(SESSION_ID), nullValue());
+        });
     }
 
     /**
diff --git a/app/src/test/java/org/opencord/aaa/impl/AaaStatisticsTest.java b/app/src/test/java/org/opencord/aaa/impl/AaaStatisticsTest.java
index c4f390c..0c11292 100644
--- a/app/src/test/java/org/opencord/aaa/impl/AaaStatisticsTest.java
+++ b/app/src/test/java/org/opencord/aaa/impl/AaaStatisticsTest.java
@@ -149,6 +149,7 @@
         aaaManager.sadisService = new MockSadisService();
         aaaManager.cfgService = new MockCfgService();
         aaaManager.storageService = new TestStorageService();
+        aaaManager.mastershipService = new MockMastershipService();
         aaaStatisticsManager = new AaaStatisticsManager();
         aaaStatisticsManager.storageService = new TestStorageService();
         aaaStatisticsManager.clusterService = new ClusterServiceAdapter();
@@ -211,39 +212,56 @@
         // (1) Supplicant start up
         Ethernet startPacket = constructSupplicantStartPacket();
         sendPacket(startPacket);
-
-        Ethernet responsePacket = (Ethernet) fetchPacket(0);
-        checkRadiusPacket(aaaManager, responsePacket, EAP.ATTR_IDENTITY);
-
-        // (2) Supplicant identify
-
-        Ethernet identifyPacket = constructSupplicantIdentifyPacket(null, EAP.ATTR_IDENTITY, (byte) 1, null);
-        sendPacket(identifyPacket);
-
-        RADIUS radiusIdentifyPacket = (RADIUS) fetchPacket(1);
-        checkRadiusPacketFromSupplicant(radiusIdentifyPacket);
-
-        assertThat(radiusIdentifyPacket.getCode(), is(RADIUS.RADIUS_CODE_ACCESS_REQUEST));
-        assertThat(new String(radiusIdentifyPacket.getAttribute(RADIUSAttribute.RADIUS_ATTR_USERNAME).getValue()),
-                is("testuser"));
-        IpAddress nasIp = IpAddress.valueOf(IpAddress.Version.INET,
-                radiusIdentifyPacket.getAttribute(RADIUSAttribute.RADIUS_ATTR_NAS_IP).getValue());
-        assertThat(nasIp.toString(), is(aaaManager.nasIpAddress.getHostAddress()));
-
-        // State machine should have been created by now
-
-        StateMachine stateMachine = aaaManager.getStateMachine(SESSION_ID);
-        assertThat(stateMachine, notNullValue());
-        assertThat(stateMachine.state(), is(StateMachine.STATE_PENDING));
-
-        // (3) RADIUS MD5 challenge
-
-        RADIUS radiusCodeAccessChallengePacket = constructRadiusCodeAccessChallengePacket(
-                RADIUS.RADIUS_CODE_ACCESS_CHALLENGE, EAP.ATTR_MD5, radiusIdentifyPacket.getIdentifier(),
-                aaaManager.radiusSecret.getBytes());
-        aaaManager.handleRadiusPacket(radiusCodeAccessChallengePacket);
-
         assertAfter(ASSERTION_DELAY, ASSERTION_LENGTH, () -> {
+            Ethernet responsePacket = (Ethernet) fetchPacket(0);
+            checkRadiusPacket(aaaManager, responsePacket, EAP.ATTR_IDENTITY);
+
+            // (2) Supplicant identify
+
+            Ethernet identifyPacket = null;
+            try {
+                identifyPacket = constructSupplicantIdentifyPacket(null,
+                                                                   EAP.ATTR_IDENTITY, (byte) 1, null);
+                sendPacket(identifyPacket);
+            } catch (Exception e) {
+                log.error(e.getMessage());
+                fail();
+            }
+            });
+        assertAfter(ASSERTION_DELAY, ASSERTION_LENGTH, () -> {
+            try {
+                RADIUS radiusIdentifyPacket = (RADIUS) fetchPacket(1);
+                checkRadiusPacketFromSupplicant(radiusIdentifyPacket);
+
+                assertThat(radiusIdentifyPacket.getCode(), is(RADIUS.RADIUS_CODE_ACCESS_REQUEST));
+                assertThat(new String(radiusIdentifyPacket.getAttribute(RADIUSAttribute.RADIUS_ATTR_USERNAME)
+                .getValue()),
+                           is("testuser"));
+                IpAddress nasIp = IpAddress.valueOf(IpAddress.Version.INET,
+                                                    radiusIdentifyPacket
+                                                    .getAttribute(RADIUSAttribute.RADIUS_ATTR_NAS_IP)
+                                                            .getValue());
+                assertThat(nasIp.toString(), is(aaaManager.nasIpAddress.getHostAddress()));
+
+                // State machine should have been created by now
+
+                StateMachine stateMachine = aaaManager.getStateMachine(SESSION_ID);
+                assertThat(stateMachine, notNullValue());
+                assertThat(stateMachine.state(), is(StateMachine.STATE_PENDING));
+
+                // (3) RADIUS MD5 challenge
+
+                RADIUS radiusCodeAccessChallengePacket = constructRadiusCodeAccessChallengePacket(
+                        RADIUS.RADIUS_CODE_ACCESS_CHALLENGE, EAP.ATTR_MD5, radiusIdentifyPacket.getIdentifier(),
+                        aaaManager.radiusSecret.getBytes());
+                aaaManager.handleRadiusPacket(radiusCodeAccessChallengePacket);
+            } catch (Exception e) {
+                log.error(e.getMessage());
+                fail();
+            }
+        });
+        assertAfter(ASSERTION_DELAY, ASSERTION_LENGTH, () -> {
+            StateMachine stateMachine = aaaManager.getStateMachine(SESSION_ID);
             Ethernet radiusChallengeMD5Packet = (Ethernet) fetchPacket(2);
             checkRadiusPacket(aaaManager, radiusChallengeMD5Packet, EAP.ATTR_MD5);
             // (4) Supplicant MD5 response
@@ -261,6 +279,7 @@
 
 
         assertAfter(ASSERTION_DELAY, ASSERTION_LENGTH, () -> {
+            StateMachine stateMachine = aaaManager.getStateMachine(SESSION_ID);
             RADIUS responseMd5RadiusPacket = (RADIUS) fetchPacket(3);
 
             try {
@@ -286,6 +305,7 @@
             aaaManager.handleRadiusPacket((successPacket));
         });
         assertAfter(ASSERTION_DELAY, ASSERTION_LENGTH, () -> {
+            StateMachine stateMachine = aaaManager.getStateMachine(SESSION_ID);
             Ethernet supplicantSuccessPacket = (Ethernet) fetchPacket(4);
 
             checkRadiusPacket(aaaManager, supplicantSuccessPacket, EAP.SUCCESS);
@@ -419,26 +439,39 @@
         sendPacket(startPacket);
 
         // (2) Supplicant identify
-
-        Ethernet identifyPacket = constructSupplicantIdentifyPacket(null, EAP.ATTR_IDENTITY, (byte) 1, null);
-        sendPacket(identifyPacket);
-
-        RADIUS radiusIdentifyPacket = (RADIUS) fetchPacket(1);
-
-        checkRadiusPacketFromSupplicant(radiusIdentifyPacket);
-
-        // State machine should have been created by now
-
-        StateMachine stateMachine = aaaManager.getStateMachine(SESSION_ID);
-
-        // (3) RADIUS MD5 challenge
-
-        RADIUS radiusCodeAccessChallengePacket = constructRadiusCodeAccessChallengePacket(
-                RADIUS.RADIUS_CODE_ACCESS_CHALLENGE, EAP.ATTR_MD5, radiusIdentifyPacket.getIdentifier(),
-                aaaManager.radiusSecret.getBytes());
-        aaaManager.handleRadiusPacket(radiusCodeAccessChallengePacket);
-
         assertAfter(ASSERTION_DELAY, ASSERTION_LENGTH, () -> {
+            Ethernet identifyPacket = null;
+            try {
+                identifyPacket = constructSupplicantIdentifyPacket(null, EAP.ATTR_IDENTITY, (byte) 1, null);
+                sendPacket(identifyPacket);
+            } catch (Exception e) {
+                log.error(e.getMessage());
+                fail();
+            }
+        });
+        assertAfter(ASSERTION_DELAY, ASSERTION_LENGTH, () -> {
+            try {
+                RADIUS radiusIdentifyPacket = (RADIUS) fetchPacket(1);
+
+                checkRadiusPacketFromSupplicant(radiusIdentifyPacket);
+
+                // State machine should have been created by now
+
+                StateMachine stateMachine = aaaManager.getStateMachine(SESSION_ID);
+
+                // (3) RADIUS MD5 challenge
+
+                RADIUS radiusCodeAccessChallengePacket = constructRadiusCodeAccessChallengePacket(
+                        RADIUS.RADIUS_CODE_ACCESS_CHALLENGE, EAP.ATTR_MD5, radiusIdentifyPacket.getIdentifier(),
+                        aaaManager.radiusSecret.getBytes());
+                aaaManager.handleRadiusPacket(radiusCodeAccessChallengePacket);
+            } catch (Exception e) {
+                log.error(e.getMessage());
+                fail();
+            }
+        });
+        assertAfter(ASSERTION_DELAY, ASSERTION_LENGTH, () -> {
+            StateMachine stateMachine = aaaManager.getStateMachine(SESSION_ID);
             Ethernet radiusChallengeMD5Packet = (Ethernet) fetchPacket(2);
 
             // (4) Supplicant MD5 response
@@ -470,6 +503,7 @@
         });
 
         assertAfter(ASSERTION_DELAY, ASSERTION_LENGTH, () -> {
+            StateMachine stateMachine = aaaManager.getStateMachine(SESSION_ID);
             Ethernet supplicantRejectedPacket = (Ethernet) fetchPacket(4);
 
             checkRadiusPacket(aaaManager, supplicantRejectedPacket, EAP.FAILURE);
@@ -511,42 +545,51 @@
         // (1) Supplicant start up
         Ethernet startPacket = constructSupplicantStartPacket();
         sendPacket(startPacket);
+        assertAfter(ASSERTION_DELAY, ASSERTION_LENGTH, () -> {
+            // (2) Supplicant identify
 
-        // (2) Supplicant identify
+            Ethernet identifyPacket = null;
+            try {
+                identifyPacket = constructSupplicantIdentifyPacket(null, EAP.ATTR_IDENTITY, (byte) 1, null);
+                sendPacket(identifyPacket);
 
-        Ethernet identifyPacket = constructSupplicantIdentifyPacket(null, EAP.ATTR_IDENTITY, (byte) 1, null);
-        sendPacket(identifyPacket);
+                RADIUS radiusIdentifyPacket = (RADIUS) fetchPacket(1);
+                checkRadiusPacketFromSupplicant(radiusIdentifyPacket);
 
-        RADIUS radiusIdentifyPacket = (RADIUS) fetchPacket(1);
-        checkRadiusPacketFromSupplicant(radiusIdentifyPacket);
+                // again creating pending state for same packet
+                constructSupplicantIdentifyPacket(null, EAP.ATTR_IDENTITY, (byte) 1, null);
+                sendPacket(identifyPacket);
+            } catch (Exception e) {
+                log.error(e.getMessage());
+                fail();
+            }
+        });
+        assertAfter(ASSERTION_DELAY, ASSERTION_LENGTH, () -> {
+            aaaManager.impl.handlePacketFromServer(null);
+            aaaManager.aaaStatisticsManager.calculatePacketRoundtripTime();
 
-        // again creating pending state for same packet
-        constructSupplicantIdentifyPacket(null, EAP.ATTR_IDENTITY, (byte) 1, null);
-        sendPacket(identifyPacket);
-        aaaManager.impl.handlePacketFromServer(null);
-        aaaManager.aaaStatisticsManager.calculatePacketRoundtripTime();
+            // creating malformed packet
+            final ByteBuffer byteBuffer = ByteBuffer.wrap(startPacket.serialize());
+            InboundPacket inPacket = new DefaultInboundPacket(connectPoint("1", 1),
+                                                              startPacket, byteBuffer);
 
-        // creating malformed packet
-        final ByteBuffer byteBuffer = ByteBuffer.wrap(startPacket.serialize());
-        InboundPacket inPacket = new DefaultInboundPacket(connectPoint("1", 1),
-                startPacket, byteBuffer);
+            PacketContext context = new TestPacketContext(127L, inPacket, null, false);
+            aaaManager.impl.handlePacketFromServer(context);
 
-        PacketContext context = new TestPacketContext(127L, inPacket, null, false);
-        aaaManager.impl.handlePacketFromServer(context);
+            // Check for increase of Stats
+            assertNotEquals(aaaStatisticsManager.getAaaStats().getEapolResIdentityMsgTrans(), ZERO);
+            assertNotEquals(aaaStatisticsManager.getAaaStats().getEapolStartReqTrans(), ZERO);
 
-        // Check for increase of Stats
-        assertNotEquals(aaaStatisticsManager.getAaaStats().getEapolResIdentityMsgTrans(), ZERO);
-        assertNotEquals(aaaStatisticsManager.getAaaStats().getEapolStartReqTrans(), ZERO);
+            assertNotEquals(aaaStatisticsManager.getAaaStats().getAccessRequestsTx(), ZERO);
+            assertNotEquals(aaaStatisticsManager.getAaaStats().getDroppedResponsesRx(), ZERO);
+            assertNotEquals(aaaStatisticsManager.getAaaStats().getPendingRequests(), ZERO);
+            assertNotEquals(aaaStatisticsManager.getAaaStats().getMalformedResponsesRx(), ZERO);
+            assertNotEquals(aaaStatisticsManager.getAaaStats().getRequestReTx(), ZERO);
+            assertNotEquals(aaaStatisticsManager.getAaaStats().getUnknownTypeRx(), ZERO);
+            assertNotEquals(aaaStatisticsManager.getAaaStats().getUnknownServerRx(), ZERO);
 
-        assertNotEquals(aaaStatisticsManager.getAaaStats().getAccessRequestsTx(), ZERO);
-        assertNotEquals(aaaStatisticsManager.getAaaStats().getDroppedResponsesRx(), ZERO);
-        assertNotEquals(aaaStatisticsManager.getAaaStats().getPendingRequests(), ZERO);
-        assertNotEquals(aaaStatisticsManager.getAaaStats().getMalformedResponsesRx(), ZERO);
-        assertNotEquals(aaaStatisticsManager.getAaaStats().getRequestReTx(), ZERO);
-        assertNotEquals(aaaStatisticsManager.getAaaStats().getUnknownTypeRx(), ZERO);
-        assertNotEquals(aaaStatisticsManager.getAaaStats().getUnknownServerRx(), ZERO);
-
-        countAaaStatistics();
+            countAaaStatistics();
+        });
     }
 
     /**
@@ -561,38 +604,55 @@
         // (1) Supplicant start up
         Ethernet startPacket = constructSupplicantStartPacket();
         sendPacket(startPacket);
+        assertAfter(ASSERTION_DELAY, ASSERTION_LENGTH, () -> {
+                        Ethernet responsePacket = (Ethernet) fetchPacket(0);
+                        checkRadiusPacket(aaaManager, responsePacket, EAP.ATTR_IDENTITY);
 
-        Ethernet responsePacket = (Ethernet) fetchPacket(0);
-        checkRadiusPacket(aaaManager, responsePacket, EAP.ATTR_IDENTITY);
+                        // (2) Supplicant identify
 
-        // (2) Supplicant identify
+            Ethernet identifyPacket = null;
+            try {
+                identifyPacket = constructSupplicantIdentifyPacket(null, EAP.ATTR_IDENTITY, (byte) 1, null);
+            } catch (Exception e) {
+                log.error(e.getMessage());
+                fail();
+            }
+            sendPacket(identifyPacket);
+                    });
+        assertAfter(ASSERTION_DELAY, ASSERTION_LENGTH, () -> {
+                        RADIUS radiusIdentifyPacket = (RADIUS) fetchPacket(1);
+            try {
+                checkRadiusPacketFromSupplicant(radiusIdentifyPacket);
+            } catch (Exception e) {
+                log.error(e.getMessage());
+                fail();
+            }
 
-        Ethernet identifyPacket = constructSupplicantIdentifyPacket(null, EAP.ATTR_IDENTITY, (byte) 1, null);
-        sendPacket(identifyPacket);
+            assertThat(radiusIdentifyPacket.getCode(), is(RADIUS.RADIUS_CODE_ACCESS_REQUEST));
+                        assertThat(new String(radiusIdentifyPacket
+                                                      .getAttribute(RADIUSAttribute.RADIUS_ATTR_USERNAME)
+                                                      .getValue()),
+                                   is("testuser"));
+                        IpAddress nasIp = IpAddress.valueOf(IpAddress.Version.INET,
+                                                            radiusIdentifyPacket
+                                                                    .getAttribute(RADIUSAttribute.RADIUS_ATTR_NAS_IP)
+                                                                    .getValue());
+                        assertThat(nasIp.toString(), is(aaaManager.nasIpAddress.getHostAddress()));
 
-        RADIUS radiusIdentifyPacket = (RADIUS) fetchPacket(1);
-        checkRadiusPacketFromSupplicant(radiusIdentifyPacket);
+                        // State machine should have been created by now
 
-        assertThat(radiusIdentifyPacket.getCode(), is(RADIUS.RADIUS_CODE_ACCESS_REQUEST));
-        assertThat(new String(radiusIdentifyPacket.getAttribute(RADIUSAttribute.RADIUS_ATTR_USERNAME).getValue()),
-                is("testuser"));
-        IpAddress nasIp = IpAddress.valueOf(IpAddress.Version.INET,
-                radiusIdentifyPacket.getAttribute(RADIUSAttribute.RADIUS_ATTR_NAS_IP).getValue());
-        assertThat(nasIp.toString(), is(aaaManager.nasIpAddress.getHostAddress()));
+                        //StateMachine stateMachine = StateMachine.lookupStateMachineBySessionId(SESSION_ID);
+                        StateMachine stateMachine = aaaManager.getStateMachine(SESSION_ID);
+                        assertThat(stateMachine, notNullValue());
+                        assertThat(stateMachine.state(), is(StateMachine.STATE_PENDING));
 
-        // State machine should have been created by now
+                        // (3) RADIUS MD5 challenge
 
-        //StateMachine stateMachine = StateMachine.lookupStateMachineBySessionId(SESSION_ID);
-        StateMachine stateMachine = aaaManager.getStateMachine(SESSION_ID);
-        assertThat(stateMachine, notNullValue());
-        assertThat(stateMachine.state(), is(StateMachine.STATE_PENDING));
-
-        // (3) RADIUS MD5 challenge
-
-        RADIUS radiusCodeAccessChallengePacket = constructRadiusCodeAccessChallengePacket(
-                RADIUS.RADIUS_CODE_ACCESS_CHALLENGE, EAP.ATTR_MD5,
-                radiusIdentifyPacket.getIdentifier(), aaaManager.radiusSecret.getBytes());
-        aaaManager.handleRadiusPacket(radiusCodeAccessChallengePacket);
+                        RADIUS radiusCodeAccessChallengePacket = constructRadiusCodeAccessChallengePacket(
+                                RADIUS.RADIUS_CODE_ACCESS_CHALLENGE, EAP.ATTR_MD5,
+                                radiusIdentifyPacket.getIdentifier(), aaaManager.radiusSecret.getBytes());
+                        aaaManager.handleRadiusPacket(radiusCodeAccessChallengePacket);
+                    });
         assertAfter(ASSERTION_DELAY, ASSERTION_LENGTH, () -> {
             Ethernet radiusChallengeMD5Packet = (Ethernet) fetchPacket(2);
             checkRadiusPacket(aaaManager, radiusChallengeMD5Packet, EAP.ATTR_MD5);
@@ -600,6 +660,8 @@
             // (4) Supplicant MD5 response
 
             Ethernet md5RadiusPacket = null;
+            StateMachine stateMachine = aaaManager.getStateMachine(SESSION_ID);
+
             try {
                 md5RadiusPacket = constructSupplicantIdentifyPacket(stateMachine, EAP.ATTR_MD5,
                                                                     stateMachine.challengeIdentifier(),
@@ -623,7 +685,7 @@
             assertThat(responseMd5RadiusPacket.getCode(), is(RADIUS.RADIUS_CODE_ACCESS_REQUEST));
 
             // State machine should be in pending state
-
+            StateMachine stateMachine = aaaManager.getStateMachine(SESSION_ID);
             assertThat(stateMachine, notNullValue());
             assertThat(stateMachine.state(), is(StateMachine.STATE_PENDING));
 
@@ -642,6 +704,7 @@
             checkRadiusPacket(aaaManager, supplicantSuccessPacket, EAP.SUCCESS);
 
             // State machine should be in authorized state
+            StateMachine stateMachine = aaaManager.getStateMachine(SESSION_ID);
 
             assertThat(stateMachine, notNullValue());
             assertThat(stateMachine.state(), is(StateMachine.STATE_AUTHORIZED));
@@ -652,6 +715,8 @@
         });
         assertAfter(ASSERTION_DELAY, ASSERTION_LENGTH, () -> {
             // State machine should be in logoff state
+            StateMachine stateMachine = aaaManager.getStateMachine(SESSION_ID);
+
             assertThat(stateMachine, notNullValue());
             assertThat(stateMachine.state(), is(StateMachine.STATE_IDLE));
 
@@ -681,40 +746,58 @@
         // (1) Supplicant start up
         Ethernet startPacket = constructSupplicantStartPacket();
         sendPacket(startPacket);
+        assertAfter(ASSERTION_DELAY, ASSERTION_LENGTH, () -> {
+            Ethernet responsePacket = (Ethernet) fetchPacket(0);
+            checkRadiusPacket(aaaManager, responsePacket, EAP.ATTR_IDENTITY);
 
-        Ethernet responsePacket = (Ethernet) fetchPacket(0);
-        checkRadiusPacket(aaaManager, responsePacket, EAP.ATTR_IDENTITY);
+            // (2) Supplicant identify
 
-        // (2) Supplicant identify
+            Ethernet identifyPacket = null;
+            try {
+                identifyPacket = constructSupplicantIdentifyPacket(null, EAP.ATTR_IDENTITY, (byte) 1, null);
+            } catch (Exception e) {
+                log.error(e.getMessage());
+                fail();
+            }
+            sendPacket(identifyPacket);
+        });
+        assertAfter(ASSERTION_DELAY, ASSERTION_LENGTH, () -> {
+            RADIUS radiusIdentifyPacket = (RADIUS) fetchPacket(1);
+            try {
+                checkRadiusPacketFromSupplicant(radiusIdentifyPacket);
 
-        Ethernet identifyPacket = constructSupplicantIdentifyPacket(null, EAP.ATTR_IDENTITY, (byte) 1, null);
-        sendPacket(identifyPacket);
 
-        RADIUS radiusIdentifyPacket = (RADIUS) fetchPacket(1);
-        checkRadiusPacketFromSupplicant(radiusIdentifyPacket);
+                assertThat(radiusIdentifyPacket.getCode(), is(RADIUS.RADIUS_CODE_ACCESS_REQUEST));
+                assertThat(new String(radiusIdentifyPacket
+                                              .getAttribute(RADIUSAttribute.RADIUS_ATTR_USERNAME)
+                                              .getValue()),
+                           is("testuser"));
+                IpAddress nasIp = IpAddress.valueOf(IpAddress.Version.INET,
+                                                    radiusIdentifyPacket
+                                                            .getAttribute(RADIUSAttribute.RADIUS_ATTR_NAS_IP)
+                                                            .getValue());
+                assertThat(nasIp.toString(), is(aaaManager.nasIpAddress.getHostAddress()));
 
-        assertThat(radiusIdentifyPacket.getCode(), is(RADIUS.RADIUS_CODE_ACCESS_REQUEST));
-        assertThat(new String(radiusIdentifyPacket.getAttribute(RADIUSAttribute.RADIUS_ATTR_USERNAME).getValue()),
-                is("testuser"));
-        IpAddress nasIp = IpAddress.valueOf(IpAddress.Version.INET,
-                  radiusIdentifyPacket.getAttribute(RADIUSAttribute.RADIUS_ATTR_NAS_IP).getValue());
-        assertThat(nasIp.toString(), is(aaaManager.nasIpAddress.getHostAddress()));
+                // State machine should have been created by now
 
-        // State machine should have been created by now
+                StateMachine stateMachine = aaaManager.getStateMachine(SESSION_ID);
+                assertThat(stateMachine, notNullValue());
+                assertThat(stateMachine.state(), is(StateMachine.STATE_PENDING));
+                Thread.sleep((aaaManager.cleanupTimerTimeOutInMins / 2) + 1);
 
-        StateMachine stateMachine = aaaManager.getStateMachine(SESSION_ID);
-        assertThat(stateMachine, notNullValue());
-        assertThat(stateMachine.state(), is(StateMachine.STATE_PENDING));
-        Thread.sleep((aaaManager.cleanupTimerTimeOutInMins / 2) + 1);
+                // State machine should be in timeout state
+                assertThat(stateMachine, notNullValue());
+                assertThat(stateMachine.state(), is(StateMachine.STATE_PENDING));
 
-        // State machine should be in timeout state
-        assertThat(stateMachine, notNullValue());
-        assertThat(stateMachine.state(), is(StateMachine.STATE_PENDING));
-
-        //Check for increase in stats
-        assertNotEquals(aaaStatisticsManager.getAaaStats().getEapolResIdentityMsgTrans(), ZERO);
-        assertNotEquals(aaaStatisticsManager.getAaaStats().getEapolStartReqTrans(), ZERO);
-       countAaaStatistics();
+                //Check for increase in stats
+                assertNotEquals(aaaStatisticsManager.getAaaStats().getEapolResIdentityMsgTrans(), ZERO);
+                assertNotEquals(aaaStatisticsManager.getAaaStats().getEapolStartReqTrans(), ZERO);
+                countAaaStatistics();
+            } catch (Exception e) {
+                log.error(e.getMessage());
+                fail();
+            }
+        });
 
     }
 
diff --git a/app/src/test/java/org/opencord/aaa/impl/AaaTestBase.java b/app/src/test/java/org/opencord/aaa/impl/AaaTestBase.java
index fdd23e9..09f5e97 100644
--- a/app/src/test/java/org/opencord/aaa/impl/AaaTestBase.java
+++ b/app/src/test/java/org/opencord/aaa/impl/AaaTestBase.java
@@ -29,8 +29,10 @@
 import org.onlab.packet.VlanId;
 import org.onosproject.cfg.ComponentConfigService;
 import org.onosproject.cfg.ConfigProperty;
+import org.onosproject.mastership.MastershipServiceAdapter;
 import org.onosproject.net.Annotations;
 import org.onosproject.net.ConnectPoint;
+import org.onosproject.net.DeviceId;
 import org.onosproject.net.Element;
 import org.onosproject.net.Port;
 import org.onosproject.net.PortNumber;
@@ -117,66 +119,67 @@
             }
         }
     }
+
     class MockComponentContext implements ComponentContext {
 
-                @Override
-                public Dictionary<String, Object> getProperties() {
-                        Dictionary<String, Object> cfgDict = new Hashtable<String, Object>();
-                        cfgDict.put("statisticsGenerationEvent", 20);
-                        return cfgDict;
-                }
+        @Override
+        public Dictionary<String, Object> getProperties() {
+            Dictionary<String, Object> cfgDict = new Hashtable<String, Object>();
+            cfgDict.put("statisticsGenerationEvent", 20);
+            return cfgDict;
+        }
 
-                @Override
-                public Object locateService(String name) {
-                        // TODO Auto-generated method stub
-                        return null;
-                }
+        @Override
+        public Object locateService(String name) {
+            // TODO Auto-generated method stub
+            return null;
+        }
 
-                @Override
-                public Object locateService(String name, ServiceReference reference) {
-                        // TODO Auto-generated method stub
-                        return null;
-                }
+        @Override
+        public Object locateService(String name, ServiceReference reference) {
+            // TODO Auto-generated method stub
+            return null;
+        }
 
-                @Override
-                public Object[] locateServices(String name) {
-                        // TODO Auto-generated method stub
-                        return null;
-                }
+        @Override
+        public Object[] locateServices(String name) {
+            // TODO Auto-generated method stub
+            return null;
+        }
 
-                @Override
-                public BundleContext getBundleContext() {
-                        // TODO Auto-generated method stub
-                        return null;
-                }
+        @Override
+        public BundleContext getBundleContext() {
+            // TODO Auto-generated method stub
+            return null;
+        }
 
-                @Override
-                public Bundle getUsingBundle() {
-                        // TODO Auto-generated method stub
-                        return null;
-                }
+        @Override
+        public Bundle getUsingBundle() {
+            // TODO Auto-generated method stub
+            return null;
+        }
 
-                @Override
-                public ComponentInstance getComponentInstance() {
-                        // TODO Auto-generated method stub
-                        return null;
-                }
+        @Override
+        public ComponentInstance getComponentInstance() {
+            // TODO Auto-generated method stub
+            return null;
+        }
 
-                @Override
-                public void enableComponent(String name) {
-                        // TODO Auto-generated method stub
-                }
+        @Override
+        public void enableComponent(String name) {
+            // TODO Auto-generated method stub
+        }
 
-                @Override
-                public void disableComponent(String name) {
-                       // TODO Auto-generated method stub
-                }
+        @Override
+        public void disableComponent(String name) {
+            // TODO Auto-generated method stub
+        }
 
-                @Override
-                public ServiceReference getServiceReference() {
-                       // TODO Auto-generated method stub
-                       return null;
-                }
+        @Override
+        public ServiceReference getServiceReference() {
+            // TODO Auto-generated method stub
+            return null;
+        }
     }
 
     /**
@@ -188,24 +191,30 @@
             return new MockPort();
         }
     }
-    private class  MockPort implements Port {
+
+    private class MockPort implements Port {
 
         @Override
         public boolean isEnabled() {
             return true;
         }
+
         public long portSpeed() {
             return 1000;
         }
+
         public Element element() {
             return null;
         }
+
         public PortNumber number() {
             return null;
         }
+
         public Annotations annotations() {
             return new MockAnnotations();
         }
+
         public Type type() {
             return Port.Type.FIBER;
         }
@@ -216,6 +225,7 @@
             public String value(String val) {
                 return "PON 1/1";
             }
+
             public Set<String> keys() {
                 return null;
             }
@@ -282,7 +292,7 @@
 
         @Override
         public Set<ConfigProperty> getProperties(String componentName) {
-           return null;
+            return null;
         }
 
         @Override
@@ -307,10 +317,17 @@
 
         @Override
         public ConfigProperty getProperty(String componentName, String attribute) {
-           return null;
+            return null;
         }
 
-}
+    }
+
+    final class MockMastershipService extends MastershipServiceAdapter {
+        @Override
+        public boolean isLocalMaster(DeviceId deviceId) {
+            return true;
+        }
+    }
 
     final class MockSubService implements BaseInformationService<SubscriberAndDeviceInformation> {
         private final VlanId uniTagMatch = VlanId.vlanId((short) 35);
@@ -331,27 +348,33 @@
                                                        techProfileId, dsBpId, usBpId,
                                                        clientNasPortId, clientCircuitId, null,
                                                        null);
+
         @Override
         public SubscriberAndDeviceInformation get(String id) {
 
-                return  sub;
+            return sub;
 
         }
 
         @Override
-        public void invalidateAll() {}
-        public void invalidateId(String id) {}
+        public void invalidateAll() {
+        }
+
+        public void invalidateId(String id) {
+        }
+
         public SubscriberAndDeviceInformation getfromCache(String id) {
             return null;
         }
     }
+
     /**
      * Mocks the DefaultPacketContext.
      */
     final class TestPacketContext extends DefaultPacketContext {
 
         TestPacketContext(long time, InboundPacket inPkt,
-                                  OutboundPacket outPkt, boolean block) {
+                          OutboundPacket outPkt, boolean block) {
             super(time, inPkt, outPkt, block);
         }
 
@@ -382,9 +405,9 @@
      * @return Ethernet packet
      */
     Ethernet constructSupplicantIdentifyPacket(StateMachine stateMachine,
-                                                       byte type,
-                                                       byte id,
-                                                       Ethernet radiusChallenge)
+                                               byte type,
+                                               byte id,
+                                               Ethernet radiusChallenge)
             throws Exception {
         Ethernet eth = new Ethernet();
         eth.setDestinationMACAddress(clientMac.toBytes());
@@ -488,7 +511,7 @@
      * Checks the contents of a RADIUS packet being sent to the RADIUS server.
      *
      * @param radiusPacket packet to check
-     * @param code expected code
+     * @param code         expected code
      */
     void checkRadiusPacket(AaaManager aaaManager, Ethernet radiusPacket, byte code) {
 
@@ -544,12 +567,12 @@
      * @return Ethernet packet
      */
     RADIUS constructRadiusCodeAccessChallengePacket(byte challengeCode, byte challengeType,
-            byte identifier, byte[] messageAuth) {
+                                                    byte identifier, byte[] messageAuth) {
 
         String challenge = "12345678901234567";
 
         EAP eap = new EAP(challengeType, (byte) 4, challengeType,
-                challenge.getBytes(Charsets.US_ASCII));
+                          challenge.getBytes(Charsets.US_ASCII));
         //eap.setIdentifier((byte) 4);
         eap.setIdentifier(identifier);
 
@@ -558,7 +581,7 @@
         //radius.setIdentifier((byte) 4);
         radius.setIdentifier(identifier);
         radius.setAttribute(RADIUSAttribute.RADIUS_ATTR_STATE,
-                challenge.getBytes(Charsets.US_ASCII));
+                            challenge.getBytes(Charsets.US_ASCII));
 
         radius.setPayload(eap);
         radius.setAttribute(RADIUSAttribute.RADIUS_ATTR_EAP_MESSAGE, eap.serialize());