Better logs and stats reporting

Change-Id: I1ef0d75398c8e0ff3fa3ac6a140c24d0394945cb
diff --git a/api/src/main/java/org/opencord/aaa/AaaStatistics.java b/api/src/main/java/org/opencord/aaa/AaaStatistics.java
index 29809f4..bb37453 100644
--- a/api/src/main/java/org/opencord/aaa/AaaStatistics.java
+++ b/api/src/main/java/org/opencord/aaa/AaaStatistics.java
@@ -19,8 +19,10 @@
 import com.google.common.base.MoreObjects;
 import com.google.common.collect.ImmutableMap;
 
+import java.util.Arrays;
 import java.util.LinkedList;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.stream.Stream;
 
 /**
  * Records metrics for the AAA application.
@@ -55,19 +57,36 @@
     public static final String EAPOL_FRAMES_TX = "eapolFramesTx";
     public static final String AUTH_STATE_IDLE = "authStateIdle";
     public static final String EAPOL_ID_REQUEST_FRAMES_TX = "eapolIdentityRequestTx";
-    public static final String EAPOL_REQUEST_FRAMES_TX = "eapolRequestFramesTx"; //TODO check
+    public static final String EAPOL_REQUEST_FRAMES_TX = "eapolRequestAuthTx";
     public static final String INVALID_PKT_TYPE = "invalidPktType";
     public static final String INVALID_BODY_LENGTH = "invalidBodyLength";
     public static final String EAPOL_VALID_FRAMES_RX = "eapolValidFramesRx";
     public static final String EAPOL_PENDING_REQUESTS = "eapolPendingRequests";
 
-    public static final String[] COUNTER_NAMES = new String[]{
-            RADIUS_ACCEPT_RESPONSES_RX,
-            RADIUS_REJECT_RESPONSES_RX,
-            RADIUS_CHALLENGE_RESPONSES_RX,
-            RADIUS_ACCESS_REQUESTS_TX,
+    // this are the stats that represent a successful EAPOL exchange
+    public static final String[] EAPOL_SM_NAMES = new String[]{
+            EAPOL_START_REQ_RX,
+
+            EAPOL_ID_REQUEST_FRAMES_TX,
+            EAPOL_ID_RESP_FRAMES_RX,
+
             RADIUS_ACCESS_REQUESTS_IDENTITY_TX,
+            RADIUS_CHALLENGE_RESPONSES_RX,
+
+            EAPOL_CHALLENGE_REQ_TX,
+            EAPOL_MD5_CHALLENGE_RESP_RX,
+
             RADIUS_ACCESS_REQUESTS_CHALLENGE_TX,
+            RADIUS_ACCEPT_RESPONSES_RX,
+
+            EAPOL_AUTH_SUCCESS_TX,
+    };
+
+    // all other EAPOL Stats
+    public static final String[] EAPOL_STATS_NAMES = new String[]{
+            EAPOL_REQUEST_FRAMES_TX,
+            RADIUS_ACCESS_REQUESTS_TX,
+            RADIUS_REJECT_RESPONSES_RX,
             RADIUS_PENDING_REQUESTS,
             TIMED_OUT_PACKETS,
             UNKNOWN_TYPE_RX,
@@ -79,25 +98,22 @@
             REQUEST_RE_TX,
             NUM_SESSIONS_EXPIRED,
             EAPOL_LOGOFF_RX,
-            EAPOL_AUTH_SUCCESS_TX,
             EAPOL_AUTH_FAILURE_TX,
-            EAPOL_START_REQ_RX,
-            EAPOL_MD5_CHALLENGE_RESP_RX,
             EAPOL_TLS_CHALLENGE_RESP,
             EAPOL_TRANS_RESP_NOT_NAK,
-            EAPOL_CHALLENGE_REQ_TX,
-            EAPOL_ID_RESP_FRAMES_RX,
             EAPOL_ID_MSG_RESP_TX,
             EAPOL_FRAMES_TX,
             AUTH_STATE_IDLE,
-            EAPOL_ID_REQUEST_FRAMES_TX,
-            EAPOL_REQUEST_FRAMES_TX,
             INVALID_PKT_TYPE,
             INVALID_BODY_LENGTH,
             EAPOL_VALID_FRAMES_RX,
             EAPOL_PENDING_REQUESTS,
     };
 
+    public static final String[] COUNTER_NAMES =
+            Stream.concat(Arrays.stream(EAPOL_SM_NAMES), Arrays.stream(EAPOL_STATS_NAMES))
+            .toArray(String[]::new);
+
     // Number of access accept packets sent to the server
     private AtomicLong radiusAcceptResponsesRx = new AtomicLong();
     // Number of access reject packets sent to the server
diff --git a/app/src/main/java/org/opencord/aaa/cli/AaaShowCountersCommand.java b/app/src/main/java/org/opencord/aaa/cli/AaaShowCountersCommand.java
index 3c08464..42e0c17 100644
--- a/app/src/main/java/org/opencord/aaa/cli/AaaShowCountersCommand.java
+++ b/app/src/main/java/org/opencord/aaa/cli/AaaShowCountersCommand.java
@@ -37,9 +37,15 @@
 
         AaaStatisticsSnapshot stats = aaaStatisticsManager.getClusterStatistics();
 
-        for (String name : AaaStatistics.COUNTER_NAMES) {
+        print("-------------------------- Expected transitions ----------------------------");
+        for (String name : AaaStatistics.EAPOL_SM_NAMES) {
+            print("%30s %10d", name, stats.get(name));
+        }
+
+        print("-------------------------------- Other stats ----------------------------------");
+        for (String name : AaaStatistics.EAPOL_STATS_NAMES) {
             print("%30s %10d", name, stats.get(name));
         }
 
     }
-}
+}
\ No newline at end of file
diff --git a/app/src/main/java/org/opencord/aaa/impl/AaaManager.java b/app/src/main/java/org/opencord/aaa/impl/AaaManager.java
index 2f1a7e5..660ad33 100644
--- a/app/src/main/java/org/opencord/aaa/impl/AaaManager.java
+++ b/app/src/main/java/org/opencord/aaa/impl/AaaManager.java
@@ -25,6 +25,7 @@
 import static org.slf4j.LoggerFactory.getLogger;
 
 import com.google.common.collect.Sets;
+
 import java.net.InetAddress;
 import java.net.UnknownHostException;
 import java.nio.ByteBuffer;
@@ -235,8 +236,8 @@
     // Configuration properties factory
     private final ConfigFactory factory =
             new ConfigFactory<ApplicationId, AaaConfig>(APP_SUBJECT_FACTORY,
-                                                         AaaConfig.class,
-                                                         "AAA") {
+                                                        AaaConfig.class,
+                                                        "AAA") {
                 @Override
                 public AaaConfig createConfig() {
                     return new AaaConfig();
@@ -252,6 +253,7 @@
 
     protected ExecutorService packetProcessorExecutor;
     protected ScheduledExecutorService serverStatusAndStateMachineTimeoutExecutor;
+
     /**
      * Builds an EAPOL packet based on the given parameters.
      *
@@ -327,7 +329,7 @@
         getConfiguredAaaServerAddress();
         radiusOperationalStatusService.initialize(nasIpAddress.getAddress(), radiusSecret, impl);
         serverStatusAndStateMachineTimeoutExecutor = Executors.newScheduledThreadPool(STATE_MACHINE_THREADS,
-                           groupedThreads("onos/aaa", "aaa-machine-%d", log));
+                                              groupedThreads("onos/aaa", "aaa-machine-%d", log));
 
         scheduledStatusServerChecker = serverStatusAndStateMachineTimeoutExecutor.scheduleAtFixedRate(
                 new ServerStatusChecker(), 0,
@@ -353,6 +355,7 @@
 
         log.info("Stopped");
     }
+
     @Modified
     public void modified(ComponentContext context) {
         Dictionary<String, Object> properties = context.getProperties();
@@ -360,7 +363,7 @@
         String s = Tools.get(properties, "operationalStatusEventGenerationPeriodInSeconds");
         operationalStatusEventGenerationPeriodInSeconds = Strings.isNullOrEmpty(s)
                 ? OPERATIONAL_STATUS_SERVER_EVENT_GENERATION_DEFAULT
-                    : Integer.parseInt(s.trim());
+                : Integer.parseInt(s.trim());
 
         s = Tools.get(properties, "operationalStatusServerTimeoutInSeconds");
         operationalStatusServerTimeoutInSeconds = Strings.isNullOrEmpty(s) ? OPERATIONAL_STATUS_SERVER_TIMEOUT_DEFAULT
@@ -370,7 +373,7 @@
         String newEvaluationModeString = Strings.isNullOrEmpty(s) ? STATUS_SERVER_MODE_DEFAULT : s.trim();
 
         radiusOperationalStatusService
-            .setOperationalStatusServerTimeoutInMillis(operationalStatusServerTimeoutInSeconds * 1000);
+                .setOperationalStatusServerTimeoutInMillis(operationalStatusServerTimeoutInSeconds * 1000);
         RadiusOperationalStatusEvaluationMode newEvaluationMode =
                 RadiusOperationalStatusEvaluationMode.getValue(newEvaluationModeString);
         if (newEvaluationMode != null) {
@@ -388,8 +391,8 @@
             if (packetProcessorExecutor != null) {
                 packetProcessorExecutor.shutdown();
             }
-            packetProcessorExecutor = newSingleThreadExecutor(groupedThreads("onos/aaa", "aaa-packet-%d", log));
-
+            packetProcessorExecutor = newSingleThreadExecutor(
+                    groupedThreads("onos/aaa", "aaa-packet-%d", log));
         }
     }
 
@@ -398,7 +401,7 @@
             impl = new SocketBasedRadiusCommunicator(appId, packetService, this);
         } else {
             impl = new PortBasedRadiusCommunicator(appId, packetService, mastershipService,
-                    deviceService, subsService, pktCustomizer, this);
+                                                   deviceService, subsService, pktCustomizer, this);
         }
     }
 
@@ -425,7 +428,7 @@
             if (newCfg.radiusHostName() != null) {
                 address = InetAddress.getByName(newCfg.radiusHostName());
             } else {
-                 address = newCfg.radiusIp();
+                address = newCfg.radiusIp();
             }
 
             configuredAaaServerAddress = address.getHostAddress();
@@ -461,13 +464,14 @@
         // Compare the calculated Message-Authenticator with the one in the message
         return Arrays.equals(newHash, messageAuthenticator);
     }
+
     public void checkForPacketFromUnknownServer(String hostAddress) {
+        if (!hostAddress.equals(configuredAaaServerAddress)) {
+            getConfiguredAaaServerAddress();
             if (!hostAddress.equals(configuredAaaServerAddress)) {
-                 getConfiguredAaaServerAddress();
-                 if (!hostAddress.equals(configuredAaaServerAddress)) {
-                     aaaStatisticsManager.getAaaStats().incrementUnknownServerRx();
-                 }
+                aaaStatisticsManager.getAaaStats().incrementUnknownServerRx();
             }
+        }
     }
 
     /**
@@ -544,8 +548,9 @@
         ConnectPoint supplicantCp = stateMachine.supplicantConnectpoint();
         switch (radiusPacket.getCode()) {
             case RADIUS.RADIUS_CODE_ACCESS_CHALLENGE:
-                log.debug("RADIUS packet: RADIUS_CODE_ACCESS_CHALLENGE for dev/port: {}/{} with MacAddress {}",
-                          supplicantCp.deviceId(), supplicantCp.port(), dstMac);
+                log.debug("RADIUS packet: RADIUS_CODE_ACCESS_CHALLENGE for dev/port: {}/{} " +
+                                  "with MacAddress {} and Identifier {}",
+                          supplicantCp.deviceId(), supplicantCp.port(), dstMac, radiusPacket.getIdentifier() & 0xff);
                 RADIUSAttribute radiusAttrState = radiusPacket.getAttribute(RADIUSAttribute.RADIUS_ATTR_STATE);
                 byte[] challengeState = null;
                 if (radiusAttrState != null) {
@@ -563,8 +568,9 @@
                     log.error(e.getMessage());
                     break;
                 }
-                log.debug("Send EAP challenge response to supplicant {} on dev/port: {}/{} with MacAddress {}",
-                          supplicantCp.deviceId(), supplicantCp.port(), dstMac);
+                log.debug("Send EAP challenge response to supplicant on dev/port: {}/{}" +
+                                  " with MacAddress {} and Identifier {}",
+                          supplicantCp.deviceId(), supplicantCp.port(), dstMac, radiusPacket.getIdentifier() & 0xff);
                 sendPacketToSupplicant(eth, stateMachine.supplicantConnectpoint(), true);
                 aaaStatisticsManager.getAaaStats().increaseChallengeResponsesRx();
                 outPacketSupp.add(eapPayload.getIdentifier());
@@ -574,8 +580,9 @@
                 machineStats.incrementTotalOctetSent(eapPayload.getLength());
                 break;
             case RADIUS.RADIUS_CODE_ACCESS_ACCEPT:
-                log.debug("RADIUS packet: RADIUS_CODE_ACCESS_ACCEPT for dev/port: {}/{} with MacAddress {}",
-                          supplicantCp.deviceId(), supplicantCp.port(), dstMac);
+                log.debug("RADIUS packet: RADIUS_CODE_ACCESS_ACCEPT for dev/port: {}/{}" +
+                                  " with MacAddress {} and Identifier {}",
+                          supplicantCp.deviceId(), supplicantCp.port(), dstMac, radiusPacket.getIdentifier() & 0xff);
                 //send an EAPOL - Success to the supplicant.
                 byte[] eapMessageSuccess =
                         radiusPacket.getAttribute(RADIUSAttribute.RADIUS_ATTR_EAP_MESSAGE).getValue();
@@ -592,8 +599,9 @@
                                          stateMachine.vlanId(),
                                          EAPOL.EAPOL_PACKET,
                                          eapPayload, stateMachine.priorityCode());
-                log.info("Send EAP success message to supplicant {} on dev/port: {}/{} with MacAddress {}",
-                         supplicantCp.deviceId(), supplicantCp.port(), dstMac);
+                log.info("Send EAP success message to supplicant on dev/port: {}/{}" +
+                                 " with MacAddress {} and Identifier {}",
+                         supplicantCp.deviceId(), supplicantCp.port(), dstMac, radiusPacket.getIdentifier() & 0xff);
                 sendPacketToSupplicant(eth, stateMachine.supplicantConnectpoint(), false);
                 aaaStatisticsManager.getAaaStats().incrementEapolAuthSuccessTrans();
 
@@ -604,8 +612,9 @@
                 machineStats.incrementTotalOctetSent(eapPayload.getLength());
                 break;
             case RADIUS.RADIUS_CODE_ACCESS_REJECT:
-                log.debug("RADIUS packet: RADIUS_CODE_ACCESS_REJECT for dev/port: {}/{} with MacAddress {}",
-                          supplicantCp.deviceId(), supplicantCp.port(), dstMac);
+                log.debug("RADIUS packet: RADIUS_CODE_ACCESS_REJECT for dev/port: {}/{}" +
+                                  " with MacAddress {} and Identifier {}",
+                          supplicantCp.deviceId(), supplicantCp.port(), dstMac, radiusPacket.getIdentifier() & 0xff);
                 //send an EAPOL - Failure to the supplicant.
                 byte[] eapMessageFailure;
                 eapPayload = new EAP();
@@ -629,8 +638,9 @@
                                          stateMachine.vlanId(),
                                          EAPOL.EAPOL_PACKET,
                                          eapPayload, stateMachine.priorityCode());
-                log.warn("Send EAP failure message to supplicant {} on dev/port: {}/{} with MacAddress {}",
-                         supplicantCp.deviceId(), supplicantCp.port(), dstMac);
+                log.warn("Send EAP failure message to supplicant {} on dev/port: {}/{}" +
+                                 " with MacAddress {} and Identifier {}", supplicantCp.deviceId(), supplicantCp.port(),
+                         dstMac, stateMachine.challengeIdentifier() & 0xff);
                 sendPacketToSupplicant(eth, stateMachine.supplicantConnectpoint(), false);
                 aaaStatisticsManager.getAaaStats().incrementEapolauthFailureTrans();
 
@@ -646,8 +656,10 @@
                                                               machineObj));
                 break;
             default:
-                log.warn("Unknown RADIUS message received with code: {} for dev/port: {}/{} with MacAddress {}",
-                         radiusPacket.getCode(), supplicantCp.deviceId(), supplicantCp.port(), dstMac);
+                log.warn("Unknown RADIUS message received with code: {} for dev/port: {}/{}" +
+                                 " with MacAddress {} and Identifier {}",
+                         radiusPacket.getCode(), supplicantCp.deviceId(), supplicantCp.port(), dstMac,
+                         radiusPacket.getIdentifier() & 0xff);
                 aaaStatisticsManager.getAaaStats().increaseUnknownTypeRx();
                 //increasing packets received to server
                 machineStats.incrementTotalPacketsReceived();
@@ -730,17 +742,31 @@
             packetProcessorExecutor.execute(() -> {
                 // Extract the original Ethernet frame from the packet information
                 InboundPacket pkt = context.inPacket();
+                if (pkt == null) {
+                    log.error("InboundPacket is null when parsed from {}", context);
+                    return;
+                }
                 Ethernet ethPkt = pkt.parsed();
                 if (ethPkt == null) {
+                    log.error("EthPkt is null when parsed from {}", pkt);
                     return;
                 }
 
                 // identify if incoming packet comes from supplicant (EAP) or RADIUS
                 switch (EthType.EtherType.lookup(ethPkt.getEtherType())) {
                     case EAPOL:
+                        if (log.isTraceEnabled()) {
+                            log.trace("Received EAPOL supplicant packet from dev/port: {} with MacAddress {}",
+                                      context.inPacket().receivedFrom(), ethPkt.getSourceMAC());
+                        }
                         handleSupplicantPacket(context.inPacket());
                         break;
                     default:
+                        if (log.isTraceEnabled()) {
+                            log.trace("Received packet-in from RADIUS server {} in enclosing packet {} from "
+                                              + "dev/port: {} with MacAddress {}", ethPkt, context.inPacket(),
+                                      context.inPacket().receivedFrom(), ethPkt.getSourceMAC());
+                        }
                         // any other packets let the specific implementation handle
                         impl.handlePacketFromServer(context);
                 }
@@ -751,7 +777,7 @@
          * Creates and initializes common fields of a RADIUS packet.
          *
          * @param stateMachine state machine for the request
-         * @param eapPacket  EAP packet
+         * @param eapPacket    EAP packet
          * @return RADIUS packet
          */
         private RADIUS getRadiusPayload(StateMachine stateMachine, byte identifier, EAP eapPacket) {
@@ -767,7 +793,7 @@
                                        stateMachine.username());
 
             radiusPayload.setAttribute(RADIUSAttribute.RADIUS_ATTR_NAS_IP,
-                    AaaManager.this.nasIpAddress.getAddress());
+                                       AaaManager.this.nasIpAddress.getAddress());
 
             radiusPayload.encapsulateMessage(eapPacket);
 
@@ -780,6 +806,7 @@
          * @param inPacket Ethernet packet coming from the supplicant
          */
         private void handleSupplicantPacket(InboundPacket inPacket) {
+
             Ethernet ethPkt = inPacket.parsed();
             // Where does it come from?
             MacAddress srcMac = ethPkt.getSourceMAC();
@@ -788,28 +815,31 @@
             PortNumber portNumber = inPacket.receivedFrom().port();
             String sessionId = sessionId(inPacket.receivedFrom());
             EAPOL eapol = (EAPOL) ethPkt.getPayload();
+
             if (log.isTraceEnabled()) {
                 log.trace("Received EAPOL packet {} in enclosing packet {} from "
-                        + "dev/port: {}/{} with MacAddress {}",
-                          eapol, ethPkt, deviceId,
-                          portNumber, srcMac);
+                                  + "dev/port: {}/{} with MacAddress {} and type {}",
+                          eapol, ethPkt, deviceId, portNumber, srcMac, eapol.getEapolType());
             }
 
             short pktlen = eapol.getPacketLength();
             byte[] eapPayLoadBuffer = eapol.serialize();
             int len = eapPayLoadBuffer.length;
             if (len != (HEADER_LENGTH + pktlen)) {
+                log.error("Invalid EAPOL pkt length {} (shoudl be {}) for packet {} from dev/port: {}/{} " +
+                          "with MacAddress {}", len, HEADER_LENGTH + pktlen, eapol, deviceId, portNumber, srcMac);
                 aaaStatisticsManager.getAaaStats().incrementInvalidBodyLength();
                 return;
             }
             if (!VALID_EAPOL_TYPE.contains(eapol.getEapolType())) {
+                log.error("Invalid EAPOL Type {} for packet {} from dev/port: {}/{} with MacAddress {}",
+                          eapol.getEapolType(), eapol, deviceId, portNumber, srcMac);
                 aaaStatisticsManager.getAaaStats().incrementInvalidPktType();
                 return;
             }
             if (pktlen >= 0 && ethPkt.getEtherType() == EthType.EtherType.EAPOL.ethType().toShort()) {
                 aaaStatisticsManager.getAaaStats().incrementValidEapolFramesRx();
             }
-
             StateMachine stateMachine = stateMachines.computeIfAbsent(sessionId, id ->
                     new StateMachine(id, serverStatusAndStateMachineTimeoutExecutor));
             stateMachine.setEapolTypeVal(eapol.getEapolType());
@@ -826,17 +856,16 @@
                     //send an EAP Request/Identify to the supplicant
                     EAP eapPayload = new EAP(EAP.REQUEST, stateMachine.identifier(), EAP.ATTR_IDENTITY, null);
                     if (ethPkt.getVlanID() != Ethernet.VLAN_UNTAGGED) {
-                       stateMachine.setPriorityCode(ethPkt.getPriorityCode());
+                        stateMachine.setPriorityCode(ethPkt.getPriorityCode());
                     }
                     Ethernet eth = buildEapolResponse(srcMac, MacAddress.valueOf(nasMacAddress),
-                                                      ethPkt.getVlanID(), EAPOL.EAPOL_PACKET,
-                                                      eapPayload, stateMachine.priorityCode());
+                                      ethPkt.getVlanID(), EAPOL.EAPOL_PACKET, eapPayload, stateMachine.priorityCode());
 
                     stateMachine.setVlanId(ethPkt.getVlanID());
-                    log.debug("Getting EAP identity from supplicant {}", stateMachine.supplicantAddress().toString());
+                    log.debug("Getting EAP identity from supplicant {} and Identifier {}",
+                              stateMachine.supplicantAddress().toString(), stateMachine.identifier() & 0xff);
                     sendPacketToSupplicant(eth, stateMachine.supplicantConnectpoint(), false);
                     aaaStatisticsManager.getAaaStats().incrementRequestIdFramesTx();
-
                     break;
                 case EAPOL.EAPOL_LOGOFF:
                     log.debug("EAP packet: EAPOL_LOGOFF from dev/port: {}/{} with MacAddress {}",
@@ -857,117 +886,39 @@
                     if (stateMachine.state() == StateMachine.STATE_IDLE) {
                         aaaStatisticsManager.getAaaStats().incrementAuthStateIdle();
                     }
-
                     break;
                 case EAPOL.EAPOL_PACKET:
-                    RADIUS radiusPayload;
+
                     // check if this is a Response/Identify or  a Response/TLS
                     EAP eapPacket = (EAP) eapol.getPayload();
                     Byte identifier = new Byte(eapPacket.getIdentifier());
-
+                    log.debug("EAP packet: EAPOL_PACKET from dev/port: {}/{} with MacAddress {} with Identifier {}",
+                              deviceId, portNumber, srcMac, identifier.doubleValue());
                     // get identifier for request and store mapping to session ID
                     RequestIdentifier radiusIdentifier = idManager.getNewIdentifier(sessionId);
+                    if (radiusIdentifier == null) {
+                        log.error("Cannot get RADIUS Identifier, dropping packet");
+                        return;
+                    }
 
                     byte dataType = eapPacket.getDataType();
+
                     switch (dataType) {
-
                         case EAP.ATTR_IDENTITY:
-                            log.debug("EAP packet: EAPOL_PACKET ATTR_IDENTITY from dev/port: {}/{} with MacAddress {}",
-                                      deviceId, portNumber, srcMac);
-                            //Setting the time of this response from RG, only when its not a re-transmission.
-                            if (stateMachine.getLastPacketReceivedTime() == 0) {
-                               stateMachine.setLastPacketReceivedTime(System.currentTimeMillis());
-                            }
-                            // request id access to RADIUS
-                            stateMachine.setUsername(eapPacket.getData());
-
-                            radiusPayload = getRadiusPayload(stateMachine, radiusIdentifier.identifier(), eapPacket);
-                            radiusPayload = pktCustomizer.customizePacket(radiusPayload, inPacket);
-                            radiusPayload.addMessageAuthenticator(radiusSecret);
-
-                            if (log.isTraceEnabled()) {
-                                log.trace("Sending ATTR_IDENTITY packet to RADIUS for supplicant at dev/port: " +
-                                                  "{}/{} with MacAddress {}", deviceId, portNumber, srcMac);
-                            }
-
-                            sendRadiusPacket(radiusPayload, inPacket);
-                            stateMachine.setWaitingForRadiusResponse(true);
-                            aaaStatisticsManager.getAaaStats().incrementRadiusReqIdTx();
-                            aaaStatisticsManager.getAaaStats().incrementEapolAtrrIdentity();
-                            // change the state to "PENDING"
-                            if (stateMachine.state() == StateMachine.STATE_PENDING) {
-                                aaaStatisticsManager.getAaaStats().increaseRequestReTx();
-                                stateMachine.incrementTotalPacketsSent();
-                                stateMachine.incrementTotalOctetSent(eapol.getPacketLength());
-                            }
-                            stateMachine.requestAccess();
+                            handleAttrIdentity(inPacket, srcMac, deviceId, portNumber,
+                                               eapol, stateMachine, eapPacket, radiusIdentifier);
                             break;
                         case EAP.ATTR_MD5:
-                            log.debug("EAP packet: EAPOL_PACKET ATTR_MD5 from dev/port: {}/{} with MacAddress {}",
-                                      deviceId, portNumber, srcMac);
-                            // verify if the EAP identifier corresponds to the
-                            // challenge identifier from the client state
-                            // machine.
-                            stateMachine.setLastPacketReceivedTime(System.currentTimeMillis());
-                            if (eapPacket.getIdentifier() == stateMachine.challengeIdentifier()) {
-                                //send the RADIUS challenge response
-                                radiusPayload = getRadiusPayload(stateMachine,
-                                        radiusIdentifier.identifier(), eapPacket);
-                                radiusPayload = pktCustomizer.customizePacket(radiusPayload, inPacket);
-
-                                if (stateMachine.challengeState() != null) {
-                                    radiusPayload.setAttribute(RADIUSAttribute.RADIUS_ATTR_STATE,
-                                            stateMachine.challengeState());
-                                }
-                                radiusPayload.addMessageAuthenticator(radiusSecret);
-                                if (outPacketSupp.contains(eapPacket.getIdentifier())) {
-                                    aaaStatisticsManager.getAaaStats().decrementPendingReqSupp();
-                                    outPacketSupp.remove(identifier);
-                                }
-                                if (log.isTraceEnabled()) {
-                                    log.trace("Sending ATTR_MD5 packet to RADIUS for supplicant at dev/port: " +
-                                                      "{}/{} with MacAddress {}", deviceId, portNumber, srcMac);
-                                }
-                                sendRadiusPacket(radiusPayload, inPacket);
-                                stateMachine.setWaitingForRadiusResponse(true);
-                                aaaStatisticsManager.getAaaStats().incrementRadiusReqChallengeTx();
-                                aaaStatisticsManager.getAaaStats().incrementEapolMd5RspChall();
-                            }
+                            handleMD5(inPacket, srcMac, deviceId, portNumber, stateMachine,
+                                      eapPacket, identifier, radiusIdentifier);
                             break;
                         case EAP.ATTR_TLS:
-                            log.debug("EAP packet: EAPOL_PACKET ATTR_TLS from dev/port: {}/{} with MacAddress {}",
-                                      deviceId, portNumber, srcMac);
-                            // request id access to RADIUS
-                            radiusPayload = getRadiusPayload(stateMachine, radiusIdentifier.identifier(), eapPacket);
-                            radiusPayload = pktCustomizer.customizePacket(radiusPayload, inPacket);
-
-                            if (stateMachine.challengeState() != null) {
-                                radiusPayload.setAttribute(RADIUSAttribute.RADIUS_ATTR_STATE,
-                                        stateMachine.challengeState());
-                            }
-                            stateMachine.setRequestAuthenticator(radiusPayload.generateAuthCode());
-                            radiusPayload.addMessageAuthenticator(radiusSecret);
-                            if (outPacketSupp.contains(eapPacket.getIdentifier())) {
-                                aaaStatisticsManager.getAaaStats().decrementPendingReqSupp();
-                                outPacketSupp.remove(identifier);
-                            }
-                            if (log.isTraceEnabled()) {
-                                log.trace("Sending ATTR_TLS packet to RADIUS for supplicant at dev/port: " +
-                                                  "{}/{} with MacAddress {}", deviceId, portNumber, srcMac);
-                            }
-                            sendRadiusPacket(radiusPayload, inPacket);
-                            stateMachine.setWaitingForRadiusResponse(true);
-                            aaaStatisticsManager.getAaaStats().incrementRadiusReqChallengeTx();
-                            aaaStatisticsManager.getAaaStats().incrementEapolTlsRespChall();
-
-                            if (stateMachine.state() != StateMachine.STATE_PENDING) {
-                                stateMachine.requestAccess();
-                            }
-
+                            handleTls(inPacket, srcMac, deviceId, portNumber, stateMachine,
+                                      eapPacket, identifier, radiusIdentifier);
                             break;
                         default:
-                            log.warn("Unknown EAP packet type from dev/port: {}/{} with MacAddress {}",
-                                     deviceId, portNumber, srcMac);
+                            log.warn("Unknown EAP packet type from dev/port: {}/{} with MacAddress {} and " +
+                                     "Identifier {}", deviceId, portNumber, srcMac, eapPacket.getIdentifier() & 0xff);
                             return;
                     }
                     break;
@@ -978,6 +929,124 @@
             aaaStatisticsManager.getAaaStats().countTransRespNotNak();
             aaaStatisticsManager.getAaaStats().countEapolResIdentityMsgTrans();
         }
+
+        private void handleAttrIdentity(InboundPacket inPacket, MacAddress srcMac, DeviceId deviceId,
+                                        PortNumber portNumber, EAPOL eapol, StateMachine stateMachine,
+                                        EAP eapPacket, RequestIdentifier radiusIdentifier) {
+            RADIUS radiusPayload;
+            log.debug("EAP packet: EAPOL_PACKET ATTR_IDENTITY from dev/port: {}/{} with MacAddress {}" +
+                              " and Identifier {}", deviceId, portNumber, srcMac, eapPacket.getIdentifier() & 0xff);
+            //Setting the time of this response from RG, only when its not a re-transmission.
+            if (stateMachine.getLastPacketReceivedTime() == 0) {
+                stateMachine.setLastPacketReceivedTime(System.currentTimeMillis());
+            }
+            // request id access to RADIUS
+            stateMachine.setUsername(eapPacket.getData());
+
+            radiusPayload = getRadiusPayload(stateMachine, radiusIdentifier.identifier(), eapPacket);
+            radiusPayload = pktCustomizer.customizePacket(radiusPayload, inPacket);
+            radiusPayload.addMessageAuthenticator(radiusSecret);
+
+            if (log.isTraceEnabled()) {
+                log.trace("Sending ATTR_IDENTITY packet to RADIUS for supplicant at dev/port: " +
+                                  "{}/{} with MacAddress {} and Identifier {}", deviceId, portNumber,
+                          srcMac, radiusIdentifier.identifier() & 0xff);
+            }
+
+            sendRadiusPacket(radiusPayload, inPacket);
+            stateMachine.setWaitingForRadiusResponse(true);
+            aaaStatisticsManager.getAaaStats().incrementRadiusReqIdTx();
+            aaaStatisticsManager.getAaaStats().incrementEapolAtrrIdentity();
+            // change the state to "PENDING"
+            if (stateMachine.state() == StateMachine.STATE_PENDING) {
+                aaaStatisticsManager.getAaaStats().increaseRequestReTx();
+                stateMachine.incrementTotalPacketsSent();
+                stateMachine.incrementTotalOctetSent(eapol.getPacketLength());
+            }
+            stateMachine.requestAccess();
+        }
+
+        private void handleMD5(InboundPacket inPacket, MacAddress srcMac, DeviceId deviceId,
+                               PortNumber portNumber, StateMachine stateMachine, EAP eapPacket,
+                               Byte identifier, RequestIdentifier radiusIdentifier) {
+            RADIUS radiusPayload;
+            log.debug("EAP packet: EAPOL_PACKET ATTR_MD5 from dev/port: {}/{} with MacAddress {}" +
+                              " and Identifier {}", deviceId, portNumber, srcMac, eapPacket.getIdentifier() & 0xff);
+            // verify if the EAP identifier corresponds to the
+            // challenge identifier from the client state
+            // machine.
+            stateMachine.setLastPacketReceivedTime(System.currentTimeMillis());
+            if (eapPacket.getIdentifier() == stateMachine.challengeIdentifier()) {
+                //send the RADIUS challenge response
+                radiusPayload = getRadiusPayload(stateMachine,
+                                                 radiusIdentifier.identifier(), eapPacket);
+                radiusPayload = pktCustomizer.customizePacket(radiusPayload, inPacket);
+
+                if (stateMachine.challengeState() != null) {
+                    radiusPayload.setAttribute(RADIUSAttribute.RADIUS_ATTR_STATE,
+                                               stateMachine.challengeState());
+                }
+                radiusPayload.addMessageAuthenticator(radiusSecret);
+                if (outPacketSupp.contains(eapPacket.getIdentifier())) {
+                    aaaStatisticsManager.getAaaStats().decrementPendingReqSupp();
+                    outPacketSupp.remove(identifier);
+                }
+                if (log.isTraceEnabled()) {
+                    log.trace("Sending ATTR_MD5 packet to RADIUS for supplicant at dev/port: {}/{}" +
+                                      " with MacAddress {} and Identifier {}", deviceId, portNumber, srcMac,
+                              radiusIdentifier.identifier() & 0xff);
+                }
+                sendRadiusPacket(radiusPayload, inPacket);
+                stateMachine.setWaitingForRadiusResponse(true);
+                aaaStatisticsManager.getAaaStats().incrementRadiusReqChallengeTx();
+                aaaStatisticsManager.getAaaStats().incrementEapolMd5RspChall();
+            } else {
+                log.error("eapolIdentifier {} and stateMachine Identifier {} do not " +
+                                  "correspond for packet from dev/port: {}/{} with MacAddress {}",
+                          eapPacket.getIdentifier() & 0xff, stateMachine.challengeIdentifier() & 0xff,
+                          deviceId, portNumber, srcMac);
+                aaaStatisticsManager.getAaaStats().incrementEapolMd5RspChall();
+                if (outPacketSupp.contains(eapPacket.getIdentifier())) {
+                    aaaStatisticsManager.getAaaStats().decrementPendingReqSupp();
+                    outPacketSupp.remove(identifier);
+                }
+                aaaStatisticsManager.getAaaStats().incrementEapolauthFailureTrans();
+            }
+        }
+
+
+        private void handleTls(InboundPacket inPacket, MacAddress srcMac, DeviceId deviceId,
+                               PortNumber portNumber, StateMachine stateMachine, EAP eapPacket,
+                               Byte identifier, RequestIdentifier radiusIdentifier) {
+            RADIUS radiusPayload;
+            log.debug("EAP packet: EAPOL_PACKET ATTR_TLS from dev/port: {}/{} with MacAddress {} " +
+                              "and Identifier {}", deviceId, portNumber, srcMac, eapPacket.getIdentifier() & 0xff);
+            // request id access to RADIUS
+            radiusPayload = getRadiusPayload(stateMachine, radiusIdentifier.identifier(), eapPacket);
+            radiusPayload = pktCustomizer.customizePacket(radiusPayload, inPacket);
+            if (stateMachine.challengeState() != null) {
+                radiusPayload.setAttribute(RADIUSAttribute.RADIUS_ATTR_STATE,
+                                           stateMachine.challengeState());
+            }
+            stateMachine.setRequestAuthenticator(radiusPayload.generateAuthCode());
+            radiusPayload.addMessageAuthenticator(radiusSecret);
+            if (outPacketSupp.contains(eapPacket.getIdentifier())) {
+                aaaStatisticsManager.getAaaStats().decrementPendingReqSupp();
+                outPacketSupp.remove(identifier);
+            }
+            if (log.isTraceEnabled()) {
+                log.trace("Sending ATTR_TLS packet to RADIUS for supplicant at dev/port: {}/{} with " +
+                                  "MacAddress {} and Identifier {}", deviceId, portNumber, srcMac,
+                          radiusIdentifier.identifier() & 0xff);
+            }
+            sendRadiusPacket(radiusPayload, inPacket);
+            stateMachine.setWaitingForRadiusResponse(true);
+            aaaStatisticsManager.getAaaStats().incrementRadiusReqChallengeTx();
+            aaaStatisticsManager.getAaaStats().incrementEapolTlsRespChall();
+            if (stateMachine.state() != StateMachine.STATE_PENDING) {
+                stateMachine.requestAccess();
+            }
+        }
     }
 
     /**
@@ -988,7 +1057,7 @@
         @Override
         public void notify(AuthenticationEvent authenticationEvent) {
             log.info("Auth event {} for {}",
-                    authenticationEvent.type(), authenticationEvent.subject());
+                     authenticationEvent.type(), authenticationEvent.subject());
 
             if (authenticationEvent.type() == AuthenticationEvent.Type.TIMEOUT) {
                 handleStateMachineTimeout(authenticationEvent.subject());
@@ -1121,7 +1190,7 @@
                     log.debug("Received PORT_REMOVED event. Clearing AAA Session with Id {}", sessionId);
 
                     flushStateMachineSession(sessionId,
-                            StateMachine.SessionTerminationReasons.PORT_REMOVED.getReason());
+                                             StateMachine.SessionTerminationReasons.PORT_REMOVED.getReason());
 
                     break;
                 case DEVICE_REMOVED:
@@ -1146,7 +1215,7 @@
             for (String session : associatedSessions) {
                 log.debug("Clearing AAA Session {} associated with Removed Device", session);
                 flushStateMachineSession(session,
-                       StateMachine.SessionTerminationReasons.DEVICE_REMOVED.getReason());
+                                         StateMachine.SessionTerminationReasons.DEVICE_REMOVED.getReason());
             }
         }
 
@@ -1164,7 +1233,7 @@
             //pushing captured machine stats to kafka
             AaaSupplicantMachineStats obj = aaaSupplicantStatsManager.getSupplicantStats(stateMachine);
             aaaSupplicantStatsManager.getMachineStatsDelegate()
-                   .notify(new AaaMachineStatisticsEvent(AaaMachineStatisticsEvent.Type.STATS_UPDATE, obj));
+                    .notify(new AaaMachineStatisticsEvent(AaaMachineStatisticsEvent.Type.STATS_UPDATE, obj));
             StateMachine removed = stateMachines.remove(sessionId);
 
             if (removed != null) {
@@ -1180,10 +1249,10 @@
             radiusOperationalStatusService.checkServerOperationalStatus();
             log.trace("--POSTING--" + radiusOperationalStatusService.getRadiusServerOperationalStatus());
             radiusOperationalStatusService.getRadiusOprStDelegate()
-                .notify(new RadiusOperationalStatusEvent(
-                        RadiusOperationalStatusEvent.Type.RADIUS_OPERATIONAL_STATUS,
-                        radiusOperationalStatusService.
-                        getRadiusServerOperationalStatus()));
+                    .notify(new RadiusOperationalStatusEvent(
+                            RadiusOperationalStatusEvent.Type.RADIUS_OPERATIONAL_STATUS,
+                            radiusOperationalStatusService.
+                                    getRadiusServerOperationalStatus()));
         }
 
     }
diff --git a/app/src/main/java/org/opencord/aaa/impl/RadiusOperationalStatusManager.java b/app/src/main/java/org/opencord/aaa/impl/RadiusOperationalStatusManager.java
index 77e58c0..377cd0a 100644
--- a/app/src/main/java/org/opencord/aaa/impl/RadiusOperationalStatusManager.java
+++ b/app/src/main/java/org/opencord/aaa/impl/RadiusOperationalStatusManager.java
@@ -51,6 +51,7 @@
     private long serverStatusOutTimeInMillis;
 
     private OperationalStatus radiusServerOperationalStatus;
+
     public static final byte AAA_REQUEST_ID_STATUS_REQUEST = 0;
     public static final byte AAA_REQUEST_ID_FAKE_ACCESS_REQUEST = 1;
 
diff --git a/app/src/main/java/org/opencord/aaa/impl/SocketBasedRadiusCommunicator.java b/app/src/main/java/org/opencord/aaa/impl/SocketBasedRadiusCommunicator.java
index 99c9396..0414cac 100755
--- a/app/src/main/java/org/opencord/aaa/impl/SocketBasedRadiusCommunicator.java
+++ b/app/src/main/java/org/opencord/aaa/impl/SocketBasedRadiusCommunicator.java
@@ -105,6 +105,7 @@
 
     @Override
     public void clearLocalState() {
+        log.info("Closing RADIUS socket: {}:{}", radiusIpAddress, radiusServerPort);
         radiusSocket.close();
         executor.shutdownNow();
     }
@@ -154,7 +155,7 @@
                 aaaManager.radiusOperationalStatusService.setStatusServerReqSent(false);
             }
         } catch (IOException e) {
-            log.info("Cannot send packet to RADIUS server", e);
+            log.warn("Cannot send packet to RADIUS server", e);
         }
     }
 
@@ -193,6 +194,10 @@
                                         .deserialize(inboundBasePacket.getData(),
                                                 0,
                                                 inboundBasePacket.getLength());
+                        if (log.isTraceEnabled()) {
+                            log.trace("Received RADIUS packet with Identifier {}",
+                                      inboundRadiusPacket.getIdentifier() & 0xff);
+                        }
                         aaaManager.aaaStatisticsManager.handleRoundtripTime(inboundRadiusPacket.getIdentifier());
                         aaaManager.handleRadiusPacket(inboundRadiusPacket);
                     } catch (DeserializationException dex) {
@@ -200,10 +205,11 @@
                         log.error("Cannot deserialize packet", dex);
                     }
                 } catch (IOException e) {
-                    log.info("Socket was closed, exiting listener thread");
+                    log.warn("Socket was closed, exiting listener thread");
                     done = true;
                 }
             }
+            log.info("UDP listener thread shutting down");
         }
     }
 
diff --git a/app/src/test/java/org/opencord/aaa/impl/IdentifierManagerTest.java b/app/src/test/java/org/opencord/aaa/impl/IdentifierManagerTest.java
new file mode 100644
index 0000000..890f2c6
--- /dev/null
+++ b/app/src/test/java/org/opencord/aaa/impl/IdentifierManagerTest.java
@@ -0,0 +1,57 @@
+/*
+ * Copyright 2017-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.opencord.aaa.impl;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+
+import static org.junit.Assert.assertNotEquals;
+import static org.slf4j.LoggerFactory.getLogger;
+
+public class IdentifierManagerTest {
+
+    IdentifierManager idManager = null;
+    private final Logger log = getLogger(getClass());
+
+    @Before
+    public void setUp() {
+        System.out.print("Set up");
+        idManager = new IdentifierManager();
+    }
+
+    @After
+    public void tearDown() {
+        System.out.print("Tear down");
+        idManager = null;
+    }
+
+    @Test
+    /**
+     * Make sure that we never get ID 1 or 0 as they are reserved for RadiusOperationalStatusManager
+     */
+    public void testIdSequence() {
+        for (int i = 1; i <= 300; i++) {
+            RequestIdentifier id = idManager.getNewIdentifier(Integer.toString(i));
+            log.trace("Id: {}", id.identifier() & 0xff);
+            assertNotEquals(id.identifier(), 0);
+            assertNotEquals(id.identifier(), 1);
+            idManager.releaseIdentifier(id);
+        }
+    }
+}