VOL-2664 Avoid overwriting vlan-filter-tasks in queue

-tag_add check is done in vlan-filter-task for add and remove cases.
-also a few log corrections are done.

Change-Id: Iec969bbbb9bffdf1236601b6a182eb0a88012416
diff --git a/python/adapters/brcm_openomci_onu/brcm_openomci_onu_handler.py b/python/adapters/brcm_openomci_onu/brcm_openomci_onu_handler.py
index 3e6760d..b3f9510 100644
--- a/python/adapters/brcm_openomci_onu/brcm_openomci_onu_handler.py
+++ b/python/adapters/brcm_openomci_onu/brcm_openomci_onu_handler.py
@@ -425,17 +425,17 @@
             if uni_id in self._queued_vlan_filter_task and tp_id in self._queued_vlan_filter_task[uni_id]:
                 self.log.info("executing-queued-vlan-filter-task",
                               uni_id=uni_id, tp_id=tp_id)
-                filter_info = self._queued_vlan_filter_task[uni_id][tp_id]
-                reactor.callLater(0, self._add_vlan_filter_task, filter_info.get("device"),
+                for filter_info in self._queued_vlan_filter_task[uni_id][tp_id]:
+                    reactor.callLater(0, self._add_vlan_filter_task, filter_info.get("device"),
                                   uni_id=uni_id, uni_port=filter_info.get("uni_port"),
                                   match_vlan = filter_info.get("match_vlan"),
                                   _set_vlan_vid= filter_info.get("set_vlan_vid"),
                                   _set_vlan_pcp = filter_info.get("set_vlan_pcp"),
                                   tp_id = filter_info.get("tp_id"))
                 # Now remove the entry from the dictionary
-                self._queued_vlan_filter_task[uni_id].clear()
+                self._queued_vlan_filter_task[uni_id][tp_id].remove(filter_info)
                 self.log.debug("executed-queued-vlan-filter-task",
-                               uni_id=uni_id)
+                               uni_id=uni_id, tp_id=tp_id)
         except Exception as e:
             self.log.error("vlan-filter-configuration-failed", uni_id=uni_id, error=e)
 
@@ -502,7 +502,7 @@
                     yield self.core_proxy.device_reason_update(self.device_id,
                                                                'tech-profile-config-download-failure-retrying')
 
-                self.log.info('downloading-tech-profile-configuration', uni_id=uni_id, tp_id=tp_id,)
+                self.log.info('downloading-tech-profile-configuration', uni_id=uni_id, tp_id=tp_id)
                 self.log.debug("tconts-gems-to-install", tconts=tconts, gem_ports=gem_ports)
 
                 self._tp_service_specific_task[uni_id][tp_path] = \
@@ -1089,13 +1089,15 @@
                           uni_id=uni_id, tp_id=tp_id)
             if uni_id not in self._queued_vlan_filter_task:
                 self._queued_vlan_filter_task[uni_id] = dict()
-            self._queued_vlan_filter_task[uni_id][tp_id] = {"device": device,
+            if tp_id not in self._queued_vlan_filter_task[uni_id]:
+                self._queued_vlan_filter_task[uni_id][tp_id] = []
+            self._queued_vlan_filter_task[uni_id][tp_id].append({"device": device,
                                                             "uni_id": uni_id,
                                                             "uni_port": uni_port,
                                                             "match_vlan": match_vlan,
                                                             "set_vlan_vid": _set_vlan_vid,
                                                             "set_vlan_pcp": _set_vlan_pcp,
-                                                            "tp_id": tp_id}
+                                                            "tp_id": tp_id})
 
     def get_tp_id_in_flow(self, flow):
         flow_metadata = fd.get_metadata_from_write_metadata ( flow )
diff --git a/python/adapters/brcm_openomci_onu/omci/brcm_tp_setup_task.py b/python/adapters/brcm_openomci_onu/omci/brcm_tp_setup_task.py
index d93224a..9d2eb09 100644
--- a/python/adapters/brcm_openomci_onu/omci/brcm_tp_setup_task.py
+++ b/python/adapters/brcm_openomci_onu/omci/brcm_tp_setup_task.py
@@ -161,7 +161,8 @@
 
     @inlineCallbacks
     def perform_service_specific_steps(self):
-        self.log.info('creating-tcont-mapper-gemport-iw')
+        self.log.info('starting-tech-profile-setup', uni_id=self._uni_port.uni_id,
+                        tconts=self._tconts, gem_ports=self._gem_ports, tp_table_id=self._tp_table_id)
 
         omci_cc = self._onu_device.omci_cc
         gem_pq_associativity = dict()
diff --git a/python/adapters/brcm_openomci_onu/omci/brcm_vlan_filter_task.py b/python/adapters/brcm_openomci_onu/omci/brcm_vlan_filter_task.py
index 23cd83d..20ec8e8 100644
--- a/python/adapters/brcm_openomci_onu/omci/brcm_vlan_filter_task.py
+++ b/python/adapters/brcm_openomci_onu/omci/brcm_vlan_filter_task.py
@@ -71,6 +71,9 @@
         self._device = omci_agent.get_device(handler.device_id)
         self._uni_port = uni_port
         self._set_vlan_id = set_vlan_id
+        self._set_vlan_pcp = set_vlan_pcp
+        self._match_vlan = match_vlan
+        self._add_tag = add_tag
         self._tp_id = tp_id
         self._results = None
         self._local_deferred = None
@@ -110,122 +113,70 @@
         """
         Perform the vlan tagging
         """
-        self.log.info('setting-vlan-tagging')
+        self.log.debug('vlan-filter-tagging-task', device_id=self._device._device_id, uni_port=self._uni_port, set_vlan_id=self._set_vlan_id, \
+            set_vlan_pcp=self._set_vlan_pcp, match_vlan=self._match_vlan, tp_id=self._tp_id, add_tag=self._add_tag)
 
         try:
-            ################################################################################
-            # VLAN Tagging Filter config
-            #
-            #  EntityID will be referenced by:
-            #            - Nothing
-            #  References:
-            #            - MacBridgePortConfigurationData for the ANI/PON side
-            #
+            if self._add_tag:
+                ################################################################################
+                # VLAN Tagging Filter config
+                #
+                #  EntityID will be referenced by:
+                #            - Nothing
+                #  References:
+                #            - MacBridgePortConfigurationData for the ANI/PON side
+                #
 
 
-            # Delete bridge ani side vlan filter
-            eid = self._mac_bridge_port_ani_entity_id + self._uni_port.entity_id + self._tp_id  # Entity ID
-            msg = VlanTaggingFilterDataFrame(eid)
-            frame = msg.delete()
-            self.log.debug('openomci-msg', omci_msg=msg)
-            self.strobe_watchdog()
-            results = yield self._device.omci_cc.send(frame)
-            self.check_status_and_state(results, 'flow-delete-vlan-tagging-filter-data')
+                # Delete bridge ani side vlan filter
+                eid = self._mac_bridge_port_ani_entity_id + self._uni_port.entity_id + self._tp_id  # Entity ID
+                msg = VlanTaggingFilterDataFrame(eid)
+                frame = msg.delete()
+                self.log.debug('openomci-msg', omci_msg=msg)
+                self.strobe_watchdog()
+                results = yield self._device.omci_cc.send(frame)
+                self.check_status_and_state(results, 'flow-delete-vlan-tagging-filter-data')
 
-            ################################################################################
-            # Create Extended VLAN Tagging Operation config (UNI-side)
-            #
-            #  EntityID relates to the VLAN TCIS later used int vlan filter task.  This only
-            #  sets up the inital MIB entry as it relates to port config, it does not set vlan
-            #  that is saved for the vlan filter task
-            #
-            #  References:
-            #            - PPTP Ethernet or VEIP UNI
-            #
+                ################################################################################
+                # Create Extended VLAN Tagging Operation config (UNI-side)
+                #
+                #  EntityID relates to the VLAN TCIS later used int vlan filter task.  This only
+                #  sets up the inital MIB entry as it relates to port config, it does not set vlan
+                #  that is saved for the vlan filter task
+                #
+                #  References:
+                #            - PPTP Ethernet or VEIP UNI
+                #
 
 
-            # Delete uni side evto
-            msg = ExtendedVlanTaggingOperationConfigurationDataFrame(
-                self._mac_bridge_service_profile_entity_id + self._uni_port.mac_bridge_port_num,
-            )
-            frame = msg.delete()
-            self.log.debug('openomci-msg', omci_msg=msg)
-            results = yield self._device.omci_cc.send(frame)
-            self.check_status_and_state(results, 'delete-extended-vlan-tagging-operation-configuration-data')
+                # Delete uni side evto
+                msg = ExtendedVlanTaggingOperationConfigurationDataFrame(
+                    self._mac_bridge_service_profile_entity_id + self._uni_port.mac_bridge_port_num,
+                )
+                frame = msg.delete()
+                self.log.debug('openomci-msg', omci_msg=msg)
+                results = yield self._device.omci_cc.send(frame)
+                self.check_status_and_state(results, 'delete-extended-vlan-tagging-operation-configuration-data')
 
 
-            # Re-Create uni side evto
-            # default to PPTP
-            association_type = 2
-            if self._uni_port.type.value == UniType.VEIP.value:
-                association_type = 10
-            elif self._uni_port.type.value == UniType.PPTP.value:
+                # Re-Create uni side evto
+                # default to PPTP
                 association_type = 2
+                if self._uni_port.type.value == UniType.VEIP.value:
+                    association_type = 10
+                elif self._uni_port.type.value == UniType.PPTP.value:
+                    association_type = 2
 
-            attributes = dict(
-                association_type=association_type,             # Assoc Type, PPTP/VEIP Ethernet UNI
-                associated_me_pointer=self._uni_port.entity_id,      # Assoc ME, PPTP/VEIP Entity Id
-
-                # See VOL-1311 - Need to set table during create to avoid exception
-                # trying to read back table during post-create-read-missing-attributes
-                # But, because this is a R/W attribute. Some ONU may not accept the
-                # value during create. It is repeated again in a set below.
-                input_tpid=self._input_tpid,    # input TPID
-                output_tpid=self._output_tpid,  # output TPID
-            )
-
-            msg = ExtendedVlanTaggingOperationConfigurationDataFrame(
-                self._mac_bridge_service_profile_entity_id + self._uni_port.mac_bridge_port_num,  # Bridge Entity ID
-                attributes=attributes
-            )
-
-            frame = msg.create()
-            self.log.debug('openomci-msg', omci_msg=msg)
-            results = yield self._device.omci_cc.send(frame)
-            self.check_status_and_state(results, 'create-extended-vlan-tagging-operation-configuration-data')
-
-            attributes = dict(
-                # Specifies the TPIDs in use and that operations in the downstream direction are
-                # inverse to the operations in the upstream direction
-                input_tpid=self._input_tpid,    # input TPID
-                output_tpid=self._output_tpid,  # output TPID
-                downstream_mode=0,              # inverse of upstream
-            )
-
-            msg = ExtendedVlanTaggingOperationConfigurationDataFrame(
-                self._mac_bridge_service_profile_entity_id + self._uni_port.mac_bridge_port_num,  # Bridge Entity ID
-                attributes=attributes
-            )
-
-            frame = msg.set()
-            self.log.debug('openomci-msg', omci_msg=msg)
-            self.strobe_watchdog()
-            results = yield self._device.omci_cc.send(frame)
-            self.check_status_and_state(results, 'set-extended-vlan-tagging-operation-configuration-data')
-
-            # Onu-Transparent
-            if self._set_vlan_id == RESERVED_TRANSPARENT_VLAN:
-                # Transparently send any single tagged packet.
-                # As the onu is to be transparent, no need to create VlanTaggingFilterData ME.
-                # Any other specific rules will take priority over this, so not setting any other vlan specific rules
                 attributes = dict(
-                    received_frame_vlan_tagging_operation_table=
-                    VlanTaggingOperation(
-                        filter_outer_priority=15,
-                        filter_outer_vid=4096,
-                        filter_outer_tpid_de=0,
-                        filter_inner_priority=14,
-                        filter_inner_vid=4096,
-                        filter_inner_tpid_de=0,
-                        filter_ether_type=0,
-                        treatment_tags_to_remove=0,
-                        treatment_outer_priority=15,
-                        treatment_outer_vid=0,
-                        treatment_outer_tpid_de=0,
-                        treatment_inner_priority=15,
-                        treatment_inner_vid=0,
-                        treatment_inner_tpid_de=4
-                    )
+                    association_type=association_type,             # Assoc Type, PPTP/VEIP Ethernet UNI
+                    associated_me_pointer=self._uni_port.entity_id,      # Assoc ME, PPTP/VEIP Entity Id
+
+                    # See VOL-1311 - Need to set table during create to avoid exception
+                    # trying to read back table during post-create-read-missing-attributes
+                    # But, because this is a R/W attribute. Some ONU may not accept the
+                    # value during create. It is repeated again in a set below.
+                    input_tpid=self._input_tpid,    # input TPID
+                    output_tpid=self._output_tpid,  # output TPID
                 )
 
                 msg = ExtendedVlanTaggingOperationConfigurationDataFrame(
@@ -233,46 +184,17 @@
                     attributes=attributes
                 )
 
-                frame = msg.set()
-                self.log.debug('openomci-msg', omci_msg=msg)
-                self.strobe_watchdog()
-                results = yield self._device.omci_cc.send(frame)
-                self.check_status_and_state(results, 'set-evto-table-transparent-vlan')
-
-            else:
-                # Re-Create bridge ani side vlan filter
-                forward_operation = 0x10  # VID investigation
-
-                msg = VlanTaggingFilterDataFrame(
-                    eid,
-                    vlan_tcis=[self._set_vlan_id],  # VLAN IDs
-                    forward_operation=forward_operation
-                )
                 frame = msg.create()
                 self.log.debug('openomci-msg', omci_msg=msg)
-                self.strobe_watchdog()
                 results = yield self._device.omci_cc.send(frame)
-                self.check_status_and_state(results, 'flow-create-vlan-tagging-filter-data')
-                # Update uni side extended vlan filter
-                # filter for untagged
+                self.check_status_and_state(results, 'create-extended-vlan-tagging-operation-configuration-data')
+
                 attributes = dict(
-                    received_frame_vlan_tagging_operation_table=
-                    VlanTaggingOperation(
-                        filter_outer_priority=15,
-                        filter_outer_vid=4096,
-                        filter_outer_tpid_de=0,
-                        filter_inner_priority=15,
-                        filter_inner_vid=4096,
-                        filter_inner_tpid_de=0,
-                        filter_ether_type=0,
-                        treatment_tags_to_remove=0,
-                        treatment_outer_priority=15,
-                        treatment_outer_vid=0,
-                        treatment_outer_tpid_de=0,
-                        treatment_inner_priority=0,
-                        treatment_inner_vid=self._set_vlan_id,
-                        treatment_inner_tpid_de=4
-                    )
+                    # Specifies the TPIDs in use and that operations in the downstream direction are
+                    # inverse to the operations in the upstream direction
+                    input_tpid=self._input_tpid,    # input TPID
+                    output_tpid=self._output_tpid,  # output TPID
+                    downstream_mode=0,              # inverse of upstream
                 )
 
                 msg = ExtendedVlanTaggingOperationConfigurationDataFrame(
@@ -284,42 +206,128 @@
                 self.log.debug('openomci-msg', omci_msg=msg)
                 self.strobe_watchdog()
                 results = yield self._device.omci_cc.send(frame)
-                self.check_status_and_state(results, 'set-evto-table-untagged')
+                self.check_status_and_state(results, 'set-extended-vlan-tagging-operation-configuration-data')
 
-                # Update uni side extended vlan filter
-                # filter for vlan 0
-                attributes = dict(
-                    received_frame_vlan_tagging_operation_table=
-                    VlanTaggingOperation(
-                        filter_outer_priority=15,  # This entry is not a double-tag rule
-                        filter_outer_vid=4096,  # Do not filter on the outer VID value
-                        filter_outer_tpid_de=0,  # Do not filter on the outer TPID field
-
-                        filter_inner_priority=8,  # Filter on inner vlan
-                        filter_inner_vid=0x0,  # Look for vlan 0
-                        filter_inner_tpid_de=0,  # Do not filter on inner TPID field
-                        filter_ether_type=0,  # Do not filter on EtherType
-
-                        treatment_tags_to_remove=1,
-                        treatment_outer_priority=15,
-                        treatment_outer_vid=0,
-                        treatment_outer_tpid_de=0,
-
-                        treatment_inner_priority=8,  # Add an inner tag and insert this value as the priority
-                        treatment_inner_vid=self._set_vlan_id,  # use this value as the VID in the inner VLAN tag
-                        treatment_inner_tpid_de=4,  # set TPID
+                # Onu-Transparent
+                if self._set_vlan_id == RESERVED_TRANSPARENT_VLAN:
+                    # Transparently send any single tagged packet.
+                    # As the onu is to be transparent, no need to create VlanTaggingFilterData ME.
+                    # Any other specific rules will take priority over this, so not setting any other vlan specific rules
+                    attributes = dict(
+                        received_frame_vlan_tagging_operation_table=
+                        VlanTaggingOperation(
+                            filter_outer_priority=15,
+                            filter_outer_vid=4096,
+                            filter_outer_tpid_de=0,
+                            filter_inner_priority=14,
+                            filter_inner_vid=4096,
+                            filter_inner_tpid_de=0,
+                            filter_ether_type=0,
+                            treatment_tags_to_remove=0,
+                            treatment_outer_priority=15,
+                            treatment_outer_vid=0,
+                            treatment_outer_tpid_de=0,
+                            treatment_inner_priority=15,
+                            treatment_inner_vid=0,
+                            treatment_inner_tpid_de=4
+                        )
                     )
-                )
-                msg = ExtendedVlanTaggingOperationConfigurationDataFrame(
-                    self._mac_bridge_service_profile_entity_id + self._uni_port.mac_bridge_port_num,  # Bridge Entity ID
-                    attributes=attributes  # See above
-                )
-                frame = msg.set()
-                self.log.debug('openomci-msg', omci_msg=msg)
-                self.strobe_watchdog()
-                results = yield self._device.omci_cc.send(frame)
-                self.check_status_and_state(results, 'set-evto-table-zero-tagged')
 
+                    msg = ExtendedVlanTaggingOperationConfigurationDataFrame(
+                        self._mac_bridge_service_profile_entity_id + self._uni_port.mac_bridge_port_num,  # Bridge Entity ID
+                        attributes=attributes
+                    )
+
+                    frame = msg.set()
+                    self.log.debug('openomci-msg', omci_msg=msg)
+                    self.strobe_watchdog()
+                    results = yield self._device.omci_cc.send(frame)
+                    self.check_status_and_state(results, 'set-evto-table-transparent-vlan')
+
+                else:
+                    # Re-Create bridge ani side vlan filter
+                    forward_operation = 0x10  # VID investigation
+
+                    msg = VlanTaggingFilterDataFrame(
+                        eid,
+                        vlan_tcis=[self._set_vlan_id],  # VLAN IDs
+                        forward_operation=forward_operation
+                    )
+                    frame = msg.create()
+                    self.log.debug('openomci-msg', omci_msg=msg)
+                    self.strobe_watchdog()
+                    results = yield self._device.omci_cc.send(frame)
+                    self.check_status_and_state(results, 'flow-create-vlan-tagging-filter-data')
+                    # Update uni side extended vlan filter
+                    # filter for untagged
+                    attributes = dict(
+                        received_frame_vlan_tagging_operation_table=
+                        VlanTaggingOperation(
+                            filter_outer_priority=15,
+                            filter_outer_vid=4096,
+                            filter_outer_tpid_de=0,
+                            filter_inner_priority=15,
+                            filter_inner_vid=4096,
+                            filter_inner_tpid_de=0,
+                            filter_ether_type=0,
+                            treatment_tags_to_remove=0,
+                            treatment_outer_priority=15,
+                            treatment_outer_vid=0,
+                            treatment_outer_tpid_de=0,
+                            treatment_inner_priority=0,
+                            treatment_inner_vid=self._set_vlan_id,
+                            treatment_inner_tpid_de=4
+                        )
+                    )
+
+                    msg = ExtendedVlanTaggingOperationConfigurationDataFrame(
+                        self._mac_bridge_service_profile_entity_id + self._uni_port.mac_bridge_port_num,  # Bridge Entity ID
+                        attributes=attributes
+                    )
+
+                    frame = msg.set()
+                    self.log.debug('openomci-msg', omci_msg=msg)
+                    self.strobe_watchdog()
+                    results = yield self._device.omci_cc.send(frame)
+                    self.check_status_and_state(results, 'set-evto-table-untagged')
+
+                    # Update uni side extended vlan filter
+                    # filter for vlan 0
+                    attributes = dict(
+                        received_frame_vlan_tagging_operation_table=
+                        VlanTaggingOperation(
+                            filter_outer_priority=15,  # This entry is not a double-tag rule
+                            filter_outer_vid=4096,  # Do not filter on the outer VID value
+                            filter_outer_tpid_de=0,  # Do not filter on the outer TPID field
+
+                            filter_inner_priority=8,  # Filter on inner vlan
+                            filter_inner_vid=0x0,  # Look for vlan 0
+                            filter_inner_tpid_de=0,  # Do not filter on inner TPID field
+                            filter_ether_type=0,  # Do not filter on EtherType
+
+                            treatment_tags_to_remove=1,
+                            treatment_outer_priority=15,
+                            treatment_outer_vid=0,
+                            treatment_outer_tpid_de=0,
+
+                            treatment_inner_priority=8,  # Add an inner tag and insert this value as the priority
+                            treatment_inner_vid=self._set_vlan_id,  # use this value as the VID in the inner VLAN tag
+                            treatment_inner_tpid_de=4,  # set TPID
+                        )
+                    )
+                    msg = ExtendedVlanTaggingOperationConfigurationDataFrame(
+                        self._mac_bridge_service_profile_entity_id + self._uni_port.mac_bridge_port_num,  # Bridge Entity ID
+                        attributes=attributes  # See above
+                    )
+                    frame = msg.set()
+                    self.log.debug('openomci-msg', omci_msg=msg)
+                    self.strobe_watchdog()
+                    results = yield self._device.omci_cc.send(frame)
+                    self.check_status_and_state(results, 'set-evto-table-zero-tagged')
+
+            else: #addTag = False
+                #TODO: needs to be implemented - future task.
+                self.log.debug('removing-vlan-filter-tag', uni_port=self._uni_port, match_vlan=self._match_vlan, )
             self.deferred.callback(self)
 
         except Exception as e:
diff --git a/python/requirements.txt b/python/requirements.txt
index 85c6640..4733885 100644
--- a/python/requirements.txt
+++ b/python/requirements.txt
@@ -1,2 +1,2 @@
 voltha-protos==3.2.3
-pyvoltha==2.3.15
+pyvoltha==2.3.18