alshabib | f0e7e70 | 2015-05-30 18:22:36 -0700 | [diff] [blame] | 1 | /* |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 2 | * Copyright 2021-present Open Networking Foundation |
alshabib | f0e7e70 | 2015-05-30 18:22:36 -0700 | [diff] [blame] | 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
alshabib | 36a4d73 | 2016-06-01 16:03:59 -0700 | [diff] [blame] | 16 | package org.opencord.olt.impl; |
alshabib | f0e7e70 | 2015-05-30 18:22:36 -0700 | [diff] [blame] | 17 | |
alshabib | f0e7e70 | 2015-05-30 18:22:36 -0700 | [diff] [blame] | 18 | import org.onlab.packet.VlanId; |
Jonathan Hart | 4f178fa | 2020-02-03 10:46:01 -0800 | [diff] [blame] | 19 | import org.onlab.util.KryoNamespace; |
Gamze Abaka | 1f62dd9 | 2020-05-07 08:58:13 +0000 | [diff] [blame] | 20 | import org.onosproject.cfg.ComponentConfigService; |
Jonathan Hart | 4f178fa | 2020-02-03 10:46:01 -0800 | [diff] [blame] | 21 | import org.onosproject.cluster.ClusterService; |
Andrea Campanella | af39b4c | 2020-05-13 14:07:44 +0200 | [diff] [blame] | 22 | import org.onosproject.cluster.LeadershipService; |
alshabib | f0e7e70 | 2015-05-30 18:22:36 -0700 | [diff] [blame] | 23 | import org.onosproject.core.ApplicationId; |
| 24 | import org.onosproject.core.CoreService; |
alshabib | 8e4fd2f | 2016-01-12 15:55:53 -0800 | [diff] [blame] | 25 | import org.onosproject.event.AbstractListenerManager; |
Andrea Campanella | af39b4c | 2020-05-13 14:07:44 +0200 | [diff] [blame] | 26 | import org.onosproject.mastership.MastershipService; |
Amit Ghosh | 1ed9aef | 2018-07-17 17:08:16 +0100 | [diff] [blame] | 27 | import org.onosproject.net.AnnotationKeys; |
Jonathan Hart | e533a42 | 2015-10-20 17:31:24 -0700 | [diff] [blame] | 28 | import org.onosproject.net.ConnectPoint; |
Amit Ghosh | 1ed9aef | 2018-07-17 17:08:16 +0100 | [diff] [blame] | 29 | import org.onosproject.net.Device; |
alshabib | f0e7e70 | 2015-05-30 18:22:36 -0700 | [diff] [blame] | 30 | import org.onosproject.net.DeviceId; |
alshabib | dec2e25 | 2016-01-15 12:20:25 -0800 | [diff] [blame] | 31 | import org.onosproject.net.Port; |
alshabib | f0e7e70 | 2015-05-30 18:22:36 -0700 | [diff] [blame] | 32 | import org.onosproject.net.PortNumber; |
| 33 | import org.onosproject.net.device.DeviceEvent; |
| 34 | import org.onosproject.net.device.DeviceListener; |
| 35 | import org.onosproject.net.device.DeviceService; |
Jonathan Hart | 4f178fa | 2020-02-03 10:46:01 -0800 | [diff] [blame] | 36 | import org.onosproject.store.serializers.KryoNamespaces; |
Jonathan Hart | 4f178fa | 2020-02-03 10:46:01 -0800 | [diff] [blame] | 37 | import org.onosproject.store.service.Serializer; |
| 38 | import org.onosproject.store.service.StorageService; |
alshabib | 36a4d73 | 2016-06-01 16:03:59 -0700 | [diff] [blame] | 39 | import org.opencord.olt.AccessDeviceEvent; |
| 40 | import org.opencord.olt.AccessDeviceListener; |
| 41 | import org.opencord.olt.AccessDeviceService; |
Gamze Abaka | 641fc07 | 2018-09-04 09:16:27 +0000 | [diff] [blame] | 42 | import org.opencord.sadis.BaseInformationService; |
| 43 | import org.opencord.sadis.SadisService; |
Amit Ghosh | 1ed9aef | 2018-07-17 17:08:16 +0100 | [diff] [blame] | 44 | import org.opencord.sadis.SubscriberAndDeviceInformation; |
Andrea Campanella | cbbb795 | 2019-11-25 06:38:41 +0000 | [diff] [blame] | 45 | import org.opencord.sadis.UniTagInformation; |
alshabib | e055967 | 2016-02-21 14:49:51 -0800 | [diff] [blame] | 46 | import org.osgi.service.component.ComponentContext; |
Carmelo Cascone | ca93116 | 2019-07-15 18:22:24 -0700 | [diff] [blame] | 47 | import org.osgi.service.component.annotations.Activate; |
| 48 | import org.osgi.service.component.annotations.Component; |
| 49 | import org.osgi.service.component.annotations.Deactivate; |
| 50 | import org.osgi.service.component.annotations.Modified; |
| 51 | import org.osgi.service.component.annotations.Reference; |
| 52 | import org.osgi.service.component.annotations.ReferenceCardinality; |
Ilayda Ozdemir | 90a9362 | 2021-02-25 09:40:58 +0000 | [diff] [blame] | 53 | import org.osgi.service.component.annotations.ReferencePolicy; |
alshabib | f0e7e70 | 2015-05-30 18:22:36 -0700 | [diff] [blame] | 54 | import org.slf4j.Logger; |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 55 | import org.slf4j.LoggerFactory; |
alshabib | f0e7e70 | 2015-05-30 18:22:36 -0700 | [diff] [blame] | 56 | |
Andrea Campanella | 7a1d7e7 | 2020-11-05 10:40:10 +0100 | [diff] [blame] | 57 | import java.util.ArrayList; |
Andrea Campanella | 7a1d7e7 | 2020-11-05 10:40:10 +0100 | [diff] [blame] | 58 | import java.util.Dictionary; |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 59 | import java.util.Iterator; |
| 60 | import java.util.LinkedList; |
Andrea Campanella | 7a1d7e7 | 2020-11-05 10:40:10 +0100 | [diff] [blame] | 61 | import java.util.List; |
| 62 | import java.util.Map; |
Andrea Campanella | 7a1d7e7 | 2020-11-05 10:40:10 +0100 | [diff] [blame] | 63 | import java.util.Properties; |
| 64 | import java.util.Set; |
Andrea Campanella | 7a1d7e7 | 2020-11-05 10:40:10 +0100 | [diff] [blame] | 65 | import java.util.concurrent.ExecutorService; |
| 66 | import java.util.concurrent.Executors; |
| 67 | import java.util.concurrent.LinkedBlockingQueue; |
| 68 | import java.util.concurrent.ScheduledExecutorService; |
| 69 | import java.util.concurrent.TimeUnit; |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 70 | import java.util.concurrent.locks.Lock; |
| 71 | import java.util.concurrent.locks.ReentrantReadWriteLock; |
Andrea Campanella | 7a1d7e7 | 2020-11-05 10:40:10 +0100 | [diff] [blame] | 72 | |
Andrea Campanella | 7a1d7e7 | 2020-11-05 10:40:10 +0100 | [diff] [blame] | 73 | import static com.google.common.base.Strings.isNullOrEmpty; |
Andrea Campanella | 7a1d7e7 | 2020-11-05 10:40:10 +0100 | [diff] [blame] | 74 | import static org.onlab.util.Tools.get; |
| 75 | import static org.onlab.util.Tools.groupedThreads; |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 76 | import static org.opencord.olt.impl.OltUtils.getPortName; |
| 77 | import static org.opencord.olt.impl.OltUtils.portWithName; |
Andrea Campanella | 7a1d7e7 | 2020-11-05 10:40:10 +0100 | [diff] [blame] | 78 | import static org.opencord.olt.impl.OsgiPropertyConstants.*; |
alshabib | f0e7e70 | 2015-05-30 18:22:36 -0700 | [diff] [blame] | 79 | |
| 80 | /** |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 81 | * OLT Application. |
alshabib | f0e7e70 | 2015-05-30 18:22:36 -0700 | [diff] [blame] | 82 | */ |
Carmelo Cascone | ca93116 | 2019-07-15 18:22:24 -0700 | [diff] [blame] | 83 | @Component(immediate = true, |
| 84 | property = { |
Carmelo Cascone | ca93116 | 2019-07-15 18:22:24 -0700 | [diff] [blame] | 85 | DEFAULT_BP_ID + ":String=" + DEFAULT_BP_ID_DEFAULT, |
Andrea Campanella | cbbb795 | 2019-11-25 06:38:41 +0000 | [diff] [blame] | 86 | DEFAULT_MCAST_SERVICE_NAME + ":String=" + DEFAULT_MCAST_SERVICE_NAME_DEFAULT, |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 87 | FLOW_PROCESSING_THREADS + ":Integer=" + FLOW_PROCESSING_THREADS_DEFAULT, |
| 88 | SUBSCRIBER_PROCESSING_THREADS + ":Integer=" + SUBSCRIBER_PROCESSING_THREADS_DEFAULT, |
| 89 | REQUEUE_DELAY + ":Integer=" + REQUEUE_DELAY_DEFAULT |
Carmelo Cascone | ca93116 | 2019-07-15 18:22:24 -0700 | [diff] [blame] | 90 | }) |
alshabib | 8e4fd2f | 2016-01-12 15:55:53 -0800 | [diff] [blame] | 91 | public class Olt |
| 92 | extends AbstractListenerManager<AccessDeviceEvent, AccessDeviceListener> |
| 93 | implements AccessDeviceService { |
alshabib | f0e7e70 | 2015-05-30 18:22:36 -0700 | [diff] [blame] | 94 | |
Carmelo Cascone | ca93116 | 2019-07-15 18:22:24 -0700 | [diff] [blame] | 95 | @Reference(cardinality = ReferenceCardinality.MANDATORY) |
alshabib | f0e7e70 | 2015-05-30 18:22:36 -0700 | [diff] [blame] | 96 | protected DeviceService deviceService; |
| 97 | |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 98 | @Reference(cardinality = ReferenceCardinality.MANDATORY) |
| 99 | protected ComponentConfigService cfgService; |
Andrea Campanella | 438e1ad | 2021-03-26 11:41:16 +0100 | [diff] [blame] | 100 | |
Carmelo Cascone | ca93116 | 2019-07-15 18:22:24 -0700 | [diff] [blame] | 101 | @Reference(cardinality = ReferenceCardinality.MANDATORY) |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 102 | protected MastershipService mastershipService; |
alshabib | f0e7e70 | 2015-05-30 18:22:36 -0700 | [diff] [blame] | 103 | |
Andrea Campanella | 438e1ad | 2021-03-26 11:41:16 +0100 | [diff] [blame] | 104 | @Reference(cardinality = ReferenceCardinality.MANDATORY) |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 105 | protected ClusterService clusterService; |
| 106 | |
| 107 | @Reference(cardinality = ReferenceCardinality.MANDATORY) |
| 108 | protected LeadershipService leadershipService; |
Andrea Campanella | 438e1ad | 2021-03-26 11:41:16 +0100 | [diff] [blame] | 109 | |
Ilayda Ozdemir | 90a9362 | 2021-02-25 09:40:58 +0000 | [diff] [blame] | 110 | @Reference(cardinality = ReferenceCardinality.OPTIONAL, |
| 111 | bind = "bindSadisService", |
| 112 | unbind = "unbindSadisService", |
| 113 | policy = ReferencePolicy.DYNAMIC) |
| 114 | protected volatile SadisService sadisService; |
Gamze Abaka | 641fc07 | 2018-09-04 09:16:27 +0000 | [diff] [blame] | 115 | |
Carmelo Cascone | ca93116 | 2019-07-15 18:22:24 -0700 | [diff] [blame] | 116 | @Reference(cardinality = ReferenceCardinality.MANDATORY) |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 117 | protected OltDeviceServiceInterface oltDeviceService; |
alshabib | e055967 | 2016-02-21 14:49:51 -0800 | [diff] [blame] | 118 | |
Carmelo Cascone | ca93116 | 2019-07-15 18:22:24 -0700 | [diff] [blame] | 119 | @Reference(cardinality = ReferenceCardinality.MANDATORY) |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 120 | protected OltFlowServiceInterface oltFlowService; |
| 121 | |
| 122 | @Reference(cardinality = ReferenceCardinality.MANDATORY) |
| 123 | protected OltMeterServiceInterface oltMeterService; |
Gamze Abaka | ad32965 | 2018-12-20 10:12:21 +0000 | [diff] [blame] | 124 | |
Jonathan Hart | 4f178fa | 2020-02-03 10:46:01 -0800 | [diff] [blame] | 125 | @Reference(cardinality = ReferenceCardinality.MANDATORY) |
| 126 | protected StorageService storageService; |
| 127 | |
| 128 | @Reference(cardinality = ReferenceCardinality.MANDATORY) |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 129 | protected CoreService coreService; |
Jonathan Hart | 4f178fa | 2020-02-03 10:46:01 -0800 | [diff] [blame] | 130 | |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 131 | protected ApplicationId appId; |
Andrea Campanella | af39b4c | 2020-05-13 14:07:44 +0200 | [diff] [blame] | 132 | |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 133 | private static final String ONOS_OLT_SERVICE = "onos/olt-service"; |
Tunahan Sezen | a07fe96 | 2021-02-24 08:24:24 +0000 | [diff] [blame] | 134 | |
Carmelo Cascone | ca93116 | 2019-07-15 18:22:24 -0700 | [diff] [blame] | 135 | /** |
Carmelo Cascone | 95ff512 | 2019-11-14 14:19:13 -0800 | [diff] [blame] | 136 | * Default bandwidth profile id that is used for authentication trap flows. |
Carmelo Cascone | ca93116 | 2019-07-15 18:22:24 -0700 | [diff] [blame] | 137 | **/ |
| 138 | protected String defaultBpId = DEFAULT_BP_ID_DEFAULT; |
Gamze Abaka | ad32965 | 2018-12-20 10:12:21 +0000 | [diff] [blame] | 139 | |
Carmelo Cascone | ca93116 | 2019-07-15 18:22:24 -0700 | [diff] [blame] | 140 | /** |
Gamze Abaka | 51a34e8 | 2020-05-08 13:03:14 +0000 | [diff] [blame] | 141 | * Default multicast service name. |
Carmelo Cascone | ca93116 | 2019-07-15 18:22:24 -0700 | [diff] [blame] | 142 | **/ |
Gamze Abaka | 51a34e8 | 2020-05-08 13:03:14 +0000 | [diff] [blame] | 143 | protected String multicastServiceName = DEFAULT_MCAST_SERVICE_NAME_DEFAULT; |
Gamze Abaka | 33feef5 | 2019-02-27 08:16:47 +0000 | [diff] [blame] | 144 | |
Saurav Das | 2d3777a | 2020-08-07 18:48:51 -0700 | [diff] [blame] | 145 | /** |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 146 | * Number of threads used to process flows. |
Saurav Das | 2d3777a | 2020-08-07 18:48:51 -0700 | [diff] [blame] | 147 | **/ |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 148 | protected int flowProcessingThreads = FLOW_PROCESSING_THREADS_DEFAULT; |
Saurav Das | 2d3777a | 2020-08-07 18:48:51 -0700 | [diff] [blame] | 149 | |
Matteo Scandolo | a4aaa97 | 2020-10-23 15:24:38 -0700 | [diff] [blame] | 150 | /** |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 151 | * Number of threads used to process flows. |
| 152 | **/ |
| 153 | protected int subscriberProcessingThreads = SUBSCRIBER_PROCESSING_THREADS_DEFAULT; |
| 154 | |
| 155 | /** |
| 156 | * Delay in ms to put an event back in the queue, used to avoid retrying things to often if conditions are not met. |
| 157 | **/ |
| 158 | protected int requeueDelay = REQUEUE_DELAY_DEFAULT; |
| 159 | |
| 160 | private final Logger log = LoggerFactory.getLogger(getClass()); |
| 161 | |
| 162 | /** |
| 163 | * A queue to asynchronously process events. |
Matteo Scandolo | a4aaa97 | 2020-10-23 15:24:38 -0700 | [diff] [blame] | 164 | */ |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 165 | protected Map<ConnectPoint, LinkedBlockingQueue<DiscoveredSubscriber>> eventsQueues; |
alshabib | f0e7e70 | 2015-05-30 18:22:36 -0700 | [diff] [blame] | 166 | |
Gamze Abaka | 641fc07 | 2018-09-04 09:16:27 +0000 | [diff] [blame] | 167 | protected BaseInformationService<SubscriberAndDeviceInformation> subsService; |
alshabib | f0e7e70 | 2015-05-30 18:22:36 -0700 | [diff] [blame] | 168 | |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 169 | /** |
| 170 | * Listener for OLT devices events. |
| 171 | */ |
| 172 | protected OltDeviceListener deviceListener = new OltDeviceListener(); |
| 173 | protected ScheduledExecutorService discoveredSubscriberExecutor = |
| 174 | Executors.newSingleThreadScheduledExecutor(groupedThreads("onos/olt", |
| 175 | "discovered-cp-%d", log)); |
Amit Ghosh | e1d3f09 | 2018-10-09 19:44:33 +0100 | [diff] [blame] | 176 | |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 177 | protected ScheduledExecutorService queueExecutor = |
| 178 | Executors.newSingleThreadScheduledExecutor(groupedThreads("onos/olt", |
| 179 | "discovered-cp-restore-%d", log)); |
Matteo Scandolo | 632f0fc | 2018-09-07 12:21:45 -0700 | [diff] [blame] | 180 | |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 181 | /** |
| 182 | * Executor used to defer flow provisioning to a different thread pool. |
| 183 | */ |
| 184 | protected ExecutorService flowsExecutor; |
Saurav Das | a9d5f44 | 2019-03-06 19:32:48 -0800 | [diff] [blame] | 185 | |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 186 | /** |
| 187 | * Executor used to defer subscriber handling from API call to a different thread pool. |
| 188 | */ |
| 189 | protected ExecutorService subscriberExecutor; |
| 190 | |
| 191 | private static final String APP_NAME = "org.opencord.olt"; |
| 192 | |
| 193 | private final ReentrantReadWriteLock queueLock = new ReentrantReadWriteLock(); |
| 194 | private final Lock queueWriteLock = queueLock.writeLock(); |
| 195 | private final Lock queueReadLock = queueLock.readLock(); |
Andrea Campanella | 0c3309d | 2020-05-29 01:51:18 -0700 | [diff] [blame] | 196 | |
alshabib | f0e7e70 | 2015-05-30 18:22:36 -0700 | [diff] [blame] | 197 | @Activate |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 198 | protected void activate(ComponentContext context) { |
| 199 | cfgService.registerProperties(getClass()); |
Andrea Campanella | 0c3309d | 2020-05-29 01:51:18 -0700 | [diff] [blame] | 200 | |
alshabib | e055967 | 2016-02-21 14:49:51 -0800 | [diff] [blame] | 201 | modified(context); |
Saurav Das | 62ad75e | 2019-03-05 12:22:22 -0800 | [diff] [blame] | 202 | |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 203 | appId = coreService.registerApplication(APP_NAME); |
Jonathan Hart | 4f178fa | 2020-02-03 10:46:01 -0800 | [diff] [blame] | 204 | KryoNamespace serializer = KryoNamespace.newBuilder() |
| 205 | .register(KryoNamespaces.API) |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 206 | .register(ConnectPoint.class) |
| 207 | .register(DiscoveredSubscriber.class) |
| 208 | .register(DiscoveredSubscriber.Status.class) |
| 209 | .register(SubscriberAndDeviceInformation.class) |
Jonathan Hart | 4f178fa | 2020-02-03 10:46:01 -0800 | [diff] [blame] | 210 | .register(UniTagInformation.class) |
Ilayda Ozdemir | 90a9362 | 2021-02-25 09:40:58 +0000 | [diff] [blame] | 211 | .register(LinkedBlockingQueue.class) |
Jonathan Hart | 4f178fa | 2020-02-03 10:46:01 -0800 | [diff] [blame] | 212 | .build(); |
| 213 | |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 214 | eventsQueues = storageService.<ConnectPoint, LinkedBlockingQueue<DiscoveredSubscriber>>consistentMapBuilder() |
| 215 | .withName("volt-subscriber-queues") |
| 216 | .withApplicationId(appId) |
Jonathan Hart | 4f178fa | 2020-02-03 10:46:01 -0800 | [diff] [blame] | 217 | .withSerializer(Serializer.using(serializer)) |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 218 | .build().asJavaMap(); |
alshabib | 4ceaed3 | 2016-03-03 18:00:58 -0800 | [diff] [blame] | 219 | |
alshabib | ba35749 | 2016-01-27 13:49:46 -0800 | [diff] [blame] | 220 | deviceService.addListener(deviceListener); |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 221 | |
| 222 | discoveredSubscriberExecutor.execute(this::processDiscoveredSubscribers); |
| 223 | eventDispatcher.addSink(AccessDeviceEvent.class, listenerRegistry); |
| 224 | log.info("Started"); |
| 225 | |
| 226 | deviceListener.handleExistingPorts(); |
alshabib | f0e7e70 | 2015-05-30 18:22:36 -0700 | [diff] [blame] | 227 | } |
| 228 | |
| 229 | @Deactivate |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 230 | protected void deactivate(ComponentContext context) { |
| 231 | cfgService.unregisterProperties(getClass(), false); |
| 232 | discoveredSubscriberExecutor.shutdown(); |
| 233 | flowsExecutor.shutdown(); |
| 234 | subscriberExecutor.shutdown(); |
| 235 | deviceListener.deactivate(); |
alshabib | f0e7e70 | 2015-05-30 18:22:36 -0700 | [diff] [blame] | 236 | log.info("Stopped"); |
| 237 | } |
| 238 | |
alshabib | e055967 | 2016-02-21 14:49:51 -0800 | [diff] [blame] | 239 | @Modified |
| 240 | public void modified(ComponentContext context) { |
| 241 | Dictionary<?, ?> properties = context != null ? context.getProperties() : new Properties(); |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 242 | if (context != null) { |
Andrea Campanella | 971d5b9 | 2020-05-07 11:20:43 +0200 | [diff] [blame] | 243 | String bpId = get(properties, DEFAULT_BP_ID); |
| 244 | defaultBpId = isNullOrEmpty(bpId) ? defaultBpId : bpId; |
Gamze Abaka | ad32965 | 2018-12-20 10:12:21 +0000 | [diff] [blame] | 245 | |
Andrea Campanella | 971d5b9 | 2020-05-07 11:20:43 +0200 | [diff] [blame] | 246 | String mcastSN = get(properties, DEFAULT_MCAST_SERVICE_NAME); |
| 247 | multicastServiceName = isNullOrEmpty(mcastSN) ? multicastServiceName : mcastSN; |
Andrea Campanella | cbbb795 | 2019-11-25 06:38:41 +0000 | [diff] [blame] | 248 | |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 249 | String flowThreads = get(properties, FLOW_PROCESSING_THREADS); |
| 250 | int oldFlowProcessingThreads = flowProcessingThreads; |
| 251 | flowProcessingThreads = isNullOrEmpty(flowThreads) ? |
| 252 | oldFlowProcessingThreads : Integer.parseInt(flowThreads.trim()); |
Saurav Das | 2d3777a | 2020-08-07 18:48:51 -0700 | [diff] [blame] | 253 | |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 254 | if (flowsExecutor == null || oldFlowProcessingThreads != flowProcessingThreads) { |
| 255 | if (flowsExecutor != null) { |
| 256 | flowsExecutor.shutdown(); |
| 257 | } |
| 258 | flowsExecutor = Executors.newFixedThreadPool(flowProcessingThreads, |
| 259 | groupedThreads(ONOS_OLT_SERVICE, |
| 260 | "flows-installer-%d")); |
| 261 | } |
Gamze Abaka | 33feef5 | 2019-02-27 08:16:47 +0000 | [diff] [blame] | 262 | |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 263 | String subscriberThreads = get(properties, SUBSCRIBER_PROCESSING_THREADS); |
| 264 | int oldSubscriberProcessingThreads = subscriberProcessingThreads; |
| 265 | subscriberProcessingThreads = isNullOrEmpty(subscriberThreads) ? |
| 266 | oldSubscriberProcessingThreads : Integer.parseInt(subscriberThreads.trim()); |
| 267 | |
| 268 | if (subscriberExecutor == null || oldSubscriberProcessingThreads != subscriberProcessingThreads) { |
| 269 | if (subscriberExecutor != null) { |
| 270 | subscriberExecutor.shutdown(); |
| 271 | } |
| 272 | subscriberExecutor = Executors.newFixedThreadPool(subscriberProcessingThreads, |
| 273 | groupedThreads(ONOS_OLT_SERVICE, |
| 274 | "subscriber-installer-%d")); |
| 275 | } |
| 276 | |
| 277 | String queueDelay = get(properties, REQUEUE_DELAY); |
| 278 | requeueDelay = isNullOrEmpty(queueDelay) ? |
| 279 | REQUEUE_DELAY_DEFAULT : Integer.parseInt(queueDelay.trim()); |
alshabib | e055967 | 2016-02-21 14:49:51 -0800 | [diff] [blame] | 280 | } |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 281 | log.info("Modified. Values = {}: {}, {}: {}, " + |
| 282 | "{}:{}, {}:{}, {}:{}", |
| 283 | DEFAULT_BP_ID, defaultBpId, |
| 284 | DEFAULT_MCAST_SERVICE_NAME, multicastServiceName, |
| 285 | FLOW_PROCESSING_THREADS, flowProcessingThreads, |
| 286 | SUBSCRIBER_PROCESSING_THREADS, subscriberProcessingThreads, |
| 287 | REQUEUE_DELAY, requeueDelay); |
alshabib | e055967 | 2016-02-21 14:49:51 -0800 | [diff] [blame] | 288 | } |
| 289 | |
Ilayda Ozdemir | 90a9362 | 2021-02-25 09:40:58 +0000 | [diff] [blame] | 290 | |
alshabib | 32232c8 | 2016-02-25 17:57:24 -0500 | [diff] [blame] | 291 | @Override |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 292 | public boolean provisionSubscriber(ConnectPoint cp) { |
| 293 | subscriberExecutor.submit(() -> { |
| 294 | Device device = deviceService.getDevice(cp.deviceId()); |
| 295 | Port port = deviceService.getPort(device.id(), cp.port()); |
| 296 | AccessDevicePort accessDevicePort = new AccessDevicePort(port); |
Hardik Windlass | 395ff37 | 2019-06-13 05:16:00 +0000 | [diff] [blame] | 297 | |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 298 | if (oltDeviceService.isNniPort(device, port.number())) { |
| 299 | log.warn("will not provision a subscriber on the NNI {}", accessDevicePort); |
| 300 | return false; |
Saurav Das | 026650f | 2020-09-21 18:56:35 -0700 | [diff] [blame] | 301 | } |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 302 | |
| 303 | log.info("Provisioning subscriber on {}", accessDevicePort); |
| 304 | |
| 305 | if (oltFlowService.isSubscriberServiceProvisioned(accessDevicePort)) { |
| 306 | log.error("Subscriber on {} is already provisioned", accessDevicePort); |
| 307 | return false; |
| 308 | } |
| 309 | |
| 310 | SubscriberAndDeviceInformation si = subsService.get(getPortName(port)); |
| 311 | if (si == null) { |
| 312 | log.error("Subscriber information not found in sadis for port {}", accessDevicePort); |
| 313 | return false; |
| 314 | } |
| 315 | DiscoveredSubscriber sub = new DiscoveredSubscriber(device, port, |
| 316 | DiscoveredSubscriber.Status.ADDED, true, si); |
| 317 | |
| 318 | // NOTE we need to keep a list of the subscribers that are provisioned on a port, |
| 319 | // regardless of the flow status |
| 320 | si.uniTagList().forEach(uti -> { |
| 321 | ServiceKey sk = new ServiceKey(accessDevicePort, uti); |
| 322 | oltFlowService.updateProvisionedSubscriberStatus(sk, true); |
| 323 | }); |
| 324 | |
| 325 | addSubscriberToQueue(sub); |
| 326 | return true; |
Andrea Campanella | 7a1d7e7 | 2020-11-05 10:40:10 +0100 | [diff] [blame] | 327 | }); |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 328 | //NOTE this only means we have taken the request in, nothing more. |
Amit Ghosh | 3193952 | 2018-08-16 13:28:21 +0100 | [diff] [blame] | 329 | return true; |
alshabib | bf23a1f | 2016-01-14 17:27:11 -0800 | [diff] [blame] | 330 | } |
| 331 | |
Andrea Campanella | cbbb795 | 2019-11-25 06:38:41 +0000 | [diff] [blame] | 332 | @Override |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 333 | public boolean removeSubscriber(ConnectPoint cp) { |
| 334 | subscriberExecutor.submit(() -> { |
| 335 | Device device = deviceService.getDevice(DeviceId.deviceId(cp.deviceId().toString())); |
| 336 | Port port = deviceService.getPort(device.id(), cp.port()); |
| 337 | AccessDevicePort accessDevicePort = new AccessDevicePort(port); |
Andrea Campanella | cbbb795 | 2019-11-25 06:38:41 +0000 | [diff] [blame] | 338 | |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 339 | if (oltDeviceService.isNniPort(device, port.number())) { |
| 340 | log.warn("will not un-provision a subscriber on the NNI {}", |
| 341 | accessDevicePort); |
Amit Ghosh | e1d3f09 | 2018-10-09 19:44:33 +0100 | [diff] [blame] | 342 | return false; |
| 343 | } |
| 344 | |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 345 | log.info("Un-provisioning subscriber on {}", accessDevicePort); |
| 346 | |
| 347 | if (!oltFlowService.isSubscriberServiceProvisioned(accessDevicePort)) { |
| 348 | log.error("Subscriber on {} is not provisioned", accessDevicePort); |
| 349 | return false; |
| 350 | } |
| 351 | |
| 352 | SubscriberAndDeviceInformation si = subsService.get(getPortName(port)); |
| 353 | if (si == null) { |
| 354 | log.error("Subscriber information not found in sadis for port {}", |
| 355 | accessDevicePort); |
| 356 | // NOTE that we are returning true so that the subscriber is removed from the queue |
| 357 | // and we can move on provisioning others |
| 358 | return false; |
| 359 | } |
| 360 | DiscoveredSubscriber sub = new DiscoveredSubscriber(device, port, |
| 361 | DiscoveredSubscriber.Status.REMOVED, true, si); |
| 362 | |
| 363 | // NOTE we need to keep a list of the subscribers that are provisioned on a port, |
| 364 | // regardless of the flow status |
| 365 | si.uniTagList().forEach(uti -> { |
| 366 | ServiceKey sk = new ServiceKey(accessDevicePort, uti); |
| 367 | oltFlowService.updateProvisionedSubscriberStatus(sk, false); |
Gamze Abaka | f59c091 | 2019-04-19 08:24:28 +0000 | [diff] [blame] | 368 | }); |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 369 | |
| 370 | addSubscriberToQueue(sub); |
Amit Ghosh | e1d3f09 | 2018-10-09 19:44:33 +0100 | [diff] [blame] | 371 | return true; |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 372 | }); |
| 373 | //NOTE this only means we have taken the request in, nothing more. |
| 374 | return true; |
| 375 | } |
| 376 | |
| 377 | @Override |
| 378 | public boolean provisionSubscriber(ConnectPoint cp, VlanId cTag, VlanId sTag, Integer tpId) { |
| 379 | log.debug("Provisioning subscriber on {}, with cTag {}, stag {}, tpId {}", |
| 380 | cp, cTag, sTag, tpId); |
| 381 | Device device = deviceService.getDevice(cp.deviceId()); |
| 382 | Port port = deviceService.getPort(device.id(), cp.port()); |
| 383 | AccessDevicePort accessDevicePort = new AccessDevicePort(port); |
| 384 | |
| 385 | if (oltDeviceService.isNniPort(device, port.number())) { |
| 386 | log.warn("will not provision a subscriber on the NNI {}", accessDevicePort); |
Amit Ghosh | e1d3f09 | 2018-10-09 19:44:33 +0100 | [diff] [blame] | 387 | return false; |
| 388 | } |
Amit Ghosh | 95e2f65 | 2017-08-23 12:49:46 +0100 | [diff] [blame] | 389 | |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 390 | SubscriberAndDeviceInformation si = new SubscriberAndDeviceInformation(); |
| 391 | UniTagInformation specificService = getUniTagInformation(getPortName(port), cTag, sTag, tpId); |
| 392 | if (specificService == null) { |
| 393 | log.error("Can't find Information for subscriber on {}, with cTag {}, " + |
| 394 | "stag {}, tpId {}", cp, cTag, sTag, tpId); |
Amit Ghosh | 3193952 | 2018-08-16 13:28:21 +0100 | [diff] [blame] | 395 | return false; |
| 396 | } |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 397 | List<UniTagInformation> uniTagInformationList = new LinkedList<>(); |
| 398 | uniTagInformationList.add(specificService); |
| 399 | si.setUniTagList(uniTagInformationList); |
| 400 | DiscoveredSubscriber sub = new DiscoveredSubscriber(device, port, |
| 401 | DiscoveredSubscriber.Status.ADDED, true, si); |
Amit Ghosh | 3193952 | 2018-08-16 13:28:21 +0100 | [diff] [blame] | 402 | |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 403 | // NOTE we need to keep a list of the subscribers that are provisioned on a port, |
| 404 | // regardless of the flow status |
| 405 | ServiceKey sk = new ServiceKey(accessDevicePort, specificService); |
| 406 | if (oltFlowService.isSubscriberServiceProvisioned(sk)) { |
| 407 | log.error("Subscriber on {} is already provisioned", sk); |
Amit Ghosh | e1d3f09 | 2018-10-09 19:44:33 +0100 | [diff] [blame] | 408 | return false; |
| 409 | } |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 410 | oltFlowService.updateProvisionedSubscriberStatus(sk, true); |
| 411 | |
| 412 | addSubscriberToQueue(sub); |
| 413 | return true; |
Amit Ghosh | 3193952 | 2018-08-16 13:28:21 +0100 | [diff] [blame] | 414 | } |
| 415 | |
| 416 | @Override |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 417 | public boolean removeSubscriber(ConnectPoint cp, VlanId cTag, VlanId sTag, Integer tpId) { |
| 418 | log.debug("Un-provisioning subscriber on {} with cTag {}, stag {}, tpId {}", |
| 419 | cp, cTag, sTag, tpId); |
| 420 | Device device = deviceService.getDevice(cp.deviceId()); |
| 421 | Port port = deviceService.getPort(device.id(), cp.port()); |
| 422 | AccessDevicePort accessDevicePort = new AccessDevicePort(port); |
| 423 | |
| 424 | if (oltDeviceService.isNniPort(device, port.number())) { |
| 425 | log.warn("will not un-provision a subscriber on the NNI {}", |
| 426 | accessDevicePort); |
| 427 | return false; |
| 428 | } |
| 429 | |
| 430 | SubscriberAndDeviceInformation si = new SubscriberAndDeviceInformation(); |
| 431 | UniTagInformation specificService = getUniTagInformation(getPortName(port), cTag, sTag, tpId); |
| 432 | if (specificService == null) { |
| 433 | log.error("Can't find Information for subscriber on {}, with cTag {}, " + |
| 434 | "stag {}, tpId {}", cp, cTag, sTag, tpId); |
| 435 | return false; |
| 436 | } |
| 437 | List<UniTagInformation> uniTagInformationList = new LinkedList<>(); |
| 438 | uniTagInformationList.add(specificService); |
| 439 | si.setUniTagList(uniTagInformationList); |
| 440 | DiscoveredSubscriber sub = new DiscoveredSubscriber(device, port, |
| 441 | DiscoveredSubscriber.Status.ADDED, true, si); |
| 442 | |
| 443 | // NOTE we need to keep a list of the subscribers that are provisioned on a port, |
| 444 | // regardless of the flow status |
| 445 | ServiceKey sk = new ServiceKey(accessDevicePort, specificService); |
| 446 | if (!oltFlowService.isSubscriberServiceProvisioned(sk)) { |
| 447 | log.error("Subscriber on {} is not provisioned", sk); |
| 448 | return false; |
| 449 | } |
| 450 | oltFlowService.updateProvisionedSubscriberStatus(sk, false); |
| 451 | |
| 452 | addSubscriberToQueue(sub); |
| 453 | return true; |
Saurav Das | 82b8e6d | 2018-10-04 15:25:12 -0700 | [diff] [blame] | 454 | } |
| 455 | |
| 456 | @Override |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 457 | public List<DeviceId> getConnectedOlts() { |
Amit Ghosh | 1ed9aef | 2018-07-17 17:08:16 +0100 | [diff] [blame] | 458 | List<DeviceId> olts = new ArrayList<>(); |
| 459 | Iterable<Device> devices = deviceService.getDevices(); |
| 460 | for (Device d : devices) { |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 461 | if (oltDeviceService.isOlt(d)) { |
Saurav Das | 82b8e6d | 2018-10-04 15:25:12 -0700 | [diff] [blame] | 462 | // So this is indeed an OLT device |
Amit Ghosh | 1ed9aef | 2018-07-17 17:08:16 +0100 | [diff] [blame] | 463 | olts.add(d.id()); |
| 464 | } |
| 465 | } |
| 466 | return olts; |
alshabib | e055967 | 2016-02-21 14:49:51 -0800 | [diff] [blame] | 467 | } |
| 468 | |
Amit Ghosh | 3193952 | 2018-08-16 13:28:21 +0100 | [diff] [blame] | 469 | /** |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 470 | * Finds the connect-point to which a subscriber is connected. |
Amit Ghosh | 3193952 | 2018-08-16 13:28:21 +0100 | [diff] [blame] | 471 | * |
| 472 | * @param id The id of the subscriber, this is the same ID as in Sadis |
| 473 | * @return Subscribers ConnectPoint if found else null |
| 474 | */ |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 475 | @Override |
| 476 | public ConnectPoint findSubscriberConnectPoint(String id) { |
Amit Ghosh | 3193952 | 2018-08-16 13:28:21 +0100 | [diff] [blame] | 477 | |
| 478 | Iterable<Device> devices = deviceService.getDevices(); |
| 479 | for (Device d : devices) { |
| 480 | for (Port p : deviceService.getPorts(d.id())) { |
| 481 | log.trace("Comparing {} with {}", p.annotations().value(AnnotationKeys.PORT_NAME), id); |
| 482 | if (p.annotations().value(AnnotationKeys.PORT_NAME).equals(id)) { |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 483 | log.debug("Found on {}", portWithName(p)); |
Amit Ghosh | 3193952 | 2018-08-16 13:28:21 +0100 | [diff] [blame] | 484 | return new ConnectPoint(d.id(), p.number()); |
| 485 | } |
| 486 | } |
| 487 | } |
| 488 | return null; |
| 489 | } |
| 490 | |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 491 | protected void processDiscoveredSubscribers() { |
| 492 | log.info("Started processDiscoveredSubscribers loop"); |
| 493 | while (true) { |
| 494 | Set<ConnectPoint> discoveredCps; |
| 495 | try { |
| 496 | queueReadLock.lock(); |
| 497 | discoveredCps = eventsQueues.keySet(); |
| 498 | } catch (Exception e) { |
| 499 | log.error("Cannot read keys from queue map", e); |
| 500 | return; |
| 501 | } finally { |
| 502 | queueReadLock.unlock(); |
| 503 | } |
Gamze Abaka | 641fc07 | 2018-09-04 09:16:27 +0000 | [diff] [blame] | 504 | |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 505 | discoveredCps.forEach(cp -> { |
| 506 | LinkedBlockingQueue<DiscoveredSubscriber> eventsQueue; |
alshabib | bf23a1f | 2016-01-14 17:27:11 -0800 | [diff] [blame] | 507 | |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 508 | try { |
| 509 | queueReadLock.lock(); |
| 510 | eventsQueue = eventsQueues.get(cp); |
| 511 | } catch (Exception e) { |
| 512 | log.error("Cannot get key from queue map", e); |
| 513 | return; |
| 514 | } finally { |
| 515 | queueReadLock.unlock(); |
Tunahan Sezen | a07fe96 | 2021-02-24 08:24:24 +0000 | [diff] [blame] | 516 | } |
Tunahan Sezen | a07fe96 | 2021-02-24 08:24:24 +0000 | [diff] [blame] | 517 | |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 518 | if (!oltDeviceService.isLocalLeader(cp.deviceId())) { |
| 519 | // if we're not local leader for this device, ignore this queue |
| 520 | if (log.isTraceEnabled()) { |
| 521 | log.trace("Ignoring queue on CP {} as not master of the device", cp); |
Andrea Campanella | 7a1d7e7 | 2020-11-05 10:40:10 +0100 | [diff] [blame] | 522 | } |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 523 | return; |
Tunahan Sezen | a07fe96 | 2021-02-24 08:24:24 +0000 | [diff] [blame] | 524 | } |
| 525 | |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 526 | flowsExecutor.execute(() -> { |
| 527 | if (!eventsQueue.isEmpty()) { |
| 528 | // we do not remove the event from the queue until it has been processed |
| 529 | // in that way we guarantee that events are processed in order |
| 530 | DiscoveredSubscriber sub = eventsQueue.peek(); |
| 531 | if (sub == null) { |
| 532 | // the queue is empty |
| 533 | return; |
Tunahan Sezen | a07fe96 | 2021-02-24 08:24:24 +0000 | [diff] [blame] | 534 | } |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 535 | |
| 536 | if (log.isTraceEnabled()) { |
| 537 | log.trace("Processing subscriber on port {} with status {}", |
| 538 | portWithName(sub.port), sub.status); |
| 539 | } |
| 540 | |
| 541 | if (sub.hasSubscriber) { |
| 542 | // this is a provision subscriber call |
| 543 | if (oltFlowService.handleSubscriberFlows(sub, defaultBpId, multicastServiceName)) { |
| 544 | if (log.isTraceEnabled()) { |
| 545 | log.trace("Provisioning of subscriber on {} completed", |
| 546 | portWithName(sub.port)); |
| 547 | } |
| 548 | removeSubscriberFromQueue(sub); |
| 549 | } |
| 550 | } else { |
| 551 | // this is a port event (ENABLED/DISABLED) |
| 552 | // means no subscriber was provisioned on that port |
| 553 | |
| 554 | if (!deviceService.isAvailable(sub.device.id()) || |
| 555 | deviceService.getPort(sub.device.id(), sub.port.number()) == null) { |
| 556 | // If the device is not connected or the port is not available do nothing |
| 557 | // This can happen when we disable and then immediately delete the device, |
| 558 | // the queue is populated but the meters and flows are already gone |
| 559 | // thus there is nothing left to do |
| 560 | return; |
| 561 | } |
| 562 | |
| 563 | if (oltFlowService.handleBasicPortFlows(sub, defaultBpId, defaultBpId)) { |
| 564 | if (log.isTraceEnabled()) { |
| 565 | log.trace("Processing of port {} completed", |
| 566 | portWithName(sub.port)); |
| 567 | } |
| 568 | removeSubscriberFromQueue(sub); |
| 569 | } |
Tunahan Sezen | a07fe96 | 2021-02-24 08:24:24 +0000 | [diff] [blame] | 570 | } |
| 571 | } |
| 572 | }); |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 573 | }); |
Tunahan Sezen | a07fe96 | 2021-02-24 08:24:24 +0000 | [diff] [blame] | 574 | |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 575 | try { |
| 576 | TimeUnit.MILLISECONDS.sleep(requeueDelay); |
| 577 | } catch (InterruptedException e) { |
| 578 | continue; |
Tunahan Sezen | a07fe96 | 2021-02-24 08:24:24 +0000 | [diff] [blame] | 579 | } |
| 580 | } |
Tunahan Sezen | a07fe96 | 2021-02-24 08:24:24 +0000 | [diff] [blame] | 581 | } |
| 582 | |
| 583 | /** |
Andrea Campanella | cbbb795 | 2019-11-25 06:38:41 +0000 | [diff] [blame] | 584 | * Checks the subscriber uni tag list and find the uni tag information. |
| 585 | * using the pon c tag, pon s tag and the technology profile id |
| 586 | * May return Optional<null> |
| 587 | * |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 588 | * @param portName port of the subscriber |
Andrea Campanella | cbbb795 | 2019-11-25 06:38:41 +0000 | [diff] [blame] | 589 | * @param innerVlan pon c tag |
| 590 | * @param outerVlan pon s tag |
| 591 | * @param tpId the technology profile id |
| 592 | * @return the found uni tag information |
| 593 | */ |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 594 | private UniTagInformation getUniTagInformation(String portName, VlanId innerVlan, |
| 595 | VlanId outerVlan, int tpId) { |
Tunahan Sezen | f0843b9 | 2021-04-30 07:13:16 +0000 | [diff] [blame] | 596 | log.debug("Getting uni tag information for {}, innerVlan: {}, outerVlan: {}, tpId: {}", |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 597 | portName, innerVlan, outerVlan, tpId); |
| 598 | SubscriberAndDeviceInformation subInfo = subsService.get(portName); |
Gamze Abaka | f59c091 | 2019-04-19 08:24:28 +0000 | [diff] [blame] | 599 | if (subInfo == null) { |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 600 | log.warn("Subscriber information doesn't exist for {}", portName); |
| 601 | return null; |
Gamze Abaka | f59c091 | 2019-04-19 08:24:28 +0000 | [diff] [blame] | 602 | } |
| 603 | |
Andrea Campanella | cbbb795 | 2019-11-25 06:38:41 +0000 | [diff] [blame] | 604 | List<UniTagInformation> uniTagList = subInfo.uniTagList(); |
| 605 | if (uniTagList == null) { |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 606 | log.warn("Uni tag list is not found for the subscriber {} on {}", subInfo.id(), portName); |
| 607 | return null; |
Andrea Campanella | cbbb795 | 2019-11-25 06:38:41 +0000 | [diff] [blame] | 608 | } |
Amit Ghosh | e1d3f09 | 2018-10-09 19:44:33 +0100 | [diff] [blame] | 609 | |
Andrea Campanella | cbbb795 | 2019-11-25 06:38:41 +0000 | [diff] [blame] | 610 | UniTagInformation service = null; |
| 611 | for (UniTagInformation tagInfo : subInfo.uniTagList()) { |
| 612 | if (innerVlan.equals(tagInfo.getPonCTag()) && outerVlan.equals(tagInfo.getPonSTag()) |
| 613 | && tpId == tagInfo.getTechnologyProfileId()) { |
| 614 | service = tagInfo; |
| 615 | break; |
Andy Bavier | 160e868 | 2019-05-07 18:32:22 -0700 | [diff] [blame] | 616 | } |
Gamze Abaka | 1efc80c | 2019-02-15 12:10:54 +0000 | [diff] [blame] | 617 | } |
Gamze Abaka | 1efc80c | 2019-02-15 12:10:54 +0000 | [diff] [blame] | 618 | |
Andrea Campanella | cbbb795 | 2019-11-25 06:38:41 +0000 | [diff] [blame] | 619 | if (service == null) { |
Matteo Scandolo | 19b56f6 | 2020-10-29 13:29:21 -0700 | [diff] [blame] | 620 | log.warn("SADIS doesn't include the service with ponCtag {} ponStag {} and tpId {} on {}", |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 621 | innerVlan, outerVlan, tpId, portName); |
Saurav Das | 82b8e6d | 2018-10-04 15:25:12 -0700 | [diff] [blame] | 622 | return null; |
| 623 | } |
Amit Ghosh | 1ed9aef | 2018-07-17 17:08:16 +0100 | [diff] [blame] | 624 | |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 625 | return service; |
Amit Ghosh | 1ed9aef | 2018-07-17 17:08:16 +0100 | [diff] [blame] | 626 | } |
| 627 | |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 628 | protected void bindSadisService(SadisService service) { |
| 629 | sadisService = service; |
| 630 | subsService = sadisService.getSubscriberInfoService(); |
| 631 | |
| 632 | log.info("Sadis-service binds to onos."); |
Amit Ghosh | 1ed9aef | 2018-07-17 17:08:16 +0100 | [diff] [blame] | 633 | } |
| 634 | |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 635 | protected void unbindSadisService(SadisService service) { |
| 636 | deviceService.removeListener(deviceListener); |
| 637 | deviceListener = null; |
| 638 | sadisService = null; |
| 639 | subsService = null; |
| 640 | log.info("Sadis-service unbinds from onos."); |
Jonathan Hart | 1d34c8b | 2018-05-05 15:37:28 -0700 | [diff] [blame] | 641 | } |
| 642 | |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 643 | protected void addSubscriberToQueue(DiscoveredSubscriber sub) { |
| 644 | ConnectPoint cp = new ConnectPoint(sub.device.id(), sub.port.number()); |
| 645 | LinkedBlockingQueue<DiscoveredSubscriber> q = null; |
| 646 | try { |
| 647 | queueReadLock.lock(); |
| 648 | q = eventsQueues.getOrDefault(cp, new LinkedBlockingQueue<>()); |
| 649 | } finally { |
| 650 | queueReadLock.unlock(); |
Ilayda Ozdemir | 90a9362 | 2021-02-25 09:40:58 +0000 | [diff] [blame] | 651 | } |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 652 | if (!q.contains(sub)) { |
| 653 | log.info("Adding subscriber to queue: {} with status {} and subscriber {}", |
| 654 | portWithName(sub.port), sub.status, sub.hasSubscriber); |
| 655 | q.add(sub); |
Andrea Campanella | 1edf883 | 2021-05-06 12:51:33 +0200 | [diff] [blame] | 656 | } else { |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 657 | log.debug("Not adding subscriber to queue as already present: {} with status {}", |
| 658 | portWithName(sub.port), sub.status); |
| 659 | // no need to update the queue in the map if nothing has changed |
| 660 | return; |
| 661 | } |
| 662 | try { |
| 663 | queueWriteLock.lock(); |
| 664 | eventsQueues.put(cp, q); |
| 665 | } catch (UnsupportedOperationException | ClassCastException | |
| 666 | NullPointerException | IllegalArgumentException e) { |
| 667 | log.error("Cannot add subscriber to queue: {}", e.getMessage()); |
| 668 | } finally { |
| 669 | queueWriteLock.unlock(); |
Jonathan Hart | 4f178fa | 2020-02-03 10:46:01 -0800 | [diff] [blame] | 670 | } |
Jonathan Hart | 4f178fa | 2020-02-03 10:46:01 -0800 | [diff] [blame] | 671 | } |
| 672 | |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 673 | protected void removeSubscriberFromQueue(DiscoveredSubscriber sub) { |
| 674 | ConnectPoint cp = new ConnectPoint(sub.device.id(), sub.port.number()); |
| 675 | LinkedBlockingQueue<DiscoveredSubscriber> q = null; |
| 676 | if (log.isTraceEnabled()) { |
| 677 | log.trace("removing subscriber {} from queue", sub); |
kdarapu | aa5da25 | 2020-04-10 15:58:05 +0530 | [diff] [blame] | 678 | } |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 679 | try { |
| 680 | queueReadLock.lock(); |
| 681 | q = eventsQueues.get(cp); |
| 682 | } finally { |
| 683 | queueReadLock.unlock(); |
| 684 | } |
| 685 | if (q == null) { |
| 686 | log.warn("Cannot find queue for connectPoint {}", cp); |
| 687 | return; |
| 688 | } |
| 689 | boolean removed = q.remove(sub); |
| 690 | if (!removed) { |
| 691 | log.warn("Subscriber {} has not been removed from queue, is it still there? {}", sub, q); |
| 692 | return; |
| 693 | } else { |
| 694 | log.debug("Subscriber {} has been removed from the queue", sub); |
Tunahan Sezen | a07fe96 | 2021-02-24 08:24:24 +0000 | [diff] [blame] | 695 | } |
| 696 | |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 697 | try { |
| 698 | queueWriteLock.lock(); |
| 699 | eventsQueues.remove(cp); // am I needed?? |
| 700 | eventsQueues.put(cp, q); |
| 701 | } catch (UnsupportedOperationException | ClassCastException | |
| 702 | NullPointerException | IllegalArgumentException e) { |
| 703 | log.error("Cannot remove subscriber {} from queue: {}", sub, e.getMessage()); |
| 704 | } finally { |
| 705 | queueWriteLock.unlock(); |
Tunahan Sezen | a07fe96 | 2021-02-24 08:24:24 +0000 | [diff] [blame] | 706 | } |
| 707 | } |
| 708 | |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 709 | protected class OltDeviceListener |
| 710 | implements DeviceListener { |
| 711 | |
| 712 | private final Logger log = LoggerFactory.getLogger(getClass()); |
| 713 | protected ExecutorService eventExecutor; |
| 714 | |
| 715 | /** |
| 716 | * Builds the listener with all the proper services and information needed. |
| 717 | */ |
| 718 | public OltDeviceListener() { |
| 719 | this.eventExecutor = Executors.newFixedThreadPool(flowProcessingThreads, |
| 720 | groupedThreads("onos/olt-device-listener-event", "event-%d", log)); |
| 721 | } |
| 722 | |
| 723 | public void deactivate() { |
| 724 | this.eventExecutor.shutdown(); |
| 725 | } |
Saurav Das | a9d5f44 | 2019-03-06 19:32:48 -0800 | [diff] [blame] | 726 | |
alshabib | f0e7e70 | 2015-05-30 18:22:36 -0700 | [diff] [blame] | 727 | @Override |
| 728 | public void event(DeviceEvent event) { |
Matteo Scandolo | 632f0fc | 2018-09-07 12:21:45 -0700 | [diff] [blame] | 729 | eventExecutor.execute(() -> { |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 730 | boolean isOlt = oltDeviceService.isOlt(event.subject()); |
| 731 | DeviceId deviceId = event.subject().id(); |
Matteo Scandolo | 632f0fc | 2018-09-07 12:21:45 -0700 | [diff] [blame] | 732 | switch (event.type()) { |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 733 | case PORT_STATS_UPDATED: |
Matteo Scandolo | 632f0fc | 2018-09-07 12:21:45 -0700 | [diff] [blame] | 734 | case DEVICE_ADDED: |
Matteo Scandolo | 632f0fc | 2018-09-07 12:21:45 -0700 | [diff] [blame] | 735 | return; |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 736 | case PORT_ADDED: |
| 737 | case PORT_UPDATED: |
| 738 | case PORT_REMOVED: |
| 739 | if (!isOlt) { |
| 740 | log.trace("Ignoring event {}, this is not an OLT device", deviceId); |
| 741 | return; |
| 742 | } |
| 743 | if (!oltDeviceService.isLocalLeader(deviceId)) { |
| 744 | log.trace("Device {} is not local to this node", deviceId); |
| 745 | return; |
| 746 | } |
| 747 | // port added, updated and removed are treated in the same way as we only care whether the port |
| 748 | // is enabled or not |
| 749 | handleOltPort(event.type(), event.subject(), event.port()); |
| 750 | return; |
| 751 | case DEVICE_AVAILABILITY_CHANGED: |
| 752 | if (!isOlt) { |
| 753 | log.trace("Ignoring event {}, this is not an OLT device", deviceId); |
| 754 | return; |
| 755 | } |
| 756 | if (deviceService.isAvailable(deviceId)) { |
| 757 | if (!oltDeviceService.isLocalLeader(deviceId)) { |
| 758 | if (log.isTraceEnabled()) { |
| 759 | log.trace("Device {} is not local to this node, not handling available device", |
| 760 | deviceId); |
| 761 | } |
| 762 | } else { |
| 763 | log.info("Handling available device: {}", deviceId); |
| 764 | handleExistingPorts(); |
| 765 | } |
| 766 | } else if (!deviceService.isAvailable(deviceId) && deviceService.getPorts(deviceId).isEmpty()) { |
| 767 | // NOTE that upon disconnection there is no mastership on the device, |
| 768 | // and we should anyway clear the local cache of the flows/meters across instances. |
| 769 | // We're only clearing the device if there are no available ports, |
| 770 | // otherwise we assume it's a temporary disconnection |
| 771 | log.info("Device {} availability changed to false and ports are empty, " + |
| 772 | "purging meters and flows", deviceId); |
| 773 | //NOTE all the instances will call these methods |
| 774 | oltFlowService.purgeDeviceFlows(deviceId); |
| 775 | oltMeterService.purgeDeviceMeters(deviceId); |
| 776 | clearQueueForDevice(deviceId); |
| 777 | } else { |
| 778 | log.info("Device {} availability changed to false, but ports are still available, " + |
| 779 | "assuming temporary disconnection. Ports: {}", |
| 780 | deviceId, deviceService.getPorts(deviceId)); |
| 781 | } |
| 782 | return; |
| 783 | case DEVICE_REMOVED: |
| 784 | if (!isOlt) { |
| 785 | log.trace("Ignoring event {}, this is not an OLT device", deviceId); |
| 786 | return; |
| 787 | } |
| 788 | log.info("Device {} Removed, purging meters and flows", deviceId); |
| 789 | oltFlowService.purgeDeviceFlows(deviceId); |
| 790 | oltMeterService.purgeDeviceMeters(deviceId); |
| 791 | clearQueueForDevice(deviceId); |
| 792 | return; |
| 793 | default: |
| 794 | if (log.isTraceEnabled()) { |
| 795 | log.trace("Not handling event: {}, ", event); |
| 796 | } |
Matteo Scandolo | 632f0fc | 2018-09-07 12:21:45 -0700 | [diff] [blame] | 797 | } |
| 798 | }); |
alshabib | f0e7e70 | 2015-05-30 18:22:36 -0700 | [diff] [blame] | 799 | } |
Gamze Abaka | 838d814 | 2019-02-21 07:06:55 +0000 | [diff] [blame] | 800 | |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 801 | protected void clearQueueForDevice(DeviceId devId) { |
| 802 | try { |
| 803 | queueWriteLock.lock(); |
| 804 | Iterator<Map.Entry<ConnectPoint, LinkedBlockingQueue<DiscoveredSubscriber>>> iter = |
| 805 | eventsQueues.entrySet().iterator(); |
| 806 | while (iter.hasNext()) { |
| 807 | Map.Entry<ConnectPoint, LinkedBlockingQueue<DiscoveredSubscriber>> entry = iter.next(); |
| 808 | if (entry.getKey().deviceId().equals(devId)) { |
| 809 | eventsQueues.remove(entry.getKey()); |
| 810 | } |
| 811 | } |
| 812 | } finally { |
| 813 | queueWriteLock.unlock(); |
Gamze Abaka | 838d814 | 2019-02-21 07:06:55 +0000 | [diff] [blame] | 814 | } |
Andrea Campanella | cbbb795 | 2019-11-25 06:38:41 +0000 | [diff] [blame] | 815 | } |
| 816 | |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 817 | protected void handleOltPort(DeviceEvent.Type type, Device device, Port port) { |
| 818 | log.info("OltDeviceListener receives event {} for port {} with status {} on device {}", type, |
| 819 | portWithName(port), port.isEnabled() ? "ENABLED" : "DISABLED", device.id()); |
| 820 | if (port.isEnabled()) { |
| 821 | if (oltDeviceService.isNniPort(device, port.number())) { |
| 822 | // NOTE in the NNI case we receive a PORT_REMOVED event with status ENABLED, thus we need to |
| 823 | // pass the floeAction to the handleNniFlows method |
| 824 | OltFlowService.FlowOperation action = OltFlowService.FlowOperation.ADD; |
| 825 | if (type == DeviceEvent.Type.PORT_REMOVED) { |
| 826 | action = OltFlowService.FlowOperation.REMOVE; |
| 827 | } |
| 828 | oltFlowService.handleNniFlows(device, port, action); |
| 829 | } else { |
| 830 | post(new AccessDeviceEvent(AccessDeviceEvent.Type.UNI_ADDED, device.id(), port)); |
| 831 | // NOTE if the subscriber was previously provisioned, |
| 832 | // then add it back to the queue to be re-provisioned |
| 833 | boolean provisionSubscriber = oltFlowService. |
| 834 | isSubscriberServiceProvisioned(new AccessDevicePort(port)); |
| 835 | SubscriberAndDeviceInformation si = subsService.get(getPortName(port)); |
| 836 | if (si == null) { |
| 837 | //NOTE this should not happen given that the subscriber was provisioned before |
| 838 | log.error("Subscriber information not found in sadis for port {}", |
| 839 | portWithName(port)); |
| 840 | return; |
| 841 | } |
| 842 | |
| 843 | DiscoveredSubscriber.Status status = DiscoveredSubscriber.Status.ADDED; |
| 844 | if (type == DeviceEvent.Type.PORT_REMOVED) { |
| 845 | status = DiscoveredSubscriber.Status.REMOVED; |
| 846 | } |
| 847 | |
| 848 | DiscoveredSubscriber sub = |
| 849 | new DiscoveredSubscriber(device, port, |
| 850 | status, provisionSubscriber, si); |
| 851 | addSubscriberToQueue(sub); |
| 852 | } |
| 853 | } else { |
| 854 | if (oltDeviceService.isNniPort(device, port.number())) { |
| 855 | // NOTE this may need to be handled on DEVICE_REMOVE as we don't disable the NNI |
| 856 | oltFlowService.handleNniFlows(device, port, OltFlowService.FlowOperation.REMOVE); |
| 857 | } else { |
| 858 | post(new AccessDeviceEvent(AccessDeviceEvent.Type.UNI_REMOVED, device.id(), port)); |
| 859 | // NOTE we are assuming that if a subscriber has default eapol |
| 860 | // it does not have subscriber flows |
| 861 | if (oltFlowService.hasDefaultEapol(port)) { |
| 862 | SubscriberAndDeviceInformation si = subsService.get(getPortName(port)); |
| 863 | if (si == null) { |
| 864 | //NOTE this should not happen given that the subscriber was provisioned before |
| 865 | log.error("Subscriber information not found in sadis for port {}", |
| 866 | portWithName(port)); |
| 867 | return; |
| 868 | } |
| 869 | DiscoveredSubscriber sub = |
| 870 | new DiscoveredSubscriber(device, port, |
| 871 | DiscoveredSubscriber.Status.REMOVED, false, si); |
| 872 | |
| 873 | addSubscriberToQueue(sub); |
| 874 | |
| 875 | } else if (oltFlowService.isSubscriberServiceProvisioned(new AccessDevicePort(port))) { |
| 876 | SubscriberAndDeviceInformation si = subsService.get(getPortName(port)); |
| 877 | if (si == null) { |
| 878 | //NOTE this should not happen given that the subscriber was provisioned before |
| 879 | log.error("Subscriber information not found in sadis for port {}", |
| 880 | portWithName(port)); |
| 881 | return; |
| 882 | } |
| 883 | DiscoveredSubscriber sub = |
| 884 | new DiscoveredSubscriber(device, port, |
| 885 | DiscoveredSubscriber.Status.REMOVED, true, si); |
| 886 | addSubscriberToQueue(sub); |
| 887 | } |
| 888 | } |
Andrea Campanella | cbbb795 | 2019-11-25 06:38:41 +0000 | [diff] [blame] | 889 | } |
Gamze Abaka | 838d814 | 2019-02-21 07:06:55 +0000 | [diff] [blame] | 890 | } |
Gamze Abaka | da282b4 | 2019-03-11 13:16:48 +0000 | [diff] [blame] | 891 | |
Matteo Scandolo | aa2adde | 2021-09-13 12:45:32 -0700 | [diff] [blame^] | 892 | /** |
| 893 | * This method is invoked on app activation in order to deal |
| 894 | * with devices and ports that are already existing in the system |
| 895 | * and thus won't trigger an event. |
| 896 | * It is also needed on instance reboot and device reconnect |
| 897 | */ |
| 898 | protected void handleExistingPorts() { |
| 899 | Iterable<DeviceId> devices = getConnectedOlts(); |
| 900 | for (DeviceId deviceId : devices) { |
| 901 | log.info("Handling existing OLT Ports for device {}", deviceId); |
| 902 | if (oltDeviceService.isLocalLeader(deviceId)) { |
| 903 | List<Port> ports = deviceService.getPorts(deviceId); |
| 904 | for (Port p : ports) { |
| 905 | if (PortNumber.LOCAL.equals(p.number()) || !p.isEnabled()) { |
| 906 | continue; |
| 907 | } |
| 908 | Device device = deviceService.getDevice(deviceId); |
| 909 | deviceListener.handleOltPort(DeviceEvent.Type.PORT_UPDATED, device, p); |
| 910 | } |
| 911 | } |
Jonathan Hart | 4f178fa | 2020-02-03 10:46:01 -0800 | [diff] [blame] | 912 | } |
| 913 | } |
| 914 | } |
Hardik Windlass | 395ff37 | 2019-06-13 05:16:00 +0000 | [diff] [blame] | 915 | } |