mpagenko | c26d4c0 | 2021-05-06 14:27:57 +0000 | [diff] [blame] | 1 | /* |
| 2 | * Copyright 2020-present Open Networking Foundation |
| 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | |
| 17 | //Package adaptercoreonu provides the utility for onu devices, flows and statistics |
| 18 | package adaptercoreonu |
| 19 | |
| 20 | import ( |
| 21 | "bufio" |
| 22 | "context" |
| 23 | "fmt" |
| 24 | "io" |
| 25 | "net/http" |
| 26 | "net/url" |
| 27 | "os" |
Andrey Pozolotin | 1394a1c | 2021-06-01 00:54:18 +0300 | [diff] [blame] | 28 | "path/filepath" |
mpagenko | c26d4c0 | 2021-05-06 14:27:57 +0000 | [diff] [blame] | 29 | "sync" |
| 30 | "time" |
| 31 | |
Girish Gowdra | 50e5642 | 2021-06-01 16:46:04 -0700 | [diff] [blame] | 32 | "github.com/opencord/voltha-lib-go/v5/pkg/log" |
mpagenko | aa3afe9 | 2021-05-21 16:20:58 +0000 | [diff] [blame] | 33 | "github.com/opencord/voltha-protos/v4/go/voltha" |
mpagenko | c26d4c0 | 2021-05-06 14:27:57 +0000 | [diff] [blame] | 34 | ) |
| 35 | |
| 36 | const cDefaultLocalDir = "/tmp" //this is the default local dir to download to |
| 37 | |
| 38 | type fileState uint32 |
| 39 | |
| 40 | //nolint:varcheck, deadcode |
| 41 | const ( |
| 42 | cFileStateUnknown fileState = iota |
| 43 | cFileStateDlStarted |
| 44 | cFileStateDlSucceeded |
| 45 | cFileStateDlFailed |
| 46 | cFileStateDlAborted |
| 47 | cFileStateDlInvalid |
| 48 | ) |
| 49 | |
| 50 | type downloadImageParams struct { |
| 51 | downloadImageName string |
| 52 | downloadImageState fileState |
| 53 | downloadImageLen int64 |
| 54 | downloadImageCrc uint32 |
| 55 | } |
| 56 | |
| 57 | type requesterChannelMap map[chan<- bool]struct{} //using an empty structure map for easier (unique) element appending |
| 58 | |
| 59 | //fileDownloadManager structure holds information needed for downloading to and storing images within the adapter |
| 60 | type fileDownloadManager struct { |
| 61 | mutexDownloadImageDsc sync.RWMutex |
| 62 | downloadImageDscSlice []downloadImageParams |
| 63 | dnldImgReadyWaiting map[string]requesterChannelMap |
| 64 | dlToAdapterTimeout time.Duration |
| 65 | } |
| 66 | |
| 67 | //newFileDownloadManager constructor returns a new instance of a fileDownloadManager |
| 68 | //mib_db (as well as not inluded alarm_db not really used in this code? VERIFY!!) |
| 69 | func newFileDownloadManager(ctx context.Context) *fileDownloadManager { |
| 70 | logger.Debug(ctx, "init-fileDownloadManager") |
| 71 | var localDnldMgr fileDownloadManager |
| 72 | localDnldMgr.downloadImageDscSlice = make([]downloadImageParams, 0) |
| 73 | localDnldMgr.dnldImgReadyWaiting = make(map[string]requesterChannelMap) |
| 74 | localDnldMgr.dlToAdapterTimeout = 10 * time.Second //default timeout, should be overwritten immediately after start |
| 75 | return &localDnldMgr |
| 76 | } |
| 77 | |
| 78 | //SetDownloadTimeout configures the timeout used to supervice the download of the image to the adapter (assumed in seconds) |
| 79 | func (dm *fileDownloadManager) SetDownloadTimeout(ctx context.Context, aDlTimeout time.Duration) { |
| 80 | dm.mutexDownloadImageDsc.Lock() |
| 81 | defer dm.mutexDownloadImageDsc.Unlock() |
| 82 | logger.Debugw(ctx, "setting download timeout", log.Fields{"timeout": aDlTimeout}) |
| 83 | dm.dlToAdapterTimeout = aDlTimeout |
| 84 | } |
| 85 | |
| 86 | //GetDownloadTimeout delivers the timeout used to supervice the download of the image to the adapter (assumed in seconds) |
| 87 | func (dm *fileDownloadManager) GetDownloadTimeout(ctx context.Context) time.Duration { |
| 88 | dm.mutexDownloadImageDsc.RLock() |
| 89 | defer dm.mutexDownloadImageDsc.RUnlock() |
| 90 | return dm.dlToAdapterTimeout |
| 91 | } |
| 92 | |
| 93 | //ImageExists returns true if the requested image already exists within the adapter |
| 94 | func (dm *fileDownloadManager) ImageExists(ctx context.Context, aImageName string) bool { |
| 95 | logger.Debugw(ctx, "checking on existence of the image", log.Fields{"image-name": aImageName}) |
| 96 | dm.mutexDownloadImageDsc.RLock() |
| 97 | defer dm.mutexDownloadImageDsc.RUnlock() |
| 98 | |
| 99 | for _, dnldImgDsc := range dm.downloadImageDscSlice { |
| 100 | if dnldImgDsc.downloadImageName == aImageName { |
| 101 | //image found (by name) |
| 102 | return true |
| 103 | } |
| 104 | } |
| 105 | //image not found (by name) |
| 106 | return false |
| 107 | } |
| 108 | |
| 109 | //StartDownload returns true if the download of the requested image could be started for the given file name and URL |
| 110 | func (dm *fileDownloadManager) StartDownload(ctx context.Context, aImageName string, aURLCommand string) error { |
| 111 | logger.Infow(ctx, "image download-to-adapter requested", log.Fields{ |
| 112 | "image-name": aImageName, "url-command": aURLCommand}) |
| 113 | loDownloadImageParams := downloadImageParams{ |
| 114 | downloadImageName: aImageName, downloadImageState: cFileStateDlStarted, |
| 115 | downloadImageLen: 0, downloadImageCrc: 0} |
| 116 | dm.mutexDownloadImageDsc.Lock() |
| 117 | dm.downloadImageDscSlice = append(dm.downloadImageDscSlice, loDownloadImageParams) |
| 118 | dm.mutexDownloadImageDsc.Unlock() |
| 119 | //try to download from http |
| 120 | err := dm.downloadFile(ctx, aURLCommand, cDefaultLocalDir, aImageName) |
| 121 | //return the result of the start-request to comfort the core processing even though the complete download may go on in background |
| 122 | return err |
| 123 | } |
| 124 | |
| 125 | //GetImageBufferLen returns the length of the specified file in bytes (file size) - as detected after download |
| 126 | func (dm *fileDownloadManager) GetImageBufferLen(ctx context.Context, aFileName string) (int64, error) { |
| 127 | dm.mutexDownloadImageDsc.RLock() |
| 128 | defer dm.mutexDownloadImageDsc.RUnlock() |
| 129 | for _, dnldImgDsc := range dm.downloadImageDscSlice { |
| 130 | if dnldImgDsc.downloadImageName == aFileName && dnldImgDsc.downloadImageState == cFileStateDlSucceeded { |
| 131 | //image found (by name) and fully downloaded |
| 132 | return dnldImgDsc.downloadImageLen, nil |
| 133 | } |
| 134 | } |
| 135 | return 0, fmt.Errorf("no downloaded image found: %s", aFileName) |
| 136 | } |
| 137 | |
| 138 | //GetDownloadImageBuffer returns the content of the requested file as byte slice |
| 139 | func (dm *fileDownloadManager) GetDownloadImageBuffer(ctx context.Context, aFileName string) ([]byte, error) { |
Andrey Pozolotin | 1394a1c | 2021-06-01 00:54:18 +0300 | [diff] [blame] | 140 | file, err := os.Open(filepath.Clean(cDefaultLocalDir + "/" + aFileName)) |
mpagenko | c26d4c0 | 2021-05-06 14:27:57 +0000 | [diff] [blame] | 141 | if err != nil { |
| 142 | return nil, err |
| 143 | } |
Andrey Pozolotin | 1394a1c | 2021-06-01 00:54:18 +0300 | [diff] [blame] | 144 | defer func() { |
| 145 | err := file.Close() |
| 146 | if err != nil { |
| 147 | logger.Errorw(ctx, "failed to close file", log.Fields{"error": err}) |
| 148 | } |
| 149 | }() |
mpagenko | c26d4c0 | 2021-05-06 14:27:57 +0000 | [diff] [blame] | 150 | |
| 151 | stats, statsErr := file.Stat() |
| 152 | if statsErr != nil { |
| 153 | return nil, statsErr |
| 154 | } |
| 155 | |
| 156 | var size int64 = stats.Size() |
| 157 | bytes := make([]byte, size) |
| 158 | |
| 159 | buffer := bufio.NewReader(file) |
| 160 | _, err = buffer.Read(bytes) |
| 161 | |
| 162 | return bytes, err |
| 163 | } |
| 164 | |
| 165 | //RequestDownloadReady receives a channel that has to be used to inform the requester in case the concerned file is downloaded |
| 166 | func (dm *fileDownloadManager) RequestDownloadReady(ctx context.Context, aFileName string, aWaitChannel chan<- bool) { |
| 167 | if dm.imageLocallyDownloaded(ctx, aFileName) { |
| 168 | //image found (by name) and fully downloaded |
| 169 | logger.Debugw(ctx, "file ready - immediate response", log.Fields{"image-name": aFileName}) |
| 170 | aWaitChannel <- true |
| 171 | return |
| 172 | } |
| 173 | //when we are here the image was not yet found or not fully downloaded - |
| 174 | // add the device specific channel to the list of waiting requesters |
| 175 | dm.mutexDownloadImageDsc.Lock() |
| 176 | defer dm.mutexDownloadImageDsc.Unlock() |
| 177 | if loRequesterChannelMap, ok := dm.dnldImgReadyWaiting[aFileName]; ok { |
| 178 | //entry for the file name already exists |
| 179 | if _, exists := loRequesterChannelMap[aWaitChannel]; !exists { |
| 180 | // requester channel does not yet exist for the image |
| 181 | loRequesterChannelMap[aWaitChannel] = struct{}{} |
| 182 | dm.dnldImgReadyWaiting[aFileName] = loRequesterChannelMap |
| 183 | logger.Debugw(ctx, "file not ready - adding new requester", log.Fields{ |
| 184 | "image-name": aFileName, "number-of-requesters": len(dm.dnldImgReadyWaiting[aFileName])}) |
| 185 | } |
| 186 | } else { |
| 187 | //entry for the file name does not even exist |
| 188 | addRequesterChannelMap := make(map[chan<- bool]struct{}) |
| 189 | addRequesterChannelMap[aWaitChannel] = struct{}{} |
| 190 | dm.dnldImgReadyWaiting[aFileName] = addRequesterChannelMap |
| 191 | logger.Debugw(ctx, "file not ready - setting first requester", log.Fields{ |
| 192 | "image-name": aFileName}) |
| 193 | } |
| 194 | } |
| 195 | |
| 196 | //RemoveReadyRequest removes the specified channel from the requester(channel) map for the given file name |
| 197 | func (dm *fileDownloadManager) RemoveReadyRequest(ctx context.Context, aFileName string, aWaitChannel chan bool) { |
| 198 | dm.mutexDownloadImageDsc.Lock() |
| 199 | defer dm.mutexDownloadImageDsc.Unlock() |
| 200 | for imageName, channelMap := range dm.dnldImgReadyWaiting { |
| 201 | if imageName == aFileName { |
| 202 | for channel := range channelMap { |
| 203 | if channel == aWaitChannel { |
| 204 | delete(dm.dnldImgReadyWaiting[imageName], channel) |
| 205 | logger.Debugw(ctx, "channel removed from the requester map", log.Fields{ |
| 206 | "image-name": aFileName, "new number-of-requesters": len(dm.dnldImgReadyWaiting[aFileName])}) |
| 207 | return //can leave directly |
| 208 | } |
| 209 | } |
| 210 | return //can leave directly |
| 211 | } |
| 212 | } |
| 213 | } |
| 214 | |
| 215 | // FileDownloadManager private (unexported) methods -- start |
| 216 | |
| 217 | //imageLocallyDownloaded returns true if the requested image already exists within the adapter |
| 218 | func (dm *fileDownloadManager) imageLocallyDownloaded(ctx context.Context, aImageName string) bool { |
| 219 | logger.Debugw(ctx, "checking if image is fully downloaded to adapter", log.Fields{"image-name": aImageName}) |
| 220 | dm.mutexDownloadImageDsc.RLock() |
| 221 | defer dm.mutexDownloadImageDsc.RUnlock() |
| 222 | |
| 223 | for _, dnldImgDsc := range dm.downloadImageDscSlice { |
| 224 | if dnldImgDsc.downloadImageName == aImageName { |
| 225 | //image found (by name) |
| 226 | if dnldImgDsc.downloadImageState == cFileStateDlSucceeded { |
| 227 | logger.Debugw(ctx, "image has been fully downloaded", log.Fields{"image-name": aImageName}) |
| 228 | return true |
| 229 | } |
| 230 | logger.Debugw(ctx, "image not yet fully downloaded", log.Fields{"image-name": aImageName}) |
| 231 | return false |
| 232 | } |
| 233 | } |
| 234 | //image not found (by name) |
| 235 | logger.Errorw(ctx, "image does not exist", log.Fields{"image-name": aImageName}) |
| 236 | return false |
| 237 | } |
| 238 | |
| 239 | //downloadFile downloads the specified file from the given http location |
| 240 | func (dm *fileDownloadManager) downloadFile(ctx context.Context, aURLCommand string, aFilePath string, aFileName string) error { |
| 241 | // Get the data |
| 242 | logger.Infow(ctx, "downloading with URL", log.Fields{"url": aURLCommand, "localPath": aFilePath}) |
| 243 | // verifying the complete URL by parsing it to its URL elements |
| 244 | urlBase, err1 := url.Parse(aURLCommand) |
| 245 | if err1 != nil { |
| 246 | logger.Errorw(ctx, "could not set base url command", log.Fields{"url": aURLCommand, "error": err1}) |
| 247 | return fmt.Errorf("could not set base url command: %s, error: %s", aURLCommand, err1) |
| 248 | } |
| 249 | urlParams := url.Values{} |
| 250 | urlBase.RawQuery = urlParams.Encode() |
| 251 | |
| 252 | //pre-check on file existence - assuming http location here |
| 253 | reqExist, errExist2 := http.NewRequest("HEAD", urlBase.String(), nil) |
| 254 | if errExist2 != nil { |
| 255 | logger.Errorw(ctx, "could not generate http head request", log.Fields{"url": urlBase.String(), "error": errExist2}) |
| 256 | return fmt.Errorf("could not generate http head request: %s, error: %s", aURLCommand, errExist2) |
| 257 | } |
| 258 | ctxExist, cancelExist := context.WithDeadline(ctx, time.Now().Add(3*time.Second)) //waiting for some fast answer |
| 259 | defer cancelExist() |
| 260 | _ = reqExist.WithContext(ctxExist) |
| 261 | respExist, errExist3 := http.DefaultClient.Do(reqExist) |
| 262 | if errExist3 != nil || respExist.StatusCode != http.StatusOK { |
| 263 | logger.Infow(ctx, "could not http head from url", log.Fields{"url": urlBase.String(), |
| 264 | "error": errExist3, "status": respExist.StatusCode}) |
| 265 | //if head is not supported by server we cannot use this test and just try to continue |
| 266 | if respExist.StatusCode != http.StatusMethodNotAllowed { |
| 267 | logger.Errorw(ctx, "http head from url: file does not exist here, aborting", log.Fields{"url": urlBase.String(), |
| 268 | "error": errExist3, "status": respExist.StatusCode}) |
| 269 | return fmt.Errorf("http head from url: file does not exist here, aborting: %s, error: %s, status: %d", |
| 270 | aURLCommand, errExist2, respExist.StatusCode) |
| 271 | } |
| 272 | } |
| 273 | defer func() { |
| 274 | deferredErr := respExist.Body.Close() |
| 275 | if deferredErr != nil { |
| 276 | logger.Errorw(ctx, "error at closing http head response body", log.Fields{"url": urlBase.String(), "error": deferredErr}) |
| 277 | } |
| 278 | }() |
| 279 | |
| 280 | //trying to download - do it in background as it may take some time ... |
| 281 | go func() { |
| 282 | req, err2 := http.NewRequest("GET", urlBase.String(), nil) |
| 283 | if err2 != nil { |
| 284 | logger.Errorw(ctx, "could not generate http request", log.Fields{"url": urlBase.String(), "error": err2}) |
| 285 | return |
| 286 | } |
| 287 | ctx, cancel := context.WithDeadline(ctx, time.Now().Add(dm.dlToAdapterTimeout)) //timeout as given from SetDownloadTimeout() |
| 288 | defer cancel() |
| 289 | _ = req.WithContext(ctx) |
| 290 | resp, err3 := http.DefaultClient.Do(req) |
| 291 | if err3 != nil || respExist.StatusCode != http.StatusOK { |
| 292 | logger.Errorw(ctx, "could not http get from url", log.Fields{"url": urlBase.String(), |
| 293 | "error": err3, "status": respExist.StatusCode}) |
| 294 | return |
| 295 | } |
| 296 | defer func() { |
| 297 | deferredErr := resp.Body.Close() |
| 298 | if deferredErr != nil { |
| 299 | logger.Errorw(ctx, "error at closing http get response body", log.Fields{"url": urlBase.String(), "error": deferredErr}) |
| 300 | } |
| 301 | }() |
| 302 | |
| 303 | // Create the file |
| 304 | aLocalPathName := aFilePath + "/" + aFileName |
| 305 | file, err := os.Create(aLocalPathName) |
| 306 | if err != nil { |
| 307 | logger.Errorw(ctx, "could not create local file", log.Fields{"path_file": aLocalPathName, "error": err}) |
| 308 | return |
| 309 | } |
| 310 | defer func() { |
| 311 | deferredErr := file.Close() |
| 312 | if deferredErr != nil { |
| 313 | logger.Errorw(ctx, "error at closing new file", log.Fields{"path_file": aLocalPathName, "error": deferredErr}) |
| 314 | } |
| 315 | }() |
| 316 | |
| 317 | // Write the body to file |
| 318 | _, err = io.Copy(file, resp.Body) |
| 319 | if err != nil { |
| 320 | logger.Errorw(ctx, "could not copy file content", log.Fields{"url": urlBase.String(), "file": aLocalPathName, "error": err}) |
| 321 | return |
| 322 | } |
| 323 | |
| 324 | fileStats, statsErr := file.Stat() |
| 325 | if err != nil { |
| 326 | logger.Errorw(ctx, "created file can't be accessed", log.Fields{"file": aLocalPathName, "stat-error": statsErr}) |
| 327 | } |
| 328 | fileSize := fileStats.Size() |
| 329 | logger.Infow(ctx, "written file size is", log.Fields{"file": aLocalPathName, "length": fileSize}) |
| 330 | |
| 331 | dm.mutexDownloadImageDsc.Lock() |
| 332 | defer dm.mutexDownloadImageDsc.Unlock() |
| 333 | for imgKey, dnldImgDsc := range dm.downloadImageDscSlice { |
| 334 | if dnldImgDsc.downloadImageName == aFileName { |
| 335 | //image found (by name) - need to write changes on the original map |
| 336 | dm.downloadImageDscSlice[imgKey].downloadImageState = cFileStateDlSucceeded |
| 337 | dm.downloadImageDscSlice[imgKey].downloadImageLen = fileSize |
| 338 | //in case upgrade process(es) was/were waiting for the file, inform them |
| 339 | for imageName, channelMap := range dm.dnldImgReadyWaiting { |
| 340 | if imageName == aFileName { |
| 341 | for channel := range channelMap { |
| 342 | // use all found channels to inform possible requesters about the existence of the file |
| 343 | channel <- true |
| 344 | delete(dm.dnldImgReadyWaiting[imageName], channel) //requester served |
| 345 | } |
| 346 | return //can leave directly |
| 347 | } |
| 348 | } |
| 349 | return //can leave directly |
| 350 | } |
| 351 | } |
| 352 | //TODO:!!! further extension could be provided here, e.g. already computing and possibly comparing the CRC, vendor check |
| 353 | }() |
| 354 | return nil |
| 355 | } |
mpagenko | aa3afe9 | 2021-05-21 16:20:58 +0000 | [diff] [blame] | 356 | |
| 357 | func (dm *fileDownloadManager) RequestDownloadState(ctx context.Context, aImageName string, |
| 358 | apDlToAdapterImageState *voltha.ImageState) { |
| 359 | logger.Debugw(ctx, "request download state for image to Adapter", log.Fields{"image-name": aImageName}) |
| 360 | dm.mutexDownloadImageDsc.RLock() |
| 361 | defer dm.mutexDownloadImageDsc.RUnlock() |
| 362 | |
| 363 | for _, dnldImgDsc := range dm.downloadImageDscSlice { |
| 364 | if dnldImgDsc.downloadImageName == aImageName { |
| 365 | //image found (by name) |
| 366 | apDlToAdapterImageState.DownloadState = voltha.ImageState_DOWNLOAD_REQUESTED |
| 367 | apDlToAdapterImageState.Reason = voltha.ImageState_NO_ERROR |
| 368 | return |
| 369 | } |
| 370 | } |
| 371 | //image not found (by name) |
| 372 | apDlToAdapterImageState.DownloadState = voltha.ImageState_DOWNLOAD_UNKNOWN |
| 373 | apDlToAdapterImageState.Reason = voltha.ImageState_NO_ERROR |
| 374 | } |
| 375 | |
| 376 | func (dm *fileDownloadManager) CancelDownload(ctx context.Context, aImageName string) { |
| 377 | logger.Debugw(ctx, "Cancel the download to Adapter", log.Fields{"image-name": aImageName}) |
| 378 | // for the moment that would only support to wait for the download end and remove the image then |
| 379 | // further reactions while still downloading can be considered with some effort, but does it make sense (synchronous load here!) |
| 380 | dm.mutexDownloadImageDsc.RLock() |
| 381 | defer dm.mutexDownloadImageDsc.RUnlock() |
| 382 | |
| 383 | tmpSlice := dm.downloadImageDscSlice[:0] |
| 384 | for _, dnldImgDsc := range dm.downloadImageDscSlice { |
| 385 | if dnldImgDsc.downloadImageName == aImageName { |
| 386 | //image found (by name) - remove the image from filesystem |
| 387 | logger.Debugw(ctx, "removing image", log.Fields{"image-name": aImageName}) |
| 388 | aLocalPathName := cDefaultLocalDir + "/" + aImageName |
| 389 | if err := os.Remove(aLocalPathName); err != nil { |
| 390 | logger.Debugw(ctx, "image not removed from filesystem", log.Fields{ |
| 391 | "image-name": aImageName, "error": err}) |
| 392 | } |
| 393 | // and in the imageDsc slice by just not appending |
| 394 | } else { |
| 395 | tmpSlice = append(tmpSlice, dnldImgDsc) |
| 396 | } |
| 397 | } |
| 398 | dm.downloadImageDscSlice = tmpSlice |
| 399 | //image not found (by name) |
| 400 | } |