mpagenko | c26d4c0 | 2021-05-06 14:27:57 +0000 | [diff] [blame^] | 1 | /* |
| 2 | * Copyright 2020-present Open Networking Foundation |
| 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | |
| 17 | //Package adaptercoreonu provides the utility for onu devices, flows and statistics |
| 18 | package adaptercoreonu |
| 19 | |
| 20 | import ( |
| 21 | "bufio" |
| 22 | "context" |
| 23 | "fmt" |
| 24 | "io" |
| 25 | "net/http" |
| 26 | "net/url" |
| 27 | "os" |
| 28 | "sync" |
| 29 | "time" |
| 30 | |
| 31 | "github.com/opencord/voltha-lib-go/v4/pkg/log" |
| 32 | ) |
| 33 | |
| 34 | const cDefaultLocalDir = "/tmp" //this is the default local dir to download to |
| 35 | |
| 36 | type fileState uint32 |
| 37 | |
| 38 | //nolint:varcheck, deadcode |
| 39 | const ( |
| 40 | cFileStateUnknown fileState = iota |
| 41 | cFileStateDlStarted |
| 42 | cFileStateDlSucceeded |
| 43 | cFileStateDlFailed |
| 44 | cFileStateDlAborted |
| 45 | cFileStateDlInvalid |
| 46 | ) |
| 47 | |
| 48 | type downloadImageParams struct { |
| 49 | downloadImageName string |
| 50 | downloadImageState fileState |
| 51 | downloadImageLen int64 |
| 52 | downloadImageCrc uint32 |
| 53 | } |
| 54 | |
| 55 | type requesterChannelMap map[chan<- bool]struct{} //using an empty structure map for easier (unique) element appending |
| 56 | |
| 57 | //fileDownloadManager structure holds information needed for downloading to and storing images within the adapter |
| 58 | type fileDownloadManager struct { |
| 59 | mutexDownloadImageDsc sync.RWMutex |
| 60 | downloadImageDscSlice []downloadImageParams |
| 61 | dnldImgReadyWaiting map[string]requesterChannelMap |
| 62 | dlToAdapterTimeout time.Duration |
| 63 | } |
| 64 | |
| 65 | //newFileDownloadManager constructor returns a new instance of a fileDownloadManager |
| 66 | //mib_db (as well as not inluded alarm_db not really used in this code? VERIFY!!) |
| 67 | func newFileDownloadManager(ctx context.Context) *fileDownloadManager { |
| 68 | logger.Debug(ctx, "init-fileDownloadManager") |
| 69 | var localDnldMgr fileDownloadManager |
| 70 | localDnldMgr.downloadImageDscSlice = make([]downloadImageParams, 0) |
| 71 | localDnldMgr.dnldImgReadyWaiting = make(map[string]requesterChannelMap) |
| 72 | localDnldMgr.dlToAdapterTimeout = 10 * time.Second //default timeout, should be overwritten immediately after start |
| 73 | return &localDnldMgr |
| 74 | } |
| 75 | |
| 76 | //SetDownloadTimeout configures the timeout used to supervice the download of the image to the adapter (assumed in seconds) |
| 77 | func (dm *fileDownloadManager) SetDownloadTimeout(ctx context.Context, aDlTimeout time.Duration) { |
| 78 | dm.mutexDownloadImageDsc.Lock() |
| 79 | defer dm.mutexDownloadImageDsc.Unlock() |
| 80 | logger.Debugw(ctx, "setting download timeout", log.Fields{"timeout": aDlTimeout}) |
| 81 | dm.dlToAdapterTimeout = aDlTimeout |
| 82 | } |
| 83 | |
| 84 | //GetDownloadTimeout delivers the timeout used to supervice the download of the image to the adapter (assumed in seconds) |
| 85 | func (dm *fileDownloadManager) GetDownloadTimeout(ctx context.Context) time.Duration { |
| 86 | dm.mutexDownloadImageDsc.RLock() |
| 87 | defer dm.mutexDownloadImageDsc.RUnlock() |
| 88 | return dm.dlToAdapterTimeout |
| 89 | } |
| 90 | |
| 91 | //ImageExists returns true if the requested image already exists within the adapter |
| 92 | func (dm *fileDownloadManager) ImageExists(ctx context.Context, aImageName string) bool { |
| 93 | logger.Debugw(ctx, "checking on existence of the image", log.Fields{"image-name": aImageName}) |
| 94 | dm.mutexDownloadImageDsc.RLock() |
| 95 | defer dm.mutexDownloadImageDsc.RUnlock() |
| 96 | |
| 97 | for _, dnldImgDsc := range dm.downloadImageDscSlice { |
| 98 | if dnldImgDsc.downloadImageName == aImageName { |
| 99 | //image found (by name) |
| 100 | return true |
| 101 | } |
| 102 | } |
| 103 | //image not found (by name) |
| 104 | return false |
| 105 | } |
| 106 | |
| 107 | //StartDownload returns true if the download of the requested image could be started for the given file name and URL |
| 108 | func (dm *fileDownloadManager) StartDownload(ctx context.Context, aImageName string, aURLCommand string) error { |
| 109 | logger.Infow(ctx, "image download-to-adapter requested", log.Fields{ |
| 110 | "image-name": aImageName, "url-command": aURLCommand}) |
| 111 | loDownloadImageParams := downloadImageParams{ |
| 112 | downloadImageName: aImageName, downloadImageState: cFileStateDlStarted, |
| 113 | downloadImageLen: 0, downloadImageCrc: 0} |
| 114 | dm.mutexDownloadImageDsc.Lock() |
| 115 | dm.downloadImageDscSlice = append(dm.downloadImageDscSlice, loDownloadImageParams) |
| 116 | dm.mutexDownloadImageDsc.Unlock() |
| 117 | //try to download from http |
| 118 | err := dm.downloadFile(ctx, aURLCommand, cDefaultLocalDir, aImageName) |
| 119 | //return the result of the start-request to comfort the core processing even though the complete download may go on in background |
| 120 | return err |
| 121 | } |
| 122 | |
| 123 | //GetImageBufferLen returns the length of the specified file in bytes (file size) - as detected after download |
| 124 | func (dm *fileDownloadManager) GetImageBufferLen(ctx context.Context, aFileName string) (int64, error) { |
| 125 | dm.mutexDownloadImageDsc.RLock() |
| 126 | defer dm.mutexDownloadImageDsc.RUnlock() |
| 127 | for _, dnldImgDsc := range dm.downloadImageDscSlice { |
| 128 | if dnldImgDsc.downloadImageName == aFileName && dnldImgDsc.downloadImageState == cFileStateDlSucceeded { |
| 129 | //image found (by name) and fully downloaded |
| 130 | return dnldImgDsc.downloadImageLen, nil |
| 131 | } |
| 132 | } |
| 133 | return 0, fmt.Errorf("no downloaded image found: %s", aFileName) |
| 134 | } |
| 135 | |
| 136 | //GetDownloadImageBuffer returns the content of the requested file as byte slice |
| 137 | func (dm *fileDownloadManager) GetDownloadImageBuffer(ctx context.Context, aFileName string) ([]byte, error) { |
| 138 | //nolint:gosec |
| 139 | file, err := os.Open(cDefaultLocalDir + "/" + aFileName) |
| 140 | if err != nil { |
| 141 | return nil, err |
| 142 | } |
| 143 | //nolint:errcheck |
| 144 | defer file.Close() |
| 145 | |
| 146 | stats, statsErr := file.Stat() |
| 147 | if statsErr != nil { |
| 148 | return nil, statsErr |
| 149 | } |
| 150 | |
| 151 | var size int64 = stats.Size() |
| 152 | bytes := make([]byte, size) |
| 153 | |
| 154 | buffer := bufio.NewReader(file) |
| 155 | _, err = buffer.Read(bytes) |
| 156 | |
| 157 | return bytes, err |
| 158 | } |
| 159 | |
| 160 | //RequestDownloadReady receives a channel that has to be used to inform the requester in case the concerned file is downloaded |
| 161 | func (dm *fileDownloadManager) RequestDownloadReady(ctx context.Context, aFileName string, aWaitChannel chan<- bool) { |
| 162 | if dm.imageLocallyDownloaded(ctx, aFileName) { |
| 163 | //image found (by name) and fully downloaded |
| 164 | logger.Debugw(ctx, "file ready - immediate response", log.Fields{"image-name": aFileName}) |
| 165 | aWaitChannel <- true |
| 166 | return |
| 167 | } |
| 168 | //when we are here the image was not yet found or not fully downloaded - |
| 169 | // add the device specific channel to the list of waiting requesters |
| 170 | dm.mutexDownloadImageDsc.Lock() |
| 171 | defer dm.mutexDownloadImageDsc.Unlock() |
| 172 | if loRequesterChannelMap, ok := dm.dnldImgReadyWaiting[aFileName]; ok { |
| 173 | //entry for the file name already exists |
| 174 | if _, exists := loRequesterChannelMap[aWaitChannel]; !exists { |
| 175 | // requester channel does not yet exist for the image |
| 176 | loRequesterChannelMap[aWaitChannel] = struct{}{} |
| 177 | dm.dnldImgReadyWaiting[aFileName] = loRequesterChannelMap |
| 178 | logger.Debugw(ctx, "file not ready - adding new requester", log.Fields{ |
| 179 | "image-name": aFileName, "number-of-requesters": len(dm.dnldImgReadyWaiting[aFileName])}) |
| 180 | } |
| 181 | } else { |
| 182 | //entry for the file name does not even exist |
| 183 | addRequesterChannelMap := make(map[chan<- bool]struct{}) |
| 184 | addRequesterChannelMap[aWaitChannel] = struct{}{} |
| 185 | dm.dnldImgReadyWaiting[aFileName] = addRequesterChannelMap |
| 186 | logger.Debugw(ctx, "file not ready - setting first requester", log.Fields{ |
| 187 | "image-name": aFileName}) |
| 188 | } |
| 189 | } |
| 190 | |
| 191 | //RemoveReadyRequest removes the specified channel from the requester(channel) map for the given file name |
| 192 | func (dm *fileDownloadManager) RemoveReadyRequest(ctx context.Context, aFileName string, aWaitChannel chan bool) { |
| 193 | dm.mutexDownloadImageDsc.Lock() |
| 194 | defer dm.mutexDownloadImageDsc.Unlock() |
| 195 | for imageName, channelMap := range dm.dnldImgReadyWaiting { |
| 196 | if imageName == aFileName { |
| 197 | for channel := range channelMap { |
| 198 | if channel == aWaitChannel { |
| 199 | delete(dm.dnldImgReadyWaiting[imageName], channel) |
| 200 | logger.Debugw(ctx, "channel removed from the requester map", log.Fields{ |
| 201 | "image-name": aFileName, "new number-of-requesters": len(dm.dnldImgReadyWaiting[aFileName])}) |
| 202 | return //can leave directly |
| 203 | } |
| 204 | } |
| 205 | return //can leave directly |
| 206 | } |
| 207 | } |
| 208 | } |
| 209 | |
| 210 | // FileDownloadManager private (unexported) methods -- start |
| 211 | |
| 212 | //imageLocallyDownloaded returns true if the requested image already exists within the adapter |
| 213 | func (dm *fileDownloadManager) imageLocallyDownloaded(ctx context.Context, aImageName string) bool { |
| 214 | logger.Debugw(ctx, "checking if image is fully downloaded to adapter", log.Fields{"image-name": aImageName}) |
| 215 | dm.mutexDownloadImageDsc.RLock() |
| 216 | defer dm.mutexDownloadImageDsc.RUnlock() |
| 217 | |
| 218 | for _, dnldImgDsc := range dm.downloadImageDscSlice { |
| 219 | if dnldImgDsc.downloadImageName == aImageName { |
| 220 | //image found (by name) |
| 221 | if dnldImgDsc.downloadImageState == cFileStateDlSucceeded { |
| 222 | logger.Debugw(ctx, "image has been fully downloaded", log.Fields{"image-name": aImageName}) |
| 223 | return true |
| 224 | } |
| 225 | logger.Debugw(ctx, "image not yet fully downloaded", log.Fields{"image-name": aImageName}) |
| 226 | return false |
| 227 | } |
| 228 | } |
| 229 | //image not found (by name) |
| 230 | logger.Errorw(ctx, "image does not exist", log.Fields{"image-name": aImageName}) |
| 231 | return false |
| 232 | } |
| 233 | |
| 234 | //downloadFile downloads the specified file from the given http location |
| 235 | func (dm *fileDownloadManager) downloadFile(ctx context.Context, aURLCommand string, aFilePath string, aFileName string) error { |
| 236 | // Get the data |
| 237 | logger.Infow(ctx, "downloading with URL", log.Fields{"url": aURLCommand, "localPath": aFilePath}) |
| 238 | // verifying the complete URL by parsing it to its URL elements |
| 239 | urlBase, err1 := url.Parse(aURLCommand) |
| 240 | if err1 != nil { |
| 241 | logger.Errorw(ctx, "could not set base url command", log.Fields{"url": aURLCommand, "error": err1}) |
| 242 | return fmt.Errorf("could not set base url command: %s, error: %s", aURLCommand, err1) |
| 243 | } |
| 244 | urlParams := url.Values{} |
| 245 | urlBase.RawQuery = urlParams.Encode() |
| 246 | |
| 247 | //pre-check on file existence - assuming http location here |
| 248 | reqExist, errExist2 := http.NewRequest("HEAD", urlBase.String(), nil) |
| 249 | if errExist2 != nil { |
| 250 | logger.Errorw(ctx, "could not generate http head request", log.Fields{"url": urlBase.String(), "error": errExist2}) |
| 251 | return fmt.Errorf("could not generate http head request: %s, error: %s", aURLCommand, errExist2) |
| 252 | } |
| 253 | ctxExist, cancelExist := context.WithDeadline(ctx, time.Now().Add(3*time.Second)) //waiting for some fast answer |
| 254 | defer cancelExist() |
| 255 | _ = reqExist.WithContext(ctxExist) |
| 256 | respExist, errExist3 := http.DefaultClient.Do(reqExist) |
| 257 | if errExist3 != nil || respExist.StatusCode != http.StatusOK { |
| 258 | logger.Infow(ctx, "could not http head from url", log.Fields{"url": urlBase.String(), |
| 259 | "error": errExist3, "status": respExist.StatusCode}) |
| 260 | //if head is not supported by server we cannot use this test and just try to continue |
| 261 | if respExist.StatusCode != http.StatusMethodNotAllowed { |
| 262 | logger.Errorw(ctx, "http head from url: file does not exist here, aborting", log.Fields{"url": urlBase.String(), |
| 263 | "error": errExist3, "status": respExist.StatusCode}) |
| 264 | return fmt.Errorf("http head from url: file does not exist here, aborting: %s, error: %s, status: %d", |
| 265 | aURLCommand, errExist2, respExist.StatusCode) |
| 266 | } |
| 267 | } |
| 268 | defer func() { |
| 269 | deferredErr := respExist.Body.Close() |
| 270 | if deferredErr != nil { |
| 271 | logger.Errorw(ctx, "error at closing http head response body", log.Fields{"url": urlBase.String(), "error": deferredErr}) |
| 272 | } |
| 273 | }() |
| 274 | |
| 275 | //trying to download - do it in background as it may take some time ... |
| 276 | go func() { |
| 277 | req, err2 := http.NewRequest("GET", urlBase.String(), nil) |
| 278 | if err2 != nil { |
| 279 | logger.Errorw(ctx, "could not generate http request", log.Fields{"url": urlBase.String(), "error": err2}) |
| 280 | return |
| 281 | } |
| 282 | ctx, cancel := context.WithDeadline(ctx, time.Now().Add(dm.dlToAdapterTimeout)) //timeout as given from SetDownloadTimeout() |
| 283 | defer cancel() |
| 284 | _ = req.WithContext(ctx) |
| 285 | resp, err3 := http.DefaultClient.Do(req) |
| 286 | if err3 != nil || respExist.StatusCode != http.StatusOK { |
| 287 | logger.Errorw(ctx, "could not http get from url", log.Fields{"url": urlBase.String(), |
| 288 | "error": err3, "status": respExist.StatusCode}) |
| 289 | return |
| 290 | } |
| 291 | defer func() { |
| 292 | deferredErr := resp.Body.Close() |
| 293 | if deferredErr != nil { |
| 294 | logger.Errorw(ctx, "error at closing http get response body", log.Fields{"url": urlBase.String(), "error": deferredErr}) |
| 295 | } |
| 296 | }() |
| 297 | |
| 298 | // Create the file |
| 299 | aLocalPathName := aFilePath + "/" + aFileName |
| 300 | file, err := os.Create(aLocalPathName) |
| 301 | if err != nil { |
| 302 | logger.Errorw(ctx, "could not create local file", log.Fields{"path_file": aLocalPathName, "error": err}) |
| 303 | return |
| 304 | } |
| 305 | defer func() { |
| 306 | deferredErr := file.Close() |
| 307 | if deferredErr != nil { |
| 308 | logger.Errorw(ctx, "error at closing new file", log.Fields{"path_file": aLocalPathName, "error": deferredErr}) |
| 309 | } |
| 310 | }() |
| 311 | |
| 312 | // Write the body to file |
| 313 | _, err = io.Copy(file, resp.Body) |
| 314 | if err != nil { |
| 315 | logger.Errorw(ctx, "could not copy file content", log.Fields{"url": urlBase.String(), "file": aLocalPathName, "error": err}) |
| 316 | return |
| 317 | } |
| 318 | |
| 319 | fileStats, statsErr := file.Stat() |
| 320 | if err != nil { |
| 321 | logger.Errorw(ctx, "created file can't be accessed", log.Fields{"file": aLocalPathName, "stat-error": statsErr}) |
| 322 | } |
| 323 | fileSize := fileStats.Size() |
| 324 | logger.Infow(ctx, "written file size is", log.Fields{"file": aLocalPathName, "length": fileSize}) |
| 325 | |
| 326 | dm.mutexDownloadImageDsc.Lock() |
| 327 | defer dm.mutexDownloadImageDsc.Unlock() |
| 328 | for imgKey, dnldImgDsc := range dm.downloadImageDscSlice { |
| 329 | if dnldImgDsc.downloadImageName == aFileName { |
| 330 | //image found (by name) - need to write changes on the original map |
| 331 | dm.downloadImageDscSlice[imgKey].downloadImageState = cFileStateDlSucceeded |
| 332 | dm.downloadImageDscSlice[imgKey].downloadImageLen = fileSize |
| 333 | //in case upgrade process(es) was/were waiting for the file, inform them |
| 334 | for imageName, channelMap := range dm.dnldImgReadyWaiting { |
| 335 | if imageName == aFileName { |
| 336 | for channel := range channelMap { |
| 337 | // use all found channels to inform possible requesters about the existence of the file |
| 338 | channel <- true |
| 339 | delete(dm.dnldImgReadyWaiting[imageName], channel) //requester served |
| 340 | } |
| 341 | return //can leave directly |
| 342 | } |
| 343 | } |
| 344 | return //can leave directly |
| 345 | } |
| 346 | } |
| 347 | //TODO:!!! further extension could be provided here, e.g. already computing and possibly comparing the CRC, vendor check |
| 348 | }() |
| 349 | return nil |
| 350 | } |