VOL-1397: Adtran-OLT - Initial containerization commit
- Need to move VERSION to base directory
Change-Id: I9d62d0607a011ce642e379fd92b35ec48b300070
diff --git a/adapters/adtran_common/download.py b/adapters/adtran_common/download.py
new file mode 100644
index 0000000..8207a99
--- /dev/null
+++ b/adapters/adtran_common/download.py
@@ -0,0 +1,523 @@
+# Copyright 2017-present Adtran, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import structlog
+import xmltodict
+from twisted.internet import reactor
+from twisted.internet.defer import returnValue, inlineCallbacks
+from pyvoltha.protos.device_pb2 import ImageDownload
+from pyvoltha.protos.common_pb2 import AdminState
+
+log = structlog.get_logger()
+
+# TODO: Following two would be good provisionable parameters
+DEFAULT_AUTO_AGE_MINUTES = 10
+DEFAULT_MAX_JOB_RUN_SECONDS = 3600 * 4 # Some OLT files are 250MB+
+
+
+class Download(object):
+ """Class to wrap an image download"""
+
+ def __init__(self, handler, request, protocols):
+ self._handler = handler
+ self._deferred = None
+ self.device_id = request.id
+ self._name = request.name
+ self._url = request.url
+ self._crc = request.crc
+ self._version = request.image_version
+ self._local = request.local_dir
+ self._save_config = request.save_config
+ self._supported_protocols = protocols
+
+ self._download_state = ImageDownload.DOWNLOAD_UNKNOWN
+ self._failure_reason = ImageDownload.UNKNOWN_ERROR
+ self._image_state = ImageDownload.IMAGE_UNKNOWN
+ self._additional_info = ''
+ self._downloaded_octets = 0
+
+ # Server profile info
+ self._server_profile_name = None
+ self._scheme = None
+ self._host = ''
+ self._port = None
+ self._path = ''
+ self._auth = None
+
+ # Download job info
+ self._download_job_name = None
+
+ self._age_out_period = DEFAULT_AUTO_AGE_MINUTES
+ self._max_execution = DEFAULT_MAX_JOB_RUN_SECONDS
+
+ def __str__(self):
+ return "ImageDownload: {}".format(self.name)
+
+ @staticmethod
+ def create(handler, request, supported_protocols):
+ """
+ Create and start a new image download
+
+ :param handler: (AdtranDeviceHandler) Device download is for
+ :param request: (ImageDownload) Request
+ :param supported_protocols: (list) download methods allowed (http, tftp, ...)
+ """
+ download = Download(handler, request, supported_protocols)
+ download._deferred = reactor.callLater(0, download.start_download)
+
+ return download
+
+ @property
+ def name(self):
+ return self._name
+
+ @property
+ def download_state(self):
+ return self._download_state
+
+ @property
+ def failure_reason(self):
+ return self._failure_reason
+
+ @property
+ def image_state(self):
+ return self._image_state
+
+ @property
+ def additional_info(self):
+ return self._additional_info
+
+ @property
+ def downloaded_bytes(self):
+ return self._downloaded_octets
+
+ @property
+ def profile_name(self):
+ return self._server_profile_name
+
+ def _cancel_deferred(self):
+ d, self._deferred = self._deferred, None
+ try:
+ if d is not None and not d.called:
+ d.cancel()
+ except Exception as e:
+ pass
+
+ @inlineCallbacks
+ def start_download(self):
+ import uuid
+ log.info('download-start', name=self.name)
+ if not self.parse_url():
+ self._download_failed()
+ returnValue('failed url parsing')
+
+ self._download_state = ImageDownload.DOWNLOAD_STARTED
+ self._failure_reason = ImageDownload.NO_ERROR
+
+ ##############################################################
+ # Configure the file server profile
+ try:
+ self._additional_info = 'Configuring Download Server profile'
+ self._server_profile_name = 'VOLTHA.download.{}'.format(uuid.uuid4())
+ profile = self.server_profile_xml
+ yield self._handler.netconf_client.edit_config(profile)
+
+ except Exception as e:
+ log.exception('server-profile', e=e)
+ self._server_profile_name = None
+ self._failure_reason = ImageDownload.UNKNOWN_ERROR
+ self._additional_info += ': Failure: {}'.format(e.message)
+ self._download_failed()
+ raise
+
+ ##############################################################
+ # Configure the software download maintenance job
+ try:
+ self._additional_info = 'Configuring Image Download Job'
+ self._download_job_name = 'VOLTHA.download.{}'.format(uuid.uuid4())
+ job = self.download_job_xml
+ yield self._handler.netconf_client.edit_config(job)
+
+ except Exception as e:
+ log.exception('server-profile', e=e)
+ self._download_job_name = None
+ self._failure_reason = ImageDownload.UNKNOWN_ERROR
+ self._additional_info += ': Failure: {}'.format(e.message)
+ self._download_failed()
+ raise
+
+ ##############################################################
+ # Schedule a task to monitor the download
+ try:
+ self._additional_info = 'Monitoring download status'
+ self._deferred = reactor.callLater(0.5, self.monitor_download_status)
+
+ except Exception as e:
+ log.exception('server-profile', e=e)
+ self._failure_reason = ImageDownload.UNKNOWN_ERROR
+ self._additional_info += ': Failure: {}'.format(e.message)
+ self._download_failed()
+ raise
+
+ returnValue('started')
+
+ def parse_url(self):
+ from urllib3 import util, exceptions
+ try:
+ results = util.parse_url(self._url)
+
+ # Server info
+ self._scheme = results.scheme.lower()
+ if self._scheme not in self._supported_protocols:
+ self._failure_reason = ImageDownload.INVALID_URL
+ self._additional_info = "Unsupported file transfer protocol: {}".format(results.scheme)
+ return False
+
+ self._host = results.host
+ self._port = results.port
+ self._path = results.path
+ self._auth = results.auth
+ return True
+
+ except exceptions.LocationValueError as e:
+ self._failure_reason = ImageDownload.INVALID_URL
+ self._additional_info = e.message
+ return False
+
+ except Exception as e:
+ self._failure_reason = ImageDownload.UNKNOWN_ERROR
+ self._additional_info = e.message
+ return False
+
+ @property
+ def server_profile_xml(self):
+ assert self._scheme in ['http', 'https', 'ftp', 'sftp', 'tftp'], 'Invalid protocol'
+
+ xml = """
+ <file-servers xmlns="http://www.adtran.com/ns/yang/adtran-file-servers">
+ <profiles>
+ <profile>"""
+
+ xml += '<name>{}</name>'.format(self._server_profile_name)
+ xml += '<connection-profile>'
+ xml += ' <host>{}</host>'.format(self._host)
+ xml += ' <port>{}</port>'.format(self._port) if self._port is not None else '<use-standard-port/>'
+
+ if self._scheme in ['http', 'https']:
+ xml += ' <protocol '
+ xml += 'xmlns:adtn-file-srv-https="http://www.adtran.com/ns/yang/adtran-file-servers-https">' +\
+ 'adtn-file-srv-https:{}'.format(self._scheme)
+ xml += ' </protocol>'
+
+ elif self._scheme == 'sftp':
+ xml += ' <protocol '
+ xml += 'xmlns:adtn-file-srv-sftp="http://www.adtran.com/ns/yang/adtran-file-servers-sftp">' +\
+ 'adtn-file-srv-sftp:sftp'
+ xml += ' </protocol>'
+
+ elif self._scheme in ['ftp', 'tftp']:
+ xml += '<protocol>adtn-file-srv:{}</protocol>'.format(self._scheme)
+
+ if self._auth is not None:
+ user_pass = self._auth.split(':')
+ xml += '<username>{}</username>'.format(user_pass[0])
+ xml += '<password>$0${}</password>'.format("".join(user_pass[1:]))
+ # And the trailer
+ xml += """
+ </connection-profile>
+ </profile>
+ </profiles>
+ </file-servers>
+ """
+ return xml
+
+ @property
+ def download_job_xml(self):
+ # TODO: May want to support notifications
+ # TODO: Not sure about this name for the entity
+ entity = 'main 0'
+ xml = """
+ <maintenance-jobs xmlns="http://www.adtran.com/ns/yang/adtran-maintenance-jobs" xmlns:adtn-phys-sw-mnt="http://www.adtran.com/ns/yang/adtran-physical-software-maintenance">
+ <maintenance-job>
+ <name>{}</name>
+ <enabled>true</enabled>
+ <notify-enabled>false</notify-enabled>
+ <maximum-execution-time>{}</maximum-execution-time>
+ <run-once>true</run-once>
+ <adtn-phys-sw-mnt:download-software>
+ <adtn-phys-sw-mnt:physical-entity>{}</adtn-phys-sw-mnt:physical-entity>
+ <adtn-phys-sw-mnt:software-name>software</adtn-phys-sw-mnt:software-name>
+ <adtn-phys-sw-mnt:remote-file>
+ <adtn-phys-sw-mnt:file-server-profile>{}</adtn-phys-sw-mnt:file-server-profile>
+ <adtn-phys-sw-mnt:filename>{}</adtn-phys-sw-mnt:filename>
+ """.format(self._download_job_name, self._max_execution, entity,
+ self._server_profile_name, self._name)
+
+ if self._path is not None:
+ xml += """
+ <adtn-phys-sw-mnt:filepath>{}</adtn-phys-sw-mnt:filepath>
+ """.format(self._path)
+
+ xml += """
+ </adtn-phys-sw-mnt:remote-file>
+ </adtn-phys-sw-mnt:download-software>
+ </maintenance-job>
+ </maintenance-jobs>
+ """
+ return xml
+
+ @property
+ def download_status_xml(self):
+ xml = """
+ <filter>
+ <maintenance-jobs-state xmlns="http://www.adtran.com/ns/yang/adtran-maintenance-jobs">
+ <maintenance-job>
+ <name>{}</name>
+ </maintenance-job>
+ </maintenance-jobs-state>
+ </filter>
+ """.format(self._download_job_name)
+ return xml
+
+ @property
+ def delete_server_profile_xml(self):
+ xml = """
+ <file-servers xmlns="http://www.adtran.com/ns/yang/adtran-file-servers">
+ <profiles operation="delete">
+ <profile>
+ <name>{}</name>
+ </profile>
+ </profiles>
+ </file-servers>
+ """.format(self._server_profile_name)
+ return xml
+
+ @property
+ def delete_download_job_xml(self):
+ xml = """
+ <maintenance-jobs xmlns="http://www.adtran.com/ns/yang/adtran-maintenance-jobs">
+ <maintenance-job operation="delete">>
+ <name>{}</name>
+ </maintenance-job>
+ </maintenance-jobs>
+ """.format(self._download_job_name)
+ return xml
+
+ @inlineCallbacks
+ def monitor_download_status(self):
+ log.debug('monitor-download', name=self.name)
+ try:
+ results = yield self._handler.netconf_client.get(self.download_status_xml)
+
+ result_dict = xmltodict.parse(results.data_xml)
+ entries = result_dict['data']['maintenance-jobs-state']['maintenance-job']
+
+ name = entries.get('name')
+ assert name == self._download_job_name, 'The job status name does not match. {} != {}'.format(name, self.name)
+ self._download_state = self.monitor_state_to_download_state(entries['state']['#text'])
+
+ completed = entries['timestamps'].get('completed-timestamp')
+ started = entries['timestamps'].get('start-timestamp')
+
+ if self._download_state == ImageDownload.DOWNLOAD_FAILED:
+ self._failure_reason = ImageDownload.UNKNOWN_ERROR
+ self._additional_info = entries['error'].get('error-message')
+
+ elif self._download_state == ImageDownload.INSUFFICIENT_SPACE:
+ self._failure_reason = ImageDownload.INSUFFICIENT_SPACE
+ self._additional_info = entries['error'].get('error-message')
+
+ elif self._download_state == ImageDownload.DOWNLOAD_STARTED:
+ self._failure_reason = ImageDownload.NO_ERROR
+ self._additional_info = 'Download started at {}'.format(started)
+
+ elif self._download_state == ImageDownload.DOWNLOAD_SUCCEEDED:
+ self._failure_reason = ImageDownload.NO_ERROR
+ self._additional_info = 'Download completed at {}'.format(completed)
+ else:
+ raise NotImplemented('Unsupported state')
+
+ done = self._download_state in [ImageDownload.DOWNLOAD_FAILED,
+ ImageDownload.DOWNLOAD_SUCCEEDED,
+ ImageDownload.INSUFFICIENT_SPACE]
+
+ except Exception as e:
+ log.exception('protocols', e=e)
+ done = False
+
+ if not done:
+ self._deferred = reactor.callLater(1, self.monitor_download_status)
+
+ returnValue('done' if done else 'not-done-yet')
+
+ def _download_failed(self):
+ log.info('download-failed', name=self.name)
+
+ self._cancel_deferred()
+ self._download_state = ImageDownload.DOWNLOAD_FAILED
+
+ # Cleanup NETCONF
+ reactor.callLater(0, self._cleanup_download_job, 20)
+ reactor.callLater(0, self._cleanup_server_profile, 20)
+ # TODO: Do we signal any completion due to failure?
+
+ def _download_complete(self):
+ log.info('download-completed', name=self.name)
+
+ self._cancel_deferred()
+ self._download_state = ImageDownload.DOWNLOAD_SUCCEEDED
+ self._downloaded_octets = 123456
+ self._failure_reason = ImageDownload.NO_ERROR
+
+ reactor.callLater(0, self._cleanup_download_job, 20)
+ reactor.callLater(0, self._cleanup_server_profile, 20)
+ # TODO: How do we signal completion?
+
+ device = self._handler.adapter_agent.get_device(self.device_id)
+ if device is not None:
+ # restore admin state to enabled
+ device.admin_state = AdminState.ENABLED
+ self._handler.adapter_agent.update_device(device)
+
+ def cancel_download(self, request):
+ log.info('cancel-sw-download', name=self.name)
+
+ self._cancel_deferred()
+
+ try:
+ # initiate cancelling software download to device at success
+ # delete image download record
+
+ self._handler.adapter_agent.delete_image_download(request)
+
+ device = self._handler.adapter_agent.get_device(self.device_id)
+ if device is not None:
+ # restore admin state to enabled
+ device.admin_state = AdminState.ENABLED
+ self._handler.adapter_agent.update_device(device)
+
+ except Exception as e:
+ log.exception(e.message)
+
+ reactor.callLater(0, self._cleanup_download_job, 20)
+ reactor.callLater(0, self._cleanup_server_profile, 20)
+
+ @inlineCallbacks
+ def _cleanup_server_profile(self, retries, attempt=1):
+ log.info('cleanup-server', name=self.name,
+ profile=self._server_profile_name,
+ attempt=attempt, remaining=retries)
+
+ if self._server_profile_name is not None:
+ try:
+ profile = self.delete_server_profile_xml
+ yield self._handler.netconf_client.edit_config(profile)
+ self._server_profile_name = None
+
+ except Exception as e:
+ log.exception(e.message)
+ if retries > 0:
+ reactor.callLater(attempt * 60, self._cleanup_download_job,
+ retries - 1, attempt + 1)
+
+ @inlineCallbacks
+ def _cleanup_download_job(self, retries, attempt=1):
+ log.info('cleanup-download', name=self.name,
+ profile=self._download_job_name,
+ attempt=attempt, remaining=retries)
+
+ if self._download_job_name is not None:
+ try:
+ job = self.delete_download_job_xml
+ yield self._handler.netconf_client.edit_config(job)
+ self._download_job_name = None
+
+ except Exception as e:
+ log.exception(e.message)
+ if retries > 0:
+ reactor.callLater(attempt * 60, self._cleanup_download_job,
+ retries - 1, attempt + 1)
+
+ @inlineCallbacks
+ def activate_image(self):
+ log.info('download-activate', name=self.name)
+
+ if self._download_state == ImageDownload.DOWNLOAD_SUCCEEDED:
+ pass # TODO: Implement
+ self._image_state = ImageDownload.IMAGE_ACTIVE
+
+ returnValue('TODO: Implement this')
+
+ @inlineCallbacks
+ def revert_image(self):
+ log.info('download-revert', name=self.name)
+
+ if self._download_state == ImageDownload.DOWNLOAD_SUCCEEDED:
+ pass # TODO: Implement
+ self._image_state = ImageDownload.IMAGE_INACTIVE
+
+ returnValue('TODO: Implement this')
+
+ def monitor_state_to_download_state(self, state):
+ if ':' in state:
+ state = state.split(':')[-1]
+ result = {
+ 'downloading-software': ImageDownload.DOWNLOAD_STARTED, # currently downloading software
+ 'storing-software': ImageDownload.DOWNLOAD_STARTED, # successfully downloaded the required software and is storing it to memory
+ 'software-stored': ImageDownload.DOWNLOAD_SUCCEEDED, # successfully downloaded the required software and has stored it successfully to memory
+ 'software-download-failed': ImageDownload.DOWNLOAD_FAILED, # unsuccessfully attemptedto download the required software
+ 'invalid-software': ImageDownload.DOWNLOAD_FAILED, # successfully downloaded the required software but the software was determined to be invalid
+ 'software-storage-failed': ImageDownload.INSUFFICIENT_SPACE, # successfully downloaded the required software but was unable to successfully stored it to memory
+ }.get(state.lower(), None)
+ log.info('download-software-state', result=result, state=state, name=self.name)
+ assert result is not None, 'Invalid state'
+ return result
+
+ def monitor_state_to_activate_state(self, state):
+ if ':' in state:
+ state = state.split(':')[-1]
+ result = {
+ 'enabling-software': ImageDownload.IMAGE_ACTIVATE, # currently enabling the software
+ 'software-enabled': ImageDownload.IMAGE_ACTIVE, # successfully enabled the required software
+ 'enable-software-failed': ImageDownload.IMAGE_INACTIVE, # unsuccessfully attempted to enable the required software revision
+ 'activating-software': ImageDownload.IMAGE_ACTIVATE, # currently activating the software
+ 'software-activated': ImageDownload.IMAGE_ACTIVE, # successfully activated the required software. The job terminated successfully
+ 'activate-software-failed': ImageDownload.IMAGE_INACTIVE, # unsuccessfully attempted to activate the required software revision
+ 'committing-software': ImageDownload.IMAGE_ACTIVATE, # currently committing the software
+ 'software-committed': ImageDownload.IMAGE_ACTIVATE, # successfully committed the required software. The job terminated successfully
+ 'commit-software-failed': ImageDownload.IMAGE_INACTIVE, # unsuccessfully attempted to commit the required software revision
+ }.get(state.lower(), None)
+ log.info('download-activate-state', result=result, state=state, name=self.name)
+ assert result is not None, 'Invalid state'
+ return result
+
+ @staticmethod
+ def clear_all(client):
+ """
+ Remove all file server profiles and download jobs
+ :param client: (ncclient) NETCONF Client to use
+ """
+ from twisted.internet import defer
+ del_fs_xml = """
+ <file-servers xmlns="http://www.adtran.com/ns/yang/adtran-file-servers">
+ <profiles operation="delete"/>
+ </file-servers>
+ """
+ del_job_xml = """
+ <maintenance-jobs operation="delete" xmlns="http://www.adtran.com/ns/yang/adtran-maintenance-jobs"/>
+ """
+ dl = [client.edit_config(del_fs_xml, ignore_delete_error=True),
+ client.edit_config(del_job_xml, ignore_delete_error=True)]
+
+ return defer.gatherResults(dl, consumeErrors=True)