[CORD-2675] Polling profiles
Change-Id: Ic47e88341a74384cb57d959bf2d01e1a8addbd8c
diff --git a/xos/synchronizer/Dockerfile.synchronizer b/xos/synchronizer/Dockerfile.synchronizer
index b0c967c..cd4da41 100644
--- a/xos/synchronizer/Dockerfile.synchronizer
+++ b/xos/synchronizer/Dockerfile.synchronizer
@@ -55,5 +55,5 @@
org.opencord.component.xos.vcs-url=$org_opencord_component_xos_vcs_url \
org.opencord.component.xos.vcs-ref=$org_opencord_component_xos_vcs_ref
-CMD bash -c "service filebeat start; cd /opt/xos/synchronizers/progran; ./run.sh"
+CMD bash -c "service filebeat start; cd /opt/xos/synchronizers/progran; python progran-synchronizer.py"
diff --git a/xos/synchronizer/run.sh b/xos/synchronizer/run.sh
deleted file mode 100755
index 63c8460..0000000
--- a/xos/synchronizer/run.sh
+++ /dev/null
@@ -1,17 +0,0 @@
-
-# Copyright 2017-present Open Networking Foundation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-python progran-synchronizer.py
diff --git a/xos/synchronizer/steps/.gitignore b/xos/synchronizer/steps/.gitignore
new file mode 100644
index 0000000..c649235
--- /dev/null
+++ b/xos/synchronizer/steps/.gitignore
@@ -0,0 +1 @@
+synchronizers
\ No newline at end of file
diff --git a/xos/synchronizer/steps/helpers.py b/xos/synchronizer/steps/helpers.py
index d52422b..8f07a66 100644
--- a/xos/synchronizer/steps/helpers.py
+++ b/xos/synchronizer/steps/helpers.py
@@ -15,6 +15,8 @@
from xosconfig import Config
from multistructlog import create_logger
from synchronizers.new_base.modelaccessor import ProgranService
+import time
+import datetime
log = create_logger(Config().get('logging'))
@@ -30,7 +32,7 @@
try:
progran_service = ProgranService.objects.all()[0]
except IndexError:
- log.error("Cannot find Progran Service, does it exists?")
+ raise Exception("Cannot find Progran Service, does it exists?")
return ProgranHelpers.get_onos_info_from_service(progran_service)
@staticmethod
@@ -41,3 +43,43 @@
'username': progran_service.onos_username,
'password': progran_service.onos_password,
}
+
+ @staticmethod
+ def get_progran_rest_errors(res):
+ res = res.json()
+ if res['Result'] == -2:
+ raise Exception(res['ErrCode'])
+
+ @staticmethod
+ def date_to_time(d):
+ if len(d) == 0:
+ return 0
+ return time.mktime(datetime.datetime.strptime(d, "%d.%m.%Y %H:%S").timetuple())
+
+ @staticmethod
+ def update_fields(model, dict, mapping={}, transformations={}):
+ dict = ProgranHelpers.convert_keys(dict, mapping, transformations)
+ for k, v in dict.iteritems():
+ if hasattr(model, k):
+ setattr(model, k, v)
+ else:
+ log.warn("%s does not have a '%s' property, not updating it" % (model.model_name, k))
+ return model
+
+ @staticmethod
+ def convert_keys(dict, mapping={}, transformations={}):
+ for k, v in dict.iteritems():
+ if k in mapping:
+ # apply custom transformations to the data
+ if k in transformations:
+ dict[k] = transformations[k](v)
+
+ # NOTE we may have different names that the field in the dict
+ dict[mapping[k]] = dict[k]
+ del dict[k]
+ return dict
+
+ @staticmethod
+ def list_diff(first, second):
+ second = set(second)
+ return [item for item in first if item not in second]
\ No newline at end of file
diff --git a/xos/synchronizer/steps/sync_imsi.py b/xos/synchronizer/steps/sync_imsi.py
index c76dafb..93af41f 100644
--- a/xos/synchronizer/steps/sync_imsi.py
+++ b/xos/synchronizer/steps/sync_imsi.py
@@ -78,6 +78,7 @@
self.run_playbook(o, create_fields)
# fetch the IMSI we just created
+ # NOTE we won't need this method once we'll have polling in place
imsi_url = "http://%s:%s/onos/progran/imsi/%s" % (base_fields['onos_url'], base_fields['onos_port'], o.imsi_number)
r = requests.get(imsi_url)
o.ue_status = r.json()['ImsiArray'][0]['UeStatus']
diff --git a/xos/synchronizer/steps/sync_progranserviceinstance.py b/xos/synchronizer/steps/sync_progranserviceinstance.py
index 14925bc..11d679d 100644
--- a/xos/synchronizer/steps/sync_progranserviceinstance.py
+++ b/xos/synchronizer/steps/sync_progranserviceinstance.py
@@ -17,14 +17,15 @@
import os
import sys
from synchronizers.new_base.SyncInstanceUsingAnsible import SyncStep
-from synchronizers.new_base.ansible_helper import run_template
-from synchronizers.new_base.modelaccessor import ProgranServiceInstance, ENodeB
+from synchronizers.new_base.modelaccessor import ProgranServiceInstance, ENodeB, Handover
from xosconfig import Config
from multistructlog import create_logger
import json
import requests
from requests.auth import HTTPBasicAuth
+import time
+import datetime
log = create_logger(Config().get('logging'))
@@ -39,6 +40,9 @@
observes = ProgranServiceInstance
+ # Poll every 5 loops of self.call
+ poll = 0
+
def sync_record(self, o):
onos = ProgranHelpers.get_progran_onos_info()
@@ -48,6 +52,8 @@
data = self.get_progran_profile_field(o)
r = requests.post(profile_url, data=json.dumps(data), auth=HTTPBasicAuth(onos['username'], onos['password']))
+
+ ProgranHelpers.get_progran_rest_errors(r)
log.info("Profile synchronized", response=r.json())
log.info("sync'ing enodeb", object=str(o), **o.tologdict())
@@ -60,6 +66,7 @@
]
}
r = requests.post(enodeb_url, data=json.dumps(data), auth=HTTPBasicAuth(onos['username'], onos['password']))
+ ProgranHelpers.get_progran_rest_errors(r)
o.active_enodeb_id = o.enodeb_id # storing the value to know when it will be deleted
log.info("EnodeB synchronized", response=r.json())
elif o.active_enodeb_id:
@@ -67,12 +74,12 @@
log.info("removing profile %s from enodeb %s" % (o.name, o.active_enodeb_id), object=str(o), **o.tologdict())
enodeb_url = "http://%s:%s/onos/progran/enodeb/%s/profile/%s" % (onos['url'], onos['port'], enb_id, o.name)
r = requests.delete(enodeb_url, auth=HTTPBasicAuth(onos['username'], onos['password']))
+ ProgranHelpers.get_progran_rest_errors(r)
o.active_enodeb_id = 0 # removing the value because it has been deleted
log.info("EnodeB synchronized", response=r.json())
o.save()
-
def get_handover_for_profile(self, o):
return {
"A3Hysteresis": o.handover.HysteresisA3,
@@ -114,4 +121,106 @@
profile_url = "http://%s:%s/onos/progran/profile/%s" % (onos['url'], onos['port'], o.name)
r = requests.delete(profile_url, auth=HTTPBasicAuth(onos['username'], onos['password']))
o.active_enodeb_id = 0 # removing the value because it has been deleted
- log.info("Profile synchronized", response=r.json())
\ No newline at end of file
+ log.info("Profile synchronized", response=r.json())
+
+ def fetch_pending(self, deleted):
+ # self.read_profiles_from_progran()
+ return super(SyncProgranServiceInstance, self).fetch_pending(deleted)
+
+ @staticmethod
+ def date_to_time(d):
+ if len(d) == 0:
+ return 0
+ return time.mktime(datetime.datetime.strptime(d, "%d.%m.%Y %H:%S").timetuple())
+
+ @staticmethod
+ def update_fields(model, dict, mapping={}, transformations={}):
+ dict = SyncProgranServiceInstance.convert_keys(dict, mapping, transformations)
+ for k, v in dict.iteritems():
+ if hasattr(model, k):
+ setattr(model, k, v)
+ else:
+ log.warn("%s does not have a '%s' property, not updating it" % (model.model_name, k))
+ return model
+
+ @staticmethod
+ def convert_keys(dict, mapping={}, transformations={}):
+ for k, v in dict.iteritems():
+ if k in mapping:
+ # apply custom transformations to the data
+ if k in transformations:
+ dict[k] = transformations[k](v)
+
+ # NOTE we may have different names that the field in the dict
+ dict[mapping[k]] = dict[k]
+ del dict[k]
+ return dict
+
+
+ def my_call(self, failed=[], deletion=False):
+ """
+ Read profile from progran and save them in xos
+ """
+ if self.poll < 5:
+ self.poll = self.poll + 1
+ else:
+ self.poll = 0
+ onos = ProgranHelpers.get_progran_onos_info()
+ profile_url = "http://%s:%s/onos/progran/profile/" % (onos['url'], onos['port'])
+ r = requests.get(profile_url, auth=HTTPBasicAuth(onos['username'], onos['password']))
+ res = r.json()['ProfileArray']
+
+ # remove default profiles
+ res = [p for p in res if "Default" not in p['Name']]
+
+ field_mapping = {
+ 'Name': 'name',
+ 'Start': 'start',
+ 'End': 'end'
+ }
+
+ field_transformations = {
+ 'Start': SyncProgranServiceInstance.date_to_time,
+ 'End': SyncProgranServiceInstance.date_to_time
+ }
+
+ handover_mapping = {
+ 'A5Hysteresis': 'HysteresisA5',
+ 'A3Hysteresis': 'HysteresisA3'
+ }
+
+ for p in res:
+
+ # checking for handovers
+ handover_dict = p['Handover']
+ handover_dict = SyncProgranServiceInstance.convert_keys(handover_dict, handover_mapping)
+ del p['Handover']
+
+ try:
+ handover = Handover.objects.get(**handover_dict)
+ log.info("handover already exists, updating it", handover=handover_dict)
+ except IndexError:
+ handover = Handover()
+ handover = SyncProgranServiceInstance.update_fields(handover, handover_dict)
+ log.info("handover is new, creating it", handover=handover_dict)
+
+ handover.save()
+
+ # checking for profiles
+ try:
+ si = ProgranServiceInstance.objects.get(name=p['Name'])
+ log.info("Profile %s already exists, updating it" % p['Name'])
+ except IndexError:
+ si = ProgranServiceInstance()
+ si.name = p['Name']
+ log.info("Profile %s is new, creating it" % p['Name'])
+
+ si = SyncProgranServiceInstance.update_fields(si, p, field_mapping, field_transformations)
+ si.handover = handover
+
+
+
+ # TODO keep track of the deleted profiles
+ # existing profiles - updated profiles = deleted profiles
+
+ si.save()
diff --git a/xos/synchronizer/steps/sync_progranserviceinstance_back.py b/xos/synchronizer/steps/sync_progranserviceinstance_back.py
new file mode 100644
index 0000000..ae97c95
--- /dev/null
+++ b/xos/synchronizer/steps/sync_progranserviceinstance_back.py
@@ -0,0 +1,126 @@
+
+# Copyright 2017-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+import sys
+
+import datetime
+import time
+
+from synchronizers.new_base.SyncInstanceUsingAnsible import SyncStep
+from synchronizers.new_base.modelaccessor import ProgranServiceInstance, ENodeB, Handover
+
+from xosconfig import Config
+from multistructlog import create_logger
+import json
+import requests
+from requests.auth import HTTPBasicAuth
+
+
+
+log = create_logger(Config().get('logging'))
+
+parentdir = os.path.join(os.path.dirname(__file__), "..")
+sys.path.insert(0, parentdir)
+sys.path.insert(0, os.path.dirname(__file__))
+from helpers import ProgranHelpers
+
+class SyncProgranServiceInstanceBack(SyncStep):
+ provides = [ProgranServiceInstance]
+
+ observes = ProgranServiceInstance
+
+
+ def call(self, failed=[], deletion=False):
+ """
+ Read profile from progran and save them in xos
+ """
+
+ if deletion == False:
+ # NOTE we won't it to run only after the delete has completed
+ return
+
+ log.info("Reading profiles from progran")
+ onos = ProgranHelpers.get_progran_onos_info()
+ profile_url = "http://%s:%s/onos/progran/profile/" % (onos['url'], onos['port'])
+ r = requests.get(profile_url, auth=HTTPBasicAuth(onos['username'], onos['password']))
+ res = r.json()['ProfileArray']
+
+ # remove default profiles
+ res = [p for p in res if "Default" not in p['Name']]
+
+ field_mapping = {
+ 'Name': 'name',
+ 'Start': 'start',
+ 'End': 'end'
+ }
+
+ field_transformations = {
+ 'Start': ProgranHelpers.date_to_time,
+ 'End': ProgranHelpers.date_to_time
+ }
+
+ handover_mapping = {
+ 'A5Hysteresis': 'HysteresisA5',
+ 'A3Hysteresis': 'HysteresisA3'
+ }
+
+ updated_profiles = []
+
+ for p in res:
+
+ # checking for handovers
+ handover_dict = p['Handover']
+ handover_dict = ProgranHelpers.convert_keys(handover_dict, handover_mapping)
+ del p['Handover']
+
+ try:
+ handover = Handover.objects.get(**handover_dict)
+ log.info("handover already exists, updating it", handover=handover_dict)
+ except IndexError:
+ handover = Handover()
+ handover = ProgranHelpers.update_fields(handover, handover_dict)
+ log.info("handover is new, creating it", handover=handover_dict)
+
+ handover.save()
+
+ # checking for profiles
+ try:
+ si = ProgranServiceInstance.objects.get(name=p['Name'])
+ log.info("Profile %s already exists, updating it" % p['Name'])
+ except IndexError:
+ si = ProgranServiceInstance()
+ si.name = p['Name']
+
+ si.no_sync = True
+
+ log.info("Profile %s is new, creating it" % p['Name'])
+
+ si = ProgranHelpers.update_fields(si, p, field_mapping, field_transformations)
+ si.handover = handover
+
+ si.save()
+
+ updated_profiles.append(si.name)
+
+ existing_profiles = [p.name for p in ProgranServiceInstance.objects.all() if not p.is_new]
+ deleted_profiles = ProgranHelpers.list_diff(existing_profiles, updated_profiles)
+
+ if len(deleted_profiles) > 0:
+ log.info("Profiles %s have been removed in progran, removing them from XOS" % str(deleted_profiles))
+ for p in deleted_profiles:
+ si = ProgranServiceInstance.objects.get(name=p)
+ si.delete()