[CORD-2675] Polling profiles

Change-Id: Ic47e88341a74384cb57d959bf2d01e1a8addbd8c
diff --git a/xos/models/progran.xproto b/xos/models/progran.xproto
index 5b1a54c..b7e5c92 100644
--- a/xos/models/progran.xproto
+++ b/xos/models/progran.xproto
@@ -48,9 +48,10 @@
     required int32 DlWifiRate = 11 [default = 100, db_index = False, null = False, blank = False];
     required int32 DlUeAllocRbRate = 12 [default = 100, db_index = False, null = False, blank = False];
     required string SubsProfile = 13 [ db_index = False, null = True, blank = True];
-    optional manytoone enodeb->ENodeB:profiles = 14 [null = True, blank = True];
-    required manytoone handover->Handover:profiles = 15 [null = False, blank = False];
-    optional int32 active_enodeb_id = 16 [null = True, blank = True, gui_hidden = True];
+    optional bool Status = 14 [ db_index = False, null = False, blank = True, default = False];
+    optional manytoone enodeb->ENodeB:profiles = 15 [null = True, blank = True];
+    required manytoone handover->Handover:profiles = 16 [null = False, blank = False];
+    optional int32 active_enodeb_id = 17 [null = True, blank = True, gui_hidden = True];
 }
 
 
diff --git a/xos/synchronizer/Dockerfile.synchronizer b/xos/synchronizer/Dockerfile.synchronizer
index b0c967c..cd4da41 100644
--- a/xos/synchronizer/Dockerfile.synchronizer
+++ b/xos/synchronizer/Dockerfile.synchronizer
@@ -55,5 +55,5 @@
       org.opencord.component.xos.vcs-url=$org_opencord_component_xos_vcs_url \
       org.opencord.component.xos.vcs-ref=$org_opencord_component_xos_vcs_ref
 
-CMD bash -c "service filebeat start; cd /opt/xos/synchronizers/progran; ./run.sh"
+CMD bash -c "service filebeat start; cd /opt/xos/synchronizers/progran; python progran-synchronizer.py"
 
diff --git a/xos/synchronizer/run.sh b/xos/synchronizer/run.sh
deleted file mode 100755
index 63c8460..0000000
--- a/xos/synchronizer/run.sh
+++ /dev/null
@@ -1,17 +0,0 @@
-
-# Copyright 2017-present Open Networking Foundation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-python progran-synchronizer.py
diff --git a/xos/synchronizer/steps/.gitignore b/xos/synchronizer/steps/.gitignore
new file mode 100644
index 0000000..c649235
--- /dev/null
+++ b/xos/synchronizer/steps/.gitignore
@@ -0,0 +1 @@
+synchronizers
\ No newline at end of file
diff --git a/xos/synchronizer/steps/helpers.py b/xos/synchronizer/steps/helpers.py
index d52422b..8f07a66 100644
--- a/xos/synchronizer/steps/helpers.py
+++ b/xos/synchronizer/steps/helpers.py
@@ -15,6 +15,8 @@
 from xosconfig import Config
 from multistructlog import create_logger
 from synchronizers.new_base.modelaccessor import ProgranService
+import time
+import datetime
 
 log = create_logger(Config().get('logging'))
 
@@ -30,7 +32,7 @@
         try:
             progran_service = ProgranService.objects.all()[0]
         except IndexError:
-            log.error("Cannot find Progran Service, does it exists?")
+            raise Exception("Cannot find Progran Service, does it exists?")
         return ProgranHelpers.get_onos_info_from_service(progran_service)
 
     @staticmethod
@@ -41,3 +43,43 @@
             'username': progran_service.onos_username,
             'password': progran_service.onos_password,
         }
+
+    @staticmethod
+    def get_progran_rest_errors(res):
+        res = res.json()
+        if res['Result'] == -2:
+            raise Exception(res['ErrCode'])
+
+    @staticmethod
+    def date_to_time(d):
+        if len(d) == 0:
+            return 0
+        return time.mktime(datetime.datetime.strptime(d, "%d.%m.%Y %H:%S").timetuple())
+
+    @staticmethod
+    def update_fields(model, dict, mapping={}, transformations={}):
+        dict = ProgranHelpers.convert_keys(dict, mapping, transformations)
+        for k, v in dict.iteritems():
+            if hasattr(model, k):
+                setattr(model, k, v)
+            else:
+                log.warn("%s does not have a '%s' property, not updating it" % (model.model_name, k))
+        return model
+
+    @staticmethod
+    def convert_keys(dict, mapping={}, transformations={}):
+        for k, v in dict.iteritems():
+            if k in mapping:
+                # apply custom transformations to the data
+                if k in transformations:
+                    dict[k] = transformations[k](v)
+
+                # NOTE we may have different names that the field in the dict
+                dict[mapping[k]] = dict[k]
+                del dict[k]
+        return dict
+
+    @staticmethod
+    def list_diff(first, second):
+        second = set(second)
+        return [item for item in first if item not in second]
\ No newline at end of file
diff --git a/xos/synchronizer/steps/sync_imsi.py b/xos/synchronizer/steps/sync_imsi.py
index c76dafb..93af41f 100644
--- a/xos/synchronizer/steps/sync_imsi.py
+++ b/xos/synchronizer/steps/sync_imsi.py
@@ -78,6 +78,7 @@
         self.run_playbook(o, create_fields)
 
         # fetch the IMSI we just created
+        # NOTE we won't need this method once we'll have polling in place
         imsi_url = "http://%s:%s/onos/progran/imsi/%s" % (base_fields['onos_url'], base_fields['onos_port'], o.imsi_number)
         r = requests.get(imsi_url)
         o.ue_status = r.json()['ImsiArray'][0]['UeStatus']
diff --git a/xos/synchronizer/steps/sync_progranserviceinstance.py b/xos/synchronizer/steps/sync_progranserviceinstance.py
index 14925bc..11d679d 100644
--- a/xos/synchronizer/steps/sync_progranserviceinstance.py
+++ b/xos/synchronizer/steps/sync_progranserviceinstance.py
@@ -17,14 +17,15 @@
 import os
 import sys
 from synchronizers.new_base.SyncInstanceUsingAnsible import SyncStep
-from synchronizers.new_base.ansible_helper import run_template
-from synchronizers.new_base.modelaccessor import ProgranServiceInstance, ENodeB
+from synchronizers.new_base.modelaccessor import ProgranServiceInstance, ENodeB, Handover
 
 from xosconfig import Config
 from multistructlog import create_logger
 import json
 import requests
 from requests.auth import HTTPBasicAuth
+import time
+import datetime
 
 
 log = create_logger(Config().get('logging'))
@@ -39,6 +40,9 @@
 
     observes = ProgranServiceInstance
 
+    # Poll every 5 loops of self.call
+    poll = 0
+
     def sync_record(self, o):
         onos = ProgranHelpers.get_progran_onos_info()
 
@@ -48,6 +52,8 @@
         data = self.get_progran_profile_field(o)
 
         r = requests.post(profile_url, data=json.dumps(data), auth=HTTPBasicAuth(onos['username'], onos['password']))
+
+        ProgranHelpers.get_progran_rest_errors(r)
         log.info("Profile synchronized", response=r.json())
 
         log.info("sync'ing enodeb", object=str(o), **o.tologdict())
@@ -60,6 +66,7 @@
                 ]
             }
             r = requests.post(enodeb_url, data=json.dumps(data), auth=HTTPBasicAuth(onos['username'], onos['password']))
+            ProgranHelpers.get_progran_rest_errors(r)
             o.active_enodeb_id = o.enodeb_id # storing the value to know when it will be deleted
             log.info("EnodeB synchronized", response=r.json())
         elif o.active_enodeb_id:
@@ -67,12 +74,12 @@
             log.info("removing profile %s from enodeb %s" % (o.name, o.active_enodeb_id), object=str(o), **o.tologdict())
             enodeb_url = "http://%s:%s/onos/progran/enodeb/%s/profile/%s" % (onos['url'], onos['port'], enb_id, o.name)
             r = requests.delete(enodeb_url, auth=HTTPBasicAuth(onos['username'], onos['password']))
+            ProgranHelpers.get_progran_rest_errors(r)
             o.active_enodeb_id = 0 # removing the value because it has been deleted
             log.info("EnodeB synchronized", response=r.json())
 
         o.save()
 
-
     def get_handover_for_profile(self, o):
         return {
             "A3Hysteresis": o.handover.HysteresisA3,
@@ -114,4 +121,106 @@
         profile_url = "http://%s:%s/onos/progran/profile/%s" % (onos['url'], onos['port'], o.name)
         r = requests.delete(profile_url, auth=HTTPBasicAuth(onos['username'], onos['password']))
         o.active_enodeb_id = 0  # removing the value because it has been deleted
-        log.info("Profile synchronized", response=r.json())
\ No newline at end of file
+        log.info("Profile synchronized", response=r.json())
+
+    def fetch_pending(self, deleted):
+        # self.read_profiles_from_progran()
+        return super(SyncProgranServiceInstance, self).fetch_pending(deleted)
+
+    @staticmethod
+    def date_to_time(d):
+        if len(d) == 0:
+            return 0
+        return time.mktime(datetime.datetime.strptime(d, "%d.%m.%Y %H:%S").timetuple())
+
+    @staticmethod
+    def update_fields(model, dict, mapping={}, transformations={}):
+        dict = SyncProgranServiceInstance.convert_keys(dict, mapping, transformations)
+        for k, v in dict.iteritems():
+            if hasattr(model, k):
+                setattr(model, k, v)
+            else:
+                log.warn("%s does not have a '%s' property, not updating it" % (model.model_name, k))
+        return model
+
+    @staticmethod
+    def convert_keys(dict, mapping={}, transformations={}):
+        for k, v in dict.iteritems():
+            if k in mapping:
+                # apply custom transformations to the data
+                if k in transformations:
+                    dict[k] = transformations[k](v)
+
+                # NOTE we may have different names that the field in the dict
+                dict[mapping[k]] = dict[k]
+                del dict[k]
+        return dict
+
+
+    def my_call(self, failed=[], deletion=False):
+        """
+        Read profile from progran and save them in xos
+        """
+        if self.poll < 5:
+            self.poll = self.poll + 1
+        else:
+            self.poll = 0
+            onos = ProgranHelpers.get_progran_onos_info()
+            profile_url = "http://%s:%s/onos/progran/profile/" % (onos['url'], onos['port'])
+            r = requests.get(profile_url, auth=HTTPBasicAuth(onos['username'], onos['password']))
+            res = r.json()['ProfileArray']
+
+            # remove default profiles
+            res = [p for p in res if "Default" not in p['Name']]
+
+            field_mapping = {
+                'Name': 'name',
+                'Start': 'start',
+                'End': 'end'
+            }
+
+            field_transformations = {
+                'Start': SyncProgranServiceInstance.date_to_time,
+                'End': SyncProgranServiceInstance.date_to_time
+            }
+
+            handover_mapping = {
+                'A5Hysteresis': 'HysteresisA5',
+                'A3Hysteresis': 'HysteresisA3'
+            }
+
+            for p in res:
+
+                # checking for handovers
+                handover_dict = p['Handover']
+                handover_dict = SyncProgranServiceInstance.convert_keys(handover_dict, handover_mapping)
+                del p['Handover']
+
+                try:
+                    handover = Handover.objects.get(**handover_dict)
+                    log.info("handover already exists, updating it", handover=handover_dict)
+                except IndexError:
+                    handover = Handover()
+                    handover = SyncProgranServiceInstance.update_fields(handover, handover_dict)
+                    log.info("handover is new, creating it", handover=handover_dict)
+
+                handover.save()
+
+                # checking for profiles
+                try:
+                    si = ProgranServiceInstance.objects.get(name=p['Name'])
+                    log.info("Profile %s already exists, updating it" % p['Name'])
+                except IndexError:
+                    si = ProgranServiceInstance()
+                    si.name = p['Name']
+                    log.info("Profile %s is new, creating it" % p['Name'])
+
+                si = SyncProgranServiceInstance.update_fields(si, p, field_mapping, field_transformations)
+                si.handover = handover
+
+
+
+                # TODO keep track of the deleted profiles
+                # existing profiles - updated profiles = deleted profiles
+
+                si.save()
diff --git a/xos/synchronizer/steps/sync_progranserviceinstance_back.py b/xos/synchronizer/steps/sync_progranserviceinstance_back.py
new file mode 100644
index 0000000..ae97c95
--- /dev/null
+++ b/xos/synchronizer/steps/sync_progranserviceinstance_back.py
@@ -0,0 +1,126 @@
+
+# Copyright 2017-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+import sys
+
+import datetime
+import time
+
+from synchronizers.new_base.SyncInstanceUsingAnsible import SyncStep
+from synchronizers.new_base.modelaccessor import ProgranServiceInstance, ENodeB, Handover
+
+from xosconfig import Config
+from multistructlog import create_logger
+import json
+import requests
+from requests.auth import HTTPBasicAuth
+
+
+
+log = create_logger(Config().get('logging'))
+
+parentdir = os.path.join(os.path.dirname(__file__), "..")
+sys.path.insert(0, parentdir)
+sys.path.insert(0, os.path.dirname(__file__))
+from helpers import ProgranHelpers
+
+class SyncProgranServiceInstanceBack(SyncStep):
+    provides = [ProgranServiceInstance]
+
+    observes = ProgranServiceInstance
+
+
+    def call(self, failed=[], deletion=False):
+        """
+        Read profile from progran and save them in xos
+        """
+
+        if deletion == False:
+            # NOTE we won't it to run only after the delete has completed
+            return
+
+        log.info("Reading profiles from progran")
+        onos = ProgranHelpers.get_progran_onos_info()
+        profile_url = "http://%s:%s/onos/progran/profile/" % (onos['url'], onos['port'])
+        r = requests.get(profile_url, auth=HTTPBasicAuth(onos['username'], onos['password']))
+        res = r.json()['ProfileArray']
+
+        # remove default profiles
+        res = [p for p in res if "Default" not in p['Name']]
+
+        field_mapping = {
+            'Name': 'name',
+            'Start': 'start',
+            'End': 'end'
+        }
+
+        field_transformations = {
+            'Start': ProgranHelpers.date_to_time,
+            'End': ProgranHelpers.date_to_time
+        }
+
+        handover_mapping = {
+            'A5Hysteresis': 'HysteresisA5',
+            'A3Hysteresis': 'HysteresisA3'
+        }
+
+        updated_profiles = []
+
+        for p in res:
+
+            # checking for handovers
+            handover_dict = p['Handover']
+            handover_dict = ProgranHelpers.convert_keys(handover_dict, handover_mapping)
+            del p['Handover']
+
+            try:
+                handover = Handover.objects.get(**handover_dict)
+                log.info("handover already exists, updating it", handover=handover_dict)
+            except IndexError:
+                handover = Handover()
+                handover = ProgranHelpers.update_fields(handover, handover_dict)
+                log.info("handover is new, creating it", handover=handover_dict)
+
+            handover.save()
+
+            # checking for profiles
+            try:
+                si = ProgranServiceInstance.objects.get(name=p['Name'])
+                log.info("Profile %s already exists, updating it" % p['Name'])
+            except IndexError:
+                si = ProgranServiceInstance()
+                si.name = p['Name']
+
+                si.no_sync = True
+
+                log.info("Profile %s is new, creating it" % p['Name'])
+
+            si = ProgranHelpers.update_fields(si, p, field_mapping, field_transformations)
+            si.handover = handover
+
+            si.save()
+
+            updated_profiles.append(si.name)
+
+        existing_profiles = [p.name for p in ProgranServiceInstance.objects.all() if not p.is_new]
+        deleted_profiles = ProgranHelpers.list_diff(existing_profiles, updated_profiles)
+
+        if len(deleted_profiles) > 0:
+            log.info("Profiles %s have been removed in progran, removing them from XOS" % str(deleted_profiles))
+            for p in deleted_profiles:
+                si = ProgranServiceInstance.objects.get(name=p)
+                si.delete()