Sapan Bhatia | 24836f1 | 2013-08-27 10:16:05 -0400 | [diff] [blame] | 1 | import os |
| 2 | import base64 |
Tony Mack | 4fa85fb | 2013-09-25 14:39:57 -0400 | [diff] [blame] | 3 | from datetime import datetime |
Sapan Bhatia | 24836f1 | 2013-08-27 10:16:05 -0400 | [diff] [blame] | 4 | from planetstack.config import Config |
Andy Bavier | e7abb62 | 2013-10-18 15:11:56 -0400 | [diff] [blame] | 5 | from util.logger import Logger, logging |
Sapan Bhatia | eba0843 | 2014-04-28 23:58:36 -0400 | [diff] [blame] | 6 | from observer.steps import * |
Andy Bavier | e7abb62 | 2013-10-18 15:11:56 -0400 | [diff] [blame] | 7 | |
Andy Bavier | 04111b7 | 2013-10-22 16:47:10 -0400 | [diff] [blame] | 8 | logger = Logger(level=logging.INFO) |
Sapan Bhatia | 24836f1 | 2013-08-27 10:16:05 -0400 | [diff] [blame] | 9 | |
Sapan Bhatia | 13c7f11 | 2013-09-02 14:19:35 -0400 | [diff] [blame] | 10 | class FailedDependency(Exception): |
Tony Mack | ce79de0 | 2013-09-24 10:12:33 -0400 | [diff] [blame] | 11 | pass |
Sapan Bhatia | 13c7f11 | 2013-09-02 14:19:35 -0400 | [diff] [blame] | 12 | |
Sapan Bhatia | 24836f1 | 2013-08-27 10:16:05 -0400 | [diff] [blame] | 13 | class SyncStep: |
Tony Mack | ce79de0 | 2013-09-24 10:12:33 -0400 | [diff] [blame] | 14 | """ A PlanetStack Sync step. |
Sapan Bhatia | 24836f1 | 2013-08-27 10:16:05 -0400 | [diff] [blame] | 15 | |
Tony Mack | ce79de0 | 2013-09-24 10:12:33 -0400 | [diff] [blame] | 16 | Attributes: |
| 17 | psmodel Model name the step synchronizes |
| 18 | dependencies list of names of models that must be synchronized first if the current model depends on them |
| 19 | """ |
| 20 | slow=False |
| 21 | def get_prop(prop): |
| 22 | try: |
| 23 | sync_config_dir = Config().sync_config_dir |
| 24 | except: |
| 25 | sync_config_dir = '/etc/planetstack/sync' |
| 26 | prop_config_path = '/'.join(sync_config_dir,self.name,prop) |
| 27 | return open(prop_config_path).read().rstrip() |
Sapan Bhatia | 24836f1 | 2013-08-27 10:16:05 -0400 | [diff] [blame] | 28 | |
Tony Mack | ce79de0 | 2013-09-24 10:12:33 -0400 | [diff] [blame] | 29 | def __init__(self, **args): |
| 30 | """Initialize a sync step |
| 31 | Keyword arguments: |
| 32 | name -- Name of the step |
| 33 | provides -- PlanetStack models sync'd by this step |
| 34 | """ |
| 35 | dependencies = [] |
Tony Mack | 387a73f | 2013-09-18 07:59:14 -0400 | [diff] [blame] | 36 | self.driver = args.get('driver') |
Sapan Bhatia | eba0843 | 2014-04-28 23:58:36 -0400 | [diff] [blame] | 37 | self.error_map = args.get('error_map') |
| 38 | |
Tony Mack | ce79de0 | 2013-09-24 10:12:33 -0400 | [diff] [blame] | 39 | try: |
| 40 | self.soft_deadline = int(self.get_prop('soft_deadline_seconds')) |
| 41 | except: |
| 42 | self.soft_deadline = 5 # 5 seconds |
Sapan Bhatia | 24836f1 | 2013-08-27 10:16:05 -0400 | [diff] [blame] | 43 | |
Tony Mack | ce79de0 | 2013-09-24 10:12:33 -0400 | [diff] [blame] | 44 | return |
Sapan Bhatia | 24836f1 | 2013-08-27 10:16:05 -0400 | [diff] [blame] | 45 | |
Sapan Bhatia | e17bc5b | 2014-04-30 00:53:06 -0400 | [diff] [blame] | 46 | def fetch_pending(self, deletion=False): |
Sapan Bhatia | 2176566 | 2014-07-23 08:59:30 -0400 | [diff] [blame] | 47 | # This is the most common implementation of fetch_pending |
| 48 | # Steps should override it if they have their own logic |
| 49 | # for figuring out what objects are outstanding. |
| 50 | main_obj = self.provides[0] |
| 51 | if (not deleted): |
| 52 | objs = main_obj.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None)) |
| 53 | else: |
| 54 | objs = main_obj.deleted_objects.all() |
| 55 | |
| 56 | return objs |
Sapan Bhatia | ca2e21f | 2013-10-02 01:10:02 -0400 | [diff] [blame] | 57 | #return Sliver.objects.filter(ip=None) |
Tony Mack | ce79de0 | 2013-09-24 10:12:33 -0400 | [diff] [blame] | 58 | |
Sapan Bhatia | ca2e21f | 2013-10-02 01:10:02 -0400 | [diff] [blame] | 59 | def check_dependencies(self, obj, failed): |
Tony Mack | ce79de0 | 2013-09-24 10:12:33 -0400 | [diff] [blame] | 60 | for dep in self.dependencies: |
Scott Baker | 105b6b7 | 2014-05-12 10:40:25 -0700 | [diff] [blame] | 61 | peer_name = dep[0].lower() + dep[1:] # django names are camelCased with the first letter lower |
| 62 | peer_object = getattr(obj, peer_name) |
Scott Baker | 0652f4e | 2014-08-19 15:49:27 -0700 | [diff] [blame] | 63 | if (peer_object and peer_object.pk==failed.pk and type(peer_object)==type(failed)): |
| 64 | raise FailedDependency("Failed dependency for %s:%s peer %s:%s failed %s:%s" % (obj.__class__.__name__, str(obj.pk\
|
| 65 | ), peer_object.__class__.__name__, str(peer_object.pk), failed.__class__.__name__, str(failed.pk))) |
Sapan Bhatia | 24836f1 | 2013-08-27 10:16:05 -0400 | [diff] [blame] | 66 | |
Sapan Bhatia | 6082336 | 2014-04-30 00:52:32 -0400 | [diff] [blame] | 67 | def call(self, failed=[], deletion=False): |
| 68 | pending = self.fetch_pending(deletion) |
Tony Mack | ce79de0 | 2013-09-24 10:12:33 -0400 | [diff] [blame] | 69 | for o in pending: |
Tony Mack | 68e818d | 2013-09-25 13:34:17 -0400 | [diff] [blame] | 70 | try: |
Sapan Bhatia | ca2e21f | 2013-10-02 01:10:02 -0400 | [diff] [blame] | 71 | for f in failed: |
Andy Bavier | e7abb62 | 2013-10-18 15:11:56 -0400 | [diff] [blame] | 72 | self.check_dependencies(o,f) # Raises exception if failed |
Sapan Bhatia | 6082336 | 2014-04-30 00:52:32 -0400 | [diff] [blame] | 73 | if (deletion): |
| 74 | self.delete_record(o) |
| 75 | o.delete(purge=True) |
| 76 | else: |
| 77 | self.sync_record(o) |
| 78 | o.enacted = datetime.now() # Is this the same timezone? XXX |
| 79 | o.backend_status = "OK" |
| 80 | o.save(update_fields=['enacted']) |
Sapan Bhatia | eba0843 | 2014-04-28 23:58:36 -0400 | [diff] [blame] | 81 | except Exception,e: |
| 82 | try: |
| 83 | o.backend_status = self.error_map.map(str(e)) |
| 84 | except: |
| 85 | o.backend_status = str(e) |
| 86 | |
Sapan Bhatia | f1d3d27 | 2014-08-18 02:24:22 -0400 | [diff] [blame] | 87 | if (o.pk): |
| 88 | o.save(update_fields=['backend_status']) |
Sapan Bhatia | eba0843 | 2014-04-28 23:58:36 -0400 | [diff] [blame] | 89 | |
| 90 | logger.log_exc("sync step failed!") |
Tony Mack | 68e818d | 2013-09-25 13:34:17 -0400 | [diff] [blame] | 91 | failed.append(o) |
Sapan Bhatia | ca2e21f | 2013-10-02 01:10:02 -0400 | [diff] [blame] | 92 | |
Tony Mack | ce79de0 | 2013-09-24 10:12:33 -0400 | [diff] [blame] | 93 | return failed |
Sapan Bhatia | 24836f1 | 2013-08-27 10:16:05 -0400 | [diff] [blame] | 94 | |
Tony Mack | 16f0474 | 2013-09-25 08:53:28 -0400 | [diff] [blame] | 95 | def __call__(self, **args): |
| 96 | return self.call(**args) |