Sapan Bhatia | 2ac8864 | 2016-01-15 10:43:19 -0500 | [diff] [blame] | 1 | import hashlib |
| 2 | import os |
| 3 | import socket |
| 4 | import sys |
| 5 | import base64 |
| 6 | import time |
| 7 | from django.db.models import F, Q |
| 8 | from xos.config import Config |
| 9 | from synchronizers.base.syncstep import SyncStep |
| 10 | from synchronizers.base.ansible import run_template_ssh |
Sapan Bhatia | 62ee8d1 | 2016-01-15 11:38:46 -0500 | [diff] [blame] | 11 | from synchronizers.base.SyncInstanceUsingAnsible import SyncInstanceUsingAnsible |
Sapan Bhatia | 2ac8864 | 2016-01-15 10:43:19 -0500 | [diff] [blame] | 12 | from core.models import Service, Slice |
Scott Baker | 664bbe5 | 2016-02-11 11:08:42 -0800 | [diff] [blame] | 13 | from services.cord.models import VCPEService, VSGTenant, VOLTTenant |
Sapan Bhatia | 2ac8864 | 2016-01-15 10:43:19 -0500 | [diff] [blame] | 14 | from services.hpc.models import HpcService, CDNPrefix |
Scott Baker | 3a01eb2 | 2016-01-15 07:57:33 -0800 | [diff] [blame] | 15 | from xos.logger import Logger, logging |
Sapan Bhatia | 2ac8864 | 2016-01-15 10:43:19 -0500 | [diff] [blame] | 16 | |
| 17 | # hpclibrary will be in steps/.. |
| 18 | parentdir = os.path.join(os.path.dirname(__file__),"..") |
| 19 | sys.path.insert(0,parentdir) |
| 20 | |
| 21 | from broadbandshield import BBS |
| 22 | |
| 23 | logger = Logger(level=logging.INFO) |
| 24 | |
| 25 | PARENTAL_MECHANISM="dnsmasq" |
| 26 | ENABLE_QUICK_UPDATE=False |
| 27 | |
Scott Baker | 664bbe5 | 2016-02-11 11:08:42 -0800 | [diff] [blame] | 28 | class SyncVSGTenant(SyncInstanceUsingAnsible): |
| 29 | provides=[VSGTenant] |
| 30 | observes=VSGTenant |
Sapan Bhatia | 2ac8864 | 2016-01-15 10:43:19 -0500 | [diff] [blame] | 31 | requested_interval=0 |
| 32 | template_name = "sync_vcpetenant.yaml" |
Srikanth Vavilapalli | ae1acd6 | 2016-01-25 20:06:43 -0500 | [diff] [blame] | 33 | service_key_name = "/opt/xos/synchronizers/vcpe/vcpe_private_key" |
Sapan Bhatia | 2ac8864 | 2016-01-15 10:43:19 -0500 | [diff] [blame] | 34 | |
| 35 | def __init__(self, *args, **kwargs): |
Scott Baker | 664bbe5 | 2016-02-11 11:08:42 -0800 | [diff] [blame] | 36 | super(SyncVSGTenant, self).__init__(*args, **kwargs) |
Sapan Bhatia | 2ac8864 | 2016-01-15 10:43:19 -0500 | [diff] [blame] | 37 | |
| 38 | def fetch_pending(self, deleted): |
| 39 | if (not deleted): |
Scott Baker | 664bbe5 | 2016-02-11 11:08:42 -0800 | [diff] [blame] | 40 | objs = VSGTenant.get_tenant_objects().filter(Q(enacted__lt=F('updated')) | Q(enacted=None),Q(lazy_blocked=False)) |
Sapan Bhatia | 2ac8864 | 2016-01-15 10:43:19 -0500 | [diff] [blame] | 41 | else: |
Scott Baker | 664bbe5 | 2016-02-11 11:08:42 -0800 | [diff] [blame] | 42 | objs = VSGTenant.get_deleted_tenant_objects() |
Sapan Bhatia | 2ac8864 | 2016-01-15 10:43:19 -0500 | [diff] [blame] | 43 | |
| 44 | return objs |
| 45 | |
| 46 | def get_vcpe_service(self, o): |
| 47 | if not o.provider_service: |
| 48 | return None |
| 49 | |
| 50 | vcpes = VCPEService.get_service_objects().filter(id=o.provider_service.id) |
| 51 | if not vcpes: |
| 52 | return None |
| 53 | |
| 54 | return vcpes[0] |
| 55 | |
| 56 | def get_extra_attributes(self, o): |
| 57 | # This is a place to include extra attributes that aren't part of the |
| 58 | # object itself. In the case of vCPE, we need to know: |
| 59 | # 1) the addresses of dnsdemux, to setup dnsmasq in the vCPE |
| 60 | # 2) CDN prefixes, so we know what URLs to send to dnsdemux |
| 61 | # 3) BroadBandShield server addresses, for parental filtering |
| 62 | # 4) vlan_ids, for setting up networking in the vCPE VM |
| 63 | |
| 64 | vcpe_service = self.get_vcpe_service(o) |
| 65 | |
| 66 | dnsdemux_ip = None |
| 67 | if vcpe_service.backend_network_label: |
| 68 | # Connect to dnsdemux using the network specified by |
| 69 | # vcpe_service.backend_network_label |
| 70 | for service in HpcService.objects.all(): |
| 71 | for slice in service.slices.all(): |
| 72 | if "dnsdemux" in slice.name: |
| 73 | for instance in slice.instances.all(): |
| 74 | for ns in instance.ports.all(): |
| 75 | if ns.ip and ns.network.labels and (vcpe_service.backend_network_label in ns.network.labels): |
| 76 | dnsdemux_ip = ns.ip |
| 77 | if not dnsdemux_ip: |
| 78 | logger.info("failed to find a dnsdemux on network %s" % vcpe_service.backend_network_label) |
| 79 | else: |
| 80 | # Connect to dnsdemux using the instance's public address |
| 81 | for service in HpcService.objects.all(): |
| 82 | for slice in service.slices.all(): |
| 83 | if "dnsdemux" in slice.name: |
| 84 | for instance in slice.instances.all(): |
| 85 | if dnsdemux_ip=="none": |
| 86 | try: |
| 87 | dnsdemux_ip = socket.gethostbyname(instance.node.name) |
| 88 | except: |
| 89 | pass |
| 90 | if not dnsdemux_ip: |
| 91 | logger.info("failed to find a dnsdemux with a public address") |
| 92 | |
| 93 | dnsdemux_ip = dnsdemux_ip or "none" |
| 94 | |
| 95 | cdn_prefixes = [] |
| 96 | for prefix in CDNPrefix.objects.all(): |
| 97 | cdn_prefixes.append(prefix.prefix) |
| 98 | |
| 99 | # Broadbandshield can either be set up internally, using vcpe_service.bbs_slice, |
| 100 | # or it can be setup externally using vcpe_service.bbs_server. |
| 101 | |
| 102 | bbs_addrs = [] |
| 103 | if vcpe_service.bbs_slice: |
| 104 | if vcpe_service.backend_network_label: |
| 105 | for bbs_instance in vcpe_service.bbs_slice.instances.all(): |
| 106 | for ns in bbs_instance.ports.all(): |
| 107 | if ns.ip and ns.network.labels and (vcpe_service.backend_network_label in ns.network.labels): |
| 108 | bbs_addrs.append(ns.ip) |
| 109 | else: |
| 110 | logger.info("unsupported configuration -- bbs_slice is set, but backend_network_label is not") |
| 111 | if not bbs_addrs: |
| 112 | logger.info("failed to find any usable addresses on bbs_slice") |
| 113 | elif vcpe_service.bbs_server: |
| 114 | bbs_addrs.append(vcpe_service.bbs_server) |
| 115 | else: |
| 116 | logger.info("neither bbs_slice nor bbs_server is configured in the vCPE") |
| 117 | |
| 118 | vlan_ids = [] |
| 119 | s_tags = [] |
| 120 | c_tags = [] |
| 121 | if o.volt: |
| 122 | vlan_ids.append(o.volt.vlan_id) # XXX remove this |
| 123 | s_tags.append(o.volt.s_tag) |
| 124 | c_tags.append(o.volt.c_tag) |
| 125 | |
| 126 | try: |
| 127 | full_setup = Config().observer_full_setup |
| 128 | except: |
| 129 | full_setup = True |
| 130 | |
| 131 | safe_macs=[] |
| 132 | if o.volt and o.volt.subscriber: |
| 133 | for user in o.volt.subscriber.users: |
| 134 | level = user.get("level",None) |
| 135 | mac = user.get("mac",None) |
| 136 | if level in ["G", "PG"]: |
| 137 | if mac: |
| 138 | safe_macs.append(mac) |
| 139 | |
| 140 | fields = {"vlan_ids": vlan_ids, # XXX remove this |
| 141 | "s_tags": s_tags, |
| 142 | "c_tags": c_tags, |
| 143 | "dnsdemux_ip": dnsdemux_ip, |
| 144 | "cdn_prefixes": cdn_prefixes, |
| 145 | "bbs_addrs": bbs_addrs, |
| 146 | "full_setup": full_setup, |
| 147 | "isolation": o.instance.isolation, |
| 148 | "safe_browsing_macs": safe_macs} |
| 149 | |
| 150 | # add in the sync_attributes that come from the SubscriberRoot object |
| 151 | |
| 152 | if o.volt and o.volt.subscriber and hasattr(o.volt.subscriber, "sync_attributes"): |
| 153 | for attribute_name in o.volt.subscriber.sync_attributes: |
| 154 | fields[attribute_name] = getattr(o.volt.subscriber, attribute_name) |
| 155 | |
| 156 | return fields |
| 157 | |
| 158 | def sync_fields(self, o, fields): |
| 159 | # the super causes the playbook to be run |
| 160 | |
Scott Baker | 664bbe5 | 2016-02-11 11:08:42 -0800 | [diff] [blame] | 161 | super(SyncVSGTenant, self).sync_fields(o, fields) |
Sapan Bhatia | 2ac8864 | 2016-01-15 10:43:19 -0500 | [diff] [blame] | 162 | |
| 163 | # now do all of our broadbandshield stuff... |
| 164 | |
| 165 | service = self.get_vcpe_service(o) |
| 166 | if not service: |
| 167 | # Ansible uses the service's keypair in order to SSH into the |
| 168 | # instance. It would be bad if the slice had no service. |
| 169 | |
| 170 | raise Exception("Slice %s is not associated with a service" % instance.slice.name) |
| 171 | |
| 172 | # Make sure the slice is configured properly |
| 173 | if (service != o.instance.slice.service): |
| 174 | raise Exception("Slice %s is associated with some service that is not %s" % (str(instance.slice), str(service))) |
| 175 | |
| 176 | # only enable filtering if we have a subscriber object (see below) |
| 177 | url_filter_enable = False |
| 178 | |
| 179 | # for attributes that come from CordSubscriberRoot |
| 180 | if o.volt and o.volt.subscriber: |
| 181 | url_filter_enable = o.volt.subscriber.url_filter_enable |
| 182 | url_filter_level = o.volt.subscriber.url_filter_level |
| 183 | url_filter_users = o.volt.subscriber.users |
| 184 | |
| 185 | if PARENTAL_MECHANISM=="broadbandshield": |
| 186 | # disable url_filter if there are no bbs_addrs |
| 187 | if url_filter_enable and (not fields.get("bbs_addrs",[])): |
| 188 | logger.info("disabling url_filter because there are no bbs_addrs") |
| 189 | url_filter_enable = False |
| 190 | |
| 191 | if url_filter_enable: |
| 192 | bbs_hostname = None |
| 193 | if service.bbs_api_hostname and service.bbs_api_port: |
| 194 | bbs_hostname = service.bbs_api_hostname |
| 195 | else: |
| 196 | # TODO: extract from slice |
| 197 | bbs_hostname = "cordcompute01.onlab.us" |
| 198 | |
| 199 | if service.bbs_api_port: |
| 200 | bbs_port = service.bbs_api_port |
| 201 | else: |
| 202 | bbs_port = 8018 |
| 203 | |
| 204 | if not bbs_hostname: |
| 205 | logger.info("broadbandshield is not configured") |
| 206 | else: |
| 207 | tStart = time.time() |
| 208 | bbs = BBS(o.bbs_account, "123", bbs_hostname, bbs_port) |
| 209 | bbs.sync(url_filter_level, url_filter_users) |
| 210 | |
| 211 | if o.hpc_client_ip: |
| 212 | logger.info("associate account %s with ip %s" % (o.bbs_account, o.hpc_client_ip)) |
| 213 | bbs.associate(o.hpc_client_ip) |
| 214 | else: |
| 215 | logger.info("no hpc_client_ip to associate") |
| 216 | |
| 217 | logger.info("bbs update time %d" % int(time.time()-tStart)) |
| 218 | |
| 219 | |
| 220 | def run_playbook(self, o, fields): |
| 221 | ansible_hash = hashlib.md5(repr(sorted(fields.items()))).hexdigest() |
| 222 | quick_update = (o.last_ansible_hash == ansible_hash) |
| 223 | |
| 224 | if ENABLE_QUICK_UPDATE and quick_update: |
| 225 | logger.info("quick_update triggered; skipping ansible recipe") |
| 226 | else: |
| 227 | if o.instance.isolation in ["container", "container_vm"]: |
Scott Baker | 664bbe5 | 2016-02-11 11:08:42 -0800 | [diff] [blame] | 228 | super(SyncVSGTenant, self).run_playbook(o, fields, "sync_vcpetenant_new.yaml") |
Sapan Bhatia | 2ac8864 | 2016-01-15 10:43:19 -0500 | [diff] [blame] | 229 | else: |
Scott Baker | 664bbe5 | 2016-02-11 11:08:42 -0800 | [diff] [blame] | 230 | super(SyncVSGTenant, self).run_playbook(o, fields) |
Sapan Bhatia | 2ac8864 | 2016-01-15 10:43:19 -0500 | [diff] [blame] | 231 | |
| 232 | o.last_ansible_hash = ansible_hash |
| 233 | |
| 234 | def delete_record(self, m): |
| 235 | pass |