Updating vEG to new docker containers, new_base and dependecies
Change-Id: I6414cae27801ff0d26b0a7b3eb2f53312d874bf0
diff --git a/xos/admin.py b/xos/admin.py
index ebe6353..e6c9cfa 100644
--- a/xos/admin.py
+++ b/xos/admin.py
@@ -18,53 +18,6 @@
from django.core.urlresolvers import reverse
from django.contrib.admin.utils import quote
-#-----------------------------------------------------------------------------
-# vEG
-#-----------------------------------------------------------------------------
-
-class VEGServiceForm(forms.ModelForm):
- bbs_api_hostname = forms.CharField(required=False)
- bbs_api_port = forms.IntegerField(required=False)
- bbs_server = forms.CharField(required=False)
- backend_network_label = forms.CharField(required=False)
- bbs_slice = forms.ModelChoiceField(queryset=Slice.objects.all(), required=False)
- dns_servers = forms.CharField(required=False)
- url_filter_kind = forms.ChoiceField(choices=VEGService.URL_FILTER_KIND_CHOICES, required=False)
- node_label = forms.CharField(required=False)
- docker_image_name = forms.CharField(required=False)
- docker_insecure_registry = forms.BooleanField(required=False)
-
- def __init__(self,*args,**kwargs):
- super (VEGServiceForm,self ).__init__(*args,**kwargs)
- if self.instance:
- self.fields['bbs_api_hostname'].initial = self.instance.bbs_api_hostname
- self.fields['bbs_api_port'].initial = self.instance.bbs_api_port
- self.fields['bbs_server'].initial = self.instance.bbs_server
- self.fields['backend_network_label'].initial = self.instance.backend_network_label
- self.fields['bbs_slice'].initial = self.instance.bbs_slice
- self.fields['dns_servers'].initial = self.instance.dns_servers
- self.fields['url_filter_kind']. initial = self.instance.url_filter_kind
- self.fields['node_label'].initial = self.instance.node_label
- self.fields['docker_image_name'].initial = self.instance.docker_image_name
- self.fields['docker_insecure_registry'].initial = self.instance.docker_insecure_registry
-
- def save(self, commit=True):
- self.instance.bbs_api_hostname = self.cleaned_data.get("bbs_api_hostname")
- self.instance.bbs_api_port = self.cleaned_data.get("bbs_api_port")
- self.instance.bbs_server = self.cleaned_data.get("bbs_server")
- self.instance.backend_network_label = self.cleaned_data.get("backend_network_label")
- self.instance.bbs_slice = self.cleaned_data.get("bbs_slice")
- self.instance.dns_servers = self.cleaned_data.get("dns_servers")
- self.instance.url_filter_kind = self.cleaned_data.get("url_filter_kind")
- self.instance.node_label = self.cleaned_data.get("node_label")
- self.instance.docker_image_name = self.cleaned_data.get("docker_image_name")
- self.instance.docker_insecure_registry = self.cleaned_data.get("docker_insecure_registry")
- return super(VEGServiceForm, self).save(commit=commit)
-
- class Meta:
- model = VEGService
- fields = '__all__'
-
class VEGServiceAdmin(ReadOnlyAwareAdmin):
model = VEGService
verbose_name = "vEG Service"
@@ -73,13 +26,12 @@
list_display_links = ('backend_status_icon', 'name', )
fieldsets = [(None, {'fields': ['backend_status_text', 'name','enabled','versionNumber', 'description', "view_url", "icon_url", "service_specific_attribute", "node_label"],
'classes':['suit-tab suit-tab-general']}),
- ("backend config", {'fields': [ "backend_network_label", "url_filter_kind", "bbs_api_hostname", "bbs_api_port", "bbs_server", "bbs_slice"],
+ ("backend config", {'fields': ["url_filter_kind"],
'classes':['suit-tab suit-tab-backend']}),
("vEG config", {'fields': ["dns_servers", "docker_image_name", "docker_insecure_registry"],
'classes':['suit-tab suit-tab-veg']}) ]
readonly_fields = ('backend_status_text', "service_specific_attribute")
inlines = [SliceInline,ServiceAttrAsTabInline,ServicePrivilegeInline]
- form = VEGServiceForm
extracontext_registered_admins = True
@@ -102,9 +54,6 @@
return VEGService.get_service_objects_by_user(request.user)
class VEGTenantForm(forms.ModelForm):
- bbs_account = forms.CharField(required=False)
- creator = forms.ModelChoiceField(queryset=User.objects.all())
- instance = forms.ModelChoiceField(queryset=Instance.objects.all(),required=False)
last_ansible_hash = forms.CharField(required=False)
wan_container_ip = forms.CharField(required=False)
wan_container_mac = forms.CharField(required=False)
@@ -112,12 +61,9 @@
def __init__(self,*args,**kwargs):
super (VEGTenantForm,self ).__init__(*args,**kwargs)
self.fields['kind'].widget.attrs['readonly'] = True
- self.fields['provider_service'].queryset = VEGService.get_service_objects().all()
+ self.fields['provider_service'].queryset = VEGService.objects.all()
if self.instance:
# fields for the attributes
- self.fields['bbs_account'].initial = self.instance.bbs_account
- self.fields['creator'].initial = self.instance.creator
- self.fields['instance'].initial = self.instance.instance
self.fields['last_ansible_hash'].initial = self.instance.last_ansible_hash
self.fields['wan_container_ip'].initial = self.instance.wan_container_ip
self.fields['wan_container_mac'].initial = self.instance.wan_container_mac
@@ -125,8 +71,8 @@
# default fields for an 'add' form
self.fields['kind'].initial = VEG_KIND
self.fields['creator'].initial = get_request().user
- if VEGService.get_service_objects().exists():
- self.fields["provider_service"].initial = VEGService.get_service_objects().all()[0]
+ if VEGService.objects.exists():
+ self.fields["provider_service"].initial = VEGService.objects.all()[0]
def save(self, commit=True):
self.instance.creator = self.cleaned_data.get("creator")
@@ -141,10 +87,10 @@
class VEGTenantAdmin(ReadOnlyAwareAdmin):
list_display = ('backend_status_icon', 'id', 'subscriber_tenant' )
list_display_links = ('backend_status_icon', 'id')
- fieldsets = [ (None, {'fields': ['backend_status_text', 'kind', 'provider_service', 'subscriber_tenant', 'service_specific_id', # 'service_specific_attribute',
- 'wan_container_ip', 'wan_container_mac', 'bbs_account', 'creator', 'instance', 'last_ansible_hash'],
+ fieldsets = [ (None, {'fields': ['backend_status_text', 'kind', 'provider_service', 'subscriber_tenant', 'service_specific_id',
+ 'wan_container_ip', 'wan_container_mac', 'creator', 'instance', 'last_ansible_hash'],
'classes':['suit-tab suit-tab-general']})]
- readonly_fields = ('backend_status_text', 'service_specific_attribute', 'bbs_account', 'wan_container_ip', 'wan_container_mac')
+ readonly_fields = ('backend_status_text', 'service_specific_attribute', 'wan_container_ip', 'wan_container_mac')
form = VEGTenantForm
suit_form_tabs = (('general','Details'),)
diff --git a/xos/models.py b/xos/models.py
index 6cb81b1..3d828e6 100644
--- a/xos/models.py
+++ b/xos/models.py
@@ -1,5 +1,5 @@
from django.db import models
-from core.models import Service, PlCoreBase, Slice, Instance, Tenant, TenantWithContainer, Node, Image, User, Flavor, Subscriber, NetworkParameter, NetworkParameterType, Port, AddressPool, User
+from core.models import Service, PlCoreBase, Slice, Instance, Tenant, TenantWithContainer, Node, Image, User, Flavor, NetworkParameter, NetworkParameterType, Port, AddressPool, User
from core.models.plcorebase import StrippedCharField
import os
from django.db import models, transaction
@@ -19,8 +19,6 @@
VEG_KIND = "vEG"
CORD_SUBSCRIBER_KIND = "CordSubscriberRoot"
-CORD_USE_VTN = getattr(Config(), "networking_use_vtn", False)
-
# -------------------------------------------
# VEG
# -------------------------------------------
@@ -30,87 +28,34 @@
URL_FILTER_KIND_CHOICES = ( (None, "None"), ("safebrowsing", "Safe Browsing"), ("answerx", "AnswerX") )
- simple_attributes = ( ("bbs_api_hostname", None),
- ("bbs_api_port", None),
- ("bbs_server", None),
- ("backend_network_label", "hpc_client"),
- ("dns_servers", "8.8.8.8"),
- ("url_filter_kind", None),
- ("node_label", None),
- ("docker_image_name", "docker.io/xosproject/veg"),
- ("docker_insecure_registry", False) )
-
- def __init__(self, *args, **kwargs):
- super(VEGService, self).__init__(*args, **kwargs)
+ url_filter_kind = StrippedCharField(max_length=30, choices=URL_FILTER_KIND_CHOICES, null=True, blank=True)
+ dns_servers = StrippedCharField(max_length=255, default="8.8.8.8")
+ node_label = StrippedCharField(max_length=30, null=True, blank=True)
+ docker_image_name = StrippedCharField(max_length=255, default="docker.io/xosproject/veg")
+ docker_insecure_registry = models.BooleanField(default=False)
class Meta:
app_label = "veg"
verbose_name = "vEG Service"
- proxy = True
- def allocate_bbs_account(self):
- vegs = VEGTenant.get_tenant_objects().all()
- bbs_accounts = [veg.bbs_account for veg in vegs]
- # There's a bit of a race here; some other user could be trying to
- # allocate a bbs_account at the same time we are.
-
- for i in range(2,21):
- account_name = "bbs%02d@onlab.us" % i
- if (account_name not in bbs_accounts):
- return account_name
-
- raise XOSConfigurationError("We've run out of available broadbandshield accounts. Delete some veg and try again.")
-
- @property
- def bbs_slice(self):
- bbs_slice_id=self.get_attribute("bbs_slice_id")
- if not bbs_slice_id:
- return None
- bbs_slices=Slice.objects.filter(id=bbs_slice_id)
- if not bbs_slices:
- return None
- return bbs_slices[0]
-
- @bbs_slice.setter
- def bbs_slice(self, value):
- if value:
- value = value.id
- self.set_attribute("bbs_slice_id", value)
-
-VEGService.setup_simple_attributes()
class VEGTenant(TenantWithContainer):
- class Meta:
- proxy = True
-
KIND = VEG_KIND
+ class Meta:
+ app_label = "veg"
+
sync_attributes = ("wan_container_ip", "wan_container_mac", "wan_container_netbits",
"wan_container_gateway_ip", "wan_container_gateway_mac",
"wan_vm_ip", "wan_vm_mac")
- default_attributes = {"instance_id": None,
- "container_id": None,
- "users": [],
- "bbs_account": None,
- "last_ansible_hash": None,
- "wan_container_ip": None}
def __init__(self, *args, **kwargs):
super(VEGTenant, self).__init__(*args, **kwargs)
self.cached_vrouter=None
@property
- def vbng(self):
- # not supported
- return None
-
- @vbng.setter
- def vbng(self, value):
- raise XOSConfigurationError("vEG.vBNG cannot be set this way -- create a new vBNG object and set it's subscriber_tenant instead")
-
- @property
def vrouter(self):
vrouter = self.get_newest_subscribed_tenant(VRouterTenant)
if not vrouter:
@@ -126,7 +71,7 @@
@vrouter.setter
def vrouter(self, value):
- raise XOSConfigurationError("vEG.vRouter cannot be set this way -- create a new vRuter object and set its subscriber_tenant instead")
+ raise XOSConfigurationError("vEGTenant.vrouter setter is not implemented")
@property
def volt(self):
@@ -138,21 +83,9 @@
return None
return volts[0]
- @property
- def bbs_account(self):
- return self.get_attribute("bbs_account", self.default_attributes["bbs_account"])
-
- @bbs_account.setter
- def bbs_account(self, value):
- return self.set_attribute("bbs_account", value)
-
- @property
- def last_ansible_hash(self):
- return self.get_attribute("last_ansible_hash", self.default_attributes["last_ansible_hash"])
-
- @last_ansible_hash.setter
- def last_ansible_hash(self, value):
- return self.set_attribute("last_ansible_hash", value)
+ @volt.setter
+ def volt(self, value):
+ raise XOSConfigurationError("VEGTenant.volt setter is not implemented")
@property
def ssh_command(self):
@@ -161,10 +94,6 @@
else:
return "no-instance"
- @ssh_command.setter
- def ssh_command(self, value):
- pass
-
def get_vrouter_field(self, name, default=None):
if self.vrouter:
return getattr(self.vrouter, name, default)
@@ -334,21 +263,6 @@
# To-do: cleanup unused instances
pass
- def manage_bbs_account(self):
- if self.deleted:
- return
-
- if self.volt and self.volt.subscriber and self.volt.subscriber.url_filter_enable:
- if not self.bbs_account:
- # make sure we use the proxied VEGService object, not the generic Service object
- veg_service = VEGService.objects.get(id=self.provider_service.id)
- self.bbs_account = veg_service.allocate_bbs_account()
- super(VEGTenant, self).save()
- else:
- if self.bbs_account:
- self.bbs_account = None
- super(VEGTenant, self).save()
-
def find_or_make_port(self, instance, network, **kwargs):
port = Port.objects.filter(instance=instance, network=network)
if port:
@@ -360,13 +274,10 @@
def get_lan_network(self, instance):
slice = self.provider_service.slices.all()[0]
- if CORD_USE_VTN:
- # there should only be one network private network, and its template should not be the management template
- lan_networks = [x for x in slice.networks.all() if x.template.visibility=="private" and (not "management" in x.template.name)]
- if len(lan_networks)>1:
- raise XOSProgrammingError("The vEG slice should only have one non-management private network")
- else:
- lan_networks = [x for x in slice.networks.all() if "lan" in x.name]
+ # there should only be one network private network, and its template should not be the management template
+ lan_networks = [x for x in slice.networks.all() if x.template.visibility == "private" and (not "management" in x.template.name)]
+ if len(lan_networks) > 1:
+ raise XOSProgrammingError("The vEG slice should only have one non-management private network")
if not lan_networks:
raise XOSProgrammingError("No lan_network")
return lan_networks[0]
@@ -409,14 +320,14 @@
# VTN-CORD needs a WAN address for the VM, so that the VM can
# be configured.
- if CORD_USE_VTN:
- tags = Tag.select_by_content_object(instance).filter(name="vm_vrouter_tenant")
- if not tags:
- vrouter = self.get_vrouter_service().get_tenant(address_pool_name="addresses_veg", subscriber_service = self.provider_service)
- vrouter.set_attribute("tenant_for_instance_id", instance.id)
- vrouter.save()
- tag = Tag(service=self.provider_service, content_object=instance, name="vm_vrouter_tenant", value="%d" % vrouter.id)
- tag.save()
+ tags = Tag.select_by_content_object(instance).filter(name="vm_vrouter_tenant")
+
+ if not tags:
+ vrouter = self.get_vrouter_service().get_tenant(address_pool_name="addresses_veg", subscriber_service=self.provider_service)
+ vrouter.set_attribute("tenant_for_instance_id", instance.id)
+ vrouter.save()
+ tag = Tag(service=self.provider_service, content_object=instance, name="vm_vrouter_tenant",value="%d" % vrouter.id)
+ tag.save()
def save(self, *args, **kwargs):
if not self.creator:
@@ -444,7 +355,6 @@
veg = veg[0]
veg.manage_container()
veg.manage_vrouter()
- veg.manage_bbs_account()
veg.cleanup_orphans()
diff --git a/xos/synchronizer/Dockerfile.synchronizer b/xos/synchronizer/Dockerfile.synchronizer
new file mode 100644
index 0000000..c1a6a6b
--- /dev/null
+++ b/xos/synchronizer/Dockerfile.synchronizer
@@ -0,0 +1,9 @@
+FROM xosproject/xos-synchronizer-base
+
+ADD . /opt/xos/synchronizers/veg
+
+ENTRYPOINT []
+
+WORKDIR "/opt/xos/synchronizers/veg"
+
+CMD bash -c "cd /opt/xos/synchronizers/veg; ./run-from-api.sh"
\ No newline at end of file
diff --git a/xos/synchronizer/Makefile b/xos/synchronizer/Makefile
new file mode 100644
index 0000000..a5bd739
--- /dev/null
+++ b/xos/synchronizer/Makefile
@@ -0,0 +1,5 @@
+synchronizer:
+ sudo docker build --rm -f Dockerfile.synchronizer -t xosproject/veg-synchronizer .
+
+up:
+ docker-compose -p veg up -d
\ No newline at end of file
diff --git a/xos/synchronizer/docker-compose.yml b/xos/synchronizer/docker-compose.yml
new file mode 100644
index 0000000..a8ff7f3
--- /dev/null
+++ b/xos/synchronizer/docker-compose.yml
@@ -0,0 +1,23 @@
+version: '2'
+
+networks:
+ xos:
+ external: true
+
+services:
+ synchronizer:
+ image: xosproject/veg-synchronizer
+ networks:
+ - xos
+ #command: bash -c "sleep 86400"
+ volumes:
+ - /opt/cord_profile/key_import/veg_rsa:/opt/xos/services/veg/keys/veg_rsa:ro
+ - /opt/cord_profile/node_key:/opt/cord_profile/node_key:ro
+ - /opt/cord/build/platform-install/credentials/xosadmin@opencord.org:/opt/xos/services/veg/credentials/xosadmin@opencord.org
+ external_links:
+ - rcordbs_xos_redis_1:redis
+ logging:
+ driver: "json-file"
+ options:
+ max-size: "1000k"
+ max-file: "5"
\ No newline at end of file
diff --git a/xos/synchronizer/manifest b/xos/synchronizer/manifest
index 959c763..4fb448b 100644
--- a/xos/synchronizer/manifest
+++ b/xos/synchronizer/manifest
@@ -47,5 +47,5 @@
model-deps
supervisor/veg-observer.conf
run.sh
-vtn_veg_synchronizer_config
-veg_stats_notifier.py
+veg_synchronizer_config
+veg_stats_notifier.py
\ No newline at end of file
diff --git a/xos/synchronizer/run-from-api.sh b/xos/synchronizer/run-from-api.sh
new file mode 100644
index 0000000..cebe5f8
--- /dev/null
+++ b/xos/synchronizer/run-from-api.sh
@@ -0,0 +1,2 @@
+export XOS_DIR=/opt/xos
+python veg-synchronizer.py -C $XOS_DIR/synchronizers/veg/veg_from_api_config
\ No newline at end of file
diff --git a/xos/synchronizer/steps/sync_monitoring_agent.yaml b/xos/synchronizer/steps/sync_monitoring_agent.yaml
index 36b7221..17dd0c0 100644
--- a/xos/synchronizer/steps/sync_monitoring_agent.yaml
+++ b/xos/synchronizer/steps/sync_monitoring_agent.yaml
@@ -2,7 +2,8 @@
- hosts: {{ instance_name }}
gather_facts: False
connection: ssh
- user: ubuntu
+ become: ubuntu
+
vars:
keystone_tenant_id: {{ keystone_tenant_id }}
keystone_user_id: {{ keystone_user_id }}
@@ -12,7 +13,7 @@
tasks:
- name: Verify if veg_stats_notifier ([] is to avoid capturing the shell process) cron job is already running
- shell: pgrep -f [v]cpe_stats_notifier | wc -l
+ shell: pgrep -f [v]veg_stats_notifier | wc -l
register: cron_job_pids_count
- name: DEBUG
diff --git a/xos/synchronizer/steps/sync_vegtenant.py b/xos/synchronizer/steps/sync_vegtenant.py
index 2b64bb1..d22b738 100644
--- a/xos/synchronizer/steps/sync_vegtenant.py
+++ b/xos/synchronizer/steps/sync_vegtenant.py
@@ -5,57 +5,36 @@
import base64
import time
from urlparse import urlparse
-from django.db.models import F, Q
from xos.config import Config
-from synchronizers.base.syncstep import SyncStep
-from synchronizers.base.ansible_helper import run_template_ssh
-from synchronizers.base.SyncInstanceUsingAnsible import SyncInstanceUsingAnsible
-from core.models import Service, Slice, Tag, ModelLink, CoarseTenant, Tenant, ServiceMonitoringAgentInfo
-from services.veg.models import VEGService, VEGTenant
+from synchronizers.new_base.SyncInstanceUsingAnsible import SyncInstanceUsingAnsible
+from synchronizers.new_base.modelaccessor import *
+from synchronizers.new_base.ansible_helper import run_template_ssh
from xos.logger import Logger, logging
-# Deal with configurations where the hpc service is not onboarded
-try:
- from services.hpc.models import HpcService, CDNPrefix
- hpc_service_onboarded=True
-except:
- hpc_service_onboarded=False
-
# hpclibrary will be in steps/..
parentdir = os.path.join(os.path.dirname(__file__),"..")
sys.path.insert(0,parentdir)
-from broadbandshield import BBS
logger = Logger(level=logging.INFO)
ENABLE_QUICK_UPDATE=False
-CORD_USE_VTN = getattr(Config(), "networking_use_vtn", False)
-
class SyncVEGTenant(SyncInstanceUsingAnsible):
provides=[VEGTenant]
observes=VEGTenant
requested_interval=0
template_name = "sync_vegtenant.yaml"
- watches = [ModelLink(CoarseTenant,via='coarsetenant'), ModelLink(ServiceMonitoringAgentInfo,via='monitoringagentinfo')]
+ watches = [ModelLink(ServiceDependency, via='servicedependency'), ModelLink(ServiceMonitoringAgentInfo, via='monitoringagentinfo')]
def __init__(self, *args, **kwargs):
super(SyncVEGTenant, self).__init__(*args, **kwargs)
- def fetch_pending(self, deleted):
- if (not deleted):
- objs = VEGTenant.get_tenant_objects().filter(Q(enacted__lt=F('updated')) | Q(enacted=None),Q(lazy_blocked=False))
- else:
- objs = VEGTenant.get_deleted_tenant_objects()
-
- return objs
-
def get_veg_service(self, o):
if not o.provider_service:
return None
- vegs = VEGService.get_service_objects().filter(id=o.provider_service.id)
+ vegs = VEGService.objects.filter(id=o.provider_service.id)
if not vegs:
return None
@@ -66,7 +45,6 @@
# object itself. In the case of vEG, we need to know:
# 1) the addresses of dnsdemux, to setup dnsmasq in the vEG
# 2) CDN prefixes, so we know what URLs to send to dnsdemux
- # 3) BroadBandShield server addresses, for parental filtering
# 4) vlan_ids, for setting up networking in the vEG VM
veg_service = self.get_veg_service(o)
@@ -83,60 +61,9 @@
if len(lines)>=2:
dnsdemux_ip = lines[0].strip()
cdn_prefixes = [x.strip() for x in lines[1:] if x.strip()]
- elif hpc_service_onboarded:
- # automatic CDN configuiration
- # it learns everything from CDN objects in XOS
- # not tested on pod.
- if veg_service.backend_network_label:
- # Connect to dnsdemux using the network specified by
- # veg_service.backend_network_label
- for service in HpcService.objects.all():
- for slice in service.slices.all():
- if "dnsdemux" in slice.name:
- for instance in slice.instances.all():
- for ns in instance.ports.all():
- if ns.ip and ns.network.labels and (veg_service.backend_network_label in ns.network.labels):
- dnsdemux_ip = ns.ip
- if not dnsdemux_ip:
- logger.info("failed to find a dnsdemux on network %s" % veg_service.backend_network_label,extra=o.tologdict())
- else:
- # Connect to dnsdemux using the instance's public address
- for service in HpcService.objects.all():
- for slice in service.slices.all():
- if "dnsdemux" in slice.name:
- for instance in slice.instances.all():
- if dnsdemux_ip=="none":
- try:
- dnsdemux_ip = socket.gethostbyname(instance.node.name)
- except:
- pass
- if not dnsdemux_ip:
- logger.info("failed to find a dnsdemux with a public address",extra=o.tologdict())
-
- for prefix in CDNPrefix.objects.all():
- cdn_prefixes.append(prefix.prefix)
dnsdemux_ip = dnsdemux_ip or "none"
- # Broadbandshield can either be set up internally, using veg_service.bbs_slice,
- # or it can be setup externally using veg_service.bbs_server.
-
- bbs_addrs = []
- if veg_service.bbs_slice:
- if veg_service.backend_network_label:
- for bbs_instance in veg_service.bbs_slice.instances.all():
- for ns in bbs_instance.ports.all():
- if ns.ip and ns.network.labels and (veg_service.backend_network_label in ns.network.labels):
- bbs_addrs.append(ns.ip)
- else:
- logger.info("unsupported configuration -- bbs_slice is set, but backend_network_label is not",extra=o.tologdict())
- if not bbs_addrs:
- logger.info("failed to find any usable addresses on bbs_slice",extra=o.tologdict())
- elif veg_service.bbs_server:
- bbs_addrs.append(veg_service.bbs_server)
- else:
- logger.info("neither bbs_slice nor bbs_server is configured in the vEG",extra=o.tologdict())
-
s_tags = []
c_tags = []
if o.volt:
@@ -167,11 +94,10 @@
fields = {"s_tags": s_tags,
"c_tags": c_tags,
"docker_remote_image_name": veg_service.docker_image_name,
- "docker_local_image_name": veg_service.docker_image_name, # veg_service.docker_image_name.split("/",1)[1].split(":",1)[0],
+ "docker_local_image_name": veg_service.docker_image_name,
"docker_opts": " ".join(docker_opts),
"dnsdemux_ip": dnsdemux_ip,
"cdn_prefixes": cdn_prefixes,
- "bbs_addrs": bbs_addrs,
"full_setup": full_setup,
"isolation": o.instance.isolation,
"safe_browsing_macs": safe_macs,
@@ -192,63 +118,6 @@
super(SyncVEGTenant, self).sync_fields(o, fields)
- # now do all of our broadbandshield stuff...
-
- service = self.get_veg_service(o)
- if not service:
- # Ansible uses the service's keypair in order to SSH into the
- # instance. It would be bad if the slice had no service.
-
- raise Exception("Slice %s is not associated with a service" % instance.slice.name)
-
- # Make sure the slice is configured properly
- if (service != o.instance.slice.service):
- raise Exception("Slice %s is associated with some service that is not %s" % (str(instance.slice), str(service)))
-
- # only enable filtering if we have a subscriber object (see below)
- url_filter_enable = False
-
- # for attributes that come from CordSubscriberRoot
- if o.volt and o.volt.subscriber:
- url_filter_enable = o.volt.subscriber.url_filter_enable
- url_filter_level = o.volt.subscriber.url_filter_level
- url_filter_users = o.volt.subscriber.devices
-
- if service.url_filter_kind == "broadbandshield":
- # disable url_filter if there are no bbs_addrs
- if url_filter_enable and (not fields.get("bbs_addrs",[])):
- logger.info("disabling url_filter because there are no bbs_addrs",extra=o.tologdict())
- url_filter_enable = False
-
- if url_filter_enable:
- bbs_hostname = None
- if service.bbs_api_hostname and service.bbs_api_port:
- bbs_hostname = service.bbs_api_hostname
- else:
- # TODO: extract from slice
- bbs_hostname = "cordcompute01.onlab.us"
-
- if service.bbs_api_port:
- bbs_port = service.bbs_api_port
- else:
- bbs_port = 8018
-
- if not bbs_hostname:
- logger.info("broadbandshield is not configured",extra=o.tologdict())
- else:
- tStart = time.time()
- bbs = BBS(o.bbs_account, "123", bbs_hostname, bbs_port)
- bbs.sync(url_filter_level, url_filter_users)
-
- if o.hpc_client_ip:
- logger.info("associate account %s with ip %s" % (o.bbs_account, o.hpc_client_ip),extra=o.tologdict())
- bbs.associate(o.hpc_client_ip)
- else:
- logger.info("no hpc_client_ip to associate",extra=o.tologdict())
-
- logger.info("bbs update time %d" % int(time.time()-tStart),extra=o.tologdict())
-
-
def run_playbook(self, o, fields):
ansible_hash = hashlib.md5(repr(sorted(fields.items()))).hexdigest()
quick_update = (o.last_ansible_hash == ansible_hash)
@@ -257,12 +126,10 @@
logger.info("quick_update triggered; skipping ansible recipe",extra=o.tologdict())
else:
if o.instance.isolation in ["container", "container_vm"]:
+ raise Exception("probably not implemented")
super(SyncVEGTenant, self).run_playbook(o, fields, "sync_vegtenant_new.yaml")
else:
- if CORD_USE_VTN:
- super(SyncVEGTenant, self).run_playbook(o, fields, template_name="sync_vegtenant_vtn.yaml")
- else:
- super(SyncVEGTenant, self).run_playbook(o, fields)
+ super(SyncVEGTenant, self).run_playbook(o, fields, template_name="sync_vegtenant_vtn.yaml")
o.last_ansible_hash = ansible_hash
@@ -305,4 +172,3 @@
template_name = "sync_monitoring_agent.yaml"
super(SyncVEGTenant, self).run_playbook(obj, fields, template_name)
- pass
diff --git a/xos/synchronizer/steps/sync_vegtenant.yaml b/xos/synchronizer/steps/sync_vegtenant.yaml
index eba2a97..59605e0 100644
--- a/xos/synchronizer/steps/sync_vegtenant.yaml
+++ b/xos/synchronizer/steps/sync_vegtenant.yaml
@@ -2,8 +2,8 @@
- hosts: {{ instance_name }}
gather_facts: False
connection: ssh
- user: ubuntu
- sudo: yes
+ become_user: ubuntu
+
vars:
cdn_enable: {{ cdn_enable }}
dnsdemux_ip: {{ dnsdemux_ip }}
@@ -99,7 +99,7 @@
dest=/etc/resolv.conf
- name: Verify if veg_stats_notifier ([] is to avoid capturing the shell process) cron job is already running
- shell: pgrep -f [v]cpe_stats_notifier | wc -l
+ shell: pgrep -f [v]veg_stats_notifier | wc -l
register: cron_job_pids_count
# - name: DEBUG
diff --git a/xos/synchronizer/steps/sync_vegtenant_new.yaml b/xos/synchronizer/steps/sync_vegtenant_new.yaml
index daa30f8..2941cae 100644
--- a/xos/synchronizer/steps/sync_vegtenant_new.yaml
+++ b/xos/synchronizer/steps/sync_vegtenant_new.yaml
@@ -2,8 +2,8 @@
- hosts: {{ instance_name }}
gather_facts: False
connection: ssh
- user: {{ username }}
- sudo: yes
+ become_user: {{ username }}
+
vars:
container_name: {{ container_name }}
cdn_enable: {{ cdn_enable }}
@@ -63,7 +63,7 @@
tasks:
- name: Verify if veg_stats_notifier ([] is to avoid capturing the shell process) cron job is already running
- shell: pgrep -f [v]cpe_stats_notifier | wc -l
+ shell: pgrep -f [v]veg_stats_notifier | wc -l
register: cron_job_pids_count
# - name: DEBUG
diff --git a/xos/synchronizer/steps/sync_vegtenant_vtn.yaml b/xos/synchronizer/steps/sync_vegtenant_vtn.yaml
index fed64ab..5a619ee 100644
--- a/xos/synchronizer/steps/sync_vegtenant_vtn.yaml
+++ b/xos/synchronizer/steps/sync_vegtenant_vtn.yaml
@@ -2,8 +2,8 @@
- hosts: {{ instance_name }}
#gather_facts: False
connection: ssh
- user: ubuntu
- sudo: yes
+ become_user: ubuntu
+
vars:
container_name: {{ container_name }}
cdn_enable: {{ cdn_enable }}
@@ -29,10 +29,6 @@
{% for prefix in cdn_prefixes %}
- {{ prefix }}
{% endfor %}
- bbs_addrs:
- {% for bbs_addr in bbs_addrs %}
- - {{ bbs_addr }}
- {% endfor %}
dns_servers:
{% for dns_server in dns_servers %}
- {{ dns_server }}
@@ -120,7 +116,7 @@
shell: touch /root/environment_is_setup
- name: Verify if veg_stats_notifier ([] is to avoid capturing the shell process) cron job is already running
- shell: pgrep -f [v]cpe_stats_notifier | wc -l
+ shell: pgrep -f [v]veg_stats_notifier | wc -l
register: cron_job_pids_count
# - name: DEBUG
diff --git a/xos/synchronizer/templates/dnsmasq_safe_servers.j2 b/xos/synchronizer/templates/dnsmasq_safe_servers.j2
index 0b3c807..0cab15a 100644
--- a/xos/synchronizer/templates/dnsmasq_safe_servers.j2
+++ b/xos/synchronizer/templates/dnsmasq_safe_servers.j2
@@ -1,4 +1,4 @@
-# This file autogenerated by vCPE observer
+# This file autogenerated by vEG observer
# It contains a list of DNS servers for dnsmasq to use.
no-resolv
diff --git a/xos/synchronizer/templates/dnsmasq_servers.j2 b/xos/synchronizer/templates/dnsmasq_servers.j2
index 7ecb319..539bd6c 100644
--- a/xos/synchronizer/templates/dnsmasq_servers.j2
+++ b/xos/synchronizer/templates/dnsmasq_servers.j2
@@ -1,4 +1,4 @@
-# This file autogenerated by vCPE observer
+# This file autogenerated by vEG observer
# It contains a list of DNS servers for dnsmasq to use.
no-resolv
diff --git a/xos/synchronizer/veg-synchronizer.py b/xos/synchronizer/veg-synchronizer.py
index 84bec4f..26b44df 100755
--- a/xos/synchronizer/veg-synchronizer.py
+++ b/xos/synchronizer/veg-synchronizer.py
@@ -5,7 +5,7 @@
import importlib
import os
import sys
-observer_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),"../../synchronizers/base")
+observer_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),"../../synchronizers/new_base")
sys.path.append(observer_path)
mod = importlib.import_module("xos-synchronizer")
mod.main()
diff --git a/xos/synchronizer/veg_from_api_config b/xos/synchronizer/veg_from_api_config
new file mode 100644
index 0000000..04e4743
--- /dev/null
+++ b/xos/synchronizer/veg_from_api_config
@@ -0,0 +1,20 @@
+# Sets options for the synchronizer
+[observer]
+name=veg
+dependency_graph=/opt/xos/synchronizers/veg/model-deps
+steps_dir=/opt/xos/synchronizers/veg/steps
+sys_dir=/opt/xos/synchronizers/veg/sys
+#logfile=/var/log/xos_backend.log
+log_file=console
+log_level=debug
+pretend=False
+backoff_disabled=True
+save_ansible_output=True
+proxy_ssh=True
+proxy_ssh_key=/opt/cord_profile/node_key
+proxy_ssh_user=root
+accessor_kind=api
+accessor_password=@/opt/xos/services/veg/credentials/xosadmin@opencord.org
+
+[networking]
+use_vtn=True
\ No newline at end of file
diff --git a/xos/synchronizer/veg_stats_notifier.py b/xos/synchronizer/veg_stats_notifier.py
index a380b58..6e0d8ca 100644
--- a/xos/synchronizer/veg_stats_notifier.py
+++ b/xos/synchronizer/veg_stats_notifier.py
@@ -138,13 +138,13 @@
rabbit_password='80608318c273f348a7c3'
rabbit_host='10.11.10.1'
vegservice_rabbit_exchange='vegservice'
-cpe_publisher_id='veg_publisher'
+veg_publisher_id='veg_publisher'
producer = None
def setup_rabbit_mq_channel():
global producer
- global rabbit_user, rabbit_password, rabbit_host, vegservice_rabbit_exchange,cpe_publisher_id
+ global rabbit_user, rabbit_password, rabbit_host, vegservice_rabbit_exchange,veg_publisher_id
vegservice_exchange = Exchange(vegservice_rabbit_exchange, "topic", durable=False)
# connections/channels
connection = BrokerConnection(rabbit_host, rabbit_user, rabbit_password)
@@ -154,23 +154,23 @@
producer = Producer(channel, exchange=vegservice_exchange, routing_key='notifications.info')
p = subprocess.Popen('hostname', shell=True, stdout=subprocess.PIPE)
(hostname, error) = p.communicate()
- cpe_publisher_id = cpe_publisher_id + '_on_' + hostname
- logger.info('cpe_publisher_id=%s',cpe_publisher_id)
+ veg_publisher_id = veg_publisher_id + '_on_' + hostname
+ logger.info('veg_publisher_id=%s',veg_publisher_id)
-def publish_cpe_stats():
+def publish_veg_stats():
global producer
- global keystone_tenant_id, keystone_user_id, cpe_publisher_id
+ global keystone_tenant_id, keystone_user_id, veg_publisher_id
- logger.debug('publish_cpe_stats invoked')
+ logger.debug('publish_veg_stats invoked')
dockercontainers = get_all_docker_containers()
- cpe_container_compute_stats = extract_compute_stats_from_all_vegs(dockercontainers)
- cpe_container_dns_stats = extract_dns_stats_from_all_vegs(dockercontainers)
+ veg_container_compute_stats = extract_compute_stats_from_all_vegs(dockercontainers)
+ veg_container_dns_stats = extract_dns_stats_from_all_vegs(dockercontainers)
- for k,v in cpe_container_dns_stats.iteritems():
+ for k,v in veg_container_dns_stats.iteritems():
msg = {'event_type': 'veg',
'message_id':six.text_type(uuid.uuid4()),
- 'publisher_id': cpe_publisher_id,
+ 'publisher_id': veg_publisher_id,
'timestamp':datetime.datetime.now().isoformat(),
'priority':'INFO',
'payload': {'veg_id':k,
@@ -196,7 +196,7 @@
compute_payload['tenant_id'] = keystone_tenant_id
msg = {'event_type': 'veg.compute.stats',
'message_id':six.text_type(uuid.uuid4()),
- 'publisher_id': cpe_publisher_id,
+ 'publisher_id': veg_publisher_id,
'timestamp':datetime.datetime.now().isoformat(),
'priority':'INFO',
'payload': compute_payload
@@ -207,7 +207,7 @@
if 'cache_size' in v:
msg = {'event_type': 'veg.dns.cache.size',
'message_id':six.text_type(uuid.uuid4()),
- 'publisher_id': cpe_publisher_id,
+ 'publisher_id': veg_publisher_id,
'timestamp':datetime.datetime.now().isoformat(),
'priority':'INFO',
'payload': {'veg_id':k,
@@ -222,7 +222,7 @@
if 'total_inserted_entries' in v:
msg = {'event_type': 'veg.dns.total_inserted_entries',
'message_id':six.text_type(uuid.uuid4()),
- 'publisher_id': cpe_publisher_id,
+ 'publisher_id': veg_publisher_id,
'timestamp':datetime.datetime.now().isoformat(),
'priority':'INFO',
'payload': {'veg_id':k,
@@ -237,7 +237,7 @@
if 'replaced_unexpired_entries' in v:
msg = {'event_type': 'veg.dns.replaced_unexpired_entries',
'message_id':six.text_type(uuid.uuid4()),
- 'publisher_id': cpe_publisher_id,
+ 'publisher_id': veg_publisher_id,
'timestamp':datetime.datetime.now().isoformat(),
'priority':'INFO',
'payload': {'veg_id':k,
@@ -252,7 +252,7 @@
if 'queries_forwarded' in v:
msg = {'event_type': 'veg.dns.queries_forwarded',
'message_id':six.text_type(uuid.uuid4()),
- 'publisher_id': cpe_publisher_id,
+ 'publisher_id': veg_publisher_id,
'timestamp':datetime.datetime.now().isoformat(),
'priority':'INFO',
'payload': {'veg_id':k,
@@ -267,7 +267,7 @@
if 'queries_answered_locally' in v:
msg = {'event_type': 'veg.dns.queries_answered_locally',
'message_id':six.text_type(uuid.uuid4()),
- 'publisher_id': cpe_publisher_id,
+ 'publisher_id': veg_publisher_id,
'timestamp':datetime.datetime.now().isoformat(),
'priority':'INFO',
'payload': {'veg_id':k,
@@ -283,7 +283,7 @@
for server in v['server_stats']:
msg = {'event_type': 'veg.dns.server.queries_sent',
'message_id':six.text_type(uuid.uuid4()),
- 'publisher_id': cpe_publisher_id,
+ 'publisher_id': veg_publisher_id,
'timestamp':datetime.datetime.now().isoformat(),
'priority':'INFO',
'payload': {'veg_id':k,
@@ -298,7 +298,7 @@
msg = {'event_type': 'veg.dns.server.queries_failed',
'message_id':six.text_type(uuid.uuid4()),
- 'publisher_id': cpe_publisher_id,
+ 'publisher_id': veg_publisher_id,
'timestamp':datetime.datetime.now().isoformat(),
'priority':'INFO',
'payload': {'veg_id':k,
@@ -312,7 +312,7 @@
logger.debug('Publishing veg.dns.server.queries_failed event: %s', msg)
def periodic_publish():
- publish_cpe_stats()
+ publish_veg_stats()
#Publish every 5minutes
threading.Timer(300, periodic_publish).start()
diff --git a/xos/synchronizer/veg_synchronizer_config b/xos/synchronizer/veg_synchronizer_config
index 0e346ef..503d8e6 100644
--- a/xos/synchronizer/veg_synchronizer_config
+++ b/xos/synchronizer/veg_synchronizer_config
@@ -7,7 +7,7 @@
name=xos
user=postgres
password=password
-host=localhost
+host=xos_db
port=5432
[api]
@@ -34,10 +34,15 @@
backoff_disabled=True
save_ansible_output=True
# set proxy_ssh to false on cloudlab
-proxy_ssh=False
full_setup=True
+proxy_ssh=True
+proxy_ssh_key=/opt/cord_profile/node_key
+proxy_ssh_user=root
enable_watchers=True
+[networking]
+use_vtn=True
+
[feefie]
client_id='vicci_dev_central'
user_id='pl'
diff --git a/xos/synchronizer/vtn_veg_synchronizer_config b/xos/synchronizer/vtn_veg_synchronizer_config
deleted file mode 100644
index 420006d..0000000
--- a/xos/synchronizer/vtn_veg_synchronizer_config
+++ /dev/null
@@ -1,47 +0,0 @@
-
-[plc]
-name=plc
-deployment=VICCI
-
-[db]
-name=xos
-user=postgres
-password=password
-host=localhost
-port=5432
-
-[api]
-host=128.112.171.237
-port=8000
-ssl_key=None
-ssl_cert=None
-ca_ssl_cert=None
-ratelimit_enabled=0
-omf_enabled=0
-mail_support_address=support@localhost
-nova_enabled=True
-
-[observer]
-name=veg
-dependency_graph=/opt/xos/synchronizers/veg/model-deps
-steps_dir=/opt/xos/synchronizers/veg/steps
-sys_dir=/opt/xos/synchronizers/veg/sys
-deleters_dir=/opt/xos/synchronizers/veg/deleters
-log_file=console
-#/var/log/hpc.log
-driver=None
-pretend=False
-backoff_disabled=True
-save_ansible_output=True
-# set proxy_ssh to false on cloudlab
-full_setup=True
-proxy_ssh=True
-proxy_ssh_key=/opt/xos/synchronizers/veg/node_key
-proxy_ssh_user=root
-
-[networking]
-use_vtn=True
-
-[feefie]
-client_id='vicci_dev_central'
-user_id='pl'
diff --git a/xos/tosca/resources/vegservice.py b/xos/tosca/resources/vegservice.py
index 85bcac7..faf2893 100644
--- a/xos/tosca/resources/vegservice.py
+++ b/xos/tosca/resources/vegservice.py
@@ -5,6 +5,6 @@
provides = "tosca.nodes.VEGService"
xos_model = VEGService
copyin_props = ["view_url", "icon_url", "enabled", "published", "public_key",
- "private_key_fn", "versionNumber", "backend_network_label",
- "dns_servers", "node_label", "docker_image_name", "docker_insecure_registry"]
+ "private_key_fn", "versionNumber", "dns_servers", "node_label",
+ "docker_image_name", "docker_insecure_registry"]