Configure the nat interface automatically
- Refactor nbhelper
- Generate device's netplan config from netbox data
This change is for [AETHER-1371]
Change-Id: Ieed77850c7405c2634bfa3b78bd2d9086b8f837b
diff --git a/scripts/base_edgeconfig.yaml b/scripts/base_edgeconfig.yaml
index e98a648..64322f0 100644
--- a/scripts/base_edgeconfig.yaml
+++ b/scripts/base_edgeconfig.yaml
@@ -1,8 +1,6 @@
---
# this is copied into every edgeconfig
-netprep_nat_if: "eno1"
-netprep_internal_if: "eno2"
netprep_router: true
netprep_netplan_file: "02-pronto"
diff --git a/scripts/edgeconfig.py b/scripts/edgeconfig.py
index 5dc21fc..2faf9de 100644
--- a/scripts/edgeconfig.py
+++ b/scripts/edgeconfig.py
@@ -33,8 +33,8 @@
},
}
- args = nbhelper.parse_cli_args(extra_args)
- nbh = nbhelper.NBHelper(args)
+ args = nbhelper.initialize(extra_args)
+ tenant = nbhelper.NBTenant()
# use base_config for additional items
yaml_out = yaml.safe_load(args.base_config.read())
@@ -44,15 +44,11 @@
# reverse zones aggregate across RFC1918 IP prefix
dns_reverse_zones = nbhelper.NBDNSReverseZones()
-
- for prefix in nbh.all_prefixes():
+ for prefix in tenant.get_prefixes().values():
nbhelper.NBDNSForwardZone.get_fwd_zone(prefix)
-
dns_reverse_zones.add_prefix(prefix)
-
dhcpd_subnet = nbhelper.NBDHCPSubnet(prefix)
-
dhcpd_if = dhcpd_subnet.dhcpd_interface
if dhcpd_if and dhcpd_if not in dhcpd_interfaces:
@@ -60,7 +56,8 @@
dhcpd_subnets.append(dhcpd_subnet)
- # yaml_out["devices"] = nbhelper.NBDevice.all_devs()
+ # yaml_out["devices"] = nbhelper.NBDevice.all_devs()
+ yaml_out["netprep_netplan"] = tenant.generate_netplan()
yaml_out["dns_forward_zones"] = nbhelper.NBDNSForwardZone.all_fwd_zones()
yaml_out["dns_reverse_zones"] = dns_reverse_zones
yaml_out["dhcpd_subnets"] = dhcpd_subnets
diff --git a/scripts/nbhelper.py b/scripts/nbhelper.py
index 75195e9..8b540dd 100644
--- a/scripts/nbhelper.py
+++ b/scripts/nbhelper.py
@@ -47,6 +47,35 @@
pynetbox.core.api.Api, yaml.SafeRepresenter.represent_none
)
+netboxapi = None
+netbox_config = None
+netbox_version = None
+
+
+def initialize(extra_args):
+ global netboxapi, netbox_config, netbox_version
+
+ args = parse_cli_args(extra_args)
+ netbox_config = yaml.safe_load(args.settings.read())
+
+ for require_args in ["api_endpoint", "token", "tenant_name"]:
+ if not netbox_config.get(require_args):
+ logger.error("The require argument: %s was not set. Stop." % require_args)
+ sys.exit(1)
+
+ netboxapi = pynetbox.api(
+ netbox_config["api_endpoint"], token=netbox_config["token"], threading=True,
+ )
+
+ if not netbox_config.get("validate_certs", True):
+ session = requests.Session()
+ session.verify = False
+ netboxapi.http_session = session
+
+ netbox_version = netboxapi.version
+
+ return args
+
def parse_cli_args(extra_args={}):
"""
@@ -66,79 +95,95 @@
"--debug", action="store_true", help="Print additional debugging information"
)
- if extra_args:
- for ename, ekwargs in extra_args.items():
- parser.add_argument(ename, **ekwargs)
+ for ename, ekwargs in extra_args.items():
+ parser.add_argument(ename, **ekwargs)
args = parser.parse_args()
-
- # only print log messages if debugging
- if args.debug:
- logger.setLevel(logging.DEBUG)
- else:
- logger.setLevel(logging.INFO)
+ log_level = logging.DEBUG if args.debug else logging.INFO
+ logger.setLevel(log_level)
return args
+def check_name_dns(name):
+
+ badchars = re.search("[^a-z0-9.-]", name.lower(), re.ASCII)
+
+ if badchars:
+ logger.error(
+ "DNS name '%s' has one or more invalid characters: '%s'",
+ name,
+ badchars.group(0),
+ )
+ sys.exit(1)
+
+ return name.lower()
+
+
+def clean_name_dns(name):
+ return re.sub("[^a-z0-9.-]", "-", name.lower(), 0, re.ASCII)
+
+
class AttrDict(dict):
def __init__(self, *args, **kwargs):
super(AttrDict, self).__init__(*args, **kwargs)
self.__dict__ = self
-class NBHelper:
- def __init__(self, args):
+class NBTenant:
+ def __init__(self):
+ self.name = netbox_config["tenant_name"]
+ self.name_segments = netbox_config.get("prefix_segments", 1)
+ self.tenant = netboxapi.tenancy.tenants.get(name=self.name)
- self.settings = yaml.safe_load(args.settings.read())
+ # NBTenant only keep the resources which owns by it
+ self.devices = list()
+ self.vms = list()
+ self.prefixes = dict()
- self.nbapi = pynetbox.api(
- self.settings["api_endpoint"], token=self.settings["token"], threading=True,
- )
+ # Get the Device and Virtual Machines from Netbox API
+ for device_data in netboxapi.dcim.devices.filter(tenant=self.tenant.slug):
+ self.devices.append(NBDevice(device_data))
- if not self.settings["validate_certs"]:
+ for vm_data in netboxapi.virtualization.virtual_machines.filter(
+ tenant=self.tenant.slug
+ ):
+ self.vms.append(NBVirtualMachine(vm_data))
- session = requests.Session()
- session.verify = False
- self.nbapi.http_session = session
+ def get_prefixes(self):
+ """Get the IP Prefixes owns by current tenant"""
- self.nb_version = self.nbapi.version
+ if self.prefixes:
+ return self.prefixes
- def all_prefixes(self):
+ vrf = netboxapi.ipam.vrfs.get(tenant=self.tenant.slug)
+ for prefix_data in netboxapi.ipam.prefixes.filter(vrf_id=vrf.id):
+ if prefix_data.description:
+ self.prefixes[prefix_data.description] = NBPrefix(
+ prefix_data, self.name_segments
+ )
+
+ return self.prefixes
+
+ def generate_netplan(self, name=""):
"""
- Return a list of prefix objects
+ Get the interface config of specific server belongs to this tenant,
+ If the name wasn't specified, return the management device config by default
"""
- p_items = []
+ target = None
- segments = 1
-
- if "prefix_segments" in self.settings:
- segments = self.settings["prefix_segments"]
-
- for prefix in self.settings["ip_prefixes"]:
- p_items.append(NBPrefix.get_prefix(self.nbapi, prefix, segments))
-
- return p_items
-
- @classmethod
- def check_name_dns(cls, name):
-
- badchars = re.search("[^a-z0-9.-]", name.lower(), re.ASCII)
-
- if badchars:
- logger.error(
- "DNS name '%s' has one or more invalid characters: '%s'",
- name,
- badchars.group(0),
+ if not name:
+ target = next(
+ filter(
+ lambda m: m.data["device_role"]["name"] == "Router",
+ self.devices + self.vms,
+ )
)
- sys.exit(1)
+ else:
+ target = next(filter(lambda m: m.name == name, self.devices + self.vms))
- return name.lower()
-
- @classmethod
- def clean_name_dns(cls, name):
- return re.sub("[^a-z0-9.-]", "-", name.lower(), 0, re.ASCII)
+ return target.generate_netplan()
@yaml.yaml_object(ydump)
@@ -146,21 +191,16 @@
prefixes = {}
- def __init__(self, api, prefix, name_segments):
-
- self.nbapi = api
- self.prefix = prefix
+ def __init__(self, data, name_segments):
+ self.data = data
self.name_segments = name_segments
-
- # get prefix information
- self.prefix_data = self.nbapi.ipam.prefixes.get(prefix=self.prefix)
- self.domain_extension = NBHelper.check_name_dns(self.prefix_data.description)
+ self.domain_extension = check_name_dns(self.data.description)
logger.debug(
- "prefix %s, domain_extension %s, data: %s",
- self.prefix,
+ "Preix %s: domain_extension %s, data: %s",
+ self.data.prefix,
self.domain_extension,
- dict(self.prefix_data),
+ dict(self.data),
)
# ip centric info
@@ -176,14 +216,18 @@
return cls.prefixes
@classmethod
- def get_prefix(cls, api, prefix, name_segments=1):
+ def get_prefix(cls, prefix, name_segments=1):
if prefix in cls.prefixes:
return cls.prefixes[prefix]
- return NBPrefix(api, prefix, name_segments)
+ data = netboxapi.ipam.prefixes.get(prefix=prefix)
+ if data:
+ return NBPrefix(data, name_segments)
+ else:
+ raise Exception("The prefix %s wasn't found in Netbox" % prefix)
def __repr__(self):
- return str(self.prefix)
+ return str(self.data.prefix)
@classmethod
def to_yaml(cls, representer, node):
@@ -204,16 +248,16 @@
"""
# get all parents of this prefix (include self)
- possible_parents = self.nbapi.ipam.prefixes.filter(contains=self.prefix)
+ possible_parents = netboxapi.ipam.prefixes.filter(contains=self.data.prefix)
- logger.debug("Prefix %s: possible parents %s", self.prefix, possible_parents)
+ logger.debug(
+ "Prefix %s: possible parents %s", self.data.prefix, possible_parents
+ )
# filter out self, return first found
for pparent in possible_parents:
- if pparent.prefix != self.prefix:
- return NBPrefix.get_prefix(
- self.nbapi, pparent.prefix, self.name_segments
- )
+ if pparent.prefix != self.data.prefix:
+ return NBPrefix.get_prefix(pparent.prefix, self.name_segments)
return None
@@ -222,7 +266,7 @@
find ip information for items (devices/vms, reserved_ips, dhcp_range) in prefix
"""
- ips = self.nbapi.ipam.ip_addresses.filter(parent=self.prefix)
+ ips = netboxapi.ipam.ip_addresses.filter(parent=self.data.prefix)
for ip in sorted(ips, key=lambda k: k["address"]):
@@ -248,22 +292,17 @@
# devices and VMs
if ip.assigned_object: # can be null if not assigned to a device/vm
aotype = ip.assigned_object_type
-
if aotype == "dcim.interface":
-
- self.aos[str(ip)] = NBDevice.get_dev(
- self.nbapi, ip.assigned_object.device.id,
+ self.aos[str(ip)] = NBDevice.get_by_id(
+ ip.assigned_object.device.id,
)
-
elif aotype == "virtualization.vminterface":
- self.aos[str(ip)] = NBVirtualMachine.get_vm(
- self.nbapi, ip.assigned_object.virtual_machine.id,
+ self.aos[str(ip)] = NBVirtualMachine.get_by_id(
+ ip.assigned_object.virtual_machine.id,
)
-
else:
logger.error("IP %s has unknown device type: %s", ip, aotype)
sys.exit(1)
-
else:
logger.warning("Unknown IP type %s, with attributes: %s", ip, dict(ip))
@@ -277,8 +316,56 @@
This parent class holds common functions for those two child classes
"""
- def __init__(self, api):
- self.nbapi = api
+ objects = dict()
+
+ def __init__(self, data):
+ self.data = data
+
+ # The AssignedObject attributes
+ self.id = data.id
+ self.name = data.name
+ self.ips = dict()
+
+ # The NetBox objects related with this AssignedObject
+ self.services = None
+ self.interfaces = list()
+ self.mgmt_interfaces = list()
+ self.interfaces_by_ip = dict()
+
+ if self.__class__ == NBDevice:
+ self.interfaces = netboxapi.dcim.interfaces.filter(device_id=self.id)
+ self.services = netboxapi.ipam.services.filter(device_id=self.id)
+ ip_addresses = netboxapi.ipam.ip_addresses.filter(device_id=self.id)
+ elif self.__class__ == NBVirtualMachine:
+ self.interfaces = netboxapi.virtualization.interfaces.filter(
+ virtual_machine_id=self.id
+ )
+ self.services = netboxapi.ipam.services.filter(virtual_machine_id=self.id)
+ ip_addresses = netboxapi.ipam.ip_addresses.filter(
+ virtual_machine_id=self.id
+ )
+
+ for ip in ip_addresses:
+ self.ips[ip.address] = ip
+ if ip.assigned_object and self.__class__ == NBDevice:
+ self.interfaces_by_ip[ip.address] = netboxapi.dcim.interfaces.get(
+ ip.assigned_object_id
+ )
+ elif ip.assigned_object and self.__class__ == NBVirtualMachine:
+ self.interfaces_by_ip[
+ ip.address
+ ] = netboxapi.virtualization.interfaces.get(ip.assigned_object_id)
+ self.interfaces_by_ip[ip.address].mgmt_only = False
+
+ logger.debug(
+ "%s id: %d, data: %s, ips: %s"
+ % (self.type, self.id, dict(self.data), self.ips)
+ )
+
+ self.netplan_config = dict()
+
+ def __repr__(self):
+ return str(dict(self.data))
def dns_name(self, ip, prefix):
"""
@@ -296,7 +383,7 @@
return name
# clean/split the device name
- name_split = NBHelper.clean_name_dns(self.data.name).split(".")
+ name_split = clean_name_dns(self.data.name).split(".")
# always add interface suffix to mgmt interfaces
if self.interfaces_by_ip[ip].mgmt_only:
@@ -377,11 +464,109 @@
Returns the interface data for the device that has the primary_ip
"""
- if self.data["primary_ip"]:
- return self.interfaces_by_ip[self.data["primary_ip"]["address"]]
+ if self.data.primary_ip:
+ return self.interfaces_by_ip[self.data.primary_ip.address]
return None
+ @property
+ def type(self):
+ return "AssignedObject"
+
+ @classmethod
+ def get_by_id(cls, obj_id):
+ raise Exception("not implemented")
+
+ @classmethod
+ def all_objects(cls):
+ return cls.objects
+
+ @classmethod
+ def to_yaml(cls, representer, node):
+ return representer.represent_dict(
+ {
+ "data": node.data,
+ "services": node.services,
+ "ips": node.ips,
+ "interfaces_by_ip": node.interfaces_by_ip,
+ }
+ )
+
+ def generate_netplan(self):
+ """
+ Get the interface config of specific server belongs to this tenant
+ """
+
+ if self.netplan_config:
+ return self.netplan_config
+
+ if not self.data:
+ logger.error(
+ "{type} {name} doesn't have data yet.".format(
+ type=self.type, name=self.name
+ )
+ )
+ sys.exit(1)
+
+ primary_ip = self.data.primary_ip.address if self.data.primary_ip else None
+ primary_if = self.interfaces_by_ip[primary_ip] if primary_ip else None
+ physical_ifs = filter(
+ lambda i: str(i.type) != "Virtual"
+ and i != primary_if
+ and i.mgmt_only == False,
+ self.interfaces,
+ )
+
+ self.netplan_config["ethernets"] = dict()
+
+ if self.data.device_role.name == "Router":
+ for address, interface in self.interfaces_by_ip.items():
+ if interface.mgmt_only == True or str(interface.type) == "Virtual":
+ continue
+
+ self.netplan_config["ethernets"].setdefault(interface.name, {})
+ self.netplan_config["ethernets"][interface.name].setdefault(
+ "addresses", []
+ ).append(address)
+
+ elif self.data.device_role.name == "Server":
+ if primary_if:
+ self.netplan_config["ethernets"][primary_if.name] = {
+ "dhcp4": "yes",
+ "dhcp4-overrides": {"route-metric": 100},
+ }
+
+ for physical_if in physical_ifs:
+ self.netplan_config["ethernets"][physical_if.name] = {
+ "dhcp4": "yes",
+ "dhcp4-overrides": {"route-metric": 200},
+ }
+ else:
+ # Exclude the device type which is not Router and Server
+ return None
+
+ # Get interfaces own by AssignedObject and is virtual (VLAN interface)
+ virtual_ifs = filter(lambda i: str(i.type) == "Virtual", self.interfaces)
+
+ for virtual_if in virtual_ifs:
+ if "vlans" not in self.netplan_config:
+ self.netplan_config["vlans"] = dict()
+
+ # vlan_object_id is the "id" on netbox, it's different from known VLAN ID
+ vlan_object_id = virtual_if.tagged_vlans[0].id
+ vlan_object = netboxapi.ipam.vlans.get(vlan_object_id)
+ virtual_if_ips = netboxapi.ipam.ip_addresses.filter(
+ interface_id=virtual_if.id
+ )
+
+ self.netplan_config["vlans"][virtual_if.name] = {
+ "id": vlan_object.vid,
+ "link": virtual_if.label,
+ "addresses": [ip.address for ip in virtual_if_ips],
+ }
+
+ return self.netplan_config
+
@yaml.yaml_object(ydump)
class NBDevice(NBAssignedObject):
@@ -390,69 +575,29 @@
Also caches all known devices in a class variable (devs)
"""
- devs = {}
+ objects = dict()
- def __init__(self, api, dev_id):
+ def __init__(self, data):
- super().__init__(api)
+ super().__init__(data)
+ self.objects[self.id] = self
- self.id = dev_id
- self.data = self.nbapi.dcim.devices.get(dev_id)
- self.services = self.nbapi.ipam.services.filter(device_id=dev_id)
-
- # not filled in unless specifically asked for (expensive for a 48 port switch)
- self.interfaces = []
- self.mgmt_interfaces = []
-
- # look up all IP's for this device
- self.ips = {
- str(ip): ip for ip in self.nbapi.ipam.ip_addresses.filter(device_id=dev_id)
- }
-
- # look up interfaces by IP
- self.interfaces_by_ip = {}
- for ip, ip_data in self.ips.items():
- if ip_data.assigned_object:
- self.interfaces_by_ip[ip] = self.nbapi.dcim.interfaces.get(
- ip_data.assigned_object_id
- )
-
- logger.debug(
- "NBDevice id: %d, data: %s, ips: %s", self.id, dict(self.data), self.ips,
- )
-
- self.devs[dev_id] = self
-
- def __repr__(self):
- return str(dict(self.data))
+ @property
+ def type(self):
+ return "NBDevice"
def get_interfaces(self):
if not self.interfaces:
- self.interfaces = self.nbapi.dcim.interfaces.filter(device_id=self.id)
+ self.interfaces = netboxapi.dcim.interfaces.filter(device_id=self.id)
return self.interfaces
@classmethod
- def get_dev(cls, api, dev_id):
- if dev_id in cls.devs:
- return cls.devs[dev_id]
+ def get_by_id(cls, obj_id):
+ obj = cls.objects.get(obj_id, None)
+ obj = obj or NBDevice(netboxapi.dcim.devices.get(obj_id))
- return NBDevice(api, dev_id)
-
- @classmethod
- def all_devs(cls):
- return cls.devs
-
- @classmethod
- def to_yaml(cls, representer, node):
- return representer.represent_dict(
- {
- "data": node.data,
- "services": node.services,
- "ips": node.ips,
- "interfaces_by_ip": node.interfaces_by_ip,
- }
- )
+ return obj
@yaml.yaml_object(ydump)
@@ -461,77 +606,34 @@
VM equivalent of NBDevice
"""
- vms = {}
+ objects = dict()
- def __init__(self, api, vm_id):
+ def __init__(self, data):
- super().__init__(api)
+ super().__init__(data)
+ self.objects[self.id] = self
- self.id = vm_id
- self.data = self.nbapi.virtualization.virtual_machines.get(vm_id)
- self.services = self.nbapi.ipam.services.filter(virtual_machine_id=vm_id)
-
- # not filled in unless specifically asked for
- self.interfaces = []
-
- # look up all IP's for this device
- self.ips = {
- str(ip): ip
- for ip in self.nbapi.ipam.ip_addresses.filter(virtual_machine_id=vm_id)
- }
-
- # look up interfaces by IP
- self.interfaces_by_ip = {}
- for ip, ip_data in self.ips.items():
- if ip_data.assigned_object:
- self.interfaces_by_ip[ip] = self.nbapi.virtualization.interfaces.get(
- ip_data.assigned_object_id
- )
- # hack as VM interfaces lack this key, and needed for services
- self.interfaces_by_ip[ip].mgmt_only = False
-
- logger.debug(
- "NBVirtualMachine id: %d, data: %s, ips: %s",
- self.id,
- dict(self.data),
- self.ips,
- )
-
- self.vms[vm_id] = self
-
- def __repr__(self):
- return str(dict(self.data))
+ @property
+ def type(self):
+ return "NBVirtualMachine"
def get_interfaces(self):
if not self.interfaces:
- self.interfaces = self.nbapi.virtualization.interfaces.filter(
+ self.interfaces = netboxapi.virtualization.interfaces.filter(
virtual_machine_id=self.id
)
return self.interfaces
@classmethod
- def get_vm(cls, api, vm_id):
- if vm_id in cls.vms:
- return cls.vms[vm_id]
-
- return NBVirtualMachine(api, vm_id)
-
- @classmethod
- def all_vms(cls):
- return cls.vms
-
- @classmethod
- def to_yaml(cls, representer, node):
- return representer.represent_dict(
- {
- "data": node.data,
- "services": node.services,
- "ips": node.ips,
- "interfaces_by_ip": node.interfaces_by_ip,
- }
+ def get_by_id(cls, obj_id):
+ obj = cls.objects.get(obj_id, None)
+ obj = obj or NBVirtualMachine(
+ netboxapi.virtualization.virtual_machines.get(obj_id)
)
+ return obj
+
@yaml.yaml_object(ydump)
class NBDNSForwardZone: