AETHER-3366 Remove OVS and simpleovs CNI from UPF
Also make connecting AiaB to external easier
Change-Id: I93e6f2cbbeb68e961e1e466923cc8b5b2a7d5d16
diff --git a/Makefile b/Makefile
index 1b3334b..0d82dee 100644
--- a/Makefile
+++ b/Makefile
@@ -20,12 +20,25 @@
TEST_APP_VALUES?= $(MAKEDIR)/5g-test-apps-values.yaml
KUBESPRAY_VERSION ?= release-2.17
-DOCKER_VERSION ?= '20.10'
-K8S_VERSION ?= v1.20.11
-HELM_VERSION ?= v3.6.3
+DOCKER_VERSION ?= '20.10'
+K8S_VERSION ?= v1.20.11
+HELM_VERSION ?= v3.6.3
+
+ENABLE_ROUTER ?= true
+ENABLE_OAISIM ?= true
+ENABLE_GNBSIM ?= true
ENABLE_SUBSCRIBER_PROXY ?= false
GNBSIM_COLORS ?= true
+DATA_IFACE ?= data
+ifeq ($(DATA_IFACE), data)
+ RAN_SUBNET := 192.168.251.0/24
+else
+ RAN_SUBNET := $(shell ip route | grep $${DATA_IFACE} | awk '/kernel/ {print $$1}')
+endif
+NODE_IP ?= $(shell ip route list default | awk -F 'src' '{ print $$2; exit }' | awk '{ print $$1 }')
+MME_IP ?=
+
HELM_GLOBAL_ARGS ?=
# Allow installing local charts or specific versions of published charts.
@@ -42,12 +55,7 @@
os_release := $(shell lsb_release -r -s)
USER := $(shell whoami)
-
-omec: $(M)/system-check $(M)/omec
-oaisim: $(M)/oaisim
-5gc: $(M)/system-check $(M)/5g-core
-
-.PHONY: omec oaisim 5gc test reset-test reset-ue 5g-core reset-5g-test clean
+.PHONY: 4g-core 5g-core oaisim test reset-test reset-ue reset-5g-test clean
$(M):
mkdir -p $(M)
@@ -107,11 +115,11 @@
-e "{'kube_version' : $(K8S_VERSION)}" \
-e "{'kube_network_plugin_multus' : True, 'multus_version' : stable, 'multus_cni_version' : 0.3.1}" \
-e "{'kube_proxy_metrics_bind_address' : 0.0.0.0:10249}" \
- -e "{'kube_pods_subnet' : 192.168.0.0/17, 'kube_service_addresses' : 192.168.128.0/17}" \
+ -e "{'kube_pods_subnet' : 192.168.84.0/24, 'kube_service_addresses' : 192.168.85.0/24}" \
-e "{'kube_apiserver_node_port_range' : 2000-36767}" \
-e "{'kubeadm_enabled': True}" \
-e "{'kube_feature_gates' : [SCTPSupport=True]}" \
- -e "{'kubelet_custom_flags' : [--allowed-unsafe-sysctls=net.*]}" \
+ -e "{'kubelet_custom_flags' : [--allowed-unsafe-sysctls=net.*, --node-ip=$(NODE_IP)]}" \
-e "{'dns_min_replicas' : 1}" \
-e "{'helm_enabled' : True, 'helm_version' : $(HELM_VERSION)}" \
cluster.yml
@@ -130,51 +138,80 @@
helm repo add aether https://charts.aetherproject.org
touch $@
-node-prep: | $(M)/k8s-ready $(M)/fabric $(M)/oaisim-lo
-
-/opt/cni/bin/simpleovs: | $(M)/k8s-ready
- sudo cp $(RESOURCEDIR)/simpleovs /opt/cni/bin/
-
/opt/cni/bin/static: | $(M)/k8s-ready
mkdir -p $(BUILD)/cni-plugins; cd $(BUILD)/cni-plugins; \
wget https://github.com/containernetworking/plugins/releases/download/v0.8.2/cni-plugins-linux-amd64-v0.8.2.tgz && \
tar xvfz cni-plugins-linux-amd64-v0.8.2.tgz
sudo cp $(BUILD)/cni-plugins/static /opt/cni/bin/
-# TODO: need to connect ONOS
-$(M)/fabric: | $(M)/setup /opt/cni/bin/simpleovs /opt/cni/bin/static
- sudo apt install -y openvswitch-switch
- sudo ovs-vsctl --may-exist add-br br-enb-net
- sudo ovs-vsctl --may-exist add-port br-enb-net enb -- set Interface enb type=internal
- sudo ip addr add 192.168.251.4/24 dev enb || true
- sudo ip link set enb up
- sudo ethtool --offload enb tx off
- sudo ip route replace 192.168.252.0/24 via 192.168.251.1 dev enb
- kubectl apply -f $(RESOURCEDIR)/router.yaml
- kubectl wait pod -n default --for=condition=Ready -l app=router --timeout=300s
- kubectl -n default exec router -- ip route add 172.250.0.0/16 via 192.168.250.3
- kubectl delete net-attach-def core-net
- touch $@
+node-prep: | $(M)/helm-ready /opt/cni/bin/static
-4g-core: | $(M)/omec
-$(M)/omec: | $(M)/helm-ready /opt/cni/bin/simpleovs /opt/cni/bin/static $(M)/fabric
- kubectl get namespace omec 2> /dev/null || kubectl create namespace omec
- helm repo update
- if [[ "${CHARTS}" == "local" || "${CHARTS}" == "local-sdcore" ]]; then helm dep up $(SD_CORE_CHART); fi
- helm upgrade --install --wait $(HELM_GLOBAL_ARGS) \
+$(M)/router-pod:
+ sudo ip link add $(DATA_IFACE) type dummy || true;
+ sudo ip link set $(DATA_IFACE) up || true;
+ DATA_IFACE=$(DATA_IFACE) envsubst < $(RESOURCEDIR)/router.yaml | kubectl apply -f -
+ kubectl wait pod -n default --for=condition=Ready -l app=router --timeout=300s
+ @touch $@
+
+$(M)/router-host:
+ sudo ip link add core link $(DATA_IFACE) type macvlan mode bridge
+ sudo ip link set core up
+ sudo ip addr add 192.168.250.1/24 dev core
+ sudo ip link add access link $(DATA_IFACE) type macvlan mode bridge
+ sudo ip link set access up
+ sudo ip addr add 192.168.252.1/24 dev access
+ sudo sysctl -w net.ipv4.ip_forward=1;
+ $(eval oiface := $(shell ip route list default | awk -F 'dev' '{ print $$2; exit }' | awk '{ print $$1 }'))
+ sudo ip route add 172.250.0.0/16 via 192.168.250.3
+ sudo iptables -t nat -A POSTROUTING -s 172.250.0.0/16 -o $(oiface) -j MASQUERADE
+ @touch $@
+
+4g-core: node-prep
+ifeq ($(ENABLE_ROUTER),true)
+ifeq ($(ENABLE_OAISIM),true)
+4g-core: $(M)/router-pod
+else
+4g-core: $(M)/router-host
+endif
+endif
+4g-core: $(M)/omec
+$(M)/omec:
+ @if [[ "${CHARTS}" == "local" || "${CHARTS}" == "local-sdcore" ]]; then \
+ helm dep up $(SD_CORE_CHART); \
+ else \
+ helm repo update; \
+ fi
+ NODE_IP=${NODE_IP} DATA_IFACE=${DATA_IFACE} RAN_SUBNET=${RAN_SUBNET} envsubst < $(4G_CORE_VALUES) | \
+ helm upgrade --create-namespace --install --wait $(HELM_GLOBAL_ARGS) \
--namespace omec \
- --values $(4G_CORE_VALUES) \
+ --values - \
sd-core \
$(SD_CORE_CHART)
- touch $@
+ @if [[ "${ENABLE_OAISIM}" == "false" ]]; then \
+ $(eval mme_ip := $(shell ip -4 -o addr show $${DATA_IFACE} | awk '{print $$4}' | cut -d'/' -f1)) \
+ echo "Your MME IP is $(mme_ip)"; \
+ fi
+ @touch $@
-$(M)/5g-core: | $(M)/helm-ready /opt/cni/bin/simpleovs /opt/cni/bin/static $(M)/fabric
- kubectl get namespace omec 2> /dev/null || kubectl create namespace omec
- helm repo update
- if [[ "${CHARTS}" == "local" || "${CHARTS}" == "local-sdcore" ]]; then helm dep up $(SD_CORE_CHART); fi
- helm upgrade --install --wait $(HELM_GLOBAL_ARGS) \
+5g-core: node-prep
+ifeq ($(ENABLE_ROUTER),true)
+ifeq ($(ENABLE_GNBSIM),true)
+5g-core: $(M)/router-pod
+else
+5g-core: $(M)/router-host
+endif
+endif
+5g-core: $(M)/5g-core
+$(M)/5g-core:
+ @if [[ "${CHARTS}" == "local" || "${CHARTS}" == "local-sdcore" ]]; then \
+ helm dep up $(SD_CORE_CHART); \
+ else \
+ helm repo update; \
+ fi
+ NODE_IP=${NODE_IP} DATA_IFACE=${DATA_IFACE} RAN_SUBNET=${RAN_SUBNET} envsubst < $(5G_CORE_VALUES) | \
+ helm upgrade --create-namespace --install --wait $(HELM_GLOBAL_ARGS) \
--namespace omec \
- --values $(5G_CORE_VALUES) \
+ --values - \
sd-core \
$(SD_CORE_CHART)
touch $@
@@ -199,27 +236,44 @@
touch $@
$(M)/oaisim-lo:
- sudo ip addr add 127.0.0.2/8 dev lo || true
- touch $@
+ @sudo ip addr add 127.0.0.2/8 dev lo || true
+ @touch $@
oaisim-standalone: | $(M)/helm-ready $(M)/ue-image $(M)/oaisim-lo
- kubectl get namespace omec 2> /dev/null || kubectl create namespace omec
- kubectl apply -f resources/busybox-sleep.yaml --namespace=omec
+ @ip link show $(DATA_IFACE) > /dev/null || (echo DATA_IFACE is not set or does not exist; exit 1)
+ @if [[ "${MME_IP}" == "" ]]; then \
+ echo MME_IP is not set; \
+ exit 1; \
+ else \
+ ping -c 3 $(MME_IP) > /dev/null || (echo MME $(MME_IP) is not reachable; exit 1) \
+ fi
+ sudo ip route add 192.168.252.0/24 via $(MME_IP)
helm repo update
- helm upgrade --install $(HELM_GLOBAL_ARGS) --namespace omec oaisim cord/oaisim -f $(OAISIM_VALUES) \
- --set images.pullPolicy=IfNotPresent
+ helm upgrade --create-namespace --install $(HELM_GLOBAL_ARGS) --namespace omec oaisim cord/oaisim -f $(OAISIM_VALUES) \
+ --set config.enb.networks.s1u.interface=$(DATA_IFACE) \
+ --set config.enb.networks.s1_mme.interface=$(DATA_IFACE) \
+ --set config.enb.mme.address=$(MME_IP) \
+ --set config.enb.mme.isLocal=false \
+ --set images.pullPolicy=IfNotPresent
kubectl rollout status -n omec statefulset ue
+ @echo "Test: registration"
@timeout 60s bash -c \
"until ip addr show oip1 | grep -q inet; \
do \
- echo 'Waiting for UE 1 gets IP address'; \
- sleep 3; \
+ echo 'Waiting for UE 1 gets IP address'; \
+ sleep 3; \
done"
- touch $(M)/oaisim $(M)/omec $(M)/fabric
+ @echo "Test: ping from UE to 8.8.8.8"
+ ping -I oip1 8.8.8.8 -c 3
+ @touch $(M)/oaisim $(M)/omec
-
-$(M)/oaisim: | $(M)/ue-image $(M)/omec $(M)/oaisim-lo
- helm upgrade --install $(HELM_GLOBAL_ARGS) --namespace omec oaisim cord/oaisim -f $(OAISIM_VALUES) \
+oaisim: | $(M)/oaisim
+$(M)/oaisim: | $(M)/ue-image $(M)/router-pod $(M)/oaisim-lo
+ sudo ip link add enb link $(DATA_IFACE) type macvlan mode bridge || true
+ sudo ip link set enb up || true
+ sudo ip addr add 192.168.251.3/24 dev enb || true
+ sudo ip route add 192.168.252.0/24 via 192.168.251.1 || true
+ helm upgrade --create-namespace --install $(HELM_GLOBAL_ARGS) --namespace omec oaisim cord/oaisim -f $(OAISIM_VALUES) \
--set images.pullPolicy=IfNotPresent
kubectl rollout status -n omec statefulset ue
@timeout 60s bash -c \
@@ -304,7 +358,30 @@
kubectl delete namespace aether-roc || true
rm -rf $(M)/roc
-test: | $(M)/fabric $(M)/omec $(M)/oaisim
+omec-clean:
+ helm delete -n omec $$(helm -n omec ls -qa) || true
+ @echo ""
+ @echo "Wait for all pods to terminate..."
+ kubectl wait -n omec --for=delete --all=true -l app!=ue pod --timeout=180s || true
+
+router-clean:
+ @kubectl delete net-attach-def router-net 2>/dev/null || true
+ @kubectl delete po router 2>/dev/null || true
+ kubectl wait --for=delete -l app=router pod --timeout=180s 2>/dev/null || true
+ sudo ip link del access || true
+ sudo ip link del core || true
+ $(eval oiface := $(shell ip route list default | awk -F 'dev' '{ print $$2; exit }' | awk '{ print $$1 }'))
+ sudo iptables -t nat -D POSTROUTING -s 172.250.0.0/16 -o $(oiface) -j MASQUERADE || true
+ @sudo ip link del data 2>/dev/null || true
+ @cd $(M); rm -f router-pod router-host
+
+oaisim-clean: reset-ue
+ @sudo ip addr del 127.0.0.2/8 dev lo 2>/dev/null || true
+ @sudo ip link del enb 2>/dev/null || true
+ @sudo ip route del 192.168.252.0/24 || true
+ @cd $(M); rm -f oaisim-lo
+
+test: | 4g-core $(M)/oaisim
@sleep 5
@echo "Test1: ping from UE to SGI network gateway"
ping -I oip1 192.168.250.1 -c 15
@@ -314,7 +391,7 @@
ping -I oip1 google.com -c 3
@echo "Finished to test"
-5g-test: | $(M)/5g-core
+5g-test: | 5g-core
@if [[ "${CHARTS}" == "release-1.6" ]]; then echo "[NOTE] 5G Test not supported for Aether 1.6, exiting..."; exit 1; fi
@echo "Test: Registration + UE initiated PDU Session Establishment + User Data packets"
@sleep 60
@@ -326,15 +403,8 @@
fi
@grep -q "Simulation Result: PASS" /tmp/gnbsim.out
-cleanup-omec:
- helm delete -n omec $$(helm -n omec ls -qa) || true
- @echo ""
- @echo "Wait for all pods to terminate..."
- kubectl wait -n omec --for=delete --all=true -l app!=ue pod --timeout=180s || true
-
-reset-test: cleanup-omec
- kubectl delete po router || true
- cd $(M); rm -f oaisim omec fabric
+reset-test: | oaisim-clean omec-clean router-clean
+ @cd $(M); rm -f omec oaisim 5g-core
reset-ue:
helm delete -n omec oaisim || true
@@ -342,7 +412,7 @@
kubectl wait -n omec --for=delete pod ue-0 || true
cd $(M); rm -f oaisim
-reset-5g-test: cleanup-omec
+reset-5g-test: omec-clean
cd $(M); rm -f 5g-core
reset-dbtestapp:
@@ -358,12 +428,7 @@
$(5G_TEST_APPS_CHART)
@echo "Finished to dbtestapp"
-clean:
- kubectl delete po router || true
- kubectl delete net-attach-def core-net || true
- sudo ovs-vsctl del-br br-access-net || true
- sudo ovs-vsctl del-br br-core-net || true
- sudo apt remove --purge openvswitch-switch -y
+clean: | oaisim-clean router-clean
source "$(VENV)/bin/activate" && cd $(BUILD)/kubespray; \
ansible-playbook -b -i inventory/local/hosts.ini reset.yml --extra-vars "reset_confirmation=yes"
@if [ -d /usr/local/etc/emulab ]; then \
diff --git a/oaisim-values.yaml b/oaisim-values.yaml
index f225b5f..31ee551 100644
--- a/oaisim-values.yaml
+++ b/oaisim-values.yaml
@@ -5,7 +5,7 @@
config:
enb:
mme:
- address: 192.168.251.4
+ address: 192.168.251.3
networks:
s1u:
interface: enb
diff --git a/resources/busybox-sleep.yaml b/resources/busybox-sleep.yaml
deleted file mode 100644
index 0e80705..0000000
--- a/resources/busybox-sleep.yaml
+++ /dev/null
@@ -1,19 +0,0 @@
-# Copyright 2022-present Open Networking Foundation
-#
-# SPDX-License-Identifier: Apache-2.0
-
-# Create a pod with app: mme label so that oaisim will start
-
-apiVersion: v1
-kind: Pod
-metadata:
- name: busybox-sleep
- labels:
- app: mme
-spec:
- containers:
- - name: busybox
- image: busybox
- args:
- - sleep
- - "1000000"
diff --git a/resources/router.yaml b/resources/router.yaml
index 8520ad8..dfba708 100644
--- a/resources/router.yaml
+++ b/resources/router.yaml
@@ -6,37 +6,12 @@
apiVersion: "k8s.cni.cncf.io/v1"
kind: NetworkAttachmentDefinition
metadata:
- name: core-net
+ name: router-net
spec:
config: '{
"cniVersion": "0.3.1",
- "type": "simpleovs",
- "ipam": {
- "type": "static"
- }
- }'
----
-apiVersion: "k8s.cni.cncf.io/v1"
-kind: NetworkAttachmentDefinition
-metadata:
- name: access-net
-spec:
- config: '{
- "cniVersion": "0.3.1",
- "type": "simpleovs",
- "ipam": {
- "type": "static"
- }
- }'
----
-apiVersion: "k8s.cni.cncf.io/v1"
-kind: NetworkAttachmentDefinition
-metadata:
- name: enb-net
-spec:
- config: '{
- "cniVersion": "0.3.1",
- "type": "simpleovs",
+ "type": "macvlan",
+ "master": "${DATA_IFACE}",
"ipam": {
"type": "static"
}
@@ -50,18 +25,19 @@
app: router
annotations:
k8s.v1.cni.cncf.io/networks: '[
- { "name": "core-net", "interface": "core-rtr", "ips": ["192.168.250.1/24"] },
- { "name": "enb-net", "interface": "enb-rtr", "ips": ["192.168.251.1/24"] },
- { "name": "access-net", "interface": "access-rtr", "ips": ["192.168.252.1/24"] }
+ { "name": "router-net", "interface": "core-gw", "ips": ["192.168.250.1/24"] },
+ { "name": "router-net", "interface": "ran-gw", "ips": ["192.168.251.1/24"] },
+ { "name": "router-net", "interface": "access-gw", "ips": ["192.168.252.1/24"] }
]'
spec:
containers:
- - name: quagga
+ - name: router
command: ["/bin/bash", "-c"]
args:
- >
sysctl -w net.ipv4.ip_forward=1;
iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE;
+ ip route add 172.250.0.0/16 via 192.168.250.3;
trap : TERM INT; sleep infinity & wait
image: opencord/quagga
securityContext:
@@ -69,5 +45,3 @@
capabilities:
add:
- NET_ADMIN
- ports:
- - containerPort: 2601
diff --git a/resources/simpleovs b/resources/simpleovs
deleted file mode 100755
index 007c582..0000000
--- a/resources/simpleovs
+++ /dev/null
@@ -1,67 +0,0 @@
-#!/bin/bash -x
-#
-# Copyright 2019-present Open Networking Foundation
-#
-# SPDX-License-Identifier: Apache-2.0
-
-set -o errexit
-set -o pipefail
-set -o nounset
-
-exec 3>&1
-exec &>>/var/log/simple-ovs-cni.log
-
-PATH="$CNI_PATH:$(dirname "${BASH_SOURCE[0]}"):$PATH"
-CNI_CONF=$(cat /dev/stdin)
-
-get_bridge() {
- echo "br-$(echo $CNI_CONF | jq -r '.name')"
-}
-
-ipam() {
- local plugin=$(echo $CNI_CONF | jq -r '.ipam.type')
- local res=$(echo $"$CNI_CONF" | "$plugin" | jq -c '.')
- echo $res
-}
-
-add_br_and_port() {
- mkdir -p /var/run/netns/
- ln -sfT $CNI_NETNS /var/run/netns/$CNI_CONTAINERID
-
- local bridge=$(get_bridge)
- local ip=$1
-
- ovs-vsctl --may-exist add-br $bridge
- ovs-vsctl add-port $bridge $CNI_IFNAME -- set Interface $CNI_IFNAME type=internal
-
- ip link set $CNI_IFNAME netns $CNI_CONTAINERID
- ip netns exec $CNI_CONTAINERID ip addr add $ip dev $CNI_IFNAME
- ip netns exec $CNI_CONTAINERID ip link set $CNI_IFNAME up
-}
-
-delete_br_and_port() {
- local bridge="br-$(echo $CNI_CONF | jq -r '.name')"
- ovs-vsctl del-port $CNI_IFNAME
- if [ -z "$(ovs-vsctl list-ports $bridge)" ]; then
- ovs-vsctl del-br $bridge
- fi
-}
-
-case $CNI_COMMAND in
-ADD)
- res=$(ipam)
- ip=$(echo $res | jq -r '.ips[0].address')
- add_br_and_port $ip
- echo '{"cniVersion":"0.3.1"}' | jq -c --arg ip $ip '.ips[0].address = $ip' >&3
- ;;
-DEL)
- set +o errexit
- ipam
- delete_br_and_port
- set -o errexit
- ;;
-*)
- echo "CNI_COMMAND=[ADD|DEL] only supported"
- exit 1
- ;;
-esac
diff --git a/sd-core-4g-values.yaml b/sd-core-4g-values.yaml
index 756d544..0c5b257 100644
--- a/sd-core-4g-values.yaml
+++ b/sd-core-4g-values.yaml
@@ -111,24 +111,27 @@
repository: "registry.opennetworking.org/docker.io/"
resources:
enabled: false
-
config:
upf:
- name: "oaisim"
sriov:
enabled: false
hugepage:
enabled: false
- cniPlugin: simpleovs
+ cniPlugin: macvlan
ipam: static
+ routes:
+ - to: ${NODE_IP}
+ via: 169.254.1.1
+ enb:
+ subnet: ${RAN_SUBNET}
+ access:
+ iface: ${DATA_IFACE}
+ core:
+ iface: ${DATA_IFACE}
cfgFiles:
upf.json:
mode: af_packet
- cpiface:
- dnn: "internet"
- hostname: "upf"
- enable_ue_ip_alloc: false
- ue_ip_pool: 172.250.0.0/16
+ hwcksum: true
5g-control-plane:
enable5G: false
diff --git a/sd-core-5g-values.yaml b/sd-core-5g-values.yaml
index 60fc8ac..fae56ab 100644
--- a/sd-core-5g-values.yaml
+++ b/sd-core-5g-values.yaml
@@ -133,7 +133,6 @@
enable: true
resources:
enabled: false
-
config:
upf:
name: "oaisim"
@@ -141,18 +140,26 @@
enabled: false
hugepage:
enabled: false
- cniPlugin: simpleovs
+ cniPlugin: macvlan
ipam: static
+ routes:
+ - to: ${NODE_IP}
+ via: 169.254.1.1
+ enb:
+ subnet: ${RAN_SUBNET}
+ access:
+ iface: ${DATA_IFACE}
+ core:
+ iface: ${DATA_IFACE}
cfgFiles:
upf.json:
+ mode: af_packet
+ hwcksum: true
log_level: "trace"
gtppsc: true
- mode: af_packet
cpiface:
dnn: "internet"
hostname: "upf"
- enable_ue_ip_alloc: false
- ue_ip_pool: 172.250.0.0/16
5g-ran-sim:
enable: true