Enable the additon of UPF(s)

Change-Id: I33e6ab2b005acb03c2dc328fb26d1e988a8eb5f2
diff --git a/Makefile b/Makefile
index b85de94..bd54a7a 100644
--- a/Makefile
+++ b/Makefile
@@ -13,12 +13,14 @@
 
 4G_CORE_VALUES       ?= $(MAKEDIR)/sd-core-4g-values.yaml
 5G_CORE_VALUES       ?= $(MAKEDIR)/sd-core-5g-values.yaml
+5G_UPF_VALUES        ?= $(MAKEDIR)/upf-5g-values.yaml
 OAISIM_VALUES        ?= $(MAKEDIR)/oaisim-values.yaml
 ROC_VALUES           ?= $(MAKEDIR)/roc-values.yaml
 ROC_DEFAULTENT_MODEL ?= $(MAKEDIR)/roc-defaultent-model.json
 ROC_4G_MODELS        ?= $(MAKEDIR)/roc-4g-models.json
 ROC_5G_MODELS        ?= $(MAKEDIR)/roc-5g-models.json
 TEST_APP_VALUES      ?= $(MAKEDIR)/5g-test-apps-values.yaml
+UPF_COUNT             = $(MAKEDIR)/upf-count.txt
 GET_HELM              = get_helm.sh
 
 KUBESPRAY_VERSION ?= release-2.17
@@ -76,6 +78,12 @@
 
 HELM_GLOBAL_ARGS ?=
 
+ifneq ("$(wildcard $(UPF_COUNT))","")
+  UPF_NUMBER = $(shell cat $(UPF_COUNT))
+else
+  UPF_NUMBER = 0
+endif
+
 # Allow installing local charts or specific versions of published charts.
 # E.g., to install the Aether 1.5 release:
 #    CHARTS=release-1.5 make test
@@ -343,8 +351,27 @@
 		--values - \
 		sd-core \
 		$(SD_CORE_CHART)
+	@echo "1" > ${UPF_COUNT}
 	touch $@
 
+# Install additional UPF(s)
+5g-upf:
+	@if [[ $(UPF_NUMBER) -lt 1 ]]; then \
+	      echo "Deploy '5g-core' before adding additional UPF(s)"; \
+				exit 1; \
+	fi
+	$(eval IP_ID=$(shell echo $$((3+$(UPF_NUMBER)))))
+	$(eval IP_UE_ID=$(shell echo $$((250-$(UPF_NUMBER)))))
+	NODE_IP=${NODE_IP} DATA_IFACE=${DATA_IFACE} RAN_SUBNET=${RAN_SUBNET} IP_ID=${IP_ID} IP_UE_ID=${IP_UE_ID} envsubst < $(5G_UPF_VALUES) | \
+	helm upgrade --create-namespace --install --wait $(HELM_GLOBAL_ARGS) \
+		--namespace upf-${UPF_NUMBER} \
+		--values - \
+		bess-upf \
+		$(UPF_CHART)
+	kubectl -n default exec -ti router -- ip route add 172.$(IP_UE_ID).0.0/16 via 192.168.250.$(IP_ID)
+	$(eval UPF_NUMBER=$(shell echo $$(($(UPF_NUMBER)+1))))
+	@echo $(UPF_NUMBER) > ${UPF_COUNT}
+
 # UE images includes kernel module, ue_ip.ko
 # which should be built in the exactly same kernel version of the host machine
 $(BUILD)/openairinterface: | $(M)/setup
@@ -582,8 +609,26 @@
 	kubectl delete namespace cattle-dashboards cattle-monitoring-system || true
 	rm $(M)/monitoring
 
-omec-clean:
+upf-clean:
+	$(eval TO_REMOVE=$(shell echo $$(($(UPF_NUMBER)-1))))
+	@echo Removing $(TO_REMOVE) additional UPFs
+	@number=$(UPF_NUMBER) ; \
+	while [[ $$number -gt 1 ]] ; do \
+					((number = number - 1)) ; \
+					((addr = 250 - number)) ; \
+					helm delete -n upf-$$number $$(helm -n upf-$$number ls -qa) || true ; \
+					echo "Wait for all pods to terminate..." ; \
+					kubectl wait -n upf-$$number --for=delete --all=true -l app!=ue pod --timeout=180s || true ; \
+					kubectl delete namespace upf-$$number || true ; \
+					kubectl -n default exec -ti router -- ip route delete 172.$$addr.0.0/16 || true ; \
+					echo "" ; \
+	done
+	@echo "1" > ${UPF_COUNT}
+
+omec-clean: upf-clean
 	helm delete -n omec $$(helm -n omec ls -qa) || true
+	kubectl delete namespace omec || true
+	@rm -f $(UPF_COUNT)
 	@echo ""
 	@echo "Wait for all pods to terminate..."
 	kubectl wait -n omec --for=delete --all=true -l app!=ue pod --timeout=180s || true
diff --git a/configs/latest b/configs/latest
index 93361a1..082e0e1 100644
--- a/configs/latest
+++ b/configs/latest
@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 SD_CORE_CHART            := aether/sd-core
+UPF_CHART                := aether/bess-upf
 
 # For installing the ROC
 AETHER_ROC_UMBRELLA_CHART := aether/aether-roc-umbrella
diff --git a/sd-core-5g-values.yaml b/sd-core-5g-values.yaml
index 77eb1bc..b60f3b0 100644
--- a/sd-core-5g-values.yaml
+++ b/sd-core-5g-values.yaml
@@ -254,8 +254,10 @@
         subnet: ${RAN_SUBNET} #this is your gNB network
       access:
         iface: ${DATA_IFACE}
+        # ip: 192.168.252.3/24   # this is the default in the Helm chart
       core:
         iface: ${DATA_IFACE}
+        # ip: 192.168.250.3/24   # this is the default in the Helm chart
       cfgFiles:
         upf.json:
           mode: af_packet  #this mode means no dpdk
@@ -282,6 +284,9 @@
     #  gnbsim: 5gc-gnbsim:0.0.1-dev-local5
   config:
     gnbsim:
+      networkTopo:
+        - upfAddr: "192.168.252.0/24"
+          upfGw: "192.168.251.1"
       singleInterface: false #default multiInterface. Works well for AIAB
       execInParallel: false #run all profiles in parallel
       goProfile:
diff --git a/upf-5g-values.yaml b/upf-5g-values.yaml
new file mode 100644
index 0000000..fcc1a81
--- /dev/null
+++ b/upf-5g-values.yaml
@@ -0,0 +1,46 @@
+# Copyright 2023-present Open Networking Foundation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+enable: true
+resources:
+  enabled: false
+images:
+  repository: "registry.opennetworking.org/docker.io/"
+  # uncomment below section to add update bess image tag
+  #tags:
+  #  bess: <bess image tag>
+  #  pfcpiface: <pfcp image tag>
+config:
+  upf:
+    name: "oaisim"
+    sriov:
+      enabled: false #default sriov is disabled in AIAB setup
+    hugepage:
+      enabled: false #should be enabled if dpdk is enabled
+    #can be any other plugin as well, remember this plugin dictates how IP address are assigned.
+    cniPlugin: macvlan
+    ipam: static
+    routes:
+      - to: ${NODE_IP}
+        via: 169.254.1.1
+    enb:
+      subnet: ${RAN_SUBNET} #this is your gNB network
+    access:
+      iface: ${DATA_IFACE}
+      ip: 192.168.252.${IP_ID}/24
+    core:
+      iface: ${DATA_IFACE}
+      ip: 192.168.250.${IP_ID}/24
+    cfgFiles:
+      upf.json:
+        mode: af_packet #this mode means no dpdk
+        hwcksum: true
+        log_level: "trace"
+        gtppsc: true #extension header is enabled in 5G. Sending QFI in pdu session extension header
+        cpiface:
+          dnn: "internet" #keep it matching with Slice dnn
+          hostname: "upf"
+          #http_port: "8080"
+          enable_ue_ip_alloc: false # if true then it means UPF allocates address from below pool
+          ue_ip_pool: "172.${IP_UE_ID}.0.0/16" # UE ip pool is used if enable_ue_ip_alloc is set to true