blob: 6b4792baa04c6b2fc0354a7d260aa09bed57963f [file] [log] [blame]
Jeremy Ronquillo5263c732020-10-13 09:42:19 -07001# Copyright 2018-present Open Networking Foundation
2#
3# SPDX-License-Identifier: LicenseRef-ONF-Member-Only-1.0
4
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -07005SHELL := /bin/bash
6BUILD ?= /tmp/build
7M ?= $(BUILD)/milestones
8MAKEDIR := $(dir $(realpath $(firstword $(MAKEFILE_LIST))))
Jeremy Ronquilloaf084f32020-08-24 13:18:47 -07009SCRIPTDIR := $(MAKEDIR)/scripts
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -070010RESOURCEDIR := $(MAKEDIR)/resources
11WORKSPACE ?= $(HOME)
Jeremy Ronquillod3cab742020-08-24 11:49:00 -070012VENV ?= $(BUILD)/venv/aiab
13AIABVALUES ?= $(MAKEDIR)/aether-in-a-box-values.yaml
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -070014
Hyunsun84f0f172020-09-23 15:40:08 -050015KUBESPRAY_VERSION ?= release-2.14
16DOCKER_VERSION ?= 19.03
17K8S_VERSION ?= v1.18.9
Scott Baker17b70d12020-09-29 23:26:21 -070018HELM_VERSION ?= v3.2.4
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -070019
20# used to start logging/monitoring and other infrastructure charts
21INFRA_CHARTS ?=
22INFRA_PREREQS = $(foreach chart,$(INFRA_CHARTS),$(M)/$(chart))
23
24KAFKA_CHART_VERSION ?= 0.13.3
25KAFKA_POD := "pod/cord-kafka-0"
26
27HELM_GLOBAL_ARGS ?=
28HELM_NEM_ARGS ?= $(HELM_GLOBAL_ARGS)
29HELM_ONOS_ARGS ?= $(HELM_GLOBAL_ARGS)
30
31cpu_family := $(shell lscpu | grep 'CPU family:' | awk '{print $$3}')
32cpu_model := $(shell lscpu | grep 'Model:' | awk '{print $$2}')
33os_vendor := $(shell lsb_release -i -s)
34os_release := $(shell lsb_release -r -s)
35
Badhrinath3e081e22020-12-02 15:02:08 -060036omec: $(M)/system-check $(M)/omec
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -070037oaisim: $(M)/oaisim
Badhrinath3e081e22020-12-02 15:02:08 -0600385gc: $(M)/system-check $(M)/5g-core
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -070039
Badhrinath3e081e22020-12-02 15:02:08 -060040.PHONY: omec oaisim 5gc test reset-test 5g-core reset-5g-test clean
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -070041
42$(M):
43 mkdir -p $(M)
44
45$(M)/system-check: | $(M)
46 @if [[ $(cpu_family) -eq 6 ]]; then \
47 if [[ $(cpu_model) -lt 60 ]]; then \
48 echo "FATAL: haswell CPU or newer is required."; \
49 exit 1; \
50 fi \
51 else \
52 echo "FATAL: unsupported CPU family."; \
53 exit 1; \
54 fi
55 @if [[ $(os_vendor) =~ (Ubuntu) ]]; then \
Hyunsun7b640512020-10-27 19:49:51 -050056 if [[ ! $(os_release) =~ (18.04) ]]; then \
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -070057 echo "WARN: $(os_vendor) $(os_release) has not been tested."; \
58 fi; \
59 if dpkg --compare-versions 4.15 gt $(shell uname -r); then \
60 echo "FATAL: kernel 4.15 or later is required."; \
61 echo "Please upgrade your kernel by running" \
62 "apt install --install-recommends linux-generic-hwe-$(os_release)"; \
63 exit 1; \
64 fi \
65 else \
66 echo "FAIL: unsupported OS."; \
67 exit 1; \
68 fi
Hyunsun7b640512020-10-27 19:49:51 -050069 @if [[ ! -d "$(WORKSPACE)/cord/aether-helm-charts" ]]; then \
70 echo "FATAL: Please clone aether-helm-charts under $(WORKSPACE)/cord directory."; \
71 exit 1; \
72 fi
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -070073 touch $@
74
75$(M)/setup: | $(M)
76 sudo $(SCRIPTDIR)/cloudlab-disksetup.sh
Hyunsun Moon82e47c22021-06-28 12:30:20 -050077 sudo apt update; sudo apt install -y software-properties-common python3 python3-pip python3-venv jq httpie ipvsadm
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -070078 touch $@
79
80$(BUILD)/kubespray: | $(M)/setup
81 mkdir -p $(BUILD)
82 cd $(BUILD); git clone https://github.com/kubernetes-incubator/kubespray.git -b $(KUBESPRAY_VERSION)
83
84$(VENV)/bin/activate: | $(M)/setup
Hyunsun Moon82e47c22021-06-28 12:30:20 -050085 python3 -m venv $(VENV)
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -070086
87$(M)/kubespray-requirements: $(BUILD)/kubespray | $(VENV)/bin/activate
88 source "$(VENV)/bin/activate" && \
89 pip install -r $(BUILD)/kubespray/requirements.txt
90 touch $@
91
92$(M)/k8s-ready: | $(M)/setup $(BUILD)/kubespray $(VENV)/bin/activate $(M)/kubespray-requirements
93 source "$(VENV)/bin/activate" && cd $(BUILD)/kubespray; \
94 ansible-playbook -b -i inventory/local/hosts.ini \
95 -e "{'override_system_hostname' : False, 'disable_swap' : True}" \
96 -e "{'docker_version' : $(DOCKER_VERSION)}" \
97 -e "{'docker_iptables_enabled' : True}" \
98 -e "{'kube_version' : $(K8S_VERSION)}" \
Hyunsun84f0f172020-09-23 15:40:08 -050099 -e "{'kube_network_plugin_multus' : True, 'multus_version' : stable, 'multus_cni_version' : 0.3.1}" \
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -0700100 -e "{'kube_proxy_metrics_bind_address' : 0.0.0.0:10249}" \
101 -e "{'kube_pods_subnet' : 192.168.0.0/17, 'kube_service_addresses' : 192.168.128.0/17}" \
102 -e "{'kube_apiserver_node_port_range' : 2000-36767}" \
103 -e "{'kubeadm_enabled': True}" \
104 -e "{'kube_feature_gates' : [SCTPSupport=True]}" \
105 -e "{'kubelet_custom_flags' : [--allowed-unsafe-sysctls=net.*]}" \
106 -e "{'dns_min_replicas' : 1}" \
107 -e "{'helm_enabled' : True, 'helm_version' : $(HELM_VERSION)}" \
108 cluster.yml
109 mkdir -p $(HOME)/.kube
110 sudo cp -f /etc/kubernetes/admin.conf $(HOME)/.kube/config
111 sudo chown $(shell id -u):$(shell id -g) $(HOME)/.kube/config
112 kubectl wait pod -n kube-system --for=condition=Ready --all
113 touch $@
114
115$(M)/helm-ready: | $(M)/k8s-ready
Woojoong Kim8425cb32021-01-08 16:18:53 -0800116 helm repo add incubator https://charts.helm.sh/incubator
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -0700117 helm repo add cord https://charts.opencord.org
118 touch $@
119
120/opt/cni/bin/simpleovs: | $(M)/k8s-ready
121 sudo cp $(RESOURCEDIR)/simpleovs /opt/cni/bin/
122
123/opt/cni/bin/static: | $(M)/k8s-ready
124 mkdir -p $(BUILD)/cni-plugins; cd $(BUILD)/cni-plugins; \
125 wget https://github.com/containernetworking/plugins/releases/download/v0.8.2/cni-plugins-linux-amd64-v0.8.2.tgz && \
126 tar xvfz cni-plugins-linux-amd64-v0.8.2.tgz
127 sudo cp $(BUILD)/cni-plugins/static /opt/cni/bin/
128
129# TODO: need to connect ONOS
130$(M)/fabric: | $(M)/setup /opt/cni/bin/simpleovs /opt/cni/bin/static
131 sudo apt install -y openvswitch-switch
132 sudo ovs-vsctl --may-exist add-br br-enb-net
133 sudo ovs-vsctl --may-exist add-port br-enb-net enb -- set Interface enb type=internal
134 sudo ip addr add 192.168.251.4/24 dev enb || true
135 sudo ip link set enb up
136 sudo ethtool --offload enb tx off
137 sudo ip route replace 192.168.252.0/24 via 192.168.251.1 dev enb
138 kubectl apply -f $(RESOURCEDIR)/router.yaml
139 kubectl wait pod -n default --for=condition=Ready -l app=router --timeout=300s
Jeremy Ronquillob2b64ea2020-12-09 13:49:03 -0800140 kubectl -n default exec router ip route add 172.250.0.0/16 via 192.168.250.3
Jeremy Ronquilloaf084f32020-08-24 13:18:47 -0700141 kubectl delete net-attach-def core-net
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -0700142 touch $@
143
144$(M)/omec: | $(M)/helm-ready /opt/cni/bin/simpleovs /opt/cni/bin/static $(M)/fabric
Scott Baker17b70d12020-09-29 23:26:21 -0700145 kubectl get namespace omec 2> /dev/null || kubectl create namespace omec
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -0700146 helm repo update
Hyunsun7b640512020-10-27 19:49:51 -0500147 helm dep up $(WORKSPACE)/cord/aether-helm-charts/omec/omec-control-plane
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -0700148 helm upgrade --install $(HELM_GLOBAL_ARGS) \
149 --namespace omec \
Jeremy Ronquillod3cab742020-08-24 11:49:00 -0700150 --values $(AIABVALUES) \
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -0700151 omec-control-plane \
Jeremy Ronquilloaf084f32020-08-24 13:18:47 -0700152 $(WORKSPACE)/cord/aether-helm-charts/omec/omec-control-plane && \
Hyunsun7b640512020-10-27 19:49:51 -0500153 kubectl wait pod -n omec --for=condition=Ready -l release=omec-control-plane --timeout=300s && \
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -0700154 helm upgrade --install $(HELM_GLOBAL_ARGS) \
155 --namespace omec \
Jeremy Ronquillod3cab742020-08-24 11:49:00 -0700156 --values $(AIABVALUES) \
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -0700157 omec-user-plane \
Jeremy Ronquilloaf084f32020-08-24 13:18:47 -0700158 $(WORKSPACE)/cord/aether-helm-charts/omec/omec-user-plane && \
Hyunsun7b640512020-10-27 19:49:51 -0500159 kubectl wait pod -n omec --for=condition=Ready -l release=omec-user-plane --timeout=300s
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -0700160 touch $@
161
Badhrinath3e081e22020-12-02 15:02:08 -0600162$(M)/5g-core: | $(M)/helm-ready /opt/cni/bin/simpleovs /opt/cni/bin/static $(M)/fabric
163 kubectl get namespace omec 2> /dev/null || kubectl create namespace omec
164 helm repo update
165 helm dep up $(WORKSPACE)/cord/aether-helm-charts/omec/5g-control-plane
166 helm upgrade --install $(HELM_GLOBAL_ARGS) \
167 --namespace omec \
168 --values $(AIABVALUES) \
Ajay Lotan Thakur980823b2021-06-23 18:55:05 -0500169 sim-app \
170 $(WORKSPACE)/cord/aether-helm-charts/omec/omec-sub-provision && \
171 kubectl wait pod -n omec --for=condition=Ready -l release=sim-app --timeout=300s
172 helm upgrade --install $(HELM_GLOBAL_ARGS) \
173 --namespace omec \
174 --values $(AIABVALUES) \
Badhrinath3e081e22020-12-02 15:02:08 -0600175 5g-core-up \
176 $(WORKSPACE)/cord/aether-helm-charts/omec/omec-user-plane && \
177 kubectl wait pod -n omec --for=condition=Ready -l release=5g-core-up --timeout=300s
178 helm upgrade --install $(HELM_GLOBAL_ARGS) \
179 --namespace omec \
180 --values $(AIABVALUES) \
181 fgc-core \
182 $(WORKSPACE)/cord/aether-helm-charts/omec/5g-control-plane && \
183 kubectl wait pod -n omec --for=condition=Ready -l release=fgc-core --timeout=300s && \
184 helm upgrade --install $(HELM_GLOBAL_ARGS) \
185 --namespace omec \
Ajay Lotan Thakurdcc602e2021-05-20 11:16:26 -0600186 --values $(AIABVALUES) \
Badhrinath3e081e22020-12-02 15:02:08 -0600187 5g-ransim-plane \
188 $(WORKSPACE)/cord/aether-helm-charts/omec/5g-ran-sim && \
189 kubectl wait pod -n omec --for=condition=Ready -l release=5g-ransim-plane --timeout=300s
190 touch $@
191
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -0700192# UE images includes kernel module, ue_ip.ko
193# which should be built in the exactly same kernel version of the host machine
194$(BUILD)/openairinterface: | $(M)/setup
195 mkdir -p $(BUILD)
196 cd $(BUILD); git clone https://github.com/opencord/openairinterface.git
197
198$(M)/ue-image: | $(M)/k8s-ready $(BUILD)/openairinterface
199 cd $(BUILD)/openairinterface; \
200 sudo docker build . --target lte-uesoftmodem \
Hyunsun7b640512020-10-27 19:49:51 -0500201 --build-arg build_base=omecproject/oai-base:1.1.0 \
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -0700202 --file Dockerfile.ue \
Hyunsun7b640512020-10-27 19:49:51 -0500203 --tag omecproject/lte-uesoftmodem:1.1.0
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -0700204 touch $@
205
206$(M)/oaisim: | $(M)/ue-image $(M)/omec
207 sudo ip addr add 127.0.0.2/8 dev lo || true
208 $(eval mme_iface=$(shell ip -4 route list default | awk -F 'dev' '{ print $$2; exit }' | awk '{ print $$1 }'))
Hyunsun7b640512020-10-27 19:49:51 -0500209 helm upgrade --install $(HELM_GLOBAL_ARGS) --namespace omec oaisim cord/oaisim -f $(AIABVALUES) \
Jeremy Ronquilloaf084f32020-08-24 13:18:47 -0700210 --set config.enb.networks.s1_mme.interface=$(mme_iface) \
211 --set images.pullPolicy=IfNotPresent
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -0700212 kubectl rollout status -n omec statefulset ue
213 @timeout 60s bash -c \
214 "until ip addr show oip1 | grep -q inet; \
215 do \
216 echo 'Waiting for UE 1 gets IP address'; \
217 sleep 3; \
218 done"
219 touch $@
220
221test: | $(M)/fabric $(M)/omec $(M)/oaisim
222 @sleep 5
223 @echo "Test1: ping from UE to SGI network gateway"
Hyunsun Moon0bd8cec2020-09-28 00:38:26 -0500224 ping -I oip1 192.168.250.1 -c 15
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -0700225 @echo "Test2: ping from UE to 8.8.8.8"
226 ping -I oip1 8.8.8.8 -c 3
Hyunsun Moon0bd8cec2020-09-28 00:38:26 -0500227 @echo "Test3: ping from UE to google.com"
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -0700228 ping -I oip1 google.com -c 3
229 @echo "Finished to test"
230
231reset-test:
Scott Baker17b70d12020-09-29 23:26:21 -0700232 helm delete -n omec oaisim || true
233 helm delete -n omec omec-control-plane || true
234 helm delete -n omec omec-user-plane || true
Jeremy Ronquillob2b64ea2020-12-09 13:49:03 -0800235 kubectl delete po router || true
236 cd $(M); rm -f oaisim omec fabric
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -0700237
Badhrinath3e081e22020-12-02 15:02:08 -0600238reset-5g-test:
Ajay Lotan Thakur980823b2021-06-23 18:55:05 -0500239 helm uninstall -n omec sim-app || true
Badhrinath3e081e22020-12-02 15:02:08 -0600240 helm uninstall -n omec fgc-core || true
241 helm uninstall -n omec 5g-core-up || true
242 helm uninstall -n omec 5g-ransim-plane || true
243 helm uninstall -n omec mongo || true
244 cd $(M); rm -f 5g-core
245
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -0700246clean: reset-test
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -0700247 kubectl delete po router || true
Jeremy Ronquilloaf084f32020-08-24 13:18:47 -0700248 kubectl delete net-attach-def core-net || true
249 sudo ovs-vsctl del-br br-access-net || true
250 sudo ovs-vsctl del-br br-core-net || true
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -0700251 sudo apt remove --purge openvswitch-switch -y
252 source "$(VENV)/bin/activate" && cd $(BUILD)/kubespray; \
253 ansible-playbook -b -i inventory/local/hosts.ini reset.yml
254 @if [ -d /usr/local/etc/emulab ]; then \
255 mount | grep /mnt/extra/kubelet/pods | cut -d" " -f3 | sudo xargs umount; \
256 sudo rm -rf /mnt/extra/kubelet; \
257 fi
258 rm -rf $(M)