blob: 6b5cc44b91cc9de2f0242f90c2bdc4b8487241ce [file] [log] [blame]
Hyunsun Moon81c8e232019-05-21 03:40:22 -06001SHELL := /bin/bash
2BUILD ?= /tmp/build
3M ?= $(BUILD)/milestones
4MAKEDIR := $(dir $(realpath $(firstword $(MAKEFILE_LIST))))
5SCRIPTDIR := $(MAKEDIR)../scripts
6RESOURCEDIR := $(MAKEDIR)/resources
7WORKSPACE ?= $(HOME)
8VENV ?= $(BUILD)/venv/ciab
9CIABVALUES ?= $(MAKEDIR)/comac-in-a-box-values.yaml
10
Hyunsun Moon07decd42019-08-19 13:59:52 -060011KUBESPRAY_VERSION ?= release-2.11
12DOCKER_VERSION ?= 18.09
13K8S_VERSION ?= v1.15.3
14HELM_VERSION ?= v2.14.2
Hyunsun Moon81c8e232019-05-21 03:40:22 -060015
16# used to start logging/monitoring and other infrastructure charts
17INFRA_CHARTS ?=
18INFRA_PREREQS = $(foreach chart,$(INFRA_CHARTS),$(M)/$(chart))
19
20KAFKA_CHART_VERSION ?= 0.13.3
21KAFKA_POD := "pod/cord-kafka-0"
22
Hyunsun Moon81c8e232019-05-21 03:40:22 -060023HELM_GLOBAL_ARGS ?=
24HELM_NEM_ARGS ?= $(HELM_GLOBAL_ARGS)
25HELM_ONOS_ARGS ?= $(HELM_GLOBAL_ARGS)
26
27cpu_family := $(shell lscpu | grep 'CPU family:' | awk '{print $$3}')
28cpu_model := $(shell lscpu | grep 'Model:' | awk '{print $$2}')
29os_vendor := $(shell lsb_release -i -s)
Hyunsun Moonad70fc12019-08-02 16:52:19 -060030os_release := $(shell lsb_release -r -s)
Hyunsun Moon81c8e232019-05-21 03:40:22 -060031
Doyoung Lee90efcf12019-09-07 13:18:35 -070032ciab: $(M)/system-check $(M)/omec $(M)/oaisim $(M)/comac-platform
Hyunsun Moon81c8e232019-05-21 03:40:22 -060033
Hyunsun Moon235d47a2019-09-03 15:47:30 -060034.PHONY: ciab test reset-test clean
Hyunsun Moon81c8e232019-05-21 03:40:22 -060035
36$(M):
37 mkdir -p $(M)
38
Hyunsun Moon07decd42019-08-19 13:59:52 -060039$(M)/system-check: | $(M)
Hyunsun Moon81c8e232019-05-21 03:40:22 -060040 @if [[ $(cpu_family) -eq 6 ]]; then \
41 if [[ $(cpu_model) -lt 60 ]]; then \
42 echo "FATAL: haswell CPU or newer is required."; \
43 exit 1; \
44 fi \
45 else \
46 echo "FATAL: unsupported CPU family."; \
47 exit 1; \
48 fi
49 @if [[ $(os_vendor) =~ (Ubuntu) ]]; then \
Hyunsun Moonad70fc12019-08-02 16:52:19 -060050 if [[ ! $(os_release) =~ (16.04) ]]; then \
51 echo "WARN: $(os_vendor) $(os_release) has not been tested."; \
52 fi; \
53 if dpkg --compare-versions 4.15 gt $(shell uname -r); then \
54 echo "FATAL: kernel 4.15 or later is required."; \
55 echo "Please upgrade your kernel by running" \
56 "apt install --install-recommends linux-generic-hwe-$(os_release)"; \
57 exit 1; \
Hyunsun Moon81c8e232019-05-21 03:40:22 -060058 fi \
59 else \
60 echo "FAIL: unsupported OS."; \
61 exit 1; \
62 fi
63 touch $@
64
65$(M)/setup: | $(M)
66 sudo $(SCRIPTDIR)/cloudlab-disksetup.sh
67 sudo apt update; sudo apt install -y software-properties-common python-pip jq httpie ipvsadm
68 touch $@
69
70$(BUILD)/kubespray: | $(M)/setup
71 mkdir -p $(BUILD)
72 cd $(BUILD); git clone https://github.com/kubernetes-incubator/kubespray.git -b $(KUBESPRAY_VERSION)
73
74$(VENV)/bin/activate: | $(M)/setup
75 sudo pip install virtualenv
76 virtualenv $(VENV) --no-site-packages
77
78$(M)/kubespray-requirements: $(BUILD)/kubespray | $(VENV)/bin/activate
79 source "$(VENV)/bin/activate" && \
80 pip install -r $(BUILD)/kubespray/requirements.txt
81 touch $@
82
83$(M)/k8s-ready: | $(M)/setup $(BUILD)/kubespray $(VENV)/bin/activate $(M)/kubespray-requirements
84 source "$(VENV)/bin/activate" && cd $(BUILD)/kubespray; \
85 ansible-playbook -b -i inventory/local/hosts.ini \
86 -e "{'override_system_hostname' : False, 'disable_swap' : True}" \
87 -e "{'docker_version' : $(DOCKER_VERSION)}" \
88 -e "{'docker_iptables_enabled' : True}" \
89 -e "{'kube_version' : $(K8S_VERSION)}" \
Hyunsun Moon5d179d42019-08-12 14:18:19 -050090 -e "{'kube_network_plugin_multus' : True, 'multus_version' : stable}" \
Hyunsun Moon81c8e232019-05-21 03:40:22 -060091 -e "{'kube_proxy_mode': iptables}" \
92 -e "{'kube_pods_subnet' : 192.168.0.0/17, 'kube_service_addresses' : 192.168.128.0/17}" \
93 -e "{'kube_apiserver_node_port_range' : 2000-36767}" \
94 -e "{'kubeadm_enabled': True}" \
95 -e "{'kube_feature_gates' : [SCTPSupport=True]}" \
96 -e "{'kubelet_custom_flags' : [--allowed-unsafe-sysctls=net.*]}" \
97 -e "{'dns_min_replicas' : 1}" \
98 -e "{'helm_enabled' : True, 'helm_version' : $(HELM_VERSION)}" \
99 cluster.yml
100 mkdir -p $(HOME)/.kube
101 sudo cp -f /etc/kubernetes/admin.conf $(HOME)/.kube/config
102 sudo chown $(shell id -u):$(shell id -g) $(HOME)/.kube/config
103 kubectl wait pod -n kube-system --for=condition=Ready --all
104 touch $@
105
106$(M)/helm-ready: | $(M)/k8s-ready
107 helm init --wait --client-only
108 helm repo add incubator https://kubernetes-charts-incubator.storage.googleapis.com/
109 helm repo add cord https://charts.opencord.org
110 touch $@
111
112$(WORKSPACE)/cord/helm-charts: | $(M)/setup
113 mkdir -p $(WORKSPACE)/cord
114 cd $(WORKSPACE)/cord; git clone https://gerrit.opencord.org/helm-charts
115
Doyoung Lee9b0973f2019-08-20 14:47:59 -0700116$(WORKSPACE)/cord/cord-platform: | $(M)/setup
117 mkdir -p $(WORKSPACE)/cord
118 cd $(WORKSPACE)/cord; git clone https://gerrit.opencord.org/cord-platform
119
120$(WORKSPACE)/cord/comac-helm-charts: | $(M)/setup
121 mkdir -p $(WORKSPACE)/cord
122 cd $(WORKSPACE)/cord; git clone https://gerrit.opencord.org/comac-helm-charts
123
124$(M)/cord-platform: | $(M)/helm-ready $(WORKSPACE)/cord/cord-platform
125 cd $(WORKSPACE)/cord/cord-platform && \
126 helm dep update cord-platform && \
Doyoung Leec7c4b5d2019-09-05 20:53:19 -0700127 helm upgrade --install $(HELM_GLOBAL_ARGS) cord-platform cord-platform -f $(CIABVALUES)
Doyoung Lee9b0973f2019-08-20 14:47:59 -0700128 touch $@
129
130$(M)/comac-platform: | $(M)/helm-ready $(WORKSPACE)/cord/comac-helm-charts $(M)/cord-platform
131 cd $(WORKSPACE)/cord/comac-helm-charts && \
132 helm dep update comac-platform && \
Doyoung Leec7c4b5d2019-09-05 20:53:19 -0700133 helm upgrade --install $(HELM_GLOBAL_ARGS) comac-platform comac-platform -f $(CIABVALUES)
Doyoung Lee9b0973f2019-08-20 14:47:59 -0700134 touch $@
135
Hyunsun Moon81c8e232019-05-21 03:40:22 -0600136/opt/cni/bin/simpleovs: | $(M)/k8s-ready
137 sudo cp $(RESOURCEDIR)/simpleovs /opt/cni/bin/
138
Hyunsun Moon07decd42019-08-19 13:59:52 -0600139/opt/cni/bin/static: | $(M)/k8s-ready
140 mkdir -p $(BUILD)/cni-plugins; cd $(BUILD)/cni-plugins; \
141 wget https://github.com/containernetworking/plugins/releases/download/v0.8.2/cni-plugins-linux-amd64-v0.8.2.tgz && \
142 tar xvfz cni-plugins-linux-amd64-v0.8.2.tgz
143 sudo cp $(BUILD)/cni-plugins/static /opt/cni/bin/
144
Hyunsun Moon89886f12019-09-05 22:16:09 -0600145# TODO: need to connect ONOS
146$(M)/fabric: | $(M)/setup /opt/cni/bin/simpleovs
147 sudo apt install -y openvswitch-switch
148 sudo ovs-vsctl --may-exist add-br br-s1u-net
149 sudo ovs-vsctl --may-exist add-port br-s1u-net s1u-enb -- set Interface s1u-enb type=internal
150 sudo ip addr add 119.0.0.4/24 dev s1u-enb || true
151 sudo ip link set s1u-enb up
Hyunsun Moon89886f12019-09-05 22:16:09 -0600152 kubectl apply -f $(RESOURCEDIR)/router.yaml
Doyoung Lee1bb116d2019-09-09 17:29:24 -0700153 kubectl wait pod -n default --for=condition=Ready -l app=router --timeout=300s
Hyunsun Moonb9d3f482019-09-07 16:38:28 -0600154 kubectl -n default exec router ip route add 16.0.0.0/8 via 13.1.1.3
Hyunsun Moon89886f12019-09-05 22:16:09 -0600155 kubectl delete net-attach-def sgi-net
156 touch $@
157
158$(M)/omec: | $(M)/helm-ready $(WORKSPACE)/cord/helm-charts /opt/cni/bin/simpleovs /opt/cni/bin/static $(M)/fabric
Hyunsun Moon07decd42019-08-19 13:59:52 -0600159 cd $(WORKSPACE)/cord/helm-charts/omec; \
160 helm dep up omec-data-plane && \
161 helm dep up omec-control-plane && \
Hyunsun Moonb9d3f482019-09-07 16:38:28 -0600162 helm upgrade --install $(HELM_GLOBAL_ARGS) --namespace omec omec-data-plane omec-data-plane -f $(CIABVALUES) && \
163 helm upgrade --install $(HELM_GLOBAL_ARGS) --namespace omec omec-control-plane omec-control-plane -f $(CIABVALUES)
164 $(WORKSPACE)/cord/helm-charts/scripts/wait_for_pods.sh omec
Hyunsun Moon81c8e232019-05-21 03:40:22 -0600165 touch $@
166
167# UE images includes kernel module, ue_ip.ko
168# which should be built in the exactly same kernel version of the host machine
169$(BUILD)/openairinterface: | $(M)/setup
170 mkdir -p $(BUILD)
171 cd $(BUILD); git clone https://github.com/opencord/openairinterface.git
172
173$(M)/ue-image: | $(M)/k8s-ready $(BUILD)/openairinterface
174 cd $(BUILD)/openairinterface; \
175 sudo docker build . --target lte-uesoftmodem \
176 --build-arg build_base=omecproject/oai-base:1.0.0 \
177 --file Dockerfile.ue \
178 --tag omecproject/lte-uesoftmodem:1.0.0
179 touch $@
180
181$(M)/oaisim: | $(M)/omec $(M)/ue-image
182 sudo ip addr add 127.0.0.2/8 dev lo || true
183 $(eval mme_iface=$(shell ip -4 route list default | awk -F 'dev' '{ print $$2; exit }' | awk '{ print $$1 }'))
184 cd $(WORKSPACE)/cord/helm-charts; \
Hyunsun Moonb9d3f482019-09-07 16:38:28 -0600185 helm upgrade --install $(HELM_GLOBAL_ARGS) --namespace omec oaisim oaisim -f $(CIABVALUES) \
Hyunsun Moon07decd42019-08-19 13:59:52 -0600186 --set config.enb.networks.s1_mme.interface=$(mme_iface)
Hyunsun Moonb9d3f482019-09-07 16:38:28 -0600187 $(WORKSPACE)/cord/helm-charts/scripts/wait_for_pods.sh omec
Hyunsun Moon81c8e232019-05-21 03:40:22 -0600188 @timeout 60s bash -c \
189 "until ip addr show oip1 | grep -q inet; \
190 do \
191 echo 'Waiting for UE 1 gets IP address'; \
192 sleep 3; \
193 done"
194 touch $@
195
Hyunsun Moon89886f12019-09-05 22:16:09 -0600196test: | $(M)/fabric $(M)/omec $(M)/oaisim
Hyunsun Moon07decd42019-08-19 13:59:52 -0600197 @timeout 120s bash -c \
Hyunsun Moon9f8b5952019-08-13 19:03:47 -0600198 "until timeout 3 ping -I oip1 13.1.1.254 -c 1 > /dev/null 2>&1; do \
199 echo 'Waiting for DP to be ready'; \
200 done"
Woojoong Kim68b57e72019-10-16 14:19:55 -0700201 @echo "Test1: ping from UE to SGI network gateway"
Hyunsun Moon9f8b5952019-08-13 19:03:47 -0600202 ping -I oip1 13.1.1.254 -c 3
Woojoong Kim68b57e72019-10-16 14:19:55 -0700203 @echo "Test2: ping from UE to 8.8.8.8"
204 ping -I oip1 8.8.8.8 -c 3
205 @echo "Test3: ping from UE to opennetworking.org and google.com"
206 ping -I oip1 opennetworking.org -c 3
207 ping -I oip1 google.com -c 3
208 @echo "Finished to test"
Hyunsun Moon81c8e232019-05-21 03:40:22 -0600209
210reset-test:
Hyunsun Moonad70fc12019-08-02 16:52:19 -0600211 helm delete --purge oaisim || true
Hyunsun Moon07decd42019-08-19 13:59:52 -0600212 helm delete --purge omec-control-plane || true
213 helm delete --purge omec-data-plane || true
Hyunsun Moon89886f12019-09-05 22:16:09 -0600214 cd $(M); rm -f oaisim omec
Hyunsun Moon81c8e232019-05-21 03:40:22 -0600215
216clean: reset-test
217 helm delete --purge $(shell helm ls -q) || true
Hyunsun Moonb9d3f482019-09-07 16:38:28 -0600218 kubectl delete po router || true
219 kubectl delete net-attach-def sgi-net || true
Hyunsun Moon81c8e232019-05-21 03:40:22 -0600220 sudo ovs-vsctl del-br br-s1u-net || true
221 sudo ovs-vsctl del-br br-sgi-net || true
222 sudo apt remove --purge openvswitch-switch -y
223 source "$(VENV)/bin/activate" && cd $(BUILD)/kubespray; \
224 ansible-playbook -b -i inventory/local/hosts.ini reset.yml
225 @if [ -d /usr/local/etc/emulab ]; then \
226 mount | grep /mnt/extra/kubelet/pods | cut -d" " -f3 | sudo xargs umount; \
227 sudo rm -rf /mnt/extra/kubelet; \
228 fi
229 rm -rf $(M)