blob: af38058504124e0784504bfa37ee1cb2666e4320 [file] [log] [blame]
Jeremy Ronquillo5263c732020-10-13 09:42:19 -07001# Copyright 2018-present Open Networking Foundation
2#
3# SPDX-License-Identifier: LicenseRef-ONF-Member-Only-1.0
4
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -07005SHELL := /bin/bash
6BUILD ?= /tmp/build
7M ?= $(BUILD)/milestones
8MAKEDIR := $(dir $(realpath $(firstword $(MAKEFILE_LIST))))
Jeremy Ronquilloaf084f32020-08-24 13:18:47 -07009SCRIPTDIR := $(MAKEDIR)/scripts
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -070010RESOURCEDIR := $(MAKEDIR)/resources
11WORKSPACE ?= $(HOME)
Jeremy Ronquillod3cab742020-08-24 11:49:00 -070012VENV ?= $(BUILD)/venv/aiab
Andy Bavierebf479c2021-09-08 15:47:58 -070013
144G_CORE_VALUES ?= $(MAKEDIR)/4g-core-values.yaml
155G_CORE_VALUES ?= $(MAKEDIR)/5g-core-values.yaml
16OAISIM_VALUES ?= $(MAKEDIR)/oaisim-values.yaml
17ROC_VALUES ?= $(MAKEDIR)/roc-values.yaml
18UPF_VALUES ?= $(MAKEDIR)/upf-values.yaml
19RANSIM_VALUES ?= $(MAKEDIR)/ransim-values.yaml
20ROC_4G_MODELS ?= $(MAKEDIR)/roc-4g-models.json
21ROC_5G_MODELS ?= $(MAKEDIR)/roc-5g-models.json
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -070022
Hyunsun84f0f172020-09-23 15:40:08 -050023KUBESPRAY_VERSION ?= release-2.14
24DOCKER_VERSION ?= 19.03
25K8S_VERSION ?= v1.18.9
Scott Baker17b70d12020-09-29 23:26:21 -070026HELM_VERSION ?= v3.2.4
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -070027
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -070028HELM_GLOBAL_ARGS ?=
Andy Baviereda466b2021-08-27 15:00:36 -070029
30# Allow installing local charts or specific versions of published charts.
31# E.g., to install the Aether 1.5 release:
32# CHARTS=release-1.5 make test
33# Default is to install from the local charts.
34CHARTS ?= local
35CONFIGFILE := configs/$(CHARTS)
36include $(CONFIGFILE)
37include configs/authentication
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -070038
39cpu_family := $(shell lscpu | grep 'CPU family:' | awk '{print $$3}')
40cpu_model := $(shell lscpu | grep 'Model:' | awk '{print $$2}')
41os_vendor := $(shell lsb_release -i -s)
42os_release := $(shell lsb_release -r -s)
43
Badhrinath3e081e22020-12-02 15:02:08 -060044omec: $(M)/system-check $(M)/omec
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -070045oaisim: $(M)/oaisim
Badhrinath3e081e22020-12-02 15:02:08 -0600465gc: $(M)/system-check $(M)/5g-core
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -070047
Badhrinath3e081e22020-12-02 15:02:08 -060048.PHONY: omec oaisim 5gc test reset-test 5g-core reset-5g-test clean
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -070049
50$(M):
51 mkdir -p $(M)
52
53$(M)/system-check: | $(M)
54 @if [[ $(cpu_family) -eq 6 ]]; then \
55 if [[ $(cpu_model) -lt 60 ]]; then \
56 echo "FATAL: haswell CPU or newer is required."; \
57 exit 1; \
58 fi \
59 else \
60 echo "FATAL: unsupported CPU family."; \
61 exit 1; \
62 fi
63 @if [[ $(os_vendor) =~ (Ubuntu) ]]; then \
Hyunsun7b640512020-10-27 19:49:51 -050064 if [[ ! $(os_release) =~ (18.04) ]]; then \
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -070065 echo "WARN: $(os_vendor) $(os_release) has not been tested."; \
66 fi; \
67 if dpkg --compare-versions 4.15 gt $(shell uname -r); then \
68 echo "FATAL: kernel 4.15 or later is required."; \
69 echo "Please upgrade your kernel by running" \
70 "apt install --install-recommends linux-generic-hwe-$(os_release)"; \
71 exit 1; \
72 fi \
73 else \
74 echo "FAIL: unsupported OS."; \
75 exit 1; \
76 fi
Hyunsun7b640512020-10-27 19:49:51 -050077 @if [[ ! -d "$(WORKSPACE)/cord/aether-helm-charts" ]]; then \
78 echo "FATAL: Please clone aether-helm-charts under $(WORKSPACE)/cord directory."; \
79 exit 1; \
80 fi
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -070081 touch $@
82
83$(M)/setup: | $(M)
84 sudo $(SCRIPTDIR)/cloudlab-disksetup.sh
Hyunsun Moon82e47c22021-06-28 12:30:20 -050085 sudo apt update; sudo apt install -y software-properties-common python3 python3-pip python3-venv jq httpie ipvsadm
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -070086 touch $@
87
88$(BUILD)/kubespray: | $(M)/setup
89 mkdir -p $(BUILD)
90 cd $(BUILD); git clone https://github.com/kubernetes-incubator/kubespray.git -b $(KUBESPRAY_VERSION)
91
92$(VENV)/bin/activate: | $(M)/setup
Hyunsun Moon82e47c22021-06-28 12:30:20 -050093 python3 -m venv $(VENV)
Wei-Yu Chen7edb7e72021-07-28 11:37:55 +080094 source "$(VENV)/bin/activate" && \
95 python -m pip install -U pip && \
96 deactivate
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -070097
98$(M)/kubespray-requirements: $(BUILD)/kubespray | $(VENV)/bin/activate
99 source "$(VENV)/bin/activate" && \
100 pip install -r $(BUILD)/kubespray/requirements.txt
101 touch $@
102
103$(M)/k8s-ready: | $(M)/setup $(BUILD)/kubespray $(VENV)/bin/activate $(M)/kubespray-requirements
104 source "$(VENV)/bin/activate" && cd $(BUILD)/kubespray; \
105 ansible-playbook -b -i inventory/local/hosts.ini \
106 -e "{'override_system_hostname' : False, 'disable_swap' : True}" \
107 -e "{'docker_version' : $(DOCKER_VERSION)}" \
108 -e "{'docker_iptables_enabled' : True}" \
109 -e "{'kube_version' : $(K8S_VERSION)}" \
Hyunsun84f0f172020-09-23 15:40:08 -0500110 -e "{'kube_network_plugin_multus' : True, 'multus_version' : stable, 'multus_cni_version' : 0.3.1}" \
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -0700111 -e "{'kube_proxy_metrics_bind_address' : 0.0.0.0:10249}" \
112 -e "{'kube_pods_subnet' : 192.168.0.0/17, 'kube_service_addresses' : 192.168.128.0/17}" \
113 -e "{'kube_apiserver_node_port_range' : 2000-36767}" \
114 -e "{'kubeadm_enabled': True}" \
115 -e "{'kube_feature_gates' : [SCTPSupport=True]}" \
116 -e "{'kubelet_custom_flags' : [--allowed-unsafe-sysctls=net.*]}" \
117 -e "{'dns_min_replicas' : 1}" \
118 -e "{'helm_enabled' : True, 'helm_version' : $(HELM_VERSION)}" \
119 cluster.yml
120 mkdir -p $(HOME)/.kube
121 sudo cp -f /etc/kubernetes/admin.conf $(HOME)/.kube/config
122 sudo chown $(shell id -u):$(shell id -g) $(HOME)/.kube/config
123 kubectl wait pod -n kube-system --for=condition=Ready --all
124 touch $@
125
126$(M)/helm-ready: | $(M)/k8s-ready
Woojoong Kim8425cb32021-01-08 16:18:53 -0800127 helm repo add incubator https://charts.helm.sh/incubator
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -0700128 helm repo add cord https://charts.opencord.org
Andy Baviereda466b2021-08-27 15:00:36 -0700129 helm repo add atomix https://charts.atomix.io
130 helm repo add onosproject https://charts.onosproject.org
131 if [ "$(REPO_PASSWORD)" ]; then \
132 helm repo add aether --username ${REPO_USERNAME} --password ${REPO_PASSWORD} https://charts.aetherproject.org; \
133 helm repo add sdran --username ${REPO_USERNAME} --password ${REPO_PASSWORD} https://sdrancharts.onosproject.org; \
134 fi
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -0700135 touch $@
136
137/opt/cni/bin/simpleovs: | $(M)/k8s-ready
138 sudo cp $(RESOURCEDIR)/simpleovs /opt/cni/bin/
139
140/opt/cni/bin/static: | $(M)/k8s-ready
141 mkdir -p $(BUILD)/cni-plugins; cd $(BUILD)/cni-plugins; \
142 wget https://github.com/containernetworking/plugins/releases/download/v0.8.2/cni-plugins-linux-amd64-v0.8.2.tgz && \
143 tar xvfz cni-plugins-linux-amd64-v0.8.2.tgz
144 sudo cp $(BUILD)/cni-plugins/static /opt/cni/bin/
145
146# TODO: need to connect ONOS
147$(M)/fabric: | $(M)/setup /opt/cni/bin/simpleovs /opt/cni/bin/static
148 sudo apt install -y openvswitch-switch
149 sudo ovs-vsctl --may-exist add-br br-enb-net
150 sudo ovs-vsctl --may-exist add-port br-enb-net enb -- set Interface enb type=internal
151 sudo ip addr add 192.168.251.4/24 dev enb || true
152 sudo ip link set enb up
153 sudo ethtool --offload enb tx off
154 sudo ip route replace 192.168.252.0/24 via 192.168.251.1 dev enb
155 kubectl apply -f $(RESOURCEDIR)/router.yaml
156 kubectl wait pod -n default --for=condition=Ready -l app=router --timeout=300s
Andy Baviereda466b2021-08-27 15:00:36 -0700157 kubectl -n default exec router -- ip route add 172.250.0.0/16 via 192.168.250.3
Jeremy Ronquilloaf084f32020-08-24 13:18:47 -0700158 kubectl delete net-attach-def core-net
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -0700159 touch $@
160
Andy Bavierebf479c2021-09-08 15:47:58 -0700161auth-secret: $(RESOURCEDIR)/aether.registry.yaml
162$(RESOURCEDIR)/aether.registry.yaml: configs/authentication
Andy Baviereda466b2021-08-27 15:00:36 -0700163 kubectl -n omec create secret docker-registry aether.registry \
164 --docker-server=https://registry.aetherproject.org \
165 --docker-username=${REGISTRY_USERNAME} \
166 --docker-password=${REGISTRY_CLI_SECRET} \
167 --dry-run=client --output=yaml > $@
168
169$(M)/omec: | $(M)/helm-ready /opt/cni/bin/simpleovs /opt/cni/bin/static $(M)/fabric $(RESOURCEDIR)/aether.registry.yaml
Scott Baker17b70d12020-09-29 23:26:21 -0700170 kubectl get namespace omec 2> /dev/null || kubectl create namespace omec
Andy Baviereda466b2021-08-27 15:00:36 -0700171 kubectl -n omec get secret aether.registry || kubectl create -f $(RESOURCEDIR)/aether.registry.yaml
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -0700172 helm repo update
Andy Baviereda466b2021-08-27 15:00:36 -0700173 if [ "$(CHARTS)" == "local" ]; then helm dep up $(OMEC_CONTROL_PLANE_CHART); fi
Andy Bavierebf479c2021-09-08 15:47:58 -0700174 helm upgrade --install --wait $(HELM_GLOBAL_ARGS) \
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -0700175 --namespace omec \
Andy Bavierebf479c2021-09-08 15:47:58 -0700176 --values $(4G_CORE_VALUES) \
177 sim-app \
178 $(OMEC_SUB_PROVISION_CHART) && \
179 helm upgrade --install --wait $(HELM_GLOBAL_ARGS) \
180 --namespace omec \
181 --values $(4G_CORE_VALUES) \
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -0700182 omec-control-plane \
Andy Baviereda466b2021-08-27 15:00:36 -0700183 $(OMEC_CONTROL_PLANE_CHART) && \
Andy Bavierebf479c2021-09-08 15:47:58 -0700184 helm upgrade --install --wait $(HELM_GLOBAL_ARGS) \
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -0700185 --namespace omec \
Andy Bavierebf479c2021-09-08 15:47:58 -0700186 --values $(UPF_VALUES) \
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -0700187 omec-user-plane \
Andy Bavierebf479c2021-09-08 15:47:58 -0700188 $(OMEC_USER_PLANE_CHART)
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -0700189 touch $@
190
Andy Baviereda466b2021-08-27 15:00:36 -0700191$(M)/5g-core: | $(M)/helm-ready /opt/cni/bin/simpleovs /opt/cni/bin/static $(M)/fabric $(RESOURCEDIR)/aether.registry.yaml
Badhrinath3e081e22020-12-02 15:02:08 -0600192 kubectl get namespace omec 2> /dev/null || kubectl create namespace omec
Andy Bavierebf479c2021-09-08 15:47:58 -0700193 kubectl -n omec get secret aether.registry || kubectl create -f $(RESOURCEDIR)/aether.registry.yaml
Badhrinath3e081e22020-12-02 15:02:08 -0600194 helm repo update
Andy Baviereda466b2021-08-27 15:00:36 -0700195 if [ "$(CHARTS)" == "local" ]; then helm dep up $(5GC_CONTROL_PLANE_CHART); fi
Andy Bavierebf479c2021-09-08 15:47:58 -0700196 helm upgrade --install --wait $(HELM_GLOBAL_ARGS) \
Badhrinath3e081e22020-12-02 15:02:08 -0600197 --namespace omec \
Andy Bavierebf479c2021-09-08 15:47:58 -0700198 --values $(5G_CORE_VALUES) \
Ajay Lotan Thakur980823b2021-06-23 18:55:05 -0500199 sim-app \
Andy Baviereda466b2021-08-27 15:00:36 -0700200 $(OMEC_SUB_PROVISION_CHART) && \
Andy Bavierebf479c2021-09-08 15:47:58 -0700201 helm upgrade --install --wait $(HELM_GLOBAL_ARGS) \
Ajay Lotan Thakur980823b2021-06-23 18:55:05 -0500202 --namespace omec \
Andy Bavierebf479c2021-09-08 15:47:58 -0700203 --values $(UPF_VALUES) \
Badhrinath3e081e22020-12-02 15:02:08 -0600204 5g-core-up \
Andy Baviereda466b2021-08-27 15:00:36 -0700205 $(OMEC_USER_PLANE_CHART) && \
Andy Bavierebf479c2021-09-08 15:47:58 -0700206 helm upgrade --install --wait $(HELM_GLOBAL_ARGS) \
Badhrinath3e081e22020-12-02 15:02:08 -0600207 --namespace omec \
Andy Bavierebf479c2021-09-08 15:47:58 -0700208 --values $(5G_CORE_VALUES) \
Badhrinath3e081e22020-12-02 15:02:08 -0600209 fgc-core \
Andy Baviereda466b2021-08-27 15:00:36 -0700210 $(5GC_CONTROL_PLANE_CHART) && \
Andy Bavierebf479c2021-09-08 15:47:58 -0700211 helm upgrade --install --wait $(HELM_GLOBAL_ARGS) \
Badhrinath3e081e22020-12-02 15:02:08 -0600212 --namespace omec \
Andy Bavierebf479c2021-09-08 15:47:58 -0700213 --values $(RANSIM_VALUES) \
Badhrinath3e081e22020-12-02 15:02:08 -0600214 5g-ransim-plane \
Andy Bavierebf479c2021-09-08 15:47:58 -0700215 $(5G_RAN_SIM_CHART)
Badhrinath3e081e22020-12-02 15:02:08 -0600216 touch $@
217
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -0700218# UE images includes kernel module, ue_ip.ko
219# which should be built in the exactly same kernel version of the host machine
220$(BUILD)/openairinterface: | $(M)/setup
221 mkdir -p $(BUILD)
222 cd $(BUILD); git clone https://github.com/opencord/openairinterface.git
223
224$(M)/ue-image: | $(M)/k8s-ready $(BUILD)/openairinterface
225 cd $(BUILD)/openairinterface; \
226 sudo docker build . --target lte-uesoftmodem \
Hyunsun7b640512020-10-27 19:49:51 -0500227 --build-arg build_base=omecproject/oai-base:1.1.0 \
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -0700228 --file Dockerfile.ue \
Hyunsun7b640512020-10-27 19:49:51 -0500229 --tag omecproject/lte-uesoftmodem:1.1.0
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -0700230 touch $@
231
232$(M)/oaisim: | $(M)/ue-image $(M)/omec
233 sudo ip addr add 127.0.0.2/8 dev lo || true
234 $(eval mme_iface=$(shell ip -4 route list default | awk -F 'dev' '{ print $$2; exit }' | awk '{ print $$1 }'))
Andy Bavierebf479c2021-09-08 15:47:58 -0700235 helm upgrade --install $(HELM_GLOBAL_ARGS) --namespace omec oaisim cord/oaisim -f $(OAISIM_VALUES) \
Jeremy Ronquilloaf084f32020-08-24 13:18:47 -0700236 --set config.enb.networks.s1_mme.interface=$(mme_iface) \
237 --set images.pullPolicy=IfNotPresent
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -0700238 kubectl rollout status -n omec statefulset ue
239 @timeout 60s bash -c \
240 "until ip addr show oip1 | grep -q inet; \
241 do \
242 echo 'Waiting for UE 1 gets IP address'; \
243 sleep 3; \
244 done"
245 touch $@
246
Andy Bavierebf479c2021-09-08 15:47:58 -0700247roc: $(M)/roc
248$(M)/roc: $(M)/helm-ready
249 kubectl get namespace aether-roc 2> /dev/null || kubectl create namespace aether-roc
250 helm repo update
251 if [ "$(CHARTS)" == "local" ]; then helm dep up $(AETHER_ROC_UMBRELLA_CHART); fi
252 helm upgrade --install --wait $(HELM_GLOBAL_ARGS) \
253 --namespace kube-system \
254 --values $(ROC_VALUES) \
255 atomix-controller \
256 $(ATOMIX_CONTROLLER_CHART)
257 helm upgrade --install --wait $(HELM_GLOBAL_ARGS) \
258 --namespace kube-system \
259 --values $(ROC_VALUES) \
260 atomix-raft-storage \
261 $(ATOMIX_RAFT_STORAGE_CHART)
262 helm upgrade --install --wait $(HELM_GLOBAL_ARGS) \
263 --namespace kube-system \
264 --values $(ROC_VALUES) \
265 onos-operator \
266 $(ONOS_OPERATOR_CHART)
267 helm upgrade --install --wait $(HELM_GLOBAL_ARGS) \
268 --namespace aether-roc \
269 --values $(ROC_VALUES) \
270 aether-roc-umbrella \
271 $(AETHER_ROC_UMBRELLA_CHART)
272 touch $@
273
274# Load the ROC 4G models. Disable loading network slice from SimApp.
275roc-4g-models: $(M)/roc
276 sed -i 's/provision-network-slice: true/provision-network-slice: false/' $(4G_CORE_VALUES)
277 sed -i 's/# syncUrl/syncUrl/' $(4G_CORE_VALUES)
278 $(eval ONOS_CLI_POD := $(shell kubectl -n aether-roc get pods -l name=onos-cli -o name))
279 echo "ONOS CLI pod: ${ONOS_CLI_POD}"
280 until kubectl -n aether-roc exec ${ONOS_CLI_POD} -- \
281 curl -s -f -L -X PATCH "http://aether-roc-api:8181/aether-roc-api" \
282 --header 'Content-Type: application/json' \
283 --data-raw "$$(cat ${ROC_4G_MODELS})"; do sleep 5; done
284
285# Load the ROC 5G models. Disable loading network slice from SimApp.
286roc-5g-models: $(M)/roc
287 sed -i 's/provision-network-slice: true/provision-network-slice: false/' $(5G_CORE_VALUES)
288 sed -i 's/# syncUrl/syncUrl/' $(5G_CORE_VALUES)
289 $(eval ONOS_CLI_POD := $(shell kubectl -n aether-roc get pods -l name=onos-cli -o name))
290 echo "ONOS CLI pod: ${ONOS_CLI_POD}"
291 until kubectl -n aether-roc exec ${ONOS_CLI_POD} -- \
292 curl -s -f -L -X PATCH "http://aether-roc-api:8181/aether-roc-api" \
293 --header 'Content-Type: application/json' \
294 --data-raw "$$(cat ${ROC_5G_MODELS})"; do sleep 5; done
295
296roc-clean:
297 @echo "This could take 2-3 minutes..."
298 sed -i 's/provision-network-slice: false/provision-network-slice: true/' $(4G_CORE_VALUES)
299 sed -i 's/ syncUrl/ # syncUrl/' $(4G_CORE_VALUES)
300 sed -i 's/provision-network-slice: false/provision-network-slice: true/' $(5G_CORE_VALUES)
301 sed -i 's/ syncUrl/ # syncUrl/' $(5G_CORE_VALUES)
302 kubectl delete namespace aether-roc || true
303 rm -rf $(M)/roc
304
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -0700305test: | $(M)/fabric $(M)/omec $(M)/oaisim
306 @sleep 5
307 @echo "Test1: ping from UE to SGI network gateway"
Hyunsun Moon0bd8cec2020-09-28 00:38:26 -0500308 ping -I oip1 192.168.250.1 -c 15
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -0700309 @echo "Test2: ping from UE to 8.8.8.8"
310 ping -I oip1 8.8.8.8 -c 3
Hyunsun Moon0bd8cec2020-09-28 00:38:26 -0500311 @echo "Test3: ping from UE to google.com"
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -0700312 ping -I oip1 google.com -c 3
313 @echo "Finished to test"
314
315reset-test:
Andy Bavierebf479c2021-09-08 15:47:58 -0700316 kubectl delete namespace omec || true
Jeremy Ronquillob2b64ea2020-12-09 13:49:03 -0800317 kubectl delete po router || true
318 cd $(M); rm -f oaisim omec fabric
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -0700319
Badhrinath3e081e22020-12-02 15:02:08 -0600320reset-5g-test:
Andy Bavierebf479c2021-09-08 15:47:58 -0700321 kubectl delete namespace omec || true
Badhrinath3e081e22020-12-02 15:02:08 -0600322 cd $(M); rm -f 5g-core
323
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -0700324clean: reset-test
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -0700325 kubectl delete po router || true
Jeremy Ronquilloaf084f32020-08-24 13:18:47 -0700326 kubectl delete net-attach-def core-net || true
327 sudo ovs-vsctl del-br br-access-net || true
328 sudo ovs-vsctl del-br br-core-net || true
Jeremy Ronquillo6be909e2020-08-24 09:36:13 -0700329 sudo apt remove --purge openvswitch-switch -y
330 source "$(VENV)/bin/activate" && cd $(BUILD)/kubespray; \
331 ansible-playbook -b -i inventory/local/hosts.ini reset.yml
332 @if [ -d /usr/local/etc/emulab ]; then \
333 mount | grep /mnt/extra/kubelet/pods | cut -d" " -f3 | sudo xargs umount; \
334 sudo rm -rf /mnt/extra/kubelet; \
335 fi
336 rm -rf $(M)