blob: f0dc1eda37417ffd11b7f9d16b4efd045a4f5cdf [file] [log] [blame]
Andy Bavier35053b62018-09-20 13:45:45 -07001SHELL := /bin/bash
2BUILD ?= /tmp
3M ?= $(BUILD)/milestones
4MAKEDIR := $(dir $(realpath $(firstword $(MAKEFILE_LIST))))
5WORKSPACE ?= $(HOME)
Andy Bavier6ce277e2018-10-02 10:29:21 -07006VIRTUALENV ?= $(WORKSPACE)/cord/test
Andy Bavier35053b62018-09-20 13:45:45 -07007SEBAVALUES ?= configs/seba-ponsim.yaml
Kailash Khalasi98dabd02018-10-19 13:00:12 -07008TESTTAGS ?= stable
Andy Bavier35053b62018-09-20 13:45:45 -07009
Zack Williams3bcdc262018-09-27 11:00:38 -070010# used to start logging/monitoring and other infrastructure charts
11INFRA_CHARTS ?=
12INFRA_PREREQS = $(foreach chart,$(INFRA_CHARTS),$(M)/$(chart))
13
Zack Williams2a6a1072019-01-07 11:15:12 -070014DOCKER_VERSION ?= "17.06"
15
16K8S_VERSION ?= "1.12.4"
17
18HELM_VERSION ?= "2.12.1"
19HELM_SHA256SUM ?= "891004bec55431b39515e2cedc4f4a06e93782aa03a4904f2bd742b168160451"
Andy Bavier35053b62018-09-20 13:45:45 -070020HELM_PLATFORM ?= "linux-amd64"
21
Zack Williams2a6a1072019-01-07 11:15:12 -070022KAFKA_CHART_VERSION ?= 0.13.3
Andy Bavier1d5b9922019-01-02 15:30:55 -070023KAFKA_POD := "pod/cord-kafka-0"
Zack Williams3bcdc262018-09-27 11:00:38 -070024
Zack Williams5c16dc82019-01-08 10:57:08 -070025ETCD_OPERATOR_VERSION ?= 0.8.3
26
Zack Williamsd0c12f02018-11-21 11:54:36 -070027# Targets
28ponsim: $(M)/siab
29
30bbsim: $(M)/bbsim-tosca
31
32both: BBSIM_DHCP_TOSCA =
33both: $(M)/siab $(M)/bbsim-tosca
Andy Bavier35053b62018-09-20 13:45:45 -070034
Andy Baviere5a16f62018-10-11 14:07:15 -070035# https://ftp.gnu.org/old-gnu/Manuals/make-3.79.1/html_chapter/make_6.html#SEC69
36latest: SEBAVALUES := configs/seba-ponsim-latest.yaml
Andy Bavier2a5a78e2019-01-08 16:47:47 -070037latest: ponsim
Andy Baviere5a16f62018-10-11 14:07:15 -070038
39stable: SEBAVALUES := configs/seba-ponsim.yaml
Andy Bavier2a5a78e2019-01-08 16:47:47 -070040stable: ponsim
Andy Baviere5a16f62018-10-11 14:07:15 -070041
Andy Bavier1d5b9922019-01-02 15:30:55 -070042siab-1.0: SEBAVALUES := configs/siab-v1.0.0.yaml
43siab-1.0: KAFKA_POD := "pod/cord-platform-kafka-0"
44# Parallel make won't work here; need to install seba-1.0.0 before siab
45siab-1.0: $(M)/seba-1.0.0 $(M)/siab
46
Andy Bavier35053b62018-09-20 13:45:45 -070047$(M)/setup:
48 mkdir -p $(M)
49 sudo apt update
50 sudo apt install -y httpie jq software-properties-common
51 sudo swapoff -a
Andy Bavier07779ea2018-10-03 15:13:22 -070052 sudo iptables -P FORWARD ACCEPT
Andy Bavier35053b62018-09-20 13:45:45 -070053 touch $@
54
55/usr/bin/docker: | $(M)/setup
56 sudo apt-key adv --keyserver keyserver.ubuntu.com --recv 0EBFCD88
57 sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(shell lsb_release -cs) stable"
58 sudo apt update
Zack Williams2a6a1072019-01-07 11:15:12 -070059 sudo apt install -y "docker-ce=${DOCKER_VERSION}*"
Andy Bavier35053b62018-09-20 13:45:45 -070060
61/usr/bin/kubeadm: | $(M)/setup /usr/bin/docker
62 curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
63 echo "deb http://apt.kubernetes.io/ kubernetes-xenial main" > /tmp/kubernetes.list
64 sudo cp /tmp/kubernetes.list /etc/apt/sources.list.d/kubernetes.list
65 sudo apt update
Zack Williams2a6a1072019-01-07 11:15:12 -070066 sudo apt install -y "kubeadm=${K8S_VERSION}-*" "kubelet=${K8S_VERSION}-*" "kubectl=${K8S_VERSION}-*"
Andy Bavier35053b62018-09-20 13:45:45 -070067
68/usr/local/bin/helm:
69 curl -L -o /tmp/helm.tgz "https://storage.googleapis.com/kubernetes-helm/helm-v${HELM_VERSION}-${HELM_PLATFORM}.tar.gz"
70 echo "${HELM_SHA256SUM} /tmp/helm.tgz" | sha256sum -c -
71 cd /tmp; tar -xzvf helm.tgz; sudo mv ${HELM_PLATFORM}/helm /usr/local/bin/helm
72 sudo chmod a+x /usr/local/bin/helm
73 rm -rf /tmp/helm.tgz /tmp/${HELM_PLATFORM}
74
75$(M)/kubeadm: | $(M)/setup /usr/bin/kubeadm
76 sudo kubeadm init --pod-network-cidr=192.168.0.0/16
Andy Baviera7c6a4e2018-09-28 13:29:43 -070077 mkdir -p $(HOME)/.kube
78 sudo cp -f /etc/kubernetes/admin.conf $(HOME)/.kube/config
79 sudo chown $(shell id -u):$(shell id -g) $(HOME)/.kube/config
Zack Williams2a6a1072019-01-07 11:15:12 -070080 kubectl apply -f https://docs.projectcalico.org/v3.3/getting-started/kubernetes/installation/hosted/rbac-kdd.yaml
81 kubectl apply -f https://docs.projectcalico.org/v3.3/getting-started/kubernetes/installation/hosted/kubernetes-datastore/calico-networking/1.7/calico.yaml
Andy Bavier35053b62018-09-20 13:45:45 -070082 kubectl taint nodes --all node-role.kubernetes.io/master-
83 touch $@
84
85$(M)/helm-init: | $(M)/kubeadm /usr/local/bin/helm
86 kubectl create serviceaccount --namespace kube-system tiller
87 kubectl create clusterrolebinding tiller-cluster-rule --clusterrole=cluster-admin --serviceaccount=kube-system:tiller
88 helm init --service-account tiller
89 until helm ls >& /dev/null; \
90 do \
91 echo "Waiting for Helm to be ready"; \
92 sleep 5; \
93 done
Zack Williams3bcdc262018-09-27 11:00:38 -070094 helm repo add incubator https://kubernetes-charts-incubator.storage.googleapis.com/
Andy Bavier1d5b9922019-01-02 15:30:55 -070095 helm repo add cord https://charts.opencord.org
Andy Bavier35053b62018-09-20 13:45:45 -070096 touch $@
97
98$(WORKSPACE)/cord/helm-charts: | $(M)/setup
99 mkdir -p $(WORKSPACE)/cord
100 cd $(WORKSPACE)/cord; git clone https://gerrit.opencord.org/helm-charts
101
102$(M)/kafka: | $(WORKSPACE)/cord/helm-charts $(M)/helm-init
Zack Williams3bcdc262018-09-27 11:00:38 -0700103 cd $(WORKSPACE)/cord/helm-charts && \
104 helm upgrade --install cord-kafka --version $(KAFKA_CHART_VERSION) -f examples/kafka-single.yaml incubator/kafka
Andy Bavier35053b62018-09-20 13:45:45 -0700105 touch $@
106
107$(M)/kafka-running: | $(M)/kafka
Andy Bavier1d5b9922019-01-02 15:30:55 -0700108 kubectl wait $(KAFKA_POD) --for condition=Ready --timeout=180s
Andy Bavier35053b62018-09-20 13:45:45 -0700109 touch $@
110
Zack Williams3bcdc262018-09-27 11:00:38 -0700111$(M)/nem-monitoring: | $(M)/kafka-running
112 cd $(WORKSPACE)/cord/helm-charts && \
113 helm dep up nem-monitoring && \
114 helm upgrade --install nem-monitoring nem-monitoring
115 $(WORKSPACE)/cord/helm-charts/scripts/wait_for_pods.sh
116 touch $@
117
118$(M)/logging: | $(M)/kafka-running
119 cd $(WORKSPACE)/cord/helm-charts && \
120 helm dep up logging && \
121 helm upgrade --install logging -f examples/logging-single.yaml logging
122 $(WORKSPACE)/cord/helm-charts/scripts/wait_for_pods.sh
123 touch $@
124
Andy Bavier35053b62018-09-20 13:45:45 -0700125# Dependency on NEM is there to force ordering for parallel install
126# The idea is to install VOLTHA / ONOS / Mininet while NEM is initializing
Zack Williams3bcdc262018-09-27 11:00:38 -0700127$(M)/onos: | $(M)/kafka-running $(M)/nem $(INFRA_PREREQS)
Andy Bavierb5651c72018-12-13 07:44:46 -0700128 cd $(WORKSPACE)/cord/helm-charts; helm upgrade --install onos onos -f $(SEBAVALUES)
Andy Bavier35053b62018-09-20 13:45:45 -0700129 touch $@
130
Andy Bavier1e3e9522018-11-06 11:12:00 -0700131$(M)/voltha: | $(M)/kafka-running $(INFRA_PREREQS) $(M)/etcd-operator-ready
Andy Bavier35053b62018-09-20 13:45:45 -0700132 cd $(WORKSPACE)/cord/helm-charts; helm dep up voltha
Andy Bavier1e3e9522018-11-06 11:12:00 -0700133 cd $(WORKSPACE)/cord/helm-charts; helm upgrade --install voltha voltha -f $(SEBAVALUES)
Andy Bavier35053b62018-09-20 13:45:45 -0700134 touch $@
135
Andy Bavier1e3e9522018-11-06 11:12:00 -0700136$(M)/etcd-operator-ready: | $(WORKSPACE)/cord/helm-charts $(M)/helm-init
Zack Williams5c16dc82019-01-08 10:57:08 -0700137 cd $(WORKSPACE)/cord/helm-charts; \
138 helm upgrade --install etcd-operator --version $(ETCD_OPERATOR_VERSION) stable/etcd-operator -f $(SEBAVALUES)
Andy Bavier1e3e9522018-11-06 11:12:00 -0700139 until kubectl get crd | grep etcdclusters; \
Andy Bavier35053b62018-09-20 13:45:45 -0700140 do \
Andy Bavier1e3e9522018-11-06 11:12:00 -0700141 echo "Waiting for etcdclusters CRD to be available"; \
Andy Bavier35053b62018-09-20 13:45:45 -0700142 sleep 5; \
143 done
144 touch $@
145
Andy Bavier1e3e9522018-11-06 11:12:00 -0700146$(M)/voltha-running: | $(M)/voltha
Kailashbff9eb72018-11-16 13:03:59 -0800147 timeout 300s bash -c "until kubectl get pod|grep etcd-cluster|grep 1/1; do echo 'Waiting for etcd-cluster to be ready'; sleep 10; done"
Andy Bavier35053b62018-09-20 13:45:45 -0700148 $(WORKSPACE)/cord/helm-charts/scripts/wait_for_pods.sh voltha
149 touch $@
150
151$(M)/ponsim: | $(M)/voltha
152 cd $(WORKSPACE)/cord/helm-charts; helm upgrade --install ponnet ponnet
153 $(WORKSPACE)/cord/helm-charts/scripts/wait_for_pods.sh kube-system
154 cd $(WORKSPACE)/cord/helm-charts; helm upgrade --install ponsimv2 ponsimv2 -f $(SEBAVALUES)
155 touch $@
156
157$(M)/pon0_fwd: | $(M)/ponsim
158 echo 8 > /tmp/pon0_group_fwd_mask
159 until sudo cp /tmp/pon0_group_fwd_mask /sys/class/net/pon0/bridge/group_fwd_mask; \
160 do \
161 echo "waiting for pon0..."; \
162 sleep 5; \
163 done
164 rm /tmp/pon0_group_fwd_mask
165 touch $@
166
167$(M)/mininet: | $(M)/onos $(M)/ponsim $(M)/pon0_fwd
168 sudo modprobe openvswitch
Andy Bavier1d5b9922019-01-02 15:30:55 -0700169 cd $(WORKSPACE)/cord/helm-charts; helm upgrade --install mininet mininet -f $(SEBAVALUES)
Andy Bavier35053b62018-09-20 13:45:45 -0700170 touch $@
171
Zack Williamsd0c12f02018-11-21 11:54:36 -0700172$(M)/nem: | $(M)/kafka-running $(INFRA_PREREQS)
Andy Bavier35053b62018-09-20 13:45:45 -0700173 cd $(WORKSPACE)/cord/helm-charts; helm dep update xos-core
Andy Baviere5a16f62018-10-11 14:07:15 -0700174 cd $(WORKSPACE)/cord/helm-charts; helm upgrade --install xos-core xos-core -f $(SEBAVALUES)
Matteo Scandolof3ceb4c2018-12-12 15:48:38 -0800175 cd $(WORKSPACE)/cord/helm-charts; helm dep update xos-profiles/seba-services
176 cd $(WORKSPACE)/cord/helm-charts; helm upgrade --install seba-services xos-profiles/seba-services -f $(SEBAVALUES)
177 cd $(WORKSPACE)/cord/helm-charts; helm dep update workflows/att-workflow
178 cd $(WORKSPACE)/cord/helm-charts; helm upgrade --install att-workflow workflows/att-workflow -f $(SEBAVALUES)
Andy Bavierde4537d2018-11-29 11:48:25 -0700179 cd $(WORKSPACE)/cord/helm-charts; helm dep update xos-profiles/base-kubernetes
180 cd $(WORKSPACE)/cord/helm-charts; helm upgrade --install base-kubernetes xos-profiles/base-kubernetes -f $(SEBAVALUES)
Andy Bavier35053b62018-09-20 13:45:45 -0700181 touch $@
182
Zack Williamsd0c12f02018-11-21 11:54:36 -0700183$(M)/nem-running: | $(M)/nem
Andy Bavier35053b62018-09-20 13:45:45 -0700184 $(WORKSPACE)/cord/helm-charts/scripts/wait_for_pods.sh
185 touch $@
186
Zack Williamsd0c12f02018-11-21 11:54:36 -0700187$(M)/ponsim-tosca: | $(M)/nem-running
Andy Bavier35053b62018-09-20 13:45:45 -0700188 timeout 60s bash -c "until ! http -a admin@opencord.org:letmein GET http://127.0.0.1:30001/xosapi/v1/core/serviceinstanceattributes|jq '.items[].backend_status'|grep -v OK; do echo 'waiting for ONOS config to sync'; sleep 5; done"
189 timeout 60s bash -c "until ! http -a admin@opencord.org:letmein GET http://127.0.0.1:30001/xosapi/v1/onos/onosapps|jq '.items[].backend_status'|grep -v OK; do echo 'waiting for ONOS apps to sync'; sleep 5; done"
Andy Bavier905ec802018-10-04 11:08:37 -0700190 timeout 60s bash -c "until http -a karaf:karaf GET http://127.0.0.1:30120/onos/v1/applications/org.opencord.kafka|jq '.state'|grep -q ACTIVE; do echo 'Waiting for Kafka app to become ACTIVE'; sleep 5; done"
Andy Baviere5a16f62018-10-11 14:07:15 -0700191 cd $(WORKSPACE)/cord/helm-charts; helm upgrade --install ponsim-pod xos-profiles/ponsim-pod -f $(SEBAVALUES)
Andy Bavier35053b62018-09-20 13:45:45 -0700192 $(WORKSPACE)/cord/helm-charts/scripts/wait_for_pods.sh
193 touch $@
194
195$(M)/siab: | $(M)/voltha-running $(M)/mininet $(M)/ponsim-tosca
196 until http -a karaf:karaf --ignore-stdin --check-status GET http://127.0.0.1:30120/onos/v1/configuration/org.opencord.olt.impl.Olt; \
197 do \
198 sleep 5; \
199 done
200 http -a karaf:karaf --ignore-stdin POST http://127.0.0.1:30120/onos/v1/configuration/org.opencord.olt.impl.Olt defaultVlan=65535
Andy Bavier905ec802018-10-04 11:08:37 -0700201 timeout 1m bash -c "until http GET http://127.0.0.1:30125/health|jq '.state'|grep -q HEALTHY; do echo 'Waiting for VOLTHA to be HEALTHY'; sleep 10; done"
202 echo "[passed] VOLTHA is HEALTHY"
203 timeout 2m bash -c "until http GET http://127.0.0.1:30125/api/v1/devices|jq '.items[].admin_state'|grep ENABLED|wc -l|grep -q 2; do echo 'Waiting for OLT and ONU to be enabled in VOLTHA'; sleep 10; done"
204 echo "[passed] OLT and ONU are enabled in VOLTHA"
205 timeout 2m bash -c "until http -a karaf:karaf GET http://127.0.0.1:30120/onos/v1/devices|jq '.devices[].available'|grep true|wc -l|grep -q 2; do echo 'Waiting for VOLTHA logical device and agg switch to be available in ONOS'; sleep 10; done"
206 echo "[passed] VOLTHA logical device and agg switch are available in ONOS"
207 timeout 2m bash -c "until http -a admin@opencord.org:letmein GET http://127.0.0.1:30001/xosapi/v1/att-workflow-driver/attworkflowdriverserviceinstances |jq '.items[0].authentication_state'|grep -q AWAITING; do echo 'waiting for ATT Workflow Driver ServiceInstance to be in AWAITING state'; sleep 10; done"
208 echo "[passed] ATT Workflow Driver ServiceInstance is in AWAITING state"
Andy Bavier35053b62018-09-20 13:45:45 -0700209 touch $@
210 echo "SEBA-in-a-Box installation finished!"
211
Andy Bavier1d5b9922019-01-02 15:30:55 -0700212$(M)/seba-1.0.0: | $(WORKSPACE)/cord/helm-charts $(M)/helm-init
213 helm install -n cord-platform cord/cord-platform --version=6.1.0 -f $(WORKSPACE)/cord/helm-charts/$(SEBAVALUES)
214 until kubectl get crd | grep etcdclusters; \
215 do \
216 echo "Waiting for etcdclusters CRD to be available"; \
217 sleep 5; \
218 done
219 touch $(M)/kafka
220 touch $(M)/onos
221 touch $(M)/etcd-operator-ready
222 helm install -n seba cord/seba --version=1.0.0 -f $(WORKSPACE)/cord/helm-charts/$(SEBAVALUES)
223 helm install -n att-workflow cord/att-workflow --version=1.0.2 -f $(WORKSPACE)/cord/helm-charts/$(SEBAVALUES)
224 touch $(M)/voltha
225 touch $(M)/nem
226 touch $@
227
Zack Williamsd0c12f02018-11-21 11:54:36 -0700228# BBSim support
229BBSIM_OLT_TOSCA ?= examples/bbsim-16.yaml
230BBSIM_DHCP_TOSCA ?= examples/bbsim-dhcp.yaml
231
232$(M)/bbsim: | $(M)/onos $(M)/voltha-running $(M)/nem-running
233 cd $(WORKSPACE)/cord/helm-charts && \
234 helm upgrade --install bbsim bbsim -f $(SEBAVALUES) && \
235 scripts/wait_for_pods.sh voltha && \
236 touch $@
237
238$(M)/bbsim-tosca: | $(M)/bbsim
239 cd $(WORKSPACE)/cord/helm-charts && \
240 echo "Loading BBSim TOSCA: $(BBSIM_DHCP_TOSCA) $(BBSIM_OLT_TOSCA)" && \
241 for tosca in $(BBSIM_DHCP_TOSCA) $(BBSIM_OLT_TOSCA) ; do \
242 curl -H "xos-username: admin@opencord.org" -H "xos-password: letmein" -X POST --data-binary @$$tosca http://127.0.0.1:30007/run ; \
243 done && \
244 touch $@
245
246run-tests: | $(M)/qatests
Andy Bavier35053b62018-09-20 13:45:45 -0700247
Andy Bavier01449932018-10-01 16:19:40 -0700248$(M)/qatests-prereqs:
249 sudo apt install -y python-pip virtualenv
Andy Bavier35053b62018-09-20 13:45:45 -0700250 touch $@
251
Andy Bavier01449932018-10-01 16:19:40 -0700252$(WORKSPACE)/cord/test/cord-tester: | $(M)/setup
253 mkdir -p $(WORKSPACE)/cord/test
254 cd $(WORKSPACE)/cord/test && git clone https://gerrit.opencord.org/cord-tester
Andy Bavier35053b62018-09-20 13:45:45 -0700255
Zack Williamsd0c12f02018-11-21 11:54:36 -0700256PYBOT_ARGS ?=
257$(M)/qatests: | $(M)/siab $(M)/qatests-prereqs $(WORKSPACE)/cord/test/cord-tester
Andy Bavier01449932018-10-01 16:19:40 -0700258 cd $(WORKSPACE)/cord/test/cord-tester/src/test/cord-api/ && \
Andy Bavier6ce277e2018-10-02 10:29:21 -0700259 WORKSPACE=$(VIRTUALENV) bash ./setup_venv.sh && \
260 source $(VIRTUALENV)/venv-cord-tester/bin/activate && \
Andy Bavier01449932018-10-01 16:19:40 -0700261 cd Tests/WorkflowValidations/ && \
Kailash48e7a3f2018-12-06 12:31:32 -0800262 robot $(PYBOT_ARGS) --removekeywords wuks -e notready -i $(TESTTAGS) SIAB.robot
Andy Bavier35053b62018-09-20 13:45:45 -0700263 touch $@
264
265remove-chart-milestones:
266 cd $(M); rm -f kafka kafka-running onos voltha etcd-operator-ready etcd-cluster \
Zack Williamsd0c12f02018-11-21 11:54:36 -0700267 voltha-running ponsim mininet nem nem-running ponsim-tosca siab bbsim bbsim-tosca \
Andy Bavier1d5b9922019-01-02 15:30:55 -0700268 logging nem-monitoring seba-1.0.0
Andy Bavier35053b62018-09-20 13:45:45 -0700269
270remove-kube-milestones:
271 cd $(M); rm -f kubeadm helm-init
272
273remove-test-milestones:
Kailashf8ff8122018-12-03 10:44:06 -0800274 cd $(M); rm -f qatests qatests-prereqs
Andy Bavier35053b62018-09-20 13:45:45 -0700275
276teardown-charts: remove-chart-milestones
Matteo Scandolo0efb3252018-12-10 15:58:45 -0800277 helm delete --purge $(shell helm ls -q | grep -v docker-registry)
Andy Bavier35053b62018-09-20 13:45:45 -0700278
Kailashf8ff8122018-12-03 10:44:06 -0800279reset-kubeadm: remove-chart-milestones remove-kube-milestones remove-test-milestones
Andy Bavier5a02e6d2019-01-16 15:54:30 -0700280 sudo kubeadm reset -f || true
Andy Bavier35053b62018-09-20 13:45:45 -0700281 sudo iptables -F && sudo iptables -t nat -F && sudo iptables -t mangle -F && sudo iptables -X
Andy Bavier5a02e6d2019-01-16 15:54:30 -0700282 sudo rm -f /var/lib/cni/networks/pon0/* || true
283 sudo rm -f /var/lib/cni/networks/pon1/* || true
284 sudo rm -f /var/lib/cni/networks/k8s-pod-network/* || true