blob: 52ee03b02f3dd4d25dd871aff06344d2e639882a [file] [log] [blame]
Test User01ed0642019-07-03 20:17:06 +00001#!/bin/bash
David K. Bainbridgeb7285432019-07-02 22:05:24 -07002# Copyright 2019 Ciena Corporation
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
David Bainbridge712afb82019-08-14 19:55:58 +000015TOTAL_START_TIME=$(date +%s)
David K. Bainbridgeb7285432019-07-02 22:05:24 -070016
David Bainbridge38dc1e82019-08-12 15:18:45 +000017FANCY=${FANCY:-1}
18if [ "$TERM X" == " X" ]; then
19 FANCY=0
20fi
21
David K. Bainbridgeb7285432019-07-02 22:05:24 -070022# trap ctrl-c and call ctrl_c()
23trap ctrl_c INT
24
25function ctrl_c() {
David Bainbridge38dc1e82019-08-12 15:18:45 +000026 echo -en $CNORM
David K. Bainbridgeb7285432019-07-02 22:05:24 -070027 echo ""
28 echo "ctrl-c trapped"
David Bainbridgec6a6eb62019-10-18 22:50:07 +000029 echo "Thank you for trying 'voltha up'"
David K. Bainbridgeb7285432019-07-02 22:05:24 -070030 exit
31}
32
David K. Bainbridgeebf38d92020-02-10 10:46:31 -080033VOLTCTL_VERSION=${VOLTCTL_VERSION:-latest}
David K. Bainbridgefe1fa8a2019-11-12 12:03:21 -080034KIND_VERSION=${KIND_VERSION:-v0.5.1}
David K. Bainbridgeb3a99372020-01-08 14:39:06 -080035VK_RELEASE=${VK_RELEASE:-master}
Test User08ebbd92019-07-03 17:15:39 +000036
David Bainbridge38dc1e82019-08-12 15:18:45 +000037BLACK=
38RED=
39GREEN=
40YELLOW=
41BLUE=
42MAGENTA=
43CYAN=
44WHITE=
45BOLD=
46NORMAL=
47ERROR=
48CEOL=
49CNORM=
50CIVIS=
51if [ $FANCY -eq 1 ]; then
52 BLACK=$(tput setaf 0)
53 RED=$(tput setaf 1)
54 GREEN=$(tput setaf 2)
55 YELLOW=$(tput setaf 3)
56 BLUE=$(tput setaf 4)
57 MAGENTA=$(tput setaf 5)
58 CYAN=$(tput setaf 6)
59 WHITE=$(tput setaf 7)
60 BOLD=$(tput bold)
61 NORMAL=$(tput sgr0)
62 ERROR="\xe2\x9c\x97\x20"
63 CEOL=$(tput el)
64 CNORM=$(tput cnorm)
65 CIVIS=$(tput civis)
66fi
Test User7d866122019-07-09 17:52:35 +000067
Test User3d7ad8e2019-07-03 06:15:44 +000068TYPE=${TYPE:-minimal}
David Bainbridge0774b232019-08-02 06:37:19 +000069NAME=${NAME:-$TYPE}
David Bainbridgee87067b2019-08-12 22:00:12 +000070WITH_TIMINGS=${WITH_TIMINGS:-no}
Test User7d866122019-07-09 17:52:35 +000071WITH_BBSIM=${WITH_BBSIM:-no}
David Bainbridge5b7b96b2019-07-25 20:29:13 +000072WITH_RADIUS=${WITH_RADIUS:-no}
Matteo Scandolo78dfee02020-02-13 16:18:03 -080073WITH_EAPOL=${WITH_EAPOL:-yes}
74WITH_DHCP=${WITH_DHCP:-yes}
75WITH_IGMP=${WITH_IGMP:-no}
David Bainbridgeb270c202019-07-26 01:44:42 +000076WITH_ONOS=${WITH_ONOS:-yes}
David Bainbridge99ac7a22019-08-31 02:26:43 +000077WITH_CHAOS=${WITH_CHAOS:-no}
David Bainbridge46505fb2019-10-01 21:13:20 +000078WITH_ADAPTERS=${WITH_ADAPTERS:-yes}
79WITH_SIM_ADAPTERS=${WITH_SIM_ADAPTERS:-yes}
80WITH_OPEN_ADAPTERS=${WITH_OPEN_ADAPTERS:-yes}
David Bainbridge05cd8822019-11-19 17:43:53 +000081ONLY_ONE=${ONLY_ONE:-yes}
David Bainbridge2b19e832019-08-16 02:40:53 +000082CONFIG_SADIS=${CONFIG_SADIS:-no}
David Bainbridge27790d62019-08-13 22:43:19 +000083INSTALL_ONOS_APPS=${INSTALL_ONOS_APPS:-no}
Test Userba1e7e72019-07-10 22:27:54 +000084JUST_K8S=${JUST_K8S:-no}
85DEPLOY_K8S=${DEPLOY_K8S:-yes}
David K. Bainbridge0e89cb92019-07-17 11:30:10 -070086INSTALL_KUBECTL=${INSTALL_KUBECTL:-yes}
87INSTALL_HELM=${INSTALL_HELM:-yes}
David Bainbridge46505fb2019-10-01 21:13:20 +000088UPDATE_HELM_REPOS=${UPDATE_HELM_REPOS:-yes}
Matteo Scandolofab98622020-01-28 15:17:20 -080089WAIT_ON_DOWN=${WAIT_ON_DOWN:-yes}
David Bainbridge5b7b96b2019-07-25 20:29:13 +000090VOLTHA_LOG_LEVEL=${VOLTHA_LOG_LEVEL:-WARN}
David Bainbridge90fd8e32019-08-21 23:32:47 +000091VOLTHA_CHART=${VOLTHA_CHART:-onf/voltha}
92VOLTHA_CHART_VERSION=${VOLTHA_CHART_VERSION:-latest}
93VOLTHA_BBSIM_CHART=${VOLTHA_BBSIM_CHART:-onf/bbsim}
Matteo Scandolodcd29f52019-10-07 15:42:42 -070094VOLTHA_BBSIM_CHART_VERSION=${VOLTHA_BBSIM_CHART_VERSION:-latest}
David Bainbridgee10f6d52019-07-25 00:28:13 +000095VOLTHA_ADAPTER_SIM_CHART=${VOLTHA_ADAPTER_SIM_CHART:-onf/voltha-adapter-simulated}
David Bainbridge90fd8e32019-08-21 23:32:47 +000096VOLTHA_ADAPTER_SIM_CHART_VERSION=${VOLTHA_ADAPTER_SIM_CHART_VERSION:-latest}
David Bainbridgee10f6d52019-07-25 00:28:13 +000097VOLTHA_ADAPTER_OPEN_OLT_CHART=${VOLTHA_ADAPTER_OPEN_OLT_CHART:-onf/voltha-adapter-openolt}
David Bainbridge90fd8e32019-08-21 23:32:47 +000098VOLTHA_ADAPTER_OPEN_OLT_CHART_VERSION=${VOLTHA_ADAPTER_OPEN_OLT_CHART_VERSION:-latest}
David Bainbridgee10f6d52019-07-25 00:28:13 +000099VOLTHA_ADAPTER_OPEN_ONU_CHART=${VOLTHA_ADAPTER_OPEN_ONU_CHART:-onf/voltha-adapter-openonu}
David Bainbridge90fd8e32019-08-21 23:32:47 +0000100VOLTHA_ADAPTER_OPEN_ONU_CHART_VERSION=${VOLTHA_ADAPTER_OPEN_ONU_CHART_VERSION:-latest}
Andrea Campanella21323292019-12-18 11:44:19 -0800101ONOS_CHART_VERSION=${ONOS_CHART_VERSION:-latest}
David Bainbridge70c0ea82019-11-14 23:25:26 +0000102EXTRA_HELM_INSTALL_ARGS=
Test Userba1e7e72019-07-10 22:27:54 +0000103
104HAVE_GO=$(which go >/dev/null 2>&1 && echo "yes" || echo "no")
105HOSTOS=$(uname -s | tr "[:upper:]" "[:lower:"])
106HOSTARCH=$(uname -m | tr "[:upper:]" "[:lower:"])
107if [ $HOSTARCH == "x86_64" ]; then
108 HOSTARCH="amd64"
109fi
Test User7d866122019-07-09 17:52:35 +0000110
111# Verify TYPE setting
112if [ $(echo ":minimal:full:" | grep -ic ":$TYPE:") -eq 0 ]; then
David K. Bainbridge0e89cb92019-07-17 11:30:10 -0700113 >&2 echo -e "${RED}${BOLD}${ERROR}ERROR:${NORMAL}${RED} Invalid \$TYPE value of '$TYPE'. Should be 'minimal' or 'full'${NORMAL}"
Test User7d866122019-07-09 17:52:35 +0000114 exit 1
115fi
116
David Bainbridge2a0b8a42019-08-16 17:39:29 +0000117# Used to verify configuration values are set to "yes" or "no" value or convert
118# equivalents to "yes" or "no"
119function verify_yes_no() {
120 local VAR=$1
121 local VAL=$(eval echo \$$VAR)
122 if [ $(echo ":y:yes:true:n:no:false:1:0:" | grep -ic ":$VAL:") -eq 0 ]; then
123 >&2 echo -e "${RED}${BOLD}${ERROR}ERROR:${NORMAL}${RED} Invalid '$VAR' value of '$VAL'. Should be 'yes' or 'no'${NORMAL}"
124 echo "INVALID"
125 return 1
126 fi
127 if [ $(echo ":y:yes:true:1:" | grep -ic ":$VAL:") -eq 1 ]; then
128 echo "yes"
129 else
130 echo "no"
131 fi
132 return 0
133}
Test User7d866122019-07-09 17:52:35 +0000134
David Bainbridge2a0b8a42019-08-16 17:39:29 +0000135ALL_YES_NO="\
136 WITH_TIMINGS \
137 WITH_BBSIM \
138 WITH_RADIUS \
Matteo Scandolo78dfee02020-02-13 16:18:03 -0800139 WITH_EAPOL \
140 WITH_DHCP \
141 WITH_IGMP \
David Bainbridge2a0b8a42019-08-16 17:39:29 +0000142 WITH_ONOS \
David Bainbridge99ac7a22019-08-31 02:26:43 +0000143 WITH_CHAOS \
David Bainbridge46505fb2019-10-01 21:13:20 +0000144 WITH_ADAPTERS \
145 WITH_SIM_ADAPTERS \
146 WITH_OPEN_ADAPTERS \
David Bainbridge2a0b8a42019-08-16 17:39:29 +0000147 CONFIG_SADIS \
148 JUST_K8S \
149 DEPLOY_K8S \
150 INSTALL_ONOS_APPS \
David Bainbridge2a0b8a42019-08-16 17:39:29 +0000151 INSTALL_KUBECTL \
152 INSTALL_HELM \
David Bainbridge46505fb2019-10-01 21:13:20 +0000153 UPDATE_HELM_REPOS \
David Bainbridge4f1b5562019-08-19 04:46:52 +0000154 WAIT_ON_DOWN \
David Bainbridge70c0ea82019-11-14 23:25:26 +0000155 ONLY_ONE \
David Bainbridge2a0b8a42019-08-16 17:39:29 +0000156 "
David Bainbridge5b7b96b2019-07-25 20:29:13 +0000157
David Bainbridge2a0b8a42019-08-16 17:39:29 +0000158ALL_OPTIONS="\
159 NAME \
160 TYPE \
161 $ALL_YES_NO \
162 VOLTHA_LOG_LEVEL \
163 VOLTHA_CHART \
David Bainbridge90fd8e32019-08-21 23:32:47 +0000164 VOLTHA_CHART_VERSION \
165 VOLTHA_BBSIM_CHART \
166 VOLTHA_BBSIM_CHART_VERSION \
David Bainbridge2a0b8a42019-08-16 17:39:29 +0000167 VOLTHA_ADAPTER_SIM_CHART \
David Bainbridge90fd8e32019-08-21 23:32:47 +0000168 VOLTHA_ADAPTER_SIM_CHART_VERSION \
David Bainbridge2a0b8a42019-08-16 17:39:29 +0000169 VOLTHA_ADAPTER_OPEN_OLT_CHART \
David Bainbridge90fd8e32019-08-21 23:32:47 +0000170 VOLTHA_ADAPTER_OPEN_OLT_CHART_VERSION \
David Bainbridge2a0b8a42019-08-16 17:39:29 +0000171 VOLTHA_ADAPTER_OPEN_ONU_CHART \
David Bainbridge90fd8e32019-08-21 23:32:47 +0000172 VOLTHA_ADAPTER_OPEN_ONU_CHART_VERSION \
Andrea Campanella21323292019-12-18 11:44:19 -0800173 ONOS_CHART_VERSION \
David Bainbridge2a0b8a42019-08-16 17:39:29 +0000174 ONOS_API_PORT \
175 ONOS_SSH_PORT \
176 VOLTHA_API_PORT \
177 VOLTHA_SSH_PORT \
178 VOLTHA_ETCD_PORT \
David K. Bainbridge8f7f30b2020-01-08 13:06:16 -0800179 VK_RELEASE \
180 KIND_VERSION \
181 VOLTCTL_VERSION \
David Bainbridge2a0b8a42019-08-16 17:39:29 +0000182 "
David Bainbridge01294952019-07-30 19:33:45 +0000183
David Bainbridge2a0b8a42019-08-16 17:39:29 +0000184# Iterate over yes/no configuration options and validate
185for VAR in $ALL_YES_NO; do
186 eval $VAR=$(verify_yes_no $VAR)
187 if [ "$(eval echo \$$VAR)" == "INVALID" ]; then
188 exit 1;
189 fi
190done
David Bainbridgeb270c202019-07-26 01:44:42 +0000191
David Bainbridge0774b232019-08-02 06:37:19 +0000192mkdir -p .voltha
193touch .voltha/ports
194HAVE=$(grep $NAME .voltha/ports)
195if [ "$HAVE X" == " X" ]; then
196 # Find free port prefix
197 START=81
198 while true; do
199 if [ $(grep -c $START .voltha/ports) -eq 0 ]; then
200 break
201 fi
202 START=$(expr $START + 1)
203 done
204 DELTA=$(expr $START - 81)
David Bainbridgea2595422019-10-22 03:54:28 +0000205 ONOS_API_PORT=${ONOS_API_PORT:-${START}81}
206 ONOS_SSH_PORT=${ONOS_SSH_PORT:-${START}01}
207 VOLTHA_API_PORT=${VOLTHA_API_PORT:-5$(expr 55 + $DELTA)55}
208 VOLTHA_SSH_PORT=${VOLTHA_SSH_PORT:-$(expr 50 + $DELTA)22}
209 VOLTHA_ETCD_PORT=${VOLTHA_ETCD_PORT:-$(expr 23 + $DELTA)79}
Test User01ed0642019-07-03 20:17:06 +0000210else
David Bainbridge0774b232019-08-02 06:37:19 +0000211 VALUES=$(echo $HAVE | sed -e 's/\s//g' | cut -d= -f2)
David Bainbridgea2595422019-10-22 03:54:28 +0000212 ONOS_API_PORT=${ONOS_API_PORT:-$(echo $VALUES | cut -d, -f1)}
213 ONOS_SSH_PORT=${ONOS_SSH_PORT:-$(echo $VALUES | cut -d, -f2)}
214 VOLTHA_API_PORT=${VOLTHA_API_PORT:-$(echo $VALUES | cut -d, -f3)}
215 VOLTHA_SSH_PORT=${VOLTHA_SSH_PORT:-$(echo $VALUES | cut -d, -f4)}
216 VOLTHA_ETCD_PORT=${VOLTHA_ETCD_PORT:-$(echo $VALUES | cut -d, -f5)}
Test User01ed0642019-07-03 20:17:06 +0000217fi
218
David Bainbridge0774b232019-08-02 06:37:19 +0000219PORTTMP=$(mktemp -u)
220cat .voltha/ports | grep -v $NAME > $PORTTMP
221echo "$NAME=$ONOS_API_PORT,$ONOS_SSH_PORT,$VOLTHA_API_PORT,$VOLTHA_SSH_PORT,$VOLTHA_ETCD_PORT" >> $PORTTMP
222cp $PORTTMP .voltha/ports
223rm -f $PORTTMP
224
David Bainbridge27790d62019-08-13 22:43:19 +0000225export ONOS_API_PORT ONOS_SSH_PORT
David Bainbridge01294952019-07-30 19:33:45 +0000226
Test Userd87942b2019-07-03 07:20:24 +0000227IDX=1
David Bainbridgee87067b2019-08-12 22:00:12 +0000228CLOCK="TIME:"
David Bainbridge38dc1e82019-08-12 15:18:45 +0000229SPIN_PARTS=
230NOT_VERIFIED=
David Bainbridgef858a022019-08-14 21:25:11 +0000231THEX=
232BUILD=
233CROSS=
234ENTER=
David Bainbridge38dc1e82019-08-12 15:18:45 +0000235VERIFIED=
236HELM=
237OLD_KEY=
238BIRD=
239HIGH_VOLTAGE=
240PLUG=
241RESTART=
242FORWARD=
243INSTALL=
244STOP=
245GO=
246DOWNLOAD=
247GEAR=
248NO_ENTRY=
249LOCK=
250
251if [ $FANCY -eq 1 ]; then
252 SPIN_PARTS="\
253 \xe2\xa2\x8e\xe2\xa1\xb0 \
254 \xe2\xa2\x8e\xe2\xa1\xa1 \
255 \xe2\xa2\x8e\xe2\xa1\x91 \
256 \xe2\xa2\x8e\xe2\xa0\xb1 \
257 \xe2\xa0\x8e\xe2\xa1\xb1 \
258 \xe2\xa2\x8a\xe2\xa1\xb1 \
259 \xe2\xa2\x8c\xe2\xa1\xb1 \
260 \xe2\xa2\x86\xe2\xa1\xb1 \
261 "
David Bainbridgee87067b2019-08-12 22:00:12 +0000262 CLOCK="\xe2\x8f\xb1"
David Bainbridgef858a022019-08-14 21:25:11 +0000263 THEX="${RED}${BOLD}\xe2\x9c\x97\x20${NORMAL}"
264 ENTER="${YELLOW}${BOLD}\xe2\x8e\x86${NORMAL}"
265 CROSS="${YELLOW}${BOLD}\xe2\x9c\x9a${NORMAL}"
266 BUILD="${YELLOW}${BOLD}\xf0\x9f\x8f\x97${NORMAL}"
267 NOT_VERIFIED="$BUILD"
268 VERIFIED="${GREEN}${BOLD}\xe2\x9c\x93\x20${NORMAL}"
269 HELM="${BLUE}${BOLD}\xE2\x8E\x88${NORMAL}"
David Bainbridge38dc1e82019-08-12 15:18:45 +0000270 OLD_KEY="\xF0\x9F\x97\x9D"
271 BIRD="\xF0\x9F\x90\xA6"
272 HIGH_VOLTAGE="\xE2\x9A\xA1"
273 PLUG="\xF0\x9F\xa7\xa9"
274 RESTART="\xf0\x9f\x94\x84"
275 FORWARD="\xE2\x87\xA8"
276 INSTALL="\xF0\x9F\x8F\x97"
277 STOP="\xf0\x9f\x9b\x91"
278 GO="\xf0\x9f\x9a\x80"
279 DOWNLOAD="\xf0\x9f\x93\xa5"
280 GEAR="\xe2\x9a\x99"
281 NO_ENTRY="\xe2\x9b\x94"
282 LOCK="\xf0\x9f\x94\x92"
283fi
David K. Bainbridge0e89cb92019-07-17 11:30:10 -0700284
David Bainbridge712afb82019-08-14 19:55:58 +0000285duration() {
David Bainbridgee87067b2019-08-12 22:00:12 +0000286 local h=$(expr $1 / 3600)
287 local m=$(expr $1 % 3600 / 60)
288 local s=$(expr $1 % 60)
289 local t=""
290
291 if [ $h -gt 0 ]; then
292 t="$t${h}h"
293 fi
294 if [ $m -gt 0 ]; then
295 t="$t${m}m"
296 fi
David Bainbridge712afb82019-08-14 19:55:58 +0000297 echo "$t${s}s"
298}
299
300printtime() {
301 local INDENT=
302 if [ "$1" == "-" ]; then
303 INDENT=" "
304 shift
305 fi
306 echo -e "$INDENT $CLOCK $(duration $1)"
David Bainbridgee87067b2019-08-12 22:00:12 +0000307}
308
David K. Bainbridgeb7285432019-07-02 22:05:24 -0700309bspin() {
Test Userd87942b2019-07-03 07:20:24 +0000310 IDX=1
David K. Bainbridgeb7285432019-07-02 22:05:24 -0700311 local INDENT=
312 if [ "$1" == "-" ]; then
313 INDENT=" "
314 shift
315 fi
David Bainbridge38dc1e82019-08-12 15:18:45 +0000316 if [ $FANCY -eq 0 ]; then
317 LINE=$(echo $* | sed -e 's/[\s+-]//g')
318 if [ "$LINE X" == " X" ]; then
319 return
320 fi
321 echo -e "$CIVIS$INDENT$*"
322 else
323 echo -en "$CIVIS$INDENT $*"
324 fi
David K. Bainbridgeb7285432019-07-02 22:05:24 -0700325}
326
327sspin() {
328 local INDENT=
329 if [ "$1" == "-" ]; then
330 INDENT=" "
331 shift
332 fi
David Bainbridge38dc1e82019-08-12 15:18:45 +0000333 if [ $FANCY -eq 0 ]; then
334 LINE=$(echo $* | sed -e 's/[\s+-]//g')
335 if [ "$LINE X" == " X" ]; then
336 return
337 fi
338 echo -e "$INDENT$*"
339 else
340 C=$(echo $SPIN_PARTS | cut '-d ' -f $IDX)
341 echo -en "\r$INDENT$C $*"
342 IDX=$(expr $IDX + 1)
343 if [ $IDX -gt 8 ]; then
344 IDX=1
345 fi
David K. Bainbridgeb7285432019-07-02 22:05:24 -0700346 fi
347}
348
349espin() {
350 local INDENT=
351 if [ "$1" == "-" ]; then
352 INDENT=" "
353 shift
354 fi
David Bainbridge38dc1e82019-08-12 15:18:45 +0000355 if [ $FANCY -eq 0 ]; then
356 LINE=$(echo $* | sed -e 's/[\s+-]//g')
357 if [ "$LINE X" == " X" ]; then
358 return
359 fi
360 echo -e "$INDENT$*"
361 else
362 echo -e "\r$INDENT$*$CNORM"
363 fi
David K. Bainbridgeb7285432019-07-02 22:05:24 -0700364}
365
David Bainbridgeac7f8072019-08-01 22:15:33 +0000366if [ "$1" == "get" -a "$2" == "voltconfig" ]; then
David Bainbridge0774b232019-08-02 06:37:19 +0000367 echo "$HOME/.volt/config-$NAME"
David Bainbridgeac7f8072019-08-01 22:15:33 +0000368 exit
369fi
370
Matteo Scandolo7ef6ba52019-10-04 09:20:37 -0700371if [ $# -ne 1 -o $(echo ":up:down:dump:" | grep -c ":$1:") -ne 1 ]; then
David Bainbridgeac7f8072019-08-01 22:15:33 +0000372 >&2 echo "What wouild you like to do today:"
373 >&2 echo " up - bring up voltha"
374 >&2 echo " down - tear down voltha"
375 >&2 echo " dump - create a debug dump of running system"
376 exit 1
377fi
378
David Bainbridge4f1b5562019-08-19 04:46:52 +0000379push_onos_config() {
Matteo Scandolo78dfee02020-02-13 16:18:03 -0800380 local TYPE=$1
381 local MSG=$2
382 local RESOURCE=$3
383 local DATA=$4
David Bainbridge4f1b5562019-08-19 04:46:52 +0000384
385 bspin - "$MSG $GEAR"
386 while true; do
Matteo Scandolo78dfee02020-02-13 16:18:03 -0800387 if [ $TYPE == "file" ]; then
388 (set -x; curl --fail -sSL --user karaf:karaf -X POST -H Content-Type:application/json http://127.0.0.1:$ONOS_API_PORT/onos/v1/$RESOURCE --data @$DATA >>$LOG 2>&1) >>$LOG 2>&1
389 fi
390 if [ $TYPE == "json" ]; then
391 (set -x; curl --fail -sSL --user karaf:karaf -X POST -H Content-Type:application/json http://127.0.0.1:$ONOS_API_PORT/onos/v1/$RESOURCE --data $DATA >>$LOG 2>&1) >>$LOG 2>&1
392 fi
David Bainbridge4f1b5562019-08-19 04:46:52 +0000393 if [ $? -eq 0 ]; then
394 break
395 fi
Matteo Scandolo9a77a732020-01-09 14:24:12 -0800396 sleep 1
David Bainbridge4f1b5562019-08-19 04:46:52 +0000397 sspin -
398 done
399 espin - $VERIFIED
400}
401
Matteo Scandolo9a77a732020-01-09 14:24:12 -0800402check_onos_app_active() {
403 local APP_ID=$1
404
405 bspin - "Checking that $APP_ID is active $CLOCK"
406 while true; do
407 (set -x; curl --fail -sSL --user karaf:karaf -X GET http://127.0.0.1:$ONOS_API_PORT/onos/v1/applications/$APP_ID | grep ACTIVE >>$LOG 2>&1) >>$LOG 2>&1
408 if [ $? -eq 0 ]; then
409 break
410 fi
411 sleep 1
412 sspin -
413 done
414 sleep 5 # OSGI components take a little longer that the app to activate
415 espin - $VERIFIED
416}
417
David Bainbridge4f1b5562019-08-19 04:46:52 +0000418override_onos_app() {
419 local APP=$1
420 local NAME=$(basename $APP | sed -e 's/^[0-9][0-9]*-//g' -e 's/-.*$//g')
421 while true; do
422 sspin -
423 # Attempt to delete old version (if it exists)
424 (set -x; curl --fail -sSL --user karaf:karaf -X DELETE http://127.0.0.1:$ONOS_API_PORT/onos/v1/applications/$NAME >>$LOG 2>&1) >>$LOG 2>&1
425 sspin -
426 if [ $? -ne 0 ]; then
427 continue
428 fi
429 (set -x; curl --fail -sSL --user karaf:karaf -X POST -H Content-Type:application/octet-stream http://127.0.0.1:$ONOS_API_PORT/onos/v1/applications?activate=true --data-binary @$APP >>$LOG 2>&1) >>$LOG 2>&1
430 if [ $? -eq 0 ]; then
431 break
432 fi
433 sleep .2
434 done
435}
436
437activate_onos_app() {
438 local MSG="$1"
439 local APP=$2
440
441 bspin - "$MSG $GO"
442 while true; do
443 sspin -
444 (set -x; curl --fail -sSL --user karaf:karaf -X POST http://127.0.0.1:$ONOS_API_PORT/onos/v1/applications/$APP/active >>$LOG 2>&1) >>$LOG 2>&1
445 if [ $? -eq 0 ]; then
446 break
447 fi
448 sleep .2
449 done
450 espin - "$VERIFIED"
451}
452
453count_pods() {
454 local NAMESPACE=$1; shift
455 if [ "$NAMESPACE" == "all-namespaces" ]; then
456 NAMESPACE="--all-namespaces"
457 else
458 NAMESPACE="-n $NAMESPACE"
459 fi
460 echo "$NAMESPACE" > /tmp/t
461 local STATES=$1; shift
462 echo "$STATES" >> /tmp/t
463 local PODS=$(kubectl get $NAMESPACE pod -o go-template="{{range .items}}{{.metadata.name}}/{{.status.phase}}/_{{range .status.containerStatuses}}{{.ready}}_{{end}} {{end}}")
464 local COUNT=0
465 local PATTERNS="$*"
466 for POD in $PODS; do
467 local NAME=$(echo $POD | cut -d/ -f 1)
468 local STATE=$(echo $POD | cut -d/ -f 2)
469 local CONTAINERS=$(echo $POD | cut -d/ -f 3 | sed -e 's/_/ /g')
470 if [ "$STATES" == "*" -o $(echo "$STATES" | grep -c ":$STATE:") -ne 0 ]; then
471 echo "IN WITH $POD" >> /tmp/t
472 local TOTAL=$(echo $CONTAINERS | wc -w)
473 local FOUND=$(echo $CONTAINERS | grep -o true | wc -l)
474 if [ $TOTAL -eq $FOUND ]; then
475 for PATTERN in $PATTERNS; do
476 echo "TEST $PATTERN AGAINST $NAME" >> /tmp/t
477 if [[ $NAME =~ $PATTERN ]]; then
478 echo "COUNT $NAME" >> /tmp/t
479 COUNT=$(expr $COUNT + 1)
480 break
481 fi
482 done
483 fi
484 fi
485 done
486 echo $COUNT >> /tmp/t
487 echo $COUNT
488}
489
490wait_for_pods() {
491 local INDENT=
492 if [ "$1" == "-" ]; then
493 INDENT=$1; shift
494 fi
495 local NAMESPACE=$1; shift
496 local EXPECT=$1; shift
497 local TYPE=$1; shift
498 local RETRY=$1; shift
499 local MESSAGE=$1; shift
500 local PATTERNS=$*
David Bainbridge99ac7a22019-08-31 02:26:43 +0000501 local STATES=":Running:"
502 if [ "$TYPE" == "not" ]; then
503 STATES="*"
504 fi
505 local HAVE=$(count_pods $NAMESPACE "$STATES" $PATTERNS)
David Bainbridge4f1b5562019-08-19 04:46:52 +0000506 local ALL=$HAVE
507 if [ "$TYPE" == "only" ]; then
508 ALL=$(count_pods "all-namespaces" "*" ".*")
509 fi
510 COUNT=$(expr 300 / 15)
511 bspin $INDENT $MESSAGE
512 sspin $INDENT
513 if [ $HAVE -ne $EXPECT -o $ALL -ne $HAVE ]; then
514 while [ $HAVE -ne $EXPECT -o $ALL -ne $HAVE ]; do
515 sspin $INDENT
516 COUNT=$(expr $COUNT - 1)
517 if [ $COUNT -eq 0 ]; then
David Bainbridge99ac7a22019-08-31 02:26:43 +0000518 HAVE=$(count_pods $NAMESPACE "$STATES" $PATTERNS)
David Bainbridge4f1b5562019-08-19 04:46:52 +0000519 ALL=$HAVE
520 if [ "$TYPE" == "only" ]; then
521 ALL=$(count_pods "all-namespaces" "*" ".*")
522 fi
523 COUNT=$(expr 300 / 15)
524 fi
525 sleep .15
526 done
527 fi
528 espin $INDENT $VERIFIED
529 if [ $HAVE -ne $EXPECT ]; then
530 return 1
531 fi
532 return 0
533}
534
David K. Bainbridgefe1fa8a2019-11-12 12:03:21 -0800535port_forward() {
536 local NS=$1
537 local SVC=$2
538 local FROM_PORT=$3
539 local TO_PORT=$4
540 local TAG=$SVC-$NAME
541
542 (set -x; _TAG=$TAG bash -c "while true; do kubectl port-forward -n $NS service/$SVC $FROM_PORT:$TO_PORT; done" >>$LOG 2>&1 &) >>$LOG 2>&1
543}
544
545kill_port_forward() {
546 local TAG=$1-$NAME
David Bainbridgec6871d22019-11-13 17:10:31 +0000547 local P_IDS=$(ps e -ww -A | grep "_TAG=$TAG" | grep -v grep | awk '{print $1}')
David Bainbridgebcf235b2019-11-12 21:33:58 +0000548 local PARENTS=
David K. Bainbridgefe1fa8a2019-11-12 12:03:21 -0800549 local KIDS=
David Bainbridgec6871d22019-11-13 17:10:31 +0000550 local UNKNOWN=
551 if [ ! -z "$P_IDS" ]; then
552 for P_ID in $P_IDS; do
553 local PP_ID=$(ps -o ppid $P_ID | tail -n +2)
554 if [ ! -z "$PP_ID" ]; then
555 if [ $PP_ID -eq 1 ]; then
556 PARENTS="$PARENTS $P_ID"
557 else
558 KIDS="$KIDS $P_ID"
559 fi
David K. Bainbridgefe1fa8a2019-11-12 12:03:21 -0800560 else
David Bainbridgec6871d22019-11-13 17:10:31 +0000561 UNKNOWN="$UNKNOWN $P_ID"
David K. Bainbridgefe1fa8a2019-11-12 12:03:21 -0800562 fi
563 done
David Bainbridgec6871d22019-11-13 17:10:31 +0000564 (set -x; kill -9 $PARENTS $KIDS $UNKNOWN >>$LOG 2>&1) >>$LOG 2>&1
David K. Bainbridgefe1fa8a2019-11-12 12:03:21 -0800565 fi
566}
567
David Bainbridgeac7f8072019-08-01 22:15:33 +0000568if [ "$1" == "down" ]; then
David Bainbridge0774b232019-08-02 06:37:19 +0000569 echo "Tearing down voltha cluster $NAME"
570 LOG="down-$NAME.log"
571 echo $(date -u +"%Y%m%dT%H%M%SZ") >$LOG
David K. Bainbridgefe1fa8a2019-11-12 12:03:21 -0800572 bspin "Remove port-forwards: onos-ui-$NAME"
573 kill_port_forward onos-ui
574 sspin "Remove port-forwards: onos-ssh-$NAME$CEOL"
575 kill_port_forward onos-ssh
576 sspin "Remove port-forwards: voltha-api-$NAME$CEOL"
577 kill_port_forward voltha-api
David K. Bainbridgefe1fa8a2019-11-12 12:03:21 -0800578 sspin "Remove port-forwards: voltha-etcd-$NAME$CEOL"
579 kill_port_forward voltha-etcd-cluster-client
580 espin "$VERIFIED Remove port-forwards$CEOL"
David Bainbridgeac7f8072019-08-01 22:15:33 +0000581 if [ $DEPLOY_K8S == "yes" ]; then
582 if [ -x ./bin/kind ]; then
David Bainbridge0774b232019-08-02 06:37:19 +0000583 bspin "Delete Kubernetes Kind Cluster"
584 (set -x; ./bin/kind delete cluster --name voltha-$NAME >>$LOG 2>&1) >>$LOG 2>&1
585 espin $VERIFIED
David Bainbridgeac7f8072019-08-01 22:15:33 +0000586 else
David Bainbridge0774b232019-08-02 06:37:19 +0000587 espin "$NO_ENTRY Delete Kubernetes Kind Cluster: kind command not found"
David Bainbridgeac7f8072019-08-01 22:15:33 +0000588 fi
589 else
590 EXISTS=$(helm list -q)
591 EXPECT="etcd-operator onos open-olt open-onu sim voltha bbsim radius"
David Bainbridge0774b232019-08-02 06:37:19 +0000592 bspin "Remove Helm Deployments"
David Bainbridgeac7f8072019-08-01 22:15:33 +0000593 for i in $EXISTS; do
594 if [ $(echo $EXPECT | grep -c $i) -eq 1 ]; then
David Bainbridge0774b232019-08-02 06:37:19 +0000595 sspin "Remove Helm Deployments: $i$CEOL"
David K. Bainbridgefe1fa8a2019-11-12 12:03:21 -0800596 (set -x; ./bin/helm delete --no-hooks --purge $i >>$LOG 2>&1) >>$LOG 2>&1
David Bainbridgeac7f8072019-08-01 22:15:33 +0000597 fi
598 done
David Bainbridge0774b232019-08-02 06:37:19 +0000599 espin "$VERIFIED Remove Helm Deployments$CEOL"
David Bainbridge4f1b5562019-08-19 04:46:52 +0000600 if [ "$WAIT_ON_DOWN" == "yes" ]; then
601 # There should only be 13 or 15 PODs in the kube-system name
602 # space and no other PODs
David Bainbridge4f1b5562019-08-19 04:46:52 +0000603 PODS="coredns-.* \
604 etcd-voltha-$NAME-control-plane \
605 kindnet-.* \
606 kube-apiserver-voltha-$NAME-control-plane \
607 kube-controller-manager-voltha-$NAME-control-plane \
608 kube-proxy-.* \
609 kube-scheduler-voltha-$NAME-control-plane \
610 tiller-deploy-.*"
611 EXPECT=$(test "$TYPE" == "minimal" && echo "13" || echo "15")
David Bainbridge99ac7a22019-08-31 02:26:43 +0000612 PODS="adapter-.* \
613 bbsim.* \
614 etcd-operator.* \
615 radius.* \
616 voltha-.*"
Matteo Scandolo73442f92020-01-29 09:53:22 -0800617
David Bainbridge99ac7a22019-08-31 02:26:43 +0000618 EXPECT=0
619
Matteo Scandolo73442f92020-01-29 09:53:22 -0800620 if [ "$WITH_ONOS" == "yes" ]; then
621 ONOS_PODS="onos-.*"
622 wait_for_pods "default" $EXPECT "not" -1 "Waiting for ONOS PODs to terminate" $ONOS_PODS
623 fi
David Bainbridge99ac7a22019-08-31 02:26:43 +0000624 wait_for_pods "voltha" $EXPECT "not" -1 "Waiting for VOLTHA PODs to terminate" $PODS
David Bainbridge4f1b5562019-08-19 04:46:52 +0000625 fi
David Bainbridgeac7f8072019-08-01 22:15:33 +0000626 fi
Matteo Scandolo7ef6ba52019-10-04 09:20:37 -0700627 exit
David Bainbridgeac7f8072019-08-01 22:15:33 +0000628fi
629
630if [ "$1" == "dump" ]; then
David Bainbridge0774b232019-08-02 06:37:19 +0000631 LOG="dump-$NAME.log"
David Bainbridgeac7f8072019-08-01 22:15:33 +0000632 TS=$(date -u +"%Y%m%dT%H%M%SZ")
David Bainbridge76514a02019-10-08 01:50:35 +0000633 if [ ! -z "$DUMP_FROM" ]; then
634 TS=$(echo $DUMP_FROM | sed -e 's/[:-]//g')
635 fi
David Bainbridgeac7f8072019-08-01 22:15:33 +0000636 WORK=$(mktemp -u -d)
David Bainbridge0774b232019-08-02 06:37:19 +0000637 DATA=$WORK/voltha-debug-dump-$NAME-$TS
David Bainbridgeac7f8072019-08-01 22:15:33 +0000638 mkdir -p $DATA
639 echo $TS > $LOG
David Bainbridge0774b232019-08-02 06:37:19 +0000640 echo -e "Capturing debug dump to voltha-debug-dump-$NAME-$TS.tgz"
David Bainbridgeac7f8072019-08-01 22:15:33 +0000641 bspin - "Copy install log"
David Bainbridge0774b232019-08-02 06:37:19 +0000642 if [ -f install-$NAME.log ]; then
643 (set -x; cp install-$NAME.log $DATA/install-$NAME.log) >>$LOG 2>&1
David Bainbridgeac7f8072019-08-01 22:15:33 +0000644 espin - $VERIFIED
645 else
David Bainbridge0774b232019-08-02 06:37:19 +0000646 espin - "$NO_ENTRY Copy install log: install-$NAME.log not found"
David Bainbridgeac7f8072019-08-01 22:15:33 +0000647 fi
648 bspin - "Dumping Kubernetes PODs"
649 (set -x; kubectl get --all-namespaces pods >> $DATA/all-pods.txt 2>&1) >>$LOG 2>&1
650 espin - $VERIFIED
651 bspin - "Dumping Kubernetes SERVICEs"
652 (set -x; kubectl get --all-namespaces svc >> $DATA/all-services.txt 2>&1) >>$LOG 2>&1
653 espin - $VERIFIED
654 bspin - "Dumping Kubernetes EVENTs"
655 (set -x; kubectl get --all-namespaces events >> $DATA/all-events.txt 2>&1) >>$LOG 2>&1
656 espin - $VERIFIED
657 bspin - "Dumping VOLTHA POD details"
Matteo Scandolodcd29f52019-10-07 15:42:42 -0700658 PODS="$(kubectl -n default get pod -o name | grep onos | sed -e 's/^/default:/g') $(kubectl get -n voltha pod -o name | sed -e 's/^/voltha:/g')"
David Bainbridgecfd7ca12019-10-06 03:31:41 +0000659 SINCE=
660 if [ ! -z "$DUMP_FROM" ]; then
661 SINCE="--since-time=$DUMP_FROM"
662 fi
David Bainbridgeac7f8072019-08-01 22:15:33 +0000663 for POD in $PODS; do
David Bainbridge1f0655a2019-10-01 22:08:48 +0000664 NS=$(echo $POD | cut -d: -f1)
665 POD=$(echo $POD | cut -d: -f2)
David Bainbridgeac7f8072019-08-01 22:15:33 +0000666 sspin - "Dumping VOLTHA POD details: $POD$CEOL"
667 mkdir -p $DATA/$POD
David Bainbridge1f0655a2019-10-01 22:08:48 +0000668 (set -x; kubectl describe -n $NS $POD >> $DATA/$POD/describe.txt 2>&1) >>$LOG 2>&1
David Bainbridgeac7f8072019-08-01 22:15:33 +0000669 sspin - "Dumping VOLTHA POD details: $POD"
David Bainbridgecfd7ca12019-10-06 03:31:41 +0000670 (set -x; kubectl logs -n $NS --all-containers $SINCE --previous $LOG_ARGS $POD >> $DATA/$POD/logs-previous.txt 2>&1) >>$LOG 2>&1
David Bainbridgeac7f8072019-08-01 22:15:33 +0000671 sspin - "Dumping VOLTHA POD details: $POD"
David Bainbridgecfd7ca12019-10-06 03:31:41 +0000672 (set -x; kubectl logs -n $NS --all-containers $SINCE $LOG_ARGS $POD >> $DATA/$POD/logs-current.txt 2>&1) >>$LOG 2>&1
David Bainbridgeac7f8072019-08-01 22:15:33 +0000673 sspin - "Dumping VOLTHA POD details: $POD"
674 done
675 espin - "$VERIFIED Dumping VOLTHA POD details$CEOL"
676 bspin - "Dumping ETCD"
677 if [ "$(which etcdctl) X" != " X" ]; then
David Bainbridge0774b232019-08-02 06:37:19 +0000678 (set -x; ETCDCTL_API=3 etcdctl --endpoints localhost:$VOLTHA_ETCD_PORT get --prefix service/voltha | hexdump -C >> $DATA/etcd.hex 2>&1) >>$LOG 2>&1
David Bainbridgeac7f8072019-08-01 22:15:33 +0000679 espin - $VERIFIED
680 else
681 espin - "$NO_ENTRY Dumping ETCD: etcdctl command not available"
682 fi
David Bainbridge0774b232019-08-02 06:37:19 +0000683 bspin - "Creating compressed TAR: voltha-debug-dump-$NAME-$TS.tgz"
684 (set -x; tar -C $WORK -zcf voltha-debug-dump-$NAME-$TS.tgz ./voltha-debug-dump-$NAME-$TS) >>$LOG 2>&1
David Bainbridgeac7f8072019-08-01 22:15:33 +0000685 espin - $VERIFIED
686 bspin - "Cleanup"
687 (set -x; rm -rf $WORK) >>$LOG 2>&1
688 espin - $VERIFIED
David Bainbridge0774b232019-08-02 06:37:19 +0000689 bspin - "$(ls -l voltha-debug-dump-$NAME-$TS.tgz)"
David Bainbridgeac7f8072019-08-01 22:15:33 +0000690 espin - $VERIFIED
691 exit
692fi
693
David Bainbridge0774b232019-08-02 06:37:19 +0000694LOG="install-$NAME.log"
David Bainbridgeac7f8072019-08-01 22:15:33 +0000695date > $LOG
David Bainbridge0774b232019-08-02 06:37:19 +0000696echo "PORTS=$ONOS_API_PORT,$ONOS_SSH_PORT,$VOLTHA_API_PORT,$VOLTHA_SSH_PORT,$VOLTHA_ETCD_PORT" >> $LOG
David Bainbridgeac7f8072019-08-01 22:15:33 +0000697
698# Output install options to log
699echo "OPTIONS" >> $LOG
David Bainbridgeac7f8072019-08-01 22:15:33 +0000700for O in $ALL_OPTIONS; do
701 VAL=$(eval echo \$$O)
David Bainbridgeac7f8072019-08-01 22:15:33 +0000702 if [ ! -z "$VAL" ]; then
703 printf " %-30s = %s\n" $O $VAL >> $LOG
704 fi
705done
706
Matteo Scandolo7ef6ba52019-10-04 09:20:37 -0700707helm_install() {
David K. Bainbridgeb7285432019-07-02 22:05:24 -0700708 local INDENT=
709 if [ "$1" == "-" ]; then
710 INDENT=$1; shift
711 fi
712 local NAMESPACE=$1; shift
David Bainbridge0774b232019-08-02 06:37:19 +0000713 local INAME=$1; shift
David K. Bainbridgeb7285432019-07-02 22:05:24 -0700714 local CHART=$1; shift
David Bainbridge90fd8e32019-08-21 23:32:47 +0000715 local CHART_VERSION=$1; shift
David K. Bainbridgeb7285432019-07-02 22:05:24 -0700716 local MESSAGE=$*
717
David Bainbridge90fd8e32019-08-21 23:32:47 +0000718 if [ "$CHART_VERSION X" != " X" -a "$CHART_VERSION" != "latest" ]; then
719 CHART_VERSION="--version $CHART_VERSION"
720 else
721 CHART_VERSION=
722 fi
723
David Bainbridge99ac7a22019-08-31 02:26:43 +0000724 local CHART_ARGS=
725 if [ -r "${INAME}-values.yaml" ]; then
726 CHART_ARGS="-f ${INAME}-values.yaml"
727 fi
728
David K. Bainbridgeb7285432019-07-02 22:05:24 -0700729 COUNT=$(expr 300 / 15)
730 bspin $INDENT $MESSAGE
David Bainbridge70c0ea82019-11-14 23:25:26 +0000731 (set -x; helm install -f $NAME-values.yaml $CHART_ARGS $EXTRA_HELM_INSTALL_ARGS --set defaults.log_level=$VOLTHA_LOG_LEVEL --namespace $NAMESPACE --name $INAME $CHART_VERSION $EXTRA_HELM_FLAGS $CHART >>$LOG 2>&1) >>$LOG 2>&1
David K. Bainbridgeb7285432019-07-02 22:05:24 -0700732 SUCCESS=$?
733 while [ $SUCCESS -ne 0 ]; do
734 sspin $INDENT
735 COUNT=$(expr $COUNT - 1)
736 if [ $COUNT -eq 0 ]; then
Andy Baviere22ff252019-12-19 11:40:31 -0700737 (set -x; helm delete --purge $INAME >>$LOG 2>&1) >>$LOG 2>&1
David Bainbridge99ac7a22019-08-31 02:26:43 +0000738 (set -x; helm install -f $NAME-values.yaml $CHART_ARGS --set defaults.log_level=$VOLTHA_LOG_LEVEL --namespace $NAMESPACE --name $INAME $CHART_VERSION $EXTRA_HELM_FLAGS $CHART >>$LOG 2>&1) >>$LOG 2>&1
Andy Baviere22ff252019-12-19 11:40:31 -0700739 SUCCESS=$?
David K. Bainbridgeb7285432019-07-02 22:05:24 -0700740 COUNT=$(expr 300 / 15)
741 fi
742 sleep .15
743 done
744 espin $INDENT $VERIFIED
745}
746
David K. Bainbridgeb7285432019-07-02 22:05:24 -0700747echo "INSTALL TYPE: $TYPE" >> $LOG
748
749bspin "Verify GOPATH"
750export GOPATH=$(pwd)
Test User3d7ad8e2019-07-03 06:15:44 +0000751mkdir -p $GOPATH/bin
David K. Bainbridgeb7285432019-07-02 22:05:24 -0700752espin $VERIFIED
753
David Bainbridgee87067b2019-08-12 22:00:12 +0000754STIME=$(date +%s)
David K. Bainbridge0e89cb92019-07-17 11:30:10 -0700755if [ "$INSTALL_KUBECTL" == "no" ]; then
756 bspin "Skip kubectl install"
757 espin $NO_ENTRY
Test Userc13bdc92019-07-03 20:57:49 +0000758else
David K. Bainbridge0e89cb92019-07-17 11:30:10 -0700759 bspin "Verify kubectl $HELM"
760 if [ -x $GOPATH/bin/kubectl ]; then
761 espin $VERIFIED
762 else
763 espin $NOT_VERIFIED
764 bspin - "Download and install Kubernetes/kubectl $DOWNLOAD"
765 (set -x; curl -o $GOPATH/bin/kubectl -sSL https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/$HOSTOS/$HOSTARCH/kubectl >>$LOG 2>&1) >>$LOG 2>&1
766 (set -x; chmod 755 $GOPATH/bin/kubectl >>$LOG 2>&1) >>$LOG 2>&1
767 espin - $VERIFIED
768 fi
Test Userc13bdc92019-07-03 20:57:49 +0000769fi
David Bainbridgee87067b2019-08-12 22:00:12 +0000770if [ "$WITH_TIMINGS" == "yes" ]; then
David Bainbridge712afb82019-08-14 19:55:58 +0000771 printtime $(expr $(date +%s) - $STIME)
David Bainbridgee87067b2019-08-12 22:00:12 +0000772fi
Test Userc13bdc92019-07-03 20:57:49 +0000773
David Bainbridgee87067b2019-08-12 22:00:12 +0000774STIME=$(date +%s)
David K. Bainbridge0e89cb92019-07-17 11:30:10 -0700775if [ "$DEPLOY_K8S" == "no" ]; then
776 bspin "Skip Kubernetes/Kind Deployment"
777 espin $NO_ENTRY
778else
Test Userba1e7e72019-07-10 22:27:54 +0000779 bspin "Verify Kubernetes/Kind $HELM"
David K. Bainbridgefe1fa8a2019-11-12 12:03:21 -0800780 IS_INSTALLED=0
781 OP_TYPE="install"
Test Userba1e7e72019-07-10 22:27:54 +0000782 if [ -x $GOPATH/bin/kind ]; then
David K. Bainbridgefe1fa8a2019-11-12 12:03:21 -0800783 OP_TYPE="upgrade"
784 if [ $($GOPATH/bin/kind --version | grep -c $KIND_VERSION) -eq 1 ]; then
785 IS_INSTALLED=1
786 espin $VERIFIED
787 fi
788 fi
789 if [ $IS_INSTALLED -eq 0 ]; then
Test Userba1e7e72019-07-10 22:27:54 +0000790 espin $NOT_VERIFIED
David K. Bainbridgefe1fa8a2019-11-12 12:03:21 -0800791 bspin - "Download and $OP_TYPE Kubernetes/kind $DOWNLOAD"
David Bainbridge9e2a6662019-07-11 17:07:57 +0000792 (set -x; curl -o $GOPATH/bin/kind -sSL https://github.com/kubernetes-sigs/kind/releases/download/$KIND_VERSION/kind-$HOSTOS-$HOSTARCH >>$LOG 2>&1) >>$LOG 2>&1
Test Userba1e7e72019-07-10 22:27:54 +0000793 (set -x; chmod 755 $GOPATH/bin/kind >>$LOG 2>&1) >>$LOG 2>&1
794 espin - $VERIFIED
795 fi
David K. Bainbridgeb7285432019-07-02 22:05:24 -0700796fi
797
David Bainbridgee87067b2019-08-12 22:00:12 +0000798if [ "$WITH_TIMINGS" == "yes" ]; then
David Bainbridge712afb82019-08-14 19:55:58 +0000799 printtime $(expr $(date +%s) - $STIME)
David Bainbridgee87067b2019-08-12 22:00:12 +0000800fi
801
802STIME=$(date +%s)
David K. Bainbridge0e89cb92019-07-17 11:30:10 -0700803if [ "$INSTALL_HELM" == "no" ]; then
804 bspin "Skip Helm Install"
805 espin $NO_ENTRY
David K. Bainbridgeb7285432019-07-02 22:05:24 -0700806else
David K. Bainbridge0e89cb92019-07-17 11:30:10 -0700807 bspin "Verify Helm $HELM"
808 if [ -x $GOPATH/bin/helm ]; then
809 espin $VERIFIED
810 else
811 espin $NOT_VERIFIED
812 bspin - "Download and install Helm $DOWNLOAD"
David Bainbridgec6a6eb62019-10-18 22:50:07 +0000813 (set -x; curl -sSL https://git.io/get_helm.sh | DESIRED_VERSION=v2.14.3 USE_SUDO=false HELM_INSTALL_DIR=$GOPATH/bin bash >>$LOG 2>&1) >>$LOG 2>&1
David K. Bainbridge0e89cb92019-07-17 11:30:10 -0700814 espin - $VERIFIED
815 fi
David K. Bainbridgeb7285432019-07-02 22:05:24 -0700816fi
David K. Bainbridgefe1fa8a2019-11-12 12:03:21 -0800817
David Bainbridgee87067b2019-08-12 22:00:12 +0000818if [ "$WITH_TIMINGS" == "yes" ]; then
David Bainbridge712afb82019-08-14 19:55:58 +0000819 printtime $(expr $(date +%s) - $STIME)
David Bainbridgee87067b2019-08-12 22:00:12 +0000820fi
David K. Bainbridgeb7285432019-07-02 22:05:24 -0700821
David Bainbridgee87067b2019-08-12 22:00:12 +0000822STIME=$(date +%s)
Test Userb5712372019-07-03 21:52:17 +0000823bspin "Verify voltctl $HIGH_VOLTAGE"
David Bainbridged2811972019-12-14 01:14:09 +0000824VOK=0
825VMESSAGE="install"
David K. Bainbridgeebf38d92020-02-10 10:46:31 -0800826export VC_VERSION="$VOLTCTL_VERSION"
827if [ "$VC_VERSION" == "latest" ]; then
828 export VC_VERSION=$(curl -sSL https://api.github.com/repos/opencord/voltctl/releases/latest | jq -r .tag_name | sed -e 's/^v//g')
829fi
Test User3d7ad8e2019-07-03 06:15:44 +0000830if [ -x $GOPATH/bin/voltctl ]; then
David Bainbridge1ba64f12019-12-18 00:40:13 +0000831 VHAVE=$($GOPATH/bin/voltctl version --clientonly -o json | jq -r .version)
David K. Bainbridgeebf38d92020-02-10 10:46:31 -0800832 if [ $? -eq 0 -a "$VHAVE" == "$VC_VERSION" ]; then
David Bainbridged2811972019-12-14 01:14:09 +0000833 VOK=1
David Bainbridge1ba64f12019-12-18 00:40:13 +0000834 espin $VERIFIED
835 else
David K. Bainbridgeebf38d92020-02-10 10:46:31 -0800836 VCHECK=$(echo -e "$VHAVE\n$VC_VERSION" | sort -V | head -1)
837 if [ "$VCHECK" == "$VHAVE" ]; then
838 VMESSAGE="upgrade"
839 else
840 VMESSAGE="downgrade"
841 fi
David Bainbridged2811972019-12-14 01:14:09 +0000842 fi
843fi
844
David Bainbridge1ba64f12019-12-18 00:40:13 +0000845if [ $VOK -eq 0 ]; then
David K. Bainbridgeb7285432019-07-02 22:05:24 -0700846 espin $NOT_VERIFIED
David Bainbridged2811972019-12-14 01:14:09 +0000847 bspin - "Download and $VMESSAGE voltctl $DOWNLOAD"
David K. Bainbridgeebf38d92020-02-10 10:46:31 -0800848 (set -x; curl -o $GOPATH/bin/voltctl -sSL https://github.com/opencord/voltctl/releases/download/v$VC_VERSION/voltctl-$VC_VERSION-$HOSTOS-$HOSTARCH >>$LOG 2>&1) >>$LOG 2>&1
Test User08ebbd92019-07-03 17:15:39 +0000849 (set -x; chmod 755 $GOPATH/bin/voltctl >>$LOG 2>&1) >>$LOG 2>&1
David K. Bainbridgeb7285432019-07-02 22:05:24 -0700850 espin - $VERIFIED
851fi
David Bainbridgee87067b2019-08-12 22:00:12 +0000852if [ "$WITH_TIMINGS" == "yes" ]; then
David Bainbridge712afb82019-08-14 19:55:58 +0000853 printtime $(expr $(date +%s) - $STIME)
David Bainbridgee87067b2019-08-12 22:00:12 +0000854fi
David K. Bainbridgeb7285432019-07-02 22:05:24 -0700855
856bspin "Verify command PATH"
Test Userba1e7e72019-07-10 22:27:54 +0000857export PATH=$GOPATH/bin:$PATH
David K. Bainbridgeb7285432019-07-02 22:05:24 -0700858espin $VERIFIED
David K. Bainbridgeb7285432019-07-02 22:05:24 -0700859
David Bainbridgee87067b2019-08-12 22:00:12 +0000860STIME=$(date +%s)
Test Userba1e7e72019-07-10 22:27:54 +0000861if [ "$DEPLOY_K8S" == "yes" ]; then
David Bainbridge0774b232019-08-02 06:37:19 +0000862 HAVE=$(kind get clusters | grep -c voltha-$NAME)
Test Userba1e7e72019-07-10 22:27:54 +0000863 bspin "Verify Kubernetes/Kind Cluster"
864 sspin
865 if [ $HAVE -eq 0 ]; then
David Bainbridge491b1bd2019-07-11 17:53:11 +0000866 espin $NOT_VERIFIED
867 bspin - "Verify cluster configuration"
David Bainbridge0774b232019-08-02 06:37:19 +0000868 if [ ! -r ./$NAME-cluster.cfg ]; then
David Bainbridge491b1bd2019-07-11 17:53:11 +0000869 espin - $NOT_VERIFIED
David Bainbridge0774b232019-08-02 06:37:19 +0000870 bspin - "Download cluster configuration: $TYPE-cluster.cfg to $NAME-cluster.cfg $DOWNLOAD"
David K. Bainbridgea00736b2020-01-08 14:33:17 -0800871 ERR_OUT=$(mktemp)
872 (set -x; curl --fail -o ./$NAME-cluster.cfg -sSL https://raw.githubusercontent.com/ciena/kind-voltha/$VK_RELEASE/$TYPE-cluster.cfg >>$LOG 2>>$ERR_OUT) >>$LOG 2>&1
873 if [ $? -ne 0 ]; then
874 espin - $THEX
875 echo -e "${RED}${BOLD}${ERROR}ERROR: $NAME-cluster.cfg${NORMAL}${RED} - $(cat $ERR_OUT)${NORMAL}"
876 echo "ERROR: $(cat $ERR_OUT)" >>$LOG
877 rm -rf $ERR_OUT ./$NAME-cluster.cfg
878 exit 1
879 fi
880 rm -rf $ERR_OUT
881 else
David K. Bainbridge0e89cb92019-07-17 11:30:10 -0700882 espin - $VERIFIED
David K. Bainbridgea00736b2020-01-08 14:33:17 -0800883 fi
David Bainbridge0774b232019-08-02 06:37:19 +0000884 kind create cluster --name voltha-$NAME --config $NAME-cluster.cfg
David Bainbridge491b1bd2019-07-11 17:53:11 +0000885 else
886 espin $VERIFIED
Test Userba1e7e72019-07-10 22:27:54 +0000887 fi
888
David Bainbridge0774b232019-08-02 06:37:19 +0000889 export KUBECONFIG="$(kind get kubeconfig-path --name="voltha-$NAME")"
Test Userba1e7e72019-07-10 22:27:54 +0000890 P="coredns-.* \
David Bainbridge0774b232019-08-02 06:37:19 +0000891 etcd-voltha-$NAME-control-plane \
Test Userba1e7e72019-07-10 22:27:54 +0000892 kindnet-.* \
David Bainbridge0774b232019-08-02 06:37:19 +0000893 kube-apiserver-voltha-$NAME-control-plane \
894 kube-controller-manager-voltha-$NAME-control-plane \
Test Userba1e7e72019-07-10 22:27:54 +0000895 kube-proxy-.* \
David Bainbridge0774b232019-08-02 06:37:19 +0000896 kube-scheduler-voltha-$NAME-control-plane"
Test Userba1e7e72019-07-10 22:27:54 +0000897
898 EXPECT=$(test "$TYPE" == "minimal" && echo "12" || echo "14")
David Bainbridge4f1b5562019-08-19 04:46:52 +0000899 wait_for_pods - "kube-system" $EXPECT "includes" -1 "Waiting for system PODs to start" $P
David K. Bainbridgeb7285432019-07-02 22:05:24 -0700900fi
David Bainbridgee87067b2019-08-12 22:00:12 +0000901if [ "$WITH_TIMINGS" == "yes" ]; then
David Bainbridge712afb82019-08-14 19:55:58 +0000902 printtime $(expr $(date +%s) - $STIME)
David Bainbridgee87067b2019-08-12 22:00:12 +0000903fi
David K. Bainbridgeb7285432019-07-02 22:05:24 -0700904
David Bainbridgee87067b2019-08-12 22:00:12 +0000905STIME=$(date +%s)
David Bainbridge4c6289c2019-09-03 21:11:32 +0000906COUNT=$(count_pods "kube-system" ":Running:" "tiller-deploy-.*")
David K. Bainbridgeb7285432019-07-02 22:05:24 -0700907bspin "Verify Helm"
Test Userba1e7e72019-07-10 22:27:54 +0000908if [ $COUNT -ne 1 ]; then
David K. Bainbridgeb7285432019-07-02 22:05:24 -0700909 espin $NOT_VERIFIED
Test Userb5712372019-07-03 21:52:17 +0000910 echo -e "Configuring Helm $GEAR"
David K. Bainbridge0e89cb92019-07-17 11:30:10 -0700911 if [ "$INSTALL_HELM" == "no" ]; then
912 bspin - "Skip Helm/Tiller Initialization"
913 espin - $NO_ENTRY
914 else
915 bspin - "Initialize Helm"
916 (set -x; helm init --upgrade >>$LOG 2>&1) >>$LOG 2>&1
917 espin - $VERIFIED
David Bainbridge4f1b5562019-08-19 04:46:52 +0000918 wait_for_pods - "kube-system" 1 "includes" -1 "Waiting for Tiller POD to start" "tiller-deploy-.*"
David K. Bainbridge0e89cb92019-07-17 11:30:10 -0700919 fi
Test Userb5712372019-07-03 21:52:17 +0000920 bspin - "Add Google Incubator repository to Helm"
David K. Bainbridgeb7285432019-07-02 22:05:24 -0700921 (set -x; helm repo add incubator https://kubernetes-charts-incubator.storage.googleapis.com >>$LOG 2>&1) >>$LOG 2>&1
922 espin - $VERIFIED
Matteo Scandolo7ef6ba52019-10-04 09:20:37 -0700923
Test Userba1e7e72019-07-10 22:27:54 +0000924 # HACK (sort-of) - the config for tiller is about to be patched, which will
925 # cause the tiller pod to be recreated. This can sometimes cause a timing
926 # issue with the "wait_for_pods" call on tiller as it may incorrectly
Matteo Scandolo7ef6ba52019-10-04 09:20:37 -0700927 # identify the running/ready tiller pod that is soon to be terminated as
Test Userba1e7e72019-07-10 22:27:54 +0000928 # what it is waiting for. To avoid this issue we do a clean scale down and
Matteo Scandolo7ef6ba52019-10-04 09:20:37 -0700929 # scale up of the pod so the script controlls when it should be expecting
Test Userba1e7e72019-07-10 22:27:54 +0000930 # things
931 (set -x; kubectl -n kube-system scale deploy tiller-deploy --replicas=0 >>$LOG 2>&1) >>$LOG 2>&1
David K. Bainbridgeb7285432019-07-02 22:05:24 -0700932
Test Userb5712372019-07-03 21:52:17 +0000933 bspin - "Add Google Stable repository to Helm"
David K. Bainbridgeb7285432019-07-02 22:05:24 -0700934 (set -x; helm repo add stable https://kubernetes-charts.storage.googleapis.com >>$LOG 2>&1) >>$LOG 2>&1
935 espin - $VERIFIED
Test Userb5712372019-07-03 21:52:17 +0000936 bspin - "Add ONF repository to Helm"
David K. Bainbridgeb7285432019-07-02 22:05:24 -0700937 (set -x; helm repo add onf https://charts.opencord.org >>$LOG 2>&1) >>$LOG 2>&1
938 espin - $VERIFIED
Test Userb5712372019-07-03 21:52:17 +0000939 bspin - "Update Helm repository cache"
David K. Bainbridgeb7285432019-07-02 22:05:24 -0700940 (set -x; helm repo update >>$LOG 2>&1) >>$LOG 2>&1
941 espin - $VERIFIED
Matteo Scandolo7ef6ba52019-10-04 09:20:37 -0700942
David K. Bainbridgeb7285432019-07-02 22:05:24 -0700943 # Create and k8s service account so that Helm can create pods
Test Userb5712372019-07-03 21:52:17 +0000944 bspin - "Create Tiller ServiceAccount"
David K. Bainbridgeb7285432019-07-02 22:05:24 -0700945 (set -x; kubectl create serviceaccount --namespace kube-system tiller >>$LOG 2>&1) >>$LOG 2>&1
946 espin - $VERIFIED
Test Userb5712372019-07-03 21:52:17 +0000947 bspin - "Create Tiller ClusterRoleBinding"
David K. Bainbridgeb7285432019-07-02 22:05:24 -0700948 (set -x; kubectl create clusterrolebinding tiller-cluster-rule --clusterrole=cluster-admin --serviceaccount=kube-system:tiller >>$LOG 2>&1) >>$LOG 2>&1
949 espin - $VERIFIED
Test Userb5712372019-07-03 21:52:17 +0000950 bspin - "Update Tiller Manifest"
David K. Bainbridgeb7285432019-07-02 22:05:24 -0700951 (set -x; kubectl patch deploy --namespace kube-system tiller-deploy -p '{"spec":{"template":{"spec":{"serviceAccount":"tiller"}}}}' >>$LOG 2>&1) >>$LOG 2>&1
Test Userba1e7e72019-07-10 22:27:54 +0000952
953 # HACK (sort-of) - part to, spin it back up
954 (set -x; kubectl -n kube-system scale deploy tiller-deploy --replicas=1 >>$LOG 2>&1) >>$LOG 2>&1
David K. Bainbridgeb7285432019-07-02 22:05:24 -0700955 espin - $VERIFIED
956else
957 espin $VERIFIED
David Bainbridge46505fb2019-10-01 21:13:20 +0000958 if [ "$UPDATE_HELM_REPOS" == "yes" ]; then
959 bspin - "Update Helm repository cache"
960 (set -x; helm repo update >>$LOG 2>&1) >>$LOG 2>&1
961 espin - $VERIFIED
962 fi
David K. Bainbridgeb7285432019-07-02 22:05:24 -0700963fi
David Bainbridge4f1b5562019-08-19 04:46:52 +0000964wait_for_pods - "kube-system" 1 "includes" -1 "Waiting for Tiller POD to start" "tiller-deploy-.*"
David Bainbridgee87067b2019-08-12 22:00:12 +0000965if [ "$WITH_TIMINGS" == "yes" ]; then
David Bainbridge712afb82019-08-14 19:55:58 +0000966 printtime $(expr $(date +%s) - $STIME)
David Bainbridgee87067b2019-08-12 22:00:12 +0000967fi
David K. Bainbridgeb7285432019-07-02 22:05:24 -0700968
David Bainbridgee87067b2019-08-12 22:00:12 +0000969STIME=$(date +%s)
David Bainbridge0774b232019-08-02 06:37:19 +0000970bspin "Verify Helm values file: $NAME-values.yaml"
971if [ ! -r "./$NAME-values.yaml" ]; then
Test Userba1e7e72019-07-10 22:27:54 +0000972 espin $NOT_VERIFIED
David Bainbridge0774b232019-08-02 06:37:19 +0000973 bspin - "Download Helm values file: $TYPE-values.yaml to $NAME-values.yaml $DOWNLOAD"
David K. Bainbridgea00736b2020-01-08 14:33:17 -0800974 ERR_OUT=$(mktemp)
975 (set -x; curl --fail -o ./$NAME-values.yaml -sSL https://raw.githubusercontent.com/ciena/kind-voltha/$VK_RELEASE/$TYPE-values.yaml >>$LOG 2>$ERR_OUT) >>$LOG 2>&1
976 if [ $? -ne 0 ]; then
977 espin - $THEX
978 echo -e "${RED}${BOLD}${ERROR}ERROR: $NAME-values.yaml${NORMAL}${RED} - $(cat $ERR_OUT)${NORMAL}"
979 echo "ERROR: $(cat $ERR_OUT)" >>$LOG
980 rm -rf $ERR_OUT ./$NAME-values.yaml
981 exit 1
982 fi
983 rm -rf $ERR_OUT
Test Userba1e7e72019-07-10 22:27:54 +0000984 espin - $VERIFIED
985else
986 espin $VERIFIED
987fi
David Bainbridgee87067b2019-08-12 22:00:12 +0000988if [ "$WITH_TIMINGS" == "yes" ]; then
David Bainbridge712afb82019-08-14 19:55:58 +0000989 printtime $(expr $(date +%s) - $STIME)
David Bainbridgee87067b2019-08-12 22:00:12 +0000990fi
Test Userba1e7e72019-07-10 22:27:54 +0000991
David Bainbridge99ac7a22019-08-31 02:26:43 +0000992STIME=$(date +%s)
993bspin "Verify or download chart specific values files $DOWNLOAD"
994VALUES_FILES="monkey-values.yaml"
David K. Bainbridgea00736b2020-01-08 14:33:17 -0800995ERR_OUT=$(mktemp)
David Bainbridge99ac7a22019-08-31 02:26:43 +0000996for i in $VALUES_FILES; do
997 if [ ! -r ./$i ]; then
David K. Bainbridgea00736b2020-01-08 14:33:17 -0800998 (set -x; curl --fail -o ./$i -sSL https://raw.githubusercontent.com/ciena/kind-voltha/$VK_RELEASE/$i >>$LOG 2>$ERR_OUT) >>$LOG 2>&1
999 if [ $? -ne 0 ]; then
1000 espin $THEX
1001 echo -e "${RED}${BOLD}${ERROR}ERROR: $i${NORMAL}${RED} - $(cat $ERR_OUT)${NORMAL}"
1002 echo "ERROR: $i - $(cat $ERR_OUT)" >>$LOG
1003 rm -rf $ERR_OUT ./$i
1004 exit 1
1005 fi
1006 rm -rf $ERR_OUT
David Bainbridge99ac7a22019-08-31 02:26:43 +00001007 fi
1008done
1009espin $VERIFIED
1010if [ "$WITH_TIMINGS" == "yes" ]; then
1011 printtime $(expr $(date +%s) - $STIME)
1012fi
1013
1014if [ "$WITH_CHAOS" == "yes" ]; then
1015 bspin "Verify or clone kube-monkey helm chart $DOWNLOAD"
1016 if [ -r ./kube-monkey ]; then
1017 espin $VERIFIED
1018 else
1019 espin $NOT_VERIFIED
1020 bspin - "GIT clone kube-monkey"
1021 (set -x; git clone https://github.com/asobti/kube-monkey kube-monkey >>$LOG 2>&1) >>$LOG 2>&1
1022 espin - $VERIFIED
1023 fi
1024fi
1025
Test Userba1e7e72019-07-10 22:27:54 +00001026if [ "$JUST_K8S" == "yes" ]; then
1027 echo "Environment deployed, not deploying VOLTHA artifacts as requested. Good bye."
1028 echo ""
1029 echo "Please issue the following commands in your terminal to ensure that you" | tee -a $LOG
1030 echo "are accessing the correct Kubernetes/Kind cluster as well as have the " | tee -a $LOG
1031 echo "tools required by VOLTHA in your command path. " | tee -a $LOG
1032 echo "" | tee -a $LOG
1033 echo -en $BOLD
1034 if [ $DEPLOY_K8S == "yes" ]; then
David Bainbridge0774b232019-08-02 06:37:19 +00001035 echo "export KUBECONFIG=\"\$(./bin/kind get kubeconfig-path --name=\"voltha-$NAME\")\"" | tee -a $LOG
Test Userba1e7e72019-07-10 22:27:54 +00001036 fi
1037 echo "export PATH=$GOPATH/bin:\$PATH" | tee -a $LOG
1038 echo -en $NORMAL
1039 echo "" | tee -a $LOG
1040 echo "Thank you for choosing kind-voltha for you quick cluster needs." | tee -a $LOG
1041 exit 0
1042fi
1043
David Bainbridgee87067b2019-08-12 22:00:12 +00001044STIME=$(date +%s)
Test Userb5712372019-07-03 21:52:17 +00001045bspin "Verify ETCD Operator $OLD_KEY"
Test User7d866122019-07-09 17:52:35 +00001046if [ $(helm list --deployed --short --namespace voltha "^etcd-operator\$" | wc -l) -ne 1 ]; then
Matteo Scandolo7ef6ba52019-10-04 09:20:37 -07001047 espin $NOT_VERIFIED
David Bainbridge90fd8e32019-08-21 23:32:47 +00001048 helm_install - voltha etcd-operator stable/etcd-operator latest "Install ETCD Operator"
David K. Bainbridgeb7285432019-07-02 22:05:24 -07001049else
1050 espin $VERIFIED
1051fi
Test User01ed0642019-07-03 20:17:06 +00001052EXPECT=$(test "$TYPE" == "minimal" && echo "1" || echo "3")
David Bainbridge4f1b5562019-08-19 04:46:52 +00001053wait_for_pods - "voltha" $EXPECT "includes" -1 "Waiting for ETCD Operator to start" "etcd-operator-.*"
David Bainbridgee87067b2019-08-12 22:00:12 +00001054if [ "$WITH_TIMINGS" == "yes" ]; then
David Bainbridge712afb82019-08-14 19:55:58 +00001055 printtime $(expr $(date +%s) - $STIME)
David Bainbridgee87067b2019-08-12 22:00:12 +00001056fi
David K. Bainbridgeb7285432019-07-02 22:05:24 -07001057
David Bainbridgee87067b2019-08-12 22:00:12 +00001058STIME=$(date +%s)
David Bainbridgeb270c202019-07-26 01:44:42 +00001059if [ $WITH_ONOS == "yes" ]; then
1060 bspin "Verify ONOS installed $BIRD"
1061 if [ $(helm list --deployed --short --namespace default "^onos\$" | wc -l) -ne 1 ]; then
1062 espin $NOT_VERIFIED
Andrea Campanella6afcb082019-12-18 16:51:30 -08001063 EXTRA_HELM_FLAGS="$SET_TAG $EXTRA_HELM_FLAGS" helm_install - default onos onf/onos $ONOS_CHART_VERSION "Install ONOS"
David Bainbridgeb270c202019-07-26 01:44:42 +00001064 else
1065 espin $VERIFIED
1066 fi
David Bainbridge4f1b5562019-08-19 04:46:52 +00001067 wait_for_pods - "default" 1 "includes" -1 "Waiting for ONOS to start" "onos-.*"
Matteo Scandolo7ef6ba52019-10-04 09:20:37 -07001068
David Bainbridgeb270c202019-07-26 01:44:42 +00001069 bspin - "Forward ONOS API port $FORWARD"
David K. Bainbridgefe1fa8a2019-11-12 12:03:21 -08001070 kill_port_forward onos-ui
1071 port_forward default onos-ui $ONOS_API_PORT 8181
David Bainbridgeb270c202019-07-26 01:44:42 +00001072 espin - $VERIFIED
1073 bspin - "Forward ONOS SSH port $FORWARD"
David K. Bainbridgefe1fa8a2019-11-12 12:03:21 -08001074 kill_port_forward onos-ssh
1075 port_forward default onos-ssh $ONOS_SSH_PORT 8101
David Bainbridgeb270c202019-07-26 01:44:42 +00001076 espin - $VERIFIED
David Bainbridged31d6122019-08-13 19:37:59 +00001077 bspin - "Verify or download ONOS configuration support files $DOWNLOAD"
Matteo Scandolo78dfee02020-02-13 16:18:03 -08001078 ONOS_FILES="olt-onos-enableExtraneousRules.json onos-aaa.json \
David Bainbridge90fd8e32019-08-21 23:32:47 +00001079 onos-dhcpl2relay.json onos-kafka.json onos-sadis-sample.json"
David Bainbridged31d6122019-08-13 19:37:59 +00001080 (set -x; mkdir -p ./onos-files >>$LOG 2>&1) >>$LOG 2>&1
David K. Bainbridgea00736b2020-01-08 14:33:17 -08001081 ERR_OUT=$(mktemp)
David Bainbridge2b19e832019-08-16 02:40:53 +00001082 for i in $ONOS_FILES; do
David Bainbridged31d6122019-08-13 19:37:59 +00001083 if [ ! -r ./onos-files/$i ]; then
David K. Bainbridgea00736b2020-01-08 14:33:17 -08001084 (set -x; curl --fail -o ./onos-files/$i -sSL https://raw.githubusercontent.com/ciena/kind-voltha/$VK_RELEASE/onos-files/$i >>$LOG 2>$ERR_OUT) >>$LOG 2>&1
1085 if [ $? -ne 0 ]; then
1086 espin - $THEX
1087 echo -e "${RED}${BOLD}${ERROR}ERROR: $i${NORMAL}${RED} - $(cat $ERR_OUT)${NORMAL}"
1088 echo "ERROR: $(cat $ERR_OUT)" >>$LOG
1089 rm -rf $ERR_OUT ./onos-files/$i
1090 exit 1
1091 fi
1092 rm -rf $ERR_OUT
David Bainbridged31d6122019-08-13 19:37:59 +00001093 fi
1094 done
Test Userba1e7e72019-07-10 22:27:54 +00001095 espin - $VERIFIED
David Bainbridge27790d62019-08-13 22:43:19 +00001096
1097 if [ $INSTALL_ONOS_APPS == "yes" ]; then
1098 bspin - "Installing custom ONOS applications"
1099 if [ -x onos-files/onos-apps -a $(ls -1 onos-files/onos-apps/*.oar 2>/dev/null | wc -l) -gt 0 ]; then
1100 for OAR in $(ls -1 onos-files/onos-apps/*.oar); do
1101 sspin - "Installing custom ONOS applications - $OAR$CEOL"
David Bainbridge2b19e832019-08-16 02:40:53 +00001102 override_onos_app $OAR
David Bainbridge27790d62019-08-13 22:43:19 +00001103 done
1104 espin - "$VERIFIED Installing custom ONOS applications$CEOL"
1105 else
1106 espin - "$NOT_VERIFIED Installing custom ONOS applications - None Found"
1107 fi
1108 fi
Matteo Scandolo7ef6ba52019-10-04 09:20:37 -07001109
Matteo Scandolo9a77a732020-01-09 14:24:12 -08001110 check_onos_app_active org.opencord.kafka
Matteo Scandolo78dfee02020-02-13 16:18:03 -08001111 push_onos_config "file" "Push ONOS Kafka Configuration" "network/configuration/apps/org.opencord.kafka" "onos-files/onos-kafka.json"
Matteo Scandolo9a77a732020-01-09 14:24:12 -08001112 check_onos_app_active org.opencord.dhcpl2relay
Matteo Scandolo78dfee02020-02-13 16:18:03 -08001113 push_onos_config "file" "Push ONOS DHCP L2 Relay Configuration" "network/configuration/apps/org.opencord.dhcpl2relay" "onos-files/onos-dhcpl2relay.json"
Matteo Scandolo9a77a732020-01-09 14:24:12 -08001114 check_onos_app_active org.opencord.olt
Matteo Scandolo78dfee02020-02-13 16:18:03 -08001115 # FIXME use WITH_DHCP and WITH_EAPOL flags to configre OLT App
1116 if [ $WITH_EAPOL == "yes" ]; then
1117 push_onos_config "json" "Enable VOLTHA ONOS EAPOL provisioning" "configuration/org.opencord.olt.impl.OltFlowService" '{"enableEapol":true}'
1118 fi
1119 if [ $WITH_DHCP == "yes" ]; then
1120 push_onos_config "json" "Enable VOLTHA ONOS DHCP provisioning" "configuration/org.opencord.olt.impl.OltFlowService" '{"enableDhcpOnProvisioning":true,"enableDhcpV4":true}'
1121 fi
1122 if [ $WITH_IGMP == "yes" ]; then
1123 push_onos_config "json" "Enable VOLTHA ONOS IGMP provisioning" "configuration/org.opencord.olt.impl.OltFlowService" '{"enableIgmpOnProvisioning":true}'
1124 fi
1125 push_onos_config "file" "Enabling extraneous rules for ONOS" "configuration/org.onosproject.net.flow.impl.FlowRuleManager" "onos-files/olt-onos-enableExtraneousRules.json"
David Bainbridge2b19e832019-08-16 02:40:53 +00001126 if [ -f onos-files/onos-sadis.json ]; then
Matteo Scandolo78dfee02020-02-13 16:18:03 -08001127 push_onos_config "file" "[optional] Push ONOS SADIS Configuration" "network/configuration/apps/org.opencord.sadis" "onos-files/onos-sadis.json"
David Bainbridge2b19e832019-08-16 02:40:53 +00001128 elif [ "$CONFIG_SADIS" == "yes" ]; then
David Bainbridge90fd8e32019-08-21 23:32:47 +00001129 SADIS_CFG=onos-files/onos-sadis-sample.json
Matteo Scandolo9a77a732020-01-09 14:24:12 -08001130 check_onos_app_active org.opencord.sadis
Matteo Scandolo78dfee02020-02-13 16:18:03 -08001131 push_onos_config "file" "[optional] Push ONOS SADIS Configuration" "network/configuration/apps/org.opencord.sadis" "$SADIS_CFG"
David Bainbridge2b19e832019-08-16 02:40:53 +00001132 fi
Test Userba1e7e72019-07-10 22:27:54 +00001133fi
David Bainbridgee87067b2019-08-12 22:00:12 +00001134if [ "$WITH_TIMINGS" == "yes" ]; then
David Bainbridge712afb82019-08-14 19:55:58 +00001135 printtime $(expr $(date +%s) - $STIME)
David Bainbridgee87067b2019-08-12 22:00:12 +00001136fi
Test Userba1e7e72019-07-10 22:27:54 +00001137
David Bainbridgee87067b2019-08-12 22:00:12 +00001138STIME=$(date +%s)
Test Userb5712372019-07-03 21:52:17 +00001139bspin "Verify VOLTHA installed $HIGH_VOLTAGE"
Test User7d866122019-07-09 17:52:35 +00001140if [ $(helm list --deployed --short --namespace voltha "^voltha\$" | wc -l) -ne 1 ]; then
David K. Bainbridgeb7285432019-07-02 22:05:24 -07001141 espin $NOT_VERIFIED
David Bainbridge70c0ea82019-11-14 23:25:26 +00001142 if [ "$ONLY_ONE" == "yes" ]; then
1143 EXTRA_HELM_INSTALL_ARGS="--set therecanbeonlyone=true"=
1144 fi
David Bainbridge90fd8e32019-08-21 23:32:47 +00001145 helm_install - voltha voltha $VOLTHA_CHART $VOLTHA_CHART_VERSION "Install VOLTHA Core"
David Bainbridge70c0ea82019-11-14 23:25:26 +00001146 EXTRA_HELM_INSTALL_ARGS=
David K. Bainbridgeb7285432019-07-02 22:05:24 -07001147else
1148 espin $VERIFIED
1149fi
Test Userba1e7e72019-07-10 22:27:54 +00001150
David Bainbridge90fd8e32019-08-21 23:32:47 +00001151VOLTHA="voltha-ofagent-.* \
David K. Bainbridgeb7285432019-07-02 22:05:24 -07001152 ro-core.* \
1153 rw-core.* \
1154 voltha-api-server-.* \
David K. Bainbridgeb7285432019-07-02 22:05:24 -07001155 voltha-etcd-cluster-.* \
1156 voltha-kafka-.* \
1157 voltha-zookeeper-.*"
David Bainbridge70c0ea82019-11-14 23:25:26 +00001158if [ "$ONLY_ONE" == "yes" ]; then
David Bainbridge1ba64f12019-12-18 00:40:13 +00001159 EXPECT=$(test "$TYPE" == "minimal" && echo "5" || echo "7")
David Bainbridge70c0ea82019-11-14 23:25:26 +00001160else
David Bainbridge1ba64f12019-12-18 00:40:13 +00001161 EXPECT=$(test "$TYPE" == "minimal" && echo "8" || echo "10")
David Bainbridge70c0ea82019-11-14 23:25:26 +00001162fi
David Bainbridge4f1b5562019-08-19 04:46:52 +00001163wait_for_pods - "voltha" $EXPECT "includes" -1 "Waiting for VOLTHA Core to start" $VOLTHA
David Bainbridgee87067b2019-08-12 22:00:12 +00001164if [ "$WITH_TIMINGS" == "yes" ]; then
David Bainbridge712afb82019-08-14 19:55:58 +00001165 printtime $(expr $(date +%s) - $STIME)
David Bainbridgee87067b2019-08-12 22:00:12 +00001166fi
David K. Bainbridgeb7285432019-07-02 22:05:24 -07001167
David Bainbridge46505fb2019-10-01 21:13:20 +00001168if [ "$WITH_ADAPTERS" == "yes" ]; then
1169 STIME=$(date +%s)
1170 EXPECT=0
1171 echo -e "Verify Adapters $PLUG"
1172 if [ "$WITH_SIM_ADAPTERS" == "yes" ]; then
1173 bspin - "Verify Simulated Adapters installed"
1174 if [ $(helm list --deployed --short --namespace voltha "^sim\$" | wc -l) -ne 1 ]; then
1175 espin - $NOT_VERIFIED
1176 helm_install - voltha sim $VOLTHA_ADAPTER_SIM_CHART $VOLTHA_ADAPTER_SIM_CHART_VERSION "Install Simulated Adapters"
1177 else
1178 espin - $VERIFIED
1179 fi
1180 EXPECT=$(expr $EXPECT + 2)
1181 fi
Matteo Scandolo7ef6ba52019-10-04 09:20:37 -07001182
David Bainbridge46505fb2019-10-01 21:13:20 +00001183 if [ "$WITH_OPEN_ADAPTERS" == "yes" ]; then
1184 bspin - "Verify OpenOLT Adapter installed"
1185 if [ $(helm list --deployed --short --namespace voltha "^open-olt\$" | wc -l) -ne 1 ]; then
1186 espin - $NOT_VERIFIED
1187 helm_install - voltha open-olt $VOLTHA_ADAPTER_OPEN_OLT_CHART $VOLTHA_ADAPTER_OPEN_OLT_CHART_VERSION "Install OpenOLT Adapter"
1188 else
1189 espin - $VERIFIED
1190 fi
1191 bspin - "Verify OpenONU Adapter installed"
1192 if [ $(helm list --deployed --short --namespace voltha "^open-onu\$" | wc -l) -ne 1 ]; then
1193 espin - $NOT_VERIFIED
1194 helm_install - voltha open-onu $VOLTHA_ADAPTER_OPEN_ONU_CHART $VOLTHA_ADAPTER_OPEN_ONU_CHART_VERSION "Install OpenONU Adapter"
1195 else
1196 espin - $VERIFIED
1197 fi
1198 EXPECT=$(expr $EXPECT + 2)
1199 fi
Matteo Scandolo7ef6ba52019-10-04 09:20:37 -07001200
David Bainbridge46505fb2019-10-01 21:13:20 +00001201 ADAPTERS="adapter-.*"
1202 wait_for_pods - "voltha" $EXPECT "includes" -1 "Waiting for adapters to start" $ADAPTERS
1203 if [ "$WITH_TIMINGS" == "yes" ]; then
1204 printtime $(expr $(date +%s) - $STIME)
1205 fi
David Bainbridgee87067b2019-08-12 22:00:12 +00001206fi
David K. Bainbridgeb7285432019-07-02 22:05:24 -07001207
Test User7d866122019-07-09 17:52:35 +00001208if [ $WITH_BBSIM == "yes" ]; then
David Bainbridgee87067b2019-08-12 22:00:12 +00001209 STIME=$(date +%s)
Test User7d866122019-07-09 17:52:35 +00001210 echo -e "Verify BBSIM $PLUG"
David Bainbridge5b7b96b2019-07-25 20:29:13 +00001211 bspin - "Verify BBSIM Installed"
Test User7d866122019-07-09 17:52:35 +00001212 if [ $(helm list --deployed --short --namespace voltha "^bbsim\$" | wc -l) -ne 1 ]; then
1213 espin - $NOT_VERIFIED
David Bainbridge90fd8e32019-08-21 23:32:47 +00001214 helm_install - voltha bbsim $VOLTHA_BBSIM_CHART $VOLTHA_BBSIM_CHART_VERSION "Install BBSIM"
Test User7d866122019-07-09 17:52:35 +00001215 else
1216 espin - $VERIFIED
1217 fi
David Bainbridge4f1b5562019-08-19 04:46:52 +00001218 wait_for_pods - "voltha" 1 "includes" -1 "Waiting for BBSIM to start" "bbsim-.*"
David Bainbridgee87067b2019-08-12 22:00:12 +00001219 if [ "$WITH_TIMINGS" == "yes" ]; then
David Bainbridge712afb82019-08-14 19:55:58 +00001220 printtime $(expr $(date +%s) - $STIME)
David Bainbridgee87067b2019-08-12 22:00:12 +00001221 fi
Test User7d866122019-07-09 17:52:35 +00001222fi
1223
David Bainbridge5b7b96b2019-07-25 20:29:13 +00001224if [ $WITH_RADIUS == "yes" ]; then
David Bainbridgee87067b2019-08-12 22:00:12 +00001225 STIME=$(date +%s)
David Bainbridge5b7b96b2019-07-25 20:29:13 +00001226 echo -e "Verify RADIUS $LOCK"
1227 bspin - "Verify RADIUS Installed"
1228 if [ $(helm list --deployed --short --namespace voltha "^radius\$" | wc -l) -ne 1 ]; then
1229 espin - $NOT_VERIFIED
David Bainbridge90fd8e32019-08-21 23:32:47 +00001230 helm_install - voltha radius onf/freeradius latest "Install RADIUS"
David Bainbridge5b7b96b2019-07-25 20:29:13 +00001231 else
1232 espin - $VERIFIED
1233 fi
David Bainbridge4f1b5562019-08-19 04:46:52 +00001234 wait_for_pods - "voltha" 1 "includes" -1 "Waiting for RADIUS to start" "radius-.*"
David Bainbridgee87067b2019-08-12 22:00:12 +00001235 if [ "$WITH_TIMINGS" == "yes" ]; then
David Bainbridge712afb82019-08-14 19:55:58 +00001236 printtime $(expr $(date +%s) - $STIME)
David Bainbridgee87067b2019-08-12 22:00:12 +00001237 fi
David Bainbridge5b7b96b2019-07-25 20:29:13 +00001238fi
1239
David Bainbridgee87067b2019-08-12 22:00:12 +00001240STIME=$(date +%s)
Test Userb5712372019-07-03 21:52:17 +00001241bspin - "Forward VOLTHA API port $FORWARD"
David K. Bainbridgefe1fa8a2019-11-12 12:03:21 -08001242kill_port_forward voltha-api
1243port_forward voltha voltha-api $VOLTHA_API_PORT 55555
David K. Bainbridgeb7285432019-07-02 22:05:24 -07001244espin - $VERIFIED
David Bainbridgeac7f8072019-08-01 22:15:33 +00001245bspin - "Forward VOLTHA ETCD port $FORWARD"
David K. Bainbridgefe1fa8a2019-11-12 12:03:21 -08001246kill_port_forward voltha-etcd-cluster-client
1247port_forward voltha voltha-etcd-cluster-client $VOLTHA_ETCD_PORT 2379
David Bainbridgeac7f8072019-08-01 22:15:33 +00001248espin - $VERIFIED
David Bainbridgee87067b2019-08-12 22:00:12 +00001249if [ "$WITH_TIMINGS" == "yes" ]; then
David Bainbridge712afb82019-08-14 19:55:58 +00001250 printtime $(expr $(date +%s) - $STIME)
David Bainbridgee87067b2019-08-12 22:00:12 +00001251fi
Test User3d7ad8e2019-07-03 06:15:44 +00001252
David Bainbridgeb270c202019-07-26 01:44:42 +00001253if [ $WITH_ONOS == "yes" -a $WITH_RADIUS == "yes" ]; then
David Bainbridge5b7b96b2019-07-25 20:29:13 +00001254 bspin "Configure ONOS RADIUS Connection $GEAR"
Andy Bavier658ebef2019-11-05 17:22:56 -07001255 (set -x; cat onos-files/onos-aaa.json | sed -e "s/:RADIUS_IP:/$(kubectl -n voltha get service/radius -o jsonpath={.spec.clusterIP})/g" | curl --fail -sSL --user karaf:karaf -X POST http://127.0.0.1:$ONOS_API_PORT/onos/v1/network/configuration/apps/org.opencord.aaa -H Content-type:application/json -d@- >>$LOG 2>&1) >>$LOG 2>&1
David Bainbridge5b7b96b2019-07-25 20:29:13 +00001256 espin $VERIFIED
1257fi
1258
David Bainbridge99ac7a22019-08-31 02:26:43 +00001259if [ "$WITH_CHAOS" == "yes" ]; then
1260 STIME=$(date +%s)
1261 echo -e "Verify kube-monkey $LOCK"
1262 bspin - "Verify kube-monkey Installed"
1263 if [ $(helm list --deployed --short --namespace kube-monkey "^monkey\$" | wc -l) -ne 1 ]; then
1264 espin - $NOT_VERIFIED
1265 helm_install - kube-monkey monkey ./kube-monkey/helm/kubemonkey latest "Install Chaos Monkey"
1266 else
1267 espin - $VERIFIED
1268 fi
1269 wait_for_pods - "kube-monkey" 1 "includes" -1 "Waiting for Chaos to start" "monkey-.*"
1270 if [ "$WITH_TIMINGS" == "yes" ]; then
1271 printtime $(expr $(date +%s) - $STIME)
1272 fi
1273fi
1274
David Bainbridge5b7b96b2019-07-25 20:29:13 +00001275bspin "Create voltctl configuration file"
Test User3d7ad8e2019-07-03 06:15:44 +00001276(set -x; mkdir -p $HOME/.volt >>$LOG 2>&1) >>$LOG 2>&1
David Bainbridge0774b232019-08-02 06:37:19 +00001277(set -x; voltctl -a v2 -s localhost:$VOLTHA_API_PORT config > $HOME/.volt/config-$NAME 2>>$LOG) >>$LOG 2>&1
David Bainbridge5b7b96b2019-07-25 20:29:13 +00001278espin $VERIFIED
Test User08ebbd92019-07-03 17:15:39 +00001279
David Bainbridge0774b232019-08-02 06:37:19 +00001280if [ ! -f "$NAME-env.sh" ]; then
1281 touch $NAME-env.sh
David Bainbridgeb07fdf72019-07-29 22:51:40 +00001282fi
1283
David Bainbridge596f30d2019-07-30 17:07:59 +00001284for O in $ALL_OPTIONS; do
1285 VAL=$(eval echo \$$O)
David Bainbridge0774b232019-08-02 06:37:19 +00001286 if [ ! -z "$VAL" -a $(grep -c "^export $O=" $NAME-env.sh) -eq 0 ]; then
1287 echo "export $O=\"$(eval echo \$$O)\"" >> $NAME-env.sh
David Bainbridgeb07fdf72019-07-29 22:51:40 +00001288 fi
1289done
1290
David Bainbridge0774b232019-08-02 06:37:19 +00001291if [ $DEPLOY_K8S == "yes" -a $(grep -c "^export KUBECONFIG=" $NAME-env.sh) -eq 0 ]; then
1292 echo "export KUBECONFIG=\"$(./bin/kind get kubeconfig-path --name=voltha-$NAME)\"" >> $NAME-env.sh
David Bainbridgeb07fdf72019-07-29 22:51:40 +00001293fi
1294
David Bainbridge0774b232019-08-02 06:37:19 +00001295if [ $(grep -c "^export VOLTCONFIG=" $NAME-env.sh) -eq 0 ]; then
1296 echo "export VOLTCONFIG=\"$HOME/.volt/config-$NAME\"" >> $NAME-env.sh
David Bainbridgeb07fdf72019-07-29 22:51:40 +00001297fi
1298
David Bainbridge0774b232019-08-02 06:37:19 +00001299if [ $(grep -c "^export PATH=" $NAME-env.sh) -eq 0 ]; then
1300 echo "export PATH=\"$GOPATH/bin:\$PATH\"" >> $NAME-env.sh
David Bainbridgeb07fdf72019-07-29 22:51:40 +00001301fi
1302
Test User7d866122019-07-09 17:52:35 +00001303echo ""
Test User08ebbd92019-07-03 17:15:39 +00001304echo "Please issue the following commands in your terminal to ensure that you" | tee -a $LOG
1305echo "are accessing the correct Kubernetes/Kind cluster as well as have the " | tee -a $LOG
1306echo "tools required by VOLTHA in your command path. " | tee -a $LOG
1307echo "" | tee -a $LOG
Test User7d866122019-07-09 17:52:35 +00001308echo -en $BOLD
Test Userba1e7e72019-07-10 22:27:54 +00001309if [ $DEPLOY_K8S == "yes" ]; then
David Bainbridge0774b232019-08-02 06:37:19 +00001310 echo "export KUBECONFIG=\"\$(./bin/kind get kubeconfig-path --name=\"voltha-$NAME\")\"" | tee -a $LOG
Test Userba1e7e72019-07-10 22:27:54 +00001311fi
David Bainbridge0774b232019-08-02 06:37:19 +00001312echo "export VOLTCONFIG=\"$HOME/.volt/config-$NAME\"" | tee -a $LOG
Test Userba1e7e72019-07-10 22:27:54 +00001313echo "export PATH=$GOPATH/bin:\$PATH" | tee -a $LOG
Test User7d866122019-07-09 17:52:35 +00001314echo -en $NORMAL
Test User08ebbd92019-07-03 17:15:39 +00001315echo "" | tee -a $LOG
1316echo "Thank you for choosing kind-voltha for you quick cluster needs." | tee -a $LOG
1317
David Bainbridge712afb82019-08-14 19:55:58 +00001318if [ "$WITH_TIMINGS" == "yes" ]; then
David Bainbridgef858a022019-08-14 21:25:11 +00001319 echo -e "$CLOCK ${BOLD}TOTAL: $(duration $(expr $(date +%s) - $TOTAL_START_TIME))${NORMAL}"
David Bainbridge712afb82019-08-14 19:55:58 +00001320fi