AETHER-3388: Added rke2 to aether-in-a-box
Change-Id: I1e4e126f601f2ef49b7b184fcfb775a00be57e76
diff --git a/Makefile b/Makefile
index 63be427..47790e9 100644
--- a/Makefile
+++ b/Makefile
@@ -20,9 +20,12 @@
TEST_APP_VALUES?= $(MAKEDIR)/5g-test-apps-values.yaml
KUBESPRAY_VERSION ?= release-2.17
-DOCKER_VERSION ?= '20.10'
-K8S_VERSION ?= v1.20.11
+DOCKER_VERSION ?= '20.10'
HELM_VERSION ?= v3.6.3
+KUBECTL_VERSION ?= v1.23.0
+
+RKE2_K8S_VERSION ?= v1.23.4+rke2r1
+K8S_VERSION ?= v1.20.11
ENABLE_ROUTER ?= true
ENABLE_OAISIM ?= true
@@ -30,6 +33,9 @@
ENABLE_SUBSCRIBER_PROXY ?= false
GNBSIM_COLORS ?= true
+K8S_INSTALL := rke2
+CTR_CMD := sudo /var/lib/rancher/rke2/bin/ctr --address /run/k3s/containerd/containerd.sock --namespace k8s.io
+
DATA_IFACE ?= data
ifeq ($(DATA_IFACE), data)
RAN_SUBNET := 192.168.251.0/24
@@ -60,7 +66,7 @@
os_release := $(shell lsb_release -r -s)
USER := $(shell whoami)
-.PHONY: 4g-core 5g-core oaisim test reset-test reset-ue reset-5g-test clean
+.PHONY: 4g-core 5g-core oaisim test reset-test reset-ue reset-5g-test node-prep clean
$(M):
mkdir -p $(M)
@@ -91,10 +97,21 @@
fi
touch $@
+ifeq ($(K8S_INSTALL),kubespray)
$(M)/setup: | $(M)
sudo $(SCRIPTDIR)/cloudlab-disksetup.sh
sudo apt update; sudo apt install -y software-properties-common python3 python3-pip python3-venv jq httpie ipvsadm
touch $@
+endif
+
+ifeq ($(K8S_INSTALL),rke2)
+$(M)/setup: | $(M)
+ sudo $(SCRIPTDIR)/cloudlab-disksetup.sh
+ sudo apt update; sudo apt install -y software-properties-common python3 python3-pip python3-venv jq httpie ipvsadm apparmor apparmor-utils
+ systemctl list-units --full -all | grep "docker.service" || sudo apt install -y docker.io
+ sudo adduser $(USER) docker || true
+ touch $@
+endif
$(BUILD)/kubespray: | $(M)/setup
mkdir -p $(BUILD)
@@ -111,6 +128,7 @@
pip install -r $(BUILD)/kubespray/requirements.txt
touch $@
+ifeq ($(K8S_INSTALL),kubespray)
$(M)/k8s-ready: | $(M)/setup $(BUILD)/kubespray $(VENV)/bin/activate $(M)/kubespray-requirements
source "$(VENV)/bin/activate" && cd $(BUILD)/kubespray; \
ansible-playbook -b -i inventory/local/hosts.ini \
@@ -142,6 +160,50 @@
helm repo add onosproject https://charts.onosproject.org
helm repo add aether https://charts.aetherproject.org
touch $@
+endif
+
+ifeq ($(K8S_INSTALL),rke2)
+$(M)/k8s-ready: | $(M)/setup
+ sudo mkdir -p /etc/rancher/rke2/
+ [ -d /usr/local/etc/emulab ] && [ ! -e /var/lib/rancher ] && sudo ln -s /var/lib/rancher /mnt/extra/rancher || true # that link gets deleted on cleanup
+ echo "cni: multus,calico" >> config.yaml
+ echo "cluster-cidr: 192.168.84.0/24" >> config.yaml
+ echo "service-cidr: 192.168.85.0/24" >> config.yaml
+ echo "kubelet-arg:" >> config.yaml
+ echo "- --allowed-unsafe-sysctls="net.*"" >> config.yaml
+ echo "- --node-ip="$(NODE_IP)"" >> config.yaml
+ echo "pause-image: k8s.gcr.io/pause:3.3" >> config.yaml
+ echo "kube-proxy-arg:" >> config.yaml
+ echo "- --metrics-bind-address="0.0.0.0:10249"" >> config.yaml
+ echo "- --proxy-mode="ipvs"" >> config.yaml
+ echo "kube-apiserver-arg:" >> config.yaml
+ echo "- --service-node-port-range="2000-36767"" >> config.yaml
+ sudo mv config.yaml /etc/rancher/rke2/
+ curl -sfL https://get.rke2.io | sudo INSTALL_RKE2_VERSION=$(RKE2_K8S_VERSION) sh -
+ sudo systemctl enable rke2-server.service
+ sudo systemctl start rke2-server.service
+ sudo /var/lib/rancher/rke2/bin/kubectl --kubeconfig /etc/rancher/rke2/rke2.yaml wait nodes --for=condition=Ready --all --timeout=300s
+ sudo /var/lib/rancher/rke2/bin/kubectl --kubeconfig /etc/rancher/rke2/rke2.yaml wait deployment -n kube-system --for=condition=available --all --timeout=300s
+ curl -LO "https://dl.k8s.io/release/$(KUBECTL_VERSION)/bin/linux/amd64/kubectl"
+ sudo chmod +x kubectl
+ sudo mv kubectl /usr/local/bin/
+ kubectl version --client
+ mkdir -p $(HOME)/.kube
+ sudo cp /etc/rancher/rke2/rke2.yaml $(HOME)/.kube/config
+ sudo chown -R $(shell id -u):$(shell id -g) $(HOME)/.kube
+ touch $@
+
+$(M)/helm-ready: | $(M)/k8s-ready
+ curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
+ chmod 700 get_helm.sh
+ sudo DESIRED_VERSION=$(HELM_VERSION) ./get_helm.sh
+ helm repo add incubator https://charts.helm.sh/incubator
+ helm repo add cord https://charts.opencord.org
+ helm repo add atomix https://charts.atomix.io
+ helm repo add onosproject https://charts.onosproject.org
+ helm repo add aether https://charts.aetherproject.org
+ touch $@
+endif
/opt/cni/bin/static: | $(M)/k8s-ready
mkdir -p $(BUILD)/cni-plugins; cd $(BUILD)/cni-plugins; \
@@ -227,7 +289,8 @@
mkdir -p $(BUILD)
cd $(BUILD); git clone https://github.com/opencord/openairinterface.git
-download-ue-image: | $(M)/k8s-ready
+ifeq ($(K8S_INSTALL),kubespray)
+download-ue-image: | $(M)/k8s-ready $(BUILD)/openairinterface
sg docker -c "docker pull ${OAISIM_UE_IMAGE} && \
docker tag ${OAISIM_UE_IMAGE} omecproject/lte-uesoftmodem:1.1.0"
touch $(M)/ue-image
@@ -239,6 +302,26 @@
--file Dockerfile.ue \
--tag omecproject/lte-uesoftmodem:1.1.0"
touch $@
+endif
+
+ifeq ($(K8S_INSTALL),rke2)
+download-ue-image: | $(M)/k8s-ready $(BUILD)/openairinterface
+ sg docker -c "docker pull ${OAISIM_UE_IMAGE} && \
+ docker tag ${OAISIM_UE_IMAGE} omecproject/lte-uesoftmodem:1.1.0 && \
+ docker save -o /tmp/lte-uesoftmodem.tar omecproject/lte-uesoftmodem:1.1.0"
+ $(CTR_CMD) images import /tmp/lte-uesoftmodem.tar
+ touch $(M)/ue-image
+
+$(M)/ue-image: $(M)/k8s-ready $(BUILD)/openairinterface
+ cd $(BUILD)/openairinterface; \
+ sg docker -c "docker build . --target lte-uesoftmodem \
+ --build-arg build_base=omecproject/oai-base:1.1.0 \
+ --file Dockerfile.ue \
+ --tag omecproject/lte-uesoftmodem:1.1.0 && \
+ docker save -o /tmp/lte-uesoftmodem.tar omecproject/lte-uesoftmodem:1.1.0"
+ $(CTR_CMD) images import /tmp/lte-uesoftmodem.tar
+ touch $@
+endif
$(M)/oaisim-lo:
@sudo ip addr add 127.0.0.2/8 dev lo || true
@@ -433,6 +516,14 @@
$(5G_TEST_APPS_CHART)
@echo "Finished to dbtestapp"
+ifeq ($(K8S_INSTALL),rke2)
+clean: | oaisim-clean router-clean
+ sudo /usr/local/bin/rke2-uninstall.sh
+ sudo rm -rf /usr/local/bin/kubectl
+ rm -rf $(M)
+endif
+
+ifeq ($(K8S_INSTALL),kubespray)
clean: | oaisim-clean router-clean
source "$(VENV)/bin/activate" && cd $(BUILD)/kubespray; \
ansible-playbook -b -i inventory/local/hosts.ini reset.yml --extra-vars "reset_confirmation=yes"
@@ -441,3 +532,4 @@
sudo rm -rf /mnt/extra/kubelet; \
fi
rm -rf $(M)
+endif
diff --git a/scripts/cloudlab-disksetup.sh b/scripts/cloudlab-disksetup.sh
index 10eb570..08bb6d0 100755
--- a/scripts/cloudlab-disksetup.sh
+++ b/scripts/cloudlab-disksetup.sh
@@ -59,7 +59,7 @@
sudo rm -rf /var/lib/libvirt/images
# create symlinks to /mnt/extra partition
-for DIR in docker kubelet openstack-helm nova libvirt/images
+for DIR in docker kubelet openstack-helm nova libvirt/images rancher
do
sudo mkdir -p "/mnt/extra/$DIR"
sudo chmod -R a+rwx "/mnt/extra/$DIR"
@@ -68,3 +68,6 @@
sudo ln -s "/mnt/extra/$DIR" "/var/lib/$DIR"
fi
done
+sudo mkdir -p "/mnt/extra/k3s"
+sudo chmod -R a+rwx "/mnt/extra/k3s"
+if [ ! -e "/run/k3s" ]; then sudo ln -s "/mnt/extra/k3s" "/run/k3s"; fi