VOL-1406: Deploy voltha 2.0 in a k8s cluster
- Also added sleep to watching rw-core to ensure that the
watching core does not respond to a request before the
core that seized it.
Change-Id: I9998975b6dda42b3bd58112115c5122bf2b49d42
diff --git a/k8s/adapters-simulated.yml b/k8s/adapters-simulated.yml
index 5579767..ec1b069 100644
--- a/k8s/adapters-simulated.yml
+++ b/k8s/adapters-simulated.yml
@@ -58,6 +58,8 @@
- "--core_topic=rwcore"
- "--simulator_topic=simulated_olt"
- "--onu_number=1"
+ - "--kv_store_host=etcd.$(NAMESPACE).svc.cluster.local"
+ - "--kv_store_port=2379"
imagePullPolicy: IfNotPresent
---
@@ -106,4 +108,6 @@
- "--kafka_cluster_port=9092"
- "--core_topic=rwcore"
- "--simulator_topic=simulated_onu"
+ - "--kv_store_host=etcd.$(NAMESPACE).svc.cluster.local"
+ - "--kv_store_port=2379"
imagePullPolicy: IfNotPresent
diff --git a/k8s/affinity-router.yml b/k8s/affinity-router.yml
index a92e442..3fcdaa2 100644
--- a/k8s/affinity-router.yml
+++ b/k8s/affinity-router.yml
@@ -12,6 +12,20 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+apiVersion: v1
+kind: Service
+metadata:
+ name: afrouter
+ namespace: voltha
+spec:
+ clusterIP: None
+ ports:
+ - name: grpc
+ port: 55555
+ targetPort: 55555
+ selector:
+ app: afrouter
+---
apiVersion: apps/v1beta1
kind: Deployment
metadata:
diff --git a/k8s/cli.yml b/k8s/cli.yml
new file mode 100644
index 0000000..bd9e38f
--- /dev/null
+++ b/k8s/cli.yml
@@ -0,0 +1,64 @@
+# Copyright 2019-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: Service
+metadata:
+ name: vcli
+ namespace: voltha
+ labels:
+ name: vcli
+spec:
+ ports:
+ - name: ssh
+ port: 5022
+ targetPort: 22
+ selector:
+ app: vcli
+---
+apiVersion: apps/v1beta1
+kind: Deployment
+metadata:
+ name: vcli
+ namespace: voltha
+spec:
+ replicas: 1
+ template:
+ metadata:
+ labels:
+ app: vcli
+ annotations:
+ cni: "calico"
+ spec:
+ containers:
+ - name: vcli
+ image: voltha-cli
+ env:
+ - name: POD_IP
+ valueFrom:
+ fieldRef:
+ fieldPath: status.podIP
+ - name: NAMESPACE
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.namespace
+ args:
+ - "/voltha/python/cli/setup.sh"
+ - "-g afrouter.$(NAMESPACE).svc.cluster.local:55555"
+ ports:
+ - containerPort: 22
+ name: ssh-port
+ imagePullPolicy: IfNotPresent
+
+
diff --git a/k8s/genie-cni-plugin-1.8.yml b/k8s/genie-cni-plugin-1.8.yml
index 44e25f7..e2b0f36 100644
--- a/k8s/genie-cni-plugin-1.8.yml
+++ b/k8s/genie-cni-plugin-1.8.yml
@@ -1,10 +1,10 @@
-# Copyright 2018 the original author or authors.
+# Copyright 2019-present Open Networking Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/k8s/kafka.yml b/k8s/kafka.yml
new file mode 100644
index 0000000..88c3a39
--- /dev/null
+++ b/k8s/kafka.yml
@@ -0,0 +1,71 @@
+# Copyright 2019-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: Service
+metadata:
+ name: kafka
+ namespace: voltha
+spec:
+ clusterIP: None
+ selector:
+ app: kafka
+ ports:
+ - protocol: TCP
+ port: 9092
+ targetPort: 9092
+---
+apiVersion: apps/v1beta1
+kind: StatefulSet
+metadata:
+ name: kafka
+ namespace: voltha
+spec:
+ serviceName: kafka
+ replicas: 3
+ updateStrategy:
+ type: RollingUpdate
+ selector:
+ matchLabels:
+ app: kafka
+ template:
+ metadata:
+ labels:
+ app: kafka
+ annotations:
+ cni: "calico"
+ spec:
+ terminationGracePeriodSeconds: 10
+ affinity:
+ podAntiAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ - labelSelector:
+ matchExpressions:
+ - key: app
+ operator: In
+ values:
+ - kafka
+ topologyKey: kubernetes.io/hostname
+ containers:
+ - name: kafka
+ image: wurstmeister/kafka:1.0.0
+ ports:
+ - containerPort: 9092
+ env:
+ - name: KAFKA_ADVERTISED_PORT
+ value: "9092"
+ - name: KAFKA_ZOOKEEPER_CONNECT
+ value: zoo1:2181,zoo2:2181,zoo3:2181
+ - name: KAFKA_HEAP_OPTS
+ value: "-Xmx256M -Xms128M"
diff --git a/k8s/namespace.yml b/k8s/namespace.yml
new file mode 100644
index 0000000..3f5d7b4
--- /dev/null
+++ b/k8s/namespace.yml
@@ -0,0 +1,22 @@
+# Copyright 2019-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Observation:
+# It appears that when a namespace is created, Kubernetes creates
+# a ServiceAccount called "default" for that new namespace.
+#
+apiVersion: v1
+kind: Namespace
+metadata:
+ name: voltha
diff --git a/k8s/ofagent.yml b/k8s/ofagent.yml
new file mode 100644
index 0000000..1f548ac
--- /dev/null
+++ b/k8s/ofagent.yml
@@ -0,0 +1,44 @@
+# Copyright 2019-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: apps/v1beta1
+kind: Deployment
+metadata:
+ name: ofagent
+ namespace: voltha
+spec:
+ replicas: 3
+ template:
+ metadata:
+ labels:
+ app: ofagent
+ annotations:
+ cni: "calico"
+ spec:
+ terminationGracePeriodSeconds: 10
+ containers:
+ - name: ofagent
+ image: voltha-ofagent
+ imagePullPolicy: IfNotPresent
+ env:
+ - name: NAMESPACE
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.namespace
+ args:
+ - "/ofagent/ofagent/main.py"
+ - "-v"
+ - "--consul=consul.$(NAMESPACE).svc.cluster.local:8500"
+ - "--controller=onos.$(NAMESPACE).svc.cluster.local:6653"
+ - "--grpc-endpoint=afrouter.$(NAMESPACE).svc.cluster.local:55555"
diff --git a/k8s/olt.yml b/k8s/olt.yml
new file mode 100644
index 0000000..92f6ec7
--- /dev/null
+++ b/k8s/olt.yml
@@ -0,0 +1,75 @@
+# Copyright 2019-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: Service
+metadata:
+ name: olt
+ namespace: voltha
+ labels:
+ name: olt
+spec:
+ ports:
+ - name: grpc
+ port: 50060
+ targetPort: 50060
+ selector:
+ app: olt
+---
+apiVersion: apps/v1beta1
+kind: Deployment
+metadata:
+ name: olt
+ namespace: voltha
+spec:
+ replicas: 1
+ template:
+ metadata:
+ labels:
+ app: olt
+ annotations:
+ cni: "calico"
+ spec:
+ containers:
+ - name: olt
+ image: "voltha-ponsim"
+ env:
+ - name: POD_IP
+ valueFrom:
+ fieldRef:
+ fieldPath: status.podIP
+ - name: NAMESPACE
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.namespace
+
+ args:
+ - "/app/ponsim"
+ - "-device_type"
+ - "OLT"
+ - "-onus"
+ - "3"
+ - "-internal_if"
+ - "eth0"
+ - "-external_if"
+ - "eth0"
+ - "-vcore_endpoint"
+ - "vcore"
+ - "-promiscuous"
+
+ ports:
+ - containerPort: 50060
+ name: grpc-port
+
+ imagePullPolicy: IfNotPresent
diff --git a/k8s/onos.yml b/k8s/onos.yml
new file mode 100644
index 0000000..3101534
--- /dev/null
+++ b/k8s/onos.yml
@@ -0,0 +1,71 @@
+# Copyright 2019-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: Service
+metadata:
+ name: onos
+ namespace: voltha
+ labels:
+ name: onos
+spec:
+ ports:
+ - name: ssh
+ port: 8101
+ targetPort: 8101
+ - name: of
+ port: 6653
+ targetPort: 6653
+ - name: ui
+ port: 8181
+ targetPort: 8181
+ selector:
+ app: onos
+---
+apiVersion: apps/v1beta1
+kind: Deployment
+metadata:
+ name: onos
+ namespace: voltha
+spec:
+ replicas: 1
+ template:
+ metadata:
+ labels:
+ app: onos
+ annotations:
+ cni: "calico"
+ spec:
+ containers:
+ - name: onos
+ image: voltha-onos
+ env:
+ - name: POD_IP
+ valueFrom:
+ fieldRef:
+ fieldPath: status.podIP
+ - name: NAMESPACE
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.namespace
+ - name: ONOS_APPS
+ value: "drivers,openflow-base"
+ ports:
+ - containerPort: 8101
+ name: ssh-port
+ - containerPort: 6653
+ name: of-port
+ - containerPort: 8181
+ name: ui-port
+ imagePullPolicy: IfNotPresent
diff --git a/k8s/onu.yml b/k8s/onu.yml
new file mode 100644
index 0000000..ca3eee3
--- /dev/null
+++ b/k8s/onu.yml
@@ -0,0 +1,75 @@
+# Copyright 2019-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: Service
+metadata:
+ name: onu
+ namespace: voltha
+ labels:
+ name: onu
+spec:
+ ports:
+ - name: grpc
+ port: 50061
+ targetPort: 50061
+ selector:
+ app: onu
+---
+apiVersion: apps/v1beta1
+kind: Deployment
+metadata:
+ name: onu
+ namespace: voltha
+spec:
+ replicas: 3
+ template:
+ metadata:
+ labels:
+ app: onu
+ annotations:
+ cni: "calico,pon0"
+ spec:
+ containers:
+ - name: onu
+ image: "voltha-ponsim"
+ env:
+ - name: POD_IP
+ valueFrom:
+ fieldRef:
+ fieldPath: status.podIP
+ - name: NAMESPACE
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.namespace
+
+ args:
+ - "/app/ponsim"
+ - "-device_type"
+ - "ONU"
+ - "-parent_addr"
+ - "olt"
+ - "-grpc_port"
+ - "50061"
+ - "-internal_if"
+ - "eth0"
+ - "-external_if"
+ - "eth1"
+ - "-promiscuous"
+
+ ports:
+ - containerPort: 50061
+ name: grpc-port
+
+ imagePullPolicy: IfNotPresent
diff --git a/k8s/operator/etcd/etcd_cluster.yml b/k8s/operator/etcd/etcd_cluster.yml
new file mode 100644
index 0000000..1bfb673
--- /dev/null
+++ b/k8s/operator/etcd/etcd_cluster.yml
@@ -0,0 +1,25 @@
+# Copyright 2019-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: "etcd.database.coreos.com/v1beta2"
+kind: "EtcdCluster"
+metadata:
+ name: etcd
+ namespace: voltha
+spec:
+ size: 3
+ version: "3.2.18"
+ pod:
+ annotations:
+ cni: "calico"
diff --git a/k8s/ro-core.yml b/k8s/ro-core.yml
index 3eb94db..2e7d78e 100644
--- a/k8s/ro-core.yml
+++ b/k8s/ro-core.yml
@@ -32,7 +32,7 @@
name: ro-core
namespace: voltha
spec:
- replicas: 1
+ replicas: 3
selector:
matchLabels:
app: ro-core
diff --git a/k8s/rw-core-pair1.yml b/k8s/rw-core-pair1.yml
index bae5092..0861d4d 100644
--- a/k8s/rw-core-pair1.yml
+++ b/k8s/rw-core-pair1.yml
@@ -68,6 +68,9 @@
- "-kafka_cluster_host=kafka.$(NAMESPACE).svc.cluster.local"
- "-kafka_cluster_port=9092"
- "-rw_core_topic=rwcore"
+ - "-kv_store_data_prefix=service/voltha"
+ - "-in_competing_mode=true"
+ - "-timeout_long_request=5000"
- "-log_level=0"
ports:
- containerPort: 50057
diff --git a/k8s/rw-core-pair2.yml b/k8s/rw-core-pair2.yml
index 133f263..59a4219 100644
--- a/k8s/rw-core-pair2.yml
+++ b/k8s/rw-core-pair2.yml
@@ -68,6 +68,9 @@
- "-kafka_cluster_host=kafka.$(NAMESPACE).svc.cluster.local"
- "-kafka_cluster_port=9092"
- "-rw_core_topic=rwcore"
+ - "-kv_store_data_prefix=service/voltha"
+ - "-in_competing_mode=true"
+ - "-timeout_long_request=5000"
- "-log_level=0"
ports:
- containerPort: 50057
diff --git a/k8s/rw-core-pair3.yml b/k8s/rw-core-pair3.yml
index 9879f35..826bf5f 100644
--- a/k8s/rw-core-pair3.yml
+++ b/k8s/rw-core-pair3.yml
@@ -68,6 +68,9 @@
- "-kafka_cluster_host=kafka.$(NAMESPACE).svc.cluster.local"
- "-kafka_cluster_port=9092"
- "-rw_core_topic=rwcore"
+ - "-kv_store_data_prefix=service/voltha"
+ - "-in_competing_mode=true"
+ - "-timeout_long_request=5000"
- "-log_level=0"
ports:
- containerPort: 50057
diff --git a/k8s/zookeeper.yml b/k8s/zookeeper.yml
new file mode 100644
index 0000000..5760f0d
--- /dev/null
+++ b/k8s/zookeeper.yml
@@ -0,0 +1,196 @@
+# Copyright 2019-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: Service
+metadata:
+ name: zoo1
+ namespace: voltha
+spec:
+ clusterIP: None
+ selector:
+ app: zookeeper-1
+ ports:
+ - name: client
+ port: 2181
+ targetPort: 2181
+ - name: follower
+ port: 2888
+ targetPort: 2888
+ - name: leader
+ port: 3888
+ targetPort: 3888
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: zoo2
+ namespace: voltha
+spec:
+ clusterIP: None
+ selector:
+ app: zookeeper-2
+ ports:
+ - name: client
+ port: 2181
+ targetPort: 2181
+ - name: follower
+ port: 2888
+ targetPort: 2888
+ - name: leader
+ port: 3888
+ targetPort: 3888
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: zoo3
+ namespace: voltha
+spec:
+ clusterIP: None
+ selector:
+ app: zookeeper-3
+ ports:
+ - name: client
+ port: 2181
+ targetPort: 2181
+ - name: follower
+ port: 2888
+ targetPort: 2888
+ - name: leader
+ port: 3888
+ targetPort: 3888
+---
+apiVersion: apps/v1beta1
+kind: StatefulSet
+metadata:
+ name: zookeeper1
+ namespace: voltha
+spec:
+ serviceName: zoo1
+ replicas: 1
+ updateStrategy:
+ type: RollingUpdate
+ selector:
+ matchLabels:
+ app: zookeeper-1
+ template:
+ metadata:
+ labels:
+ app: zookeeper-1
+ annotations:
+ cni: "calico"
+ spec:
+ terminationGracePeriodSeconds: 0
+ containers:
+ - name: zoo1
+ image: zookeeper:3.4.11
+ ports:
+ - containerPort: 2181
+ - containerPort: 2888
+ - containerPort: 3888
+ env:
+ - name: NAMESPACE
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.namespace
+ - name: ZOO_MY_ID
+ value: "1"
+ - name: ZOO_SERVERS
+ value: >
+ server.1=zookeeper1-0.zoo1.$(NAMESPACE).svc.cluster.local:2888:3888
+ server.2=zookeeper2-0.zoo2.$(NAMESPACE).svc.cluster.local:2888:3888
+ server.3=zookeeper3-0.zoo3.$(NAMESPACE).svc.cluster.local:2888:3888
+---
+apiVersion: apps/v1beta1
+kind: StatefulSet
+metadata:
+ name: zookeeper2
+ namespace: voltha
+spec:
+ serviceName: zoo2
+ replicas: 1
+ updateStrategy:
+ type: RollingUpdate
+ selector:
+ matchLabels:
+ app: zookeeper-2
+ template:
+ metadata:
+ labels:
+ app: zookeeper-2
+ annotations:
+ cni: "calico"
+ spec:
+ terminationGracePeriodSeconds: 0
+ containers:
+ - name: zoo2
+ image: zookeeper:3.4.11
+ ports:
+ - containerPort: 2181
+ - containerPort: 2888
+ - containerPort: 3888
+ env:
+ - name: NAMESPACE
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.namespace
+ - name: ZOO_MY_ID
+ value: "2"
+ - name: ZOO_SERVERS
+ value: >
+ server.1=zookeeper1-0.zoo1.$(NAMESPACE).svc.cluster.local:2888:3888
+ server.2=zookeeper2-0.zoo2.$(NAMESPACE).svc.cluster.local:2888:3888
+ server.3=zookeeper3-0.zoo3.$(NAMESPACE).svc.cluster.local:2888:3888
+---
+apiVersion: apps/v1beta1
+kind: StatefulSet
+metadata:
+ name: zookeeper3
+ namespace: voltha
+spec:
+ serviceName: zoo3
+ replicas: 1
+ updateStrategy:
+ type: RollingUpdate
+ selector:
+ matchLabels:
+ app: zookeeper-3
+ template:
+ metadata:
+ labels:
+ app: zookeeper-3
+ annotations:
+ cni: "calico"
+ spec:
+ terminationGracePeriodSeconds: 0
+ containers:
+ - name: zoo3
+ image: zookeeper:3.4.11
+ ports:
+ - containerPort: 2181
+ - containerPort: 2888
+ - containerPort: 3888
+ env:
+ - name: NAMESPACE
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.namespace
+ - name: ZOO_MY_ID
+ value: "3"
+ - name: ZOO_SERVERS
+ value: >
+ server.1=zookeeper1-0.zoo1.$(NAMESPACE).svc.cluster.local:2888:3888
+ server.2=zookeeper2-0.zoo2.$(NAMESPACE).svc.cluster.local:2888:3888
+ server.3=zookeeper3-0.zoo3.$(NAMESPACE).svc.cluster.local:2888:3888
diff --git a/rw_core/core/transaction.go b/rw_core/core/transaction.go
index a8bcfd6..397299e 100644
--- a/rw_core/core/transaction.go
+++ b/rw_core/core/transaction.go
@@ -217,6 +217,10 @@
default:
acquired = false
}
+ // Ensure the request watcher does not reply before the request server
+ if !acquired {
+ time.Sleep(1 * time.Second)
+ }
return acquired
}