VOL-1406: Deploy voltha 2.0 in a k8s cluster

- Also added sleep to watching rw-core to ensure that the
  watching core does not respond to a request before the
  core that seized it.

Change-Id: I9998975b6dda42b3bd58112115c5122bf2b49d42
diff --git a/k8s/adapters-simulated.yml b/k8s/adapters-simulated.yml
index 5579767..ec1b069 100644
--- a/k8s/adapters-simulated.yml
+++ b/k8s/adapters-simulated.yml
@@ -58,6 +58,8 @@
             - "--core_topic=rwcore"
             - "--simulator_topic=simulated_olt"
             - "--onu_number=1"
+            - "--kv_store_host=etcd.$(NAMESPACE).svc.cluster.local"
+            - "--kv_store_port=2379"
           imagePullPolicy: IfNotPresent
 ---
 
@@ -106,4 +108,6 @@
             - "--kafka_cluster_port=9092"
             - "--core_topic=rwcore"
             - "--simulator_topic=simulated_onu"
+            - "--kv_store_host=etcd.$(NAMESPACE).svc.cluster.local"
+            - "--kv_store_port=2379"
           imagePullPolicy: IfNotPresent
diff --git a/k8s/affinity-router.yml b/k8s/affinity-router.yml
index a92e442..3fcdaa2 100644
--- a/k8s/affinity-router.yml
+++ b/k8s/affinity-router.yml
@@ -12,6 +12,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+apiVersion: v1
+kind: Service
+metadata:
+  name: afrouter
+  namespace: voltha
+spec:
+  clusterIP: None
+  ports:
+    - name: grpc
+      port: 55555
+      targetPort: 55555
+  selector:
+    app: afrouter
+---
 apiVersion: apps/v1beta1
 kind: Deployment
 metadata:
diff --git a/k8s/cli.yml b/k8s/cli.yml
new file mode 100644
index 0000000..bd9e38f
--- /dev/null
+++ b/k8s/cli.yml
@@ -0,0 +1,64 @@
+# Copyright 2019-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: vcli
+  namespace: voltha
+  labels:
+    name: vcli
+spec:
+  ports:
+    - name: ssh
+      port: 5022
+      targetPort: 22
+  selector:
+    app: vcli
+---
+apiVersion: apps/v1beta1
+kind: Deployment
+metadata:
+  name: vcli
+  namespace: voltha
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: vcli
+      annotations:
+        cni: "calico"
+    spec:
+      containers:
+        - name: vcli
+          image: voltha-cli
+          env:
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+          args:
+            - "/voltha/python/cli/setup.sh"
+            - "-g afrouter.$(NAMESPACE).svc.cluster.local:55555"
+          ports:
+            - containerPort: 22
+              name: ssh-port
+          imagePullPolicy: IfNotPresent
+
+
diff --git a/k8s/genie-cni-plugin-1.8.yml b/k8s/genie-cni-plugin-1.8.yml
index 44e25f7..e2b0f36 100644
--- a/k8s/genie-cni-plugin-1.8.yml
+++ b/k8s/genie-cni-plugin-1.8.yml
@@ -1,10 +1,10 @@
-# Copyright 2018 the original author or authors.
+# Copyright 2019-present Open Networking Foundation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
-#      http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/k8s/kafka.yml b/k8s/kafka.yml
new file mode 100644
index 0000000..88c3a39
--- /dev/null
+++ b/k8s/kafka.yml
@@ -0,0 +1,71 @@
+# Copyright 2019-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: kafka
+  namespace: voltha
+spec:
+  clusterIP: None
+  selector:
+    app: kafka
+  ports:
+  - protocol: TCP
+    port: 9092
+    targetPort: 9092
+---
+apiVersion: apps/v1beta1
+kind: StatefulSet
+metadata:
+  name: kafka
+  namespace: voltha
+spec:
+  serviceName: kafka
+  replicas: 3
+  updateStrategy:
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      app: kafka
+  template:
+    metadata:
+      labels:
+        app: kafka
+      annotations:
+        cni: "calico"
+    spec:
+      terminationGracePeriodSeconds: 10
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+                matchExpressions:
+                  - key: app
+                    operator: In
+                    values:
+                      - kafka
+              topologyKey: kubernetes.io/hostname
+      containers:
+      - name: kafka
+        image: wurstmeister/kafka:1.0.0
+        ports:
+        - containerPort: 9092
+        env:
+        - name: KAFKA_ADVERTISED_PORT
+          value: "9092"
+        - name: KAFKA_ZOOKEEPER_CONNECT
+          value: zoo1:2181,zoo2:2181,zoo3:2181
+        - name: KAFKA_HEAP_OPTS
+          value: "-Xmx256M -Xms128M"
diff --git a/k8s/namespace.yml b/k8s/namespace.yml
new file mode 100644
index 0000000..3f5d7b4
--- /dev/null
+++ b/k8s/namespace.yml
@@ -0,0 +1,22 @@
+# Copyright 2019-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Observation:
+#    It appears that when a namespace is created, Kubernetes creates
+#    a ServiceAccount called "default" for that new namespace.
+#
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: voltha
diff --git a/k8s/ofagent.yml b/k8s/ofagent.yml
new file mode 100644
index 0000000..1f548ac
--- /dev/null
+++ b/k8s/ofagent.yml
@@ -0,0 +1,44 @@
+# Copyright 2019-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: apps/v1beta1
+kind: Deployment
+metadata:
+  name: ofagent
+  namespace: voltha
+spec:
+  replicas: 3
+  template:
+    metadata:
+      labels:
+        app: ofagent
+      annotations:
+        cni: "calico"
+    spec:
+      terminationGracePeriodSeconds: 10
+      containers:
+      - name: ofagent
+        image: voltha-ofagent
+        imagePullPolicy: IfNotPresent
+        env:
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        args:
+        - "/ofagent/ofagent/main.py"
+        - "-v"
+        - "--consul=consul.$(NAMESPACE).svc.cluster.local:8500"
+        - "--controller=onos.$(NAMESPACE).svc.cluster.local:6653"
+        - "--grpc-endpoint=afrouter.$(NAMESPACE).svc.cluster.local:55555"
diff --git a/k8s/olt.yml b/k8s/olt.yml
new file mode 100644
index 0000000..92f6ec7
--- /dev/null
+++ b/k8s/olt.yml
@@ -0,0 +1,75 @@
+# Copyright 2019-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: Service
+metadata:
+   name: olt
+   namespace: voltha
+   labels:
+     name: olt
+spec:
+  ports:
+    - name: grpc
+      port: 50060
+      targetPort: 50060
+  selector:
+    app: olt
+---
+apiVersion: apps/v1beta1
+kind: Deployment
+metadata:
+  name: olt
+  namespace: voltha
+spec:
+   replicas: 1
+   template:
+     metadata:
+       labels:
+         app: olt
+       annotations:
+         cni: "calico"
+     spec:
+       containers:
+         - name: olt
+           image: "voltha-ponsim"
+           env:
+             - name: POD_IP
+               valueFrom:
+                 fieldRef:
+                   fieldPath: status.podIP
+             - name: NAMESPACE
+               valueFrom:
+                 fieldRef:
+                   fieldPath: metadata.namespace
+
+           args:
+             - "/app/ponsim"
+             - "-device_type"
+             - "OLT"
+             - "-onus"
+             - "3"
+             - "-internal_if"
+             - "eth0"
+             - "-external_if"
+             - "eth0"
+             - "-vcore_endpoint"
+             - "vcore"
+             - "-promiscuous"
+
+           ports:
+            - containerPort: 50060
+              name: grpc-port
+
+           imagePullPolicy: IfNotPresent
diff --git a/k8s/onos.yml b/k8s/onos.yml
new file mode 100644
index 0000000..3101534
--- /dev/null
+++ b/k8s/onos.yml
@@ -0,0 +1,71 @@
+# Copyright 2019-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: Service
+metadata:
+   name: onos
+   namespace: voltha
+   labels:
+     name: onos
+spec:
+  ports:
+    - name: ssh
+      port: 8101
+      targetPort: 8101
+    - name: of
+      port: 6653
+      targetPort: 6653
+    - name: ui
+      port: 8181
+      targetPort: 8181
+  selector:
+    app: onos
+---
+apiVersion: apps/v1beta1
+kind: Deployment
+metadata:
+  name: onos
+  namespace: voltha
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: onos
+      annotations:
+        cni: "calico"
+    spec:
+      containers:
+        - name: onos
+          image: voltha-onos
+          env:
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: ONOS_APPS
+              value: "drivers,openflow-base"
+          ports:
+            - containerPort: 8101
+              name: ssh-port
+            - containerPort: 6653
+              name: of-port
+            - containerPort: 8181
+              name: ui-port
+          imagePullPolicy: IfNotPresent
diff --git a/k8s/onu.yml b/k8s/onu.yml
new file mode 100644
index 0000000..ca3eee3
--- /dev/null
+++ b/k8s/onu.yml
@@ -0,0 +1,75 @@
+# Copyright 2019-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: Service
+metadata:
+   name: onu
+   namespace: voltha
+   labels:
+     name: onu
+spec:
+  ports:
+    - name: grpc
+      port: 50061
+      targetPort: 50061
+  selector:
+    app: onu
+---
+apiVersion: apps/v1beta1
+kind: Deployment
+metadata:
+  name: onu
+  namespace: voltha
+spec:
+   replicas: 3
+   template:
+     metadata:
+       labels:
+         app: onu
+       annotations:
+         cni: "calico,pon0"
+     spec:
+       containers:
+         - name: onu
+           image: "voltha-ponsim"
+           env:
+             - name: POD_IP
+               valueFrom:
+                 fieldRef:
+                   fieldPath: status.podIP
+             - name: NAMESPACE
+               valueFrom:
+                 fieldRef:
+                   fieldPath: metadata.namespace
+
+           args:
+             - "/app/ponsim"
+             - "-device_type"
+             - "ONU"
+             - "-parent_addr"
+             - "olt"
+             - "-grpc_port"
+             - "50061"
+             - "-internal_if"
+             - "eth0"
+             - "-external_if"
+             - "eth1"
+             - "-promiscuous"
+
+           ports:
+            - containerPort: 50061
+              name: grpc-port
+
+           imagePullPolicy: IfNotPresent
diff --git a/k8s/operator/etcd/etcd_cluster.yml b/k8s/operator/etcd/etcd_cluster.yml
new file mode 100644
index 0000000..1bfb673
--- /dev/null
+++ b/k8s/operator/etcd/etcd_cluster.yml
@@ -0,0 +1,25 @@
+# Copyright 2019-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: "etcd.database.coreos.com/v1beta2"
+kind: "EtcdCluster"
+metadata:
+  name: etcd
+  namespace: voltha
+spec:
+  size: 3
+  version: "3.2.18"
+  pod:
+    annotations:
+      cni: "calico"
diff --git a/k8s/ro-core.yml b/k8s/ro-core.yml
index 3eb94db..2e7d78e 100644
--- a/k8s/ro-core.yml
+++ b/k8s/ro-core.yml
@@ -32,7 +32,7 @@
   name: ro-core
   namespace: voltha
 spec:
-  replicas: 1 
+  replicas: 3
   selector:
     matchLabels:
       app: ro-core
diff --git a/k8s/rw-core-pair1.yml b/k8s/rw-core-pair1.yml
index bae5092..0861d4d 100644
--- a/k8s/rw-core-pair1.yml
+++ b/k8s/rw-core-pair1.yml
@@ -68,6 +68,9 @@
             - "-kafka_cluster_host=kafka.$(NAMESPACE).svc.cluster.local"
             - "-kafka_cluster_port=9092"
             - "-rw_core_topic=rwcore"
+            - "-kv_store_data_prefix=service/voltha"
+            - "-in_competing_mode=true"
+            - "-timeout_long_request=5000"
             - "-log_level=0"
           ports:
             - containerPort: 50057
diff --git a/k8s/rw-core-pair2.yml b/k8s/rw-core-pair2.yml
index 133f263..59a4219 100644
--- a/k8s/rw-core-pair2.yml
+++ b/k8s/rw-core-pair2.yml
@@ -68,6 +68,9 @@
             - "-kafka_cluster_host=kafka.$(NAMESPACE).svc.cluster.local"
             - "-kafka_cluster_port=9092"
             - "-rw_core_topic=rwcore"
+            - "-kv_store_data_prefix=service/voltha"
+            - "-in_competing_mode=true"
+            - "-timeout_long_request=5000"
             - "-log_level=0"
           ports:
             - containerPort: 50057
diff --git a/k8s/rw-core-pair3.yml b/k8s/rw-core-pair3.yml
index 9879f35..826bf5f 100644
--- a/k8s/rw-core-pair3.yml
+++ b/k8s/rw-core-pair3.yml
@@ -68,6 +68,9 @@
             - "-kafka_cluster_host=kafka.$(NAMESPACE).svc.cluster.local"
             - "-kafka_cluster_port=9092"
             - "-rw_core_topic=rwcore"
+            - "-kv_store_data_prefix=service/voltha"
+            - "-in_competing_mode=true"
+            - "-timeout_long_request=5000"
             - "-log_level=0"
           ports:
             - containerPort: 50057
diff --git a/k8s/zookeeper.yml b/k8s/zookeeper.yml
new file mode 100644
index 0000000..5760f0d
--- /dev/null
+++ b/k8s/zookeeper.yml
@@ -0,0 +1,196 @@
+# Copyright 2019-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: zoo1
+  namespace: voltha
+spec:
+  clusterIP: None
+  selector:
+    app: zookeeper-1
+  ports:
+  - name: client
+    port: 2181
+    targetPort: 2181
+  - name: follower
+    port: 2888
+    targetPort: 2888
+  - name: leader
+    port: 3888
+    targetPort: 3888
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: zoo2
+  namespace: voltha
+spec:
+  clusterIP: None
+  selector:
+    app: zookeeper-2
+  ports:
+  - name: client
+    port: 2181
+    targetPort: 2181
+  - name: follower
+    port: 2888
+    targetPort: 2888
+  - name: leader
+    port: 3888
+    targetPort: 3888
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: zoo3
+  namespace: voltha
+spec:
+  clusterIP: None
+  selector:
+    app: zookeeper-3
+  ports:
+  - name: client
+    port: 2181
+    targetPort: 2181
+  - name: follower
+    port: 2888
+    targetPort: 2888
+  - name: leader
+    port: 3888
+    targetPort: 3888
+---
+apiVersion: apps/v1beta1
+kind: StatefulSet
+metadata:
+  name: zookeeper1
+  namespace: voltha
+spec:
+  serviceName: zoo1
+  replicas: 1
+  updateStrategy:
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      app: zookeeper-1
+  template:
+    metadata:
+      labels:
+        app: zookeeper-1
+      annotations:
+        cni: "calico"
+    spec:
+      terminationGracePeriodSeconds: 0
+      containers:
+      - name: zoo1
+        image: zookeeper:3.4.11
+        ports:
+        - containerPort: 2181
+        - containerPort: 2888
+        - containerPort: 3888
+        env:
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: ZOO_MY_ID
+          value: "1"
+        - name: ZOO_SERVERS
+          value: >
+            server.1=zookeeper1-0.zoo1.$(NAMESPACE).svc.cluster.local:2888:3888
+            server.2=zookeeper2-0.zoo2.$(NAMESPACE).svc.cluster.local:2888:3888
+            server.3=zookeeper3-0.zoo3.$(NAMESPACE).svc.cluster.local:2888:3888
+---
+apiVersion: apps/v1beta1
+kind: StatefulSet
+metadata:
+  name: zookeeper2
+  namespace: voltha
+spec:
+  serviceName: zoo2
+  replicas: 1
+  updateStrategy:
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      app: zookeeper-2
+  template:
+    metadata:
+      labels:
+        app: zookeeper-2
+      annotations:
+        cni: "calico"
+    spec:
+      terminationGracePeriodSeconds: 0
+      containers:
+      - name: zoo2
+        image: zookeeper:3.4.11
+        ports:
+        - containerPort: 2181
+        - containerPort: 2888
+        - containerPort: 3888
+        env:
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: ZOO_MY_ID
+          value: "2"
+        - name: ZOO_SERVERS
+          value: >
+            server.1=zookeeper1-0.zoo1.$(NAMESPACE).svc.cluster.local:2888:3888
+            server.2=zookeeper2-0.zoo2.$(NAMESPACE).svc.cluster.local:2888:3888
+            server.3=zookeeper3-0.zoo3.$(NAMESPACE).svc.cluster.local:2888:3888
+---
+apiVersion: apps/v1beta1
+kind: StatefulSet
+metadata:
+  name: zookeeper3
+  namespace: voltha
+spec:
+  serviceName: zoo3
+  replicas: 1
+  updateStrategy:
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      app: zookeeper-3
+  template:
+    metadata:
+      labels:
+        app: zookeeper-3
+      annotations:
+        cni: "calico"
+    spec:
+      terminationGracePeriodSeconds: 0
+      containers:
+      - name: zoo3
+        image: zookeeper:3.4.11
+        ports:
+        - containerPort: 2181
+        - containerPort: 2888
+        - containerPort: 3888
+        env:
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: ZOO_MY_ID
+          value: "3"
+        - name: ZOO_SERVERS
+          value: >
+            server.1=zookeeper1-0.zoo1.$(NAMESPACE).svc.cluster.local:2888:3888
+            server.2=zookeeper2-0.zoo2.$(NAMESPACE).svc.cluster.local:2888:3888
+            server.3=zookeeper3-0.zoo3.$(NAMESPACE).svc.cluster.local:2888:3888