AETHER-510 Export maintenance window metric
Change-Id: I54aa39ec9cbc95162b2d6068f62348daa7da4d72
diff --git a/edge-monitoring/Makefile b/edge-monitoring/Makefile
new file mode 100644
index 0000000..0fc83de
--- /dev/null
+++ b/edge-monitoring/Makefile
@@ -0,0 +1,73 @@
+# Copyright 2020-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Secret iCal URL for ACE downtime calendar
+SECRET_ICAL_URL ?=
+
+# Namespace to install secret in, for testing
+NAMESPACE ?= edge-monitoring
+
+# set default shell
+SHELL = bash -e -o pipefail
+
+# Variables
+VERSION ?= $(shell cat ./VERSION)
+VENVDIR := venv
+
+## Docker related
+DOCKER_REGISTRY ?=
+DOCKER_REPOSITORY ?=
+DOCKER_BUILD_ARGS ?=
+DOCKER_TAG ?= ${VERSION}
+ADAPTER_IMAGENAME := ${DOCKER_REGISTRY}${DOCKER_REPOSITORY}edge-monitoring-server:${DOCKER_TAG}
+
+
+# This should to be the first and default target in this Makefile
+help:
+ @echo "Usage: make [<target>]"
+ @echo "where available targets are:"
+ @echo
+ @echo "build : Build the edge-monitoring-server docker image"
+ @echo "help : Print this help"
+ @echo "docker-push : Push the docker image to an external repository"
+ @echo "venv : Build local Python virtualenv"
+ @echo "test : Run edge-monitoring-server unit tests"
+ @echo "clean : Remove files created by the build and tests"
+ @echo "distclean : Remove venv directory"
+ @echo
+
+build: docker-build
+
+docker-build:
+ docker build $(DOCKER_BUILD_ARGS) \
+ -t ${ADAPTER_IMAGENAME} \
+ -f ./Dockerfile.server .
+
+docker-push:
+ docker push ${ADAPTER_IMAGENAME}
+
+venv: ${VENVDIR}
+ virtualenv --python=python3.7 ${VENVDIR};\
+ source ./${VENVDIR}/bin/activate ; set -u ;\
+ pip install -r requirements.txt
+
+clean:
+ find . -name '*.pyc' | xargs rm -f
+
+distclean: clean
+ rm -rf ${VENVDIR}
+
+test: venv
+ source ./${VENVDIR}/bin/activate ; set -u ;\
+ python3 test_edge_monitoring_server.py
diff --git a/edge-monitoring/VERSION b/edge-monitoring/VERSION
new file mode 100644
index 0000000..0d91a54
--- /dev/null
+++ b/edge-monitoring/VERSION
@@ -0,0 +1 @@
+0.3.0
diff --git a/edge-monitoring/edge_monitoring_server.py b/edge-monitoring/edge_monitoring_server.py
index 41ed675..29d66f5 100755
--- a/edge-monitoring/edge_monitoring_server.py
+++ b/edge-monitoring/edge_monitoring_server.py
@@ -14,10 +14,16 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import os
import time
+import datetime
+import pytz
+import threading
+from icalevents.icalevents import events
from flask import Flask, jsonify, abort, request, Response
import prometheus_client as prom
+SECRET_ICAL_URL = os.environ.get("SECRET_ICAL_URL")
NO_RESULT_THRESHOLD = 720
app = Flask(__name__)
@@ -28,7 +34,7 @@
'control_plane': 'connected',
'user_plane': 'connected'
},
- 'last_update': time.time()
+ 'last_update': time.time(),
}
]
@@ -40,9 +46,45 @@
"connected": 2
}
+room_mapping = {
+ "production-edge-onf-menlo": "(Compute)-MP-1-Aether Production",
+ "production-edge-example": "(Compute)-MP-1-Aether Production" # for testing
+}
+
cp_status = prom.Gauge("aetheredge_status_control_plane", "Control plane status code", ["name"])
up_status = prom.Gauge("aetheredge_status_user_plane", "User plane status code", ["name"])
last_update = prom.Gauge("aetheredge_last_update", "Last reported test result", ["name"])
+maint_window = prom.Gauge("aetheredge_in_maintenance_window", "Currently in a maintenance window", ["name"])
+
+def is_my_event(event, name):
+ for field in ["summary", "location", "description"]:
+ if name in getattr(event, field, ""):
+ return True
+ return False
+
+def in_maintenance_window(events, name, now):
+ for event in events:
+ if event.start < now and event.end > now:
+ if is_my_event(event, name):
+ return True
+ if name in room_mapping and is_my_event(event, room_mapping[name]):
+ return True
+ return False
+
+def pull_maintenance_events():
+ while(True):
+ now = datetime.datetime.now(pytz.utc)
+ try:
+ es = events(SECRET_ICAL_URL, start = now)
+ except Exception as e:
+ print(e)
+ else:
+ for edge in edges:
+ if 'maintenance' not in edge:
+ edge['maintenance'] = {}
+ edge['maintenance']['in_window'] = in_maintenance_window(es, edge['name'], now)
+ edge['maintenance']['last_update'] = time.time()
+ time.sleep(60)
def time_out_stale_results():
for edge in edges:
@@ -63,10 +105,14 @@
cp_status.labels(edge['name']).set(status_codes[edge['status']['control_plane']])
up_status.labels(edge['name']).set(status_codes[edge['status']['user_plane']])
last_update.labels(edge['name']).set(edge['last_update'])
+ if 'maintenance' in edge:
+ maint_window.labels(edge['name']).set(int(edge['maintenance']['in_window']))
res.append(prom.generate_latest(cp_status))
res.append(prom.generate_latest(up_status))
res.append(prom.generate_latest(last_update))
+ res.append(prom.generate_latest(maint_window))
+
return Response(res, mimetype="text/plain")
@@ -121,4 +167,8 @@
if __name__ == '__main__':
+ if SECRET_ICAL_URL:
+ print(" * Starting maintenance calendar polling thread")
+ t = threading.Thread(target=pull_maintenance_events)
+ t.start()
app.run(debug=True, host='0.0.0.0', port=80)
diff --git a/edge-monitoring/edge_monitoring_server_k8s.yaml b/edge-monitoring/edge_monitoring_server_k8s.yaml
index 7fb384e..1041cfe 100644
--- a/edge-monitoring/edge_monitoring_server_k8s.yaml
+++ b/edge-monitoring/edge_monitoring_server_k8s.yaml
@@ -29,9 +29,15 @@
spec:
containers:
- name: server
- image: docker.io/omecproject/edge-monitoring-server:0.2.0
+ image: docker.io/omecproject/edge-monitoring-server:0.3.0
imagePullPolicy: Always
command: ["python", "edge_monitoring_server.py"]
+ env:
+ - name: SECRET_ICAL_URL
+ valueFrom:
+ secretKeyRef:
+ name: calendar-secret
+ key: iCalUrl
livenessProbe:
httpGet:
path: /edges/healthz
diff --git a/edge-monitoring/requirements.txt b/edge-monitoring/requirements.txt
index d87fc2e..7e08052 100644
--- a/edge-monitoring/requirements.txt
+++ b/edge-monitoring/requirements.txt
@@ -3,3 +3,5 @@
git+git://github.com/sch3m4/pyadb@master#egg=pyadb
python-daemon
prometheus-client
+pytz
+icalevents
diff --git a/edge-monitoring/test_edge_monitoring_server.py b/edge-monitoring/test_edge_monitoring_server.py
new file mode 100755
index 0000000..c6a6e03
--- /dev/null
+++ b/edge-monitoring/test_edge_monitoring_server.py
@@ -0,0 +1,87 @@
+#!/usr/bin/env python
+
+# Copyright 2020-present Open Networking Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+import edge_monitoring_server as ems
+import datetime
+import pytz
+
+
+class MyEvent:
+ def __init__ (self, location = "", description = "", summary = "", start = None, end = None):
+ self.location = location
+ self.description = description
+ self.summary = summary
+ self.start = start
+ self.end = end
+
+class MyEventNoLoc:
+ def __init__ (self, description = "", summary = ""):
+ self.description = description
+ self.summary = summary
+
+
+class TestEdgeMonitoringServer(unittest.TestCase):
+ def test_match_location(self):
+ event = MyEvent(location = "production-edge-onf-menlo, (Compute)-MP-1-Aether Production")
+ self.assertTrue(ems.is_my_event(event, "production-edge-onf-menlo"))
+ self.assertTrue(ems.is_my_event(event, "(Compute)-MP-1-Aether Production"))
+
+ def test_match_description(self):
+ event = MyEvent(description = "production-edge-onf-menlo, (Compute)-MP-1-Aether Production")
+ self.assertTrue(ems.is_my_event(event, "production-edge-onf-menlo"))
+ self.assertTrue(ems.is_my_event(event, "(Compute)-MP-1-Aether Production"))
+
+ def test_match_summary(self):
+ event = MyEvent(summary = "production-edge-onf-menlo, (Compute)-MP-1-Aether Production")
+ self.assertTrue(ems.is_my_event(event, "production-edge-onf-menlo"))
+ self.assertTrue(ems.is_my_event(event, "(Compute)-MP-1-Aether Production"))
+
+ def test_no_match(self):
+ event = MyEvent(summary = "production-edge-onf-menlo, (Compute)-MP-1-Aether Production")
+ self.assertFalse(ems.is_my_event(event, "production-edge-intel"))
+ self.assertFalse(ems.is_my_event(event, "(Compute)-MP-1-Aether Staging"))
+
+ def test_missing_field(self):
+ event = MyEventNoLoc(description = "production-edge-onf-menlo, (Compute)-MP-1-Aether Production")
+ self.assertTrue(ems.is_my_event(event, "production-edge-onf-menlo"))
+
+ def test_in_window(self):
+ events = []
+ now = datetime.datetime.now(pytz.utc)
+ events.append(MyEvent(location = "(Compute)-MP-1-Aether Production",
+ start = now - datetime.timedelta(hours=1),
+ end = now + datetime.timedelta(hours=1)))
+ self.assertTrue(ems.in_maintenance_window(events, "production-edge-onf-menlo", now))
+ self.assertFalse(ems.in_maintenance_window(events, "production-edge-onf-tucson", now))
+
+ def test_not_in_window(self):
+ events = []
+ now = datetime.datetime.now(pytz.utc)
+ events.append(MyEvent(location = "production-edge-onf-menlo",
+ start = now + datetime.timedelta(hours=1),
+ end = now + datetime.timedelta(hours=2)))
+ self.assertFalse(ems.in_maintenance_window(events, "production-edge-onf-menlo", now))
+
+ def test_no_events(self):
+ events = []
+ now = datetime.datetime.now(pytz.utc)
+ self.assertFalse(ems.in_maintenance_window(events, "production-edge-onf-menlo", now))
+
+
+if __name__ == '__main__':
+ suite = unittest.TestLoader().loadTestsFromTestCase(TestEdgeMonitoringServer)
+ unittest.TextTestRunner(verbosity=2).run(suite)