[SEBA-233]
Change to using kafka for GUI events, instead of redis
Change-Id: I205f18632f15b44a5447530af33e94dd8c88674e
diff --git a/.gitignore b/.gitignore
index 5c29422..1f0579c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,3 +5,5 @@
.nyc_output
coverage
.idea
+package-lock.json
+
diff --git a/Dockerfile b/Dockerfile
index 7663336..d08e937 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -2,7 +2,7 @@
# To build use: docker build -t xosproject/xos-ws .
# To run use: docker run -p 3000:3000 -d xosproject/xos-ws
-FROM node:7.9.0
+FROM node:10-alpine
# Set environment variables
ENV CODE_SOURCE .
@@ -11,6 +11,9 @@
# Create app directory
WORKDIR ${CODE_DEST}
+# install librdkafka
+RUN apk --no-cache add -U python make bash g++
+
# Copy over app dependencies and source files
COPY ${CODE_SOURCE}/package.json ${CODE_DEST}/
COPY ${CODE_SOURCE}/src/ ${CODE_DEST}/src/
@@ -39,4 +42,3 @@
org.opencord.vcs-commit-date=$org_opencord_vcs_commit_date
CMD [ "npm", "start" ]
-
diff --git a/package.json b/package.json
index 1e67777..51d0b29 100644
--- a/package.json
+++ b/package.json
@@ -1,31 +1,37 @@
{
"name": "xos_nb_rest",
- "version": "1.0.0",
+ "version": "2.0.0-dev0",
"description": "Northbound REST and WebSocket interfaces for XOS",
"main": "src/server.js",
"scripts": {
"start": "node src/server.js",
"test": "LOG_LEVEL=error nyc --reporter=html mocha spec/**/*.spec.js",
"test:dev": "mocha -w spec/**/*.spec.js",
- "lint": "eslint ."
+ "lint": "eslint spec src"
},
"author": "Open Networking Foundation",
"license": "Apache-2.0",
+ "repository": {
+ "type": "git",
+ "url": "git://gerrit.opencord.org/xos-rest-gw.git"
+ },
"dependencies": {
"body-parser": "^1.15.2",
"cors": "^2.8.1",
"express": "^4.14.0",
"lodash": "^4.17.2",
+ "node-rdkafka": "^2.4.1",
"node-yaml-config": "0.0.4",
"redis": "^2.6.3",
"socket.io": "^1.7.1",
"superagent": "^3.1.0",
+ "test": "^0.6.0",
"winston": "^2.3.0",
"yargs": "^6.4.0"
},
"devDependencies": {
"chai": "^3.5.0",
- "eslint": "^3.11.1",
+ "eslint": "^3.19.0",
"eslint-plugin-import": "^2.2.0",
"fakeredis": "^1.0.3",
"mocha": "^3.2.0",
diff --git a/spec/kafka.spec.js b/spec/kafka.spec.js
new file mode 100644
index 0000000..17df04e
--- /dev/null
+++ b/spec/kafka.spec.js
@@ -0,0 +1,158 @@
+/*
+ * Copyright 2017-present Open Networking Foundation
+
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+
+ * http://www.apache.org/licenses/LICENSE-2.0
+
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+(function () {
+ 'use strict';
+
+ const chai = require('chai');
+ const expect = chai.expect;
+ const sinon = require('sinon');
+ const sinonChai = require('sinon-chai');
+ const mockery = require('mockery');
+ chai.use(sinonChai);
+
+ const socketSpy = sinon.spy();
+
+ const mockSocket = {
+ get: () => {
+ return {
+ emit: socketSpy
+ }
+ }
+ };
+
+ const trigger = {}
+
+ const mockStream = {
+ on: (event, cb) => {
+ trigger[event] = cb
+ },
+ consumer: {
+ on: sinon.spy()
+ }
+ }
+
+ const fakekafka = {
+ KafkaConsumer: {
+ createReadStream: () => mockStream
+ }
+ }
+
+ const channelName = 'Site';
+ const msgTopic = 'xos.gui_events';
+
+ describe('The event system', () => {
+
+ before((done) => {
+
+ // Enable mockery to mock objects
+ mockery.enable({
+ warnOnReplace: false,
+ warnOnUnregistered: false
+ });
+
+ // Override the node-rdkafka module with our fakekafka instance
+ mockery.registerMock('node-rdkafka', fakekafka);
+
+ // mock the socketIo client to have a spy
+ mockery.registerMock('./websocket.js', mockSocket);
+
+ require('../src/controllers/kafka.js');
+ setTimeout(() => {
+ done();
+ }, 1000);
+ });
+
+ after(() => {
+ mockery.disable();
+ });
+
+ // run after each test
+ beforeEach(() => {
+ socketSpy.reset();
+ });
+
+ it('should send a websocket event when text Kafka event is received', (done) => {
+ trigger.data({topic:msgTopic,
+ key:channelName,
+ timestamp:1234,
+ value:'I am sending a message.',
+ });
+
+ setTimeout(() => {
+ expect(socketSpy).to.have.been.called;
+ expect(socketSpy).to.have.been.calledWith('update', {
+ model: channelName,
+ msg: 'I am sending a message.'
+ });
+ done();
+ }, 500)
+ });
+
+ it('should send a websocket event when JSON Kafka event is received', (done) => {
+ trigger.data({topic:msgTopic,
+ key:channelName,
+ timestamp:2345,
+ value:JSON.stringify({msg: 'JSON Message'}),
+ });
+
+ setTimeout(() => {
+ expect(socketSpy).to.have.been.called;
+ expect(socketSpy).to.have.been.calledWith('update', {
+ model: channelName,
+ msg: {msg: 'JSON Message'}
+ });
+ done();
+ }, 1000)
+ });
+
+ it('should send a websocket event with msg: Deleted when JSON object has deleted:true', (done) => {
+ trigger.data({topic:msgTopic,
+ key:channelName,
+ timestamp:3456,
+ value:JSON.stringify({msg: 'Deleted', deleted: true}),
+ });
+
+ setTimeout(() => {
+ expect(socketSpy).to.have.been.called;
+ expect(socketSpy).to.have.been.calledWith('remove', {
+ model: channelName,
+ msg: {
+ msg: 'Deleted',
+ deleted: true
+ },
+ deleted: true
+ });
+
+ done();
+ }, 1000)
+ });
+
+ it('should not send a websocket event if the Kafka key is Diag', (done) => {
+ trigger.data({topic:msgTopic,
+ key:'Diag',
+ timestamp:4567,
+ value:JSON.stringify({msg: 'Diag Message'}),
+ });
+
+ setTimeout(() => {
+ expect(socketSpy).not.to.have.been.called;
+ done();
+ }, 1000)
+ });
+
+ });
+})();
diff --git a/spec/redis.spec.js b/spec/redis.spec.js
deleted file mode 100644
index 05ddadc..0000000
--- a/spec/redis.spec.js
+++ /dev/null
@@ -1,146 +0,0 @@
-
-/*
- * Copyright 2017-present Open Networking Foundation
-
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
-
- * http://www.apache.org/licenses/LICENSE-2.0
-
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-(function () {
- 'use strict';
-
- const chai = require('chai');
- const expect = chai.expect;
- const sinon = require('sinon');
- const sinonChai = require('sinon-chai');
- const mockery = require('mockery');
- chai.use(sinonChai);
- const fakeredis = require('fakeredis');
-
- const client = fakeredis.createClient('test-client');
- const publisher = fakeredis.createClient('test-client');
-
- const socketSpy = sinon.spy();
- const mockSocket = {
- get: () => {
- return {
- emit: socketSpy
- }
- }
- };
-
- const mockRequest = {
- get: () => {
- return {
- end: (fn) => {
- fn(null, {body: [
- {name: 'Slice'},
- {name: 'Site'}
- ]});
- }
- }
- }
- }
- const channelName = 'Site';
-
- describe('The event system', () => {
-
- before((done) => {
-
- // Enable mockery to mock objects
- mockery.enable({
- warnOnReplace: false,
- warnOnUnregistered: false
- });
-
- // Stub the createClient method to *always* return the client created above
- sinon.stub(fakeredis, 'createClient', () => client);
-
- // Override the redis module with our fakeredis instance
- mockery.registerMock('redis', fakeredis);
-
- // Override the superagent module with our mockRequest instance
- mockery.registerMock('superagent', mockRequest);
-
- // mock the socketIo client to have a spy
- mockery.registerMock('./websocket.js', mockSocket);
-
- require('../src/controllers/redis.js');
- setTimeout(() => {
- done();
- }, 1000);
- });
-
- after(() => {
- mockery.disable();
- fakeredis.createClient.restore();
- });
-
- // run after each test
- beforeEach(() => {
- client.unsubscribe(channelName);
- client.subscribe(channelName);
- publisher.flushdb();
- socketSpy.reset();
- });
-
- it('should send a websocket event when it receive a redis event that is not JSON', (done) => {
- publisher.publish(channelName, 'I am sending a message.');
- setTimeout(() => {
- expect(socketSpy).to.have.been.called;
- expect(socketSpy).to.have.been.calledWith('update', {
- model: channelName,
- msg: 'I am sending a message.'
- });
- done();
- }, 500)
- });
-
- it('should send a websocket event when it receive a redis event that is JSON', (done) => {
- publisher.publish(channelName, JSON.stringify({msg: 'Json Message'}));
- setTimeout(() => {
- expect(socketSpy).to.have.been.called;
- expect(socketSpy).to.have.been.calledWith('update', {
- model: channelName,
- msg: {msg: 'Json Message'}
- });
- done();
- }, 1000)
- });
-
- it('should send a websocket event when an object has been removed', (done) => {
- publisher.publish(channelName, JSON.stringify({msg: 'Deleted', deleted: true}));
- setTimeout(() => {
- expect(socketSpy).to.have.been.called;
- expect(socketSpy).to.have.been.calledWith('remove', {
- model: channelName,
- msg: {
- msg: 'Deleted',
- deleted: true
- },
- deleted: true
- });
-
- done();
- }, 1000)
- });
-
- it('should not send a websocket event if the channel is Diag', (done) => {
- publisher.publish('Diag', JSON.stringify({msg: 'Json Message'}));
- setTimeout(() => {
- expect(socketSpy).not.to.have.been.called;
- done();
- }, 1000)
- });
- });
-})();
\ No newline at end of file
diff --git a/src/config/config.js b/src/config/config.js
index 7ce4c0c..b533ddb 100644
--- a/src/config/config.js
+++ b/src/config/config.js
@@ -1,4 +1,3 @@
-
/*
* Copyright 2017-present Open Networking Foundation
@@ -15,11 +14,10 @@
* limitations under the License.
*/
-
(function () {
'use strict';
-
- // NOTE do we still need CLI args?
+
+ // NOTE do we still need CLI args?
// Won't be better to use NODE_ENV and the native node-yaml-config feature
const argv = require('yargs').argv;
@@ -40,12 +38,10 @@
}
module.exports = {
- redis: {
- host: (config && config.redis) ? config.redis.host : 'redis',
- port: (config && config.redis) ? config.redis.port : 6379
- },
gateway: {
port: (config && config.gateway) ? config.gateway.port : 3000
- }
+ },
+ kafka_bootstrap_servers: ( config && config.kafka_bootstrap_servers ) ?
+ config.kafka_bootstrap_servers : ['cord-kafka:9092']
};
-})();
\ No newline at end of file
+})();
diff --git a/src/controllers/kafka.js b/src/controllers/kafka.js
new file mode 100644
index 0000000..f754a65
--- /dev/null
+++ b/src/controllers/kafka.js
@@ -0,0 +1,84 @@
+/*
+ * Copyright 2018-present Open Networking Foundation
+
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+
+ * http://www.apache.org/licenses/LICENSE-2.0
+
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+(function () {
+ 'use strict';
+
+ const socketIo = require('./websocket.js');
+ const config = require('../config/config.js');
+ const logger = require('../config/logger.js');
+ const socket = socketIo.get();
+
+ // docs: https://github.com/Blizzard/node-rdkafka
+ var kafka = require('node-rdkafka');
+
+ logger.log('debug',`Using librdkafka version: ${kafka.librdkafkaVersion}, kafka features: '${kafka.features}'`);
+ logger.log('debug',`Connecting to broker: ${config.kafka_bootstrap_servers}`);
+
+ var stream = kafka.KafkaConsumer.createReadStream({
+ 'metadata.broker.list': config.kafka_bootstrap_servers,
+ 'group.id': 'xos-ws',
+ 'socket.keepalive.enable': true,
+ 'enable.auto.commit': false
+ }, {}, {
+ topics: ['xos.gui_events'],
+ });
+
+ stream.on('ready', function () {
+ logger.log('info', 'Kafka connected');
+ });
+
+ stream.on('error', function (err) {
+ logger.log('error', err);
+ });
+
+ stream.consumer.on('event.error', function (err) {
+ logger.log('error', err);
+ });
+
+ stream.on('data', function (msg) {
+ logger.log('debug', `Topic: ${msg.topic}, Key: ${msg.key}, Timestamp: ${msg.timestamp}`);
+
+ // strip diag messages
+ // NOTE: have to coerce to string (due to FFI?)
+ if (msg.key.toString() === 'Diag') {
+ return;
+ }
+
+ let msgobj;
+
+ try {
+ // TODO find the user that needs to be notified for msg.object update
+ msgobj = JSON.parse(msg.value)
+ }
+
+ catch(e) {
+ // stringify the event if it is not JSON
+ msgobj = msg.value.toString()
+ }
+
+ if (msgobj.deleted) {
+ logger.log('info', 'Remove on: ' + msg.key + ': ' + msg.value);
+ socket.emit('remove', {model: msg.key.toString(), msg: msgobj, deleted: true});
+ }
+ else {
+ logger.log('info', 'Update on: ' + msg.key + ': ' + msg.value);
+ socket.emit('update', {model: msg.key.toString(), msg: msgobj});
+ }
+
+ });
+
+})();
diff --git a/src/controllers/redis.js b/src/controllers/redis.js
deleted file mode 100644
index 7e86000..0000000
--- a/src/controllers/redis.js
+++ /dev/null
@@ -1,77 +0,0 @@
-
-/*
- * Copyright 2017-present Open Networking Foundation
-
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
-
- * http://www.apache.org/licenses/LICENSE-2.0
-
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-(function () {
- 'use strict';
-
- const socketIo = require('./websocket.js');
- const config = require('../config/config.js');
- const logger = require('../config/logger.js');
- const request = require('superagent');
- const socket = socketIo.get();
- const _ = require('lodash');
-
- var redis = require('redis');
-
- var client = redis.createClient({
- host: config.redis.host,
- port: config.redis.port
- });
-
- client.on('error', function (err) {
- logger.log('error', err);
- });
-
- client.on('ready', function () {
- logger.log('info', 'Redis connected');
- });
-
- client.on('subscribe', function (channel) {
- logger.log('debug', `Subscribed to channel: ${channel}`);
- });
-
- client.on('pmessage', function (pattern, channel, message) {
- if (channel === 'Diag') {
- return;
- }
-
-
- let msg;
- try {
- msg = JSON.parse(message);
- // TODO find the user that needs to be notified for msg.object update
- }
- catch(e) {
- // send the event also if it is not JSON
- msg = message;
- }
-
- if (msg.deleted) {
- logger.log('warn', 'Remove on: ' + channel + ': ' + message);
- socket.emit('remove', {model: channel, msg: msg, deleted: true});
- }
- else {
- logger.log('warn', 'Update on: ' + channel + ': ' + message);
- socket.emit('update', {model: channel, msg: msg});
- }
- });
-
- // subscribe to all channels
- client.psubscribe('*');
-
-})();
\ No newline at end of file
diff --git a/src/controllers/websocket.js b/src/controllers/websocket.js
index 30a7885..e04e13a 100644
--- a/src/controllers/websocket.js
+++ b/src/controllers/websocket.js
@@ -1,4 +1,3 @@
-
/*
* Copyright 2017-present Open Networking Foundation
@@ -15,7 +14,6 @@
* limitations under the License.
*/
-
(function () {
'use strict';
@@ -54,4 +52,4 @@
// const socket = socketIo.get();
// socket.emit('eventName', data);
-})();
\ No newline at end of file
+})();
diff --git a/src/server.js b/src/server.js
index 0cebf74..54fc65f 100644
--- a/src/server.js
+++ b/src/server.js
@@ -18,7 +18,7 @@
(function () {
'use strict';
-
+
const express = require('express');
const app = express();
const config = require('./config/config.js').gateway;
@@ -48,8 +48,8 @@
// once server is ready setup WebSocket
socketIo.create(server);
- // start redis
- require('./controllers/redis.js');
+ // start kafka
+ require('./controllers/kafka.js');
});
app.server = server;
return server;
@@ -70,4 +70,4 @@
start: startServer,
stop: stopServer
};
-})();
\ No newline at end of file
+})();