CORD-239 refactor of harvester uservice
Change-Id: I0fdb587267b6c5fb1c53bb35d77cd5921b937b6d
diff --git a/harvester/Dockerfile b/harvester/Dockerfile
index 18bac9d..360b39d 100644
--- a/harvester/Dockerfile
+++ b/harvester/Dockerfile
@@ -11,12 +11,19 @@
## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
## See the License for the specific language governing permissions and
## limitations under the License.
-FROM python:2.7-alpine
+FROM golang:1.6-alpine
MAINTAINER Open Networking Laboratory <info@onlab.us>
-RUN apk update && apk add bind
+RUN apk --update add openssh-client git bind
-ADD dhcpharvester.py /dhcpharvester.py
+RUN go get github.com/tools/godep
+ADD . /go/src/gerrit.opencord.com/maas/harvester
+
+WORKDIR /go/src/gerrit.opencord.com/maas/harvester
+RUN /go/bin/godep restore || true
+
+WORKDIR /go
+RUN go install gerrit.opencord.com/maas/harvester
LABEL org.label-schema.name="harvester" \
org.label-schema.description="Provides DHCP havesting and insertion into DNS" \
@@ -24,4 +31,4 @@
org.label-schema.vendor="Open Networking Labratory" \
org.label-schema.schema-version="1.0"
-ENTRYPOINT [ "python", "/dhcpharvester.py" ]
+ENTRYPOINT ["/go/bin/harvester"]
diff --git a/harvester/Godeps/Godeps.json b/harvester/Godeps/Godeps.json
new file mode 100644
index 0000000..bda40ad
--- /dev/null
+++ b/harvester/Godeps/Godeps.json
@@ -0,0 +1,54 @@
+{
+ "ImportPath": "gerrit.opencord.org/maas/cord-provisioner",
+ "GoVersion": "go1.6",
+ "GodepVersion": "v72",
+ "Deps": [
+ {
+ "ImportPath": "github.com/tatsushid/go-fastping",
+ "Rev": "d7bb493dee3e090e2ffb6914adddf17c1e7c026c"
+ },
+ {
+ "ImportPath": "github.com/gorilla/context",
+ "Comment": "v1.1-4-gaed02d1",
+ "Rev": "aed02d124ae4a0e94fea4541c8effd05bf0c8296"
+ },
+ {
+ "ImportPath": "github.com/gorilla/mux",
+ "Comment": "v1.1-13-g9fa818a",
+ "Rev": "9fa818a44c2bf1396a17f9d5a3c0f6dd39d2ff8e"
+ },
+ {
+ "ImportPath": "github.com/kelseyhightower/envconfig",
+ "Comment": "1.1.0-17-g91921eb",
+ "Rev": "91921eb4cf999321cdbeebdba5a03555800d493b"
+ },
+ {
+ "ImportPath": "github.com/Sirupsen/logrus",
+ "Rev": "f3cfb454f4c209e6668c95216c4744b8fddb2356"
+ },
+ {
+ "ImportPath": "golang.org/x/net/icmp",
+ "Rev": "88c1a61b3dd4d98651f24775ca288fac5d2544ce"
+ },
+ {
+ "ImportPath": "golang.org/x/net/internal/iana",
+ "Rev": "88c1a61b3dd4d98651f24775ca288fac5d2544ce"
+ },
+ {
+ "ImportPath": "golang.org/x/net/internal/netreflect",
+ "Rev": "88c1a61b3dd4d98651f24775ca288fac5d2544ce"
+ },
+ {
+ "ImportPath": "golang.org/x/net/internal/nettest",
+ "Rev": "88c1a61b3dd4d98651f24775ca288fac5d2544ce"
+ },
+ {
+ "ImportPath": "golang.org/x/net/ipv4",
+ "Rev": "88c1a61b3dd4d98651f24775ca288fac5d2544ce"
+ },
+ {
+ "ImportPath": "golang.org/x/net/ipv6",
+ "Rev": "88c1a61b3dd4d98651f24775ca288fac5d2544ce"
+ }
+ ]
+}
diff --git a/harvester/README.md b/harvester/README.md
index 509c26d..d247528 100644
--- a/harvester/README.md
+++ b/harvester/README.md
@@ -1,14 +1,20 @@
# DHCP/DNS Name and IP Harvester
-This Python application and Docker image provide an utility that periodically parses the DHCP leases files and updates the `bind9` DNS configuration so that hosts
-that are assigned IP addresses dynamically from DHCP can be looked up via DNS.
+This µservice and Docker image provide an utility that periodically parses the
+DHCP leases database file and updates the `bind9` DNS configuration so that
+hosts that are assigned IP addresses dynamically from DHCP can be looked up via
+DNS.
### Integration
-There are several keys to making all this work. The utility needs to be able to read the DHCP lease file as well as write a file to a location that can be read
-by the DNS server; so more than likely this utility should be run on the same host that is running DHCP and DNS. Additionally, this utility needs to be able to
-run the bind9 utility `rndc` to reload the DNS zone. This means that it needs a `DNSSEC` key and secret to access the DNS server.
+There are several keys to making all this work. The utility needs to be able to
+read the DHCP lease database file as well as write a file to a location that
+can be read by the DNS server; so more than likely this utility should be run
+on the same host that is running DHCP and DNS. Additionally, this utility needs
+to be able to run the bind9 utility `rndc` to reload the DNS zone. This means
+that it needs a `DNSSEC` key and secret to access the DNS server.
-Lastly, this utility generates a file that can be `$include`-ed into a bind9 zone file, so the original zone file needs to be augmented with a `$INCLUDE` statement
-that includes the files to which the uility is configured to write via the `-dest` command line option.
+Lastly, this utility generates a file that can be `$include`-ed into a bind9
+zone file, so the original zone file needs to be augmented with a `$INCLUDE`
+statement that includes the files to which the uility is configured to write.
### Docker Build
To build the docker image use the command:
@@ -16,29 +22,55 @@
docker build -t harvester .
```
+### configuration
+The utility is configured the environment following the 12 factor application
+principles. The available configuration settings are:
+
+| OPTION | DEFAULT | DESCRIPTION |
+| --- | --- | --- |
+| PORT | 4246 | port on which the service will listen for requests |
+| LISTEN | 0.0.0.0 | IP on which the service will listen for requests |
+| LOG_LEVEL | warning | log output level |
+| LOG_FORMAT | text | format of log messages |
+| DHCP_LEASE_FILE | /harvester/dhcpd.leases | lease file to parse for lease information |
+| OUTPUT_FILE | | name of file to output discovered lease in bind9 format |
+| OUTPUT_FORMAT | {{.ClientHostname}}\tIN A {{.IPAddress}}\t; {{.HardwareAddress}} | specifies the single entry format when outputing to a file |
+| VERIFY_LEASES | true | verifies leases with a ping |
+| VERIFY_TIMEOUT | 1s | max timeout (RTT) to wait for verification pings |
+| VERIFY_WITH_UDP | false | use UDP instead of raw sockets for ping verification |
+| QUERY_PERIOD | 30s | period at which the DHCP lease file is processed |
+| QUIET_PERIOD | 2s | period to wait between accepting parse requests |
+| REQUEST_TIMEOUT | 10s | period to wait for processing when requesting a DHCP lease database parsing |
+| RNDC_UPDATE | false | determines if the harvester reloads the DNS servers after harvest |
+| RNDC_ADDRESS | 127.0.0.1 | IP address of the DNS server to contact via RNDC |
+| RNDC_PORT | 954 | port of the DNS server to contact via RNDC |
+| RNDC_KEY_FILE | /key/rndc.conf.maas | key file, with default, to contact DNS server |
+| RNDC_ZONE | cord.lab | zone to reload |
+
+When configuring the µservice via the environment the name of the option should
+be prefixed with `HARVESTER_`.
+
### Docker Run
To run the utility, a docker command similar to what is below may be used
```
-docker run -d --name=dhcpharvester \
- -v `pwd`/key:/key -v /var/lib/maas/dhcp:/dhcp -v /etc/bind/maas:/bind harvester \
- -f '^(?!cord)' -u -s 192.168.42.231 -p 954 -k /key/mykey.conf -z cord.lab -r 5m \
- -y -t 1s
+docker run -d --name=harvester
```
-### API
-There is a simple REST API on this utility so that an external client can asynchronously invoke the DHCP harvest behavior. The REST API is
-synchronous in that the request will not return a response until the harvest is complete. To invoke the request a `HTTP PUT` request needs
-be sent to the utility, such as by curl:
-```
-curl -XPOST http://<apiserver>:<apiport>/harvest
-```
-Currently there is not security around this so it could be abused. There is some protection so that if the system is sent multple request
-if won't actually reharvest until a quiet period has expired. The purpose is to not allow the system to be overloaded.
+### REST API
+
+| RESOURCE | METHOD | DESCRIPTION |
+| --- | --- | --- |
+| /harvest | POST | Requests the processing of the DHCP lease database |
+| /lease | GET | Returns the list of DHCP leases harvested from the DHCP server |
+| /lease/{ip} | GET | Returns a single DHCP lease associated with the given IP |
+| /lease/hostname/{name} | GET | Returns a single DHCP lease associated with the given client hostname |
+| /lease/hardware/{mac} | GET | Returns a single DHCP lease associated with the given hardware addreaa |
+
+Currently there is no security around this so it could be abused. There is some
+protection so that if the system is sent multiple requests it won't actually
+re-harvest until a quiet period has expired. The purpose is to not allow the
+system to be overloaded.
### Implementation Details
-Internally the implementation uses threads and queues to communicate between the threads when the utility is in the mode to periodically
-harvest.
-
-For the verification of IP addresses, i.e. pinging the hosts, worker threads are used to support concurrency, thus making the verification
-process faster.
+Verification of leases is performed by doing an ICMP ping to the host.
diff --git a/harvester/dhcpharvester.py b/harvester/dhcpharvester.py
deleted file mode 100755
index 95e69a6..0000000
--- a/harvester/dhcpharvester.py
+++ /dev/null
@@ -1,627 +0,0 @@
-#!/usr/bin/python
-## Copyright 2016 Open Networking Laboratory
-##
-## Licensed under the Apache License, Version 2.0 (the "License");
-## you may not use this file except in compliance with the License.
-## You may obtain a copy of the License at
-##
-## http://www.apache.org/licenses/LICENSE-2.0
-##
-## Unless required by applicable law or agreed to in writing, software
-## distributed under the License is distributed on an "AS IS" BASIS,
-## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-## See the License for the specific language governing permissions and
-## limitations under the License.
-import sys, threading, thread, subprocess, re, time, datetime, bisect, BaseHTTPServer
-from optparse import OptionParser
-from Queue import Queue
-
-def parse_timestamp(raw_str):
- tokens = raw_str.split()
-
- if len(tokens) == 1:
- if tokens[0].lower() == 'never':
- return 'never';
-
- else:
- raise Exception('Parse error in timestamp')
-
- elif len(tokens) == 3:
- return datetime.datetime.strptime(' '.join(tokens[1:]),
- '%Y/%m/%d %H:%M:%S')
-
- else:
- raise Exception('Parse error in timestamp')
-
-def timestamp_is_ge(t1, t2):
- if t1 == 'never':
- return True
-
- elif t2 == 'never':
- return False
-
- else:
- return t1 >= t2
-
-
-def timestamp_is_lt(t1, t2):
- if t1 == 'never':
- return False
-
- elif t2 == 'never':
- return t1 != 'never'
-
- else:
- return t1 < t2
-
-
-def timestamp_is_between(t, tstart, tend):
- return timestamp_is_ge(t, tstart) and timestamp_is_lt(t, tend)
-
-
-def parse_hardware(raw_str):
- tokens = raw_str.split()
-
- if len(tokens) == 2:
- return tokens[1]
-
- else:
- raise Exception('Parse error in hardware')
-
-
-def strip_endquotes(raw_str):
- return raw_str.strip('"')
-
-
-def identity(raw_str):
- return raw_str
-
-
-def parse_binding_state(raw_str):
- tokens = raw_str.split()
-
- if len(tokens) == 2:
- return tokens[1]
-
- else:
- raise Exception('Parse error in binding state')
-
-
-def parse_next_binding_state(raw_str):
- tokens = raw_str.split()
-
- if len(tokens) == 3:
- return tokens[2]
-
- else:
- raise Exception('Parse error in next binding state')
-
-
-def parse_rewind_binding_state(raw_str):
- tokens = raw_str.split()
-
- if len(tokens) == 3:
- return tokens[2]
-
- else:
- raise Exception('Parse error in next binding state')
-
-def parse_res_fixed_address(raw_str):
- return raw_str
-
-def parse_res_hardware(raw_str):
- tokens = raw_str.split()
- return tokens[1]
-
-def parse_reservation_file(res_file):
- valid_keys = {
- 'hardware' : parse_res_hardware,
- 'fixed-address' : parse_res_fixed_address,
- }
-
- res_db = {}
- res_rec = {}
- in_res = False
- for line in res_file:
- if line.lstrip().startswith('#'):
- continue
- tokens = line.split()
-
- if len(tokens) == 0:
- continue
-
- key = tokens[0].lower()
-
- if key == 'host':
- if not in_res:
- res_rec = {'hostname' : tokens[1]}
- in_res = True
-
- else:
- raise Exception("Parse error in reservation file")
- elif key == '}':
- if in_res:
- for k in valid_keys:
- if callable(valid_keys[k]):
- res_rec[k] = res_rec.get(k, '')
- else:
- res_rec[k] = False
-
- hostname = res_rec['hostname']
-
- if hostname in res_db:
- res_db[hostname].insert(0, res_rec)
-
- else:
- res_db[hostname] = [res_rec]
-
- res_rec = {}
- in_res = False
-
- else:
- raise Exception('Parse error in reservation file')
-
- elif key in valid_keys:
- if in_res:
- value = line[(line.index(key) + len(key)):]
- value = value.strip().rstrip(';').rstrip()
-
- if callable(valid_keys[key]):
- res_rec[key] = valid_keys[key](value)
- else:
- res_rec[key] = True
-
- else:
- raise Exception('Parse error in reservation file')
-
- else:
- if in_res:
- raise Exception('Parse error in reservation file')
-
- if in_res:
- raise Exception('Parse error in reservation file')
-
- # Turn the leases into an array
- results = []
- for res in res_db:
- results.append({
- 'client-hostname' : res_db[res][0]['hostname'],
- 'hardware' : res_db[res][0]['hardware'],
- 'ip_address' : res_db[res][0]['fixed-address'],
- })
- return results
-
-
-def parse_leases_file(leases_file):
- valid_keys = {
- 'starts': parse_timestamp,
- 'ends': parse_timestamp,
- 'tstp': parse_timestamp,
- 'tsfp': parse_timestamp,
- 'atsfp': parse_timestamp,
- 'cltt': parse_timestamp,
- 'hardware': parse_hardware,
- 'binding': parse_binding_state,
- 'next': parse_next_binding_state,
- 'rewind': parse_rewind_binding_state,
- 'uid': strip_endquotes,
- 'client-hostname': strip_endquotes,
- 'option': identity,
- 'set': identity,
- 'on': identity,
- 'abandoned': None,
- 'bootp': None,
- 'reserved': None,
- }
-
- leases_db = {}
-
- lease_rec = {}
- in_lease = False
- in_failover = False
-
- for line in leases_file:
- if line.lstrip().startswith('#'):
- continue
-
- tokens = line.split()
-
- if len(tokens) == 0:
- continue
-
- key = tokens[0].lower()
-
- if key == 'lease':
- if not in_lease:
- ip_address = tokens[1]
-
- lease_rec = {'ip_address' : ip_address}
- in_lease = True
-
- else:
- raise Exception('Parse error in leases file')
-
- elif key == 'failover':
- in_failover = True
- elif key == '}':
- if in_lease:
- for k in valid_keys:
- if callable(valid_keys[k]):
- lease_rec[k] = lease_rec.get(k, '')
- else:
- lease_rec[k] = False
-
- ip_address = lease_rec['ip_address']
-
- if ip_address in leases_db:
- leases_db[ip_address].insert(0, lease_rec)
-
- else:
- leases_db[ip_address] = [lease_rec]
-
- lease_rec = {}
- in_lease = False
-
- elif in_failover:
- in_failover = False
- continue
- else:
- raise Exception('Parse error in leases file')
-
- elif key in valid_keys:
- if in_lease:
- value = line[(line.index(key) + len(key)):]
- value = value.strip().rstrip(';').rstrip()
-
- if callable(valid_keys[key]):
- lease_rec[key] = valid_keys[key](value)
- else:
- lease_rec[key] = True
-
- else:
- raise Exception('Parse error in leases file')
-
- else:
- if in_lease:
- raise Exception('Parse error in leases file')
-
- if in_lease:
- raise Exception('Parse error in leases file')
-
- return leases_db
-
-
-def round_timedelta(tdelta):
- return datetime.timedelta(tdelta.days,
- tdelta.seconds + (0 if tdelta.microseconds < 500000 else 1))
-
-
-def timestamp_now():
- n = datetime.datetime.utcnow()
- return datetime.datetime(n.year, n.month, n.day, n.hour, n.minute,
- n.second)# + (0 if n.microsecond < 500000 else 1))
-
-
-def lease_is_active(lease_rec, as_of_ts):
- return lease_rec['binding'] != 'free' and timestamp_is_between(as_of_ts, lease_rec['starts'],
- lease_rec['ends'])
-
-
-def ipv4_to_int(ipv4_addr):
- parts = ipv4_addr.split('.')
- return (int(parts[0]) << 24) + (int(parts[1]) << 16) + \
- (int(parts[2]) << 8) + int(parts[3])
-
-def select_active_leases(leases_db, as_of_ts):
- retarray = []
- sortedarray = []
-
- for ip_address in leases_db:
- lease_rec = leases_db[ip_address][0]
-
- if lease_is_active(lease_rec, as_of_ts):
- ip_as_int = ipv4_to_int(ip_address)
- insertpos = bisect.bisect(sortedarray, ip_as_int)
- sortedarray.insert(insertpos, ip_as_int)
- retarray.insert(insertpos, lease_rec)
-
- return retarray
-
-def matched(list, target):
- if list == None:
- return False
-
- for r in list:
- if re.match(r, target) != None:
- return True
- return False
-
-def convert_to_seconds(time_val):
- num = int(time_val[:-1])
- if time_val.endswith('s'):
- return num
- elif time_val.endswith('m'):
- return num * 60
- elif time_val.endswith('h'):
- return num * 60 * 60
- elif time_val.endswith('d'):
- return num * 60 * 60 * 24
-
-def ping(ip, timeout):
- cmd = ['ping', '-c', '1', '-w', timeout, ip]
- try:
- out = subprocess.check_output(cmd)
- return True
- except subprocess.CalledProcessError as e:
- return False
-
-def ping_worker(list, to, respQ):
- for lease in list:
- respQ.put(
- {
- 'verified': ping(lease['ip_address'], to),
- 'lease' : lease,
- })
-
-def interruptable_get(q):
- r = None
- while True:
- try:
- return q.get(timeout=1000)
- except Queue.Empty:
- pass
-
-##############################################################################
-
-def harvest(options):
-
- ifilter = None
- if options.include != None:
- ifilter = options.include.translate(None, ' ').split(',')
-
- rfilter = None
- if options.filter != None:
- rfilter = options.filter.split(',')
-
- myfile = open(options.leases, 'r')
- leases = parse_leases_file(myfile)
- myfile.close()
-
- reservations = []
- try:
- with open(options.reservations, 'r') as res_file:
- reservations = parse_reservation_file(res_file)
- res_file.close()
- except (IOError) as e:
- pass
-
- now = timestamp_now()
- report_dataset = select_active_leases(leases, now) + reservations
-
- verified = []
- if options.verify:
-
- # To verify is lease information is valid, i.e. that the host which got the lease still responding
- # we ping the host. Not perfect, but good for the main use case. As the lease file can get long
- # a little concurrency is used. The lease list is divided amoung workers and each worker takes
- # a share.
- respQ = Queue()
- to = str(convert_to_seconds(options.timeout))
- share = int(len(report_dataset) / options.worker_count)
- extra = len(report_dataset) % options.worker_count
- start = 0
- for idx in range(0, options.worker_count):
- end = start + share
- if extra > 0:
- end = end + 1
- extra = extra - 1
- worker = threading.Thread(target=ping_worker, args=(report_dataset[start:end], to, respQ))
- worker.daemon = True
- worker.start()
- start = end
-
- # All the verification work has been farmed out to worker threads, so sit back and wait for reponses.
- # Once all responses are received we are done. Probably should put a time out here as well, but for
- # now we expect a response for every lease, either positive or negative
- count = 0
- while count != len(report_dataset):
- resp = interruptable_get(respQ)
- count = count + 1
- if resp['verified']:
- print("INFO: verified host '%s' with address '%s'" % (resp['lease']['client-hostname'], resp['lease']['ip_address']))
- verified.append(resp['lease'])
- else:
- print("INFO: dropping host '%s' with address '%s' (not verified)" % (resp['lease']['client-hostname'], resp['lease']['ip_address']))
- else:
- verified = report_dataset
-
- # Look for duplicate names and add the compressed MAC as a suffix
- names = {}
- for lease in verified:
- # If no client hostname use MAC
- name = lease['client-hostname']
- if 'client-hostname' not in lease or len(name) == 0:
- name = "UNK-" + lease['hardware'].translate(None, ':').upper()
-
- if name in names:
- names[name] = '+'
- else:
- names[name] = '-'
-
- size = 0
- count = 0
- for lease in verified:
- name = lease['client-hostname']
- if 'client-hostname' not in lease or len(name) == 0:
- name = "UNK-" + lease['hardware'].translate(None, ':').upper()
-
- if (ifilter != None and name in ifilter) or matched(rfilter, name):
- if names[name] == '+':
- lease['client-hostname'] = name + '-' + lease['hardware'].translate(None, ':').upper()
- size = max(size, len(lease['client-hostname']))
- count += 1
-
- if options.dest == '-':
- out=sys.stdout
- else:
- out=open(options.dest, 'w+')
-
- for lease in verified:
- name = lease['client-hostname']
- if 'client-hostname' not in lease or len(name) == 0:
- name = "UNK-" + lease['hardware'].translate(None, ':').upper()
-
- if ifilter != None and name in ifilter or matched(rfilter, name):
- out.write(format(name, '<'+str(size)) + ' IN A ' + lease['ip_address'] + ' ; ' + lease['hardware'] +'\n')
- if options.dest != '-':
- out.close()
- return count
-
-def reload_zone(rndc, server, port, key, zone):
- cmd = [rndc, '-s', server]
- if key != None:
- cmd.extend(['-c', key])
- cmd.extend(['-p', port, 'reload'])
- if zone != None:
- cmd.append(zone)
-
- try:
- out = subprocess.check_output(cmd)
- print("INFO: [%s UTC] updated DNS sever" % time.asctime(time.gmtime()))
- except subprocess.CalledProcessError as e:
- print("ERROR: failed to update DNS server, exit code %d" % e.returncode)
- print(e.output)
-
-def handleRequestsUsing(requestQ):
- return lambda *args: ApiHandler(requestQ, *args)
-
-class ApiHandler(BaseHTTPServer.BaseHTTPRequestHandler):
- def __init__(s, requestQ, *args):
- s.requestQ = requestQ
- BaseHTTPServer.BaseHTTPRequestHandler.__init__(s, *args)
-
- def do_HEAD(s):
- s.send_response(200)
- s.send_header("Content-type", "application/json")
- s.end_headers()
-
- def do_POST(s):
- if s.path == '/harvest':
- waitQ = Queue()
- s.requestQ.put(waitQ)
- resp = waitQ.get(block=True, timeout=None)
- s.send_response(200)
- s.send_header('Content-type', 'application/json')
- s.end_headers()
-
- if resp == "QUIET":
- s.wfile.write('{ "response" : "QUIET" }')
- else:
- s.wfile.write('{ "response" : "OK" }')
-
- else:
- s.send_response(404)
-
- def do_GET(s):
- """Respond to a GET request."""
- s.send_response(404)
-
-def do_api(hostname, port, requestQ):
- server_class = BaseHTTPServer.HTTPServer
- httpd = server_class((hostname, int(port)), handleRequestsUsing(requestQ))
- print("INFO: [%s UTC] Start API server on %s:%s" % (time.asctime(time.gmtime()), hostname, port))
- try:
- httpd.serve_forever()
- except KeyboardInterrupt:
- pass
- httpd.server_close()
- print("INFO: [%s UTC] Stop API server on %s:%s" % (time.asctime(time.gmtime()), hostname, port))
-
-def harvester(options, requestQ):
- quiet = convert_to_seconds(options.quiet)
- last = -1
- resp = "OK"
- while True:
- responseQ = requestQ.get(block=True, timeout=None)
- if last == -1 or (time.time() - last) > quiet:
- work_field(options)
- last = time.time()
- resp = "OK"
- else:
- resp = "QUIET"
-
- if responseQ != None:
- responseQ.put(resp)
-
-def work_field(options):
- start = datetime.datetime.now()
- print("INFO: [%s UTC] starting to harvest hosts from DHCP" % (time.asctime(time.gmtime())))
- count = harvest(options)
- end = datetime.datetime.now()
- delta = end - start
- print("INFO: [%s UTC] harvested %d hosts, taking %d seconds" % (time.asctime(time.gmtime()), count, delta.seconds))
- if options.update:
- reload_zone(options.rndc, options.server, options.port, options.key, options.zone)
-
-def main():
- parser = OptionParser()
- parser.add_option('-l', '--leases', dest='leases', default='/dhcp/dhcpd.leases',
- help="specifies the DHCP lease file from which to harvest")
- parser.add_option('-x', '--reservations', dest='reservations', default='/etc/dhcp/dhcpd.reservations',
- help="specified the reservation file as ISC DHCP doesn't update the lease file for fixed addresses")
- parser.add_option('-d', '--dest', dest='dest', default='/bind/dhcp_harvest.inc',
- help="specifies the file to write the additional DNS information")
- parser.add_option('-i', '--include', dest='include', default=None,
- help="list of hostnames to include when harvesting DNS information")
- parser.add_option('-f', '--filter', dest='filter', default=None,
- help="list of regex expressions to use as an include filter")
- parser.add_option('-r', '--repeat', dest='repeat', default=None,
- help="continues to harvest DHCP information every specified interval")
- parser.add_option('-c', '--command', dest='rndc', default='rndc',
- help="shell command to execute to cause reload")
- parser.add_option('-k', '--key', dest='key', default=None,
- help="rndc key file to use to access DNS server")
- parser.add_option('-s', '--server', dest='server', default='127.0.0.1',
- help="server to reload after generating updated dns information")
- parser.add_option('-p', '--port', dest='port', default='954',
- help="port on server to contact to reload server")
- parser.add_option('-z', '--zone', dest='zone', default=None,
- help="zone to reload after generating updated dns information")
- parser.add_option('-u', '--update', dest='update', default=False, action='store_true',
- help="update the DNS server, by reloading the zone")
- parser.add_option('-y', '--verify', dest='verify', default=False, action='store_true',
- help="verify the hosts with a ping before pushing them to DNS")
- parser.add_option('-t', '--timeout', dest='timeout', default='1s',
- help="specifies the duration to wait for a verification ping from a host")
- parser.add_option('-a', '--apiserver', dest='apiserver', default='0.0.0.0',
- help="specifies the interfaces on which to listen for API requests")
- parser.add_option('-e', '--apiport', dest='apiport', default='8954',
- help="specifies the port on which to listen for API requests")
- parser.add_option('-q', '--quiet', dest='quiet', default='1m',
- help="specifieds a minimum quiet period between actually harvest times.")
- parser.add_option('-w', '--workers', dest='worker_count', type='int', default=5,
- help="specifies the number of workers to use when verifying IP addresses")
-
- (options, args) = parser.parse_args()
-
- # Kick off a thread to listen for HTTP requests to force a re-evaluation
- requestQ = Queue()
- api = threading.Thread(target=do_api, args=(options.apiserver, options.apiport, requestQ))
- api.daemon = True
- api.start()
-
- if options.repeat == None:
- work_field(options)
- else:
- secs = convert_to_seconds(options.repeat)
- farmer = threading.Thread(target=harvester, args=(options, requestQ))
- farmer.daemon = True
- farmer.start()
- while True:
- cropQ = Queue()
- requestQ.put(cropQ)
- interruptable_get(cropQ)
- time.sleep(secs)
-
-if __name__ == "__main__":
- main()
diff --git a/harvester/handlers.go b/harvester/handlers.go
new file mode 100644
index 0000000..56316eb
--- /dev/null
+++ b/harvester/handlers.go
@@ -0,0 +1,136 @@
+// Copyright 2016 Open Networking Laboratory
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package main
+
+import (
+ "encoding/json"
+ "github.com/gorilla/mux"
+ "net/http"
+ "strings"
+ "time"
+)
+
+// listLeaseHandler returns a list of all known leases
+func (app *application) listLeasesHandler(w http.ResponseWriter, r *http.Request) {
+
+ // convert data map of leases to a slice
+ app.interchange.RLock()
+ leases := make([]Lease, len(app.leases))
+ i := 0
+ for _, lease := range app.leases {
+ leases[i] = *lease
+ i += 1
+ }
+ app.interchange.RUnlock()
+
+ w.Header().Set("Content-Type", "application/json")
+ encoder := json.NewEncoder(w)
+ encoder.Encode(leases)
+}
+
+// getLeaseHandler return a single known lease
+func (app *application) getLeaseHandler(w http.ResponseWriter, r *http.Request) {
+ vars := mux.Vars(r)
+ ip, ok := vars["ip"]
+ if !ok || strings.TrimSpace(ip) == "" {
+ w.WriteHeader(http.StatusBadRequest)
+ return
+ }
+ app.interchange.RLock()
+ lease, ok := app.leases[ip]
+ app.interchange.RUnlock()
+ if !ok {
+ w.WriteHeader(http.StatusNotFound)
+ return
+ }
+
+ w.Header().Set("Content-Type", "application/json")
+ encoder := json.NewEncoder(w)
+ encoder.Encode(lease)
+}
+
+// getLeaseByHardware return a single known lease by its MAC address
+func (app *application) getLeaseByHardware(w http.ResponseWriter, r *http.Request) {
+ vars := mux.Vars(r)
+ mac, ok := vars["mac"]
+ if !ok || strings.TrimSpace(mac) == "" {
+ w.WriteHeader(http.StatusBadRequest)
+ return
+ }
+ app.interchange.RLock()
+ lease, ok := app.byHardware[mac]
+ app.interchange.RUnlock()
+ if !ok {
+ w.WriteHeader(http.StatusNotFound)
+ return
+ }
+
+ w.Header().Set("Content-Type", "application/json")
+ encoder := json.NewEncoder(w)
+ encoder.Encode(lease)
+}
+
+// getLeaseByHostname return a single known lease by its hostname
+func (app *application) getLeaseByHostname(w http.ResponseWriter, r *http.Request) {
+ vars := mux.Vars(r)
+ name, ok := vars["name"]
+ if !ok || strings.TrimSpace(name) == "" {
+ w.WriteHeader(http.StatusBadRequest)
+ return
+ }
+ app.interchange.RLock()
+ lease, ok := app.byHostname[name]
+ app.interchange.RUnlock()
+ if !ok {
+ w.WriteHeader(http.StatusNotFound)
+ return
+ }
+
+ w.Header().Set("Content-Type", "application/json")
+ encoder := json.NewEncoder(w)
+ encoder.Encode(lease)
+}
+
+// doHarvestHandler request a harvest of lease information and return if it was completed or during the quiet period
+func (app *application) doHarvestHandler(w http.ResponseWriter, r *http.Request) {
+ app.log.Info("Manual harvest invocation")
+ responseChan := make(chan uint)
+ app.requests <- &responseChan
+ select {
+ case response := <-responseChan:
+ switch response {
+ case responseOK:
+ w.Header().Set("Content-Type", "application/json")
+ encoder := json.NewEncoder(w)
+ encoder.Encode(struct {
+ Response string `json:"response"`
+ }{
+ Response: "OK",
+ })
+ case responseQuiet:
+ w.Header().Set("Content-Type", "application/json")
+ encoder := json.NewEncoder(w)
+ encoder.Encode(struct {
+ Response string `json:"response"`
+ }{
+ Response: "QUIET",
+ })
+ default:
+ w.WriteHeader(http.StatusInternalServerError)
+ }
+ case <-time.After(app.RequestTimeout):
+ app.log.Error("Request to process DHCP lease file timed out")
+ w.WriteHeader(http.StatusInternalServerError)
+ }
+}
diff --git a/harvester/harvest-compose.yml b/harvester/harvest-compose.yml
deleted file mode 100644
index c21504d..0000000
--- a/harvester/harvest-compose.yml
+++ /dev/null
@@ -1,13 +0,0 @@
-harvester:
- image: cord/dhcpharvester
- container_name: harvester
- restart: never
- labels:
- - "lab.cord.component=Controller"
- volumes:
- - "/var/lib/maas/dhcp:/dhcp"
- - "/etc/bind/maas:/bind"
- - "/home/ubuntu/compose-services/dhcpharvester/key:/key"
- ports:
- - "8954:8954"
- command: [ "--server", "192.168.42.231", "--port", "954", "--key", "/key/mykey.conf", "--zone", "cord.lab", "--update", "--verify", "--timeout", "1s", "--repeat", "5m", "--quiet", "2s", "--workers", "10", "--filter", "^" ]
diff --git a/harvester/harvester.go b/harvester/harvester.go
new file mode 100644
index 0000000..9d47830
--- /dev/null
+++ b/harvester/harvester.go
@@ -0,0 +1,135 @@
+// Copyright 2016 Open Networking Laboratory
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package main
+
+import (
+ "fmt"
+ "github.com/Sirupsen/logrus"
+ "github.com/gorilla/mux"
+ "github.com/kelseyhightower/envconfig"
+ "net/http"
+ "strconv"
+ "sync"
+ "text/template"
+ "time"
+)
+
+// application application configuration and internal state
+type application struct {
+ Port int `default:"4246" desc:"port on which the service will listen for requests"`
+ Listen string `default:"0.0.0.0" desc:"IP on which the service will listen for requests"`
+ LogLevel string `default:"warning" envconfig:"LOG_LEVEL" desc:"log output level"`
+ LogFormat string `default:"text" envconfig:"LOG_FORMAT" desc:"format of log messages"`
+ DHCPLeaseFile string `default:"/harvester/dhcpd.leases" envconfig:"DHCP_LEASE_FILE" desc:"lease file to parse for lease information"`
+ OutputFile string `envconfig:"OUTPUT_FILE" desc:"name of file to output discovered lease in bind9 format"`
+ OutputFormat string `default:"{{.ClientHostname}}\tIN A {{.IPAddress}}\t; {{.HardwareAddress}}" envconfig:"OUTPUT_FORMAT" desc:"specifies the single entry format when outputing to a file"`
+ VerifyLeases bool `default:"true" envconfig:"VERIFY_LEASES" desc:"verifies leases with a ping"`
+ VerifyTimeout time.Duration `default:"1s" envconfig:"VERIFY_TIMEOUT" desc:"max timeout (RTT) to wait for verification pings"`
+ VerifyWithUDP bool `default:"false" envconfig:"VERIFY_WITH_UDP" desc:"use UDP instead of raw sockets for ping verification"`
+ QueryPeriod time.Duration `default:"30s" envconfig:"QUERY_PERIOD" desc:"period at which the DHCP lease file is processed"`
+ QuietPeriod time.Duration `default:"2s" envconfing:"QUIET_PERIOD" desc:"period to wait between accepting parse requests"`
+ RequestTimeout time.Duration `default:"10s" envconfig:"REQUEST_TIMEOUT" desc:"period to wait for processing when requesting a DHCP lease database parsing"`
+ RNDCUpdate bool `default:"false" envconfig:"RNDC_UPDATE" desc:"determines if the harvester reloads the DNS servers after harvest"`
+ RNDCAddress string `default:"127.0.0.1" envconfig:"RNDC_ADDRESS" desc:"IP address of the DNS server to contact via RNDC"`
+ RNDCPort int `default:"954" envconfig:"RNDC_PORT" desc:"port of the DNS server to contact via RNDC"`
+ RNDCKeyFile string `default:"/key/rndc.conf.maas" envconfig:"RNDC_KEY_FILE" desc:"key file, with default, to contact DNS server"`
+ RNDCZone string `default:"cord.lab" envconfig:"RNDC_ZONE" desc:"zone to reload"`
+
+ log *logrus.Logger `ignored:"true"`
+ interchange sync.RWMutex `ignored:"true"`
+ leases map[string]*Lease `ignored:"true"`
+ byHardware map[string]*Lease `ignored:"true"`
+ byHostname map[string]*Lease `ignored:"true"`
+ outputTemplate *template.Template `ignored:"true"`
+ requests chan *chan uint `ignored:"true"`
+}
+
+func main() {
+ // initialize application state
+ app := &application{
+ log: logrus.New(),
+ requests: make(chan *chan uint, 100),
+ }
+
+ // process and validate the application configuration
+ err := envconfig.Process("HARVESTER", app)
+ if err != nil {
+ app.log.Fatalf("unable to parse configuration options : %s", err)
+ }
+ switch app.LogFormat {
+ case "json":
+ app.log.Formatter = &logrus.JSONFormatter{}
+ default:
+ app.log.Formatter = &logrus.TextFormatter{
+ FullTimestamp: true,
+ ForceColors: true,
+ }
+ }
+ level, err := logrus.ParseLevel(app.LogLevel)
+ if err != nil {
+ level = logrus.WarnLevel
+ }
+ app.log.Level = level
+
+ app.outputTemplate, err = template.New("harvester").Parse(app.OutputFormat)
+ if err != nil {
+ app.log.Fatalf("invalid output file format specified : %s", err)
+ }
+
+ // output the configuration
+ app.log.Infof(`Configuration:
+ LISTEN: %s
+ PORT: %d
+ LOG_LEVEL: %s
+ LOG_FORMAT: %s
+ DHCP_LEASE_FILE: %s
+ OUTPUT_FILE: %s
+ OUTPUT_FORMAT: %s
+ VERIFY_LEASES: %t
+ VERIFY_TIMEOUT: %s
+ VERIFY_WITH_UDP: %t
+ QUERY_PERIOD: %s
+ QUIET_PERIOD: %s
+ REQUEST_TIMEOUT: %s
+ RNDC_UPDATE: %t
+ RNDC_ADDRESS: %s
+ RNDC_PORT: %d
+ RNDC_KEY_FILE: %s
+ RNDC_ZONE: %s`,
+ app.Listen, app.Port,
+ app.LogLevel, app.LogFormat,
+ app.DHCPLeaseFile, app.OutputFile, strconv.Quote(app.OutputFormat),
+ app.VerifyLeases, app.VerifyTimeout, app.VerifyWithUDP,
+ app.QueryPeriod, app.QuietPeriod, app.RequestTimeout,
+ app.RNDCUpdate, app.RNDCAddress, app.RNDCPort, app.RNDCKeyFile, app.RNDCZone)
+
+ // establish REST end points
+ router := mux.NewRouter()
+ router.HandleFunc("/lease/", app.listLeasesHandler).Methods("GET")
+ router.HandleFunc("/lease/{ip}", app.getLeaseHandler).Methods("GET")
+ router.HandleFunc("/lease/hardware/{mac}", app.getLeaseByHardware).Methods("GET")
+ router.HandleFunc("/lease/hostname/{name}", app.getLeaseByHostname).Methods("GET")
+ router.HandleFunc("/harvest/", app.doHarvestHandler).Methods("POST")
+ router.HandleFunc("/harvest", app.doHarvestHandler).Methods("POST")
+ http.Handle("/", router)
+
+ // start DHCP lease file synchronization handler
+ go app.syncRequestHandler(app.requests)
+
+ // start loop to periodically synchronize DHCP lease file
+ go app.syncFromDHCPLeaseFileLoop(app.requests)
+
+ // listen for REST requests
+ http.ListenAndServe(fmt.Sprintf("%s:%d", app.Listen, app.Port), nil)
+}
diff --git a/harvester/key/mykey.conf b/harvester/key/mykey.conf
deleted file mode 100644
index 5c1ee5a..0000000
--- a/harvester/key/mykey.conf
+++ /dev/null
@@ -1,8 +0,0 @@
-key "rndc-maas-key" {
- algorithm hmac-md5;
- secret "3wUD5ethlazwlMKLGe2PViPJoPl2Cen5r9BePqwyHac=";
-};
-
-options {
- default-key "rndc-maas-key";
-};
diff --git a/harvester/lease.go b/harvester/lease.go
new file mode 100644
index 0000000..109f5cc
--- /dev/null
+++ b/harvester/lease.go
@@ -0,0 +1,123 @@
+// Copyright 2016 Open Networking Laboratory
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package main
+
+import (
+ "encoding/json"
+ "fmt"
+ "net"
+ "strings"
+ "time"
+)
+
+// BindingState type used to maintain lease state
+type BindingState uint
+
+// constant values of lease binding state
+const (
+ Unknown BindingState = 0
+ Free BindingState = 1
+ Active BindingState = 2
+ Expired BindingState = 3
+ Released BindingState = 4
+ Abandoned BindingState = 5
+ Reset BindingState = 6
+ Backup BindingState = 7
+ Reserved BindingState = 8
+ Bootp BindingState = 9
+)
+
+// String return a string value for a lease binding state
+func (s *BindingState) String() string {
+ switch *s {
+ case 1:
+ return "Free"
+ case 2:
+ return "Active"
+ case 3:
+ return "Expired"
+ case 4:
+ return "Released"
+ case 5:
+ return "Abandoned"
+ case 6:
+ return "Reset"
+ case 7:
+ return "Backup"
+ case 8:
+ return "Reserved"
+ case 9:
+ return "Bootp"
+ default:
+ return "Unknown"
+ }
+}
+
+// Lease DHCP lease information
+type Lease struct {
+ BindingState BindingState `json:"binding-state"`
+ IPAddress net.IP `json:"ip-address"`
+ ClientHostname string `json:"client-hostname"`
+ HardwareAddress net.HardwareAddr `json:"hardware-address"`
+ Starts time.Time `json:"starts"`
+ Ends time.Time `json:"ends"`
+}
+
+// MarshalJSON custom marshaller for DHCP lease
+func (l *Lease) MarshalJSON() ([]byte, error) {
+
+ // a custom marshaller is required because the net.Hardware marshals to a string
+ // that is not in the standard MAC address format by default as well as the
+ // binding state is marshalled to a human readable string
+ type Alias Lease
+ return json.Marshal(&struct {
+ HardwareAddress string `json:"hardware-address"`
+ BindingState string `json:"binding-state"`
+ *Alias
+ }{
+ HardwareAddress: l.HardwareAddress.String(),
+ BindingState: l.BindingState.String(),
+ Alias: (*Alias)(l),
+ })
+}
+
+// parseBindingState conversts from a string to a valid binding state constant
+func parseBindingState(bindingState string) (BindingState, error) {
+ switch strings.ToLower(bindingState) {
+ case "free":
+ return Free, nil
+ case "active":
+ return Active, nil
+ case "expired":
+ return Expired, nil
+ case "released":
+ return Released, nil
+ case "abandoned":
+ return Abandoned, nil
+ case "reset":
+ return Reset, nil
+ case "backup":
+ return Backup, nil
+ case "reserved":
+ return Reserved, nil
+ case "bootp":
+ return Bootp, nil
+ case "unknown":
+ fallthrough
+ default:
+ return Unknown, nil
+ }
+
+ return 0, fmt.Errorf("Unknown lease binding state '%s'", bindingState)
+}
diff --git a/harvester/parse.go b/harvester/parse.go
new file mode 100644
index 0000000..d61e791
--- /dev/null
+++ b/harvester/parse.go
@@ -0,0 +1,237 @@
+// Copyright 2016 Open Networking Laboratory
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package main
+
+import (
+ "bufio"
+ "fmt"
+ "net"
+ "os"
+ "os/exec"
+ "strconv"
+ "strings"
+ "text/tabwriter"
+ "time"
+)
+
+// leaseFilterFunc provides a mechanism to filter which leases are returned by lease file parser
+type leaseFilterFunc func(lease *Lease) bool
+
+const (
+ // returns if a parse requests is processed or denied because of quiet period
+ responseQuiet uint = 0
+ responseOK uint = 1
+
+ // time format for parsing time stamps in lease file
+ dateTimeLayout = "2006/1/2 15:04:05"
+
+ bindFileFormat = "{{.ClientHostname}}\tIN A {{.IPAddress}}\t; {{.HardwareAddress}}"
+)
+
+// parseLease parses a single lease from the lease file
+func parseLease(scanner *bufio.Scanner, lease *Lease) error {
+ var err error
+ for scanner.Scan() {
+ fields := strings.Fields(scanner.Text())
+ if len(fields) > 0 {
+ switch fields[0] {
+ case "}":
+ // If no client-hostname was specified, generate one
+ if len(lease.ClientHostname) == 0 {
+ lease.ClientHostname = strings.ToUpper("UNK-" +
+ strings.Replace(lease.HardwareAddress.String(), ":", "", -1))
+ }
+ return nil
+ case "client-hostname":
+ lease.ClientHostname = strings.Trim(fields[1], "\";")
+ case "hardware":
+ lease.HardwareAddress, err = net.ParseMAC(strings.Trim(fields[2], ";"))
+ if err != nil {
+ return err
+ }
+ case "binding":
+ lease.BindingState, err = parseBindingState(strings.Trim(fields[2], ";"))
+ if err != nil {
+ return err
+ }
+ case "starts":
+ lease.Starts, err = time.Parse(dateTimeLayout,
+ fields[2]+" "+strings.Trim(fields[3], ";"))
+ if err != nil {
+ return err
+ }
+ case "ends":
+ lease.Ends, err = time.Parse(dateTimeLayout,
+ fields[2]+" "+strings.Trim(fields[3], ";"))
+ if err != nil {
+ return err
+ }
+ }
+ }
+ }
+ return nil
+}
+
+// parseLeaseFile parses the entire lease file
+func parseLeaseFile(filename string, filterFunc leaseFilterFunc) (map[string]*Lease, error) {
+ leases := make(map[string]*Lease)
+
+ file, err := os.Open(filename)
+ if err != nil {
+ return nil, err
+ }
+ defer file.Close()
+
+ scanner := bufio.NewScanner(file)
+ scanner.Split(bufio.ScanLines)
+ for scanner.Scan() {
+ fields := strings.Fields(scanner.Text())
+ if len(fields) > 0 && fields[0] == "lease" {
+ lease := Lease{}
+ lease.IPAddress = net.ParseIP(fields[1])
+ parseLease(scanner, &lease)
+ if filterFunc(&lease) {
+ leases[lease.IPAddress.String()] = &lease
+ }
+ }
+ }
+
+ if err = scanner.Err(); err != nil {
+ return nil, err
+ }
+
+ return leases, nil
+}
+
+// syncRequestHandler accepts requests to parse the lease file and either processes or ignores because of quiet period
+func (app *application) syncRequestHandler(requests chan *chan uint) {
+
+ // track the last time file was processed to enforce quiet period
+ var last *time.Time = nil
+
+ // process requests on the channel
+ for response := range requests {
+ now := time.Now()
+
+ // if the request is made during the quiet period then drop the request to prevent
+ // a storm
+ if last != nil && now.Sub(*last) < app.QuietPeriod {
+ app.log.Warn("Request received during query quiet period, will not harvest")
+ if response != nil {
+ *response <- responseQuiet
+ }
+ continue
+ }
+
+ // process the lease database
+ app.log.Infof("Synchronizing DHCP lease database")
+ leases, err := parseLeaseFile(app.DHCPLeaseFile,
+ func(lease *Lease) bool {
+ return lease.BindingState != Free &&
+ lease.Ends.After(now) &&
+ lease.Starts.Before(now)
+ })
+ if err != nil {
+ app.log.Errorf("Unable to parse DHCP lease file at '%s' : %s",
+ app.DHCPLeaseFile, err)
+ } else {
+ // if configured to verify leases with a ping do so
+ if app.VerifyLeases {
+ app.log.Infof("Verifing %d discovered leases", len(leases))
+ _, err := app.verifyLeases(leases)
+ if err != nil {
+ app.log.Errorf("unexpected error while verifing leases : %s", err)
+ app.log.Infof("Discovered %d active, not verified because of error, DHCP leases",
+ len(leases))
+ } else {
+ app.log.Infof("Discovered %d active and verified DHCP leases", len(leases))
+ }
+ } else {
+ app.log.Infof("Discovered %d active, not not verified, DHCP leases", len(leases))
+ }
+
+ // if configured to output the lease information to a file, do so
+ if len(app.OutputFile) > 0 {
+ app.log.Infof("Writing lease information to file '%s'", app.OutputFile)
+ out, err := os.Create(app.OutputFile)
+ if err != nil {
+ app.log.Errorf(
+ "unexpected error while attempting to open file `%s' for output : %s",
+ app.OutputFile, err)
+ } else {
+ table := tabwriter.NewWriter(out, 1, 0, 4, ' ', 0)
+ for _, lease := range leases {
+ if err := app.outputTemplate.Execute(table, lease); err != nil {
+ app.log.Errorf(
+ "unexpected error while writing leases to file '%s' : %s",
+ app.OutputFile, err)
+ break
+ }
+ fmt.Fprintln(table)
+ }
+ table.Flush()
+ }
+ out.Close()
+ }
+
+ // if configured to reload the DNS server, then use the RNDC command to do so
+ if app.RNDCUpdate {
+ cmd := exec.Command("rndc", "-s", app.RNDCAddress, "-p", strconv.Itoa(app.RNDCPort),
+ "-c", app.RNDCKeyFile, "reload", app.RNDCZone)
+ err := cmd.Run()
+ if err != nil {
+ app.log.Errorf("Unexplected error while attempting to reload zone '%s' on DNS server '%s:%d' : %s", app.RNDCZone, app.RNDCAddress, app.RNDCPort, err)
+ } else {
+ app.log.Infof("Successfully reloaded DNS zone '%s' on server '%s:%d' via RNDC command",
+ app.RNDCZone, app.RNDCAddress, app.RNDCPort)
+ }
+ }
+
+ // process the results of the parse to internal data structures
+ app.interchange.Lock()
+ app.leases = leases
+ app.byHostname = make(map[string]*Lease)
+ app.byHardware = make(map[string]*Lease)
+ for _, lease := range leases {
+ app.byHostname[lease.ClientHostname] = lease
+ app.byHardware[lease.HardwareAddress.String()] = lease
+ }
+ leases = nil
+ app.interchange.Unlock()
+ }
+ if last == nil {
+ last = &time.Time{}
+ }
+ *last = time.Now()
+
+ if response != nil {
+ *response <- responseOK
+ }
+ }
+}
+
+// syncFromDHCPLeaseFileLoop periodically request a lease file processing
+func (app *application) syncFromDHCPLeaseFileLoop(requests chan *chan uint) {
+ responseChan := make(chan uint)
+ for {
+ requests <- &responseChan
+ select {
+ case _ = <-responseChan:
+ // request completed
+ case <-time.After(app.RequestTimeout):
+ app.log.Error("request to process DHCP lease file timed out")
+ }
+ time.Sleep(app.QueryPeriod)
+ }
+}
diff --git a/harvester/verify.go b/harvester/verify.go
new file mode 100644
index 0000000..22ae332
--- /dev/null
+++ b/harvester/verify.go
@@ -0,0 +1,63 @@
+// Copyright 2016 Open Networking Laboratory
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package main
+
+import (
+ "github.com/tatsushid/go-fastping"
+ "net"
+ "time"
+)
+
+// NOTE: the go-fastping utility calls its handlers (OnRecv, OnIdle) from a single thread as such
+// the code below does not have to serialize access to the "nonverified" array as access is
+// serialized the the fastping utility calling from a single thread.
+
+// verifyLeases verifies that the lease is valid by using an ICMP ping
+func (app *application) verifyLeases(leases map[string]*Lease) (map[string]*Lease, error) {
+ nonverified := make(map[string]bool)
+
+ // Populate the non-verified list from all the leases and then we will remove those
+ // that are verified
+ for ip, _ := range leases {
+ nonverified[ip] = true
+ }
+
+ pinger := fastping.NewPinger()
+ for _, lease := range leases {
+ pinger.AddIPAddr(&net.IPAddr{IP: lease.IPAddress})
+ }
+
+ if app.VerifyWithUDP {
+ pinger.Network("udp")
+ }
+ pinger.MaxRTT = app.VerifyTimeout
+
+ // when a ping response is received remove that lease from the non-verified list
+ pinger.OnRecv = func(addr *net.IPAddr, rtt time.Duration) {
+ app.log.Infof("Verified lease for IP address '%s' with RTT of '%s'", addr.String(), rtt)
+ delete(nonverified, addr.String())
+ }
+ err := pinger.Run()
+ if err != nil {
+ return nil, err
+ }
+
+ // Remove unverified leases from list
+ for ip, _ := range nonverified {
+ app.log.Infof("Discarding lease for IP address '%s', could not be verified", ip)
+ delete(leases, ip)
+ }
+
+ return leases, nil
+}