Static Jenkins Site Generator
- Private Jenkins job scraping w/API key
- Added Gilroy font to match main public website
- Link back to ONF website for products
- Add more products
Change-Id: I3ed2dc1e371c564ee483ab83fd110a88d818bca7
diff --git a/siterender.py b/siterender.py
new file mode 100644
index 0000000..315f8df
--- /dev/null
+++ b/siterender.py
@@ -0,0 +1,257 @@
+#!/usr/bin/env python3
+
+# SPDX-FileCopyrightText: © 2020 Open Networking Foundation <support@opennetworking.org>
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import absolute_import
+
+import argparse
+import datetime
+import jinja2
+import json
+import logging
+import os
+
+# create shared logger
+logging.basicConfig()
+logger = logging.getLogger("sjsgsr")
+
+# script starttime, used for timeboxing
+starttime = datetime.datetime.now(datetime.timezone.utc)
+startdate = datetime.datetime(
+ starttime.year, starttime.month, starttime.day, tzinfo=datetime.timezone.utc
+)
+
+
+def parse_sitebuilder_args():
+ """
+ parse CLI arguments
+ """
+
+ parser = argparse.ArgumentParser(description="Jenkins job results site renderer")
+
+ def readable_dir(path):
+ if os.path.isdir(path) and os.access(path, os.R_OK):
+ return path
+ raise argparse.ArgumentTypeError("%s is not a directory or unreadable" % path)
+
+ # Flags
+ parser.add_argument(
+ "--product_dir",
+ default="products",
+ type=readable_dir,
+ help="Directory containing product JSON created by buildcollector.py",
+ )
+ parser.add_argument(
+ "--template_dir",
+ default="templates",
+ type=readable_dir,
+ help="Directory with Jinja2 templates",
+ )
+ parser.add_argument(
+ "--site_dir",
+ default="site",
+ type=readable_dir,
+ help="Directory to write the static site into",
+ )
+ parser.add_argument(
+ "--debug", action="store_true", help="Print additional debugging information"
+ )
+
+ return parser.parse_args()
+
+
+def jinja_env(template_dir):
+ """
+ Returns a Jinja2 enviroment loading files from template_dir
+ """
+
+ env = jinja2.Environment(
+ loader=jinja2.FileSystemLoader(template_dir),
+ autoescape=jinja2.select_autoescape(["html"]),
+ undefined=jinja2.StrictUndefined,
+ trim_blocks=True,
+ lstrip_blocks=True,
+ )
+
+ # Jinja2 filters
+ def tsdatetime(value, fmt="%Y-%m-%d %H:%M %Z"):
+ # timestamps given ms precision, divide by 1000
+ dateval = datetime.datetime.fromtimestamp(
+ value // 1000, tz=datetime.timezone.utc
+ )
+ return dateval.strftime(fmt)
+
+ def tsdate(value, fmt="%Y-%m-%d"):
+ # timestamps given ms precision, divide by 1000
+ dateval = datetime.datetime.fromtimestamp(
+ value // 1000, tz=datetime.timezone.utc
+ )
+ return dateval.strftime(fmt)
+
+ def timebox(value, boxsize=24, count=7):
+ """
+ Given a list of builds, groups in them into within a sequential range
+ Boxsize is given in hours
+ Count is number of boxes to return
+ Time starts from now()
+ """
+ retbox = []
+
+ nowms = startdate.timestamp() * 1000 # ms precision datetime
+
+ hourms = 60 * 60 * 1000 # ms in an hour
+
+ for box in range(-1, count - 1):
+
+ startms = nowms - (box * hourms * boxsize)
+ endms = nowms - ((box + 1) * hourms * boxsize)
+ timebox_builds = 0
+ success_count = 0
+
+ builds = []
+
+ for bdata in value:
+ # loops multiple times over entire list of builds, could be
+ # optimized
+
+ bt = int(bdata["timestamp"])
+
+ if startms > bt > endms:
+ timebox_builds += 1
+
+ builds.append(bdata)
+
+ if bdata["result"] == "SUCCESS":
+ success_count += 1
+
+ # determine overall status for the timebox
+ if timebox_builds == 0:
+ status = "NONE"
+ elif timebox_builds == success_count:
+ status = "SUCCESS"
+ elif success_count == 0:
+ status = "FAILURE"
+ else: # timebox_builds > success_count
+ status = "UNSTABLE"
+
+ retbox.append(
+ {
+ "result": status,
+ "box_start": int(endms),
+ "outcome": "%d of %d" % (success_count, timebox_builds),
+ "builds": builds,
+ }
+ )
+
+ return retbox
+
+ env.filters["tsdatetime"] = tsdatetime
+ env.filters["tsdate"] = tsdate
+ env.filters["timebox"] = timebox
+
+ return env
+
+
+def clean_name(name):
+ """
+ Clean up a name string. Currently only replaces spaces with underscores
+ """
+ return name.replace(" ", "_")
+
+
+def render_to_file(j2env, context, template_name, path):
+ """
+ Render out a template to file
+ """
+ parent_dir = os.path.dirname(path)
+ os.makedirs(parent_dir, exist_ok=True)
+
+ template = j2env.get_template(template_name)
+
+ with open(path, "w") as outfile:
+ outfile.write(template.render(context))
+
+
+def json_file_load(path):
+ """
+ Get data from local file, return data as a dict
+ """
+
+ with open(path) as jf:
+ try:
+ data = json.loads(jf.read())
+ except json.decoder.JSONDecodeError:
+ logger.exception("Unable to decode JSON from file: '%s'", path)
+
+ return data
+
+
+# main function that calls other functions
+if __name__ == "__main__":
+
+ args = parse_sitebuilder_args()
+
+ # only print log messages if debugging
+ if args.debug:
+ logger.setLevel(logging.DEBUG)
+ else:
+ logger.setLevel(logging.CRITICAL)
+
+ j2env = jinja_env(args.template_dir)
+
+ buildtime = starttime.strftime("%Y-%m-%d %H:%M %Z")
+
+ # init list of projects
+ projects = {}
+
+ # list of products
+ prodfiles = os.listdir(args.product_dir)
+
+ for prodfile in prodfiles:
+
+ # load product, and set buildtime
+ logger.debug("loading file file: '%s'", prodfile)
+ product = json_file_load("%s/%s" % (args.product_dir, prodfile))
+ product.update({"buildtime": buildtime})
+
+ projname = product["onf_project"]
+
+ # build product filename, write out template
+ site_prod_filename = "%s/%s/%s/index.html" % (
+ args.site_dir,
+ projname,
+ clean_name(product["product_name"]),
+ )
+ render_to_file(j2env, product, "product.html", site_prod_filename)
+
+ # product to project list
+ if projname not in projects:
+ projects[projname] = [product]
+ else:
+ projects[projname].append(product)
+
+ # list of projects
+ for projname in sorted(projects.keys()):
+
+ proj_filename = "%s/%s/index.html" % (args.site_dir, projname)
+
+ products = projects[projname]
+
+ project_context = {
+ "buildtime": buildtime,
+ "project_name": projname,
+ "products": products,
+ }
+
+ render_to_file(j2env, project_context, "project.html", proj_filename)
+
+ # render index file
+ index_filename = "%s/index.html" % args.site_dir
+
+ index_context = {
+ "buildtime": buildtime,
+ "projects": projects,
+ }
+
+ render_to_file(j2env, index_context, "index.html", index_filename)