siterender.py

#!/usr/bin/env python3

# SPDX-FileCopyrightText: © 2020 Open Networking Foundation <support@opennetworking.org>
# SPDX-License-Identifier: Apache-2.0

from __future__ import absolute_import

import argparse
import datetime
import jinja2
import json
import logging
import os

# create shared logger
logging.basicConfig()
logger = logging.getLogger("sjsgsr")

# script starttime, used for timeboxing
starttime = datetime.datetime.now(datetime.timezone.utc)
startdate = datetime.datetime(
    starttime.year, starttime.month, starttime.day, tzinfo=datetime.timezone.utc
)


def parse_sitebuilder_args():
    """
    parse CLI arguments
    """

    parser = argparse.ArgumentParser(description="Jenkins job results site renderer")

    def readable_dir(path):
        if os.path.isdir(path) and os.access(path, os.R_OK):
            return path
        raise argparse.ArgumentTypeError("%s is not a directory or unreadable" % path)

    # Flags
    parser.add_argument(
        "--product_dir",
        default="products",
        type=readable_dir,
        help="Directory containing product JSON created by buildcollector.py",
    )
    parser.add_argument(
        "--template_dir",
        default="templates",
        type=readable_dir,
        help="Directory with Jinja2 templates",
    )
    parser.add_argument(
        "--site_dir",
        default="site",
        type=readable_dir,
        help="Directory to write the static site into",
    )
    parser.add_argument(
        "--debug", action="store_true", help="Print additional debugging information"
    )

    return parser.parse_args()


def jinja_env(template_dir):
    """
    Returns a Jinja2 enviroment loading files from template_dir
    """

    env = jinja2.Environment(
        loader=jinja2.FileSystemLoader(template_dir),
        autoescape=jinja2.select_autoescape(["html"]),
        undefined=jinja2.StrictUndefined,
        trim_blocks=True,
        lstrip_blocks=True,
    )

    # Jinja2 filters
    def tsdatetime(value, fmt="%Y-%m-%d %H:%M %Z"):
        # timestamps given ms precision, divide by 1000
        dateval = datetime.datetime.fromtimestamp(
            value // 1000, tz=datetime.timezone.utc
        )
        return dateval.strftime(fmt)

    def tsdate(value, fmt="%Y-%m-%d"):
        # timestamps given ms precision, divide by 1000
        dateval = datetime.datetime.fromtimestamp(
            value // 1000, tz=datetime.timezone.utc
        )
        return dateval.strftime(fmt)

    def timebox(value, boxsize=24, count=7):
        """
        Given a list of builds, groups in them into within a sequential range
        Boxsize is given in hours
        Count is number of boxes to return
        Time starts from now()
        """
        retbox = []

        nowms = startdate.timestamp() * 1000  # ms precision datetime

        hourms = 60 * 60 * 1000  # ms in an hour

        for box in range(-1, count - 1):

            startms = nowms - (box * hourms * boxsize)
            endms = nowms - ((box + 1) * hourms * boxsize)
            timebox_builds = 0
            success_count = 0

            builds = []

            for bdata in value:
                # loops multiple times over entire list of builds, could be
                # optimized

                bt = int(bdata["timestamp"])

                if startms > bt > endms:
                    timebox_builds += 1

                    builds.append(bdata)

                    if bdata["result"] == "SUCCESS":
                        success_count += 1

            # determine overall status for the timebox
            if timebox_builds == 0:
                status = "NONE"
            elif timebox_builds == success_count:
                status = "SUCCESS"
            elif success_count == 0:
                status = "FAILURE"
            else:  # timebox_builds > success_count
                status = "UNSTABLE"

            retbox.append(
                {
                    "result": status,
                    "box_start": int(endms),
                    "outcome": "%d of %d" % (success_count, timebox_builds),
                    "builds": builds,
                }
            )

        return retbox

    env.filters["tsdatetime"] = tsdatetime
    env.filters["tsdate"] = tsdate
    env.filters["timebox"] = timebox

    return env


def clean_name(name):
    """
    Clean up a name string. Currently only replaces spaces with underscores
    """
    return name.replace(" ", "_")


def render_to_file(j2env, context, template_name, path):
    """
    Render out a template to file
    """
    parent_dir = os.path.dirname(path)
    os.makedirs(parent_dir, exist_ok=True)

    template = j2env.get_template(template_name)

    with open(path, "w") as outfile:
        outfile.write(template.render(context))


def json_file_load(path):
    """
    Get data from local file, return data as a dict
    """

    with open(path) as jf:
        try:
            data = json.loads(jf.read())
        except json.decoder.JSONDecodeError:
            logger.exception("Unable to decode JSON from file: '%s'", path)

    return data


# main function that calls other functions
if __name__ == "__main__":

    args = parse_sitebuilder_args()

    # only print log messages if debugging
    if args.debug:
        logger.setLevel(logging.DEBUG)
    else:
        logger.setLevel(logging.CRITICAL)

    j2env = jinja_env(args.template_dir)

    buildtime = starttime.strftime("%Y-%m-%d %H:%M %Z")

    # init list of projects
    projects = {}

    # list of products
    prodfiles = os.listdir(args.product_dir)

    for prodfile in prodfiles:

        # load product, and set buildtime
        logger.debug("loading file file: '%s'", prodfile)
        product = json_file_load("%s/%s" % (args.product_dir, prodfile))
        product.update({"buildtime": buildtime})

        projname = product["onf_project"]

        # build product filename, write out template
        site_prod_filename = "%s/%s/%s/index.html" % (
            args.site_dir,
            projname,
            clean_name(product["product_name"]),
        )
        render_to_file(j2env, product, "product.html", site_prod_filename)

        # product to project list
        if projname not in projects:
            projects[projname] = [product]
        else:
            projects[projname].append(product)

    # list of projects
    for projname in sorted(projects.keys()):

        proj_filename = "%s/%s/index.html" % (args.site_dir, projname)

        products = projects[projname]

        project_context = {
            "buildtime": buildtime,
            "project_name": projname,
            "products": products,
        }

        render_to_file(j2env, project_context, "project.html", proj_filename)

    # render index file
    index_filename = "%s/index.html" % args.site_dir

    index_context = {
        "buildtime": buildtime,
        "projects": projects,
    }

    render_to_file(j2env, index_context, "index.html", index_filename)