buildbot: build documentation, add nightly upload of master docs, other tweaks
* master.cfg: Add a "build-docs" builder to test generation of HTML and PDF
docs into the commit checks. With nightly=true property, upload generated
docs to a static dir on the master. Run from a NightlyScheduler.
Add the properties from the internal worker config to the buildbot
BuildSlave properties, so they're visible in the web UI. Filter through
a workers2publicprops helper, to whitelist allowed props.
diff --git a/infra/buildbot/master/master.cfg b/infra/buildbot/master/master.cfg
index 899d6fa..00e56dd 100644
--- a/infra/buildbot/master/master.cfg
+++ b/infra/buildbot/master/master.cfg
@@ -16,18 +16,52 @@
# password defs
execfile("pass.cfg")
+# filter a given 'workers' entry into a property list
+# suitable for public display
+def workers2publicprops (worker):
+ publicprops = [ "os", "version", "vm", "pkg", "texi", "cc",
+ "latent", ]
+ return { k:worker[k] for k in worker if k in publicprops }
+
+# vm: non-VM are assumed faster and used for initial build
+# pkg: rpm, sysv, dpkg - only do test rpm builds at moment
+# texi: True or "true" if we can use for doc building
+# cc: List of tuples of installed compilers, with:
+# (<tag>, <version>, <command>)
+# tag: gcc, clang, sunpro
+# latent: VM spun up on demand via LatentSlave, uses "session" for
+# the libvirt URI.
+# session: libvirt URI to use for latent workers. Default will be set on
+# latent VMs if not specified.
+# hd_image: libvirt image to use
workers = {
"fedora-24": {
"os": "Fedora",
"version": "24",
"vm": False,
"pkg": "rpm",
+ "texi": True,
+ "cc": [ ("gcc", "6.3.1"),
+ ("clang", "3.8.1"),
+ ("gcc", "3.4.6", "gcc34"),
+ ],
+ },
+ "fedora-26": {
+ "os": "Fedora",
+ "version": "26",
+ "vm": False,
+ "pkg": "rpm",
+ "cc": [ ("gcc", "7.0.1"),
+ ("clang", "3.9.0"),
+ ("gcc", "3.4.6", "gcc34"),
+ ],
},
"centos-7": {
"os": "CentOS",
"version": "7",
"vm": False,
"pkg": "rpm",
+ "cc": [ ("gcc", "4.8.5") ],
},
"debian-8": {
"os": "Debian",
@@ -35,6 +69,7 @@
"vm": True,
"pkg": "dpkg",
"latent": True,
+ "cc": [ ("gcc", "4.9.2") ],
"hd_image": "/var/lib/libvirt/images/debian8.qcow2",
},
"debian-9": {
@@ -42,6 +77,7 @@
"version": "9",
"vm": True,
"pkg": "dpkg",
+ "cc": [ ("gcc", "6.3.0") ],
"latent": True,
"hd_image": "/var/lib/libvirt/images/debian9.qcow2",
},
@@ -51,6 +87,7 @@
"vm": True,
"pkg": "",
"latent": True,
+ "cc": [ ("clang", "3.4.1") ],
"hd_image": "/var/lib/libvirt/images/freebsd103.qcow2",
},
"freebsd-11": {
@@ -58,6 +95,7 @@
"version": "11",
"vm": True,
"pkg": "",
+ "cc": [ ("gcc", "4.9.4"), ("clang", "3.8.0"), ],
"latent": True,
"hd_image": "/var/lib/libvirt/images/freebsd110.qcow2",
},
@@ -67,6 +105,9 @@
"vm": True,
"pkg": "sysv",
"latent": True,
+ "cc": [ ("gcc", "6.3.0"), ("sunpro", "12.0"),
+ ("gcc", "4.4.4")
+ ],
"hd_image": "/var/lib/libvirt/images/buildbot-oi-hipster.qcow2",
},
}
@@ -83,7 +124,7 @@
analyses_builders = [ "clang-analyzer" ]
# default Libvirt session
-for w in (w for w in workers.values () if ("latent" in w)
+for w in (w for w in workers.values () if ("latent" in w and w["latent"])
and ("session" not in w)):
w["session"] = 'qemu+ssh://buildbot@sagan.jakma.org/system'
@@ -93,12 +134,17 @@
rpmbuilders = ["rpm-" + kw for kw in workers if workers[kw]["pkg"] == "rpm"]
+# compilers
+# not using yet
+# [kw for kw in workers if len([v for (c,v) in workers[kw]["cc"] if c == "gcc"]) > 0 ]
+
allbuilders = []
allbuilders += osbuilders
allbuilders += rpmbuilders
allbuilders += analyses_builders
allbuilders += ["commit-builder"]
allbuilders += ["build-distcheck"]
+allbuilders += ["build-docs" ]
# Force merging of requests.
# c['mergeRequests'] = lambda *args, **kwargs: True
@@ -112,7 +158,9 @@
for w in (w for w in workers.values() if ("latent" not in w)
or (w["latent"] == False)):
- c['slaves'].append(buildslave.BuildSlave(w["bot"], w["pass"]))
+ c['slaves'].append(buildslave.BuildSlave(w["bot"], w["pass"],
+ properties=workers2publicprops (w),
+ ))
for w in (w for w in workers.values()
if ("latent" in w)
@@ -123,6 +171,7 @@
w["pass"],
util.Connection(w["session"]),
w["hd_image"],
+ properties=workers2publicprops (w),
))
# 'protocols' contains information about protocols which master will use for
@@ -197,6 +246,11 @@
name="trigger-rpm",
builderNames=rpmbuilders))
+# Doc build check (non-nightly, so no upload)
+c['schedulers'].append(schedulers.Triggerable(
+ name="trigger-build-docs",
+ builderNames=["build-docs"]))
+
# Try and force schedulers
c['schedulers'].append(schedulers.ForceScheduler(
name="force",
@@ -207,10 +261,23 @@
builderNames=osbuilders
+ rpmbuilders
+ ["build-distcheck",
- "clang-analyzer" ],
+ "clang-analyzer",
+ "build-docs" ],
userpass=users,
port=8031))
+## nightly docs build
+c['schedulers'].append(schedulers.Nightly(
+ name="nightly-docs",
+ branch="master",
+ builderNames=[ "build-docs" ],
+ hour=3,
+ minute=0,
+ onlyIfChanged=True,
+ properties = { "nightly": True },
+))
+
+
####### BUILDERS
c['builders'] = []
@@ -218,12 +285,15 @@
# what steps, and which slaves can execute them. Note that any particular build will
# only take place on one slave.
-common_steps = [
-steps.Git(repourl=quaggagit, mode='incremental'),
-steps.ShellCommand(command=["./update-autotools"]),
-steps.Configure(),
-steps.ShellCommand(command=["make", "clean"]),
-steps.Compile(),
+common_setup = [
+ steps.Git(repourl=quaggagit, mode='incremental'),
+ steps.ShellCommand(command=["./update-autotools"],
+ description="generating autoconf",
+ descriptionDone="autoconf"),
+ steps.Configure(command="../build/configure"),
+ steps.ShellCommand(command=["make", "clean"],
+ description="cleaning",
+ descriptionDone="make clean"),
]
### Default 'check' build, builder instantiated for each OS
@@ -238,6 +308,8 @@
factory.addStep(steps.ShellCommand(command=["make", "clean"],
description="cleaning",
descriptionDone="clean"))
+
+#factory.addSteps(common_setup)
factory.addStep(steps.Compile(command=["make", "-j", "2", "all"]))
factory.addStep(steps.ShellCommand(command=["make", "check"],
description="checking",
@@ -348,6 +420,95 @@
)
)
+### Build documentation
+
+def build_is_nightly (step):
+ n = step.getProperty("nightly")
+ if n == True or n == "True" or n == "true":
+ return True
+ return False
+
+f = util.BuildFactory ()
+f.addStep(steps.Git(repourl=quaggagit, mode='full'))
+f.addStep(steps.ShellCommand(command=["./update-autotools"],
+ description="run autotools",
+ descriptionDone="autotools"))
+f.addStep(steps.Configure(command=["../build/configure"],
+ workdir="docs"))
+f.addStep(steps.ShellCommand(command=["make", "V=99", "quagga.html"],
+ description="making split HTML doc",
+ descriptionDone="docs: split HTML",
+ workdir="docs/doc",
+ haltOnFailure=True,
+))
+#f.addStep(steps.FileUpload(
+# slavesrc="build/doc/fig-normal-processing.png",
+# masterdest = "public_html/docs/nightly/quagga/",
+# name = "Upload Fig 1",
+# doStepIf=build_is_nightly,
+#))
+#f.addStep(steps.FileUpload(
+# slavesrc="build/doc/fig-rs-processing.png",
+# masterdest = "public_html/docs/nightly/quagga/",
+# name = "Upload Fig 2",
+# doStepIf=build_is_nightly,
+#))
+f.addStep(steps.MultipleFileUpload(
+ slavesrcs=[ "doc/fig-rs-processing.png",
+ "doc/fig-normal-processing.png" ],
+ masterdest = "public_html/docs/nightly/quagga/",
+ name = "Upload Figures",
+ doStepIf=build_is_nightly,
+))
+f.addStep(steps.DirectoryUpload(
+ slavesrc="quagga.html",
+ masterdest = "public_html/docs/nightly/quagga",
+ compress = 'bz2',
+ name = "Upload split HTML",
+ url = "/docs/nightly/quagga/index.html",
+ workdir="docs/doc",
+ doStepIf=build_is_nightly,
+))
+f.addStep(steps.RemoveDirectory(
+ dir="docs/doc/quagga.html",
+))
+f.addStep(steps.ShellCommand(command=["make", "V=99",
+ "MAKEINFOFLAGS=--no-split",
+ "quagga.html"],
+ description="making one-page HTML doc",
+ descriptionDone="docs: one-page HTML",
+ workdir="docs/doc",
+ haltOnFailure=True
+))
+f.addStep(steps.FileUpload(
+ slavesrc="quagga.html",
+ masterdest = "public_html/docs/nightly/quagga/quagga.html",
+ name = "Upload single HTML",
+ url = "/docs/nightly/quagga/quagga.html",
+ workdir="docs/doc",
+ doStepIf=build_is_nightly,
+))
+f.addStep(steps.ShellCommand(command=["make", "V=99", "quagga.pdf"],
+ description="making PDF docs",
+ descriptionDone="docs: PDF",
+ workdir="docs/doc"
+))
+f.addStep(steps.FileUpload(
+ slavesrc="quagga.pdf",
+ masterdest = "public_html/docs/nightly/quagga/quagga.pdf",
+ name = "Upload PDF",
+ url = "/docs/nightly/quagga/quagga.pdf",
+ workdir="docs/doc",
+ doStepIf=build_is_nightly,
+))
+
+c['builders'].append(
+ util.BuilderConfig(name="build-docs",
+ slavenames=[w["bot"] for w in workers.values()
+ if "texi" in w and w["texi"] == True ],
+ factory=f
+))
+
### Co-ordination builds used to sequence parallel builds via Triggerable
# to understand this you have to read this list and the Triggered schedulers
@@ -367,7 +528,8 @@
updateSourceStamp=True
))
f.addStep(steps.Trigger (
- schedulerNames = [ "trigger-build-analyses", "trigger-distcheck" ],
+ schedulerNames = [ "trigger-build-analyses", "trigger-distcheck",
+ "trigger-build-docs" ],
waitForFinish=True,
updateSourceStamp=True
))
@@ -383,7 +545,6 @@
factory=f)
)
-
####### STATUS TARGETS
# 'status' is a list of Status Targets. The results of each build will be