infra/buildbot/master/master.cfg

# -*- python -*-
# ex: set syntax=python:

from buildbot.plugins import *
from buildbot.plugins import buildslave, util

# This is a sample buildmaster config file. It must be installed as
# 'master.cfg' in your buildmaster's base directory.

# This is the dictionary that the buildmaster pays attention to. We also use
# a shorter alias to save typing.
c = BuildmasterConfig = {}

quaggagit = 'git://git.sv.gnu.org/quagga.git'

# password defs
execfile("pass.cfg")

workers = {
	"fedora-24": { 
	  "os": "Fedora",
	  "version": "24",
	  "vm": False,
	  "pkg": "rpm",
	}, 
	"centos-7": {
	  "os": "CentOS",
	  "version": "7",
	  "vm": False,
	  "pkg": "rpm",
	},
	"debian-8": {
	  "os": "Debian",
	  "version": "8",
	  "vm": True,	
  	  "pkg": "dpkg",
  	  "latent": True,
  	  "hd_image": "/var/lib/libvirt/images/debian8.qcow2",
  	},
  	"debian-9": { 
	  "os": "Debian",
	  "version": "9",
	  "vm": True,	
  	  "pkg": "dpkg",
  	  "latent": True,
  	  "hd_image": "/var/lib/libvirt/images/debian9.qcow2",
  	},
  	"freebsd-10": { 
	  "os": "FreeBSD",
	  "version": "10",
	  "vm": True,	
  	  "pkg": "",
  	  "latent": True,
  	  "hd_image": "/var/lib/libvirt/images/freebsd103.qcow2",
  	}, 
  	"freebsd-11": { 
	  "os": "FreeBSD",
	  "version": "11",
	  "vm": True,	
  	  "pkg": "",
  	  "latent": True,
  	  "hd_image": "/var/lib/libvirt/images/freebsd110.qcow2",
  	},
}

# ensure "latent" is set to false, where not set.
# add in the passwords
for kw in workers:
	w = workers[kw]
	w["bot"] = "buildbot-" + kw
	if "latent" not in w:
		w["latent"] = False
	w["pass"] = workers_pass[kw]

analyses_builders = [ "clang-analyzer" ]

# default Libvirt session
for w in (w for w in workers.values () if ("latent" in w) 
					and ("session" not in w)):
	w["session"] = 'qemu+ssh://buildbot@sagan.jakma.org/system'

osbuilders = ["build-" + kw for kw in workers]

allbuilders =  []
allbuilders += osbuilders
allbuilders += ["rpm-" + kw for kw in workers if workers[kw]["pkg"] == "rpm"]
allbuilders += analyses_builders
allbuilders += ["commit-builder"]
allbuilders += ["build-distcheck"]

# Force merging of requests.
c['mergeRequests'] = lambda *args, **kwargs: True

####### BUILDSLAVES
c['slaves'] = []

# The 'slaves' list defines the set of recognized buildslaves. Each element is
# a BuildSlave object, specifying a unique slave name and password.  The same
# slave name and password must be configured on the slave.

for w in (w for w in workers.values() if ("latent" not in w)
				      or (w["latent"] == False)):
	c['slaves'].append(buildslave.BuildSlave(w["bot"], w["pass"]))

for w in (w for w in workers.values()
		  if ("latent" in w) 
		     and w["latent"]
		     and "hd_image" in w):
	c['slaves'].append(buildslave.LibVirtSlave(
					w["bot"],
					w["pass"],
					util.Connection(w["session"]),
					w["hd_image"],
	))

# 'protocols' contains information about protocols which master will use for
# communicating with slaves.
# You must define at least 'port' option that slaves could connect to your master
# with this protocol.
# 'port' must match the value configured into the buildslaves (with their
# --master option)
c['protocols'] = {'pb': {'port': 9989}}

####### CHANGESOURCES

# the 'change_source' setting tells the buildmaster how it should find out
# about source code changes.  Here we point to the buildbot clone of pyflakes.

c['change_source'] = []
c['change_source'].append(changes.GitPoller(
	quaggagit,
        workdir='gitpoller-workdir', 
	branches=['master','volatile/next'],
        pollinterval=300))

####### SCHEDULERS

# Configure the Schedulers, which decide how to react to incoming changes. 

# We want a first line of 'quick' builds, which then trigger further builds.
#
# A control-flow builder, "commit-builder", used to sequence the 'real'
# sets of builders, via Triggers.

c['schedulers'] = []
c['schedulers'].append(schedulers.SingleBranchScheduler(
                            name="master-change",
                            change_filter=util.ChangeFilter(branch='master'),
                            treeStableTimer=10,
                            builderNames=[ "commit-builder" ]))

c['schedulers'].append(schedulers.SingleBranchScheduler(
                            name="next-change",
                            change_filter=util.ChangeFilter(
                            	branch='volatile/next'),
                            treeStableTimer=10,
                            builderNames=[ "commit-builder" ] ))

# Initial build checks on faster, non-VM
c['schedulers'].append(schedulers.Triggerable(
	name="trigger-build-first",
	builderNames=list("build-" + kw
				for kw in workers
					if workers[kw]["vm"] == False)))

# Build using remaining builders, after firstbuilders.
c['schedulers'].append(schedulers.Triggerable(
	name="trigger-build-rest",
	builderNames=list("build-" + kw
				for w in workers
					if workers[kw]["vm"] == True)))

# Analyses tools, e.g. CLang Analyzer scan-build
c['schedulers'].append(schedulers.Triggerable(
		       name="trigger-build-analyses",	
		       builderNames=analyses_builders))
# Dist check
c['schedulers'].append(schedulers.Triggerable(
		       name="trigger-distcheck",	
		       builderNames=["build-distcheck"]))
# RPM check and build
c['schedulers'].append(schedulers.Triggerable(
		       name="trigger-rpm",	
		       builderNames=["rpm-" + kw
				      for kw in workers 
				      if workers[kw]["pkg"] == "rpm"]))

# Try and force schedulers
c['schedulers'].append(schedulers.ForceScheduler(
                       name="force",
                       builderNames=allbuilders))

c['schedulers'].append(schedulers.Try_Userpass(
		       name="try",
		       builderNames=["build-" + kw for kw in workers]
		       		    + ["rpm-" + kw for kw in workers
		       		    	if workers[kw]["pkg"] == "rpm"]
				     + ["build-distcheck", 
					 "clang-analyzer" ],
			userpass=users,
			port=8031))

####### BUILDERS
c['builders'] = []

# The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
# what steps, and which slaves can execute them.  Note that any particular build will
# only take place on one slave.

common_steps = [
steps.Git(repourl=quaggagit, mode='incremental'),
steps.ShellCommand(command=["./update-autotools"]),
steps.Configure(),
steps.ShellCommand(command=["make", "clean"]),
steps.Compile(),
]

### Default 'check' build, builder instantiated for each OS

factory = util.BuildFactory()
# check out the source
factory.addStep(steps.Git(repourl=quaggagit, mode='incremental'))
factory.addStep(steps.ShellCommand(command=["./update-autotools"],
				   description="generating autoconf",
				   descriptionDone="autoconf"))
factory.addStep(steps.Configure())
factory.addStep(steps.ShellCommand(command=["make", "clean"],
				   description="cleaning",
				   descriptionDone="clean"))
factory.addStep(steps.Compile(command=["make", "-j", "2", "all"]))
factory.addStep(steps.ShellCommand(command=["make", "check"],
				   description="checking",
				   descriptionDone="make check"))

# create builder for every OS, for every buildbot
# XXX: at moment this assumes 1:1 OS<->bot
for kw in workers:
   c['builders'].append(util.BuilderConfig(
	name="build-" + kw,
	slavenames=workers[kw]["bot"],
	factory=factory))

### distcheck Builder, executed on any available bot
factory = util.BuildFactory()
# check out the source
factory.addStep(steps.Git(repourl=quaggagit, mode='incremental'))
factory.addStep(steps.ShellCommand(command=["./update-autotools"],
				   description="generating autoconf",
				   descriptionDone="autoconf"))
factory.addStep(steps.Configure())
factory.addStep(steps.ShellCommand(command=["make", "clean"],
				   description="cleaning",
				   descriptionDone="make clean"))
factory.addStep(steps.ShellCommand(command=["make", "distcheck"],
				   description="run make distcheck",
				   descriptionDone="make distcheck"))
c['builders'].append(
	util.BuilderConfig(name="build-distcheck",
	slavenames=list(w["bot"] for w in workers.values()),
	factory=factory,
))

### LLVM clang-analyzer build, executed on any available non-VM bot

f = util.BuildFactory()
# check out the source
f.addStep(steps.Git(repourl=quaggagit, mode='incremental',
		    getDescription=True))
f.addStep(steps.ShellCommand(command=["./update-autotools"],
			     description="run autotools",
			     descriptionDone="autoconf"))
f.addStep(steps.Configure())
f.addStep(steps.ShellCommand(command=["make", "clean"],
			     description="cleaning",
			     descriptionDone="make clean"))

f.addStep(steps.SetProperty(property="clang-id",
	value=util.Interpolate("%(prop:commit-description)s-%(prop:buildnumber)s")))

f.addStep(steps.SetProperty(property="clang-output-dir",
	value=util.Interpolate("../CLANG-%(prop:clang-id)s")))
f.addStep(steps.SetProperty(property="clang-uri",
	value=util.Interpolate("/clang-analyzer/%(prop:clang-id)s")))
# relative to buildbot master working directory
f.addStep(steps.SetProperty(property="clang-upload-dir",
	value=util.Interpolate("public_html/clang-analyzer/%(prop:clang-id)s")))

f.addStep(steps.Compile(command=["scan-build",
			      	 "-analyze-headers",
			      	 "-o",
			      	 util.Interpolate("%(prop:clang-output-dir)s"),
				 "make", "-j", "all"]))
f.addStep(steps.DirectoryUpload(
	  slavesrc=util.Interpolate("%(prop:clang-output-dir)s"),
	  masterdest = util.Interpolate("%(prop:clang-upload-dir)s"),
	  compress = 'bz2',
	  name = "clang report",
	  url = util.Interpolate("%(prop:clang-uri)s"),
))
f.addStep(steps.RemoveDirectory(
	dir=util.Interpolate("%(prop:clang-output-dir)s")
))


c['builders'].append(
    util.BuilderConfig(name="clang-analyzer",
      slavenames=list(w["bot"] for w in workers.values() if not w["vm"]),
      factory=f))


### RPM: check and build
f = util.BuildFactory ()

# check out the source
f.addStep(steps.Git(repourl=quaggagit, mode='full'))
f.addStep(steps.ShellCommand(command=["./update-autotools"],
				   description="run autotools",
				   descriptionDone="autotools"))
f.addStep(steps.Configure())
f.addStep(steps.ShellCommand(command=["make", "dist"],
				     description="run make dist",
				     descriptionDone="make dist"))
# not imported somehow
#f.addStep(steps.RpmLint(fileloc="redhat/quagga.spec"))
f.addStep(steps.ShellCommand(command=["rpmlint", "-i", "redhat/quagga.spec"],
				description="run rpmlint",
				descriptionDone="rpmlint"))
f.addStep(steps.RpmBuild(specfile="redhat/quagga.spec"))
#			 rpmdir=util.Interpolate("%(prop:builddir)s/rpm")))

# XXX: assuming 1:1 OS:buildbot mapping
for kw in (kw for kw in workers if workers[kw]["pkg"] == "rpm"):
	c['builders'].append(
    		util.BuilderConfig(name="rpm-" + kw,
      				   slavenames="buildbot-" + kw,
      				   factory=f
      		)
      	)

### Co-ordination builds used to sequence parallel builds via Triggerable

# to understand this you have to read this list and the Triggered schedulers
# to see what sets of builds are being sequenced. Bit clunky, but Buildbot
# doesn't have a way to just specify a pipeline of groups of builders more
# cleanly.

f = util.BuildFactory()
f.addStep(steps.Trigger (
	schedulerNames = [ "trigger-build-first" ],
	waitForFinish=True,
	updateSourceStamp=True
))
f.addStep(steps.Trigger (
	schedulerNames = [ "trigger-build-rest" ],
	waitForFinish=True,
	updateSourceStamp=True
))
f.addStep(steps.Trigger (
	schedulerNames = [ "trigger-build-analyses", "trigger-distcheck" ],
	waitForFinish=True,
	updateSourceStamp=True
))
f.addStep(steps.Trigger (
	schedulerNames = [ "trigger-rpm" ],
	waitForFinish=True,
	updateSourceStamp=True
))

c['builders'].append(
    util.BuilderConfig(name="commit-builder",
      slavenames=[w["bot"] for w in workers.values() if not w["vm"]],
      factory=f)
)


####### STATUS TARGETS

# 'status' is a list of Status Targets. The results of each build will be
# pushed to these targets. buildbot/status/*.py has a variety to choose from,
# including web pages, email senders, and IRC bots.

c['status'] = []

from buildbot.status import html
from buildbot.status.web import authz, auth

authz_cfg=authz.Authz(
    # change any of these to True to enable; see the manual for more
    # options
    #auth=auth.BasicAuth([("pyflakes","pyflakes")]),
    auth=util.BasicAuth(users),
    gracefulShutdown = False,
    forceBuild = 'auth', # use this to test your slave once it is set up
    forceAllBuilds = 'auth',  # ..or this
    pingBuilder = 'auth',
    stopBuild = 'auth',
    stopAllBuilds = 'auth',
    cancelPendingBuild = 'auth',
    cancelAllPendingBuilds = 'auth',
    pauseSlave = 'auth',    
)
c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg))

c['status'].append(status.MailNotifier(
	fromaddr="buildbot@quagga.net",
	extraRecipients=["paul@jakma.org"],
	sendToInterestedUsers=False,
))

c['status'].append (status.IRC(
	"irc.freenode.net", "bb-quagga",
	useColors=True,
	channels=[{"channel": "#quagga"}],
	notify_events={
		'exception': 1,
		'successToFailure': 1,
		'failureToSuccess': 1,
	},
))

####### PROJECT IDENTITY

# the 'title' string will appear at the top of this buildbot
# installation's html.WebStatus home page (linked to the
# 'titleURL') and is embedded in the title of the waterfall HTML page.

c['title'] = "Quagga"
c['titleURL'] = "https://www.quagga.net/"

# the 'buildbotURL' string should point to the location where the buildbot's
# internal web server (usually the html.WebStatus page) is visible. This
# typically uses the port number set in the Waterfall 'status' entry, but
# with an externally-visible host name which the buildbot cannot figure out
# without some help.

c['buildbotURL'] = "http://buildbot.quagga.net/"

####### DB URL

c['db'] = {
    # This specifies what database buildbot uses to store its state.  You can leave
    # this at its default for all but the largest installations.
    'db_url' : "sqlite:///state.sqlite",
}

#### debug
c['debugPassword'] = debugPassword