# changelog entries are parsed recursively.
# Supported distributions are: squeeze wheezy jessie trusty utopic
distros = auto
+
+# Email Maintainer (specified in debian/control) when a build fails
+email_maintainer = True
+
+# Email Uploaders (specified in debian/control) when a build fails
+email_uploaders = True
================================================================================
"""
import os
import re
import sys
-import urllib2
+import xml.etree
try:
if sys.version_info[:2] >= (3, 2):
from ConfigParser import SafeConfigParser as ConfigParser
import debian.changelog
+import gitlab
import jenkins
_LOG_LEVEL = logging.INFO
_LOG_FORMAT = '%(asctime)s %(name)s [%(process)d] %(levelname)s: %(message)s'
JENKINS_URL = "https://jenkins/"
-# TODO: Use special user
-JENKINS_USER = "bdrung"
+JENKINS_USER = "jenkins"
+JENKIKS_TOKEN_FILE = os.path.expanduser("~/.config/jenkins_token")
+GITLAB_URL = "https://gitlab.pb.local"
+GITLAB_TOKEN_FILE = os.path.expanduser("~/.config/gitlab_token")
TEMPLATE_JOB = "debian-package-template2"
-JENKINS_TRIGGER_TOKEN = "BuildIt"
SUPPORTED_DISTROS = [
"squeeze",
"wheezy",
FALLBACK_DISTRO = "squeeze"
-def get_config_parser():
- """Create a ConfigParser object and set the default values"""
- config = ConfigParser()
- config.add_section('debian')
- config.set('debian', 'enabled', 'auto')
- config.set('debian', 'distros', 'auto')
- return config
+class BuildchainConfig(ConfigParser):
+ """Enhanced ConfigParser object
+
+ This object is initialised with default values and the configuration is
+ read from the optional buildchain_file parameter.
+ """
+
+ def __init__(self, buildchain_file=None): # pylint: disable=W0231
+ ConfigParser.__init__(self)
+ self._set_defaults()
+ if buildchain_file is not None:
+ self.readfp(buildchain_file)
+
+ def _set_defaults(self):
+ """Create a ConfigParser object and set the default values"""
+ self.add_section('debian')
+ self.set('debian', 'enabled', 'auto')
+ self.set('debian', 'distros', 'auto')
+ self.set('debian', 'email_maintainer', 'True')
+ self.set('debian', 'email_uploaders', 'True')
+
+
+class JenkinsJob(object):
+ def __init__(self, jenkins_client, job_name, config_xml, logger):
+ self.jenkins_client = jenkins_client
+ self.job_name = job_name
+ self.config_xml = xml.etree.ElementTree.fromstring(config_xml)
+ self.logger = logger
+
+ @classmethod
+ def from_template(cls, jenkins_client, job_name, substitutions, logger):
+ """Creates a Jenkins job with the given name from the template."""
+ logger.info("Retreiving Job template from '{job}' job...".format(job=TEMPLATE_JOB))
+ config_xml = jenkins_client.get_job_config(TEMPLATE_JOB)
+
+ if not config_xml:
+ logger.error("Job template '{job}' not found.".format(job=TEMPLATE_JOB))
+ sys.exit(1)
+
+ jenkins_job = cls(jenkins_client, job_name, config_xml, logger)
+ jenkins_job.fill_template(substitutions)
+
+ return jenkins_job
+
+ def fill_template(self, substitutions):
+ """Fills the template job configuration with the given substitutions
+
+ substitutions is a dict.
+ """
+ self.logger.info("Setting following variables:\n{dict}".format(dict=substitutions))
+ config_xml_text = xml.etree.ElementTree.tostring(self.config_xml)
+ for key, value in substitutions.iteritems():
+ config_xml_text = re.sub('@' + key + '@', value, config_xml_text)
+
+ self.config_xml = xml.etree.ElementTree.fromstring(config_xml_text)
+ # Update description
+ self.config_xml.find('description').text = substitutions['description']
+ # Enable job
+ self.config_xml.find('disabled').text = 'false'
+
+ def update_job(self):
+ """Sends the (possibly modified) job configuration XML to the Jenkins server
+
+ This action creates/updates the job on the Jenkins server.
+ """
+ config_xml_text = xml.etree.ElementTree.tostring(self.config_xml)
+ if self.jenkins_client.job_exists(self.job_name):
+ self.logger.info("Updating Jenkins job '{job}'...".format(job=self.job_name))
+ self.jenkins_client.reconfig_job(self.job_name, config_xml_text)
+ else:
+ self.logger.info("Creating Jenkins job '{job}'...".format(job=self.job_name))
+ self.jenkins_client.create_job(self.job_name, config_xml_text)
+
+ def trigger_job(self, params):
+ trigger_token = self.config_xml.find('authToken').text
+ self.logger.info("Triggering Jenkins job '{job}' with parameters {params}..."
+ .format(job=self.job_name, params=params))
+ self.jenkins_client.build_job(self.job_name, params, trigger_token)
+ job_url = self.jenkins_client.get_job_info(self.job_name)['url']
+ self.logger.info("Triggered job {url}".format(url=job_url))
def get_git_tree(data, commit):
return data['repository']['homepage'] + '/raw/' + commit['id'] + '/'
-def read_buildchain_config(config, data, commit, logger):
+def read_buildchain_config(data, gitlab_client, logger):
"""Read .buildchain configuration file from git repository
Try to read the .buildchain configuration file from the git repository and
update the config object"""
- git_tree = get_git_tree(data, commit)
- try:
- response = urllib2.urlopen(os.path.join(git_tree, '.buildchain'))
- buildinfo = response.read()
- logger.info(".buildinfo content:\n" + buildinfo)
- config.readfp(io.BytesIO(buildinfo))
- except urllib2.HTTPError as error:
- if error.code == 404:
- logger.info("No .buildinfo file found in {repo}".format(repo=commit['url']))
- else:
- logger.error("Retreiving .buildinfo failed with HTTP code {code}."
- .format(code=error.code))
- raise
+ buildchain = gitlab_client.getrawblob(data['project_id'], data['after'], '.buildchain')
+ if buildchain is False:
+ url = os.path.join(data['repository']['homepage'], 'tree', data['after'])
+ logger.info("No .buildchain file found in {url}".format(url=url))
+ config = BuildchainConfig()
+ else:
+ logger.info(".buildchain content:\n{content}".format(content=buildchain))
+ config = BuildchainConfig(io.BytesIO(buildchain))
+ return config
-def get_debian_changelog(config, git_tree, logger):
+def get_debian_changelog(config, data, gitlab_client, logger):
"""Make sure the project is enabled. Then retreive the debian/changelog file"""
if config.get('debian', 'enabled') == 'auto':
enabled = 'auto'
"Doing nothing.")
sys.exit(0)
- try:
- response = urllib2.urlopen(os.path.join(git_tree, 'debian', 'changelog'))
- changelog = response.read()
- except urllib2.HTTPError as error:
- if error.code == 404:
- if enabled == 'auto':
- logger.info("The enabled variable of the Debian package build was set to 'auto' "
- "and no debian/changelog file found. Doing nothing.")
- sys.exit(0)
- else:
- logger.error("The Debian package build is enabled, but no debian/changelog file "
- "found in the git repository.")
- sys.exit(1)
+ changelog = gitlab_client.getrawblob(data['project_id'], data['after'], 'debian/changelog')
+ if changelog is False:
+ if enabled == 'auto':
+ logger.info("The enabled variable of the Debian package build was set to 'auto' "
+ "and no debian/changelog file found. Doing nothing.")
+ sys.exit(0)
else:
- logger.error("Retreiving debian/changelog failed with HTTP code {code}."
- .format(code=error.code))
- raise
+ logger.error("The Debian package build is enabled, but no debian/changelog file "
+ "found in the git repository.")
+ sys.exit(1)
changelog = debian.changelog.Changelog(changelog)
return changelog
-def get_distros(config, changelog, logger):
- """Return list of distributions to build
+def get_distros(config, changelog, branch, logger):
+ """Return list of distributions to build the package for
+
+ There are multiple ways to retreive the distributions to build:
- The distros variable can be set to a list of space-separated distributions or to
- 'auto' in the .buildchain configuration file. In 'auto' mode, debian/changelog
+ 1) Check the branch name to start with a specific distribution. In this case
+ the package is just build for this distribution.
+
+ 2) The .buildchain configuration variable 'distros' can be set to a list of
+ space-separated distributions or to 'auto'. In 'auto' mode, debian/changelog
will be parsed and the distros from there will be used. If distributions is set
to 'UNRELEASED', the previous changelog entries are parsed recursively.
The list of distributions is checked for invalid or unsupported distributions
and duplicate entries are removed from the list.
"""
+ for distro in SUPPORTED_DISTROS:
+ if re.match(distro + '(-dev|-(bugfix|feature|hotfix|poc)-.*)?$', branch):
+ distros = [distro]
+ logger.info("Distributions to built: {distro} (due to branch name {name})"
+ .format(distro=distro, name=branch))
+ return distros
+
+ if not re.match('(master|develop|(bugfix|feature|hotfix|poc)/.*|)$', branch):
+ branches = ("master develop hotfix/* bugfix/* feature/* poc/* ${distro} ${distro}-dev "
+ "${distro}-hotfix-* ${distro}-bugfix-* ${distro}-feature-* ${distro}-poc-*")
+ logger.info("Branch name '{name}' does not match any of these branches: {branches}"
+ .format(name=branch, branches=branches))
+ return []
+
distros = config.get('debian', 'distros')
if distros == 'auto':
logger.info("Distros was set to 'auto'. "
return distros
-def update_jenkins_job(config, jenkins_client, job_name, data, branch, logger):
- logger.info("Retreiving Job template from '{job}' job...".format(job=TEMPLATE_JOB))
- config_xml = jenkins_client.get_job_config(TEMPLATE_JOB)
- if not config_xml:
- logger.error("Job template '{job}' not found.".format(job=TEMPLATE_JOB))
- sys.exit(1)
+def get_job_name(source_name, distribution, branch):
+ """Return the corresponding Jenkins job name
+
+ The Jenkins job name is "<source name>_<upload distro>" where <source name> is the
+ Debian source package name and <upload distro> is the upload target distribution.
+
+ The mapping from branch to upload distro is:
- replacements = {}
- replacements['gitrepo'] = data['repository']['url']
- replacements['branch'] = branch
- replacements['homepage'] = data['repository']['homepage']
- # TODO: fill useful list of recipients
- replacements['recipients'] = 'benjamin.drung@profitbricks.com'
- for key, value in replacements.iteritems():
- print(key, value)
- config_xml = re.sub('@' + key + '@', value, config_xml)
- # Enable job
- config_xml = re.sub('<disabled>true</disabled>', '<disabled>false</disabled>', config_xml)
- # Update description
-
- if jenkins_client.job_exists(job_name):
- logger.info("Updating Jenkins job {job}...".format(job=job_name))
- jenkins_client.reconfig_job(job_name, config_xml)
+ master -> ${distribution}
+ develop -> ${distribution}-dev
+ hotfix/* -> ${distribution}-hotfix-*
+ bugfix/* -> ${distribution}-bugfix-*
+ feature/* -> ${distribution}-feature-*
+ ${distribution}* -> ${distribution}*
+ """
+ for distro in SUPPORTED_DISTROS:
+ if re.match(distro + '(-dev|-(bugfix|feature|hotfix|poc)-.*)?$', branch):
+ distro = branch
+ break
else:
- logger.info("Creating Jenkins job {job}...".format(job=job_name))
- jenkins_client.create_job(job_name, config_xml)
+ if re.match('(master|hotfix/.*)$', branch):
+ distro = distribution
+ elif branch == 'develop':
+ distro = distribution + '-dev'
+ else:
+ match = re.match('(bugfix|feature|poc)/(.*)', branch)
+ if match:
+ distro = distribution + '-' + match.group(1) + '-' + match.group(2)
+ else:
+ raise Exception("Unsupported branch name '{name}' to build a Debian package from."
+ .format(name=branch))
+
+ job_name = source_name + '_' + distro
+ return job_name
def main():
logging.basicConfig(format=_LOG_FORMAT, level=_LOG_LEVEL)
logger = logging.getLogger(os.path.basename(__file__))
- config = get_config_parser()
- # TODO: retreive api token in secure way
- apitoken = open(os.path.dirname(sys.argv[0]) + '/.apitoken').read().strip()
- jenkins_client = jenkins.Jenkins(JENKINS_URL, JENKINS_USER, apitoken)
+
+ # Read payload from environment
if 'payload' not in os.environ:
logger.error("No 'payload' environment variable set. Please specify the Gitlab webhook "
"JSON data in the 'payload' environment variable.")
sys.exit(1)
logger.info('Received push event:\n{event}'.format(event=json.dumps(data, indent=4)))
- # Just process the latest commit
- if len(data['commits']) > 0:
- commit = data['commits'][0]
- logger.info("Processing commit {commit} from {repo}..."
- .format(commit=commit['id'], repo=data['repository']['homepage']))
-
- git_tree = get_git_tree(data, commit)
- read_buildchain_config(config, data, commit, logger)
- changelog = get_debian_changelog(config, git_tree, logger)
- source_name = changelog.package
- branch = re.sub('^refs/heads/', '', data['ref'])
- for distro in get_distros(config, changelog, logger):
- job_name = distro + '_' + source_name
- if branch != 'master':
- job_name += '_' + branch
- update_jenkins_job(config, jenkins_client, job_name, data, branch, logger)
- # TODO: update debian_build.py to use commit['id'] instead of branch
- params = {
- 'GIT_BRANCH_NAME': branch,
- 'REBUILD_DIST': '',
- }
- url = jenkins_client.build_job_url(job_name, params, JENKINS_TRIGGER_TOKEN)
- logger.info("Triggering Jenkins job '{job}' by calling {url}..."
- .format(job=job_name, url=url))
- # TODO: Extract JENKINS_TRIGGER_TOKEN from job description
- jenkins_client.build_job(job_name, params, JENKINS_TRIGGER_TOKEN)
+ # Connect to Jenkins
+ if not os.path.isfile(JENKIKS_TOKEN_FILE):
+ logger.error("File '{file}' not found. Please place the Jenkins API token into this file."
+ .format(file=JENKIKS_TOKEN_FILE))
+ sys.exit(1)
+ jenkins_token = open(JENKIKS_TOKEN_FILE).read().strip()
+ logger.info("Connecting to Jenkins API on {url} as user '{user}'..."
+ .format(url=JENKINS_URL, user=JENKINS_USER))
+ jenkins_client = jenkins.Jenkins(JENKINS_URL, JENKINS_USER, jenkins_token)
+
+ # Connect to Gitlab
+ if not os.path.isfile(GITLAB_TOKEN_FILE):
+ logger.error("File '{file}' not found. Please place the Gitlab API token into this file."
+ .format(file=GITLAB_TOKEN_FILE))
+ sys.exit(1)
+ gitlab_token = open(GITLAB_TOKEN_FILE).read().strip()
+ logger.info("Connecting to Gitlab API on {url}...".format(url=GITLAB_URL))
+ gitlab_client = gitlab.Gitlab(GITLAB_URL, gitlab_token, verify_ssl=False)
+
+ logger.info("Processing commit {commit} from {repo}..."
+ .format(commit=data['after'], repo=data['repository']['homepage']))
+
+ config = read_buildchain_config(data, gitlab_client, logger)
+ changelog = get_debian_changelog(config, data, gitlab_client, logger)
+ source_name = changelog.package
+ branch = re.sub('^refs/heads/', '', data['ref'])
+ for distro in get_distros(config, changelog, branch, logger):
+ job_name = get_job_name(source_name, distro, branch)
+ logger.info("Creating/updating Jenkins job '{name}'...".format(name=job_name))
+ description = ("Debian package build job for the source package <b>{package}</b> from "
+ '<a href="{url}">{url}</a><br/>Warning: This job is '
+ "automatically generated by the {script} script. Do not edit this file "
+ "manually, because these changes will get lost on the next git push. "
+ "Instead configure the job via the .buildchain file in your git "
+ "repository.".format(package=changelog.package,
+ url=data['repository']['homepage'],
+ script=os.path.basename(sys.argv[0])))
+ substitutions = {
+ 'gitrepo': data['repository']['url'],
+ 'branch': branch,
+ 'homepage': data['repository']['homepage'],
+ # TODO: fill useful list of recipients
+ 'recipients': 'benjamin.drung@profitbricks.com',
+ 'description': description,
+ }
+ jenkins_job = JenkinsJob.from_template(jenkins_client, job_name, substitutions, logger)
+ jenkins_job.update_job()
+ jenkins_job.trigger_job({'GIT_BRANCH_NAME': data['after']})
if __name__ == '__main__':
main()
projects / profitbricks / jenkins-build-scripts.git / commitdiff