Add gitlab_jenkins_trigger.py script

diff --git a/gitlab_jenkins_trigger.py b/gitlab_jenkins_trigger.py
new file mode 100755 (executable)
index 0000000..2a74316
--- /dev/null
+++ b/gitlab_jenkins_trigger.py
@@ -0,0 +1,266 @@
+#!/usr/bin/python
+
+# Copyright (C) 2014, ProfitBricks GmbH
+# Authors: Benjamin Drung <benjamin.drung@profitbricks.com>
+#
+# Permission to use, copy, modify, and/or distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+"""Create/Update Debian build Jenkins jobs based on Gitlab webhooks
+
+The behavior of the gitlab_jenkins_trigger can be configured by placing a .buildchain
+configuration file into the git repository. See get_config_parser() for the default
+values of the .buildchain configuration.
+
+Example .buildchain file:
+================================================================================
+# This is a sample .buildchain configuration file
+# The section is [debian]
+
+[debian]
+
+# enable or disable Debian package builds (default: auto)
+# In auto mode, the debian build is enabled if debian/changelog exists.
+# Otherwise it will be disabled.
+enabled = True
+
+# distros can be set to a list of space-separated distributions or to 'auto'.
+# In 'auto' mode, debian/changelog will be parsed and the distros
+# from there will be used. Multiple distributions are separated by spaces
+# in debian/changelog. If distributions is set to 'UNRELEASED', the previous
+# changelog entries are parsed recursively.
+# Supported distributions are: squeeze wheezy jessie trusty utopic
+distros = auto
+================================================================================
+
+"""
+
+from __future__ import print_function
+
+import io
+import json
+import logging
+import os
+import re
+import sys
+import urllib2
+
+try:
+    if sys.version_info[:2] >= (3, 2):
+        from configparser import ConfigParser
+    else:
+        from configparser import SafeConfigParser as ConfigParser
+except ImportError:
+    from ConfigParser import SafeConfigParser as ConfigParser
+
+import debian.changelog
+import jenkins
+
+_LOG_LEVEL = logging.INFO
+_LOG_FORMAT = '%(asctime)s %(name)s [%(process)d] %(levelname)s: %(message)s'
+
+JENKINS_URL = "https://jenkins/"
+# TODO: Use special user
+JENKINS_USER = "bdrung"
+TEMPLATE_JOB = "debian-package-template2"
+JENKINS_TRIGGER_TOKEN = "BuildIt"
+SUPPORTED_DISTROS = [
+    "squeeze",
+    "wheezy",
+    "jessie",
+    "trusty",
+    "utopic",
+]
+FALLBACK_DISTRO = "squeeze"
+
+
+def get_config_parser():
+    """Create a ConfigParser object and set the default values"""
+    config = ConfigParser()
+    config.add_section('debian')
+    config.set('debian', 'enabled', 'auto')
+    config.set('debian', 'distros', 'auto')
+    return config
+
+
+def get_git_tree(data, commit):
+    """Return the base directory URL for the given git repository"""
+    return data['repository']['homepage'] + '/raw/' + commit['id'] + '/'
+
+
+def read_buildchain_config(config, data, commit, logger):
+    """Read .buildchain configuration file from git repository
+
+    Try to read the .buildchain configuration file from the git repository and
+    update the config object"""
+    git_tree = get_git_tree(data, commit)
+    try:
+        response = urllib2.urlopen(os.path.join(git_tree, '.buildchain'))
+        buildinfo = response.read()
+        logger.info(".buildinfo content:\n" + buildinfo)
+        config.readfp(io.BytesIO(buildinfo))
+    except urllib2.HTTPError as error:
+        if error.code == 404:
+            logger.info("No .buildinfo file found in {repo}".format(repo=commit['url']))
+        else:
+            logger.error("Retreiving .buildinfo failed with HTTP code {code}."
+                         .format(code=error.code))
+            raise
+
+
+def get_debian_changelog(config, git_tree, logger):
+    """Make sure the project is enabled. Then retreive the debian/changelog file"""
+    if config.get('debian', 'enabled') == 'auto':
+        enabled = 'auto'
+    else:
+        enabled = config.getboolean('debian', 'enabled')
+        if not enabled:
+            logger.info("Debian package build disabled in .buildchain configuration. "
+                        "Doing nothing.")
+            sys.exit(0)
+
+    try:
+        response = urllib2.urlopen(os.path.join(git_tree, 'debian', 'changelog'))
+        changelog = response.read()
+    except urllib2.HTTPError as error:
+        if error.code == 404:
+            if enabled == 'auto':
+                logger.info("The enabled variable of the Debian package build was set to 'auto' "
+                            "and no debian/changelog file found. Doing nothing.")
+                sys.exit(0)
+            else:
+                logger.error("The Debian package build is enabled, but no debian/changelog file "
+                             "found in the git repository.")
+                sys.exit(1)
+        else:
+            logger.error("Retreiving debian/changelog failed with HTTP code {code}."
+                         .format(code=error.code))
+            raise
+    changelog = debian.changelog.Changelog(changelog)
+    return changelog
+
+
+def get_distros(config, changelog, logger):
+    """Return list of distributions to build
+
+    The distros variable can be set to a list of space-separated distributions or to
+    'auto' in the .buildchain configuration file. In 'auto' mode, debian/changelog
+    will be parsed and the distros from there will be used. If distributions is set
+    to 'UNRELEASED', the previous changelog entries are parsed recursively.
+
+    The list of distributions is checked for invalid or unsupported distributions
+    and duplicate entries are removed from the list.
+    """
+    distros = config.get('debian', 'distros')
+    if distros == 'auto':
+        logger.info("Distros was set to 'auto'. "
+                    "Extracting distributions from debian/changelog...")
+        # Search for the last release (where distribution is not UNRELEASED)
+        for block in changelog:
+            distros = block.distributions
+            if distros != "UNRELEASED":
+                break
+    logger.info("Specified distributions: {distros}".format(distros=distros))
+    distros = [d for d in distros.split(' ') if d]
+
+    # Check distros list to be valid
+    for i in range(len(distros)):
+        if distros[i] not in SUPPORTED_DISTROS:
+            logger.warn("Unsuppored distribution '{old}' specified. Fallback to '{new}' instead. "
+                        "Please specify a supported distribution: {supported}"
+                        .format(old=distros[i], new=FALLBACK_DISTRO,
+                                supported=" ".join(SUPPORTED_DISTROS)))
+            distros[i] = FALLBACK_DISTRO
+
+    # Remove duplicate entries
+    distros = [ii for n, ii in enumerate(distros) if ii not in distros[:n]]
+
+    logger.info("Distributions to built: {distros}".format(distros=" ".join(distros)))
+    return distros
+
+
+def update_jenkins_job(config, jenkins_client, job_name, data, branch, logger):
+    logger.info("Retreiving Job template from '{job}' job...".format(job=TEMPLATE_JOB))
+    config_xml = jenkins_client.get_job_config(TEMPLATE_JOB)
+    if not config_xml:
+        logger.error("Job template '{job}' not found.".format(job=TEMPLATE_JOB))
+        sys.exit(1)
+
+    replacements = {}
+    replacements['gitrepo'] = data['repository']['url']
+    replacements['branch'] = branch
+    replacements['homepage'] = data['repository']['homepage']
+    # TODO: fill useful list of recipients
+    replacements['recipients'] = 'benjamin.drung@profitbricks.com'
+    for key, value in replacements.iteritems():
+        print(key, value)
+        config_xml = re.sub('@' + key + '@', value, config_xml)
+    # Enable job
+    config_xml = re.sub('<disabled>true</disabled>', '<disabled>false</disabled>', config_xml)
+    # Update description
+
+    if jenkins_client.job_exists(job_name):
+        logger.info("Updating Jenkins job {job}...".format(job=job_name))
+        jenkins_client.reconfig_job(job_name, config_xml)
+    else:
+        logger.info("Creating Jenkins job {job}...".format(job=job_name))
+        jenkins_client.create_job(job_name, config_xml)
+
+
+def main():
+    logging.basicConfig(format=_LOG_FORMAT, level=_LOG_LEVEL)
+    logger = logging.getLogger(os.path.basename(__file__))
+    config = get_config_parser()
+    # TODO: retreive api token in secure way
+    apitoken = open(os.path.dirname(sys.argv[0]) + '/.apitoken').read().strip()
+    jenkins_client = jenkins.Jenkins(JENKINS_URL, JENKINS_USER, apitoken)
+    if 'payload' not in os.environ:
+        logger.error("No 'payload' environment variable set. Please specify the Gitlab webhook "
+                     "JSON data in the 'payload' environment variable.")
+        sys.exit(1)
+    try:
+        data = json.loads(os.environ['payload'])
+    except ValueError:
+        logger.error("Failed to decode JSON payload from 'payload' environment variable: "
+                     "'{payload}'".format(payload=os.environ['payload']))
+        sys.exit(1)
+    logger.info('Received push event:\n{event}'.format(event=json.dumps(data, indent=4)))
+
+    # Just process the latest commit
+    if len(data['commits']) > 0:
+        commit = data['commits'][0]
+        logger.info("Processing commit {commit} from {repo}..."
+                    .format(commit=commit['id'], repo=data['repository']['homepage']))
+
+        git_tree = get_git_tree(data, commit)
+        read_buildchain_config(config, data, commit, logger)
+        changelog = get_debian_changelog(config, git_tree, logger)
+        source_name = changelog.package
+        branch = re.sub('^refs/heads/', '', data['ref'])
+        for distro in get_distros(config, changelog, logger):
+            job_name = distro + '_' + source_name
+            if branch != 'master':
+                job_name += '_' + branch
+            update_jenkins_job(config, jenkins_client, job_name, data, branch, logger)
+            # TODO: update debian_build.py to use commit['id'] instead of branch
+            params = {
+                'GIT_BRANCH_NAME': branch,
+                'REBUILD_DIST': '',
+            }
+            url = jenkins_client.build_job_url(job_name, params, JENKINS_TRIGGER_TOKEN)
+            logger.info("Triggering Jenkins job '{job}' by calling {url}..."
+                        .format(job=job_name, url=url))
+            # TODO: Extract JENKINS_TRIGGER_TOKEN from job description
+            jenkins_client.build_job(job_name, params, JENKINS_TRIGGER_TOKEN)
+
+if __name__ == '__main__':
+    main()