From c830dbdbefa994a6fcdfd741610fba527a52c8bc Mon Sep 17 00:00:00 2001 From: Mathias Klette Date: Wed, 8 May 2013 00:42:07 +0200 Subject: [PATCH] debian_build: make multiple dists work --- debian_build.py | 141 ++++++++++++++++++++++++++---------------------- 1 file changed, 77 insertions(+), 64 deletions(-) diff --git a/debian_build.py b/debian_build.py index 50b037a..64b4e53 100755 --- a/debian_build.py +++ b/debian_build.py @@ -24,6 +24,7 @@ import sys import time from ConfigParser import SafeConfigParser from multiprocessing import cpu_count +from pprint import pformat # import 3rd parties from debian import changelog @@ -44,7 +45,7 @@ logger = logger_init(__file__) # jenkins environment - parameters ENV.setdefault('NO_UPLOAD','') ENV.setdefault('DISTRIBUTION','squeeze') -GIT_REPO_PATH = ENV['GIT_REPO_PATH'] +#GIT_REPO_PATH = ENV['GIT_REPO_PATH'] # local constants BUILD_START = datetime.datetime.now() @@ -59,13 +60,13 @@ AUTO_CHANGELOG_REPONAMES = ( 'storage-docs', ) STABLE_DISTRIBUTIONS = ( - 'stable', - 'production', + 'stable', + 'production', 'production-proposed-updates' ) STABLE_BRANCHES_RE = re.compile( '^(origin/)?(master|hotfix/.*)$') -DEVELOP_BRANCHES_RE = re.compile( +UNSTABLE_BRANCHES_RE = re.compile( '^(origin/)?(develop|pre-staging|release/.*)$') EXPERIMENTAL_BRANCHES_RE = re.compile( '^(origin/)?(experimental|(feature|poc|bugfix)/.*)$') @@ -77,12 +78,12 @@ def getopts(): '--distribution', dest='distribution', choices = [ - 'stable', - 'production', - 'production-proposed-updates', - 'unstable', - 'pre-staging', - 'staging', + 'stable', + 'production', + 'production-proposed-updates', + 'unstable', + 'pre-staging', + 'staging', 'experimental' ], default='unstable', @@ -106,14 +107,14 @@ def getopts(): return parser.parse_args() if __name__ == '__main__': - logger.debug('Environment: %s' %(ENV)) + logger.debug('Environment:\n%s' %(pformat(ENV))) # Act I: prepare variables # .. command line arguments # FIXME: not used right now, shall it be used? options, args = getopts() - # .. actions related, define defaults + # .. actions, define defaults do_autoincrement = False do_cidb = False do_cleanup = True @@ -122,6 +123,13 @@ if __name__ == '__main__': do_tagging = False do_uploads = True + cl = changelog.Changelog() + cl.parse_changelog(open('debian/changelog')) + curr_name = cl.package + curr_version = cl.full_version + curr_dist = cl.distributions + daily_date = BUILD_START.strftime('%Y%m%d%H%M%S') + # .. repository related curr_commit = ENV['GIT_COMMIT'] gitrepo = git.Repo('.') @@ -143,28 +151,14 @@ if __name__ == '__main__': } ) - # .. changelog related - daily_date = BUILD_START.strftime('%Y%m%d%H%M%S') - - cl = changelog.Changelog() - cl.parse_changelog(open('debian/changelog')) - curr_name = cl.package - curr_version = cl.full_version - curr_dist = cl.distributions - logger.info( - 'Current Changelog: Package {name} with version {version} in distribution {dist}'.format( - name = curr_name, - version = curr_version, - dist = curr_dist, - )) - # Act II: make decissions if re.match(STABLE_BRANCHES_RE, ENV['GIT_BRANCH']): - new_dist = 'production-proposed-updates' + new_dist = 'stable' new_version = '{version}'.format( - version = curr_version, + version = curr_version, ) + pb_suite = 'master-' + ENV['DISTRIBUTION'] # replace valid debian version chars that are invalid for git tagging new_tag = curr_version.replace('~','_') new_tag = new_tag.replace(':',',') @@ -179,61 +173,69 @@ if __name__ == '__main__': # .. fail if we already found the tag we would create upon a successfull build, # except, the existing tag uses the same commit as we were triggered with remote_tag = [tag for tag in gitrepo.tags if tag.name == new_tag] - if len(remote_tag) > 0: + if len(remote_tag) > 0: if remote_tag[0].commit.id == curr_commit: - logger.info('Tag was already created for this commit.') + logger.debug('Tag was already created for this commit.') do_tagging = False else: logger.error('Tag was already created for another commit.') exit_error() else: do_tagging = True - # .. just used for GitBuildPackage() - pb_suite = new_dist else: - # TODO: tagging, cidb and liveboot makes no sense yet for other dists - # TODO: align reprepro to also use naming scheme for squeeze - pb_suite = 'master-' + ENV['DISTRIBUTION'] + # FIXME: tagging, cidb and liveboot makes no sense yet for other dists + # FIXME: align reprepro to also use naming scheme for squeeze dput_obj.contents.update({ 'incoming': '/srv/profitbricks-repository/incoming/profitbricks-wheezy', }) - - elif re.match(DEVELOP_BRANCHES_RE, ENV['GIT_BRANCH']): - new_dist = 'pre-staging' + + elif re.match(UNSTABLE_BRANCHES_RE, ENV['GIT_BRANCH']): + new_dist = 'unstable' new_version = '{version}~develop{date}+{build}+{commit}'.format( - version = curr_version, - date = daily_date, - build = ENV['BUILD_NUMBER'], + version = curr_version, + date = daily_date, + build = ENV['BUILD_NUMBER'], commit = curr_commit[0:7], ) + pb_suite = 'develop-' + ENV['DISTRIBUTION'] # reset actions # .. always include successful build packages into CIDB if ENV['DISTRIBUTION'] == 'squeeze': do_cidb = True - pb_suite = new_dist else: - pb_suite = 'develop-' + ENV['DISTRIBUTION'] + dput_obj.contents.update({ + 'incoming': '/srv/profitbricks-repository/incoming/profitbricks-wheezy', + }) elif re.match(EXPERIMENTAL_BRANCHES_RE, ENV['GIT_BRANCH']): - new_dist ='dev-'+ ENV['GIT_BRANCH'].replace("/","-") + new_dist = 'dev-' + ENV['GIT_BRANCH'].replace("/","-") new_version = '{version}~experimental{date}+{build}+{commit}'.format( - version = curr_version, - date = daily_date, - build = ENV['BUILD_NUMBER'], + version = curr_version, + date = daily_date, + build = ENV['BUILD_NUMBER'], commit = curr_commit[0:7], ) + pb_suite = 'experimental-' + ENV['DISTRIBUTION'] dput_obj.contents.update({ - 'incoming': '/srv/dev-repository/incoming/', 'post_upload_command': 'ssh reprepro@alexandria.pb.local /srv/dev-repository/bin/pb_processincoming', }) - pb_suite = new_dist + if ENV['DISTRIBUTION'] == 'squeeze': + dput_obj.contents.update({ + 'incoming': '/srv/dev-repository/incoming/', + }) else: - pb_suite = 'experimental-' + ENV['DISTRIBUTION'] + # reprepro uses dynamic distributions based on the distribution + # value given in the package. to not interferre with experimental + # squeeze package, we need to include the distribution here. + new_dist = 'dev-' + ENV['GIT_BRANCH'].replace("/","-") + ENV['DISTRIBUTION'] + dput_obj.contents.update({ + 'incoming': '/srv/dev-repository/incoming/wheezy', + }) else: logger.error('Don\'t know how to handle branch "{branch}".'.format( branch = ENV['GIT_BRANCH'], - ) + )) exit_error() #FIXME: enable this once Florian acks. @@ -248,6 +250,13 @@ if __name__ == '__main__': # version = '%s~alpha%s+%s+%s' %(version, daily_date, BUILD_NUMBER, curr_commit[0:7]) # changelog_distro = 'unstable' + logger.debug('Distribution: "%s" => "%s"' %(curr_dist,new_dist)) + logger.debug('Version: "%s" => "%s"' %(curr_version,new_version)) + logger.debug('PB_SUITE: ' + pb_suite) + if do_tagging: + logger.debug('Tag to create: ' + new_tag) + logger.debug('Upload configuration:\n%s' %(pformat(dput_obj.contents))) + # ACT III: do something actually # .. do some housekeeping first @@ -271,13 +280,13 @@ if __name__ == '__main__': # .. and also cleanup existing *orig.tar.gz if os.path.isdir(BUILD_AREA): - files = [os.path.join(BUILD_AREA, file) + files = [os.path.join(BUILD_AREA, file) for file in fnmatch.filter( os.listdir(BUILD_AREA), '*.orig.tar.gz' )] if len(files) > 0: - logger.debug('Delete previous upstream tarball(s)') + logger.info('Delete previous upstream tarball(s)') cmd = [BIN_SUDO, BIN_RM, '-v'] cmd.extend(files) ret = subprocess.Popen( @@ -290,7 +299,7 @@ if __name__ == '__main__': env={'':''} ).wait() logger.debug('"{cmd}" returns {exitcode}.'.format( - cmd = cmd, + cmd = cmd, exitcode = ret )) figlet('Cleanup OK') @@ -348,6 +357,8 @@ if __name__ == '__main__': if ret: raise Exception( '"%s" returned with exitcode: %s' %(' '.join(cmd), ret)) + else: + logger.info('Changelog updated by Jenkins using debian_build.py!') # let me see the first two changelog entries: @@ -396,7 +407,9 @@ if __name__ == '__main__': try: if os.path.exists(reports_file): subprocess.check_call(['/bin/tar', 'xzvf', '-C', ENV['WORKSPACE'], reports_file]) - subprocess.check_call(['/usr/bin/sudo' , '/bin/rm', reports_file]) + subprocess.check_call(['/usr/bin/sudo' , '/bin/rm', '-v', reports_file]) + else: + logger.info('No reports found.') figlet('Reports OK') except: pass @@ -435,8 +448,8 @@ if __name__ == '__main__': # construct path for changes file changes_file = os.path.join( - ENV['WORKSPACE'], - '../build-area/result/', + ENV['WORKSPACE'], + '../build-area/result/', '{name}_{version}_amd64.changes'.format( name = curr_name, version = new_version, @@ -469,12 +482,12 @@ if __name__ == '__main__': package_instances=[] try: package_instances = add_package_instances( - "profitbricks", - ENV['JOB_NAME'], - ENV['BUILD_NUMBER'], - changes_file, - new_version, - BUILD_START, + "profitbricks", + ENV['JOB_NAME'], + ENV['BUILD_NUMBER'], + changes_file, + new_version, + BUILD_START, BUILD_END, ) figlet('CIDB OK') -- 2.39.5