From 87dc50e801d8354070a605391d3dfd2b6608f377 Mon Sep 17 00:00:00 2001 From: Holger Levsen Date: Thu, 27 Sep 2012 16:34:11 +0200 Subject: [PATCH] script to add existign packages to db added --- common_code.py | 31 +++++++++++++++++++++++++++++++ db_add.py | 15 ++++++++++++--- debian_build.py | 23 ++--------------------- debian_packages2db.py | 32 ++++++++++++++++++++------------ debian_repos2db.sh | 7 ++++++- 5 files changed, 71 insertions(+), 37 deletions(-) create mode 100755 common_code.py diff --git a/common_code.py b/common_code.py new file mode 100755 index 0000000..f2db8db --- /dev/null +++ b/common_code.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import sys +import logging +from logging import Formatter + +def logger_init(): + log_format = '%(asctime)s %(name)s[%(process)d] %(levelname)s: %(message)s' + formatter = Formatter(log_format) + logging.basicConfig( + format=log_format, + level=logging.DEBUG + ) + logger = logging.getLogger(__file__) + logger.setLevel(logging.DEBUG) + return logger + +def logger_error(msg): + print msg + logger.error(msg) + +def exit_ok(): + sys.exit(0) + +def exit_error(): + print "################################################################" + sys.exitfunc = lambda: None + sys.exit(1) + + diff --git a/db_add.py b/db_add.py index b9ce111..cec8255 100755 --- a/db_add.py +++ b/db_add.py @@ -6,15 +6,13 @@ import re import pwd import sys import errno -import logging import datetime import subprocess import time import fileinput import psycopg2 +from common_code import * from cidb import * -from logging import Formatter - def db_add_job(con, name): cur = con.cursor() @@ -120,4 +118,15 @@ def add_package_instance(origin, job_name, build_number, changes_file, version, logger.info("CIDB update OK.") return db_package_instance_id +def add_package_version(origin, package, version): + logger = logger_init() + con = db_connect() + #logger.debug("save %s to database" %(job_name)) + db_origin_id = db_add_origin(con, origin) + db_package_id = db_add_package (con, package) + db_package_instance_id = db_add_package_instance (con, db_package_id, db_origin_id, 'NULL', version) + logger.debug("INSERT success package version %s %s" %( package,version)) + #logger.debug('Cmd returned with status %d' %(cmdobj.returncode)) + con.commit() + return db_package_instance_id diff --git a/debian_build.py b/debian_build.py index 63eb8ce..f599a78 100755 --- a/debian_build.py +++ b/debian_build.py @@ -19,6 +19,7 @@ import time import urllib import fileinput import psycopg2 +from common_code import * from add_liveboot_request import add_liveboot_request from db_add import * from cidb import * @@ -92,22 +93,10 @@ DPUT_CF = os.path.join(CWD, '..', 'dput.cf') NO_UPLOAD = os.environ.get('NO_UPLOAD') -log_format = '%(asctime)s %(name)s[%(process)d] %(levelname)s: %(message)s' -formatter = Formatter(log_format) -logging.basicConfig( - format=log_format, - level=logging.DEBUG -) -logger = logging.getLogger(__file__) -logger.setLevel(logging.DEBUG) class HudsonUrl(urllib.FancyURLopener): pass -def logger_error(msg): - print msg - logger.error(msg) - def read_file(path): try: fh = open(path, 'r', 1) @@ -190,15 +179,6 @@ def remove_git_target_workspace(): return True - -def exit_ok(): - sys.exit(0) - -def exit_error(): - print "################################################################" - sys.exitfunc = lambda: None - sys.exit(1) - def getopts(): usage = '%prog [options]' parser = optparse.OptionParser(usage=usage, version='%prog 0.1') @@ -213,6 +193,7 @@ def getopts(): return parser.parse_args() if __name__ == '__main__': + logger = logger_init() logger.debug('Called in: %s' % os.getcwd()) logger.debug('Called with: %s' % " ".join(sys.argv)) logger.debug( diff --git a/debian_packages2db.py b/debian_packages2db.py index e46994a..a32ad7e 100755 --- a/debian_packages2db.py +++ b/debian_packages2db.py @@ -10,36 +10,44 @@ import subprocess import time import fileinput import string +import tempfile +from common_code import * from db_add import * if __name__ == '__main__': - if len(sys.argv) != 2: - print("usage: %s $packages_file") + if len(sys.argv) != 3: + print("usage: %s $packages_file $origin") sys.exit(1) packages_file = sys.argv[1] + origin = sys.argv[2] - cmd = ['grep-dctrl', '-n', '-s', 'Package,Version', '', packages_file] + tmp = tempfile.NamedTemporaryFile(delete=False) + cmd = ['egrep', '(^Package:|^Version:)', packages_file] print cmd - grep_dctrl = subprocess.Popen( + egrep = subprocess.Popen( cmd, - stdout=subprocess.PIPE, + stdout=tmp, close_fds=True, shell=False, + bufsize=-1 ) - ret = grep_dctrl.wait() + ret = egrep.wait() if ret: print '%s was not successfull, return code was %s ' % (' '.join(cmd), ret) sys.exit(1) + tmp.seek(0) + pkg_name = ' ' while pkg_name: - pkg_name = grep_dctrl.stdout.readline() - pkg_name = pkg_name.strip() + pkg_name = tmp.readline() if pkg_name == '': break - version = grep_dctrl.stdout.readline() - version = version.strip() - empty = grep_dctrl.stdout.readline() - print 'would add %s %s' % (pkg_name,version) + pkg_name = pkg_name.strip().split()[1] + version = tmp.readline() + version = version.strip().split()[1] + print 'would add %s %s %s' % (pkg_name,version,origin) + add_package_version(origin, pkg_name, version) + os.remove(tmp.name) diff --git a/debian_repos2db.sh b/debian_repos2db.sh index db6ff9a..0604c17 100755 --- a/debian_repos2db.sh +++ b/debian_repos2db.sh @@ -9,11 +9,16 @@ REPO_SECTIONS[1]="main contrib non-free" REPO_BASE_URL[1]="http://alexandria.profitbricks.localdomain/debian-mirror/dists/SUITE/SECTION/binary-amd64/" for RUN in 0 1 ; do + if [ "$RUN" == 0 ] ; then + ORIGIN=profitbricks + else + ORIGIN=debian + fi for SUITE in ${REPO_SUITES[$RUN]} ; do for SECTION in ${REPO_SECTIONS[$RUN]} ; do URL=$(echo ${REPO_BASE_URL[$RUN]} | sed -s "s#SUITE#$SUITE#" | sed -s "s#SECTION#$SECTION#")/Packages wget $URL -o /dev/null - [ -s Packages ] && ./debian_packages2db.py Packages + [ -s Packages ] && ./debian_packages2db.py Packages $ORIGIN rm Packages done done -- 2.39.5