work in progress: debian package importer

diff --git a/.gitignore b/.gitignore
index 9fa524a7549a3b97eb3e6b51ae0c5b3295eb6083..3220327edf0c1633890156d61689b8cff78f6181 100644 (file)
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1,4 @@
 .*\.swp
+*/*\.pyc
+*\.pyc
+Packages*

diff --git a/db_add.py b/db_add.py
new file mode 100755 (executable)
index 0000000..b9ce111
--- /dev/null
+++ b/db_add.py
@@ -0,0 +1,123 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import os
+import re
+import pwd
+import sys
+import errno
+import logging
+import datetime
+import subprocess
+import time
+import fileinput
+import psycopg2
+from cidb import *
+from logging import Formatter
+
+
+def db_add_job(con, name):
+    cur = con.cursor()
+    cur.execute("SAVEPOINT a")
+    try:
+        cur.execute("INSERT INTO jenkins_job(name) VALUES('%s') RETURNING id" % (name))
+    except psycopg2.DatabaseError as e:
+    # record already exist
+        cur.execute("ROLLBACK TO a")  # have to rollback after failed command
+        cur.execute("SELECT id FROM jenkins_job WHERE name='%s'" % (name))
+        #logger.debug("INSERT INTO jenkins_job(name) VALUES('%s')" % (name))
+    return cur.fetchone()[0]
+
+def db_add_build(con, number, db_id):
+    cur = con.cursor()
+    cur.execute("SAVEPOINT a")
+    try:
+        cur.execute("INSERT INTO jenkins_build(jenkins_build_number, jenkins_job_id) VALUES(%s, %s) RETURNING id" % (number, db_id))
+    except psycopg2.DatabaseError as e:
+        cur.execute("ROLLBACK TO a")  # have to rollback after failed command
+        #logger.debug("INSERT INTO jenkins_build(jenkins_build_number, jenkins_job_id) VALUES(%s, %s) RETURNING id" % (number, db_id))
+    return cur.fetchone()[0]
+
+def db_add_build_result(con, build_id, start, end):
+    cur = con.cursor()
+    cur.execute("SAVEPOINT a")
+    try:
+        cur.execute("INSERT INTO jenkins_build_result(jenkins_build_id, build_start, build_end, successful) VALUES(%s, '%s', '%s', %s) RETURNING id" % (build_id, start, end, 'true'))
+    except psycopg2.DatabaseError as e:
+        cur.execute("ROLLBACK TO a")  # have to rollback after failed command
+        #logger.debug("INSERT INTO jenkins_build_result(jenkins_build_id, build_start, build_end, successful) VALUES(%s, '%s', '%s', %s) RETURNING id" % (build_id, start, end, 'true'))
+    return cur.fetchone()[0]
+
+def db_add_package(con, package):
+    cur = con.cursor()
+    cur.execute("SAVEPOINT a")
+    try:
+        cur.execute("INSERT INTO deb_package(name) VALUES('%s') RETURNING id" % (package))
+    except psycopg2.DatabaseError as e:
+    # record already exist
+        cur.execute("ROLLBACK TO a")  # have to rollback after failed command
+        cur.execute("SELECT id FROM deb_package WHERE name='%s'" % (package))
+        #logger.debug("INSERT INTO deb_package(name) VALUES('%s') RETURNING id" % (package))
+    return cur.fetchone()[0]
+
+def db_add_package_instance(con, package_id, origin_id, build_result_id, version):
+    cur = con.cursor()
+    cur.execute("SAVEPOINT a")
+    try:
+        cur.execute("INSERT INTO deb_package_instance(deb_package_id, deb_package_origin_id, jenkins_build_result_id, version, available) VALUES(%s, %s, %s, '%s', 'true') RETURNING id" % (package_id, origin_id, build_result_id, version)) 
+    except psycopg2.DatabaseError as e:
+    # record already exist - this MUST NOT HAPPEN
+        cur.execute("ROLLBACK TO a")  # have to rollback after failed command
+        logger_error("FAILED: INSERT INTO deb_package_instance(deb_package_id, deb_package_origin_id, jenkins_build_result_id, version, available) VALUES(%s, %s, %s, '%s', 'true') RETURNING id" % (package_id, origin_id, build_result_id, version))
+        exit_error()
+    return cur.fetchone()[0]
+
+def db_add_origin(con, origin):
+    cur = con.cursor()
+    cur.execute("SAVEPOINT a")
+    try:
+        cur.execute("INSERT INTO deb_package_origin(origin) VALUES('%s') RETURNING id" % (origin))
+    except psycopg2.DatabaseError as e:
+    # record already exist
+        cur.execute("ROLLBACK TO a")  # have to rollback after failed command
+        cur.execute("SELECT id FROM deb_package_origin WHERE origin='%s'" % (origin))
+        #logger.debug("INSERT INTO deb_package_origin(origin) VALUES('%s') RETURNING id" % (origin))
+    return cur.fetchone()[0]
+
+def add_package_instance(origin, job_name, build_number, changes_file, version, start, end):
+    con = db_connect()
+    #logger.debug("save %s to database" %(job_name))
+    db_origin_id = db_add_origin(con, origin) 
+    #logger.debug("INSERT success origin %s" % origin)
+    db_job_id = db_add_job(con, job_name) 
+    #logger.debug("INSERT success job %s" % job_name)
+    # maybe FIXME: we could add the build already once its started, but for package builds we dont wanna add failures to the db at all
+    db_build_id = db_add_build (con, build_number, db_job_id)
+    db_build_result_id = db_add_build_result (con, db_build_id, start, end)
+    #logger.debug("INSERT success build number %s" % build_number)
+    #logger.debug("processing %s" %(changes_file))
+    cmd = ["dcmd", '--deb', changes_file]
+    cmdobj = subprocess.Popen(
+            cmd,
+            shell=False,
+            cwd='/',
+            close_fds=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            env={'':''}
+    )
+    #logger.debug('Calling "%s":' %(' '.join(cmd)))
+    ret = cmdobj.wait()
+    for line in cmdobj.stdout.readlines():
+        package = os.path.basename(line.strip()).split('_')[0]
+        #logger.debug('package: %s' %(package))
+        db_package_id = db_add_package (con, package)
+        #logger.debug("INSERT success package %s" % package)
+        db_package_instance_id = db_add_package_instance (con, db_package_id, db_origin_id, db_build_result_id, version)
+        logger.debug("INSERT success package version %s %s" %( package,version))
+    #logger.debug('Cmd returned with status %d' %(cmdobj.returncode))
+    con.commit()
+    logger.info("CIDB update OK.")
+    return db_package_instance_id
+
+

diff --git a/debian_build.py b/debian_build.py
index 71a95cbcf34ebf25ace6e398f6057c80a33ee006..63eb8ce2aee738abf36e3771885368f07c90bc1d 100755 (executable)
--- a/debian_build.py
+++ b/debian_build.py
@@ -20,6 +20,7 @@ import urllib
 import fileinput
 import psycopg2
 from add_liveboot_request import add_liveboot_request
+from db_add import *
 from cidb import *
 from glob import glob
 from ftplib import FTP
@@ -117,110 +118,6 @@ def read_file(path):
         fh.close()
         return result
 
-def db_add_job(con, name):
-    cur = con.cursor()
-    cur.execute("SAVEPOINT a")
-    try:
-        cur.execute("INSERT INTO jenkins_job(name) VALUES('%s') RETURNING id" % (name))
-    except psycopg2.DatabaseError as e:
-    # record already exist
-        cur.execute("ROLLBACK TO a")  # have to rollback after failed command
-        cur.execute("SELECT id FROM jenkins_job WHERE name='%s'" % (name))
-        #logger.debug("INSERT INTO jenkins_job(name) VALUES('%s')" % (name))
-    return cur.fetchone()[0]
-
-def db_add_build(con, number, db_id):
-    cur = con.cursor()
-    cur.execute("SAVEPOINT a")
-    try:
-        cur.execute("INSERT INTO jenkins_build(jenkins_build_number, jenkins_job_id) VALUES(%s, %s) RETURNING id" % (number, db_id))
-    except psycopg2.DatabaseError as e:
-        cur.execute("ROLLBACK TO a")  # have to rollback after failed command
-        #logger.debug("INSERT INTO jenkins_build(jenkins_build_number, jenkins_job_id) VALUES(%s, %s) RETURNING id" % (number, db_id))
-    return cur.fetchone()[0]
-
-def db_add_build_result(con, build_id, start, end):
-    cur = con.cursor()
-    cur.execute("SAVEPOINT a")
-    try:
-        cur.execute("INSERT INTO jenkins_build_result(jenkins_build_id, build_start, build_end, successful) VALUES(%s, '%s', '%s', %s) RETURNING id" % (build_id, start, end, 'true'))
-    except psycopg2.DatabaseError as e:
-        cur.execute("ROLLBACK TO a")  # have to rollback after failed command
-        #logger.debug("INSERT INTO jenkins_build_result(jenkins_build_id, build_start, build_end, successful) VALUES(%s, '%s', '%s', %s) RETURNING id" % (build_id, start, end, 'true'))
-    return cur.fetchone()[0]
-
-def db_add_package(con, package):
-    cur = con.cursor()
-    cur.execute("SAVEPOINT a")
-    try:
-        cur.execute("INSERT INTO deb_package(name) VALUES('%s') RETURNING id" % (package))
-    except psycopg2.DatabaseError as e:
-    # record already exist
-        cur.execute("ROLLBACK TO a")  # have to rollback after failed command
-        cur.execute("SELECT id FROM deb_package WHERE name='%s'" % (package))
-        #logger.debug("INSERT INTO deb_package(name) VALUES('%s') RETURNING id" % (package))
-    return cur.fetchone()[0]
-
-def db_add_package_instance(con, package_id, origin_id, build_result_id, version):
-    cur = con.cursor()
-    cur.execute("SAVEPOINT a")
-    try:
-        cur.execute("INSERT INTO deb_package_instance(deb_package_id, deb_package_origin_id, jenkins_build_result_id, version, available) VALUES(%s, %s, %s, '%s', 'true') RETURNING id" % (package_id, origin_id, build_result_id, version)) 
-    except psycopg2.DatabaseError as e:
-    # record already exist - this MUST NOT HAPPEN
-        cur.execute("ROLLBACK TO a")  # have to rollback after failed command
-        logger_error("FAILED: INSERT INTO deb_package_instance(deb_package_id, deb_package_origin_id, jenkins_build_result_id, version, available) VALUES(%s, %s, %s, '%s', 'true') RETURNING id" % (package_id, origin_id, build_result_id, version))
-        exit_error()
-    return cur.fetchone()[0]
-
-def db_add_origin(con, origin):
-    cur = con.cursor()
-    cur.execute("SAVEPOINT a")
-    try:
-        cur.execute("INSERT INTO deb_package_origin(origin) VALUES('%s') RETURNING id" % (origin))
-    except psycopg2.DatabaseError as e:
-    # record already exist
-        cur.execute("ROLLBACK TO a")  # have to rollback after failed command
-        cur.execute("SELECT id FROM deb_package_origin WHERE origin='%s'" % (origin))
-        #logger.debug("INSERT INTO deb_package_origin(origin) VALUES('%s') RETURNING id" % (origin))
-    return cur.fetchone()[0]
-
-def add_package_instance(origin, job_name, build_number, changes_file, version, start, end):
-    con = db_connect()
-    #logger.debug("save %s to database" %(job_name))
-    db_origin_id = db_add_origin(con, origin) 
-    #logger.debug("INSERT success origin %s" % origin)
-    db_job_id = db_add_job(con, job_name) 
-    #logger.debug("INSERT success job %s" % job_name)
-    # maybe FIXME: we could add the build already once its started, but for package builds we dont wanna add failures to the db at all
-    db_build_id = db_add_build (con, build_number, db_job_id)
-    db_build_result_id = db_add_build_result (con, db_build_id, start, end)
-    #logger.debug("INSERT success build number %s" % build_number)
-    #logger.debug("processing %s" %(changes_file))
-    cmd = ["dcmd", '--deb', changes_file]
-    cmdobj = subprocess.Popen(
-            cmd,
-            shell=False,
-            cwd='/',
-            close_fds=True,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.STDOUT,
-            env={'':''}
-    )
-    #logger.debug('Calling "%s":' %(' '.join(cmd)))
-    ret = cmdobj.wait()
-    for line in cmdobj.stdout.readlines():
-        package = os.path.basename(line.strip()).split('_')[0]
-        #logger.debug('package: %s' %(package))
-        db_package_id = db_add_package (con, package)
-        #logger.debug("INSERT success package %s" % package)
-        db_package_instance_id = db_add_package_instance (con, db_package_id, db_origin_id, db_build_result_id, version)
-        logger.debug("INSERT success package version %s %s" %( package,version))
-    #logger.debug('Cmd returned with status %d' %(cmdobj.returncode))
-    con.commit()
-    logger.info("CIDB update OK.")
-    return db_package_instance_id
-
 def dput_package_upload(changes_path):
     try:
         cmd = [BIN_DPUT, '-c', '%s' %(DPUT_CF), '--no-upload-log', 'profitbricks', '%s' %(changes_path)]

diff --git a/debian_packages2db.py b/debian_packages2db.py
new file mode 100755 (executable)
index 0000000..e46994a
--- /dev/null
+++ b/debian_packages2db.py
@@ -0,0 +1,45 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import os
+import re
+import sys
+import datetime
+import platform
+import subprocess
+import time
+import fileinput
+import string
+from db_add import *
+
+
+if __name__ == '__main__':
+    if len(sys.argv) != 2:
+        print("usage: %s $packages_file")
+        sys.exit(1)
+    packages_file = sys.argv[1]
+
+    cmd = ['grep-dctrl', '-n', '-s', 'Package,Version', '', packages_file]
+    print cmd
+    grep_dctrl =  subprocess.Popen(
+            cmd,
+            stdout=subprocess.PIPE,
+            close_fds=True,
+            shell=False,
+    )
+    ret = grep_dctrl.wait()
+    if ret:
+        print '%s was not successfull, return code was %s ' % (' '.join(cmd), ret)
+        sys.exit(1)
+
+    pkg_name = ' '
+    while pkg_name:
+        pkg_name = grep_dctrl.stdout.readline()
+        pkg_name = pkg_name.strip()
+        if pkg_name == '':
+            break
+        version = grep_dctrl.stdout.readline()
+        version = version.strip()
+       empty = grep_dctrl.stdout.readline()
+        print 'would add %s %s' % (pkg_name,version)
+

diff --git a/debian_repos2db.sh b/debian_repos2db.sh
new file mode 100755 (executable)
index 0000000..db6ff9a
--- /dev/null
+++ b/debian_repos2db.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+REPO_SUITES[0]="production-approved-updates production-proposed-updates production pre-staging"
+REPO_SECTIONS[0]="main contrib non-free"
+REPO_BASE_URL[0]="http://alexandria/profitbricks-repository/dists/SUITE/SECTION/binary-amd64/"
+
+REPO_SUITES[1]="squeeze profitbricks-backports profitbricks-backports-proposed-updates profitbricks-backports-pre-staging"
+REPO_SECTIONS[1]="main contrib non-free"
+REPO_BASE_URL[1]="http://alexandria.profitbricks.localdomain/debian-mirror/dists/SUITE/SECTION/binary-amd64/"
+
+for RUN in 0 1 ; do
+       for SUITE in ${REPO_SUITES[$RUN]} ; do
+               for SECTION in ${REPO_SECTIONS[$RUN]} ; do
+                       URL=$(echo ${REPO_BASE_URL[$RUN]} | sed -s "s#SUITE#$SUITE#" | sed -s "s#SECTION#$SECTION#")/Packages
+                       wget  $URL -o /dev/null
+                       [ -s Packages ] && ./debian_packages2db.py Packages
+                       rm Packages
+               done
+       done
+done