From 712640f62325968d2abee41275ba662d040303f5 Mon Sep 17 00:00:00 2001 From: Holger Levsen Date: Thu, 15 Nov 2012 12:59:43 +0100 Subject: [PATCH] actual delete liveboot images in db and filesystem --- cleanup_liveboot.py | 53 ++++++++++++++++++++++++++++++++++ keep_liveboot.py | 37 ------------------------ liveboot_garbage_collection.sh | 9 +++--- liveboot_rsync.sh | 18 +++--------- 4 files changed, 62 insertions(+), 55 deletions(-) create mode 100755 cleanup_liveboot.py delete mode 100755 keep_liveboot.py diff --git a/cleanup_liveboot.py b/cleanup_liveboot.py new file mode 100755 index 0000000..79573cb --- /dev/null +++ b/cleanup_liveboot.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import os +import re +import pwd +import sys +import errno +import datetime +import subprocess +import time +import fileinput +import psycopg2 +from common_code import * +from cidb import * + +logger = logger_init() + +def liveboot_to_keep(name): + con = db_connect() + cur = con.cursor() + try: + cur.execute("SELECT keep_image FROM liveboot WHERE build_name = '%s' and keep_image = true" % (name)) + except psycopg2.DatabaseError as e: + logger.debug("SELECT keep_image FROM liveboot WHERE build_name = '%s' and keep_image = true" % (name)) + return False + if cur.fetchone(): + return True + +def set_liveboot_to_delete(name): + con = db_connect() + cur = con.cursor() + try: + cur.execute("UPDATE liveboot SET deleted = true WHERE build_name = '%s'" % (name)) + con.commit() + except psycopg2.DatabaseError as e: + logger.debug("UPDATE liveboot SET deleted = true WHERE build_name = '%s'" % (name)) + return False + +if __name__ == '__main__': + if len(sys.argv) != 2 or sys.argv[1] not in ('keep', 'delete'): + print("usage: %s keep $liveboot_name") + print(" will return true or false, depending if 'keep_image' is set for this image") + print("usage: %s delete $liveboot_name") + print(" will set 'keep_image' to false") + sys.exit(1) + action = sys.argv[1] + name = sys.argv[2] + if action = "keep" and not liveboot_to_keep(name): + sys.exit(1) + if action = "delete": + set_liveboot_to_delete(name) + diff --git a/keep_liveboot.py b/keep_liveboot.py deleted file mode 100755 index e9ff5a9..0000000 --- a/keep_liveboot.py +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import os -import re -import pwd -import sys -import errno -import datetime -import subprocess -import time -import fileinput -import psycopg2 -from common_code import * -from cidb import * - -logger = logger_init() - -def liveboot_to_keep(name): - con = db_connect() - cur = con.cursor() - try: - cur.execute("SELECT keep_image FROM liveboot WHERE build_name = '%s' and keep_image = true" % (name)) - except psycopg2.DatabaseError as e: - logger.debug("SELECT keep_image FROM liveboot WHERE build_name = '%s' and keep_image = true" % (name)) - return False - if cur.fetchone(): - return True - -if __name__ == '__main__': - if len(sys.argv) != 2: - print("usage: %s $liveboot_name") - sys.exit(1) - name = sys.argv[1] - if not liveboot_to_keep(name): - sys.exit(1) - diff --git a/liveboot_garbage_collection.sh b/liveboot_garbage_collection.sh index be41ab3..f817473 100755 --- a/liveboot_garbage_collection.sh +++ b/liveboot_garbage_collection.sh @@ -8,7 +8,7 @@ set -e #export # no configuration below this block -AMOUNT_TO_KEEP=15 +AMOUNT_TO_KEEP=25 LIVEBOOT_IMAGE_PATH="/srv/mirror/liveboot" echo @@ -23,7 +23,7 @@ SCRIPT_PATH=$(dirname $0) FIRST_PASS=$(ls -1r $LIVEBOOT_IMAGE_PATH) SECOND_PASS="" for DIRECTORY in $FIRST_PASS ; do - if $SCRIPT_PATH/keep_liveboot.py "$DIRECTORY" ; then + if $SCRIPT_PATH/cleanup_liveboot.py "keep" "$DIRECTORY" ; then echo -n "$DIRECTORY: " echo keep flag set, keeping it. else @@ -37,8 +37,8 @@ for DIRECTORY in $SECOND_PASS ; do let "KEPT=KEPT+1" else echo "deleted." - echo rm -rf "$LIVEBOOT_IMAGE_PATH/$DIRECTORY" - echo FIXME: still need to remove the liveboot from cidb... + $SCRIPT_PATH/cleanup_liveboot.py "delete" "$DIRECTORY" + rm -rf "$LIVEBOOT_IMAGE_PATH/$DIRECTORY" fi done echo @@ -49,4 +49,5 @@ ls -1r $LIVEBOOT_IMAGE_PATH echo echo "Free diskspace on sagunt:" df -h $LIVEBOOT_IMAGE_PATH +echo echo "The liveboot images are copied to other servers which might not have as much space available." diff --git a/liveboot_rsync.sh b/liveboot_rsync.sh index d709251..d27d765 100755 --- a/liveboot_rsync.sh +++ b/liveboot_rsync.sh @@ -25,22 +25,12 @@ RSYNC_DST4="liveboot@10.252.1.1/liveboot/" # dc1+2 (mgmt1) RSYNC_DST5="liveboot@10.1.1.1/liveboot/" -# TODO: implement better garbage collection, allow to keep images -# TODO: needs to update the db too -# FIXME: locking # -# cleanup $RSYNC_SRC, keep MAX number of images +# check if another rsync is running # -MAX=15 -cd $RSYNC_SRC -COUNT=0 -for i in $(ls -atd1 ./liveboot-*) ; do - let COUNT=COUNT+1 - if [ $COUNT -gt $MAX ] ; then - sudo rm -rfv ./$i - else - echo "keeping $i" - fi +while [ $(ps fax | grep rsync | grep -v grep | grep $RSYNC_SRC 2>/dev/null|wc -l) -gt 0 ] ; do + echo "Another rsync run in progress, sleeping." + sleep 15 done # -- 2.39.5