From 00961466e35452d0c0fa47586a5b9f79a1a51fb1 Mon Sep 17 00:00:00 2001 From: Frank Brehm Date: Fri, 9 Mar 2018 12:34:32 +0100 Subject: [PATCH] Adding and using lib/webhooks/errors.py, lib/webhooks/obj.py and lib/webhooks/handler.py --- lib/webhooks/__init__.py | 2 +- lib/webhooks/base_app.py | 186 ++-------- lib/webhooks/common.py | 145 +++++++- lib/webhooks/errors.py | 204 +++++++++++ lib/webhooks/handler.py | 730 +++++++++++++++++++++++++++++++++++++++ lib/webhooks/obj.py | 328 ++++++++++++++++++ 6 files changed, 1442 insertions(+), 153 deletions(-) create mode 100644 lib/webhooks/errors.py create mode 100644 lib/webhooks/handler.py create mode 100644 lib/webhooks/obj.py diff --git a/lib/webhooks/__init__.py b/lib/webhooks/__init__.py index 9281c23..f7ee2b6 100644 --- a/lib/webhooks/__init__.py +++ b/lib/webhooks/__init__.py @@ -1,6 +1,6 @@ #!/bin/env python3 # -*- coding: utf-8 -*- -__version__ = '0.7.2' +__version__ = '0.8.1' # vim: ts=4 et list diff --git a/lib/webhooks/base_app.py b/lib/webhooks/base_app.py index 2c6978d..6c2925b 100644 --- a/lib/webhooks/base_app.py +++ b/lib/webhooks/base_app.py @@ -6,6 +6,7 @@ @copyright: © 2017 by Frank Brehm, Berlin @summary: The module for the base application object. """ +from __future__ import absolute_import # Standard modules import sys @@ -27,9 +28,14 @@ import yaml # Own modules import webhooks -from webhooks.common import pp, to_bytes, to_bool +from .common import pp, to_bytes, to_bool + +from .obj import BaseObjectError, BaseObject + +from .handler import BaseHandler __version__ = webhooks.__version__ + LOG = logging.getLogger(__name__) DEFAULT_FROM_EMAIL = 'puppetmaster@pixelpark.com' DEFAULT_FROM_SENDER = 'Puppetmaster <{}>'.format(DEFAULT_FROM_EMAIL) @@ -38,13 +44,21 @@ DEFAULT_TO_SENDER = 'Puppet <{}>'.format(DEFAULT_TO_EMAIL) # ============================================================================= -class BaseHookApp(object): +class BaseHookError(BaseObjectError): + """ + Base error class useable by all descendand objects. + """ + + pass + + +# ============================================================================= +class BaseHookApp(BaseObject): """ Base class for the application objects. """ cgi_bin_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) - base_dir = os.path.dirname(cgi_bin_dir) puppetlabs_cfg_dir = os.sep + os.path.join('etc', 'puppetlabs') puppet_envs_dir = os.path.join(puppetlabs_cfg_dir, 'code', 'environments') @@ -54,37 +68,21 @@ class BaseHookApp(object): dev_re = re.compile(r'^dev') # ------------------------------------------------------------------------- - def __init__(self, appname=None, verbose=0, version=__version__): + def __init__(self, appname=None, base_dir=None, verbose=0, version=__version__): """Constructor.""" + if not base_dir: + base_dir = os.path.dirname(self.cgi_bin_dir) + if not getattr(self, 'description', None): self.description = "Base gitlab webhook application." - self._appname = None - """ - @ivar: name of the current running application - @type: str - """ - if appname: - v = str(appname).strip() - if v: - self._appname = v - if not self._appname: - self._appname = os.path.basename(sys.argv[0]) - - self._version = version - """ - @ivar: version string of the current object or application - @type: str - """ - - self._verbose = verbose - """ - @ivar: verbosity level (0 - 9) - @type: int - """ - self._start_verbose = verbose + super(BaseHookApp, self).__init__( + appname=appname, verbose=verbose, version=version, + base_dir=base_dir, initialized=False, + ) + self._start_verbose = self.verbose self._simulate = False self.data = None @@ -125,38 +123,7 @@ class BaseHookApp(object): self.search_curl_bin() - # ----------------------------------------------------------- - @property - def appname(self): - """The name of the current running application.""" - return self._appname - - @appname.setter - def appname(self, value): - if value: - v = str(value).strip() - if v: - self._appname = v - - # ----------------------------------------------------------- - @property - def version(self): - """The version string of the current object or application.""" - return self._version - - # ----------------------------------------------------------- - @property - def verbose(self): - """The verbosity level.""" - return getattr(self, '_verbose', 0) - - @verbose.setter - def verbose(self, value): - v = int(value) - if v >= 0: - self._verbose = v - else: - LOG.warn("Wrong verbose level %r, must be >= 0", value) + return # ----------------------------------------------------------- @property @@ -196,19 +163,7 @@ class BaseHookApp(object): return os.path.join(self.puppet_envs_dir, cur_env) # ------------------------------------------------------------------------- - def __str__(self): - """ - Typecasting function for translating object structure - into a string - - @return: structure as string - @rtype: str - """ - - return pp(self.as_dict()) - - # ------------------------------------------------------------------------- - def as_dict(self): + def as_dict(self, short=True): """ Transforms the elements of the object into a dict @@ -216,16 +171,8 @@ class BaseHookApp(object): @rtype: dict """ - res = {} - for key in self.__dict__: - if key.startswith('_') and not key.startswith('__'): - continue - res[key] = self.__dict__[key] - res['__class_name__'] = self.__class__.__name__ - res['appname'] = self.appname - res['verbose'] = self.verbose + res = super(BaseHookApp, self).as_dict(short=short) res['simulate'] = self.simulate - res['base_dir'] = self.base_dir res['cgi_bin_dir'] = self.cgi_bin_dir res['log_directory'] = self.log_directory res['error_logfile'] = self.error_logfile @@ -296,81 +243,18 @@ class BaseHookApp(object): sys.stderr.write("\nSimulation mode - nothing is really done.\n\n") self.simulate = True - # ------------------------------------------------------------------------- - def get_cmd(self, cmd): - - if os.path.isabs(cmd): - if not os.path.exists(cmd): - LOG.error("Command {!r} does not exists.".format(cmd)) - return None - if not os.access(cmd, os.X_OK): - LOG.error("Command {!r} is not executable.".format(cmd)) - return None - return os.path.normpath(cmd) - - path_list = [] - cmd_abs = None - - search_path = os.environ.get('PATH', None) - if not search_path: - search_path = os.defpath - - search_path_list = [ - '/opt/pixelpark/bin', - '/opt/puppetlabs/puppet/bin', - '/www/bin', - ] - - search_path_list += search_path.split(os.pathsep) - - default_path = [ - '/usr/local/sbin', - '/usr/local/bin', - '/usr/sbin', - '/usr/bin', - '/sbin', - '/bin', - ] - search_path_list += default_path - - for d in search_path_list: - if not os.path.exists(d): - continue - if not os.path.isdir(d): - continue - d_abs = os.path.realpath(d) - if d_abs not in path_list: - path_list.append(d_abs) - - if self.verbose > 1: - LOG.debug("Searching for command {c!r} in:\n{p}".format( - c=cmd, p=pp(path_list))) - - for d in path_list: - p = os.path.join(d, cmd) - if os.path.exists(p): - if self.verbose > 2: - LOG.debug("Found {!r} ...".format(p)) - if os.access(p, os.X_OK): - cmd_abs = p - break - else: - LOG.debug("Command {!r} is not executable.".format(p)) - - if cmd_abs: - LOG.debug("Found {c!r} in {p!r}.".format(c=cmd, p=cmd_abs)) - else: - LOG.error("Command {!r} not found.".format(cmd)) - - return cmd_abs - # ------------------------------------------------------------------------- def search_curl_bin(self): - cmd = self.get_cmd('curl') + searcher = BaseHandler( + appname=self.appname, verbose=self.verbose, base_dir=self.base_dir) + + cmd = searcher.get_cmd('curl') + del searcher if not cmd: sys.exit(9) self.curl_bin = cmd + return # ------------------------------------------------------------------------- def read_config(self): diff --git a/lib/webhooks/common.py b/lib/webhooks/common.py index fef2002..4e17c40 100644 --- a/lib/webhooks/common.py +++ b/lib/webhooks/common.py @@ -14,15 +14,19 @@ import logging import re import pprint import platform +import locale # Own modules -__version__ = '0.2.0' +__version__ = '0.3.1' LOG = logging.getLogger(__name__) RE_YES = re.compile(r'^\s*(?:y(?:es)?|true)\s*$', re.IGNORECASE) RE_NO = re.compile(r'^\s*(?:no?|false|off)\s*$', re.IGNORECASE) +RE_B2H_FINAL_ZEROES = re.compile(r'0+$') +RE_B2H_FINAL_SIGNS = re.compile(r'\D+$') + # ============================================================================= def pp(value, indent=4, width=99, depth=None): @@ -183,6 +187,145 @@ def to_bool(value): return bool(value) +# ============================================================================= +def caller_search_path(): + """ + Builds a search path for executables from environment $PATH + including some standard paths. + + @return: all existing search paths + @rtype: list + """ + + path_list = [] + search_path = os.environ['PATH'] + if not search_path: + search_path = os.defpath + + search_path_list = [ + '/opt/pixelpark/bin', + '/opt/puppetlabs/puppet/bin', + '/www/bin', + '/opt/PPlocal/bin', + ] + + for d in search_path.split(os.pathsep): + search_path_list.append(d) + + default_path = [ + '/bin', + '/usr/bin', + '/usr/local/bin', + '/sbin', + '/usr/sbin', + '/usr/local/sbin', + '/usr/ucb', + '/usr/sfw/bin', + '/opt/csw/bin', + '/usr/openwin/bin', + '/usr/ccs/bin', + ] + + for d in default_path: + search_path_list.append(d) + + for d in search_path_list: + if not os.path.exists(d): + continue + if not os.path.isdir(d): + continue + d_abs = os.path.realpath(d) + if d_abs not in path_list: + path_list.append(d_abs) + + return path_list + + +# ============================================================================= +def bytes2human( + value, si_conform=False, precision=None, format_str='{value} {unit}'): + """ + Converts the given value in bytes into a human readable format. + The limit for electing the next higher prefix is at 1500. + + It raises a ValueError on invalid values. + + @param value: the value to convert + @type value: long + @param si_conform: use factor 1000 instead of 1024 for kB a.s.o., + if do so, than the units are for example MB instead MiB. + @type si_conform: bool + @param precision: how many digits after the decimal point have to stay + in the result + @type precision: int + @param format_str: a format string to format the result. + @type format_str: str + + @return: the value in a human readable format together with the unit + @rtype: str + + """ + + val = int(value) + + if not val: + return format_str.format(value=0, unit='Bytes') + + base = 1024 + prefixes = { + 1: 'KiB', + 2: 'MiB', + 3: 'GiB', + 4: 'TiB', + 5: 'PiB', + 6: 'EiB', + 7: 'ZiB', + 8: 'YiB', + } + if si_conform: + base = 1000 + prefixes = { + 1: 'kB', + 2: 'MB', + 3: 'GB', + 4: 'TB', + 5: 'PB', + 6: 'EB', + 7: 'ZB', + 8: 'YB', + } + + exponent = 0 + + float_val = float(val) + while float_val >= (2 * base) and exponent < 8: + float_val /= base + exponent += 1 + + unit = '' + if not exponent: + precision = None + unit = 'Bytes' + if val == 1: + unit = 'Byte' + value_str = locale.format_string('%d', val) + return format_str.format(value=value_str, unit=unit) + + unit = prefixes[exponent] + value_str = '' + if precision is None: + value_str = locale.format_string('%f', float_val) + value_str = RE_B2H_FINAL_ZEROES.sub('', value_str) + value_str = RE_B2H_FINAL_SIGNS.sub('', value_str) + else: + value_str = locale.format_string('%.*f', (precision, float_val)) + + if not exponent: + return value_str + + return format_str.format(value=value_str, unit=unit) + + # ============================================================================= if __name__ == "__main__": diff --git a/lib/webhooks/errors.py b/lib/webhooks/errors.py new file mode 100644 index 0000000..684b467 --- /dev/null +++ b/lib/webhooks/errors.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@author: Frank Brehm +@summary: module for some common used error classes +""" + +# Standard modules +import errno + + +__version__ = '0.1.1' + +# ============================================================================= +class PpError(Exception): + """ + Base error class for all other self defined exceptions. + """ + + pass + + +# ============================================================================= +class PpAppError(PpError): + + pass + + +# ============================================================================= +class InvalidMailAddressError(PpError): + """Class for a exception in case of a malformed mail address.""" + + # ------------------------------------------------------------------------- + def __init__(self, address, msg=None): + + self.address = address + self.msg = msg + + # ------------------------------------------------------------------------- + def __str__(self): + + msg = "Wrong mail address {a!r} ({c})".format( + a=self.address, c=self.address.__class__.__name__) + if self.msg: + msg += ': ' + self.msg + else: + msg += '.' + return msg + + +# ============================================================================= +class FunctionNotImplementedError(PpError, NotImplementedError): + """ + Error class for not implemented functions. + """ + + # ------------------------------------------------------------------------- + def __init__(self, function_name, class_name): + """ + Constructor. + + @param function_name: the name of the not implemented function + @type function_name: str + @param class_name: the name of the class of the function + @type class_name: str + + """ + + self.function_name = function_name + if not function_name: + self.function_name = '__unkown_function__' + + self.class_name = class_name + if not class_name: + self.class_name = '__unkown_class__' + + # ------------------------------------------------------------------------- + def __str__(self): + """ + Typecasting into a string for error output. + """ + + msg = "Function {func}() has to be overridden in class {cls!r}." + return msg.format(func=self.function_name, cls=self.class_name) + +# ============================================================================= +class IoTimeoutError(PpError, IOError): + """ + Special error class indicating a timout error on a read/write operation + """ + + # ------------------------------------------------------------------------- + def __init__(self, strerror, timeout, filename=None): + """ + Constructor. + + @param strerror: the error message about the operation + @type strerror: str + @param timeout: the timout in seconds leading to the error + @type timeout: float + @param filename: the filename leading to the error + @type filename: str + + """ + + t_o = None + try: + t_o = float(timeout) + except ValueError: + pass + self.timeout = t_o + + if t_o is not None: + strerror += " (timeout after {:0.1f} secs)".format(t_o) + + if filename is None: + super(IoTimeoutError, self).__init__(errno.ETIMEDOUT, strerror) + else: + super(IoTimeoutError, self).__init__( + errno.ETIMEDOUT, strerror, filename) + +# ============================================================================= +class ReadTimeoutError(IoTimeoutError): + """ + Special error class indicating a timout error on reading of a file. + """ + + # ------------------------------------------------------------------------- + def __init__(self, timeout, filename): + """ + Constructor. + + @param timeout: the timout in seconds leading to the error + @type timeout: float + @param filename: the filename leading to the error + @type filename: str + + """ + + strerror = "Timeout error on reading" + super(ReadTimeoutError, self).__init__(strerror, timeout, filename) + + +# ============================================================================= +class WriteTimeoutError(IoTimeoutError): + """ + Special error class indicating a timout error on a writing into a file. + """ + + # ------------------------------------------------------------------------- + def __init__(self, timeout, filename): + """ + Constructor. + + @param timeout: the timout in seconds leading to the error + @type timeout: float + @param filename: the filename leading to the error + @type filename: str + + """ + + strerror = "Timeout error on writing" + super(WriteTimeoutError, self).__init__(strerror, timeout, filename) + +# ============================================================================= +class CouldntOccupyLockfileError(PpError): + """ + Special error class indicating, that a lockfile couldn't coccupied + after a defined time. + """ + + # ----------------------------------------------------- + def __init__(self, lockfile, duration, tries): + """ + Constructor. + + @param lockfile: the lockfile, which could't be occupied. + @type lockfile: str + @param duration: The duration in seconds, which has lead to this situation + @type duration: float + @param tries: the number of tries creating the lockfile + @type tries: int + + """ + + self.lockfile = str(lockfile) + self.duration = float(duration) + self.tries = int(tries) + + # ----------------------------------------------------- + def __str__(self): + + return "Couldn't occupy lockfile {!r} in {:0.1f} seconds with {} tries.".format( + self.lockfile, self.duration, self.tries) + + +# ============================================================================= + +if __name__ == "__main__": + pass + +# ============================================================================= + +# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 diff --git a/lib/webhooks/handler.py b/lib/webhooks/handler.py new file mode 100644 index 0000000..e2fee78 --- /dev/null +++ b/lib/webhooks/handler.py @@ -0,0 +1,730 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@author: Frank Brehm +@contact: frank.brehm@pixelpark.com +@copyright: © 2018 by Frank Brehm, Publicies Pixelpark GmbH, Berlin +@summary: A base handler module for a handler object, that can call or spawn + OS commands and read and write files. +""" +from __future__ import absolute_import + +# Standard modules +import os +import logging +import subprocess +import pwd +import signal +import errno +import locale +import time +import pipes +from fcntl import fcntl, F_GETFL, F_SETFL + +# Third party modules +import six + +# Own modules +from .common import caller_search_path + +from .errors import ReadTimeoutError, WriteTimeoutError + +from .obj import BaseObjectError +from .obj import BaseObject + +__version__ = '0.1.1' + +log = logging.getLogger(__name__) + + +# Some module varriables +CHOWN_CMD = os.sep + os.path.join('bin', 'chown') +ECHO_CMD = os.sep + os.path.join('bin', 'echo') +SUDO_CMD = os.sep + os.path.join('usr', 'bin', 'sudo') + + +# ============================================================================= +class BaseHandlerError(BaseObjectError): + """Base error class for all exceptions happened during + execution this application""" + + pass + + +# ============================================================================= +class CommandNotFoundError(BaseHandlerError): + """ + Special exception, if one ore more OS commands were not found. + + """ + + # ------------------------------------------------------------------------- + def __init__(self, cmd_list): + """ + Constructor. + + @param cmd_list: all not found OS commands. + @type cmd_list: list + + """ + + self.cmd_list = None + if cmd_list is None: + self.cmd_list = ["Unknown OS command."] + elif isinstance(cmd_list, list): + self.cmd_list = cmd_list + else: + self.cmd_list = [cmd_list] + + if len(self.cmd_list) < 1: + raise ValueError("Empty command list given.") + + # ------------------------------------------------------------------------- + def __str__(self): + """ + Typecasting into a string for error output. + """ + + cmds = ', '.join([("'" + str(x) + "'") for x in self.cmd_list]) + msg = "Could not found OS command" + if len(self.cmd_list) != 1: + msg += 's' + msg += ": " + cmds + return msg + + +# ============================================================================= +class BaseHandler(BaseObject): + """ + Base class for handler objects. + """ + + # ------------------------------------------------------------------------- + def __init__( + self, appname=None, verbose=0, version=__version__, base_dir=None, + initialized=None, simulate=False, sudo=False, quiet=False): + """ + Initialisation of the base handler object. + + @raise CommandNotFoundError: if the commands 'chmod', 'echo' and + 'sudo' could not be found. + @raise BaseHandlerError: on a uncoverable error. + + @param appname: name of the current running application + @type appname: str + @param verbose: verbose level + @type verbose: int + @param version: the version string of the current object or application + @type version: str + @param base_dir: the base directory of all operations + @type base_dir: str + @param initialized: initialisation is complete after __init__() + of this object + @type initialized: bool + @param simulate: don't execute actions, only display them + @type simulate: bool + @param sudo: should the command executed by sudo by default + @type sudo: bool + @param quiet: don't display ouput of action after calling + @type quiet: bool + + @return: None + """ + + super(BaseHandler, self).__init__( + appname=appname, verbose=verbose, version=version, + base_dir=base_dir, initialized=False, + ) + + failed_commands = [] + + self._simulate = bool(simulate) + """ + @ivar: don't execute actions, only display them + @type: bool + """ + + self._quiet = quiet + """ + @ivar: don't display ouput of action after calling + (except output on STDERR) + @type: bool + """ + + self._sudo = sudo + """ + @ivar: should the command executed by sudo by default + @type: bool + """ + + self._chown_cmd = CHOWN_CMD + """ + @ivar: the chown command for changing ownership of file objects + @type: str + """ + if not os.path.exists(self.chown_cmd) or not os.access( + self.chown_cmd, os.X_OK): + self._chown_cmd = self.get_command('chown') + if not self.chown_cmd: + failed_commands.append('chown') + + self._echo_cmd = ECHO_CMD + """ + @ivar: the echo command for simulating execution + @type: str + """ + if not os.path.exists(self.echo_cmd) or not os.access( + self.echo_cmd, os.X_OK): + self._echo_cmd = self.get_command('echo') + if not self.echo_cmd: + failed_commands.append('echo') + + self._sudo_cmd = SUDO_CMD + """ + @ivar: the sudo command for execute commands as root + @type: str + """ + if not os.path.exists(self.sudo_cmd) or not os.access( + self._sudo_cmd, os.X_OK): + self.sudo_cmd = self.get_command('sudo') + if not self.sudo_cmd: + failed_commands.append('sudo') + + # Some commands are missing + if failed_commands: + raise CommandNotFoundError(failed_commands) + + if initialized is None: + self.initialized = True + else: + self.initialized = initialized + if self.verbose > 3: + log.debug("Initialized.") + + # ----------------------------------------------------------- + @property + def simulate(self): + """Simulation mode.""" + return self._simulate + + @simulate.setter + def simulate(self, value): + self._simulate = bool(value) + + # ----------------------------------------------------------- + @property + def quiet(self): + """Don't display ouput of action after calling.""" + return self._quiet + + @quiet.setter + def quiet(self, value): + self._quiet = bool(value) + + # ----------------------------------------------------------- + @property + def sudo(self): + """Should the command executed by sudo by default.""" + return self._sudo + + @sudo.setter + def sudo(self, value): + self._sudo = bool(value) + + # ----------------------------------------------------------- + @property + def chown_cmd(self): + """The absolute path to the OS command 'chown'.""" + return self._chown_cmd + + # ----------------------------------------------------------- + @property + def echo_cmd(self): + """The absolute path to the OS command 'echo'.""" + return self._echo_cmd + + # ----------------------------------------------------------- + @property + def sudo_cmd(self): + """The absolute path to the OS command 'sudo'.""" + return self._sudo_cmd + + # ------------------------------------------------------------------------- + def as_dict(self, short=True): + """ + Transforms the elements of the object into a dict + + @param short: don't include local properties in resulting dict. + @type short: bool + + @return: structure as dict + @rtype: dict + """ + + res = super(BaseHandler, self).as_dict(short=short) + res['simulate'] = self.simulate + res['quiet'] = self.quiet + res['sudo'] = self.sudo + res['chown_cmd'] = self.chown_cmd + res['echo_cmd'] = self.echo_cmd + res['sudo_cmd'] = self.sudo_cmd + + return res + + # ------------------------------------------------------------------------- + def __repr__(self): + """Typecasting into a string for reproduction.""" + + out = "<%s(" % (self.__class__.__name__) + + fields = [] + fields.append("appname=%r" % (self.appname)) + fields.append("verbose=%r" % (self.verbose)) + fields.append("version=%r" % (self.version)) + fields.append("base_dir=%r" % (self.base_dir)) + fields.append("initialized=%r" % (self.initialized)) + fields.append("simulate=%r" % (self.simulate)) + fields.append("sudo=%r" % (self.sudo)) + fields.append("quiet=%r" % (self.quiet)) + + out += ", ".join(fields) + ")>" + return out + + # ------------------------------------------------------------------------- + def get_command(self, cmd, quiet=False): + """ + Searches the OS search path for the given command and gives back the + normalized position of this command. + If the command is given as an absolute path, it check the existence + of this command. + + @param cmd: the command to search + @type cmd: str + @param quiet: No warning message, if the command could not be found, + only a debug message + @type quiet: bool + + @return: normalized complete path of this command, or None, + if not found + @rtype: str or None + + """ + + if self.verbose > 2: + log.debug("Searching for command {!r} ...".format(cmd)) + + # Checking an absolute path + if os.path.isabs(cmd): + if not os.path.exists(cmd): + log.warning("Command {!r} doesn't exists.".format(cmd)) + return None + if not os.access(cmd, os.X_OK): + log.warning("Command {!r} is not executable.".format(cmd)) + return None + return os.path.normpath(cmd) + + # Checking a relative path + for d in caller_search_path(): + if self.verbose > 3: + log.debug("Searching command in {!r} ...".format(d)) + p = os.path.join(d, cmd) + if os.path.exists(p): + if self.verbose > 2: + log.debug("Found {!r} ...".format(p)) + if os.access(p, os.X_OK): + return os.path.normpath(p) + else: + log.debug("Command {!r} is not executable.".format(p)) + + # command not found, sorry + if quiet: + if self.verbose > 2: + log.debug("Command {!r} not found.".format(cmd)) + else: + log.warning("Command {!r} not found.".format(cmd)) + + return None + + # ------------------------------------------------------------------------- + def call( + self, cmd, sudo=None, simulate=None, quiet=None, shell=False, + stdout=None, stderr=None, bufsize=0, drop_stderr=False, + close_fds=False, hb_handler=None, hb_interval=2.0, + poll_interval=0.2, **kwargs): + """ + Executing a OS command. + + @param cmd: the cmd you wanne call + @type cmd: list of strings or str + @param sudo: execute the command with sudo + @type sudo: bool (or none, if self.sudo will be be asked) + @param simulate: simulate execution or not, + if None, self.simulate will asked + @type simulate: bool or None + @param quiet: quiet execution independend of self.quiet + @type quiet: bool + @param shell: execute the command with a shell + @type shell: bool + @param stdout: file descriptor for stdout, + if not given, self.stdout is used + @type stdout: int + @param stderr: file descriptor for stderr, + if not given, self.stderr is used + @type stderr: int + @param bufsize: size of the buffer for stdout + @type bufsize: int + @param drop_stderr: drop all output on stderr, independend + of any value of stderr + @type drop_stderr: bool + @param close_fds: closing all open file descriptors + (except 0, 1 and 2) on calling subprocess.Popen() + @type close_fds: bool + @param kwargs: any optional named parameter (must be one + of the supported suprocess.Popen arguments) + @type kwargs: dict + + @return: tuple of:: + - return value of calling process, + - output on STDOUT, + - output on STDERR + + """ + + cmd_list = cmd + if isinstance(cmd, str): + cmd_list = [cmd] + + pwd_info = pwd.getpwuid(os.geteuid()) + + if sudo is None: + sudo = self.sudo + if sudo: + cmd_list.insert(0, self.sudo_cmd) + + if simulate is None: + simulate = self.simulate + if simulate: + cmd_list.insert(0, self.echo_cmd) + quiet = False + + if quiet is None: + quiet = self.quiet + + use_shell = bool(shell) + + cmd_list = [str(element) for element in cmd_list] + cmd_str = ' '.join(map(lambda x: pipes.quote(x), cmd_list)) + + if not quiet or self.verbose > 1: + log.debug("Executing: {}".format(cmd_list)) + + if quiet and self.verbose > 1: + log.debug("Quiet execution") + + used_stdout = subprocess.PIPE + if stdout is not None: + used_stdout = stdout + + used_stderr = subprocess.PIPE + if drop_stderr: + used_stderr = None + elif stderr is not None: + used_stderr = stderr + + cur_locale = locale.getlocale() + cur_encoding = cur_locale[1] + if (cur_locale[1] is None or cur_locale[1] == '' or + cur_locale[1].upper() == 'C' or + cur_locale[1].upper() == 'POSIX'): + cur_encoding = 'UTF-8' + + cmd_obj = subprocess.Popen( + cmd_list, + shell=use_shell, + cwd=self.base_dir, + close_fds=close_fds, + stderr=used_stderr, + stdout=used_stdout, + bufsize=bufsize, + env={'USER': pwd_info.pw_name}, + **kwargs + ) + + # Display Output of executable + stdoutdata = '' + stderrdata = '' + if six.PY3: + stdoutdata = bytearray() + stderrdata = bytearray() + + if hb_handler is not None: + + if not quiet or self.verbose > 1: + log.debug(( + "Starting asynchronous communication with '{cmd}', " + "heartbeat interval is {interval:0.1f} seconds.").format( + cmd=cmd_str, interval=hb_interval)) + + out_flags = fcntl(cmd_obj.stdout, F_GETFL) + err_flags = fcntl(cmd_obj.stderr, F_GETFL) + fcntl(cmd_obj.stdout, F_SETFL, out_flags | os.O_NONBLOCK) + fcntl(cmd_obj.stderr, F_SETFL, err_flags | os.O_NONBLOCK) + + start_time = time.time() + + while True: + + if self.verbose > 3: + log.debug("Checking for the end of the communication ...") + if cmd_obj.poll() is not None: + cmd_obj.wait() + break + + # Heartbeat handling ... + cur_time = time.time() + time_diff = cur_time - start_time + if time_diff >= hb_interval: + if not quiet or self.verbose > 1: + log.debug("Time to execute the heartbeat handler.") + if hb_handler: + hb_handler() + start_time = cur_time + if self.verbose > 3: + log.debug("Sleeping {:0.2f} seconds ...".format(poll_interval)) + time.sleep(poll_interval) + + # Reading out file descriptors + if used_stdout is not None: + try: + stdoutdata += os.read(cmd_obj.stdout.fileno(), 1024) + if self.verbose > 3: + log.debug(" stdout is now: {!r}".format(stdoutdata)) + except OSError: + pass + if used_stderr is not None: + try: + stderrdata += os.read(cmd_obj.stderr.fileno(), 1024) + if self.verbose > 3: + log.debug(" stderr is now: {!r}".format(stderrdata)) + except OSError: + pass + else: + if not quiet or self.verbose > 1: + log.debug("Starting synchronous communication with '{}'.".format(cmd_str)) + (stdoutdata, stderrdata) = cmd_obj.communicate() + + if not quiet or self.verbose > 1: + log.debug("Finished communication with '{}'.".format(cmd_str)) + + if stderrdata: + if six.PY3: + if self.verbose > 2: + log.debug("Decoding {what} from {enc!r}.".format( + what='STDERR', enc=cur_encoding)) + stderrdata = stderrdata.decode(cur_encoding) + if quiet and not self.verbose: + pass + else: + msg = "Output on {where}: {what!r}.".format( + where="STDERR", what=stderrdata.strip()) + if quiet: + log.debug(msg) + else: + self.handle_error(msg, self.appname) + + if stdoutdata: + if six.PY3: + if self.verbose > 2: + log.debug("Decoding {what} from {enc!r}.".format( + what='STDOUT', enc=cur_encoding)) + stdoutdata = stdoutdata.decode(cur_encoding) + do_out = False + if self.verbose: + if quiet: + if self.verbose > 3: + do_out = True + else: + do_out = False + else: + do_out = True + else: + if not quiet: + do_out = True + if do_out: + msg = "Output on {where}: {what!r}.".format( + where="STDOUT", what=stderrdata.strip()) + log.debug(msg) + + ret = cmd_obj.wait() + if not quiet or self.verbose > 1: + log.debug("Returncode: {}".format(ret)) + + return (ret, stdoutdata, stderrdata) + + # ------------------------------------------------------------------------- + def read_file(self, filename, timeout=2, quiet=False): + """ + Reads the content of the given filename. + + @raise IOError: if file doesn't exists or isn't readable + @raise ReadTimeoutError: on timeout reading the file + + @param filename: name of the file to read + @type filename: str + @param timeout: the amount in seconds when this method should timeout + @type timeout: int + @param quiet: increases the necessary verbosity level to + put some debug messages + @type quiet: bool + + @return: file content + @rtype: str + + """ + + needed_verbose_level = 1 + if quiet: + needed_verbose_level = 3 + + def read_alarm_caller(signum, sigframe): + ''' + This nested function will be called in event of a timeout + + @param signum: the signal number (POSIX) which happend + @type signum: int + @param sigframe: the frame of the signal + @type sigframe: object + ''' + + raise ReadTimeoutError(timeout, filename) + + timeout = abs(int(timeout)) + + if not os.path.isfile(filename): + raise IOError( + errno.ENOENT, "File doesn't exists.", filename) + if not os.access(filename, os.R_OK): + raise IOError( + errno.EACCES, 'Read permission denied.', filename) + + if self.verbose > needed_verbose_level: + log.debug("Reading file content of {!r} ...".format(filename)) + + signal.signal(signal.SIGALRM, read_alarm_caller) + signal.alarm(timeout) + + content = '' + fh = open(filename, 'r') + for line in fh.readlines(): + content += line + fh.close() + + signal.alarm(0) + + return content + + # ------------------------------------------------------------------------- + def write_file(self, filename, content, timeout=2, must_exists=True, quiet=False): + """ + Writes the given content into the given filename. + It should only be used for small things, because it writes unbuffered. + + @raise IOError: if file doesn't exists or isn't writeable + @raise WriteTimeoutError: on timeout writing into the file + + @param filename: name of the file to write + @type filename: str + @param content: the content to write into the file + @type content: str + @param timeout: the amount in seconds when this method should timeout + @type timeout: int + @param must_exists: the file must exists before writing + @type must_exists: bool + @param quiet: increases the necessary verbosity level to + put some debug messages + @type quiet: bool + + @return: None + + """ + + def write_alarm_caller(signum, sigframe): + ''' + This nested function will be called in event of a timeout + + @param signum: the signal number (POSIX) which happend + @type signum: int + @param sigframe: the frame of the signal + @type sigframe: object + ''' + + raise WriteTimeoutError(timeout, filename) + + verb_level1 = 0 + verb_level2 = 1 + verb_level3 = 3 + if quiet: + verb_level1 = 2 + verb_level2 = 3 + verb_level3 = 4 + + timeout = int(timeout) + + if must_exists: + if not os.path.isfile(filename): + raise IOError(errno.ENOENT, "File doesn't exists.", filename) + + if os.path.exists(filename): + if not os.access(filename, os.W_OK): + if self.simulate: + log.error("Write permission to {!r} denied.".format(filename)) + else: + raise IOError(errno.EACCES, 'Write permission denied.', filename) + else: + parent_dir = os.path.dirname(filename) + if not os.access(parent_dir, os.W_OK): + if self.simulate: + log.error("Write permission to {!r} denied.".format(parent_dir)) + else: + raise IOError(errno.EACCES, 'Write permission denied.', parent_dir) + + if self.verbose > verb_level1: + if self.verbose > verb_level2: + log.debug("Write {what!r} into {to!r}.".format( + what=content, to=filename)) + else: + log.debug("Writing {!r} ...".format(filename)) + + if self.simulate: + if self.verbose > verb_level2: + log.debug("Simulating write into {!r}.".format(filename)) + return + + signal.signal(signal.SIGALRM, write_alarm_caller) + signal.alarm(timeout) + + # Open filename for writing unbuffered + if self.verbose > verb_level3: + log.debug("Opening {!r} for write unbuffered ...".format(filename)) + fh = open(filename, 'w', 0) + + try: + fh.write(content) + finally: + if self.verbose > verb_level3: + log.debug("Closing {!r} ...".format(filename)) + fh.close() + + signal.alarm(0) + + return + +# ============================================================================= + +if __name__ == "__main__": + + pass + +# ============================================================================= + +# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 list diff --git a/lib/webhooks/obj.py b/lib/webhooks/obj.py new file mode 100644 index 0000000..9b850aa --- /dev/null +++ b/lib/webhooks/obj.py @@ -0,0 +1,328 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@author: Frank Brehm +@contact: frank.brehm@pixelpark.com +@copyright: © 2018 by Frank Brehm, Publicies Pixelpark GmbH, Berlin +""" +from __future__ import absolute_import + +# Standard modules +import sys +import os +import logging +import datetime +import traceback + +# Third party modules + +# Own modules +from .common import pp, to_bytes + +from .errors import PpError + +__version__ = '0.2.5' + +LOG = logging.getLogger(__name__) + + +# ============================================================================= +class BaseObjectError(PpError): + """ + Base error class useable by all descendand objects. + """ + + pass + + +# ============================================================================= +class BaseObject(object): + """ + Base class for all objects. + """ + + # ------------------------------------------------------------------------- + def __init__( + self, appname=None, verbose=0, version=__version__, base_dir=None, + initialized=False): + """ + Initialisation of the base object. + + Raises an exception on a uncoverable error. + """ + + self._appname = None + """ + @ivar: name of the current running application + @type: str + """ + if appname: + v = str(appname).strip() + if v: + self._appname = v + if not self._appname: + self._appname = os.path.basename(sys.argv[0]) + + self._version = version + """ + @ivar: version string of the current object or application + @type: str + """ + + self._verbose = int(verbose) + """ + @ivar: verbosity level (0 - 9) + @type: int + """ + if self._verbose < 0: + msg = "Wrong verbose level {!r}, must be >= 0".format(verbose) + raise ValueError(msg) + + self._initialized = False + """ + @ivar: initialisation of this object is complete + after __init__() of this object + @type: bool + """ + + self._base_dir = base_dir + """ + @ivar: base directory used for different purposes, must be an existent + directory. Defaults to directory of current script daemon.py. + @type: str + """ + if base_dir: + if not os.path.exists(base_dir): + msg = "Base directory {!r} does not exists.".format(base_dir) + self.handle_error(msg) + self._base_dir = None + elif not os.path.isdir(base_dir): + msg = "Base directory {!r} is not a directory.".format(base_dir) + self.handle_error(msg) + self._base_dir = None + if not self._base_dir: + self._base_dir = os.path.dirname(os.path.abspath(sys.argv[0])) + + self._initialized = bool(initialized) + + # ----------------------------------------------------------- + @property + def appname(self): + """The name of the current running application.""" + if hasattr(self, '_appname'): + return self._appname + return os.path.basename(sys.argv[0]) + + @appname.setter + def appname(self, value): + if value: + v = str(value).strip() + if v: + self._appname = v + + # ----------------------------------------------------------- + @property + def version(self): + """The version string of the current object or application.""" + return getattr(self, '_version', __version__) + + # ----------------------------------------------------------- + @property + def verbose(self): + """The verbosity level.""" + return getattr(self, '_verbose', 0) + + @verbose.setter + def verbose(self, value): + v = int(value) + if v >= 0: + self._verbose = v + else: + LOG.warn("Wrong verbose level {!r}, must be >= 0".format(value)) + + # ----------------------------------------------------------- + @property + def initialized(self): + """The initialisation of this object is complete.""" + return getattr(self, '_initialized', False) + + @initialized.setter + def initialized(self, value): + self._initialized = bool(value) + + # ----------------------------------------------------------- + @property + def base_dir(self): + """The base directory used for different purposes.""" + return self._base_dir + + @base_dir.setter + def base_dir(self, value): + if value.startswith('~'): + value = os.path.expanduser(value) + if not os.path.exists(value): + msg = "Base directory {!r} does not exists.".format(value) + LOG.error(msg) + elif not os.path.isdir(value): + msg = "Base directory {!r} is not a directory.".format(value) + LOG.error(msg) + else: + self._base_dir = value + + # ------------------------------------------------------------------------- + def __str__(self): + """ + Typecasting function for translating object structure + into a string + + @return: structure as string + @rtype: str + """ + + return pp(self.as_dict(short=True)) + + # ------------------------------------------------------------------------- + def __repr__(self): + """Typecasting into a string for reproduction.""" + + out = "<%s(" % (self.__class__.__name__) + + fields = [] + fields.append("appname={!r}".format(self.appname)) + fields.append("verbose={!r}".format(self.verbose)) + fields.append("version={!r}".format(self.version)) + fields.append("base_dir={!r}".format(self.base_dir)) + fields.append("initialized={!r}".format(self.initialized)) + + out += ", ".join(fields) + ")>" + return out + + # ------------------------------------------------------------------------- + def as_dict(self, short=True): + """ + Transforms the elements of the object into a dict + + @param short: don't include local properties in resulting dict. + @type short: bool + + @return: structure as dict + @rtype: dict + """ + + res = self.__dict__ + res = {} + for key in self.__dict__: + if short and key.startswith('_') and not key.startswith('__'): + continue + val = self.__dict__[key] + if isinstance(val, BaseObject): + res[key] = val.as_dict(short=short) + else: + res[key] = val + res['__class_name__'] = self.__class__.__name__ + res['appname'] = self.appname + res['version'] = self.version + res['verbose'] = self.verbose + res['initialized'] = self.initialized + res['base_dir'] = self.base_dir + + return res + + # ------------------------------------------------------------------------- + def handle_error( + self, error_message=None, exception_name=None, do_traceback=False): + """ + Handle an error gracefully. + + Print a traceback and continue. + + @param error_message: the error message to display + @type error_message: str + @param exception_name: name of the exception class + @type exception_name: str + @param do_traceback: allways show a traceback + @type do_traceback: bool + + """ + + msg = 'Exception happened: ' + if exception_name is not None: + exception_name = exception_name.strip() + if exception_name: + msg = exception_name + ': ' + else: + msg = '' + if error_message: + msg += str(error_message) + else: + msg += 'undefined error.' + + root_log = logging.getLogger() + has_handlers = False + if root_log.handlers: + has_handlers = True + + if has_handlers: + LOG.error(msg) + if do_traceback: + LOG.error(traceback.format_exc()) + else: + curdate = datetime.datetime.now() + curdate_str = "[" + curdate.isoformat(' ') + "]: " + msg = curdate_str + msg + "\n" + if hasattr(sys.stderr, 'buffer'): + sys.stderr.buffer.write(to_bytes(msg)) + else: + sys.stderr.write(msg) + if do_traceback: + traceback.print_exc() + + return + + # ------------------------------------------------------------------------- + def handle_info(self, message, info_name=None): + """ + Shows an information. This happens both to STDERR and to all + initialized log handlers. + + @param message: the info message to display + @type message: str + @param info_name: Title of information + @type info_name: str + + """ + + msg = '' + if info_name is not None: + info_name = info_name.strip() + if info_name: + msg = info_name + ': ' + msg += str(message).strip() + + root_log = logging.getLogger() + has_handlers = False + if root_log.handlers: + has_handlers = True + + if has_handlers: + LOG.info(msg) + else: + curdate = datetime.datetime.now() + curdate_str = "[" + curdate.isoformat(' ') + "]: " + msg = curdate_str + msg + "\n" + if hasattr(sys.stderr, 'buffer'): + sys.stderr.buffer.write(to_bytes(msg)) + else: + sys.stderr.write(msg) + + return + +# ============================================================================= + +if __name__ == "__main__": + + pass + +# ============================================================================= + +# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 -- 2.39.5