From fa7f0e8a2fe944adafd684d1180597f1efeb0389 Mon Sep 17 00:00:00 2001 From: Frank Brehm Date: Thu, 23 Aug 2018 12:00:24 +0200 Subject: [PATCH] Starting with get-forge-modules --- get-forge-modules | 29 ++++++ lib/webhooks/base_app.py | 59 ++++++++--- lib/webhooks/get_forge_modules.py | 157 ++++++++++++++++++++++++++++++ 3 files changed, 230 insertions(+), 15 deletions(-) create mode 100755 get-forge-modules create mode 100644 lib/webhooks/get_forge_modules.py diff --git a/get-forge-modules b/get-forge-modules new file mode 100755 index 0000000..e9a3073 --- /dev/null +++ b/get-forge-modules @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Standard modules +import os +import sys +import logging + +# own modules: +basedir = os.path.abspath(os.path.dirname(__file__)) +libdir = os.path.join(basedir, 'lib') + +sys.path.insert(0, libdir) + +from webhooks.get_forge_modules import GetForgeModulesApp + +MY_APPNAME = os.path.basename(sys.argv[0]) +LOG = logging.getLogger(MY_APPNAME) + +app = GetForgeModulesApp(appname=MY_APPNAME) + +if app.verbose > 2: + LOG.debug("{c} object:\n{o}".format(c=app.__class__.__name__, o=app)) + +app() + +sys.exit(0) + +# vim: ts=4 et diff --git a/lib/webhooks/base_app.py b/lib/webhooks/base_app.py index f7a2f90..6f413e3 100644 --- a/lib/webhooks/base_app.py +++ b/lib/webhooks/base_app.py @@ -79,6 +79,9 @@ class BaseHookApp(BaseObject): if not getattr(self, 'description', None): self.description = "Base gitlab webhook application." + self.data_dir = self.default_data_dir + self._read_stdin = True + super(BaseHookApp, self).__init__( appname=appname, verbose=verbose, version=version, base_dir=base_dir, initialized=False, @@ -95,7 +98,6 @@ class BaseHookApp(BaseObject): self.full_name = None self.git_ssh_url = None self.do_sudo = True - self.data_dir = self.default_data_dir self.cmdline_args = None @@ -132,6 +134,8 @@ class BaseHookApp(BaseObject): appname=self.appname, verbose=self.verbose, base_dir=self.base_dir, simulate=self.simulate) + self.post_init() + return # ----------------------------------------------------------- @@ -144,6 +148,16 @@ class BaseHookApp(BaseObject): def simulate(self, value): self._simulate = to_bool(value) + # ----------------------------------------------------------- + @property + def read_stdin(self): + """Flag, tzhat STDIN shoud be read.""" + return getattr(self, '_read_stdin', True) + + @read_stdin.setter + def read_stdin(self, value): + self._read_stdin = to_bool(value) + # ----------------------------------------------------------- @property def log_directory(self): @@ -191,6 +205,7 @@ class BaseHookApp(BaseObject): res['env_dir'] = self.env_dir res['fileserver_dir'] = self.fileserver_dir res['default_data_dir'] = self.default_data_dir + res['read_stdin'] = self.read_stdin return res @@ -457,6 +472,14 @@ class BaseHookApp(BaseObject): if flush: file.flush() + # ------------------------------------------------------------------------- + def post_init(self): + """ Dummy function, which is called after initialization of the application object. + May be overridden in child classes. + """ + + pass + # ------------------------------------------------------------------------- def __call__(self): """Helper method to make the resulting object callable.""" @@ -474,7 +497,10 @@ class BaseHookApp(BaseObject): if self.verbose > 2: LOG.debug("Encoding of STDIN now: {!r}".format(input_stream.encoding)) - self.data = input_stream.read() + if self.read_stdin: + self.data = input_stream.read() + else: + self.data = '{}' try: self.json_data = json.loads(self.data) if self.verbose > 1: @@ -483,8 +509,14 @@ class BaseHookApp(BaseObject): try: if self.pre_run(): self.run() + except BaseHookError as e: + msg = "Got a {n} performing {a}: {e}".format( + n=e.__class__.__name__, a=self.appname, e=e) + self.error_data.append(msg) + LOG.error(msg) except Exception as e: - msg = "Got a {n} performing the deploy: {e}".format(n=e.__class__.__name__, e=e) + msg = "Got a {n} performing {a}: {e}".format( + n=e.__class__.__name__, a=self.appname, e=e) msg += "\n\nTraceback:\n{}".format(traceback.format_exc()) self.error_data.append(msg) LOG.error(msg) @@ -504,6 +536,9 @@ class BaseHookApp(BaseObject): # ------------------------------------------------------------------------- def pre_run(self): + if not self.json_data and not self.read_stdin: + return True + self.ref = self.json_data['ref'].split('/')[-1] self.namespace = self.json_data['project']['namespace'] self.name = self.json_data['project']['name'] @@ -559,22 +594,16 @@ class BaseHookApp(BaseObject): LOG.debug("Checking availibility of data directory {!r} ...".format(self.data_dir)) if not os.path.exists(self.data_dir): - msg = "Data directory {!r} does not exists.".format(self.data_dir) - LOG.error(msg) - self.print_out(msg + "\n") - return False + raise BaseHookError( + "Data directory {!r} does not exists.".format(self.data_dir)) if not os.path.isdir(self.data_dir): - msg = "Path for data directory {!r} is not a directory.".format(self.data_dir) - LOG.error(msg) - self.print_out(msg + "\n") - return False + raise BaseHookError( + "Path for data directory {!r} is not a directory.".format(self.data_dir)) if not os.access(self.data_dir, os.W_OK): - msg = "Data directory {!r} is not writeable.".format(self.data_dir) - LOG.error(msg) - self.print_out(msg + "\n") - return False + raise BaseHookError( + "Data directory {!r} is not writeable.".format(self.data_dir)) return True diff --git a/lib/webhooks/get_forge_modules.py b/lib/webhooks/get_forge_modules.py new file mode 100644 index 0000000..328911c --- /dev/null +++ b/lib/webhooks/get_forge_modules.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@author: Frank Brehm +@contact: frank.brehm@pixelpark.com +@copyright: © 2017 by Frank Brehm, Berlin +@summary: The module for the deploy application object. +""" + +# Standard modules +import os +import logging +import textwrap +import copy +import pipes +import json +import re +import glob + +# Third party modules +import six +import yaml +import requests + +# Own modules +import webhooks + +from webhooks.common import pp, to_str + +from webhooks.base_app import BaseHookError, BaseHookApp + +__version__ = webhooks.__version__ +LOG = logging.getLogger(__name__) + + +# ============================================================================= +class GetForgeModulesError(BaseHookError): + + pass + + +# ============================================================================= +class GetForgeModulesApp(BaseHookApp): + """ + Class for the application objects. + """ + + default_forge_uri = 'https://forgeapi.puppet.com/v3/modules' + default_puppet_root_env_dir = os.sep + os.path.join('etc', 'puppetlabs', 'code', 'environments') + default_http_timeout = 30 + + open_args = {} + if six.PY3: + open_args = { + 'encoding': 'utf-8', + 'errors': 'surrogateescape', + } + + # ------------------------------------------------------------------------- + def __init__(self, appname=None, verbose=0, version=__version__): + """Constructor.""" + + self.cache_file = None + self.environments = [] + self.puppet_root_env_dir = self.default_puppet_root_env_dir + + self.description = textwrap.dedent('''\ + Collects information about all used Puppet Modules from {url} + and cache those information in {cdir!r}. + ''').strip().format( + url=self.default_forge_uri, cdir=self.default_data_dir) + + super(GetForgeModulesApp, self).__init__( + appname=appname, verbose=verbose, version=version) + + # ------------------------------------------------------------------------- + def post_init(self): + + self.read_stdin = False + self.cache_file = os.path.join(self.data_dir, self.appname + '.yaml') + + + self.initialized = True + + # ------------------------------------------------------------------------- + def as_dict(self, short=True): + """ + Transforms the elements of the object into a dict + + @return: structure as dict + @rtype: dict + """ + + res = super(GetForgeModulesApp, self).as_dict() + + res['default_forge_uri'] = self.default_forge_uri + res['default_http_timeout'] = self.default_http_timeout + res['default_puppet_root_env_dir'] = self.default_puppet_root_env_dir + res['open_args'] = self.open_args + + return res + + # ------------------------------------------------------------------------- + def evaluate_config(self, config, yaml_file): + + super(GetForgeModulesApp, self).evaluate_config(config, yaml_file) + + if 'puppet_env_dir' in config and config['puppet_env_dir']: + path = config['puppet_env_dir'] + if os.path.isabs(path): + self.puppet_root_env_dir = os.path.normapath(path) + else: + msg = ( + "Path for the Puppet environments {p!r} must be " + "an absolute path in {f!r}.").format( + p=path, f=yaml_file) + self.error_data.append(msg) + + # ------------------------------------------------------------------------- + def run(self): + """Main routine.""" + + msg = "Here I go again ..." + LOG.info(msg) + self.print_out(msg) + + self.check_data_dir() + self._init_puppet_environments() + + # ------------------------------------------------------------------------- + def _init_puppet_environments(self): + + LOG.debug("Collecting all Puppet environments in {!r}.".format(self.puppet_root_env_dir)) + pattern = os.path.join(self.puppet_root_env_dir, '*') + dirs = glob.glob(pattern) + + self.environments = [] + for path in dirs: + if os.path.isdir(path): + env = os.path.basename(path) + self.environments.append(env) + + self.environments.sort(key=str.lower) + if self.verbose > 1: + LOG.debug("Found Puppet environments:\n{}".format(pp(self.environments))) + if not self.environments: + raise GetForgeModulesError( + "Did not found environments in {!r}.".format(self.puppet_root_env_dir)) + +# ============================================================================= + +if __name__ == "__main__": + + pass + +# ============================================================================= +# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 list -- 2.39.5