From 621d518383988d2469a805173b6963002eca60b9 Mon Sep 17 00:00:00 2001 From: Frank Brehm Date: Thu, 23 Aug 2018 17:24:20 +0200 Subject: [PATCH] Finish reading Puppetfile and metadata.json --- lib/webhooks/__init__.py | 2 +- lib/webhooks/get_forge_modules.py | 76 ++++++++++++++++++++++++++++++- lib/webhooks/module_info.py | 68 ++++++++++++++++++++++++++- 3 files changed, 142 insertions(+), 4 deletions(-) diff --git a/lib/webhooks/__init__.py b/lib/webhooks/__init__.py index f94029c..a425805 100644 --- a/lib/webhooks/__init__.py +++ b/lib/webhooks/__init__.py @@ -1,6 +1,6 @@ #!/bin/env python3 # -*- coding: utf-8 -*- -__version__ = '0.9.2' +__version__ = '0.9.3' # vim: ts=4 et list diff --git a/lib/webhooks/get_forge_modules.py b/lib/webhooks/get_forge_modules.py index 1d39f0a..98821e1 100644 --- a/lib/webhooks/get_forge_modules.py +++ b/lib/webhooks/get_forge_modules.py @@ -161,6 +161,68 @@ class GetForgeModulesApp(BaseHookApp): for env in self.environments: self.read_puppetfile(env) + self.read_metadata_files(env) + + if self.verbose > 2: + LOG.debug("Found modules:\n{}".format(pp(self.modules))) + + # ------------------------------------------------------------------------- + def read_metadata_files(self, env): + + pattern = os.path.join(self.puppet_root_env_dir, env, 'modules', '*') + if self.verbose > 2: + LOG.debug("Globbing pattern for module directories: {!r}".format(pattern)) + + for module_dir in glob.glob(pattern): + module_info = self.get_meta_module_info(module_dir, env) + if module_info: + full_name = module_info.full_name + if full_name in self.modules: + self.modules[full_name].merge_in(module_info) + else: + self.modules[full_name] = module_info + + # ------------------------------------------------------------------------- + def get_meta_module_info(self, module_dir, env): + + if self.verbose > 2: + LOG.debug("Get module information from {!r}.".format(module_dir)) + + if not os.path.exists(module_dir): + LOG.warn("Directory {!r} does not exists.".format(module_dir)) + return None + + if not os.path.isdir(module_dir): + LOG.warn("Path {!r} is not a directory".format(module_dir)) + return None + + metadata_file = os.path.join(module_dir, 'metadata.json') + if not os.path.exists(metadata_file): + LOG.warn("Metadatafile {!r} does not exists.".format(metadata_file)) + return None + if not os.path.isfile(metadata_file): + LOG.warn("Metadatafile {!r} is not a regular file.".format(metadata_file)) + return None + if not os.access(metadata_file, os.R_OK): + LOG.warn("Metadatafile {!r} is readable.".format(metadata_file)) + return None + if self.verbose > 2: + LOG.debug("Reading and evaluating {!r}.".format(metadata_file)) + + json_data = None + try: + with open(metadata_file, 'r', **self.open_args) as fh: + json_data = json.load(fh) + except ValueError as e: + LOG.warn( "Could not interprete {f!r} as a regular JSON file: {e}".format( + f=metadata_file, e=e)) + return None + if not json_data: + LOG.warn("Did not found any data in {!r}.".format(metadata_file)) + return None + + return ModuleInfo.init_from_json( + json_data, env, appname=self.appname, verbose=self.verbose, base_dir=self.base_dir) # ------------------------------------------------------------------------- def read_puppetfile(self, env): @@ -196,11 +258,17 @@ class GetForgeModulesApp(BaseHookApp): if self.re_comma_at_end.search(line): continue - if self.verbose > 2: + if self.verbose > 3: LOG.debug("Evaluating line {!r}...".format(prev_line)) module_info = ModuleInfo.init_from_puppetfile_line( appname=self.appname, verbose=self.verbose, base_dir=self.base_dir, line=prev_line, env=env) + if module_info: + full_name = module_info.full_name + if full_name in self.modules: + self.modules[full_name].merge_in(module_info) + else: + self.modules[full_name] = module_info prev_line = '' @@ -210,6 +278,12 @@ class GetForgeModulesApp(BaseHookApp): module_info = ModuleInfo.init_from_puppetfile_line( appname=self.appname, verbose=self.verbose, base_dir=self.base_dir, line=prev_line, env=env) + if module_info: + full_name = module_info.full_name + if full_name in self.modules: + self.modules[full_name].merge_in(module_info) + else: + self.modules[full_name] = module_info # ------------------------------------------------------------------------- def init_puppet_environments(self): diff --git a/lib/webhooks/module_info.py b/lib/webhooks/module_info.py index 88395e9..9cb4f07 100644 --- a/lib/webhooks/module_info.py +++ b/lib/webhooks/module_info.py @@ -22,7 +22,7 @@ from .common import pp, to_str from .obj import BaseObjectError from .obj import BaseObject -__version__ = '0.2.0' +__version__ = '0.3.0' LOG = logging.getLogger(__name__) @@ -33,6 +33,12 @@ class ModuleInfoError(BaseObjectError): pass +# ============================================================================= +class ModuleInfoTypeError(ModuleInfoError, TypeError): + + pass + + # ============================================================================= class ModuleInfo(BaseObject): """Class for encapsulating information about a Puppet module.""" @@ -42,7 +48,7 @@ class ModuleInfo(BaseObject): re_pf_line_version = re.compile(r"^\s*'([^']+)'") re_def_token = re.compile(r'^\s*(?:,\s*)?([^,]+)(?:\s*,|$)') re_empty = re.compile(r'^\s*(?:,\s*)?$') - re_key_val_pair = re.compile(r"^\s*:?([a-z]+)\s*=>\s*'([^']+)'\s*$", re.IGNORECASE) + re_key_val_pair = re.compile(r'^\s*:?([a-z]+)\s*=>\s*[\'"]([^\'"]+)[\'"]\s*$', re.IGNORECASE) re_v_at_start = re.compile(r"^v", re.IGNORECASE) # ------------------------------------------------------------------------- @@ -107,6 +113,12 @@ class ModuleInfo(BaseObject): self._vendor = _vendor self._full_name_orig = self.full_name + # ------------------------------------------------------------------------- + @property + def version_upstream(self): + """The current version number of the module from upstream.""" + return self._version_upstream + # ------------------------------------------------------------------------- @property def name(self): @@ -148,6 +160,10 @@ class ModuleInfo(BaseObject): else: self._full_name_orig = None + # ------------------------------------------------------------------------- + def __repr__(self): + return str(self) + # ------------------------------------------------------------------------- def as_dict(self, short=True): """ @@ -163,9 +179,57 @@ class ModuleInfo(BaseObject): res['vendor'] = self.vendor res['full_name'] = self.full_name res['full_name_orig'] = self.full_name_orig + res['version_upstream'] = self.version_upstream return res + # ------------------------------------------------------------------------- + def merge_in(self, other): + + if not isinstance(other, ModuleInfo): + raise ModuleInfoTypeError(( + "Parameter {p!r} is not of class ModuleInfoTypeError, but of class {c} " + " instead.").format(p=other, c=other.__class__.__name__)) + + if other.version_upstream and not self.version_upstream: + self._version_upstream = other.version_upstream + + if other.repo and not self.repo: + self.repo = other.repo + + for env in other.local_versions.keys(): + if env not in self.local_versions: + self.local_versions[env] = other.local_versions[env] + + for env in other.expected_versions.keys(): + if env not in self.expected_versions: + self.expected_versions[env] = other.expected_versions[env] + + # ------------------------------------------------------------------------- + @classmethod + def init_from_json( + cls, json_data, env, appname=None, verbose=0, base_dir=None): + + if 'name' not in json_data: + LOG.warn("Did not found module name in json.") + return None + + module_info = None + + try: + module_info = cls( + appname=appname, verbose=verbose, base_dir=base_dir, + full_name=json_data['name'], + ) + except ModuleInfoError as e: + LOG.warn("{c}: {e}".format(c=e.__class__.__name__, e=e)) + return None + + if 'version' in json_data: + module_info.local_versions[env] = json_data['version'].strip() + + return module_info + # ------------------------------------------------------------------------- @classmethod def init_from_puppetfile_line( -- 2.39.5