From: Frank Brehm Date: Fri, 24 May 2024 14:53:51 +0000 (+0200) Subject: First usable version of the ConsulHandler. X-Git-Tag: 1.9.1^2~6 X-Git-Url: https://git.uhu-banane.org/?a=commitdiff_plain;h=d318756c7cb92e82e51191394a20758caba87fe0;p=pixelpark%2Fcreate-terraform.git First usable version of the ConsulHandler. --- diff --git a/lib/create_terraform/consul.py b/lib/create_terraform/consul.py index 1b3718c..d86e27c 100644 --- a/lib/create_terraform/consul.py +++ b/lib/create_terraform/consul.py @@ -9,12 +9,19 @@ from __future__ import absolute_import, print_function # Standard module +import base64 +import copy +import json import logging import os +import socket +import urllib.parse +import urllib3 # Third party modules -from fb_tools.common import to_bool +from fb_tools.common import pp, to_bool from fb_tools.handling_obj import HandlingObject +from fb_tools.obj import FbGenericBaseObject import requests from requests.exceptions import RequestException @@ -29,10 +36,12 @@ from . import MAX_PORT_NUMBER from . import __version__ as GLOBAL_VERSION from .errors import ConsulHandlerError +from .errors import ConsulRequestError +from .errors import ConsulApiError from .xlate import XLATOR -__version__ = '0.1.0' +__version__ = '0.2.0' LOG = logging.getLogger(__name__) LOGLEVEL_REQUESTS_SET = False @@ -40,6 +49,131 @@ LOGLEVEL_REQUESTS_SET = False _ = XLATOR.gettext +# ============================================================================= +class ConsulKeyQueryParams(FbGenericBaseObject): + """Class for encapsulating and handling Consul API key/value queries.""" + + valid_params = ('dc', 'recurse', 'raw', 'keys') + + # ------------------------------------------------------------------------- + def __init__(self, **kwargs): + """Initialize a ConsulHandler object.""" + self._dc = '' + self._recurse = False + self._raw = False + self._keys = False + + for arg in kwargs.keys(): + if arg not in self.valid_params: + msg = _('Invalid parameter {p!r} given in init of a {c} object.').format( + p=arg, c=self.__class__.__name__) + raise AttributeError(msg) + setattr(self, arg, kwargs[arg]) + + # ----------------------------------------------------------- + @property + def dc(self): + """Specifies the datacenter to query.""" + return self._dc + + @dc.setter + def dc(self, value): + if value is None: + self._dc = '' + return + + self._dc = str(value).strip() + + # ----------------------------------------------------------- + @property + def recurse(self): + """ + Specifies if the lookup should be recursive. + + Treat the key of the query as a prefix instead of a literal match. + """ + return self._recurse + + @recurse.setter + def recurse(self, value): + self._recurse = to_bool(value) + + # ----------------------------------------------------------- + @property + def raw(self): + """ + Specifies the response is just the raw value of the key, without any encoding or metadata. + """ + return self._raw + + @raw.setter + def raw(self, value): + self._raw = to_bool(value) + + # ----------------------------------------------------------- + @property + def keys(self): + """ + Specifies to return only keys (no values or metadata). + + Specifying this parameter implies recurse. + """ + return self._keys + + @keys.setter + def keys(self, value): + self._keys = to_bool(value) + + # ------------------------------------------------------------------------- + def params_str(self, urlencoded=False): + """Generate a string with all non-default params.""" + pairs = [] + + for param in self.valid_params: + value = getattr(self, param) + if value: + if urlencoded: + if param == 'dc': + val = urllib.parse.quote(value) + else: + val = 'true' + else: + val = repr(value) + pairs.append(param + '=' + val) + + if urlencoded: + return '&'.join(pairs) + return ', '.join(pairs) + + # ------------------------------------------------------------------------- + def __repr__(self): + """Typecast into a string for reproduction.""" + out = '<%s(' % (self.__class__.__name__) + out += self.params_str() + '>' + + return out + + # ------------------------------------------------------------------------- + def as_dict(self, short=True): + """ + Transform the elements of the object into a dict. + + @param short: don't include local properties in resulting dict. + @type short: bool + + @return: structure as dict + @rtype: dict + """ + res = super(ConsulKeyQueryParams, self).as_dict(short=short) + + res['dc'] = self.dc + res['recurse'] = self.recurse + res['raw'] = self.raw + res['keys'] = self.keys + + return res + + # ============================================================================= class ConsulHandler(HandlingObject): """ @@ -233,10 +367,10 @@ class ConsulHandler(HandlingObject): return res # ------------------------------------------------------------------------- - def _build_url(self, key): + def _build_url(self, key, **params): - if not os.path.isabs(path): - msg = _('The path {!r} must be an absolute path.').format(path) + if os.path.isabs(key): + msg = _('The key {!r} must not be an absolute path.').format(key) raise ValueError(msg) url = 'http://{}'.format(self.consul_server) @@ -252,10 +386,143 @@ class ConsulHandler(HandlingObject): url += self.path_prefix + '/' + key + if params: + params_obj = ConsulKeyQueryParams(**params) + param_str = params_obj.params_str(True) + if param_str: + url += '?' + param_str + return url + # ------------------------------------------------------------------------- + def perform_request( # noqa: C901 + self, key, method='GET', content_type=None, data=None, headers=None, + may_simulate=False, raw_data=False, **params): + """Perform the underlying API request.""" + if headers is None: + headers = {} + + url = self._build_url(key, **params) + if self.verbose > 1: + LOG.debug(_('URL to request: {!r}').format(url)) + LOG.debug(_('Request method: {!r}').format(method)) + + if data and self.verbose > 1: + data_out = '{!r}'.format(data) + try: + data_out = json.loads(data) + except ValueError: + pass + else: + data_out = pp(data_out) + LOG.debug('Data:\n{}'.format(data_out)) + if self.verbose > 2: + LOG.debug('RAW data:\n{}'.format(data)) + + headers.update({'User-Agent': self.user_agent}) + if content_type: + headers.update({'Content-Type': content_type}) + if self.verbose > 1: + head_out = copy.copy(headers) + LOG.debug('Headers:\n{}'.format(pp(head_out))) + + if may_simulate and self.simulate: + LOG.debug(_('Simulation mode, Request will not be sent.')) + return '' + + try: + + session = requests.Session() + if self.mocked: + self.start_mocking(session) + response = session.request( + method, url, data=data, headers=headers, timeout=self.timeout) + + except RequestException as e: + raise ConsulRequestError(str(e), url, e.request, e.response) + + except ( + socket.timeout, urllib3.exceptions.ConnectTimeoutError, + urllib3.exceptions.MaxRetryError, + requests.exceptions.ConnectTimeout) as e: + msg = _('Got a {c} on connecting to {h!r}: {e}.').format( + c=e.__class__.__name__, h=self.master_server, e=e) + raise ConsulHandlerError(msg) + + try: + self._eval_response(url, response) + + except ValueError: + raise ConsulHandlerError(_('Failed to parse the response'), response.text) + + if self.verbose > 3: + LOG.debug('RAW response: {!r}.'.format(response.text)) + if not response.text: + return '' + + if raw_data: + return response.text + + json_response = response.json() + if self.verbose > 3: + LOG.debug('JSON response:\n{}'.format(pp(json_response))) + + return json_response + + # ------------------------------------------------------------------------- + def _eval_response(self, url, response): + + if response.ok: + return + + err = response.json() + code = response.status_code + msg = err['error'] + LOG.debug(_('Got an error response code {code}: {msg}').format(code=code, msg=msg)) + + raise ConsulApiError(code, msg, url) + + # ------------------------------------------------------------------------- + def get_keys(self, key=''): + """Return a recursive list of all keys below the given start point.""" + result = self.perform_request(key, keys=True) + if not result: + return [] + + result.sort(key=str.lower) + return result + + # ------------------------------------------------------------------------- + def get_key(self, key): + """ + Return the value of the given key. + + if the key does not exists, it returns None. + """ + result = self.perform_request(key) + if not result: + return None + + first_data = result[0] + if not 'Value': + return None + + encoded_data = first_data['Value'] + json_data = base64.b64decode(encoded_data) + value = json.loads(str(json_data)) + return value + + # ------------------------------------------------------------------------- + def start_mocking(self, session): + """Start mocking mode of this class for unit testing.""" + if not self.mocked: + return + LOG.debug(_('Preparing mocking ...')) + import requests_mock + adapter = requests_mock.Adapter() + session.mount('mock', adapter) # vim: ts=4 et list diff --git a/lib/create_terraform/errors.py b/lib/create_terraform/errors.py index fd84da7..e9c2b70 100644 --- a/lib/create_terraform/errors.py +++ b/lib/create_terraform/errors.py @@ -16,7 +16,7 @@ from fb_tools.config import ConfigError from .xlate import XLATOR -__version__ = '1.4.0' +__version__ = '1.5.0' _ = XLATOR.gettext ngettext = XLATOR.ngettext @@ -125,6 +125,60 @@ class ConsulHandlerError(TerraformHandlerError): pass + +# ============================================================================= +class ConsulApiError(ConsulHandlerError): + """Base class for more complex exceptions.""" + + # ------------------------------------------------------------------------- + def __init__(self, code, msg, uri=None): + """Initialize the ConsuleApiError object.""" + self.code = code + self.msg = msg + self.uri = uri + + # ------------------------------------------------------------------------- + def __str__(self): + """Typecast into a string.""" + if self.uri: + msg = _('Got a {code} error code from {uri!r}: {msg}').format( + code=self.code, uri=self.uri, msg=self.msg) + else: + msg = _('Got a {code} error code: {msg}').format(code=self.code, msg=self.msg) + + return msg + + +# ============================================================================= +class ConsulRequestError(ConsulHandlerError): + """Raised, when some other exceptions occured on a HTTP(S) request.""" + + # ------------------------------------------------------------------------- + def __init__(self, msg, uri=None, request=None, response=None): + """Initialize the ConsulRequestError object.""" + self.msg = msg + self.uri = uri + self.request = request + self.response = response + + # ------------------------------------------------------------------------- + def __str__(self): + """Typecast into a string.""" + msg = _('Got an error requesting {uri!r}: {msg}').format(uri=self.uri, msg=self.msg) + if self.request: + cls = '' + if not isinstance(self.request, str): + cls = self.request.__class__.__name__ + ' - ' + msg += ' / Request: {c}{e}'.format(c=cls, e=self.request) + if self.response: + cls = '' + if not isinstance(self.response, str): + cls = self.response.__class__.__name__ + ' - ' + msg += ' / Response: {c}{e}'.format(c=cls, e=self.response) + + return msg + + # ============================================================================= if __name__ == "__main__": diff --git a/lib/create_terraform/handler/__init__.py b/lib/create_terraform/handler/__init__.py index ec40597..8826e83 100644 --- a/lib/create_terraform/handler/__init__.py +++ b/lib/create_terraform/handler/__init__.py @@ -40,13 +40,15 @@ from .vmware import CrTfHandlerVmwMixin from .. import MIN_VERSION_TERRAFORM, MAX_VERSION_TERRAFORM from .. import MIN_VERSION_VSPHERE_PROVIDER +from ..consul import ConsulHandler + from ..errors import AbortExecution # from ..tools import password_input from ..xlate import XLATOR -__version__ = '4.1.1' +__version__ = '4.2.0' LOG = logging.getLogger(__name__) _ = XLATOR.gettext @@ -193,6 +195,8 @@ class CreateTerraformHandler( self.eval_errors = 0 + self.consul = None + super(CreateTerraformHandler, self).__init__( appname=appname, verbose=verbose, version=version, base_dir=base_dir, simulate=simulate, force=force, terminal_has_colors=terminal_has_colors, @@ -235,6 +239,11 @@ class CreateTerraformHandler( "regular file.").format(str(self.private_key)) raise ExpectedHandlerError(msg) + self.consul = ConsulHandler( + appname=appname, verbose=verbose, base_dir=base_dir, simulate=simulate, force=force, + terminal_has_colors=terminal_has_colors, initialized=True + ) + if initialized: self.initialized = True