Refactoring method explore_vsphere_templates().

diff --git a/lib/create_terraform/handler/__init__.py b/lib/create_terraform/handler/__init__.py
index fd9c8df31aa7e1c9ca2919744b58793506a1002a..dd54c3d49898651c5caf3aef7df194eee979406d 100644 (file)
--- a/lib/create_terraform/handler/__init__.py
+++ b/lib/create_terraform/handler/__init__.py
@@ -46,7 +46,7 @@ from ..errors import AbortExecution
 
 from ..xlate import XLATOR
 
-__version__ = '4.0.1'
+__version__ = '4.0.2'
 LOG = logging.getLogger(__name__)
 
 _ = XLATOR.gettext
@@ -141,7 +141,7 @@ class CreateTerraformHandler(
 
         self.vsphere_folders = []
 
-        self.vsphere_vm_cache = []
+        self.vsphere_vm_cache = {}
 
         self.vsphere_user = None
         self.vsphere_password = None

diff --git a/lib/create_terraform/handler/vmware.py b/lib/create_terraform/handler/vmware.py
index 9b06821d89e7c6f6833a701b287964b3bdd575f1..1e5e7da5bbc9dd741657d1159facc95c7d205b3c 100644 (file)
--- a/lib/create_terraform/handler/vmware.py
+++ b/lib/create_terraform/handler/vmware.py
@@ -33,7 +33,7 @@ from ..slim_vm import SlimVm
 
 from ..xlate import XLATOR
 
-__version__ = '0.2.2'
+__version__ = '0.3.0'
 LOG = logging.getLogger(__name__)
 
 _ = XLATOR.gettext
@@ -251,9 +251,6 @@ class CrTfHandlerVmwMixin():
     # -------------------------------------------------------------------------·
     def exec_cache_vmw_vms(self):
 
-        # if self.stop_at_step == 'vmw-vms':
-        #     self.incr_verbosity()
-
         self.explore_vsphere_vms()
 
         if self.eval_errors:
@@ -386,19 +383,19 @@ class CrTfHandlerVmwMixin():
         self.all_vms = {}
         re_vm = re.compile(r'.*')
 
-        for vs_name in self.vsphere:
+        for vsphere_name in self.vsphere:
 
-            if vs_name not in self.all_vms:
-                self.all_vms[vs_name] = {}
+            if vsphere_name not in self.all_vms:
+                self.all_vms[vsphere_name] = {}
 
-            vm_list = self.vsphere[vs_name].get_vms(re_vm, name_only=True)
+            vm_list = self.vsphere[vsphere_name].get_vms(re_vm, name_only=True)
             for vm_tuple in vm_list:
                 vm_name = vm_tuple[0]
                 vm_path = vm_tuple[1]
-                if vm_name in self.all_vms[vs_name]:
-                    self.all_vms[vs_name][vm_name].append(vm_path)
+                if vm_name in self.all_vms[vsphere_name]:
+                    self.all_vms[vsphere_name][vm_name].append(vm_path)
                 else:
-                    self.all_vms[vs_name][vm_name] = [vm_path]
+                    self.all_vms[vsphere_name][vm_name] = [vm_path]
 
         if self.verbose > 2:
             msg = _("All existing VMs and templates:")
@@ -458,24 +455,34 @@ class CrTfHandlerVmwMixin():
 
         LOG.info(_("Exploring and caching all vSphere VMs and templates ..."))
 
+        total_vms = 0
+        slim_vms = []
+        slim_verbose = 2
+
         for vsphere_name in self.vsphere:
 
+            if vsphere_name not in self.vsphere_vm_cache:
+                self.vsphere_vm_cache[vsphere_name] = []
+
             re_vm = re.compile(r'.*')
             vm_list = self.vsphere[vsphere_name].get_vms(re_vm, as_obj=True, stop_at_found=False)
 
             for vm in vm_list:
-                slim_vm = SlimVm(vsphere_name, vm.name, vm.path, is_template=vm.template)
-                self.vsphere_vm_cache.append(slim_vm)
+                self.vsphere_vm_cache[vsphere_name].append(vm)
+                total_vms += 1
+                if self.verbose >= slim_verbose:
+                    slim_vm = SlimVm(vsphere_name, vm.name, vm.path, is_template=vm.template)
+                    slim_vms.append(slim_vm)
 
         msg = ngettext(
             'Found one VM or template in vSphere.',
-            'Found {nr} VMs and templates in vSphere.', len(self.vsphere_vm_cache))
-        msg = msg.format(nr=len(self.vsphere_vm_cache))
+            'Found {nr} VMs and templates in vSphere.', total_vms)
+        msg = msg.format(nr=total_vms)
         LOG.debug(msg)
 
-        if self.verbose > 1:
+        if self.verbose >= slim_verbose:
             msg = _("All explored vSphere VMs and templates:")
-            LOG.debug(msg + "\n" + pp(self.vsphere_vm_cache))
+            LOG.debug(msg + "\n" + pp(slim_vms))
 
     # -------------------------------------------------------------------------·
     def explore_vsphere_templates(self):
@@ -487,6 +494,9 @@ class CrTfHandlerVmwMixin():
             if vsphere_name not in self.vsphere_templates:
                 self.vsphere_templates[vsphere_name] = {}
 
+            if vsphere_name not in self.vsphere_vm_cache:
+                self.vsphere_vm_cache[vsphere_name] = []
+
             self.config.vsphere[vsphere_name].used_templates = []
 
             for vm in self.vms:
@@ -502,6 +512,7 @@ class CrTfHandlerVmwMixin():
             msg += "\n" + pp(self.config.vsphere[vsphere_name].used_templates)
             LOG.debug(msg)
 
+            found_template = False
             for template_name in self.config.vsphere[vsphere_name].used_templates:
 
                 if template_name in self.vsphere_templates[vsphere_name]:
@@ -509,15 +520,31 @@ class CrTfHandlerVmwMixin():
 
                 LOG.debug(_("Searching for template {t!r} in VSPhere {v!r} ...").format(
                     t=template_name, v=vsphere_name))
-                re_vm = re.compile(r'^' + re.escape(template_name) + r'$', re.IGNORECASE)
-                vm_list = self.vsphere[vsphere_name].get_vms(
-                    re_vm, as_obj=True, stop_at_found=True)
-                if vm_list:
-                    vm = vm_list[0]
-                    tname = vm.name.lower()
-                    if tname not in self.vsphere_templates[vsphere_name]:
-                        self.vsphere_templates[vsphere_name][template_name] = vm
-                else:
+                pat_vm = r'^' + re.escape(template_name) + r'$'
+                re_vm = re.compile(pat_vm, re.IGNORECASE)
+
+                for vm in self.vsphere_vm_cache[vsphere_name]:
+                    if self.verbose >= 2:
+                        LOG.debug(_('Checking VM {n!r} against pattern {p!r} ...').format(
+                            n=vm.name, p=pat_vm))
+                    if re_vm.match(vm.name):
+                        if vm.template:
+                            msg = _(
+                                'Found VMWare template {n!r} in vSphere {vs!r}, path '
+                                '{path!r}.').format(n=vm.name, vs=vsphere_name, path=vm.path)
+                            LOG.debug(msg)
+                        else:
+                            msg = _(
+                                'Found VM {n!r} in vSphere {vs!r}, path {path!r}, which is not '
+                                'a template, but it will be used as a template.').format(
+                                n=vm.name, vs=vsphere_name, path=vm.path)
+                            LOG.warn(msg)
+                        tname = vm.name.lower()
+                        if tname not in self.vsphere_templates[vsphere_name]:
+                            self.vsphere_templates[vsphere_name][template_name] = vm
+                        found_template = True
+
+                if not found_template:
                     LOG.error(_("Template {t!r} not found in VSPhere {v!r}.").format(
                         t=template_name, v=vsphere_name))
                     self.eval_errors += 1
@@ -587,22 +614,22 @@ class CrTfHandlerVmwMixin():
             print(" * {} ".format(vm.fqdn), end='', flush=True)
             if self.verbose:
                 print()
-            vs_name = vm.vsphere
-            vsphere = self.vsphere[vs_name]
+            vsphere_name = vm.vsphere
+            vsphere = self.vsphere[vsphere_name]
 
             vm_paths = None
-            if vs_name in self.all_vms:
-                if vm.fqdn in self.all_vms[vs_name]:
-                    vm_paths = self.all_vms[vs_name][vm.fqdn]
+            if vsphere_name in self.all_vms:
+                if vm.fqdn in self.all_vms[vsphere_name]:
+                    vm_paths = self.all_vms[vsphere_name][vm.fqdn]
 
             if vm_paths:
                 msg = _('[{m}]  - VM is already existing in VSphere {v!r}, path {p!r}.').format(
-                    m=self.colored('Existing', 'YELLOW'), v=vs_name, p=pp(vm_paths))
+                    m=self.colored('Existing', 'YELLOW'), v=vsphere_name, p=pp(vm_paths))
                 print(msg, end='', flush=True)
                 if self.verbose:
                     print()
 
-                vm_info = vsphere.get_vm(vm.fqdn, vsphere_name=vs_name, as_obj=True)
+                vm_info = vsphere.get_vm(vm.fqdn, vsphere_name=vsphere_name, as_obj=True)
                 if self.verbose > 2:
                     LOG.debug(_("VM info:") + "\n" + pp(vm_info.as_dict(bare=True)))
                 ds = vm_info.config_path_storage
@@ -644,19 +671,19 @@ class CrTfHandlerVmwMixin():
         if self.verbose:
             if self.used_dc_clusters:
                 out_lines = []
-                for vs_name in self.used_dc_clusters:
-                    for cluster in self.used_dc_clusters[vs_name]:
+                for vsphere_name in self.used_dc_clusters:
+                    for cluster in self.used_dc_clusters[vsphere_name]:
                         out_lines.append('  * VSphere {v!r}: {c}'.format(
-                            v=vs_name, c=cluster))
+                            v=vsphere_name, c=cluster))
                 out = '\n'.join(out_lines)
                 LOG.debug(_("Used datastore clusters:") + "\n" + out)
             else:
                 LOG.debug(_("No datastore clusters are used."))
             if self.used_datastores:
                 out_lines = []
-                for vs_name in self.used_datastores:
-                    for ds in self.used_datastores[vs_name]:
-                        out_lines.append('  * VSphere {v!r}: {ds}'.format(v=vs_name, ds=ds))
+                for vsphere_name in self.used_datastores:
+                    for ds in self.used_datastores[vsphere_name]:
+                        out_lines.append('  * VSphere {v!r}: {ds}'.format(v=vsphere_name, ds=ds))
                 out = '\n'.join(out_lines)
                 LOG.debug(_("Used datastors:") + "\n" + out)
             else:
@@ -686,8 +713,8 @@ class CrTfHandlerVmwMixin():
                 disk = vm.disks[unit_number]
                 needed_gb += disk.size_gb
 
-        vs_name = vm.vsphere
-        vsphere = self.vsphere[vs_name]
+        vsphere_name = vm.vsphere
+        vsphere = self.vsphere[vsphere_name]
 
         found = False
         for cluster_name in vsphere.ds_clusters.keys():
@@ -695,7 +722,7 @@ class CrTfHandlerVmwMixin():
                 if self.verbose > 2:
                     LOG.debug(_(
                         "Found datastore cluster {c!r} in VSphere {v!r} for VM {n!r}.").format(
-                        n=vm.name, v=vs_name, c=vm.ds_cluster))
+                        n=vm.name, v=vsphere_name, c=vm.ds_cluster))
                 if vm.ds_cluster != cluster_name:
                     LOG.debug(_("Setting datastore cluster for VM {n!r} to {c!r} ...").format(
                         n=vm.name, c=cluster_name))
@@ -705,13 +732,13 @@ class CrTfHandlerVmwMixin():
                     LOG.debug(_(
                         "Free space of cluster {c!r} in VSphere {v!r} before provisioning: "
                         "{a:0.1f} GiB.").format(
-                        c=cluster_name, v=vs_name, a=ds_cluster.avail_space_gb))
+                        c=cluster_name, v=vsphere_name, a=ds_cluster.avail_space_gb))
                 if ds_cluster.avail_space_gb < needed_gb:
                     LOG.error(_(
                         "Datastore cluster {d!r} in VSphere {v!r} has not sufficient space for "
                         "storage of VM {vm!r} (needed {n:0.1f} GiB, available {a:0.1f} "
                         "GiB).").format(
-                            d=cluster_name, v=vs_name, vm=vm.name, n=needed_gb,
+                            d=cluster_name, v=vsphere_name, vm=vm.name, n=needed_gb,
                             a=ds_cluster.avail_space_gb))
                     self.eval_errors += 1
                 else:
@@ -720,17 +747,17 @@ class CrTfHandlerVmwMixin():
                         LOG.debug(_(
                             "Free space in cluster {c!r} in VSphere {v!r} after provisioning: "
                             "{a:0.1f} GiB.").format(
-                            c=cluster_name, v=vs_name, a=ds_cluster.avail_space_gb))
+                            c=cluster_name, v=vsphere_name, a=ds_cluster.avail_space_gb))
                 found = True
-                if vs_name not in self.used_dc_clusters:
-                    self.used_dc_clusters[vs_name] = []
-                if cluster_name not in self.used_dc_clusters[vs_name]:
-                    self.used_dc_clusters[vs_name].append(cluster_name)
+                if vsphere_name not in self.used_dc_clusters:
+                    self.used_dc_clusters[vsphere_name] = []
+                if cluster_name not in self.used_dc_clusters[vsphere_name]:
+                    self.used_dc_clusters[vsphere_name].append(cluster_name)
                 break
 
         if not found:
             LOG.error(_("Datastore cluster {c!r} of VM {n!r} not found in VSphere {v!r}.").format(
-                n=vm.name, c=vm.ds_cluster, v=vs_name))
+                n=vm.name, c=vm.ds_cluster, v=vsphere_name))
             self.eval_errors += 1
 
     # -------------------------------------------------------------------------·
@@ -758,8 +785,8 @@ class CrTfHandlerVmwMixin():
                 disk = vm.disks[unit_number]
                 needed_gb += disk.size_gb
 
-        vs_name = vm.vsphere
-        vsphere = self.vsphere[vs_name]
+        vsphere_name = vm.vsphere
+        vsphere = self.vsphere[vsphere_name]
 
         vm_cluster = None
         for cluster in vsphere.clusters:
@@ -778,7 +805,7 @@ class CrTfHandlerVmwMixin():
                 if ds_name.lower() == vm.datastore.lower():
                     if self.verbose > 2:
                         LOG.debug(_("Found datastore {d!r} for VM {n!r} in VSPhere {v!r}.").format(
-                            n=vm.name, d=vm.datastore, v=vs_name))
+                            n=vm.name, d=vm.datastore, v=vsphere_name))
                     if ds_name not in vm_cluster.datastores:
                         LOG.warn(_("Datastore {d!r} not available in cluster {c!r}.").format(
                             d=ds_name, c=vm.cluster))
@@ -801,24 +828,24 @@ class CrTfHandlerVmwMixin():
                     break
             if not found:
                 LOG.error(_("Datastore {d!r} of VM {n!r} not found in VSPhere {v!r}.").format(
-                    n=vm.name, d=vm.datastore, v=vs_name))
+                    n=vm.name, d=vm.datastore, v=vsphere_name))
                 self.eval_errors += 1
-            if vs_name not in self.used_datastores:
-                self.used_datastores[vs_name] = []
-            if found_ds_name not in self.used_datastores[vs_name]:
-                self.used_datastores[vs_name].append(found_ds_name)
+            if vsphere_name not in self.used_datastores:
+                self.used_datastores[vsphere_name] = []
+            if found_ds_name not in self.used_datastores[vsphere_name]:
+                self.used_datastores[vsphere_name].append(found_ds_name)
             return
 
         ds_name = vsphere.datastores.find_ds(
             needed_gb, vm.ds_type, use_ds=copy.copy(vm_cluster.datastores), no_k8s=True)
         if ds_name:
             LOG.debug(_("Found datastore {d!r} for VM {n!r} in VSPhere {v!r}.").format(
-                d=ds_name, n=vm.name, v=vs_name))
+                d=ds_name, n=vm.name, v=vsphere_name))
             vm.datastore = ds_name
-            if vs_name not in self.used_datastores:
-                self.used_datastores[vs_name] = []
-            if ds_name not in self.used_datastores[vs_name]:
-                self.used_datastores[vs_name].append(ds_name)
+            if vsphere_name not in self.used_datastores:
+                self.used_datastores[vsphere_name] = []
+            if ds_name not in self.used_datastores[vsphere_name]:
+                self.used_datastores[vsphere_name].append(ds_name)
         else:
             self.eval_errors += 1
 
@@ -836,10 +863,10 @@ class CrTfHandlerVmwMixin():
         if self.verbose:
 
             lines = []
-            for vs_name in self.used_networks:
-                for nw in self.used_networks[vs_name]:
+            for vsphere_name in self.used_networks:
+                for nw in self.used_networks[vsphere_name]:
                     lines.append('  * VSphere {v!r}: {n}'.format(
-                        v=vs_name, n=nw))
+                        v=vsphere_name, n=nw))
             out = '\n'.join(lines)
             LOG.debug(_("Used networks:") + "\n" + out)
 
@@ -858,16 +885,16 @@ class CrTfHandlerVmwMixin():
     # -------------------------------------------------------------------------·
     def _validate_interfaces_vm(self, vm):
 
-        vs_name = vm.vsphere
+        vsphere_name = vm.vsphere
         LOG.debug(_("Checking interfaces of VM {n!r} in VSPhere {v!r} ...").format(
-            n=vm.name, v=vs_name))
+            n=vm.name, v=vsphere_name))
 
         if not vm.interfaces:
             LOG.error(_("No interfaces defined for VM {!r}.").format(vm.name))
             self.eval_errors += 1
             return
 
-        vsphere = self.vsphere[vs_name]
+        vsphere = self.vsphere[vsphere_name]
 
         vm_cluster = None
         for cluster in vsphere.clusters:
@@ -883,12 +910,12 @@ class CrTfHandlerVmwMixin():
         for iface in vm.interfaces:
             i += 1
             self._validate_interface_of_vm(
-                vm_name=vm.name, iface=iface, vs_name=vs_name, vm_cluster=vm_cluster, i=i)
+                vm_name=vm.name, iface=iface, vsphere_name=vsphere_name, vm_cluster=vm_cluster, i=i)
 
     # -------------------------------------------------------------------------·
-    def _validate_interface_of_vm(self, vm_name, iface, vs_name, vm_cluster, i=0):
+    def _validate_interface_of_vm(self, vm_name, iface, vsphere_name, vm_cluster, i=0):
 
-        vsphere = self.vsphere[vs_name]
+        vsphere = self.vsphere[vsphere_name]
 
         if self.verbose > 1:
             LOG.debug(_("Checking interface {i} of VM {n!r} ...").format(
@@ -988,18 +1015,18 @@ class CrTfHandlerVmwMixin():
                 if iface.netmask_v6 is None:
                     iface.netmask_v6 = net.network.prefixlen
 
-        if vs_name not in self.used_networks:
-            self.used_networks[vs_name] = []
-        if network not in self.used_networks[vs_name]:
-            self.used_networks[vs_name].append(network)
+        if vsphere_name not in self.used_networks:
+            self.used_networks[vsphere_name] = []
+        if network not in self.used_networks[vsphere_name]:
+            self.used_networks[vsphere_name].append(network)
 
     # -------------------------------------------------------------------------·
     def ensure_vsphere_folders(self):
 
-        vs_name = None
-        for vs_name in self.vsphere.keys():
+        vsphere_name = None
+        for vsphere_name in self.vsphere.keys():
             break
-        vsphere = self.vsphere[vs_name]
+        vsphere = self.vsphere[vsphere_name]
 
         print()
         LOG.info(_("Ensuring existence of all necessary vSphere VM folders."))