introduce easy installation of specs

Author: Martin Hatina

dnf/base.py

@@ -24,6 +24,8 @@
 from __future__ import print_function
 from __future__ import unicode_literals
 
+import argparse
+
 import libdnf.transaction
 
 from dnf.comps import CompsQuery
@@ -32,7 +34,7 @@
 from dnf.module.metadata_loader import ModuleMetadataLoader
 from dnf.module.repo_module_dict import RepoModuleDict
 from dnf.module.repo_module_version import RepoModuleVersion
-from dnf.util import first
+from dnf.util import _parse_specs
 from dnf.db.history import SwdbInterface
 from dnf.yum import misc
 from functools import reduce
@@ -113,6 +115,7 @@ def __init__(self, conf=None):
         self._allow_erasing = False
         self._repo_set_imported_gpg_keys = set()
         self.repo_module_dict = RepoModuleDict(self)
+        self.output = None
 
     def __enter__(self):
         return self
@@ -577,6 +580,8 @@ def reset(self, sack=False, repos=False, goal=False):
             self._goal = None
             if self._sack is not None:
                 self._goal = dnf.goal.Goal(self._sack)
+            if self._module_persistor is not None:
+                self._module_persistor.reset()
             self.history.close()
             self._comps_trans = dnf.comps.TransactionBunch()
             self._transaction = None
@@ -1581,13 +1586,13 @@ def reason_fn(pkgname):
 
         return dnf.comps.Solver(self.history, self._comps, reason_fn)
 
-    def environment_install(self, env_id, types, exclude=None, strict=True):
+    def environment_install(self, env_id, types, exclude=None, strict=True, exclude_groups=None):
         assert dnf.util.is_string_type(env_id)
         solver = self._build_comps_solver()
         types = self._translate_comps_pkg_types(types)
         trans = dnf.comps.install_or_skip(solver._environment_install,
                                           env_id, types, exclude or set(),
-                                          strict)
+                                          strict, exclude_groups)
         if not trans:
             return 0
         return self._add_comps_trans(trans)
@@ -1643,7 +1648,7 @@ def _pattern_to_pkgname(pattern):
                      grp_id, trans.install)
         return self._add_comps_trans(trans)
 
-    def env_group_install(self, patterns, types, strict=True):
+    def env_group_install(self, patterns, types, strict=True, exclude=None, exclude_groups=None):
         q = CompsQuery(self.comps, self.history,
                        CompsQuery.ENVIRONMENTS | CompsQuery.GROUPS,
                        CompsQuery.AVAILABLE | CompsQuery.INSTALLED)
@@ -1657,9 +1662,10 @@ def env_group_install(self, patterns, types, strict=True):
                 done = False
                 continue
             for group_id in res.groups:
-                cnt += self.group_install(group_id, types, strict=strict)
+                cnt += self.group_install(group_id, types, exclude=exclude, strict=strict)
             for env_id in res.environments:
-                cnt += self.environment_install(env_id, types, strict=strict)
+                cnt += self.environment_install(env_id, types, exclude=exclude, strict=strict,
+                                                exclude_groups=exclude_groups)
         if not done and strict:
             raise dnf.exceptions.Error(_('Nothing to do.'))
         return cnt
@@ -1754,6 +1760,10 @@ def _install_multiarch(self, query, reponame=None, strict=True):
             self._goal.install(select=sltr, optional=(not strict))
         return len(available)
 
+    def enable_module(self, specs, save_immediately=False):
+        for spec in specs:
+            self.repo_module_dict.enable(spec, save_immediately)
+
     def install_module(self, specs, strict=True):
         """
         Install module based on provided specs
@@ -1763,6 +1773,97 @@ def install_module(self, specs, strict=True):
         """
         return self.repo_module_dict.install(specs, strict)
 
+    def _categorize_specs(self, install, exclude):
+        """
+        Categorize :param install and :param exclude list into two groups each (packages and groups)
+
+        :param install: list of specs, whether packages ('foo') or groups/modules ('@bar')
+        :param exclude: list of specs, whether packages ('foo') or groups/modules ('@bar')
+        :return: categorized install and exclude specs (stored in argparse.Namespace class)
+
+        To access packages use: specs.pkg_specs,
+        to access groups use: specs.grp_specs
+        """
+        install_specs = argparse.Namespace()
+        exclude_specs = argparse.Namespace()
+        _parse_specs(install_specs, install)
+        _parse_specs(exclude_specs, exclude)
+
+        return install_specs, exclude_specs
+
+    def _exclude_package_specs(self, exclude_specs):
+        glob_excludes = [exclude for exclude in exclude_specs.pkg_specs
+                         if dnf.util.is_glob_pattern(exclude)]
+        excludes = [exclude for exclude in exclude_specs.pkg_specs
+                    if exclude not in glob_excludes]
+
+        exclude_query = self.sack.query().filter(name=excludes)
+        glob_exclude_query = self.sack.query().filter(name__glob=glob_excludes)
+
+        self.sack.add_excludes(exclude_query)
+        self.sack.add_excludes(glob_exclude_query)
+
+    def _exclude_groups(self, group_specs):
+        group_excludes = []
+
+        for group_spec in group_specs:
+            if '/' in group_spec:
+                split = group_spec.split('/')
+                group_spec = split[0]
+
+            environment = self.comps.environment_by_pattern(group_spec)
+            if environment:
+                for group in environment.groups_iter():
+                    for pkg in group.packages_iter():
+                        group_excludes.append(pkg.name)
+            else:
+                group = self.comps.group_by_pattern(group_spec)
+                if not group:
+                    continue
+
+                for pkg in group.packages_iter():
+                    group_excludes.append(pkg.name)
+
+        exclude_query = self.sack.query().filter(name=group_excludes)
+        self.sack.add_excludes(exclude_query)
+
+    def _install_groups(self, group_specs, excludes, skipped, strict=True):
+        for group_spec in group_specs:
+            try:
+                types = self.conf.group_package_types
+
+                if '/' in group_spec:
+                    split = group_spec.split('/')
+                    group_spec = split[0]
+                    types = split[1].split(',')
+
+                self.env_group_install([group_spec], types, strict, excludes.pkg_specs,
+                                       excludes.grp_specs)
+            except dnf.exceptions.Error:
+                skipped.append("@" + group_spec)
+
+    def install_specs(self, install, exclude=None, reponame=None, strict=True, forms=None):
+        if exclude is None:
+            exclude = []
+
+        skipped = []
+        install_specs, exclude_specs = self._categorize_specs(install, exclude)
+
+        self._exclude_package_specs(exclude_specs)
+        for spec in install_specs.pkg_specs:
+            try:
+                self.install(spec, reponame=reponame, strict=strict, forms=forms)
+            except dnf.exceptions.Error:
+                skipped.append(spec)
+
+        groups = self.install_module(install_specs.grp_specs, strict)
+
+        self.read_comps(arch_filter=True)
+        self._exclude_groups(exclude_specs.grp_specs)
+        self._install_groups(groups, exclude_specs, skipped, strict)
+
+        return skipped
+
     def install(self, pkg_spec, reponame=None, strict=True, forms=None):
         # :api
         """Mark package(s) given by pkg_spec and reponame for installation."""

dnf/cli/option_parser.py

@@ -20,6 +20,7 @@
 
 from __future__ import unicode_literals
 from dnf.i18n import _
+from dnf.util import _parse_specs
 
 import argparse
 import dnf.exceptions
@@ -118,19 +119,7 @@ def __call__(self, parser, namespace, values, opt_str):
 
     class ParseSpecGroupFileCallback(argparse.Action):
         def __call__(self, parser, namespace, values, opt_str):
-            setattr(namespace, "filenames", [])
-            setattr(namespace, "grp_specs", [])
-            setattr(namespace, "pkg_specs", [])
-            for value in values:
-                schemes = dnf.pycomp.urlparse.urlparse(value)[0]
-                if value.endswith('.rpm'):
-                    namespace.filenames.append(value)
-                elif schemes and schemes in ('http', 'ftp', 'file', 'https'):
-                    namespace.filenames.append(value)
-                elif value.startswith('@'):
-                    namespace.grp_specs.append(value[1:])
-                else:
-                    namespace.pkg_specs.append(value)
+            _parse_specs(namespace, values)
 
     class PkgNarrowCallback(argparse.Action):
         def __init__(self, *args, **kwargs):

dnf/comps.py

@@ -83,14 +83,14 @@ def _fn_display_order(group):
 
 
 def install_or_skip(install_fnc, grp_or_env_id, types, exclude=None,
-                    strict=True):
+                    strict=True, exclude_groups=None):
     """Either mark in persistor as installed given `grp_or_env` (group
        or environment) or skip it (if it's already installed).
        `install_fnc` has to be Solver._group_install
        or Solver._environment_install.
        """
     try:
-        return install_fnc(grp_or_env_id, types, exclude, strict)
+        return install_fnc(grp_or_env_id, types, exclude, strict, exclude_groups)
     except dnf.comps.CompsError as e:
         logger.warning("%s, %s", ucd(e)[:-1], _("skipping."))
 
@@ -550,18 +550,22 @@ def _removable_grp(self, group_id):
         assert dnf.util.is_string_type(group_id)
         return self.history.env.is_removable_group(group_id)
 
-    def _environment_install(self, env_id, pkg_types, exclude, strict=True):
+    def _environment_install(self, env_id, pkg_types, exclude, strict=True, exclude_groups=None):
         assert dnf.util.is_string_type(env_id)
         comps_env = self.comps._environment_by_id(env_id)
         swdb_env = self.history.env.new(env_id, comps_env.name, comps_env.ui_name, pkg_types)
         self.history.env.install(swdb_env)
 
         trans = TransactionBunch()
         for comps_group in comps_env.mandatory_groups:
+            if exclude_groups and comps_group in exclude_groups:
+                continue
             trans += self._group_install(comps_group.id, pkg_types, exclude, strict)
             swdb_env.addGroup(comps_group.id, True, MANDATORY)
 
         for comps_group in comps_env.optional_groups:
+            if exclude_groups and comps_group in exclude_groups:
+                continue
             swdb_env.addGroup(comps_group.id, False, OPTIONAL)
             # TODO: if a group is already installed, mark it as installed?
         return trans
@@ -611,7 +615,7 @@ def _environment_upgrade(self, env_id):
         self.history.env.upgrade(swdb_env)
         return trans
 
-    def _group_install(self, group_id, pkg_types, exclude=None, strict=True):
+    def _group_install(self, group_id, pkg_types, exclude=None, strict=True, exclude_groups=None):
         assert dnf.util.is_string_type(group_id)
         comps_group = self.comps._group_by_id(group_id)
         if not comps_group:

dnf/module/exceptions.py

@@ -51,6 +51,12 @@ def __init__(self, module_spec):
         super(EnabledStreamException, self).__init__(value)
 
 
+class InstallMultipleStreamsException(dnf.exceptions.Error):
+    def __init__(self, module_spec):
+        value = "Cannot install more streams from module '{}' at the same time".format(module_spec)
+        super(InstallMultipleStreamsException, self).__init__(value)
+
+
 class DifferentStreamEnabledException(dnf.exceptions.Error):
     def __init__(self, module_spec):
         value = "Different stream enabled for module: {}".format(module_spec)

dnf/module/repo_module.py

@@ -80,6 +80,9 @@ def enable(self, stream, assumeyes=False):
         if self.conf.enabled._get() and self.conf.stream._get() == stream:
             return
 
+        if not self.parent.base.conf.assumeno and not self.parent.base.output:
+            assumeyes = True
+
         if self.conf.stream._get() is not "" and \
                 str(self.conf.stream._get()) != str(stream) and \
                 not assumeyes:
@@ -92,7 +95,8 @@ def enable(self, stream, assumeyes=False):
             else:
                 raise EnabledStreamException("{}:{}".format(self.name, stream))
 
-        self.parent.base._module_persistor.set_data(self, stream=stream, enabled=True)
+        self.parent.base._module_persistor.set_data(self, stream=stream, enabled=True,
+                                                    version=-1, profiles=[])
 
     def disable(self):
         self.parent.base._module_persistor.set_data(self, enabled=False, profiles=[])

dnf/module/repo_module_dict.py

@@ -27,7 +27,7 @@
 from dnf.module.exceptions import NoStreamSpecifiedException, NoModuleException, \
     EnabledStreamException, ProfileNotInstalledException, NoProfileToRemoveException, \
     VersionLockedException, CannotLockVersionException, \
-    DifferentStreamEnabledException
+    DifferentStreamEnabledException, InstallMultipleStreamsException
 from dnf.module.repo_module import RepoModule
 from dnf.module.subject import ModuleSubject
 from dnf.selector import Selector
@@ -339,9 +339,11 @@ def get_best_versions(self, module_specs):
                 skipped.append(module_spec)
                 continue
 
-            key = "{}:{}".format(module_version.name, module_version.stream)
+            key = module_version.name
             if key in best_versions:
                 best_version, profiles, default_profiles = best_versions[key]
+                if best_version.stream != module_version.stream:
+                    raise InstallMultipleStreamsException(module_version.name)
 
                 if module_form.profile:
                     profiles.append(module_form.profile)

dnf/util.py

@@ -47,6 +47,34 @@
 """DNF Utilities."""
 
 
+def _parse_specs(namespace, values):
+    """
+    Categorize :param values list into packages, groups and filenames
+
+    :param namespace: argparse.Namespace, where specs will be stored
+    :param values: list of specs, whether packages ('foo') or groups/modules ('@bar')
+                   or filenames ('*.rmp', 'http://*', ...)
+
+    To access packages use: specs.pkg_specs,
+    to access groups use: specs.grp_specs,
+    to access filenames use: specs.filenames
+    """
+
+    setattr(namespace, "filenames", [])
+    setattr(namespace, "grp_specs", [])
+    setattr(namespace, "pkg_specs", [])
+    for value in values:
+        schemes = dnf.pycomp.urlparse.urlparse(value)[0]
+        if value.endswith('.rpm'):
+            namespace.filenames.append(value)
+        elif schemes and schemes in ('http', 'ftp', 'file', 'https'):
+            namespace.filenames.append(value)
+        elif value.startswith('@'):
+            namespace.grp_specs.append(value[1:])
+        else:
+            namespace.pkg_specs.append(value)
+
+
 def _non_repo_handle(conf=None):
     handle = librepo.Handle()
     handle.useragent = dnf.const.USER_AGENT