yum - only instantiate YumBase once (#63713)
* yum - only instantiate YumBase once
Previously, this code was re-instantiating the `YumBase` object
many times which is unnecessary and slow. However, we must do it
twice in the `state: absent` case because the `yumSack` and
`rpmSack` data of the previously instantiated object becomes
invalid and is no longer useful post transaction when we verify
that the package removal did in fact take place. Also, this patch
removes the repetitive re-processing of enable/disable of repos in
various places.
Here's a display of the speed increase against a RHEL7 host:
```yaml
- hosts: rhel7
remote_user: root
tasks:
- name: Install generic packages
yum:
state: present
name:
- iptraf-ng
- screen
- erlang
- name: Remove generic packages
yum:
state: absent
name:
- iptraf-ng
- screen
- erlang
```
Before this patch:
```
real 0m52.728s
user 0m5.645s
sys 0m0.482s
```
After this patch:
```
real 0m17.139s
user 0m3.238s
sys 0m0.277s
```
Fixes #63588
Fixes #63551
Signed-off-by: Adam Miller <admiller@redhat.com>
* add changelog
Signed-off-by: Adam Miller <admiller@redhat.com>
This commit is contained in:
Adam Miller
ansibot
parent
0923ed56c7
commit
8c43697e3b
2 changed files with 81 additions and 86 deletions
|
|
@ -0,0 +1,4 @@
|
|||
bugfixes:
|
||||
- yum - performance bugfix, the YumBase object was being instantiated
|
||||
multiple times unnecessarily, which lead to considerable overhead when
|
||||
operating against large sets of packages.
|
||||
|
|
@ -391,6 +391,7 @@ class YumModule(YumDnf):
|
|||
|
||||
self.pkg_mgr_name = "yum"
|
||||
self.lockfile = '/var/run/yum.pid'
|
||||
self._yum_base = None
|
||||
|
||||
def _enablerepos_with_error_checking(self, yumbase):
|
||||
# NOTE: This seems unintuitive, but it mirrors yum's CLI behavior
|
||||
|
|
@ -454,34 +455,51 @@ class YumModule(YumDnf):
|
|||
# another copy seems to be running
|
||||
return True
|
||||
|
||||
@property
|
||||
def yum_base(self):
|
||||
my = yum.YumBase()
|
||||
my.preconf.debuglevel = 0
|
||||
my.preconf.errorlevel = 0
|
||||
my.preconf.plugins = True
|
||||
my.preconf.enabled_plugins = self.enable_plugin
|
||||
my.preconf.disabled_plugins = self.disable_plugin
|
||||
if self.releasever:
|
||||
my.preconf.releasever = self.releasever
|
||||
if self.installroot != '/':
|
||||
# do not setup installroot by default, because of error
|
||||
# CRITICAL:yum.cli:Config Error: Error accessing file for config file:////etc/yum.conf
|
||||
# in old yum version (like in CentOS 6.6)
|
||||
my.preconf.root = self.installroot
|
||||
my.conf.installroot = self.installroot
|
||||
if self.conf_file and os.path.exists(self.conf_file):
|
||||
my.preconf.fn = self.conf_file
|
||||
if os.geteuid() != 0:
|
||||
if hasattr(my, 'setCacheDir'):
|
||||
my.setCacheDir()
|
||||
else:
|
||||
cachedir = yum.misc.getCacheDir()
|
||||
my.repos.setCacheDir(cachedir)
|
||||
my.conf.cache = 0
|
||||
if self.disable_excludes:
|
||||
my.conf.disable_excludes = self.disable_excludes
|
||||
if self._yum_base:
|
||||
return self._yum_base
|
||||
else:
|
||||
# Only init once
|
||||
self._yum_base = yum.YumBase()
|
||||
self._yum_base.preconf.debuglevel = 0
|
||||
self._yum_base.preconf.errorlevel = 0
|
||||
self._yum_base.preconf.plugins = True
|
||||
self._yum_base.preconf.enabled_plugins = self.enable_plugin
|
||||
self._yum_base.preconf.disabled_plugins = self.disable_plugin
|
||||
if self.releasever:
|
||||
self._yum_base.preconf.releasever = self.releasever
|
||||
if self.installroot != '/':
|
||||
# do not setup installroot by default, because of error
|
||||
# CRITICAL:yum.cli:Config Error: Error accessing file for config file:////etc/yum.conf
|
||||
# in old yum version (like in CentOS 6.6)
|
||||
self._yum_base.preconf.root = self.installroot
|
||||
self._yum_base.conf.installroot = self.installroot
|
||||
if self.conf_file and os.path.exists(self.conf_file):
|
||||
self._yum_base.preconf.fn = self.conf_file
|
||||
if os.geteuid() != 0:
|
||||
if hasattr(self._yum_base, 'setCacheDir'):
|
||||
self._yum_base.setCacheDir()
|
||||
else:
|
||||
cachedir = yum.misc.getCacheDir()
|
||||
self._yum_base.repos.setCacheDir(cachedir)
|
||||
self._yum_base.conf.cache = 0
|
||||
if self.disable_excludes:
|
||||
self._yum_base.conf.disable_excludes = self.disable_excludes
|
||||
|
||||
return my
|
||||
# A sideeffect of accessing conf is that the configuration is
|
||||
# loaded and plugins are discovered
|
||||
self.yum_base.conf
|
||||
|
||||
try:
|
||||
self._enablerepos_with_error_checking(self._yum_base)
|
||||
|
||||
for rid in self.disablerepo:
|
||||
self.yum_base.repos.disableRepo(rid)
|
||||
except Exception as e:
|
||||
self.module.fail_json(msg="Failure talking to yum: %s" % to_native(e))
|
||||
|
||||
return self._yum_base
|
||||
|
||||
def po_to_envra(self, po):
|
||||
if hasattr(po, 'ui_envra'):
|
||||
|
|
@ -492,11 +510,10 @@ class YumModule(YumDnf):
|
|||
def is_group_env_installed(self, name):
|
||||
name_lower = name.lower()
|
||||
|
||||
my = self.yum_base()
|
||||
if yum.__version_info__ >= (3, 4):
|
||||
groups_list = my.doGroupLists(return_evgrps=True)
|
||||
groups_list = self.yum_base.doGroupLists(return_evgrps=True)
|
||||
else:
|
||||
groups_list = my.doGroupLists()
|
||||
groups_list = self.yum_base.doGroupLists()
|
||||
|
||||
# list of the installed groups on the first index
|
||||
groups = groups_list[0]
|
||||
|
|
@ -520,15 +537,10 @@ class YumModule(YumDnf):
|
|||
if not repoq:
|
||||
pkgs = []
|
||||
try:
|
||||
my = self.yum_base()
|
||||
for rid in self.disablerepo:
|
||||
my.repos.disableRepo(rid)
|
||||
self._enablerepos_with_error_checking(my)
|
||||
|
||||
e, m, _ = my.rpmdb.matchPackageNames([pkgspec])
|
||||
e, m, _ = self.yum_base.rpmdb.matchPackageNames([pkgspec])
|
||||
pkgs = e + m
|
||||
if not pkgs and not is_pkg:
|
||||
pkgs.extend(my.returnInstalledPackagesByDep(pkgspec))
|
||||
pkgs.extend(self.yum_base.returnInstalledPackagesByDep(pkgspec))
|
||||
except Exception as e:
|
||||
self.module.fail_json(msg="Failure talking to yum: %s" % to_native(e))
|
||||
|
||||
|
|
@ -574,15 +586,10 @@ class YumModule(YumDnf):
|
|||
|
||||
pkgs = []
|
||||
try:
|
||||
my = self.yum_base()
|
||||
for rid in self.disablerepo:
|
||||
my.repos.disableRepo(rid)
|
||||
self._enablerepos_with_error_checking(my)
|
||||
|
||||
e, m, _ = my.pkgSack.matchPackageNames([pkgspec])
|
||||
e, m, _ = self.yum_base.pkgSack.matchPackageNames([pkgspec])
|
||||
pkgs = e + m
|
||||
if not pkgs:
|
||||
pkgs.extend(my.returnPackagesByDep(pkgspec))
|
||||
pkgs.extend(self.yum_base.returnPackagesByDep(pkgspec))
|
||||
except Exception as e:
|
||||
self.module.fail_json(msg="Failure talking to yum: %s" % to_native(e))
|
||||
|
||||
|
|
@ -613,16 +620,12 @@ class YumModule(YumDnf):
|
|||
updates = []
|
||||
|
||||
try:
|
||||
my = self.yum_base()
|
||||
for rid in self.disablerepo:
|
||||
my.repos.disableRepo(rid)
|
||||
self._enablerepos_with_error_checking(my)
|
||||
|
||||
pkgs = my.returnPackagesByDep(pkgspec) + my.returnInstalledPackagesByDep(pkgspec)
|
||||
pkgs = self.yum_base.returnPackagesByDep(pkgspec) + \
|
||||
self.yum_base.returnInstalledPackagesByDep(pkgspec)
|
||||
if not pkgs:
|
||||
e, m, _ = my.pkgSack.matchPackageNames([pkgspec])
|
||||
e, m, _ = self.yum_base.pkgSack.matchPackageNames([pkgspec])
|
||||
pkgs = e + m
|
||||
updates = my.doPackageLists(pkgnarrow='updates').updates
|
||||
updates = self.yum_base.doPackageLists(pkgnarrow='updates').updates
|
||||
except Exception as e:
|
||||
self.module.fail_json(msg="Failure talking to yum: %s" % to_native(e))
|
||||
|
||||
|
|
@ -653,13 +656,9 @@ class YumModule(YumDnf):
|
|||
|
||||
pkgs = []
|
||||
try:
|
||||
my = self.yum_base()
|
||||
for rid in self.disablerepo:
|
||||
my.repos.disableRepo(rid)
|
||||
self._enablerepos_with_error_checking(my)
|
||||
|
||||
try:
|
||||
pkgs = my.returnPackagesByDep(req_spec) + my.returnInstalledPackagesByDep(req_spec)
|
||||
pkgs = self.yum_base.returnPackagesByDep(req_spec) + \
|
||||
self.yum_base.returnInstalledPackagesByDep(req_spec)
|
||||
except Exception as e:
|
||||
# If a repo with `repo_gpgcheck=1` is added and the repo GPG
|
||||
# key was never accepted, querying this repo will throw an
|
||||
|
|
@ -668,14 +667,15 @@ class YumModule(YumDnf):
|
|||
# the key and try again.
|
||||
if 'repomd.xml signature could not be verified' in to_native(e):
|
||||
self.module.run_command(self.yum_basecmd + ['makecache'])
|
||||
pkgs = my.returnPackagesByDep(req_spec) + my.returnInstalledPackagesByDep(req_spec)
|
||||
pkgs = self.yum_base.returnPackagesByDep(req_spec) + \
|
||||
self.yum_base.returnInstalledPackagesByDep(req_spec)
|
||||
else:
|
||||
raise
|
||||
if not pkgs:
|
||||
e, m, _ = my.pkgSack.matchPackageNames([req_spec])
|
||||
e, m, _ = self.yum_base.pkgSack.matchPackageNames([req_spec])
|
||||
pkgs.extend(e)
|
||||
pkgs.extend(m)
|
||||
e, m, _ = my.rpmdb.matchPackageNames([req_spec])
|
||||
e, m, _ = self.yum_base.rpmdb.matchPackageNames([req_spec])
|
||||
pkgs.extend(e)
|
||||
pkgs.extend(m)
|
||||
except Exception as e:
|
||||
|
|
@ -767,21 +767,20 @@ class YumModule(YumDnf):
|
|||
@contextmanager
|
||||
def set_env_proxy(self):
|
||||
# setting system proxy environment and saving old, if exists
|
||||
my = self.yum_base()
|
||||
namepass = ""
|
||||
scheme = ["http", "https"]
|
||||
old_proxy_env = [os.getenv("http_proxy"), os.getenv("https_proxy")]
|
||||
try:
|
||||
# "_none_" is a special value to disable proxy in yum.conf/*.repo
|
||||
if my.conf.proxy and my.conf.proxy not in ("_none_",):
|
||||
if my.conf.proxy_username:
|
||||
namepass = namepass + my.conf.proxy_username
|
||||
proxy_url = my.conf.proxy
|
||||
if my.conf.proxy_password:
|
||||
namepass = namepass + ":" + my.conf.proxy_password
|
||||
elif '@' in my.conf.proxy:
|
||||
namepass = my.conf.proxy.split('@')[0].split('//')[-1]
|
||||
proxy_url = my.conf.proxy.replace("{0}@".format(namepass), "")
|
||||
if self.yum_base.conf.proxy and self.yum_base.conf.proxy not in ("_none_",):
|
||||
if self.yum_base.conf.proxy_username:
|
||||
namepass = namepass + self.yum_base.conf.proxy_username
|
||||
proxy_url = self.yum_base.conf.proxy
|
||||
if self.yum_base.conf.proxy_password:
|
||||
namepass = namepass + ":" + self.yum_base.conf.proxy_password
|
||||
elif '@' in self.yum_base.conf.proxy:
|
||||
namepass = self.yum_base.conf.proxy.split('@')[0].split('//')[-1]
|
||||
proxy_url = self.yum_base.conf.proxy.replace("{0}@".format(namepass), "")
|
||||
|
||||
if namepass:
|
||||
namepass = namepass + '@'
|
||||
|
|
@ -792,7 +791,7 @@ class YumModule(YumDnf):
|
|||
)
|
||||
else:
|
||||
for item in scheme:
|
||||
os.environ[item + "_proxy"] = my.conf.proxy
|
||||
os.environ[item + "_proxy"] = self.yum_base.conf.proxy
|
||||
yield
|
||||
except yum.Errors.YumBaseError:
|
||||
raise
|
||||
|
|
@ -1139,6 +1138,7 @@ class YumModule(YumDnf):
|
|||
# of the process
|
||||
|
||||
# at this point we check to see if the pkg is no longer present
|
||||
self._yum_base = None # previous YumBase package index is now invalid
|
||||
for pkg in pkgs:
|
||||
if pkg.startswith('@'):
|
||||
installed = self.is_group_env_installed(pkg)
|
||||
|
|
@ -1471,9 +1471,9 @@ class YumModule(YumDnf):
|
|||
can be done for remove and absent action
|
||||
|
||||
As solution I would advice to cal
|
||||
try: my.repos.disableRepo(disablerepo)
|
||||
try: self.yum_base.repos.disableRepo(disablerepo)
|
||||
and
|
||||
try: my.repos.enableRepo(enablerepo)
|
||||
try: self.yum_base.repos.enableRepo(enablerepo)
|
||||
right before any yum_cmd is actually called regardless
|
||||
of yum action.
|
||||
|
||||
|
|
@ -1490,19 +1490,14 @@ class YumModule(YumDnf):
|
|||
if self.update_cache:
|
||||
self.module.run_command(self.yum_basecmd + ['clean', 'expire-cache'])
|
||||
|
||||
my = self.yum_base()
|
||||
try:
|
||||
if self.disablerepo:
|
||||
for rid in self.disablerepo:
|
||||
my.repos.disableRepo(rid)
|
||||
current_repos = my.repos.repos.keys()
|
||||
current_repos = self.yum_base.repos.repos.keys()
|
||||
if self.enablerepo:
|
||||
try:
|
||||
self._enablerepos_with_error_checking(my)
|
||||
new_repos = my.repos.repos.keys()
|
||||
new_repos = self.yum_base.repos.repos.keys()
|
||||
for i in new_repos:
|
||||
if i not in current_repos:
|
||||
rid = my.repos.getRepo(i)
|
||||
rid = self.yum_base.repos.getRepo(i)
|
||||
a = rid.repoXML.repoid # nopep8 - https://github.com/ansible/ansible/pull/21475#pullrequestreview-22404868
|
||||
current_repos = new_repos
|
||||
except yum.Errors.YumBaseError as e:
|
||||
|
|
@ -1598,13 +1593,9 @@ class YumModule(YumDnf):
|
|||
# the system then users will see an error message using the yum API.
|
||||
# Use repoquery in those cases.
|
||||
|
||||
my = self.yum_base()
|
||||
# A sideeffect of accessing conf is that the configuration is
|
||||
# loaded and plugins are discovered
|
||||
my.conf
|
||||
repoquery = None
|
||||
try:
|
||||
yum_plugins = my.plugins._plugins
|
||||
yum_plugins = self.yum_base.plugins._plugins
|
||||
except AttributeError:
|
||||
pass
|
||||
else:
|
||||
|
|
|
|||
Loading…
Reference in a new issue