630 lines
22 KiB
Python
630 lines
22 KiB
Python
# dnfpayload.py
|
|
# DNF/rpm software payload management.
|
|
#
|
|
# Copyright (C) 2013 Red Hat, Inc.
|
|
#
|
|
# This copyrighted material is made available to anyone wishing to use,
|
|
# modify, copy, or redistribute it subject to the terms and conditions of
|
|
# the GNU General Public License v.2, or (at your option) any later version.
|
|
# This program is distributed in the hope that it will be useful, but WITHOUT
|
|
# ANY WARRANTY expressed or implied, including the implied warranties of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
|
|
# Public License for more details. You should have received a copy of the
|
|
# GNU General Public License along with this program; if not, write to the
|
|
# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
|
# 02110-1301, USA. Any Red Hat trademarks that are incorporated in the
|
|
# source code or documentation are not subject to the GNU General Public
|
|
# License and may only be used or replicated with the express permission of
|
|
# Red Hat, Inc.
|
|
#
|
|
# Red Hat Author(s): Ales Kozumplik <akozumpl@redhat.com>
|
|
#
|
|
|
|
from blivet.size import Size
|
|
from pyanaconda.flags import flags
|
|
from pyanaconda.i18n import _
|
|
from pyanaconda.progress import progressQ
|
|
|
|
import ConfigParser
|
|
import collections
|
|
import itertools
|
|
import logging
|
|
import multiprocessing
|
|
import operator
|
|
import pyanaconda.constants as constants
|
|
from pykickstart.constants import GROUP_ALL, GROUP_DEFAULT, KS_MISSING_IGNORE
|
|
import pyanaconda.errors as errors
|
|
import pyanaconda.iutil
|
|
import pyanaconda.localization
|
|
import pyanaconda.packaging as packaging
|
|
import sys
|
|
import time
|
|
|
|
log = logging.getLogger("packaging")
|
|
|
|
try:
|
|
import dnf
|
|
import dnf.exceptions
|
|
import dnf.repo
|
|
import dnf.callback
|
|
import rpm
|
|
except ImportError as e:
|
|
log.error("dnfpayload: component import failed: %s", e)
|
|
dnf = None
|
|
rpm = None
|
|
|
|
DNF_CACHE_DIR = '/tmp/dnf.cache'
|
|
DNF_PACKAGE_CACHE_DIR_SUFFIX = 'dnf.package.cache'
|
|
DOWNLOAD_MPOINTS = {'/tmp',
|
|
'/',
|
|
'/mnt/sysimage',
|
|
'/mnt/sysimage/home',
|
|
'/mnt/sysimage/tmp',
|
|
'/mnt/sysimage/var',
|
|
}
|
|
REPO_DIRS = ['/etc/yum.repos.d',
|
|
'/etc/anaconda.repos.d',
|
|
'/tmp/updates/anaconda.repos.d',
|
|
'/tmp/product/anaconda.repos.d']
|
|
|
|
def _failure_limbo():
|
|
progressQ.send_quit(1)
|
|
while True:
|
|
time.sleep(10000)
|
|
|
|
def _df_map():
|
|
"""Return (mountpoint -> size available) mapping."""
|
|
output = pyanaconda.iutil.execWithCapture('df', ['--output=target,avail'])
|
|
output = output.rstrip()
|
|
lines = output.splitlines()
|
|
structured = {}
|
|
for line in lines:
|
|
items = line.split()
|
|
key = items[0]
|
|
val = items[1]
|
|
if not key.startswith('/'):
|
|
continue
|
|
structured[key] = Size(int(val)*1024)
|
|
return structured
|
|
|
|
def _paced(fn):
|
|
"""Execute `fn` no more often then every 2 seconds."""
|
|
def paced_fn(self, *args):
|
|
now = time.time()
|
|
if now - self.last_time < 2:
|
|
return
|
|
self.last_time = now
|
|
return fn(self, *args)
|
|
return paced_fn
|
|
|
|
def _pick_mpoint(df, requested):
|
|
def reasonable_mpoint(mpoint):
|
|
return mpoint in DOWNLOAD_MPOINTS
|
|
|
|
# reserve extra
|
|
requested = requested + Size("150 MB")
|
|
sufficients = {key : val for (key,val) in df.items() if val > requested
|
|
and reasonable_mpoint(key)}
|
|
log.info('Sufficient mountpoints found: %s', sufficients)
|
|
|
|
if not len(sufficients):
|
|
return None
|
|
# default to the biggest one:
|
|
return sorted(sufficients.iteritems(), key=operator.itemgetter(1),
|
|
reverse=True)[0][0]
|
|
|
|
class PayloadRPMDisplay(dnf.callback.LoggingTransactionDisplay):
|
|
def __init__(self, queue):
|
|
super(PayloadRPMDisplay, self).__init__()
|
|
self._queue = queue
|
|
self._last_ts = None
|
|
self.cnt = 0
|
|
|
|
def event(self, package, action, te_current, te_total, ts_current, ts_total):
|
|
if action == self.PKG_INSTALL and te_current == 0:
|
|
# do not report same package twice
|
|
if self._last_ts == ts_current:
|
|
return
|
|
self._last_ts = ts_current
|
|
|
|
msg = '%s.%s (%d/%d)' % \
|
|
(package.name, package.arch, ts_current, ts_total)
|
|
self.cnt += 1
|
|
self._queue.put(('install', msg))
|
|
elif action == self.TRANS_POST:
|
|
self._queue.put(('post', None))
|
|
|
|
class DownloadProgress(dnf.callback.DownloadProgress):
|
|
def __init__(self):
|
|
self.downloads = collections.defaultdict(int)
|
|
self.last_time = time.time()
|
|
self.total_files = 0
|
|
self.total_size = Size(0)
|
|
|
|
@_paced
|
|
def _update(self):
|
|
msg = _('Downloading %(total_files)s RPMs, '
|
|
'%(downloaded)s / %(total_size)s (%(percent)d%%) done.')
|
|
downloaded = Size(sum(self.downloads.values()))
|
|
vals = {
|
|
'downloaded' : downloaded,
|
|
'percent' : int(100 * downloaded/self.total_size),
|
|
'total_files' : self.total_files,
|
|
'total_size' : self.total_size
|
|
}
|
|
progressQ.send_message(msg % vals)
|
|
|
|
def end(self, payload, status, err_msg):
|
|
nevra = str(payload)
|
|
if status is dnf.callback.STATUS_OK:
|
|
self.downloads[nevra] = payload.download_size
|
|
self._update()
|
|
return
|
|
log.critical("Failed to download '%s': %d - %s", nevra, status, err_msg)
|
|
|
|
def progress(self, payload, done):
|
|
nevra = str(payload)
|
|
self.downloads[nevra] = done
|
|
self._update()
|
|
|
|
def start(self, total_files, total_size):
|
|
self.total_files = total_files
|
|
self.total_size = Size(total_size)
|
|
|
|
def do_transaction(base, queue):
|
|
try:
|
|
display = PayloadRPMDisplay(queue)
|
|
base.do_transaction(display=display)
|
|
except BaseException as e:
|
|
log.error('The transaction process has ended abruptly')
|
|
log.info(e)
|
|
queue.put(('quit', str(e)))
|
|
|
|
class DNFPayload(packaging.PackagePayload):
|
|
def __init__(self, data):
|
|
packaging.PackagePayload.__init__(self, data)
|
|
if rpm is None or dnf is None:
|
|
raise packaging.PayloadError("unsupported payload type")
|
|
|
|
self._base = None
|
|
self._configure()
|
|
|
|
def unsetup(self):
|
|
super(DNFPayload, self).unsetup()
|
|
self._base = None
|
|
|
|
def _add_repo(self, ksrepo):
|
|
"""Add a repo to the dnf repo object
|
|
|
|
:param ksrepo: Kickstart Repository to add
|
|
:type ksrepo: Kickstart RepoData object.
|
|
:returns: None
|
|
"""
|
|
repo = dnf.repo.Repo(ksrepo.name, DNF_CACHE_DIR)
|
|
url = ksrepo.baseurl
|
|
mirrorlist = ksrepo.mirrorlist
|
|
if url:
|
|
repo.baseurl = [url]
|
|
if mirrorlist:
|
|
repo.mirrorlist = mirrorlist
|
|
repo.sslverify = not (ksrepo.noverifyssl or flags.noverifyssl)
|
|
repo.enable()
|
|
self._base.repos.add(repo)
|
|
|
|
# Load the metadata to verify that the repo is valid
|
|
try:
|
|
self._base.repos[repo.id].load()
|
|
except dnf.exceptions.RepoError as e:
|
|
raise packaging.MetadataError(e)
|
|
|
|
log.info("added repo: '%s'", ksrepo.name)
|
|
|
|
def addRepo(self, ksrepo):
|
|
"""Add a repo to dnf and kickstart repo lists
|
|
|
|
:param ksrepo: Kickstart Repository to add
|
|
:type ksrepo: Kickstart RepoData object.
|
|
:returns: None
|
|
"""
|
|
self._add_repo(ksrepo)
|
|
super(DNFPayload, self).addRepo(ksrepo)
|
|
|
|
def _apply_selections(self):
|
|
if self.data.packages.nocore:
|
|
log.info("skipping core group due to %%packages --nocore; system may not be complete")
|
|
else:
|
|
self._select_group('core')
|
|
|
|
for pkg_name in self.data.packages.packageList:
|
|
log.info("selecting package: '%s'", pkg_name)
|
|
try:
|
|
self._install_package(pkg_name)
|
|
except packaging.NoSuchPackage as e:
|
|
self._miss(e)
|
|
|
|
for group in self.data.packages.groupList:
|
|
if group.name == 'core':
|
|
continue
|
|
try:
|
|
default = group.include in (GROUP_ALL,
|
|
GROUP_DEFAULT)
|
|
optional = group.include == GROUP_ALL
|
|
self._select_group(group.name, default=default, optional=optional)
|
|
except packaging.NoSuchGroup as e:
|
|
self._miss(e)
|
|
|
|
for package in self.requiredPackages:
|
|
self._install_package(package, required=True)
|
|
|
|
for group in self.requiredGroups:
|
|
self._select_group(group, required=True)
|
|
|
|
self._select_kernel_package()
|
|
self._install_package('dnf')
|
|
|
|
def _bump_tx_id(self):
|
|
if self.txID is None:
|
|
self.txID = 1
|
|
else:
|
|
self.txID += 1
|
|
return self.txID
|
|
|
|
def _configure(self):
|
|
self._base = dnf.Base()
|
|
conf = self._base.conf
|
|
conf.cachedir = DNF_CACHE_DIR
|
|
conf.logdir = '/tmp/'
|
|
# disable console output completely:
|
|
conf.debuglevel = 0
|
|
conf.errorlevel = 0
|
|
self._base.logging.setup_from_dnf_conf(conf)
|
|
|
|
conf.releasever = self._getReleaseVersion(None)
|
|
conf.installroot = pyanaconda.iutil.getSysroot()
|
|
conf.prepend_installroot('persistdir')
|
|
|
|
# NSS won't survive the forking we do to shield out chroot during
|
|
# transaction, disable it in RPM:
|
|
conf.tsflags.append('nocrypto')
|
|
|
|
conf.reposdir = REPO_DIRS
|
|
|
|
@property
|
|
def _download_space(self):
|
|
transaction = self._base.transaction
|
|
if transaction is None:
|
|
return Size(0)
|
|
|
|
size = sum(tsi.installed.downloadsize for tsi in transaction)
|
|
return Size(size)
|
|
|
|
def _install_package(self, pkg_name, required=False):
|
|
try:
|
|
return self._base.install(pkg_name)
|
|
except dnf.exceptions.MarkingError:
|
|
raise packaging.NoSuchPackage(pkg_name, required=required)
|
|
|
|
def _miss(self, exn):
|
|
if self.data.packages.handleMissing == KS_MISSING_IGNORE:
|
|
return
|
|
|
|
log.error('Missed: %r', exn)
|
|
if errors.errorHandler.cb(exn) == errors.ERROR_RAISE:
|
|
# The progress bar polls kind of slowly, thus installation could
|
|
# still continue for a bit before the quit message is processed.
|
|
# Doing a sys.exit also ensures the running thread quits before
|
|
# it can do anything else.
|
|
progressQ.send_quit(1)
|
|
sys.exit(1)
|
|
|
|
def _pick_download_location(self):
|
|
required = self._download_space
|
|
df_map = _df_map()
|
|
mpoint = _pick_mpoint(df_map, required)
|
|
log.info("Download space required: %s, use filesystem at: %s", required,
|
|
mpoint)
|
|
if mpoint is None:
|
|
msg = "Not enough disk space to download the packages."
|
|
raise packaging.PayloadError(msg)
|
|
|
|
pkgdir = '%s/%s' % (mpoint, DNF_PACKAGE_CACHE_DIR_SUFFIX)
|
|
for repo in self._base.repos.iter_enabled():
|
|
repo.pkgdir = pkgdir
|
|
|
|
def _select_group(self, group_id, default=True, optional=False, required=False):
|
|
grp = self._base.comps.group_by_pattern(group_id)
|
|
if grp is None:
|
|
raise packaging.NoSuchGroup(group_id, required=required)
|
|
types = {'mandatory'}
|
|
if default:
|
|
types.add('default')
|
|
if optional:
|
|
types.add('optional')
|
|
exclude = self.data.packages.excludedList
|
|
self._base.group_install(grp, types, exclude=exclude)
|
|
|
|
def _select_kernel_package(self):
|
|
kernels = self.kernelPackages
|
|
for kernel in kernels:
|
|
try:
|
|
self._install_package(kernel)
|
|
except packaging.NoSuchPackage:
|
|
log.info('kernel: no such package %s', kernel)
|
|
else:
|
|
log.info('kernel: selected %s', kernel)
|
|
break
|
|
else:
|
|
log.error('kernel: failed to select a kernel from %s', kernels)
|
|
|
|
def _sync_metadata(self, dnf_repo):
|
|
try:
|
|
dnf_repo.load()
|
|
except dnf.exceptions.RepoError as e:
|
|
id_ = dnf_repo.id
|
|
if id_ == self.baseRepo:
|
|
raise packaging.MetadataError(str(e))
|
|
log.info('_sync_metadata: addon repo error: %s', e)
|
|
self.disableRepo(id_)
|
|
|
|
@property
|
|
def baseRepo(self):
|
|
# is any locking needed here as in the yumpayload?
|
|
repo_names = [constants.BASE_REPO_NAME] + self.DEFAULT_REPOS
|
|
for repo in self._base.repos.iter_enabled():
|
|
if repo.id in repo_names:
|
|
return repo.id
|
|
return None
|
|
|
|
@property
|
|
def environments(self):
|
|
environments = self._base.comps.environments_iter()
|
|
return [env.id for env in environments]
|
|
|
|
@property
|
|
def groups(self):
|
|
groups = self._base.comps.groups_iter()
|
|
return [g.id for g in groups]
|
|
|
|
@property
|
|
def mirrorEnabled(self):
|
|
return True
|
|
|
|
@property
|
|
def repos(self):
|
|
# known repo ids
|
|
return [r.id for r in self._base.repos.values()]
|
|
|
|
@property
|
|
def spaceRequired(self):
|
|
transaction = self._base.transaction
|
|
if transaction is None:
|
|
return Size("3000 MB")
|
|
|
|
size = sum(tsi.installed.installsize for tsi in transaction)
|
|
# add 35% to account for the fact that the above method is laughably
|
|
# inaccurate:
|
|
size *= 1.35
|
|
return Size(size)
|
|
|
|
def _isGroupVisible(self, grpid):
|
|
grp = self._base.comps.group_by_pattern(grpid)
|
|
if grp is None:
|
|
raise packaging.NoSuchGroup(grpid)
|
|
return grp.visible
|
|
|
|
def _groupHasInstallableMembers(self, grpid):
|
|
return True
|
|
|
|
def checkSoftwareSelection(self):
|
|
log.info("checking software selection")
|
|
self._bump_tx_id()
|
|
self._base.reset(goal=True)
|
|
self._apply_selections()
|
|
|
|
try:
|
|
if self._base.resolve():
|
|
log.debug("checking dependencies: success.")
|
|
else:
|
|
log.debug("empty transaction")
|
|
except dnf.exceptions.DepsolveError as e:
|
|
msg = str(e)
|
|
log.warning(msg)
|
|
raise packaging.DependencyError([msg])
|
|
|
|
log.info("%d packages selected totalling %s",
|
|
len(self._base.transaction), self.spaceRequired)
|
|
|
|
def disableRepo(self, repo_id):
|
|
try:
|
|
self._base.repos[repo_id].disable()
|
|
log.info("Disabled '%s'", repo_id)
|
|
except KeyError:
|
|
pass
|
|
super(DNFPayload, self).disableRepo(repo_id)
|
|
|
|
def enableRepo(self, repo_id):
|
|
try:
|
|
self._base.repos[repo_id].enable()
|
|
log.info("Enabled '%s'", repo_id)
|
|
except KeyError:
|
|
pass
|
|
super(DNFPayload, self).enableRepo(repo_id)
|
|
|
|
def environmentDescription(self, environmentid):
|
|
env = self._base.comps.environment_by_pattern(environmentid)
|
|
if env is None:
|
|
raise packaging.NoSuchGroup(environmentid)
|
|
return (env.ui_name, env.ui_description)
|
|
|
|
def environmentGroups(self, environmentid):
|
|
env = self._base.comps.environment_by_pattern(environmentid)
|
|
if env is None:
|
|
raise packaging.NoSuchGroup(environmentid)
|
|
group_ids = (id_.name for id_ in env.group_ids)
|
|
option_ids = (id_.name for id_ in env.option_ids)
|
|
return list(itertools.chain(group_ids, option_ids))
|
|
|
|
def environmentHasOption(self, environmentid, grpid):
|
|
env = self._base.comps.environment_by_pattern(environmentid)
|
|
if env is None:
|
|
raise packaging.NoSuchGroup(environmentid)
|
|
return grpid in (id_.name for id_ in env.option_ids)
|
|
|
|
def environmentOptionIsDefault(self, environmentid, grpid):
|
|
env = self._base.comps.environment_by_pattern(environmentid)
|
|
if env is None:
|
|
raise packaging.NoSuchGroup(environmentid)
|
|
return False
|
|
|
|
def groupDescription(self, grpid):
|
|
""" Return name/description tuple for the group specified by id. """
|
|
grp = self._base.comps.group_by_pattern(grpid)
|
|
if grp is None:
|
|
raise packaging.NoSuchGroup(grpid)
|
|
return (grp.ui_name, grp.ui_description)
|
|
|
|
def gatherRepoMetadata(self):
|
|
map(self._sync_metadata, self._base.repos.iter_enabled())
|
|
self._base.fill_sack(load_system_repo=False)
|
|
self._base.read_comps()
|
|
self._refreshEnvironmentAddons()
|
|
|
|
def install(self):
|
|
progressQ.send_message(_('Starting package installation process'))
|
|
|
|
# Add the rpm macros to the global transaction environment
|
|
for macro in self.rpmMacros:
|
|
rpm.addMacro(macro[0], macro[1])
|
|
|
|
if self.install_device:
|
|
self._setupMedia(self.install_device)
|
|
try:
|
|
self.checkSoftwareSelection()
|
|
self._pick_download_location()
|
|
except packaging.PayloadError as e:
|
|
if errors.errorHandler.cb(e) == errors.ERROR_RAISE:
|
|
_failure_limbo()
|
|
|
|
pkgs_to_download = self._base.transaction.install_set
|
|
log.info('Downloading pacakges.')
|
|
progressQ.send_message(_('Downloading packages'))
|
|
progress = DownloadProgress()
|
|
try:
|
|
self._base.download_packages(pkgs_to_download, progress)
|
|
except dnf.exceptions.DownloadError as e:
|
|
msg = 'Failed to download the following packages: %s' % str(e)
|
|
exc = packaging.PayloadInstallError(msg)
|
|
if errors.errorHandler.cb(exc) == errors.ERROR_RAISE:
|
|
_failure_limbo()
|
|
|
|
log.info('Downloading packages finished.')
|
|
|
|
pre_msg = _("Preparing transaction from installation source")
|
|
progressQ.send_message(pre_msg)
|
|
|
|
queue = multiprocessing.Queue()
|
|
process = multiprocessing.Process(target=do_transaction,
|
|
args=(self._base, queue))
|
|
process.start()
|
|
(token, msg) = queue.get()
|
|
while token not in ('post', 'quit'):
|
|
if token == 'install':
|
|
msg = _("Installing %s") % msg
|
|
progressQ.send_message(msg)
|
|
(token, msg) = queue.get()
|
|
|
|
if token == 'quit':
|
|
_failure_limbo()
|
|
|
|
post_msg = _("Performing post-installation setup tasks")
|
|
progressQ.send_message(post_msg)
|
|
process.join()
|
|
self._base.close()
|
|
|
|
def isRepoEnabled(self, repo_id):
|
|
try:
|
|
return self._base.repos[repo_id].enabled
|
|
except (dnf.exceptions.RepoError, KeyError):
|
|
return super(DNFPayload, self).isRepoEnabled(repo_id)
|
|
|
|
def languageGroups(self):
|
|
locales = [self.data.lang.lang] + self.data.lang.addsupport
|
|
match_fn = pyanaconda.localization.langcode_matches_locale
|
|
gids = set()
|
|
gl_tuples = ((g.id, g.lang_only) for g in self._base.comps.groups_iter())
|
|
for (gid, lang) in gl_tuples:
|
|
for locale in locales:
|
|
if match_fn(lang, locale):
|
|
gids.add(gid)
|
|
log.info('languageGroups: %s', gids)
|
|
return list(gids)
|
|
|
|
def preInstall(self, packages=None, groups=None):
|
|
super(DNFPayload, self).preInstall()
|
|
self.requiredPackages = packages
|
|
self.requiredGroups = groups
|
|
self.addDriverRepos()
|
|
|
|
def release(self):
|
|
pass
|
|
|
|
def reset(self, root=None, releasever=None):
|
|
super(DNFPayload, self).reset()
|
|
self.txID = None
|
|
self._base.reset(sack=True, repos=True)
|
|
|
|
def selectEnvironment(self, environmentid):
|
|
env = self._base.comps.environment_by_pattern(environmentid)
|
|
map(self.selectGroup, (id_.name for id_ in env.group_ids))
|
|
|
|
def updateBaseRepo(self, fallback=True, root=None, checkmount=True):
|
|
log.info('configuring base repo')
|
|
self.reset()
|
|
url, mirrorlist, sslverify = self._setupInstallDevice(self.storage,
|
|
checkmount)
|
|
method = self.data.method
|
|
|
|
if method.method:
|
|
try:
|
|
self._base.conf.releasever = self._getReleaseVersion(url)
|
|
log.debug("releasever from %s is %s", url, self._base.conf.releasever)
|
|
except ConfigParser.MissingSectionHeaderError as e:
|
|
log.error("couldn't set releasever from base repo (%s): %s",
|
|
method.method, e)
|
|
|
|
try:
|
|
base_ksrepo = self.data.RepoData(
|
|
name=constants.BASE_REPO_NAME, baseurl=url,
|
|
mirrorlist=mirrorlist, noverifyssl=not sslverify)
|
|
self._add_repo(base_ksrepo)
|
|
except (packaging.MetadataError, packaging.PayloadError) as e:
|
|
log.error("base repo (%s/%s) not valid -- removing it",
|
|
method.method, url)
|
|
self._base.repos.pop(constants.BASE_REPO_NAME, None)
|
|
if not fallback:
|
|
for repo in self._base.repos.iter_enabled():
|
|
self.disableRepo(repo.id)
|
|
return
|
|
|
|
# this preserves the method details while disabling it
|
|
method.method = None
|
|
self.install_device = None
|
|
|
|
if not method.method:
|
|
# only when there's no repo set via method use the repos from the
|
|
# install image itself:
|
|
log.info('Loading repositories config on the filesystem.')
|
|
self._base.read_all_repos()
|
|
|
|
for ksrepo in self.data.repo.dataList():
|
|
self._add_repo(ksrepo)
|
|
|
|
ksnames = [r.name for r in self.data.repo.dataList()]
|
|
ksnames.append(constants.BASE_REPO_NAME)
|
|
for repo in self._base.repos.iter_enabled():
|
|
id_ = repo.id
|
|
if 'source' in id_ or 'debuginfo' in id_:
|
|
self.disableRepo(id_)
|
|
elif constants.isFinal and 'rawhide' in id_:
|
|
self.disableRepo(id_)
|