You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
qubes-installer-qubes-os/anaconda/pyanaconda/storage_utils.py

616 lines
25 KiB

#
# Copyright (C) 2014 Red Hat, Inc.
#
# This copyrighted material is made available to anyone wishing to use,
# modify, copy, or redistribute it subject to the terms and conditions of
# the GNU General Public License v.2, or (at your option) any later version.
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY expressed or implied, including the implied warranties of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details. You should have received a copy of the
# GNU General Public License along with this program; if not, write to the
# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA. Any Red Hat trademarks that are incorporated in the
# source code or documentation are not subject to the GNU General Public
# License and may only be used or replicated with the express permission of
# Red Hat, Inc.
#
"""UI-independent storage utility functions"""
import re
import locale
import os
from contextlib import contextmanager
from blivet import arch
from blivet import util
from blivet import udev
from blivet.size import Size
from blivet.errors import StorageError
from blivet.platform import platform as _platform
from blivet.devicefactory import DEVICE_TYPE_LVM
from blivet.devicefactory import DEVICE_TYPE_LVM_THINP
from blivet.devicefactory import DEVICE_TYPE_BTRFS
from blivet.devicefactory import DEVICE_TYPE_MD
from blivet.devicefactory import DEVICE_TYPE_PARTITION
from blivet.devicefactory import DEVICE_TYPE_DISK
from pyanaconda.i18n import _, N_
from pyanaconda import isys
from pyanaconda.constants import productName
from pyanaconda.errors import errorHandler, ERROR_RAISE
from pykickstart.constants import AUTOPART_TYPE_PLAIN, AUTOPART_TYPE_BTRFS
from pykickstart.constants import AUTOPART_TYPE_LVM, AUTOPART_TYPE_LVM_THINP
import logging
log = logging.getLogger("anaconda")
# TODO: all those constants and mappings should go to blivet
DEVICE_TEXT_LVM = N_("LVM")
DEVICE_TEXT_LVM_THINP = N_("LVM Thin Provisioning")
DEVICE_TEXT_MD = N_("RAID")
DEVICE_TEXT_PARTITION = N_("Standard Partition")
DEVICE_TEXT_BTRFS = N_("Btrfs")
DEVICE_TEXT_DISK = N_("Disk")
DEVICE_TEXT_MAP = {DEVICE_TYPE_LVM: DEVICE_TEXT_LVM,
DEVICE_TYPE_MD: DEVICE_TEXT_MD,
DEVICE_TYPE_PARTITION: DEVICE_TEXT_PARTITION,
DEVICE_TYPE_BTRFS: DEVICE_TEXT_BTRFS,
DEVICE_TYPE_LVM_THINP: DEVICE_TEXT_LVM_THINP,
DEVICE_TYPE_DISK: DEVICE_TEXT_DISK}
PARTITION_ONLY_FORMAT_TYPES = ("macefi", "prepboot", "biosboot", "appleboot")
MOUNTPOINT_DESCRIPTIONS = {"Swap": N_("The 'swap' area on your computer is used by the operating\n"
"system when running low on memory."),
"Boot": N_("The 'boot' area on your computer is where files needed\n"
"to start the operating system are stored."),
"Root": N_("The 'root' area on your computer is where core system\n"
"files and applications are stored."),
"Home": N_("The 'home' area on your computer is where all your personal\n"
"data is stored."),
"BIOS Boot": N_("The BIOS boot partition is required to enable booting\n"
"from GPT-partitioned disks on BIOS hardware."),
"PReP Boot": N_("The PReP boot partition is required as part of the\n"
"boot loader configuration on some PPC platforms.")
}
AUTOPART_CHOICES = ((N_("Standard Partition"), AUTOPART_TYPE_PLAIN),
(N_("Btrfs"), AUTOPART_TYPE_BTRFS),
(N_("LVM"), AUTOPART_TYPE_LVM),
(N_("LVM Thin Provisioning"), AUTOPART_TYPE_LVM_THINP))
AUTOPART_DEVICE_TYPES = {AUTOPART_TYPE_LVM: DEVICE_TYPE_LVM,
AUTOPART_TYPE_LVM_THINP: DEVICE_TYPE_LVM_THINP,
AUTOPART_TYPE_PLAIN: DEVICE_TYPE_PARTITION,
AUTOPART_TYPE_BTRFS: DEVICE_TYPE_BTRFS}
NAMED_DEVICE_TYPES = (DEVICE_TYPE_BTRFS, DEVICE_TYPE_LVM, DEVICE_TYPE_MD, DEVICE_TYPE_LVM_THINP)
CONTAINER_DEVICE_TYPES = (DEVICE_TYPE_LVM, DEVICE_TYPE_BTRFS, DEVICE_TYPE_LVM_THINP)
udev_device_dict_cache = None
def size_from_input(input_str, units=None):
""" Get a Size object from an input string.
:param str input_str: a string forming some representation of a size
:param units: use these units if none specified in input_str
:type units: str or NoneType
:returns: a Size object corresponding to input_str
:rtype: :class:`blivet.size.Size` or NoneType
Units default to bytes if no units in input_str or units.
"""
if not input_str:
# Nothing to parse
return None
# A string ending with a digit contains no units information.
if re.search(r'[\d.%s]$' % locale.nl_langinfo(locale.RADIXCHAR), input_str):
input_str += units or ""
try:
size = Size(input_str)
except ValueError:
return None
return size
def device_type_from_autopart(autopart_type):
"""Get device type matching the given autopart type."""
return AUTOPART_DEVICE_TYPES.get(autopart_type, None)
class UIStorageFilter(logging.Filter):
"""Logging filter for UI storage events"""
def filter(self, record):
record.name = "storage.ui"
return True
@contextmanager
def ui_storage_logger():
"""Context manager that applies the UIStorageFilter for its block"""
storage_log = logging.getLogger("blivet")
storage_filter = UIStorageFilter()
storage_log.addFilter(storage_filter)
yield
storage_log.removeFilter(storage_filter)
class SanityException(Exception):
pass
class SanityError(SanityException):
pass
class SanityWarning(SanityException):
pass
class LUKSDeviceWithoutKeyError(SanityError):
pass
def sanity_check(storage, min_ram=isys.MIN_RAM):
"""
Run a series of tests to verify the storage configuration.
This function is called at the end of partitioning so that
we can make sure you don't have anything silly (like no /,
a really small /, etc).
:param storage: an instance of the :class:`blivet.Blivet` class to check
:param min_ram: minimum RAM (in MiB) needed for the installation with swap
space available
:rtype: a list of SanityExceptions
:return: a list of accumulated errors and warnings
"""
exns = []
checkSizes = [('/usr', Size("250 MiB")), ('/tmp', Size("50 MiB")), ('/var', Size("384 MiB")),
('/home', Size("100 MiB")), ('/boot', Size("200 MiB"))]
mustbeonlinuxfs = ['/', '/var', '/tmp', '/usr', '/home', '/usr/share', '/usr/lib']
mustbeonroot = ['/bin', '/dev', '/sbin', '/etc', '/lib', '/root', '/mnt', 'lost+found', '/proc']
filesystems = storage.mountpoints
root = storage.fsset.root_device
swaps = storage.fsset.swap_devices
if root:
if root.size < Size("250 MiB"):
exns.append(
SanityWarning(_("Your root partition is less than 250 "
"megabytes which is usually too small to "
"install %s.") % (productName,)))
else:
exns.append(
SanityError(_("You have not defined a root partition (/), "
"which is required for installation of %s "
"to continue.") % (productName,)))
# Prevent users from installing on s390x with (a) no /boot volume, (b) the
# root volume on LVM, and (c) the root volume not restricted to a single
# PV
# NOTE: There is not really a way for users to create a / volume
# restricted to a single PV. The backend support is there, but there are
# no UI hook-ups to drive that functionality, but I do not personally
# care. --dcantrell
if arch.is_s390() and '/boot' not in storage.mountpoints and root:
if root.type == 'lvmlv' and not root.single_pv:
exns.append(
SanityError(_("This platform requires /boot on a dedicated "
"partition or logical volume. If you do not "
"want a /boot volume, you must place / on a "
"dedicated non-LVM partition.")))
# FIXME: put a check here for enough space on the filesystems. maybe?
for (mount, size) in checkSizes:
if mount in filesystems and filesystems[mount].size < size:
exns.append(
SanityWarning(_("Your %(mount)s partition is less than "
"%(size)s which is lower than recommended "
"for a normal %(productName)s install.")
% {'mount': mount, 'size': size,
'productName': productName}))
# storage.mountpoints is a property that returns a new dict each time, so
# iterating over it is thread-safe.
for (mount, device) in filesystems.items():
problem = filesystems[mount].check_size()
if problem < 0:
exns.append(
SanityError(_("Your %(mount)s partition is too small for %(format)s formatting "
"(allowable size is %(minSize)s to %(maxSize)s)")
% {"mount": mount, "format": device.format.name,
"minSize": device.min_size, "maxSize": device.max_size}))
elif problem > 0:
exns.append(
SanityError(_("Your %(mount)s partition is too large for %(format)s formatting "
"(allowable size is %(minSize)s to %(maxSize)s)")
% {"mount":mount, "format": device.format.name,
"minSize": device.min_size, "maxSize": device.max_size}))
if storage.bootloader and not storage.bootloader.skip_bootloader:
stage1 = storage.bootloader.stage1_device
if not stage1:
exns.append(
SanityError(_("No valid boot loader target device found. "
"See below for details.")))
pe = _platform.stage1_missing_error
if pe:
exns.append(SanityError(_(pe)))
else:
storage.bootloader.is_valid_stage1_device(stage1)
exns.extend(SanityError(msg) for msg in storage.bootloader.errors)
exns.extend(SanityWarning(msg) for msg in storage.bootloader.warnings)
stage2 = storage.bootloader.stage2_device
if stage1 and not stage2:
exns.append(SanityError(_("You have not created a bootable partition.")))
else:
storage.bootloader.is_valid_stage2_device(stage2)
exns.extend(SanityError(msg) for msg in storage.bootloader.errors)
exns.extend(SanityWarning(msg) for msg in storage.bootloader.warnings)
if not storage.bootloader.check():
exns.extend(SanityError(msg) for msg in storage.bootloader.errors)
#
# check that GPT boot disk on BIOS system has a BIOS boot partition
#
if _platform.weight(fstype="biosboot") and \
stage1 and stage1.is_disk and \
getattr(stage1.format, "labelType", None) == "gpt":
missing = True
for part in [p for p in storage.partitions if p.disk == stage1]:
if part.format.type == "biosboot":
missing = False
break
if missing:
exns.append(
SanityError(_("Your BIOS-based system needs a special "
"partition to boot from a GPT disk label. "
"To continue, please create a 1MiB "
"'biosboot' type partition.")))
if not swaps:
installed = util.total_memory()
required = Size("%s MiB" % (min_ram + isys.NO_SWAP_EXTRA_RAM))
if installed < required:
exns.append(
SanityError(_("You have not specified a swap partition. "
"%(requiredMem)s of memory is required to continue installation "
"without a swap partition, but you only have %(installedMem)s.")
% {"requiredMem": required,
"installedMem": installed}))
else:
exns.append(
SanityWarning(_("You have not specified a swap partition. "
"Although not strictly required in all cases, "
"it will significantly improve performance "
"for most installations.")))
no_uuid = [s for s in swaps if s.format.exists and not s.format.uuid]
if no_uuid:
exns.append(
SanityWarning(_("At least one of your swap devices does not have "
"a UUID, which is common in swap space created "
"using older versions of mkswap. These devices "
"will be referred to by device path in "
"/etc/fstab, which is not ideal since device "
"paths can change under a variety of "
"circumstances. ")))
for (mountpoint, dev) in filesystems.items():
if mountpoint in mustbeonroot:
exns.append(
SanityError(_("This mount point is invalid. The %s directory must "
"be on the / file system.") % mountpoint))
if mountpoint in mustbeonlinuxfs and (not dev.format.mountable or not dev.format.linux_native):
exns.append(
SanityError(_("The mount point %s must be on a linux file system.") % mountpoint))
if storage.root_device and storage.root_device.format.exists:
e = storage.must_format(storage.root_device)
if e:
exns.append(SanityError(e))
exns += verify_LUKS_devices_have_key(storage)
exns += check_mounted_partitions(storage)
return exns
def verify_LUKS_devices_have_key(storage):
"""
Verify that all non-existant LUKS devices have some way of obtaining
a key.
Note: LUKS device creation will fail without a key.
:rtype: generator of str
:returns: a generator of error messages, may yield no error messages
"""
for dev in (d for d in storage.devices if \
d.format.type == "luks" and \
not d.format.exists and \
not d.format.has_key):
yield LUKSDeviceWithoutKeyError(_("Encryption requested for LUKS device %s but no encryption key specified for this device.") % (dev.name,))
def check_mounted_partitions(storage):
""" Check the selected disks to make sure all their partitions are unmounted.
:rtype: generator of str
:returns: a generator of error messages, may yield no error messages
"""
for disk in storage.disks:
if not disk.partitioned:
continue
for part in disk.format.partitions:
part_dev = storage.devicetree.get_device_by_path(part.path)
if part_dev and part_dev.protected:
log.debug("Not checking protected %s for being mounted, assuming live image mount", part.path)
continue
if part.busy:
yield SanityError(_("%s is currently mounted and cannot be used for the "
"installation. Please unmount it and retry.") % part.path)
def bound_size(size, device, old_size):
""" Returns a size bounded by the maximum and minimum size for
the device.
:param size: the candidate size
:type size: :class:`blivet.size.Size`
:param device: the device being displayed
:type device: :class:`blivet.devices.StorageDevice`
:param old_size: the fallback size
:type old_size: :class:`blivet.size.Size`
:returns: a size to which to set the device
:rtype: :class:`blivet.size.Size`
If size is 0, interpreted as set size to maximum possible.
If no maximum size is available, reset size to old_size, but
log a warning.
"""
max_size = device.max_size
min_size = device.min_size
if not size:
if max_size:
log.info("No size specified, using maximum size for this device (%d).", max_size)
size = max_size
else:
log.warning("No size specified and no maximum size available, setting size back to original size (%d).", old_size)
size = old_size
else:
if max_size:
if size > max_size:
log.warning("Size specified (%d) is greater than the maximum size for this device (%d), using maximum size.", size, max_size)
size = max_size
else:
log.warning("Unknown upper bound on size. Using requested size (%d).", size)
if size < min_size:
log.warning("Size specified (%d) is less than the minimum size for this device (%d), using minimum size.", size, min_size)
size = min_size
return size
def try_populate_devicetree(devicetree):
"""
Try to populate the given devicetree while catching errors and dealing with
some special ones in a nice way (giving user chance to do something about
them).
:param devicetree: devicetree to try to populate
:type decicetree: :class:`blivet.devicetree.DeviceTree`
"""
while True:
try:
devicetree.populate()
except StorageError as e:
if errorHandler.cb(e) == ERROR_RAISE:
raise
else:
continue
else:
break
return
class StorageSnapshot(object):
"""R/W snapshot of storage (i.e. a :class:`blivet.Blivet` instance)"""
def __init__(self, storage=None):
"""
Create new instance of the class
:param storage: if given, its snapshot is created
:type storage: :class:`blivet.Blivet`
"""
if storage:
self._storage_snap = storage.copy()
else:
self._storage_snap = None
@property
def storage(self):
return self._storage_snap
@property
def created(self):
return bool(self._storage_snap)
def create_snapshot(self, storage):
"""Create (and save) snapshot of storage"""
self._storage_snap = storage.copy()
def dispose_snapshot(self):
"""
Dispose (unref) the snapshot
.. note::
In order to free the memory taken by the snapshot, all references
returned by :property:`self.storage` have to be unrefed too.
"""
self._storage_snap = None
def reset_to_snapshot(self, storage, dispose=False):
"""
Reset storage to snapshot (**modifies :param:`storage` in place**)
:param storage: :class:`blivet.Blivet` instance to reset to the created snapshot
:param bool dispose: whether to dispose the snapshot after reset or not
:raises ValueError: if no snapshot is available (was not created before)
"""
if not self.created:
raise ValueError("No snapshot created, cannot reset")
# we need to create a new copy from the snapshot first -- simple
# assignment from the snapshot would result in snapshot being modified
# by further changes of 'storage'
new_copy = self._storage_snap.copy()
storage.devicetree = new_copy.devicetree
storage.roots = new_copy.roots
storage.fsset = new_copy.fsset
if dispose:
self.dispose_snapshot()
# a snapshot of early storage as we got it from scanning disks without doing any
# changes
on_disk_storage = StorageSnapshot()
def filter_unsupported_disklabel_devices(devices):
""" Return input list minus any devices that exist on an unsupported disklabel. """
return [d for d in devices
if not any(not getattr(p, "disklabel_supported", True) for p in d.ancestors)]
def device_name_is_disk(device_name, devicetree=None, refresh_udev_cache=False):
"""Report if the given device name corresponds to a disk device.
Check if the device name is a disk device or not. This function uses
the provided Blivet devicetree for the checking and Blivet udev module
if no devicetree is provided.
Please note that the udev based check uses an internal cache that is generated
when this function is first called in the udev checking mode. This basically
means that udev devices added later will not be taken into account.
If this is a problem for your usecase then use the refresh_udev_cache option
to force a refresh of the udev cache.
:param str device_name: name of the device to check
:param devicetree: device tree to look up devices in (optional)
:type devicetree: :class:`blivet.DeviceTree`
:param bool refresh_udev_cache: governs if the udev device cache should be refreshed
:returns: True if the device name corresponds to a disk, False if not
:rtype: bool
"""
if devicetree is None:
global udev_device_dict_cache
if device_name:
if udev_device_dict_cache is None or refresh_udev_cache:
# Lazy load the udev dick that contains the {device_name : udev_device,..,}
# mappings. The operation could be quite costly due to udev_settle() calls,
# so we cache it in this non-elegant way.
# An unfortunate side effect of this is that udev devices that show up after
# this function is called for the first time will not be taken into account.
udev_device_dict_cache = {udev.device_get_name(d): d for d in udev.get_devices()}
udev_device = udev_device_dict_cache.get(device_name)
return udev_device and udev.device_is_realdisk(udev_device)
else:
return False
else:
device = devicetree.get_device_by_name(device_name)
return device and device.is_disk
def device_matches(spec, devicetree=None, disks_only=False):
"""Return names of block devices matching the provided specification.
:param str spec: a device identifier (name, UUID=<uuid>, &c)
:keyword devicetree: device tree to look up devices in (optional)
:type devicetree: :class:`blivet.DeviceTree`
:param bool disks_only: if only disk devices matching the spec should be returned
:returns: names of matching devices
:rtype: list of str
The spec can contain multiple "sub specs" delimited by a |, for example:
"sd*|hd*|vd*"
In such case we resolve the specs from left to right and return all
unique matches, for example:
["sda", "sda1", "sda2", "sdb", "sdb1", "vdb"]
If disks_only is specified we only return
disk devices matching the spec. For the example above
the output with disks_only=True would be:
["sda", "sdb", "vdb"]
Also note that parse methods will not have access to a devicetree, while execute
methods will. The devicetree is superior in that it can resolve md
array names and in that it reflects scheduled device removals, but for
normal local disks udev.resolve_devspec should suffice.
"""
matches = []
# the device specifications might contain multiple "sub specs" separated by a |
# - the specs are processed from left to right
for single_spec in spec.split("|"):
full_spec = single_spec
if not full_spec.startswith("/dev/"):
full_spec = os.path.normpath("/dev/" + full_spec)
# the regular case
single_spec_matches = udev.resolve_glob(full_spec)
for match in single_spec_matches:
if match not in matches:
# skip non-disk devices in disk-only mode
if disks_only and not device_name_is_disk(match):
continue
matches.append(match)
dev_name = None
# Use spec here instead of full_spec to preserve the spec and let the
# called code decide whether to treat the spec as a path instead of a name.
if devicetree is None:
# we run the spec through resolve_devspec() here as unlike resolve_glob()
# it can also resolve labels and UUIDs
dev_name = udev.resolve_devspec(single_spec)
if disks_only and dev_name:
if not device_name_is_disk(dev_name):
dev_name = None # not a disk
else:
# devicetree can also handle labels and UUIDs
device = devicetree.resolve_device(single_spec)
if device:
dev_name = device.name
if disks_only and not device_name_is_disk(dev_name, devicetree=devicetree):
dev_name = None # not a disk
# The dev_name variable can be None if the spec is not not found or is not valid,
# but we don't want that ending up in the list.
if dev_name and dev_name not in matches:
matches.append(dev_name)
return matches