qubes-linux-kernel/patches.xen/xen-pcpu-hotplug

From: Jiang, Yunhong <yunhong.jiang@intel.com>	
Subject: xen/acpi: Export host physical CPU information to dom0
References: bnc#651066
Patch-mainline: n/a

This patch expose host's physical CPU information to dom0 in sysfs, so
that dom0's management tools can control the physical CPU if needed.

It also provides interface in sysfs to logical online/offline a
physical CPU.

Notice: The information in dom0 is synced with xen hypervisor
asynchronously.

From: Jiang, Yunhong <yunhong.jiang@intel.com>	
Subject: Add cpu hotplug support for 2.6.32 branch

Add physical CPU hotplug support to origin/xen/next-2.6.32 branch.
Please notice that, even with this change, the acpi_processor->id is 
still always -1. This is because several workaround in PM side depends
on acpi_processor->id == -1. As the CPU hotplug logic does not depends
on acpi_processor->id, I'd still keep it no changes.

But we need change the acpi_processor->id in the future.

Signed-off-by: Jiang, Yunhong <yunhong.jiang@intel.com>

jb: ported over glue logic; retry loops around XENPF_get_cpuinfo;
    cleanup.
Acked-by: jbeulich@novell.com

--- head-2011-01-30.orig/arch/x86/kernel/acpi/processor_extcntl_xen.c	2011-02-02 15:09:40.000000000 +0100
+++ head-2011-01-30/arch/x86/kernel/acpi/processor_extcntl_xen.c	2011-02-02 15:09:57.000000000 +0100
@@ -181,9 +181,69 @@ static int xen_tx_notifier(struct acpi_p
 {
 	return -EINVAL;
 }
+
 static int xen_hotplug_notifier(struct acpi_processor *pr, int event)
 {
-	return -EINVAL;
+	int ret = -EINVAL;
+#ifdef CONFIG_ACPI_HOTPLUG_CPU
+	acpi_status status = 0;
+	acpi_object_type type;
+	uint32_t apic_id;
+	int device_decl = 0;
+	unsigned long long pxm;
+	xen_platform_op_t op = {
+		.interface_version  = XENPF_INTERFACE_VERSION,
+	};
+
+	status = acpi_get_type(pr->handle, &type);
+	if (ACPI_FAILURE(status)) {
+		pr_warning("can't get object type for acpi_id %#x\n",
+			   pr->acpi_id);
+		return -ENXIO;
+	}
+
+	switch (type) {
+	case ACPI_TYPE_PROCESSOR:
+		break;
+	case ACPI_TYPE_DEVICE:
+		device_decl = 1;
+		break;
+	default:
+		pr_warning("unsupported object type %#x for acpi_id %#x\n",
+			   type, pr->acpi_id);
+		return -EOPNOTSUPP;
+	}
+
+	apic_id = acpi_get_cpuid(pr->handle, ~device_decl, pr->acpi_id);
+	if (apic_id < 0) {
+		pr_warning("can't get apic_id for acpi_id %#x\n",
+			   pr->acpi_id);
+		return -ENODATA;
+	}
+
+	status = acpi_evaluate_integer(pr->handle, "_PXM", NULL, &pxm);
+	if (ACPI_FAILURE(status)) {
+		pr_warning("can't get pxm for acpi_id %#x\n",
+			   pr->acpi_id);
+		return -ENODATA;
+	}
+
+	switch (event) {
+	case HOTPLUG_TYPE_ADD:
+		op.cmd = XENPF_cpu_hotadd;
+		op.u.cpu_add.apic_id = apic_id;
+		op.u.cpu_add.acpi_id = pr->acpi_id;
+		op.u.cpu_add.pxm = pxm;
+		ret = HYPERVISOR_platform_op(&op);
+		break;
+	case HOTPLUG_TYPE_REMOVE:
+		pr_warning("Xen doesn't support CPU hot remove\n");
+		ret = -EOPNOTSUPP;
+		break;
+	}
+#endif
+
+	return ret;
 }
 
 static struct processor_extcntl_ops xen_extcntl_ops = {
@@ -194,8 +254,10 @@ static int __init init_extcntl(void)
 {
 	unsigned int pmbits = (xen_start_info->flags & SIF_PM_MASK) >> 8;
 
+#ifndef CONFIG_ACPI_HOTPLUG_CPU
 	if (!pmbits)
 		return 0;
+#endif
 	if (pmbits & XEN_PROCESSOR_PM_CX)
 		xen_extcntl_ops.pm_ops[PM_TYPE_IDLE] = xen_cx_notifier;
 	if (pmbits & XEN_PROCESSOR_PM_PX)
--- head-2011-01-30.orig/drivers/acpi/processor_driver.c	2011-02-01 15:03:10.000000000 +0100
+++ head-2011-01-30/drivers/acpi/processor_driver.c	2011-02-02 15:09:57.000000000 +0100
@@ -82,7 +82,7 @@ MODULE_LICENSE("GPL");
 static int acpi_processor_add(struct acpi_device *device);
 static int acpi_processor_remove(struct acpi_device *device, int type);
 static void acpi_processor_notify(struct acpi_device *device, u32 event);
-static acpi_status acpi_processor_hotadd_init(acpi_handle handle, int *p_cpu);
+static acpi_status acpi_processor_hotadd_init(struct acpi_processor *pr);
 static int acpi_processor_handle_eject(struct acpi_processor *pr);
 
 
@@ -324,8 +324,7 @@ static int acpi_processor_get_info(struc
 	 *  they are physically not present.
 	 */
 	if (pr->id == -1) {
-		if (ACPI_FAILURE
-		    (acpi_processor_hotadd_init(pr->handle, &pr->id)) &&
+		if (ACPI_FAILURE(acpi_processor_hotadd_init(pr)) &&
 		    acpi_get_cpuid(pr->handle, ~device_declaration,
 				   pr->acpi_id) < 0) {
 			return -ENODEV;
@@ -789,13 +788,26 @@ processor_walk_namespace_cb(acpi_handle 
 	return (AE_OK);
 }
 
-static acpi_status acpi_processor_hotadd_init(acpi_handle handle, int *p_cpu)
+static acpi_status acpi_processor_hotadd_init(struct acpi_processor *pr)
 {
+	acpi_handle handle = pr->handle;
+	int *p_cpu = &pr->id;
+
+#ifdef CONFIG_XEN
+	if (xen_pcpu_index(pr->acpi_id, 1) != -1)
+		return AE_OK;
+#endif
 
 	if (!is_processor_present(handle)) {
 		return AE_ERROR;
 	}
 
+	if (processor_cntl_external()) {
+		processor_notify_external(pr, PROCESSOR_HOTPLUG,
+					  HOTPLUG_TYPE_ADD);
+		return AE_OK;
+	}
+
 	if (acpi_map_lsapic(handle, p_cpu))
 		return AE_ERROR;
 
@@ -809,10 +821,11 @@ static acpi_status acpi_processor_hotadd
 
 static int acpi_processor_handle_eject(struct acpi_processor *pr)
 {
-#ifdef CONFIG_XEN
-	if (pr->id == -1)
+	if (processor_cntl_external()) {
+		processor_notify_external(pr, PROCESSOR_HOTPLUG,
+					  HOTPLUG_TYPE_REMOVE);
 		return (0);
-#endif
+	}
 
 	if (cpu_online(pr->id))
 		cpu_down(pr->id);
@@ -822,7 +835,7 @@ static int acpi_processor_handle_eject(s
 	return (0);
 }
 #else
-static acpi_status acpi_processor_hotadd_init(acpi_handle handle, int *p_cpu)
+static acpi_status acpi_processor_hotadd_init(struct acpi_processor *pr)
 {
 	return AE_ERROR;
 }
--- head-2011-01-30.orig/drivers/acpi/processor_extcntl.c	2011-02-01 15:03:03.000000000 +0100
+++ head-2011-01-30/drivers/acpi/processor_extcntl.c	2011-02-02 15:09:57.000000000 +0100
@@ -83,10 +83,13 @@ int processor_notify_external(struct acp
 
 		ret = processor_extcntl_ops->pm_ops[type](pr, event);
 		break;
+#ifdef CONFIG_ACPI_HOTPLUG_CPU
 	case PROCESSOR_HOTPLUG:
 		if (processor_extcntl_ops->hotplug)
 			ret = processor_extcntl_ops->hotplug(pr, type);
+		xen_pcpu_hotplug(type);
 		break;
+#endif
 	default:
 		pr_err("Unsupported processor event %d.\n", event);
 		break;
--- head-2011-01-30.orig/drivers/xen/core/Makefile	2011-02-02 15:09:52.000000000 +0100
+++ head-2011-01-30/drivers/xen/core/Makefile	2011-02-02 15:09:57.000000000 +0100
@@ -5,6 +5,7 @@
 obj-y := evtchn.o gnttab.o reboot.o machine_reboot.o firmware.o
 
 obj-$(CONFIG_PCI)		+= pci.o
+obj-$(CONFIG_ACPI_HOTPLUG_CPU)	+= pcpu.o
 obj-$(CONFIG_PROC_FS)		+= xen_proc.o
 obj-$(CONFIG_SYS_HYPERVISOR)	+= hypervisor_sysfs.o
 obj-$(CONFIG_HOTPLUG_CPU)	+= cpu_hotplug.o
--- /dev/null	1970-01-01 00:00:00.000000000 +0000
+++ head-2011-01-30/drivers/xen/core/pcpu.c	2011-02-02 15:09:57.000000000 +0100
@@ -0,0 +1,416 @@
+/*
+ * pcpu.c - management physical cpu in dom0 environment
+ */
+#include <linux/acpi.h>
+#include <linux/cpu.h>
+#include <linux/interrupt.h>
+#include <linux/kobject.h>
+#include <linux/list.h>
+#include <linux/spinlock.h>
+#include <linux/sysdev.h>
+#include <asm/hypervisor.h>
+#include <xen/interface/platform.h>
+#include <xen/evtchn.h>
+#include <acpi/processor.h>
+
+struct pcpu {
+	struct list_head pcpu_list;
+	struct sys_device sysdev;
+	uint32_t xen_id;
+	uint32_t apic_id;
+	uint32_t acpi_id;
+	uint32_t flags;
+};
+
+static inline int xen_pcpu_online(uint32_t flags)
+{
+	return !!(flags & XEN_PCPU_FLAGS_ONLINE);
+}
+
+static DEFINE_MUTEX(xen_pcpu_lock);
+
+/* No need for irq disable since hotplug notify is in workqueue context */
+#define get_pcpu_lock() mutex_lock(&xen_pcpu_lock);
+#define put_pcpu_lock() mutex_unlock(&xen_pcpu_lock);
+
+static LIST_HEAD(xen_pcpus);
+
+static int xen_pcpu_down(uint32_t xen_id)
+{
+	xen_platform_op_t op = {
+		.cmd			= XENPF_cpu_offline,
+		.interface_version	= XENPF_INTERFACE_VERSION,
+		.u.cpu_ol.cpuid	= xen_id,
+	};
+
+	return HYPERVISOR_platform_op(&op);
+}
+
+static int xen_pcpu_up(uint32_t xen_id)
+{
+	xen_platform_op_t op = {
+		.cmd			= XENPF_cpu_online,
+		.interface_version	= XENPF_INTERFACE_VERSION,
+		.u.cpu_ol.cpuid	= xen_id,
+	};
+
+	return HYPERVISOR_platform_op(&op);
+}
+
+static ssize_t show_online(struct sys_device *dev,
+			   struct sysdev_attribute *attr,
+			   char *buf)
+{
+	struct pcpu *cpu = container_of(dev, struct pcpu, sysdev);
+
+	return sprintf(buf, "%d\n", xen_pcpu_online(cpu->flags));
+}
+
+static ssize_t store_online(struct sys_device *dev,
+			    struct sysdev_attribute *attr,
+			    const char *buf, size_t count)
+{
+	struct pcpu *cpu = container_of(dev, struct pcpu, sysdev);
+	ssize_t ret;
+
+	switch (buf[0]) {
+	case '0':
+		ret = xen_pcpu_down(cpu->xen_id);
+		break;
+	case '1':
+		ret = xen_pcpu_up(cpu->xen_id);
+		break;
+	default:
+		ret = -EINVAL;
+	}
+
+	if (ret >= 0)
+		ret = count;
+	return ret;
+}
+
+static SYSDEV_ATTR(online, 0644, show_online, store_online);
+
+static ssize_t show_apicid(struct sys_device *dev,
+			   struct sysdev_attribute *attr,
+			   char *buf)
+{
+	struct pcpu *cpu = container_of(dev, struct pcpu, sysdev);
+
+	return sprintf(buf, "%#x\n", cpu->apic_id);
+}
+static SYSDEV_ATTR(apic_id, 0444, show_apicid, NULL);
+
+static ssize_t show_acpiid(struct sys_device *dev,
+			   struct sysdev_attribute *attr,
+			   char *buf)
+{
+	struct pcpu *cpu = container_of(dev, struct pcpu, sysdev);
+
+	return sprintf(buf, "%#x\n", cpu->acpi_id);
+}
+static SYSDEV_ATTR(acpi_id, 0444, show_acpiid, NULL);
+
+static struct sysdev_class xen_pcpu_sysdev_class = {
+	.name = "xen_pcpu",
+};
+
+static int xen_pcpu_free(struct pcpu *pcpu)
+{
+	if (!pcpu)
+		return 0;
+
+	sysdev_remove_file(&pcpu->sysdev, &attr_online);
+	sysdev_remove_file(&pcpu->sysdev, &attr_apic_id);
+	sysdev_remove_file(&pcpu->sysdev, &attr_acpi_id);
+	sysdev_unregister(&pcpu->sysdev);
+	list_del(&pcpu->pcpu_list);
+	kfree(pcpu);
+
+	return 0;
+}
+
+static inline int same_pcpu(struct xenpf_pcpuinfo *info,
+			    struct pcpu *pcpu)
+{
+	return (pcpu->apic_id == info->apic_id) &&
+		(pcpu->xen_id == info->xen_cpuid);
+}
+
+/*
+ * Return 1 if online status changed
+ */
+static int xen_pcpu_online_check(struct xenpf_pcpuinfo *info,
+				 struct pcpu *pcpu)
+{
+	int result = 0;
+
+	if (info->xen_cpuid != pcpu->xen_id)
+		return 0;
+
+	if (xen_pcpu_online(info->flags) && !xen_pcpu_online(pcpu->flags)) {
+		/* the pcpu is onlined */
+		pcpu->flags |= XEN_PCPU_FLAGS_ONLINE;
+		kobject_uevent(&pcpu->sysdev.kobj, KOBJ_ONLINE);
+		result = 1;
+	} else if (!xen_pcpu_online(info->flags) &&
+		   xen_pcpu_online(pcpu->flags))  {
+		/* The pcpu is offlined now */
+		pcpu->flags &= ~XEN_PCPU_FLAGS_ONLINE;
+		kobject_uevent(&pcpu->sysdev.kobj, KOBJ_OFFLINE);
+		result = 1;
+	}
+
+	return result;
+}
+
+static int pcpu_sysdev_init(struct pcpu *cpu)
+{
+	int error;
+
+	error = sysdev_register(&cpu->sysdev);
+	if (error) {
+		pr_warning("xen_pcpu_add: Failed to register pcpu\n");
+		kfree(cpu);
+		return -1;
+	}
+	sysdev_create_file(&cpu->sysdev, &attr_online);
+	sysdev_create_file(&cpu->sysdev, &attr_apic_id);
+	sysdev_create_file(&cpu->sysdev, &attr_acpi_id);
+	return 0;
+}
+
+static struct pcpu *get_pcpu(unsigned int xen_id)
+{
+	struct pcpu *pcpu;
+
+	list_for_each_entry(pcpu, &xen_pcpus, pcpu_list)
+		if (pcpu->xen_id == xen_id)
+			return pcpu;
+
+	return NULL;
+}
+
+static struct pcpu *init_pcpu(struct xenpf_pcpuinfo *info)
+{
+	struct pcpu *pcpu;
+
+	if (info->flags & XEN_PCPU_FLAGS_INVALID)
+		return NULL;
+
+	/* The PCPU is just added */
+	pcpu = kzalloc(sizeof(struct pcpu), GFP_KERNEL);
+	if (!pcpu)
+		return NULL;
+
+	INIT_LIST_HEAD(&pcpu->pcpu_list);
+	pcpu->xen_id = info->xen_cpuid;
+	pcpu->apic_id = info->apic_id;
+	pcpu->acpi_id = info->acpi_id;
+	pcpu->flags = info->flags;
+
+	pcpu->sysdev.cls = &xen_pcpu_sysdev_class;
+	pcpu->sysdev.id = info->xen_cpuid;
+
+	if (pcpu_sysdev_init(pcpu)) {
+		kfree(pcpu);
+		return NULL;
+	}
+
+	list_add_tail(&pcpu->pcpu_list, &xen_pcpus);
+	return pcpu;
+}
+
+#define PCPU_NO_CHANGE			0
+#define PCPU_ADDED			1
+#define PCPU_ONLINE_OFFLINE		2
+#define PCPU_REMOVED			3
+/*
+ * Caller should hold the pcpu lock
+ * < 0: Something wrong
+ * 0: No changes
+ * > 0: State changed
+ */
+static struct pcpu *_sync_pcpu(unsigned int cpu_num, unsigned int *max_id,
+			       int *result)
+{
+	struct pcpu *pcpu;
+	struct xenpf_pcpuinfo *info;
+	xen_platform_op_t op = {
+		.cmd                = XENPF_get_cpuinfo,
+		.interface_version  = XENPF_INTERFACE_VERSION,
+	};
+	int ret;
+
+	*result = -1;
+
+	info = &op.u.pcpu_info;
+	info->xen_cpuid = cpu_num;
+
+	do {
+		ret = HYPERVISOR_platform_op(&op);
+	} while (ret == -EBUSY);
+	if (ret)
+		return NULL;
+
+	if (max_id)
+		*max_id = op.u.pcpu_info.max_present;
+
+	pcpu = get_pcpu(cpu_num);
+
+	if (info->flags & XEN_PCPU_FLAGS_INVALID) {
+		/* The pcpu has been removed */
+		*result = PCPU_NO_CHANGE;
+		if (pcpu) {
+			xen_pcpu_free(pcpu);
+			*result = PCPU_REMOVED;
+		}
+		return NULL;
+	}
+
+
+	if (!pcpu) {
+		*result = PCPU_ADDED;
+		pcpu = init_pcpu(info);
+		if (pcpu == NULL) {
+			pr_warning("Failed to init pcpu %x\n",
+				   info->xen_cpuid);
+			*result = -1;
+		}
+	} else {
+		*result = PCPU_NO_CHANGE;
+		/*
+		 * Old PCPU is replaced with a new pcpu, this means
+		 * several virq is missed, will it happen?
+		 */
+		if (!same_pcpu(info, pcpu)) {
+			pr_warning("Pcpu %x changed!\n", pcpu->xen_id);
+			pcpu->apic_id = info->apic_id;
+			pcpu->acpi_id = info->acpi_id;
+		}
+		if (xen_pcpu_online_check(info, pcpu))
+			*result = PCPU_ONLINE_OFFLINE;
+	}
+	return pcpu;
+}
+
+/*
+ * Sync dom0's pcpu information with xen hypervisor's
+ */
+static int xen_sync_pcpus(void)
+{
+	/*
+	 * Boot cpu always have cpu_id 0 in xen
+	 */
+	unsigned int cpu_num = 0, max_id = 0;
+	int result = 0;
+	struct pcpu *pcpu;
+
+	get_pcpu_lock();
+
+	while ((result >= 0) && (cpu_num <= max_id)) {
+		pcpu = _sync_pcpu(cpu_num, &max_id, &result);
+
+		switch (result)	{
+		case PCPU_NO_CHANGE:
+		case PCPU_ADDED:
+		case PCPU_ONLINE_OFFLINE:
+		case PCPU_REMOVED:
+			break;
+		default:
+			pr_warning("Failed to sync pcpu %x\n", cpu_num);
+			break;
+		}
+		cpu_num++;
+	}
+
+	if (result < 0) {
+		struct pcpu *tmp;
+
+		list_for_each_entry_safe(pcpu, tmp, &xen_pcpus, pcpu_list)
+			xen_pcpu_free(pcpu);
+	}
+
+	put_pcpu_lock();
+
+	return 0;
+}
+
+static void xen_pcpu_dpc(struct work_struct *work)
+{
+	if (xen_sync_pcpus() < 0)
+		pr_warning("xen_pcpu_dpc: Failed to sync pcpu information\n");
+}
+static DECLARE_WORK(xen_pcpu_work, xen_pcpu_dpc);
+
+static irqreturn_t xen_pcpu_interrupt(int irq, void *dev_id)
+{
+	schedule_work(&xen_pcpu_work);
+
+	return IRQ_HANDLED;
+}
+
+int xen_pcpu_hotplug(int type)
+{
+	schedule_work(&xen_pcpu_work);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(xen_pcpu_hotplug);
+
+int xen_pcpu_index(uint32_t id, bool is_acpiid)
+{
+	unsigned int cpu_num, max_id;
+	xen_platform_op_t op = {
+		.cmd                = XENPF_get_cpuinfo,
+		.interface_version  = XENPF_INTERFACE_VERSION,
+	};
+	struct xenpf_pcpuinfo *info = &op.u.pcpu_info;
+
+	for (max_id = cpu_num = 0; cpu_num <= max_id; ++cpu_num) {
+		int ret;
+
+		info->xen_cpuid = cpu_num;
+		do {
+			ret = HYPERVISOR_platform_op(&op);
+		} while (ret == -EBUSY);
+		if (ret)
+			continue;
+
+		if (info->max_present > max_id)
+			max_id = info->max_present;
+		if (id == (is_acpiid ? info->acpi_id : info->apic_id))
+			return cpu_num;
+	}
+
+	return -1;
+}
+EXPORT_SYMBOL_GPL(xen_pcpu_index);
+
+static int __init xen_pcpu_init(void)
+{
+	int err;
+
+	if (!is_initial_xendomain())
+		return 0;
+
+	err = sysdev_class_register(&xen_pcpu_sysdev_class);
+	if (err) {
+		pr_warning("xen_pcpu_init: "
+			   "Failed to register sysdev class (%d)\n", err);
+		return err;
+	}
+
+	xen_sync_pcpus();
+
+	if (!list_empty(&xen_pcpus))
+		err = bind_virq_to_irqhandler(VIRQ_PCPU_STATE, 0,
+					      xen_pcpu_interrupt, 0,
+					      "pcpu", NULL);
+	if (err < 0)
+		pr_warning("xen_pcpu_init: "
+			   "Failed to bind pcpu_state virq (%d)\n", err);
+
+	return err;
+}
+subsys_initcall(xen_pcpu_init);
--- head-2011-01-30.orig/include/acpi/processor.h	2011-02-01 15:03:10.000000000 +0100
+++ head-2011-01-30/include/acpi/processor.h	2011-02-02 15:09:57.000000000 +0100
@@ -509,6 +509,8 @@ static inline void xen_convert_psd_pack(
 	xpsd->num_processors = apsd->num_processors;
 }
 
+extern int xen_pcpu_hotplug(int type);
+extern int xen_pcpu_index(uint32_t id, bool is_acpiid);
 #endif /* CONFIG_XEN */
 
 #endif