From: jbeulich@novell.com
Subject: properly predict phys<->mach translations
Patch-mainline: obsolete

--- head-2009-07-28.orig/arch/x86/include/mach-xen/asm/maddr_32.h	2009-07-28 12:14:16.000000000 +0200
+++ head-2009-07-28/arch/x86/include/mach-xen/asm/maddr_32.h	2009-07-29 10:56:35.000000000 +0200
@@ -30,17 +30,19 @@ extern unsigned int   machine_to_phys_or
 
 static inline unsigned long pfn_to_mfn(unsigned long pfn)
 {
-	if (xen_feature(XENFEAT_auto_translated_physmap))
+	if (unlikely(xen_feature(XENFEAT_auto_translated_physmap)))
 		return pfn;
-	BUG_ON(max_mapnr && pfn >= max_mapnr);
+	if (likely(max_mapnr))
+		BUG_ON(pfn >= max_mapnr);
 	return phys_to_machine_mapping[pfn] & ~FOREIGN_FRAME_BIT;
 }
 
 static inline int phys_to_machine_mapping_valid(unsigned long pfn)
 {
-	if (xen_feature(XENFEAT_auto_translated_physmap))
+	if (unlikely(xen_feature(XENFEAT_auto_translated_physmap)))
 		return 1;
-	BUG_ON(max_mapnr && pfn >= max_mapnr);
+	if (likely(max_mapnr))
+		BUG_ON(pfn >= max_mapnr);
 	return (phys_to_machine_mapping[pfn] != INVALID_P2M_ENTRY);
 }
 
@@ -48,7 +50,7 @@ static inline unsigned long mfn_to_pfn(u
 {
 	unsigned long pfn;
 
-	if (xen_feature(XENFEAT_auto_translated_physmap))
+	if (unlikely(xen_feature(XENFEAT_auto_translated_physmap)))
 		return mfn;
 
 	if (unlikely((mfn >> machine_to_phys_order) != 0))
@@ -95,17 +97,18 @@ static inline unsigned long mfn_to_pfn(u
 static inline unsigned long mfn_to_local_pfn(unsigned long mfn)
 {
 	unsigned long pfn = mfn_to_pfn(mfn);
-	if ((pfn < max_mapnr)
-	    && !xen_feature(XENFEAT_auto_translated_physmap)
-	    && (phys_to_machine_mapping[pfn] != mfn))
+	if (likely(pfn < max_mapnr)
+	    && likely(!xen_feature(XENFEAT_auto_translated_physmap))
+	    && unlikely(phys_to_machine_mapping[pfn] != mfn))
 		return max_mapnr; /* force !pfn_valid() */
 	return pfn;
 }
 
 static inline void set_phys_to_machine(unsigned long pfn, unsigned long mfn)
 {
-	BUG_ON(max_mapnr && pfn >= max_mapnr);
-	if (xen_feature(XENFEAT_auto_translated_physmap)) {
+	if (likely(max_mapnr))
+		BUG_ON(pfn >= max_mapnr);
+	if (unlikely(xen_feature(XENFEAT_auto_translated_physmap))) {
 		BUG_ON(pfn != mfn && mfn != INVALID_P2M_ENTRY);
 		return;
 	}
--- head-2009-07-28.orig/arch/x86/include/mach-xen/asm/maddr_64.h	2009-07-28 12:14:16.000000000 +0200
+++ head-2009-07-28/arch/x86/include/mach-xen/asm/maddr_64.h	2009-07-29 10:56:35.000000000 +0200
@@ -25,17 +25,19 @@ extern unsigned int   machine_to_phys_or
 
 static inline unsigned long pfn_to_mfn(unsigned long pfn)
 {
-	if (xen_feature(XENFEAT_auto_translated_physmap))
+	if (unlikely(xen_feature(XENFEAT_auto_translated_physmap)))
 		return pfn;
-	BUG_ON(max_mapnr && pfn >= max_mapnr);
+	if (likely(max_mapnr))
+		BUG_ON(pfn >= max_mapnr);
 	return phys_to_machine_mapping[pfn] & ~FOREIGN_FRAME_BIT;
 }
 
 static inline int phys_to_machine_mapping_valid(unsigned long pfn)
 {
-	if (xen_feature(XENFEAT_auto_translated_physmap))
+	if (unlikely(xen_feature(XENFEAT_auto_translated_physmap)))
 		return 1;
-	BUG_ON(max_mapnr && pfn >= max_mapnr);
+	if (likely(max_mapnr))
+		BUG_ON(pfn >= max_mapnr);
 	return (phys_to_machine_mapping[pfn] != INVALID_P2M_ENTRY);
 }
 
@@ -43,7 +45,7 @@ static inline unsigned long mfn_to_pfn(u
 {
 	unsigned long pfn;
 
-	if (xen_feature(XENFEAT_auto_translated_physmap))
+	if (unlikely(xen_feature(XENFEAT_auto_translated_physmap)))
 		return mfn;
 
 	if (unlikely((mfn >> machine_to_phys_order) != 0))
@@ -90,17 +92,18 @@ static inline unsigned long mfn_to_pfn(u
 static inline unsigned long mfn_to_local_pfn(unsigned long mfn)
 {
 	unsigned long pfn = mfn_to_pfn(mfn);
-	if ((pfn < max_mapnr)
-	    && !xen_feature(XENFEAT_auto_translated_physmap)
-	    && (phys_to_machine_mapping[pfn] != mfn))
+	if (likely(pfn < max_mapnr)
+	    && likely(!xen_feature(XENFEAT_auto_translated_physmap))
+	    && unlikely(phys_to_machine_mapping[pfn] != mfn))
 		return max_mapnr; /* force !pfn_valid() */
 	return pfn;
 }
 
 static inline void set_phys_to_machine(unsigned long pfn, unsigned long mfn)
 {
-	BUG_ON(max_mapnr && pfn >= max_mapnr);
-	if (xen_feature(XENFEAT_auto_translated_physmap)) {
+	if (likely(max_mapnr))
+		BUG_ON(pfn >= max_mapnr);
+	if (unlikely(xen_feature(XENFEAT_auto_translated_physmap))) {
 		BUG_ON(pfn != mfn && mfn != INVALID_P2M_ENTRY);
 		return;
 	}
--- head-2009-07-28.orig/arch/x86/include/mach-xen/asm/pgtable_types.h	2009-07-28 13:14:11.000000000 +0200
+++ head-2009-07-28/arch/x86/include/mach-xen/asm/pgtable_types.h	2009-07-29 10:56:35.000000000 +0200
@@ -207,7 +207,7 @@ typedef struct { pgdval_t pgd; } pgd_t;
 #define __pgd_ma(x) ((pgd_t) { (x) } )
 static inline pgd_t xen_make_pgd(pgdval_t val)
 {
-	if (val & _PAGE_PRESENT)
+	if (likely(val & _PAGE_PRESENT))
 		val = pte_phys_to_machine(val);
 	return (pgd_t) { val };
 }
@@ -217,10 +217,10 @@ static inline pgdval_t xen_pgd_val(pgd_t
 {
 	pgdval_t ret = __pgd_val(pgd);
 #if PAGETABLE_LEVELS == 2 && CONFIG_XEN_COMPAT <= 0x030002
-	if (ret)
+	if (likely(ret))
 		ret = machine_to_phys(ret) | _PAGE_PRESENT;
 #else
-	if (ret & _PAGE_PRESENT)
+	if (likely(ret & _PAGE_PRESENT))
 		ret = pte_machine_to_phys(ret);
 #endif
 	return ret;
@@ -237,7 +237,7 @@ typedef struct { pudval_t pud; } pud_t;
 #define __pud_ma(x) ((pud_t) { (x) } )
 static inline pud_t xen_make_pud(pudval_t val)
 {
-	if (val & _PAGE_PRESENT)
+	if (likely(val & _PAGE_PRESENT))
 		val = pte_phys_to_machine(val);
 	return (pud_t) { val };
 }
@@ -246,7 +246,7 @@ static inline pud_t xen_make_pud(pudval_
 static inline pudval_t xen_pud_val(pud_t pud)
 {
 	pudval_t ret = __pud_val(pud);
-	if (ret & _PAGE_PRESENT)
+	if (likely(ret & _PAGE_PRESENT))
 		ret = pte_machine_to_phys(ret);
 	return ret;
 }
@@ -266,7 +266,7 @@ typedef struct { pmdval_t pmd; } pmd_t;
 #define __pmd_ma(x)	((pmd_t) { (x) } )
 static inline pmd_t xen_make_pmd(pmdval_t val)
 {
-	if (val & _PAGE_PRESENT)
+	if (likely(val & _PAGE_PRESENT))
 		val = pte_phys_to_machine(val);
 	return (pmd_t) { val };
 }
@@ -276,10 +276,10 @@ static inline pmdval_t xen_pmd_val(pmd_t
 {
 	pmdval_t ret = __pmd_val(pmd);
 #if CONFIG_XEN_COMPAT <= 0x030002
-	if (ret)
+	if (likely(ret))
 		ret = pte_machine_to_phys(ret) | _PAGE_PRESENT;
 #else
-	if (ret & _PAGE_PRESENT)
+	if (likely(ret & _PAGE_PRESENT))
 		ret = pte_machine_to_phys(ret);
 #endif
 	return ret;
@@ -308,7 +308,7 @@ static inline pmdval_t pmd_flags(pmd_t p
 #define __pte_ma(x) ((pte_t) { .pte = (x) } )
 static inline pte_t xen_make_pte(pteval_t val)
 {
-	if ((val & (_PAGE_PRESENT|_PAGE_IOMAP)) == _PAGE_PRESENT)
+	if (likely((val & (_PAGE_PRESENT|_PAGE_IOMAP)) == _PAGE_PRESENT))
 		val = pte_phys_to_machine(val);
 	return (pte_t) { .pte = val };
 }
@@ -317,7 +317,7 @@ static inline pte_t xen_make_pte(pteval_
 static inline pteval_t xen_pte_val(pte_t pte)
 {
 	pteval_t ret = __pte_val(pte);
-	if ((pte.pte_low & (_PAGE_PRESENT|_PAGE_IOMAP)) == _PAGE_PRESENT)
+	if (likely((pte.pte_low & (_PAGE_PRESENT|_PAGE_IOMAP)) == _PAGE_PRESENT))
 		ret = pte_machine_to_phys(ret);
 	return ret;
 }