[PATCH v2 3/3] xen/x86: Change stub page allocation/free logic

[Jason Andryuk <jason.andryuk@xxxxxxx>]

Today the inine tracking of the stub page is problematic.  0xcc is used
to indicate unused, but it is also a "clear value."  A !CONFIG_PV build
with smt=0 will bring up CPU0, bring up CPU1, bring down CPU1, and free
the in-use stub page.  Subsequent CPU onlining can write to the re-used
page.

Each stub page accomodates 32 stub regions, and each CPU uses an offset
into its portion of the page.  Each CPU used a CPU-specific mapping of
the whole page.  The virtual address of the CPU-specific mapping is
fixed, so it can be used to track the stub page.

Remove the actual free-ing from cpu_smpboot_free().  Use the stub_va PTE
to track the underlying page.  destroy_xen_mapping() would clear the
mapping, so replace it with modify_xen_mappings() to retain the PFN in
the PTE (with NX set).

In alloc_stub_page(), check for a valid PFN in the stub_va PTE.  When
found, it will be used.  This handles re-onlining a CPU.  Otherwise the
existing logic is retained to use a passed in mfn or allocate one.
These paths handle to bringing up new CPUs.

If all CPUs for a stub page are offlined, the page will be dangling and
unusable.  But it will be re-used if CPUs are re-onlined.

Signed-off-by: Jason Andryuk <jason.andryuk@xxxxxxx>
---
 xen/arch/x86/smpboot.c | 30 +++++++++++++++++++-----------
 1 file changed, 19 insertions(+), 11 deletions(-)

diff --git a/xen/arch/x86/smpboot.c b/xen/arch/x86/smpboot.c
index 7241dba621..11937175a9 100644
--- a/xen/arch/x86/smpboot.c
+++ b/xen/arch/x86/smpboot.c
@@ -647,11 +647,21 @@ unsigned long alloc_stub_page(unsigned int cpu, unsigned 
long *mfn)
 {
     unsigned long stub_va;
     struct page_info *pg;
+    mfn_t stub_mfn;
 
     BUILD_BUG_ON(STUBS_PER_PAGE & (STUBS_PER_PAGE - 1));
 
-    if ( *mfn )
+    stub_va = XEN_VIRT_END - FIXADDR_X_SIZE - (cpu + 1) * PAGE_SIZE;
+    stub_mfn = page_walk_mfn(virt_to_mfn(idle_pg_table), stub_va);
+    if ( mfn_valid(stub_mfn) )
+    {
+        *mfn = mfn_x(stub_mfn);
+        pg = mfn_to_page(stub_mfn);
+    }
+    else if ( *mfn )
+    {
         pg = mfn_to_page(_mfn(*mfn));
+    }
     else
     {
         nodeid_t node = cpu_to_node(cpu);
@@ -664,7 +674,6 @@ unsigned long alloc_stub_page(unsigned int cpu, unsigned 
long *mfn)
         unmap_domain_page(memset(__map_domain_page(pg), 0xcc, PAGE_SIZE));
     }
 
-    stub_va = XEN_VIRT_END - FIXADDR_X_SIZE - (cpu + 1) * PAGE_SIZE;
     if ( map_pages_to_xen(stub_va, page_to_mfn(pg), 1,
                           PAGE_HYPERVISOR_RX | MAP_SMALL_PAGES) )
     {
@@ -990,19 +999,18 @@ static void cpu_smpboot_free(unsigned int cpu, bool 
remove)
     {
         mfn_t mfn = _mfn(per_cpu(stubs.mfn, cpu));
         unsigned char *stub_page = map_domain_page(mfn);
-        unsigned int i;
 
         memset(stub_page + STUB_BUF_CPU_OFFS(cpu), 0xcc, STUB_BUF_SIZE);
-        for ( i = 0; i < STUBS_PER_PAGE; ++i )
-            if ( stub_page[i * STUB_BUF_SIZE] != 0xcc )
-                break;
         unmap_domain_page(stub_page);
-        destroy_xen_mappings(per_cpu(stubs.addr, cpu) & PAGE_MASK,
-                             (per_cpu(stubs.addr, cpu) | ~PAGE_MASK) + 1);
+        /*
+         * destroy_xen_mappings() clears the PFN from the PTE, but we want to
+         * keep it for potential reuse if re-onlined.  Pass _PAGE_PRESENT to
+         * retain the PFN.
+         */
+        modify_xen_mappings(per_cpu(stubs.addr, cpu) & PAGE_MASK,
+                            (per_cpu(stubs.addr, cpu) | ~PAGE_MASK) + 1,
+                            _PAGE_PRESENT | _PAGE_NX);
         per_cpu(stubs.addr, cpu) = 0;
-        per_cpu(stubs.mfn, cpu) = 0;
-        if ( i == STUBS_PER_PAGE )
-            free_domheap_page(mfn_to_page(mfn));
     }
 
     if ( IS_ENABLED(CONFIG_PV32) )
-- 
2.54.0