UPSTREAM: KVM: arm64: Fix host stage-2 PGD refcount

The KVM page-table library refcounts the pages of concatenated stage-2
PGDs individually. However, when running KVM in protected mode, the
host's stage-2 PGD is currently managed by EL2 as a single high-order
compound page, which can cause the refcount of the tail pages to reach 0
when they shouldn't, hence corrupting the page-table.

Fix this by introducing a new hyp_split_page() helper in the EL2 page
allocator (matching the kernel's split_page() function), and make use of
it from host_s2_zalloc_pages_exact().

Fixes: 1025c8c0c6ac ("KVM: arm64: Wrap the host with a stage 2")
Acked-by: Will Deacon <will@kernel.org>
Suggested-by: Will Deacon <will@kernel.org>
Signed-off-by: Quentin Perret <qperret@google.com>
Signed-off-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20211005090155.734578-5-qperret@google.com
(cherry picked from commit 1d58a17ef54599506d44c45ac95be27273a4d2b1)
Bug: 187129171
Signed-off-by: Connor O'Brien <connoro@google.com>
Change-Id: Idf0d8328710517aee822f77a2097295734c78e7a

arch/arm64/kvm/hyp/include/nvhe/gfp.h

@@ -59,6 +59,7 @@ static inline void hyp_set_page_refcounted(struct hyp_page *p)
 
 /* Allocation */
 void *hyp_alloc_pages(struct hyp_pool *pool, unsigned int order);
+void hyp_split_page(struct hyp_page *page);
 void hyp_get_page(void *addr);
 void hyp_put_page(void *addr);
 

arch/arm64/kvm/hyp/nvhe/mem_protect.c

@@ -36,7 +36,18 @@ static const u8 pkvm_hyp_id = 1;
 
 static void *host_s2_zalloc_pages_exact(size_t size)
 {
-	return hyp_alloc_pages(&host_s2_mem, get_order(size));
+	void *addr = hyp_alloc_pages(&host_s2_mem, get_order(size));
+
+	hyp_split_page(hyp_virt_to_page(addr));
+
+	/*
+	 * The size of concatenated PGDs is always a power of two of PAGE_SIZE,
+	 * so there should be no need to free any of the tail pages to make the
+	 * allocation exact.
+	 */
+	WARN_ON(size != (PAGE_SIZE << get_order(size)));
+
+	return addr;
 }
 
 static void *host_s2_zalloc_page(void *pool)

arch/arm64/kvm/hyp/nvhe/page_alloc.c

@@ -140,6 +140,20 @@ void hyp_get_page(void *addr)
 	hyp_page_ref_inc(p);
 }
 
+void hyp_split_page(struct hyp_page *p)
+{
+	unsigned short order = p->order;
+	unsigned int i;
+
+	p->order = 0;
+	for (i = 1; i < (1 << order); i++) {
+		struct hyp_page *tail = p + i;
+
+		tail->order = 0;
+		hyp_set_page_refcounted(tail);
+	}
+}
+
 void *hyp_alloc_pages(struct hyp_pool *pool, unsigned int order)
 {
 	unsigned int i = order;