summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorKent Overstreet <kent.overstreet@gmail.com>2019-02-11 17:04:33 -0500
committerKent Overstreet <kent.overstreet@gmail.com>2019-04-03 12:44:06 -0400
commitbf55d426aaf035ba15e9ca0be286c4945e8c9864 (patch)
tree5c38bfb1e65e4575645414e005ba51d2f1c65e14
parentebf8f73a7e25f4f0c70365d5186332198094aea5 (diff)
Propagate gfp_t when allocating pte entries from __vmalloc
This fixes a lockdep recursion when using __vmalloc from places that aren't GFP_KERNEL safe. Signed-off-by: Kent Overstreet <kent.overstreet@gmail.com>
-rw-r--r--arch/alpha/include/asm/pgalloc.h11
-rw-r--r--arch/arc/include/asm/pgalloc.h9
-rw-r--r--arch/arm/include/asm/pgalloc.h11
-rw-r--r--arch/arm/mm/idmap.c2
-rw-r--r--arch/arm/mm/mmu.c5
-rw-r--r--arch/arm/mm/pgd.c8
-rw-r--r--arch/arm64/include/asm/pgalloc.h17
-rw-r--r--arch/arm64/mm/hugetlbpage.c8
-rw-r--r--arch/csky/include/asm/pgalloc.h4
-rw-r--r--arch/hexagon/include/asm/pgalloc.h5
-rw-r--r--arch/ia64/include/asm/pgalloc.h14
-rw-r--r--arch/ia64/mm/hugetlbpage.c4
-rw-r--r--arch/ia64/mm/init.c6
-rw-r--r--arch/m68k/include/asm/mcf_pgalloc.h12
-rw-r--r--arch/m68k/include/asm/motorola_pgalloc.h7
-rw-r--r--arch/m68k/include/asm/sun3_pgalloc.h12
-rw-r--r--arch/m68k/mm/kmap.c5
-rw-r--r--arch/m68k/sun3x/dvma.c6
-rw-r--r--arch/microblaze/include/asm/pgalloc.h6
-rw-r--r--arch/microblaze/mm/pgtable.c6
-rw-r--r--arch/mips/include/asm/pgalloc.h14
-rw-r--r--arch/mips/mm/hugetlbpage.c4
-rw-r--r--arch/mips/mm/ioremap.c6
-rw-r--r--arch/nds32/include/asm/pgalloc.h14
-rw-r--r--arch/nds32/kernel/dma.c4
-rw-r--r--arch/nios2/include/asm/pgalloc.h8
-rw-r--r--arch/nios2/mm/ioremap.c6
-rw-r--r--arch/openrisc/include/asm/pgalloc.h2
-rw-r--r--arch/openrisc/mm/ioremap.c8
-rw-r--r--arch/parisc/include/asm/pgalloc.h16
-rw-r--r--arch/parisc/kernel/pci-dma.c6
-rw-r--r--arch/parisc/mm/hugetlbpage.c4
-rw-r--r--arch/powerpc/include/asm/book3s/32/pgalloc.h4
-rw-r--r--arch/powerpc/include/asm/book3s/64/pgalloc.h20
-rw-r--r--arch/powerpc/include/asm/nohash/32/pgalloc.h6
-rw-r--r--arch/powerpc/include/asm/nohash/64/pgalloc.h14
-rw-r--r--arch/powerpc/kvm/book3s_64_mmu_radix.c2
-rw-r--r--arch/powerpc/mm/hugetlbpage.c8
-rw-r--r--arch/powerpc/mm/pgtable-book3e.c6
-rw-r--r--arch/powerpc/mm/pgtable-book3s64.c14
-rw-r--r--arch/powerpc/mm/pgtable-hash64.c6
-rw-r--r--arch/powerpc/mm/pgtable-radix.c12
-rw-r--r--arch/powerpc/mm/pgtable_32.c6
-rw-r--r--arch/riscv/include/asm/pgalloc.h11
-rw-r--r--arch/s390/include/asm/pgalloc.h25
-rw-r--r--arch/s390/mm/hugetlbpage.c6
-rw-r--r--arch/s390/mm/pgalloc.c10
-rw-r--r--arch/s390/mm/pgtable.c6
-rw-r--r--arch/s390/mm/vmem.c2
-rw-r--r--arch/sh/include/asm/pgalloc.h7
-rw-r--r--arch/sh/mm/hugetlbpage.c4
-rw-r--r--arch/sh/mm/init.c4
-rw-r--r--arch/sh/mm/pgtable.c8
-rw-r--r--arch/sparc/include/asm/pgalloc_32.h6
-rw-r--r--arch/sparc/include/asm/pgalloc_64.h12
-rw-r--r--arch/sparc/mm/hugetlbpage.c4
-rw-r--r--arch/sparc/mm/init_64.c10
-rw-r--r--arch/sparc/mm/srmmu.c2
-rw-r--r--arch/um/include/asm/pgalloc.h2
-rw-r--r--arch/um/include/asm/pgtable-3level.h3
-rw-r--r--arch/um/kernel/mem.c17
-rw-r--r--arch/um/kernel/skas/mmu.c4
-rw-r--r--arch/unicore32/include/asm/pgalloc.h8
-rw-r--r--arch/unicore32/mm/pgd.c2
-rw-r--r--arch/x86/include/asm/pgalloc.h30
-rw-r--r--arch/x86/kernel/espfix_64.c2
-rw-r--r--arch/x86/kernel/tboot.c6
-rw-r--r--arch/x86/mm/pgtable.c4
-rw-r--r--arch/x86/platform/efi/efi_64.c9
-rw-r--r--arch/xtensa/include/asm/pgalloc.h4
-rw-r--r--include/asm-generic/4level-fixup.h6
-rw-r--r--include/asm-generic/5level-fixup.h6
-rw-r--r--include/asm-generic/pgtable-nop4d-hack.h2
-rw-r--r--include/asm-generic/pgtable-nop4d.h2
-rw-r--r--include/asm-generic/pgtable-nopmd.h2
-rw-r--r--include/asm-generic/pgtable-nopud.h2
-rw-r--r--include/linux/mm.h40
-rw-r--r--include/linux/vmalloc.h2
-rw-r--r--lib/ioremap.c8
-rw-r--r--mm/hugetlb.c11
-rw-r--r--mm/kasan/init.c8
-rw-r--r--mm/memory.c51
-rw-r--r--mm/migrate.c6
-rw-r--r--mm/mremap.c6
-rw-r--r--mm/userfaultfd.c6
-rw-r--r--mm/vmalloc.c49
-rw-r--r--mm/zsmalloc.c2
-rw-r--r--virt/kvm/arm/mmu.c6
88 files changed, 378 insertions, 393 deletions
diff --git a/arch/alpha/include/asm/pgalloc.h b/arch/alpha/include/asm/pgalloc.h
index 02f9f91bb4f0..6b8336865e0c 100644
--- a/arch/alpha/include/asm/pgalloc.h
+++ b/arch/alpha/include/asm/pgalloc.h
@@ -39,9 +39,9 @@ pgd_free(struct mm_struct *mm, pgd_t *pgd)
}
static inline pmd_t *
-pmd_alloc_one(struct mm_struct *mm, unsigned long address)
+pmd_alloc_one(struct mm_struct *mm, unsigned long address, gfp_t gfp)
{
- pmd_t *ret = (pmd_t *)__get_free_page(GFP_KERNEL|__GFP_ZERO);
+ pmd_t *ret = (pmd_t *)get_zeroed_page(gfp);
return ret;
}
@@ -52,10 +52,9 @@ pmd_free(struct mm_struct *mm, pmd_t *pmd)
}
static inline pte_t *
-pte_alloc_one_kernel(struct mm_struct *mm)
+pte_alloc_one_kernel(struct mm_struct *mm, gfp_t gfp)
{
- pte_t *pte = (pte_t *)__get_free_page(GFP_KERNEL|__GFP_ZERO);
- return pte;
+ return (pte_t *)get_zeroed_page(gfp);
}
static inline void
@@ -67,7 +66,7 @@ pte_free_kernel(struct mm_struct *mm, pte_t *pte)
static inline pgtable_t
pte_alloc_one(struct mm_struct *mm)
{
- pte_t *pte = pte_alloc_one_kernel(mm);
+ pte_t *pte = pte_alloc_one_kernel(mm, GFP_KERNEL);
struct page *page;
if (!pte)
diff --git a/arch/arc/include/asm/pgalloc.h b/arch/arc/include/asm/pgalloc.h
index 9c9b5a5ebf2e..491535bb2bfc 100644
--- a/arch/arc/include/asm/pgalloc.h
+++ b/arch/arc/include/asm/pgalloc.h
@@ -90,14 +90,9 @@ static inline int __get_order_pte(void)
return get_order(PTRS_PER_PTE * sizeof(pte_t));
}
-static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm)
+static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, gfp_t gfp)
{
- pte_t *pte;
-
- pte = (pte_t *) __get_free_pages(GFP_KERNEL | __GFP_ZERO,
- __get_order_pte());
-
- return pte;
+ return (pte_t *) __get_free_pages(gfp|__GFP_ZERO, __get_order_pte());
}
static inline pgtable_t
diff --git a/arch/arm/include/asm/pgalloc.h b/arch/arm/include/asm/pgalloc.h
index 17ab72f0cc4e..f21ba862f667 100644
--- a/arch/arm/include/asm/pgalloc.h
+++ b/arch/arm/include/asm/pgalloc.h
@@ -27,9 +27,10 @@
#ifdef CONFIG_ARM_LPAE
-static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
+static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr,
+ gfp_t gfp)
{
- return (pmd_t *)get_zeroed_page(GFP_KERNEL);
+ return (pmd_t *)get_zeroed_page(gfp);
}
static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
@@ -48,7 +49,7 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
/*
* Since we have only two-level page tables, these are trivial
*/
-#define pmd_alloc_one(mm,addr) ({ BUG(); ((pmd_t *)2); })
+#define pmd_alloc_one(mm,addr,gfp) ({ BUG(); ((pmd_t *)2); })
#define pmd_free(mm, pmd) do { } while (0)
#define pud_populate(mm,pmd,pte) BUG()
@@ -81,11 +82,11 @@ static inline void clean_pte_table(pte_t *pte)
* +------------+
*/
static inline pte_t *
-pte_alloc_one_kernel(struct mm_struct *mm)
+pte_alloc_one_kernel(struct mm_struct *mm, gfp_t gfp)
{
pte_t *pte;
- pte = (pte_t *)__get_free_page(PGALLOC_GFP);
+ pte = (pte_t *)get_zeroed_page(gfp);
if (pte)
clean_pte_table(pte);
diff --git a/arch/arm/mm/idmap.c b/arch/arm/mm/idmap.c
index 1d1edd064199..64b44901565f 100644
--- a/arch/arm/mm/idmap.c
+++ b/arch/arm/mm/idmap.c
@@ -27,7 +27,7 @@ static void idmap_add_pmd(pud_t *pud, unsigned long addr, unsigned long end,
unsigned long next;
if (pud_none_or_clear_bad(pud) || (pud_val(*pud) & L_PGD_SWAPPER)) {
- pmd = pmd_alloc_one(&init_mm, addr);
+ pmd = pmd_alloc_one(&init_mm, addr, GFP_KERNEL);
if (!pmd) {
pr_warn("Failed to allocate identity pmd.\n");
return;
diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c
index f5cc1ccfea3d..d31ca40274e1 100644
--- a/arch/arm/mm/mmu.c
+++ b/arch/arm/mm/mmu.c
@@ -980,10 +980,11 @@ void __init create_mapping_late(struct mm_struct *mm, struct map_desc *md,
bool ng)
{
#ifdef CONFIG_ARM_LPAE
- pud_t *pud = pud_alloc(mm, pgd_offset(mm, md->virtual), md->virtual);
+ pud_t *pud = pud_alloc(mm, pgd_offset(mm, md->virtual), md->virtual,
+ GFP_KERNEL);
if (WARN_ON(!pud))
return;
- pmd_alloc(mm, pud, 0);
+ pmd_alloc(mm, pud, 0, GFP_KERNEL);
#endif
__create_mapping(mm, md, late_alloc, ng);
}
diff --git a/arch/arm/mm/pgd.c b/arch/arm/mm/pgd.c
index a1606d950251..6c3a6406726e 100644
--- a/arch/arm/mm/pgd.c
+++ b/arch/arm/mm/pgd.c
@@ -57,11 +57,11 @@ pgd_t *pgd_alloc(struct mm_struct *mm)
* Allocate PMD table for modules and pkmap mappings.
*/
new_pud = pud_alloc(mm, new_pgd + pgd_index(MODULES_VADDR),
- MODULES_VADDR);
+ MODULES_VADDR, GFP_KERNEL);
if (!new_pud)
goto no_pud;
- new_pmd = pmd_alloc(mm, new_pud, 0);
+ new_pmd = pmd_alloc(mm, new_pud, 0, GFP_KERNEL);
if (!new_pmd)
goto no_pmd;
#endif
@@ -72,11 +72,11 @@ pgd_t *pgd_alloc(struct mm_struct *mm)
* contains the machine vectors. The vectors are always high
* with LPAE.
*/
- new_pud = pud_alloc(mm, new_pgd, 0);
+ new_pud = pud_alloc(mm, new_pgd, 0, GFP_KERNEL);
if (!new_pud)
goto no_pud;
- new_pmd = pmd_alloc(mm, new_pud, 0);
+ new_pmd = pmd_alloc(mm, new_pud, 0, GFP_KERNEL);
if (!new_pmd)
goto no_pmd;
diff --git a/arch/arm64/include/asm/pgalloc.h b/arch/arm64/include/asm/pgalloc.h
index 52fa47c73bf0..54199d52ea6e 100644
--- a/arch/arm64/include/asm/pgalloc.h
+++ b/arch/arm64/include/asm/pgalloc.h
@@ -26,14 +26,14 @@
#define check_pgt_cache() do { } while (0)
-#define PGALLOC_GFP (GFP_KERNEL | __GFP_ZERO)
#define PGD_SIZE (PTRS_PER_PGD * sizeof(pgd_t))
#if CONFIG_PGTABLE_LEVELS > 2
-static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
+static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr,
+ gfp_t gfp)
{
- return (pmd_t *)__get_free_page(PGALLOC_GFP);
+ return (pmd_t *)get_zeroed_page(gfp);
}
static inline void pmd_free(struct mm_struct *mm, pmd_t *pmdp)
@@ -60,9 +60,10 @@ static inline void __pud_populate(pud_t *pudp, phys_addr_t pmdp, pudval_t prot)
#if CONFIG_PGTABLE_LEVELS > 3
-static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
+static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr,
+ gfp_t gfp)
{
- return (pud_t *)__get_free_page(PGALLOC_GFP);
+ return (pud_t *)get_zeroed_page(gfp);
}
static inline void pud_free(struct mm_struct *mm, pud_t *pudp)
@@ -91,9 +92,9 @@ extern pgd_t *pgd_alloc(struct mm_struct *mm);
extern void pgd_free(struct mm_struct *mm, pgd_t *pgdp);
static inline pte_t *
-pte_alloc_one_kernel(struct mm_struct *mm)
+pte_alloc_one_kernel(struct mm_struct *mm, gfp_t gfp)
{
- return (pte_t *)__get_free_page(PGALLOC_GFP);
+ return (pte_t *)get_zeroed_page(gfp);
}
static inline pgtable_t
@@ -101,7 +102,7 @@ pte_alloc_one(struct mm_struct *mm)
{
struct page *pte;
- pte = alloc_pages(PGALLOC_GFP, 0);
+ pte = alloc_pages(GFP_KERNEL|__GFP_ZERO, 0);
if (!pte)
return NULL;
if (!pgtable_page_ctor(pte)) {
diff --git a/arch/arm64/mm/hugetlbpage.c b/arch/arm64/mm/hugetlbpage.c
index 28cbc22d7e30..a1dc42544495 100644
--- a/arch/arm64/mm/hugetlbpage.c
+++ b/arch/arm64/mm/hugetlbpage.c
@@ -210,14 +210,14 @@ pte_t *huge_pte_alloc(struct mm_struct *mm,
pte_t *ptep = NULL;
pgdp = pgd_offset(mm, addr);
- pudp = pud_alloc(mm, pgdp, addr);
+ pudp = pud_alloc(mm, pgdp, addr, GFP_KERNEL);
if (!pudp)
return NULL;
if (sz == PUD_SIZE) {
ptep = (pte_t *)pudp;
} else if (sz == (PAGE_SIZE * CONT_PTES)) {
- pmdp = pmd_alloc(mm, pudp, addr);
+ pmdp = pmd_alloc(mm, pudp, addr, GFP_KERNEL);
WARN_ON(addr & (sz - 1));
/*
@@ -233,9 +233,9 @@ pte_t *huge_pte_alloc(struct mm_struct *mm,
pud_none(READ_ONCE(*pudp)))
ptep = huge_pmd_share(mm, addr, pudp);
else
- ptep = (pte_t *)pmd_alloc(mm, pudp, addr);
+ ptep = (pte_t *)pmd_alloc(mm, pudp, addr, GFP_KERNEL);
} else if (sz == (PMD_SIZE * CONT_PMDS)) {
- pmdp = pmd_alloc(mm, pudp, addr);
+ pmdp = pmd_alloc(mm, pudp, addr, GFP_KERNEL);
WARN_ON(addr & (sz - 1));
return (pte_t *)pmdp;
}
diff --git a/arch/csky/include/asm/pgalloc.h b/arch/csky/include/asm/pgalloc.h
index d213bb47b717..1611a84be5b4 100644
--- a/arch/csky/include/asm/pgalloc.h
+++ b/arch/csky/include/asm/pgalloc.h
@@ -24,12 +24,12 @@ static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd,
extern void pgd_init(unsigned long *p);
-static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm)
+static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, gfp_t gfp)
{
pte_t *pte;
unsigned long i;
- pte = (pte_t *) __get_free_page(GFP_KERNEL);
+ pte = (pte_t *) __get_free_page(gfp);
if (!pte)
return NULL;
diff --git a/arch/hexagon/include/asm/pgalloc.h b/arch/hexagon/include/asm/pgalloc.h
index d36183887b60..2c42f912f4e0 100644
--- a/arch/hexagon/include/asm/pgalloc.h
+++ b/arch/hexagon/include/asm/pgalloc.h
@@ -74,10 +74,9 @@ static inline struct page *pte_alloc_one(struct mm_struct *mm)
}
/* _kernel variant gets to use a different allocator */
-static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm)
+static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, gfp_t gfp)
{
- gfp_t flags = GFP_KERNEL | __GFP_ZERO;
- return (pte_t *) __get_free_page(flags);
+ return (pte_t *) get_zeroed_page(gfp);
}
static inline void pte_free(struct mm_struct *mm, struct page *pte)
diff --git a/arch/ia64/include/asm/pgalloc.h b/arch/ia64/include/asm/pgalloc.h
index c9e481023c25..dd99d58a8946 100644
--- a/arch/ia64/include/asm/pgalloc.h
+++ b/arch/ia64/include/asm/pgalloc.h
@@ -40,9 +40,10 @@ pgd_populate(struct mm_struct *mm, pgd_t * pgd_entry, pud_t * pud)
pgd_val(*pgd_entry) = __pa(pud);
}
-static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
+static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr,
+ gfp_t gfp)
{
- return quicklist_alloc(0, GFP_KERNEL, NULL);
+ return quicklist_alloc(0, gfp, NULL);
}
static inline void pud_free(struct mm_struct *mm, pud_t *pud)
@@ -58,9 +59,10 @@ pud_populate(struct mm_struct *mm, pud_t * pud_entry, pmd_t * pmd)
pud_val(*pud_entry) = __pa(pmd);
}
-static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
+static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr,
+ gfp_t gfp)
{
- return quicklist_alloc(0, GFP_KERNEL, NULL);
+ return quicklist_alloc(0, gfp, NULL);
}
static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
@@ -99,9 +101,9 @@ static inline pgtable_t pte_alloc_one(struct mm_struct *mm)
return page;
}
-static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm)
+static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, gfp_t gfp)
{
- return quicklist_alloc(0, GFP_KERNEL, NULL);
+ return quicklist_alloc(0, gfp, NULL);
}
static inline void pte_free(struct mm_struct *mm, pgtable_t pte)
diff --git a/arch/ia64/mm/hugetlbpage.c b/arch/ia64/mm/hugetlbpage.c
index d16e419fd712..01e08edc9d90 100644
--- a/arch/ia64/mm/hugetlbpage.c
+++ b/arch/ia64/mm/hugetlbpage.c
@@ -35,9 +35,9 @@ huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz)
pte_t *pte = NULL;
pgd = pgd_offset(mm, taddr);
- pud = pud_alloc(mm, pgd, taddr);
+ pud = pud_alloc(mm, pgd, taddr, GFP_KERNEL);
if (pud) {
- pmd = pmd_alloc(mm, pud, taddr);
+ pmd = pmd_alloc(mm, pud, taddr, GFP_KERNEL);
if (pmd)
pte = pte_alloc_map(mm, pmd, taddr);
}
diff --git a/arch/ia64/mm/init.c b/arch/ia64/mm/init.c
index 29d841525ca1..63a91fdcd32e 100644
--- a/arch/ia64/mm/init.c
+++ b/arch/ia64/mm/init.c
@@ -216,13 +216,13 @@ put_kernel_page (struct page *page, unsigned long address, pgprot_t pgprot)
pgd = pgd_offset_k(address); /* note: this is NOT pgd_offset()! */
{
- pud = pud_alloc(&init_mm, pgd, address);
+ pud = pud_alloc(&init_mm, pgd, address, GFP_KERNEL);
if (!pud)
goto out;
- pmd = pmd_alloc(&init_mm, pud, address);
+ pmd = pmd_alloc(&init_mm, pud, address, GFP_KERNEL);
if (!pmd)
goto out;
- pte = pte_alloc_kernel(pmd, address);
+ pte = pte_alloc_kernel(pmd, address, GFP_KERNEL);
if (!pte)
goto out;
if (!pte_none(*pte))
diff --git a/arch/m68k/include/asm/mcf_pgalloc.h b/arch/m68k/include/asm/mcf_pgalloc.h
index 4399d712f6db..95384360cf44 100644
--- a/arch/m68k/include/asm/mcf_pgalloc.h
+++ b/arch/m68k/include/asm/mcf_pgalloc.h
@@ -12,15 +12,9 @@ extern inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
extern const char bad_pmd_string[];
-extern inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm)
+extern inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, gfp_t gfp)
{
- unsigned long page = __get_free_page(GFP_DMA);
-
- if (!page)
- return NULL;
-
- memset((void *)page, 0, PAGE_SIZE);
- return (pte_t *) (page);
+ return (pte_t *) get_zeroed_page(gfp|GFP_DMA);
}
extern inline pmd_t *pmd_alloc_kernel(pgd_t *pgd, unsigned long address)
@@ -29,7 +23,7 @@ extern inline pmd_t *pmd_alloc_kernel(pgd_t *pgd, unsigned long address)
}
#define pmd_alloc_one_fast(mm, address) ({ BUG(); ((pmd_t *)1); })
-#define pmd_alloc_one(mm, address) ({ BUG(); ((pmd_t *)2); })
+#define pmd_alloc_one(mm, address, gfp) ({ BUG(); ((pmd_t *)2); })
#define pmd_populate(mm, pmd, page) (pmd_val(*pmd) = \
(unsigned long)(page_address(page)))
diff --git a/arch/m68k/include/asm/motorola_pgalloc.h b/arch/m68k/include/asm/motorola_pgalloc.h
index d04d9ba9b976..e9b598f96b57 100644
--- a/arch/m68k/include/asm/motorola_pgalloc.h
+++ b/arch/m68k/include/asm/motorola_pgalloc.h
@@ -8,11 +8,11 @@
extern pmd_t *get_pointer_table(void);
extern int free_pointer_table(pmd_t *);
-static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm)
+static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, gfp_t gfp)
{
pte_t *pte;
- pte = (pte_t *)__get_free_page(GFP_KERNEL|__GFP_ZERO);
+ pte = (pte_t *)get_zeroed_page(gfp);
if (pte) {
__flush_page_to_ram(pte);
flush_tlb_kernel_page(pte);
@@ -67,7 +67,8 @@ static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t page,
}
-static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long address)
+static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long address,
+ gfp_t gfp)
{
return get_pointer_table();
}
diff --git a/arch/m68k/include/asm/sun3_pgalloc.h b/arch/m68k/include/asm/sun3_pgalloc.h
index 1456c5eecbd9..18324d4a33f1 100644
--- a/arch/m68k/include/asm/sun3_pgalloc.h
+++ b/arch/m68k/include/asm/sun3_pgalloc.h
@@ -15,7 +15,7 @@
extern const char bad_pmd_string[];
-#define pmd_alloc_one(mm,address) ({ BUG(); ((pmd_t *)2); })
+#define pmd_alloc_one(mm,address,gfp) ({ BUG(); ((pmd_t *)2); })
static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
@@ -35,15 +35,9 @@ do { \
tlb_remove_page((tlb), pte); \
} while (0)
-static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm)
+static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, gfp_t gfp)
{
- unsigned long page = __get_free_page(GFP_KERNEL);
-
- if (!page)
- return NULL;
-
- memset((void *)page, 0, PAGE_SIZE);
- return (pte_t *) (page);
+ return (pte_t *) get_zeroed_page(gfp);
}
static inline pgtable_t pte_alloc_one(struct mm_struct *mm)
diff --git a/arch/m68k/mm/kmap.c b/arch/m68k/mm/kmap.c
index 40a3b327da07..8de716049a84 100644
--- a/arch/m68k/mm/kmap.c
+++ b/arch/m68k/mm/kmap.c
@@ -196,7 +196,7 @@ void __iomem *__ioremap(unsigned long physaddr, unsigned long size, int cachefla
printk ("\npa=%#lx va=%#lx ", physaddr, virtaddr);
#endif
pgd_dir = pgd_offset_k(virtaddr);
- pmd_dir = pmd_alloc(&init_mm, pgd_dir, virtaddr);
+ pmd_dir = pmd_alloc(&init_mm, pgd_dir, virtaddr, GFP_KERNEL);
if (!pmd_dir) {
printk("ioremap: no mem for pmd_dir\n");
return NULL;
@@ -208,7 +208,8 @@ void __iomem *__ioremap(unsigned long physaddr, unsigned long size, int cachefla
virtaddr += PTRTREESIZE;
size -= PTRTREESIZE;
} else {
- pte_dir = pte_alloc_kernel(pmd_dir, virtaddr);
+ pte_dir = pte_alloc_kernel(pmd_dir, virtaddr,
+ GFP_KERNEL);
if (!pte_dir) {
printk("ioremap: no mem for pte_dir\n");
return NULL;
diff --git a/arch/m68k/sun3x/dvma.c b/arch/m68k/sun3x/dvma.c
index 89e630e66555..86ffbe278513 100644
--- a/arch/m68k/sun3x/dvma.c
+++ b/arch/m68k/sun3x/dvma.c
@@ -95,7 +95,8 @@ inline int dvma_map_cpu(unsigned long kaddr,
pmd_t *pmd;
unsigned long end2;
- if((pmd = pmd_alloc(&init_mm, pgd, vaddr)) == NULL) {
+ pmd = pmd_alloc(&init_mm, pgd, vaddr, GFP_KERNEL);
+ if (!pmd) {
ret = -ENOMEM;
goto out;
}
@@ -109,7 +110,8 @@ inline int dvma_map_cpu(unsigned long kaddr,
pte_t *pte;
unsigned long end3;
- if((pte = pte_alloc_kernel(pmd, vaddr)) == NULL) {
+ pte = pte_alloc_kernel(pmd, vaddr, GFP_KERNEL);
+ if (!pte) {
ret = -ENOMEM;
goto out;
}
diff --git a/arch/microblaze/include/asm/pgalloc.h b/arch/microblaze/include/asm/pgalloc.h
index f4cc9ffc449e..240e0bcd143d 100644
--- a/arch/microblaze/include/asm/pgalloc.h
+++ b/arch/microblaze/include/asm/pgalloc.h
@@ -106,9 +106,9 @@ static inline void free_pgd_slow(pgd_t *pgd)
* the pgd will always be present..
*/
#define pmd_alloc_one_fast(mm, address) ({ BUG(); ((pmd_t *)1); })
-#define pmd_alloc_one(mm, address) ({ BUG(); ((pmd_t *)2); })
+#define pmd_alloc_one(mm, address, gfp) ({ BUG(); ((pmd_t *)2); })
-extern pte_t *pte_alloc_one_kernel(struct mm_struct *mm);
+extern pte_t *pte_alloc_one_kernel(struct mm_struct *mm, gfp_t gfp);
static inline struct page *pte_alloc_one(struct mm_struct *mm)
{
@@ -166,7 +166,7 @@ static inline void pte_free(struct mm_struct *mm, struct page *ptepage)
* We don't have any real pmd's, and this code never triggers because
* the pgd will always be present..
*/
-#define pmd_alloc_one(mm, address) ({ BUG(); ((pmd_t *)2); })
+#define pmd_alloc_one(mm, address, gfp) ({ BUG(); ((pmd_t *)2); })
#define pmd_free(mm, x) do { } while (0)
#define __pmd_free_tlb(tlb, x, addr) pmd_free((tlb)->mm, x)
#define pgd_populate(mm, pmd, pte) BUG()
diff --git a/arch/microblaze/mm/pgtable.c b/arch/microblaze/mm/pgtable.c
index c2ce1e42b888..796c422af7ac 100644
--- a/arch/microblaze/mm/pgtable.c
+++ b/arch/microblaze/mm/pgtable.c
@@ -144,7 +144,7 @@ int map_page(unsigned long va, phys_addr_t pa, int flags)
/* Use upper 10 bits of VA to index the first level map */
pd = pmd_offset(pgd_offset_k(va), va);
/* Use middle 10 bits of VA to index the second-level map */
- pg = pte_alloc_kernel(pd, va); /* from powerpc - pgtable.c */
+ pg = pte_alloc_kernel(pd, va, GFP_KERNEL); /* from powerpc - pgtable.c */
/* pg = pte_alloc_kernel(&init_mm, pd, va); */
if (pg != NULL) {
@@ -235,11 +235,11 @@ unsigned long iopa(unsigned long addr)
return pa;
}
-__ref pte_t *pte_alloc_one_kernel(struct mm_struct *mm)
+__ref pte_t *pte_alloc_one_kernel(struct mm_struct *mm, gfp_t gfp)
{
pte_t *pte;
if (mem_init_done) {
- pte = (pte_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO);
+ pte = (pte_t *)get_zeroed_page(gfp);
} else {
pte = (pte_t *)early_get_page();
if (pte)
diff --git a/arch/mips/include/asm/pgalloc.h b/arch/mips/include/asm/pgalloc.h
index 27808d9461f4..7e832f978afd 100644
--- a/arch/mips/include/asm/pgalloc.h
+++ b/arch/mips/include/asm/pgalloc.h
@@ -50,9 +50,9 @@ static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)
free_pages((unsigned long)pgd, PGD_ORDER);
}
-static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm)
+static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, gfp_t gfp)
{
- return (pte_t *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, PTE_ORDER);
+ return (pte_t *)__get_free_pages(gfp | __GFP_ZERO, PTE_ORDER);
}
static inline struct page *pte_alloc_one(struct mm_struct *mm)
@@ -89,11 +89,12 @@ do { \
#ifndef __PAGETABLE_PMD_FOLDED
-static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long address)
+static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long address,
+ gfp_t gfp)
{
pmd_t *pmd;
- pmd = (pmd_t *) __get_free_pages(GFP_KERNEL, PMD_ORDER);
+ pmd = (pmd_t *) __get_free_pages(gfp, PMD_ORDER);
if (pmd)
pmd_init((unsigned long)pmd, (unsigned long)invalid_pte_table);
return pmd;
@@ -110,11 +111,12 @@ static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
#ifndef __PAGETABLE_PUD_FOLDED
-static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long address)
+static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long address,
+ gfp_t gfp)
{
pud_t *pud;
- pud = (pud_t *) __get_free_pages(GFP_KERNEL, PUD_ORDER);
+ pud = (pud_t *) __get_free_pages(gfp, PUD_ORDER);
if (pud)
pud_init((unsigned long)pud, (unsigned long)invalid_pmd_table);
return pud;
diff --git a/arch/mips/mm/hugetlbpage.c b/arch/mips/mm/hugetlbpage.c
index cef152234312..27843e10f680 100644
--- a/arch/mips/mm/hugetlbpage.c
+++ b/arch/mips/mm/hugetlbpage.c
@@ -29,9 +29,9 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr,
pte_t *pte = NULL;
pgd = pgd_offset(mm, addr);
- pud = pud_alloc(mm, pgd, addr);
+ pud = pud_alloc(mm, pgd, addr, GFP_KERNEL);
if (pud)
- pte = (pte_t *)pmd_alloc(mm, pud, addr);
+ pte = (pte_t *)pmd_alloc(mm, pud, addr, GFP_KERNEL);
return pte;
}
diff --git a/arch/mips/mm/ioremap.c b/arch/mips/mm/ioremap.c
index 1601d90b087b..40da8f0ba70b 100644
--- a/arch/mips/mm/ioremap.c
+++ b/arch/mips/mm/ioremap.c
@@ -56,7 +56,7 @@ static inline int remap_area_pmd(pmd_t * pmd, unsigned long address,
phys_addr -= address;
BUG_ON(address >= end);
do {
- pte_t * pte = pte_alloc_kernel(pmd, address);
+ pte_t *pte = pte_alloc_kernel(pmd, address, GFP_KERNEL);
if (!pte)
return -ENOMEM;
remap_area_pte(pte, address, end - address, address + phys_addr, flags);
@@ -82,10 +82,10 @@ static int remap_area_pages(unsigned long address, phys_addr_t phys_addr,
pmd_t *pmd;
error = -ENOMEM;
- pud = pud_alloc(&init_mm, dir, address);
+ pud = pud_alloc(&init_mm, dir, address, GFP_KERNEL);
if (!pud)
break;
- pmd = pmd_alloc(&init_mm, pud, address);
+ pmd = pmd_alloc(&init_mm, pud, address, GFP_KERNEL);
if (!pmd)
break;
if (remap_area_pmd(pmd, address, end - address,
diff --git a/arch/nds32/include/asm/pgalloc.h b/arch/nds32/include/asm/pgalloc.h
index 3c5fee5b5759..b187a2f1275c 100644
--- a/arch/nds32/include/asm/pgalloc.h
+++ b/arch/nds32/include/asm/pgalloc.h
@@ -12,8 +12,8 @@
/*
* Since we have only two-level page tables, these are trivial
*/
-#define pmd_alloc_one(mm, addr) ({ BUG(); ((pmd_t *)2); })
-#define pmd_free(mm, pmd) do { } while (0)
+#define pmd_alloc_one(mm, addr, gfp) ({ BUG(); ((pmd_t *)2); })
+#define pmd_free(mm, pmd) do { } while (0)
#define pgd_populate(mm, pmd, pte) BUG()
#define pmd_pgtable(pmd) pmd_page(pmd)
@@ -22,15 +22,9 @@ extern void pgd_free(struct mm_struct *mm, pgd_t * pgd);
#define check_pgt_cache() do { } while (0)
-static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm)
+static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, gfp_t gfp)
{
- pte_t *pte;
-
- pte =
- (pte_t *) __get_free_page(GFP_KERNEL | __GFP_RETRY_MAYFAIL |
- __GFP_ZERO);
-
- return pte;
+ return (pte_t *) get_zeroed_page(gfp | __GFP_RETRY_MAYFAIL);
}
static inline pgtable_t pte_alloc_one(struct mm_struct *mm)
diff --git a/arch/nds32/kernel/dma.c b/arch/nds32/kernel/dma.c
index d0dbd4fe9645..920a00376214 100644
--- a/arch/nds32/kernel/dma.c
+++ b/arch/nds32/kernel/dma.c
@@ -300,7 +300,7 @@ static int __init consistent_init(void)
do {
pgd = pgd_offset(&init_mm, CONSISTENT_BASE);
- pmd = pmd_alloc(&init_mm, pgd, CONSISTENT_BASE);
+ pmd = pmd_alloc(&init_mm, pgd, CONSISTENT_BASE, GFP_KERNEL);
if (!pmd) {
pr_err("%s: no pmd tables\n", __func__);
ret = -ENOMEM;
@@ -310,7 +310,7 @@ static int __init consistent_init(void)
* It's not necessary to warn here. */
/* WARN_ON(!pmd_none(*pmd)); */
- pte = pte_alloc_kernel(pmd, CONSISTENT_BASE);
+ pte = pte_alloc_kernel(pmd, CONSISTENT_BASE, GFP_KERNEL);
if (!pte) {
ret = -ENOMEM;
break;
diff --git a/arch/nios2/include/asm/pgalloc.h b/arch/nios2/include/asm/pgalloc.h
index 3a149ead1207..2ce9bd5399d3 100644
--- a/arch/nios2/include/asm/pgalloc.h
+++ b/arch/nios2/include/asm/pgalloc.h
@@ -37,13 +37,9 @@ static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)
free_pages((unsigned long)pgd, PGD_ORDER);
}
-static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm)
+static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, gf_t gfp)
{
- pte_t *pte;
-
- pte = (pte_t *) __get_free_pages(GFP_KERNEL|__GFP_ZERO, PTE_ORDER);
-
- return pte;
+ return (pte_t *)__get_free_pages(gfp|__GFP_ZERO, PTE_ORDER);
}
static inline pgtable_t pte_alloc_one(struct mm_struct *mm)
diff --git a/arch/nios2/mm/ioremap.c b/arch/nios2/mm/ioremap.c
index 3a28177a01eb..50c38da029ce 100644
--- a/arch/nios2/mm/ioremap.c
+++ b/arch/nios2/mm/ioremap.c
@@ -61,7 +61,7 @@ static inline int remap_area_pmd(pmd_t *pmd, unsigned long address,
if (address >= end)
BUG();
do {
- pte_t *pte = pte_alloc_kernel(pmd, address);
+ pte_t *pte = pte_alloc_kernel(pmd, address, GFP_KERNEL);
if (!pte)
return -ENOMEM;
@@ -90,10 +90,10 @@ static int remap_area_pages(unsigned long address, unsigned long phys_addr,
pmd_t *pmd;
error = -ENOMEM;
- pud = pud_alloc(&init_mm, dir, address);
+ pud = pud_alloc(&init_mm, dir, address, GFP_KERNEL);
if (!pud)
break;
- pmd = pmd_alloc(&init_mm, pud, address);
+ pmd = pmd_alloc(&init_mm, pud, address, GFP_KERNEL);
if (!pmd)
break;
if (remap_area_pmd(pmd, address, end - address,
diff --git a/arch/openrisc/include/asm/pgalloc.h b/arch/openrisc/include/asm/pgalloc.h
index 149c82ee4b8b..f33f2a450450 100644
--- a/arch/openrisc/include/asm/pgalloc.h
+++ b/arch/openrisc/include/asm/pgalloc.h
@@ -70,7 +70,7 @@ static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)
free_page((unsigned long)pgd);
}
-extern pte_t *pte_alloc_one_kernel(struct mm_struct *mm);
+extern pte_t *pte_alloc_one_kernel(struct mm_struct *mm, gfp_t gfp);
static inline struct page *pte_alloc_one(struct mm_struct *mm)
{
diff --git a/arch/openrisc/mm/ioremap.c b/arch/openrisc/mm/ioremap.c
index 270d1c9bc0d6..a603c517f092 100644
--- a/arch/openrisc/mm/ioremap.c
+++ b/arch/openrisc/mm/ioremap.c
@@ -118,17 +118,17 @@ EXPORT_SYMBOL(iounmap);
* the memblock infrastructure.
*/
-pte_t __ref *pte_alloc_one_kernel(struct mm_struct *mm)
+pte_t __ref *pte_alloc_one_kernel(struct mm_struct *mm, gfp_t gfp)
{
pte_t *pte;
if (likely(mem_init_done)) {
- pte = (pte_t *) __get_free_page(GFP_KERNEL);
+ pte = (pte_t *) get_zeroed_page(gfp);
} else {
pte = (pte_t *) __va(memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE));
+ if (pte)
+ clear_page(pte);
}
- if (pte)
- clear_page(pte);
return pte;
}
diff --git a/arch/parisc/include/asm/pgalloc.h b/arch/parisc/include/asm/pgalloc.h
index d05c678c77c4..705f5fffbd67 100644
--- a/arch/parisc/include/asm/pgalloc.h
+++ b/arch/parisc/include/asm/pgalloc.h
@@ -62,12 +62,10 @@ static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, pmd_t *pmd)
(__u32)(__pa((unsigned long)pmd) >> PxD_VALUE_SHIFT));
}
-static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long address)
+static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long address,
+ gfp_t gfp)
{
- pmd_t *pmd = (pmd_t *)__get_free_pages(GFP_KERNEL, PMD_ORDER);
- if (pmd)
- memset(pmd, 0, PAGE_SIZE<<PMD_ORDER);
- return pmd;
+ return (pmd_t *)__get_free_pages(gfp|__GFP_ZERO, PMD_ORDER);
}
static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
@@ -94,7 +92,7 @@ static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
* inside the pgd, so has no extra memory associated with it.
*/
-#define pmd_alloc_one(mm, addr) ({ BUG(); ((pmd_t *)2); })
+#define pmd_alloc_one(mm, addr, gfp) ({ BUG(); ((pmd_t *)2); })
#define pmd_free(mm, x) do { } while (0)
#define pgd_populate(mm, pmd, pte) BUG()
@@ -134,11 +132,9 @@ pte_alloc_one(struct mm_struct *mm)
return page;
}
-static inline pte_t *
-pte_alloc_one_kernel(struct mm_struct *mm)
+static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, gfp_t gfp)
{
- pte_t *pte = (pte_t *)__get_free_page(GFP_KERNEL|__GFP_ZERO);
- return pte;
+ return (pte_t *)get_zeroed_page(gfp);
}
static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
diff --git a/arch/parisc/kernel/pci-dma.c b/arch/parisc/kernel/pci-dma.c
index 239162355b58..285688e735ff 100644
--- a/arch/parisc/kernel/pci-dma.c
+++ b/arch/parisc/kernel/pci-dma.c
@@ -113,7 +113,7 @@ static inline int map_pmd_uncached(pmd_t * pmd, unsigned long vaddr,
if (end > PGDIR_SIZE)
end = PGDIR_SIZE;
do {
- pte_t * pte = pte_alloc_kernel(pmd, vaddr);
+ pte_t *pte = pte_alloc_kernel(pmd, vaddr, GFP_KERNEL);
if (!pte)
return -ENOMEM;
if (map_pte_uncached(pte, orig_vaddr, end - vaddr, paddr_ptr))
@@ -134,8 +134,8 @@ static inline int map_uncached_pages(unsigned long vaddr, unsigned long size,
dir = pgd_offset_k(vaddr);
do {
pmd_t *pmd;
-
- pmd = pmd_alloc(NULL, dir, vaddr);
+
+ pmd = pmd_alloc(NULL, dir, vaddr, GFP_KERNEL);
if (!pmd)
return -ENOMEM;
if (map_pmd_uncached(pmd, vaddr, end - vaddr, &paddr))
diff --git a/arch/parisc/mm/hugetlbpage.c b/arch/parisc/mm/hugetlbpage.c
index d77479ae3af2..6351549539fe 100644
--- a/arch/parisc/mm/hugetlbpage.c
+++ b/arch/parisc/mm/hugetlbpage.c
@@ -61,9 +61,9 @@ pte_t *huge_pte_alloc(struct mm_struct *mm,
addr &= HPAGE_MASK;
pgd = pgd_offset(mm, addr);
- pud = pud_alloc(mm, pgd, addr);
+ pud = pud_alloc(mm, pgd, addr, GFP_KERNEL);
if (pud) {
- pmd = pmd_alloc(mm, pud, addr);
+ pmd = pmd_alloc(mm, pud, addr, GFP_KERNEL);
if (pmd)
pte = pte_alloc_map(mm, pmd, addr);
}
diff --git a/arch/powerpc/include/asm/book3s/32/pgalloc.h b/arch/powerpc/include/asm/book3s/32/pgalloc.h
index 3633502e102c..9032660c0e5a 100644
--- a/arch/powerpc/include/asm/book3s/32/pgalloc.h
+++ b/arch/powerpc/include/asm/book3s/32/pgalloc.h
@@ -42,7 +42,7 @@ static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)
* We don't have any real pmd's, and this code never triggers because
* the pgd will always be present..
*/
-/* #define pmd_alloc_one(mm,address) ({ BUG(); ((pmd_t *)2); }) */
+/* #define pmd_alloc_one(mm,address,gfp) ({ BUG(); ((pmd_t *)2); }) */
#define pmd_free(mm, x) do { } while (0)
#define __pmd_free_tlb(tlb,x,a) do { } while (0)
/* #define pgd_populate(mm, pmd, pte) BUG() */
@@ -61,7 +61,7 @@ static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmdp,
#define pmd_pgtable(pmd) ((pgtable_t)pmd_page_vaddr(pmd))
-extern pte_t *pte_alloc_one_kernel(struct mm_struct *mm);
+extern pte_t *pte_alloc_one_kernel(struct mm_struct *mm, gfp_t gfp);
extern pgtable_t pte_alloc_one(struct mm_struct *mm);
void pte_frag_destroy(void *pte_frag);
pte_t *pte_fragment_alloc(struct mm_struct *mm, int kernel);
diff --git a/arch/powerpc/include/asm/book3s/64/pgalloc.h b/arch/powerpc/include/asm/book3s/64/pgalloc.h
index 9c1173283b96..1582c8b29814 100644
--- a/arch/powerpc/include/asm/book3s/64/pgalloc.h
+++ b/arch/powerpc/include/asm/book3s/64/pgalloc.h
@@ -39,8 +39,8 @@ extern struct vmemmap_backing *vmemmap_list;
extern struct kmem_cache *pgtable_cache[];
#define PGT_CACHE(shift) pgtable_cache[shift]
-extern pte_t *pte_fragment_alloc(struct mm_struct *, int);
-extern pmd_t *pmd_fragment_alloc(struct mm_struct *, unsigned long);
+extern pte_t *pte_fragment_alloc(struct mm_struct *, int, gfp_t);
+extern pmd_t *pmd_fragment_alloc(struct mm_struct *, unsigned long, gfp_t);
extern void pte_fragment_free(unsigned long *, int);
extern void pmd_fragment_free(unsigned long *);
extern void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int shift);
@@ -114,12 +114,13 @@ static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, pud_t *pud)
pgd_set(pgd, __pgtable_ptr_val(pud) | PGD_VAL_BITS);
}
-static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
+static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr,
+ gfp_t gfp)
{
pud_t *pud;
pud = kmem_cache_alloc(PGT_CACHE(PUD_CACHE_INDEX),
- pgtable_gfp_flags(mm, GFP_KERNEL));
+ pgtable_gfp_flags(mm, gfp));
/*
* Tell kmemleak to ignore the PUD, that means don't scan it for
* pointers and don't consider it a leak. PUDs are typically only
@@ -152,9 +153,10 @@ static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pud,
pgtable_free_tlb(tlb, pud, PUD_INDEX);
}
-static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
+static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr,
+ gfp_t gfp)
{
- return pmd_fragment_alloc(mm, addr);
+ return pmd_fragment_alloc(mm, addr, gfp);
}
static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
@@ -190,14 +192,14 @@ static inline pgtable_t pmd_pgtable(pmd_t pmd)
return (pgtable_t)pmd_page_vaddr(pmd);
}
-static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm)
+static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, gfp_t gfp)
{
- return (pte_t *)pte_fragment_alloc(mm, 1);
+ return (pte_t *)pte_fragment_alloc(mm, 1, gfp);
}
static inline pgtable_t pte_alloc_one(struct mm_struct *mm)
{
- return (pgtable_t)pte_fragment_alloc(mm, 0);
+ return (pgtable_t)pte_fragment_alloc(mm, 0, GFP_KERNEL);
}
static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
diff --git a/arch/powerpc/include/asm/nohash/32/pgalloc.h b/arch/powerpc/include/asm/nohash/32/pgalloc.h
index bd186e85b4f7..8a5a944251c7 100644
--- a/arch/powerpc/include/asm/nohash/32/pgalloc.h
+++ b/arch/powerpc/include/asm/nohash/32/pgalloc.h
@@ -42,8 +42,8 @@ static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)
* We don't have any real pmd's, and this code never triggers because
* the pgd will always be present..
*/
-/* #define pmd_alloc_one(mm,address) ({ BUG(); ((pmd_t *)2); }) */
-#define pmd_free(mm, x) do { } while (0)
+/* #define pmd_alloc_one(mm,address,gfp) ({ BUG(); ((pmd_t *)2); }) */
+#define pmd_free(mm, x) do { } while (0)
#define __pmd_free_tlb(tlb,x,a) do { } while (0)
/* #define pgd_populate(mm, pmd, pte) BUG() */
@@ -79,7 +79,7 @@ static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmdp,
#define pmd_pgtable(pmd) ((pgtable_t)pmd_page_vaddr(pmd))
#endif
-extern pte_t *pte_alloc_one_kernel(struct mm_struct *mm);
+extern pte_t *pte_alloc_one_kernel(struct mm_struct *mm, gfp_t gfp);
extern pgtable_t pte_alloc_one(struct mm_struct *mm);
void pte_frag_destroy(void *pte_frag);
pte_t *pte_fragment_alloc(struct mm_struct *mm, int kernel);
diff --git a/arch/powerpc/include/asm/nohash/64/pgalloc.h b/arch/powerpc/include/asm/nohash/64/pgalloc.h
index 66d086f85bd5..e30f21916a2a 100644
--- a/arch/powerpc/include/asm/nohash/64/pgalloc.h
+++ b/arch/powerpc/include/asm/nohash/64/pgalloc.h
@@ -51,10 +51,11 @@ static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)
#define pgd_populate(MM, PGD, PUD) pgd_set(PGD, (unsigned long)PUD)
-static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
+static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr,
+ gfp_t gfp)
{
return kmem_cache_alloc(PGT_CACHE(PUD_INDEX_SIZE),
- pgtable_gfp_flags(mm, GFP_KERNEL));
+ pgtable_gfp_flags(mm, gfp));
}
static inline void pud_free(struct mm_struct *mm, pud_t *pud)
@@ -81,10 +82,11 @@ static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd,
#define pmd_pgtable(pmd) pmd_page(pmd)
-static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
+static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr,
+ gfp_t gfp)
{
return kmem_cache_alloc(PGT_CACHE(PMD_CACHE_INDEX),
- pgtable_gfp_flags(mm, GFP_KERNEL));
+ pgtable_gfp_flags(mm, gfp));
}
static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
@@ -93,9 +95,9 @@ static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
}
-static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm)
+static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, gfp_t gfp)
{
- return (pte_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO);
+ return (pte_t *)get_zeroed_page(gfp);
}
static inline pgtable_t pte_alloc_one(struct mm_struct *mm)
diff --git a/arch/powerpc/kvm/book3s_64_mmu_radix.c b/arch/powerpc/kvm/book3s_64_mmu_radix.c
index 1b821c6efdef..ab79c7cd5e3e 100644
--- a/arch/powerpc/kvm/book3s_64_mmu_radix.c
+++ b/arch/powerpc/kvm/book3s_64_mmu_radix.c
@@ -580,7 +580,7 @@ int kvmppc_create_pte(struct kvm *kvm, pgd_t *pgtable, pte_t pte,
if (pgd_present(*pgd))
pud = pud_offset(pgd, gpa);
else
- new_pud = pud_alloc_one(kvm->mm, gpa);
+ new_pud = pud_alloc_one(kvm->mm, gpa, GFP_KERNEL);
pmd = NULL;
if (pud && pud_present(*pud) && !pud_huge(*pud))
diff --git a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c
index 9e732bb2c84a..f66c42c93395 100644
--- a/arch/powerpc/mm/hugetlbpage.c
+++ b/arch/powerpc/mm/hugetlbpage.c
@@ -153,7 +153,7 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz
hpdp = (hugepd_t *)pg;
} else {
pdshift = PUD_SHIFT;
- pu = pud_alloc(mm, pg, addr);
+ pu = pud_alloc(mm, pg, addr, GFP_KERNEL);
if (pshift == PUD_SHIFT)
return (pte_t *)pu;
else if (pshift > PMD_SHIFT) {
@@ -161,7 +161,7 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz
hpdp = (hugepd_t *)pu;
} else {
pdshift = PMD_SHIFT;
- pm = pmd_alloc(mm, pu, addr);
+ pm = pmd_alloc(mm, pu, addr, GFP_KERNEL);
if (pshift == PMD_SHIFT)
/* 16MB hugepage */
return (pte_t *)pm;
@@ -177,13 +177,13 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz
hpdp = (hugepd_t *)pg;
} else {
pdshift = PUD_SHIFT;
- pu = pud_alloc(mm, pg, addr);
+ pu = pud_alloc(mm, pg, addr, GFP_KERNEL);
if (pshift >= PUD_SHIFT) {
ptl = pud_lockptr(mm, pu);
hpdp = (hugepd_t *)pu;
} else {
pdshift = PMD_SHIFT;
- pm = pmd_alloc(mm, pu, addr);
+ pm = pmd_alloc(mm, pu, addr, GFP_KERNEL);
ptl = pmd_lockptr(mm, pm);
hpdp = (hugepd_t *)pm;
}
diff --git a/arch/powerpc/mm/pgtable-book3e.c b/arch/powerpc/mm/pgtable-book3e.c
index e0ccf36714b2..ded846507c17 100644
--- a/arch/powerpc/mm/pgtable-book3e.c
+++ b/arch/powerpc/mm/pgtable-book3e.c
@@ -80,13 +80,13 @@ int map_kernel_page(unsigned long ea, unsigned long pa, pgprot_t prot)
BUILD_BUG_ON(TASK_SIZE_USER64 > PGTABLE_RANGE);
if (slab_is_available()) {
pgdp = pgd_offset_k(ea);
- pudp = pud_alloc(&init_mm, pgdp, ea);
+ pudp = pud_alloc(&init_mm, pgdp, ea, GFP_KERNEL);
if (!pudp)
return -ENOMEM;
- pmdp = pmd_alloc(&init_mm, pudp, ea);
+ pmdp = pmd_alloc(&init_mm, pudp, ea, GFP_KERNEL);
if (!pmdp)
return -ENOMEM;
- ptep = pte_alloc_kernel(pmdp, ea);
+ ptep = pte_alloc_kernel(pmdp, ea, GFP_KERNEL);
if (!ptep)
return -ENOMEM;
} else {
diff --git a/arch/powerpc/mm/pgtable-book3s64.c b/arch/powerpc/mm/pgtable-book3s64.c
index ecd31569a120..00570a92c0f1 100644
--- a/arch/powerpc/mm/pgtable-book3s64.c
+++ b/arch/powerpc/mm/pgtable-book3s64.c
@@ -262,15 +262,14 @@ static pmd_t *get_pmd_from_cache(struct mm_struct *mm)
return (pmd_t *)ret;
}
-static pmd_t *__alloc_for_pmdcache(struct mm_struct *mm)
+static pmd_t *__alloc_for_pmdcache(struct mm_struct *mm, gfp_t gfp)
{
void *ret = NULL;
struct page *page;
- gfp_t gfp = GFP_KERNEL_ACCOUNT | __GFP_ZERO;
- if (mm == &init_mm)
- gfp &= ~__GFP_ACCOUNT;
- page = alloc_page(gfp);
+ if (mm != &init_mm)
+ gfp |= __GFP_ACCOUNT;
+ page = alloc_page(gfp|__GFP_ZERO);
if (!page)
return NULL;
if (!pgtable_pmd_page_ctor(page)) {
@@ -303,7 +302,8 @@ static pmd_t *__alloc_for_pmdcache(struct mm_struct *mm)
return (pmd_t *)ret;
}
-pmd_t *pmd_fragment_alloc(struct mm_struct *mm, unsigned long vmaddr)
+pmd_t *pmd_fragment_alloc(struct mm_struct *mm, unsigned long vmaddr,
+ gfp_t gfp)
{
pmd_t *pmd;
@@ -311,7 +311,7 @@ pmd_t *pmd_fragment_alloc(struct mm_struct *mm, unsigned long vmaddr)
if (pmd)
return pmd;
- return __alloc_for_pmdcache(mm);
+ return __alloc_for_pmdcache(mm, gfp);
}
void pmd_fragment_free(unsigned long *pmd)
diff --git a/arch/powerpc/mm/pgtable-hash64.c b/arch/powerpc/mm/pgtable-hash64.c
index c08d49046a96..d90deb67d81e 100644
--- a/arch/powerpc/mm/pgtable-hash64.c
+++ b/arch/powerpc/mm/pgtable-hash64.c
@@ -152,13 +152,13 @@ int hash__map_kernel_page(unsigned long ea, unsigned long pa, pgprot_t prot)
BUILD_BUG_ON(TASK_SIZE_USER64 > H_PGTABLE_RANGE);
if (slab_is_available()) {
pgdp = pgd_offset_k(ea);
- pudp = pud_alloc(&init_mm, pgdp, ea);
+ pudp = pud_alloc(&init_mm, pgdp, ea, GFP_KERNEL);
if (!pudp)
return -ENOMEM;
- pmdp = pmd_alloc(&init_mm, pudp, ea);
+ pmdp = pmd_alloc(&init_mm, pudp, ea, GFP_KERNEL);
if (!pmdp)
return -ENOMEM;
- ptep = pte_alloc_kernel(pmdp, ea);
+ ptep = pte_alloc_kernel(pmdp, ea, GFP_KERNEL);
if (!ptep)
return -ENOMEM;
set_pte_at(&init_mm, ea, ptep, pfn_pte(pa >> PAGE_SHIFT, prot));
diff --git a/arch/powerpc/mm/pgtable-radix.c b/arch/powerpc/mm/pgtable-radix.c
index 931156069a81..11007d789d88 100644
--- a/arch/powerpc/mm/pgtable-radix.c
+++ b/arch/powerpc/mm/pgtable-radix.c
@@ -149,21 +149,21 @@ static int __map_kernel_page(unsigned long ea, unsigned long pa,
* boot.
*/
pgdp = pgd_offset_k(ea);
- pudp = pud_alloc(&init_mm, pgdp, ea);
+ pudp = pud_alloc(&init_mm, pgdp, ea, GFP_KERNEL);
if (!pudp)
return -ENOMEM;
if (map_page_size == PUD_SIZE) {
ptep = (pte_t *)pudp;
goto set_the_pte;
}
- pmdp = pmd_alloc(&init_mm, pudp, ea);
+ pmdp = pmd_alloc(&init_mm, pudp, ea, GFP_KERNEL);
if (!pmdp)
return -ENOMEM;
if (map_page_size == PMD_SIZE) {
ptep = pmdp_ptep(pmdp);
goto set_the_pte;
}
- ptep = pte_alloc_kernel(pmdp, ea);
+ ptep = pte_alloc_kernel(pmdp, ea, GFP_KERNEL);
if (!ptep)
return -ENOMEM;
@@ -198,21 +198,21 @@ void radix__change_memory_range(unsigned long start, unsigned long end,
for (idx = start; idx < end; idx += PAGE_SIZE) {
pgdp = pgd_offset_k(idx);
- pudp = pud_alloc(&init_mm, pgdp, idx);
+ pudp = pud_alloc(&init_mm, pgdp, idx, GFP_KERNEL);
if (!pudp)
continue;
if (pud_huge(*pudp)) {
ptep = (pte_t *)pudp;
goto update_the_pte;
}
- pmdp = pmd_alloc(&init_mm, pudp, idx);
+ pmdp = pmd_alloc(&init_mm, pudp, idx, GFP_KERNEL);
if (!pmdp)
continue;
if (pmd_huge(*pmdp)) {
ptep = pmdp_ptep(pmdp);
goto update_the_pte;
}
- ptep = pte_alloc_kernel(pmdp, idx);
+ ptep = pte_alloc_kernel(pmdp, idx, GFP_KERNEL);
if (!ptep)
continue;
update_the_pte:
diff --git a/arch/powerpc/mm/pgtable_32.c b/arch/powerpc/mm/pgtable_32.c
index ded71126ce4c..62bf682a2dbb 100644
--- a/arch/powerpc/mm/pgtable_32.c
+++ b/arch/powerpc/mm/pgtable_32.c
@@ -43,12 +43,12 @@ EXPORT_SYMBOL(ioremap_bot); /* aka VMALLOC_END */
extern char etext[], _stext[], _sinittext[], _einittext[];
-__ref pte_t *pte_alloc_one_kernel(struct mm_struct *mm)
+__ref pte_t *pte_alloc_one_kernel(struct mm_struct *mm, gfp_t gfp)
{
if (!slab_is_available())
return memblock_alloc(PTE_FRAG_SIZE, PTE_FRAG_SIZE);
- return (pte_t *)pte_fragment_alloc(mm, 1);
+ return (pte_t *)pte_fragment_alloc(mm, 1, gfp);
}
pgtable_t pte_alloc_one(struct mm_struct *mm)
@@ -214,7 +214,7 @@ int map_kernel_page(unsigned long va, phys_addr_t pa, pgprot_t prot)
/* Use upper 10 bits of VA to index the first level map */
pd = pmd_offset(pud_offset(pgd_offset_k(va), va), va);
/* Use middle 10 bits of VA to index the second-level map */
- pg = pte_alloc_kernel(pd, va);
+ pg = pte_alloc_kernel(pd, va, GFP_KERNEL);
if (pg != 0) {
err = 0;
/* The PTE should never be already set nor present in the
diff --git a/arch/riscv/include/asm/pgalloc.h b/arch/riscv/include/asm/pgalloc.h
index 94043cf83c90..991c8d268e7a 100644
--- a/arch/riscv/include/asm/pgalloc.h
+++ b/arch/riscv/include/asm/pgalloc.h
@@ -67,10 +67,10 @@ static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)
#ifndef __PAGETABLE_PMD_FOLDED
-static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
+static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr,
+ gfp_t gfp)
{
- return (pmd_t *)__get_free_page(
- GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_ZERO);
+ return (pmd_t *)get_zeroed_page(gfp|__GFP_RETRY_MAYFAIL);
}
static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
@@ -82,10 +82,9 @@ static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
#endif /* __PAGETABLE_PMD_FOLDED */
-static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm)
+static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, gfp_t gfp)
{
- return (pte_t *)__get_free_page(
- GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_ZERO);
+ return (pte_t *)get_zeroed_page(gfp | __GFP_RETRY_MAYFAIL);
}
static inline struct page *pte_alloc_one(struct mm_struct *mm)
diff --git a/arch/s390/include/asm/pgalloc.h b/arch/s390/include/asm/pgalloc.h
index bccb8f4a63e2..49fb02562735 100644
--- a/arch/s390/include/asm/pgalloc.h
+++ b/arch/s390/include/asm/pgalloc.h
@@ -19,10 +19,10 @@
#define CRST_ALLOC_ORDER 2
-unsigned long *crst_table_alloc(struct mm_struct *);
+unsigned long *crst_table_alloc(struct mm_struct *, gfp_t);
void crst_table_free(struct mm_struct *, unsigned long *);
-unsigned long *page_table_alloc(struct mm_struct *);
+unsigned long *page_table_alloc(struct mm_struct *, gfp_t);
struct page *page_table_alloc_pgste(struct mm_struct *mm);
void page_table_free(struct mm_struct *, unsigned long *);
void page_table_free_rcu(struct mmu_gather *, unsigned long *, unsigned long);
@@ -48,9 +48,10 @@ static inline unsigned long pgd_entry_type(struct mm_struct *mm)
int crst_table_upgrade(struct mm_struct *mm, unsigned long limit);
void crst_table_downgrade(struct mm_struct *);
-static inline p4d_t *p4d_alloc_one(struct mm_struct *mm, unsigned long address)
+static inline p4d_t *p4d_alloc_one(struct mm_struct *mm, unsigned long address,
+ gfp_t gfp)
{
- unsigned long *table = crst_table_alloc(mm);
+ unsigned long *table = crst_table_alloc(mm, gfp);
if (table)
crst_table_init(table, _REGION2_ENTRY_EMPTY);
@@ -58,18 +59,20 @@ static inline p4d_t *p4d_alloc_one(struct mm_struct *mm, unsigned long address)
}
#define p4d_free(mm, p4d) crst_table_free(mm, (unsigned long *) p4d)
-static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long address)
+static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long address,
+ gfp_t gfp)
{
- unsigned long *table = crst_table_alloc(mm);
+ unsigned long *table = crst_table_alloc(mm, gfp);
if (table)
crst_table_init(table, _REGION3_ENTRY_EMPTY);
return (pud_t *) table;
}
#define pud_free(mm, pud) crst_table_free(mm, (unsigned long *) pud)
-static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long vmaddr)
+static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long vmaddr,
+ gfp_t gfp)
{
- unsigned long *table = crst_table_alloc(mm);
+ unsigned long *table = crst_table_alloc(mm, gfp);
if (!table)
return NULL;
@@ -104,7 +107,7 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
static inline pgd_t *pgd_alloc(struct mm_struct *mm)
{
- unsigned long *table = crst_table_alloc(mm);
+ unsigned long *table = crst_table_alloc(mm, GFP_KERNEL);
if (!table)
return NULL;
@@ -139,8 +142,8 @@ static inline void pmd_populate(struct mm_struct *mm,
/*
* page table entry allocation/free routines.
*/
-#define pte_alloc_one_kernel(mm) ((pte_t *)page_table_alloc(mm))
-#define pte_alloc_one(mm) ((pte_t *)page_table_alloc(mm))
+#define pte_alloc_one_kernel(mm, gfp) ((pte_t *)page_table_alloc(mm, gfp))
+#define pte_alloc_one(mm) ((pte_t *)page_table_alloc(mm, GFP_KERNEL))
#define pte_free_kernel(mm, pte) page_table_free(mm, (unsigned long *) pte)
#define pte_free(mm, pte) page_table_free(mm, (unsigned long *) pte)
diff --git a/arch/s390/mm/hugetlbpage.c b/arch/s390/mm/hugetlbpage.c
index b0246c705a19..eeb14683696d 100644
--- a/arch/s390/mm/hugetlbpage.c
+++ b/arch/s390/mm/hugetlbpage.c
@@ -192,14 +192,14 @@ pte_t *huge_pte_alloc(struct mm_struct *mm,
pmd_t *pmdp = NULL;
pgdp = pgd_offset(mm, addr);
- p4dp = p4d_alloc(mm, pgdp, addr);
+ p4dp = p4d_alloc(mm, pgdp, addr, GFP_KERNEL);
if (p4dp) {
- pudp = pud_alloc(mm, p4dp, addr);
+ pudp = pud_alloc(mm, p4dp, addr, GFP_KERNEL);
if (pudp) {
if (sz == PUD_SIZE)
return (pte_t *) pudp;
else if (sz == PMD_SIZE)
- pmdp = pmd_alloc(mm, pudp, addr);
+ pmdp = pmd_alloc(mm, pudp, addr, GFP_KERNEL);
}
}
return (pte_t *) pmdp;
diff --git a/arch/s390/mm/pgalloc.c b/arch/s390/mm/pgalloc.c
index db6bb2f97a2c..b8c309de98c4 100644
--- a/arch/s390/mm/pgalloc.c
+++ b/arch/s390/mm/pgalloc.c
@@ -53,9 +53,9 @@ __initcall(page_table_register_sysctl);
#endif /* CONFIG_PGSTE */
-unsigned long *crst_table_alloc(struct mm_struct *mm)
+unsigned long *crst_table_alloc(struct mm_struct *mm, gfp_t gfp)
{
- struct page *page = alloc_pages(GFP_KERNEL, 2);
+ struct page *page = alloc_pages(gfp, 2);
if (!page)
return NULL;
@@ -87,7 +87,7 @@ int crst_table_upgrade(struct mm_struct *mm, unsigned long end)
rc = 0;
notify = 0;
while (mm->context.asce_limit < end) {
- table = crst_table_alloc(mm);
+ table = crst_table_alloc(mm, GFP_KERNEL);
if (!table) {
rc = -ENOMEM;
break;
@@ -179,7 +179,7 @@ void page_table_free_pgste(struct page *page)
/*
* page table entry allocation/free routines.
*/
-unsigned long *page_table_alloc(struct mm_struct *mm)
+unsigned long *page_table_alloc(struct mm_struct *mm, gfp_t gfp)
{
unsigned long *table;
struct page *page;
@@ -209,7 +209,7 @@ unsigned long *page_table_alloc(struct mm_struct *mm)
return table;
}
/* Allocate a fresh page */
- page = alloc_page(GFP_KERNEL);
+ page = alloc_page(gfp);
if (!page)
return NULL;
if (!pgtable_page_ctor(page)) {
diff --git a/arch/s390/mm/pgtable.c b/arch/s390/mm/pgtable.c
index f2cc7da473e4..8284f99b9de4 100644
--- a/arch/s390/mm/pgtable.c
+++ b/arch/s390/mm/pgtable.c
@@ -418,13 +418,13 @@ static pmd_t *pmd_alloc_map(struct mm_struct *mm, unsigned long addr)
pmd_t *pmd;
pgd = pgd_offset(mm, addr);
- p4d = p4d_alloc(mm, pgd, addr);
+ p4d = p4d_alloc(mm, pgd, addr, GFP_KERNEL);
if (!p4d)
return NULL;
- pud = pud_alloc(mm, p4d, addr);
+ pud = pud_alloc(mm, p4d, addr, GFP_KERNEL);
if (!pud)
return NULL;
- pmd = pmd_alloc(mm, pud, addr);
+ pmd = pmd_alloc(mm, pud, addr, GFP_KERNEL);
return pmd;
}
diff --git a/arch/s390/mm/vmem.c b/arch/s390/mm/vmem.c
index 0472e27febdf..47ffefab75c4 100644
--- a/arch/s390/mm/vmem.c
+++ b/arch/s390/mm/vmem.c
@@ -54,7 +54,7 @@ pte_t __ref *vmem_pte_alloc(void)
pte_t *pte;
if (slab_is_available())
- pte = (pte_t *) page_table_alloc(&init_mm);
+ pte = (pte_t *) page_table_alloc(&init_mm, GFP_KERNEL);
else
pte = (pte_t *) memblock_phys_alloc(size, size);
if (!pte)
diff --git a/arch/sh/include/asm/pgalloc.h b/arch/sh/include/asm/pgalloc.h
index 8ad73cb31121..bd51502e8b2a 100644
--- a/arch/sh/include/asm/pgalloc.h
+++ b/arch/sh/include/asm/pgalloc.h
@@ -12,7 +12,8 @@ extern void pgd_free(struct mm_struct *mm, pgd_t *pgd);
#if PAGETABLE_LEVELS > 2
extern void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd);
-extern pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long address);
+extern pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long address,
+ gfp_t gfp);
extern void pmd_free(struct mm_struct *mm, pmd_t *pmd);
#endif
@@ -32,9 +33,9 @@ static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd,
/*
* Allocate and free page tables.
*/
-static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm)
+static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, gfp_t gfp)
{
- return quicklist_alloc(QUICK_PT, GFP_KERNEL, NULL);
+ return quicklist_alloc(QUICK_PT, gfp, NULL);
}
static inline pgtable_t pte_alloc_one(struct mm_struct *mm)
diff --git a/arch/sh/mm/hugetlbpage.c b/arch/sh/mm/hugetlbpage.c
index 960deb1f24a1..1eb4932cdb64 100644
--- a/arch/sh/mm/hugetlbpage.c
+++ b/arch/sh/mm/hugetlbpage.c
@@ -32,9 +32,9 @@ pte_t *huge_pte_alloc(struct mm_struct *mm,
pgd = pgd_offset(mm, addr);
if (pgd) {
- pud = pud_alloc(mm, pgd, addr);
+ pud = pud_alloc(mm, pgd, addr, GFP_KERNEL);
if (pud) {
- pmd = pmd_alloc(mm, pud, addr);
+ pmd = pmd_alloc(mm, pud, addr, GFP_KERNEL);
if (pmd)
pte = pte_alloc_map(mm, pmd, addr);
}
diff --git a/arch/sh/mm/init.c b/arch/sh/mm/init.c
index a8e5c0e00fca..9273d7e20174 100644
--- a/arch/sh/mm/init.c
+++ b/arch/sh/mm/init.c
@@ -53,13 +53,13 @@ static pte_t *__get_pte_phys(unsigned long addr)
return NULL;
}
- pud = pud_alloc(NULL, pgd, addr);
+ pud = pud_alloc(NULL, pgd, addr, GFP_KERNEL);
if (unlikely(!pud)) {
pud_ERROR(*pud);
return NULL;
}
- pmd = pmd_alloc(NULL, pud, addr);
+ pmd = pmd_alloc(NULL, pud, addr, GFP_KERNEL);
if (unlikely(!pmd)) {
pmd_ERROR(*pmd);
return NULL;
diff --git a/arch/sh/mm/pgtable.c b/arch/sh/mm/pgtable.c
index 5c8f9247c3c2..972f54fa09ea 100644
--- a/arch/sh/mm/pgtable.c
+++ b/arch/sh/mm/pgtable.c
@@ -2,8 +2,6 @@
#include <linux/mm.h>
#include <linux/slab.h>
-#define PGALLOC_GFP GFP_KERNEL | __GFP_ZERO
-
static struct kmem_cache *pgd_cachep;
#if PAGETABLE_LEVELS > 2
static struct kmem_cache *pmd_cachep;
@@ -32,7 +30,7 @@ void pgtable_cache_init(void)
pgd_t *pgd_alloc(struct mm_struct *mm)
{
- return kmem_cache_alloc(pgd_cachep, PGALLOC_GFP);
+ return kmem_cache_alloc(pgd_cachep, GFP_KERNEL|__GFP_ZERO);
}
void pgd_free(struct mm_struct *mm, pgd_t *pgd)
@@ -46,9 +44,9 @@ void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
set_pud(pud, __pud((unsigned long)pmd));
}
-pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long address)
+pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long address, gfp_t gfp)
{
- return kmem_cache_alloc(pmd_cachep, PGALLOC_GFP);
+ return kmem_cache_alloc(pmd_cachep, gfp|__GFP_ZERO);
}
void pmd_free(struct mm_struct *mm, pmd_t *pmd)
diff --git a/arch/sparc/include/asm/pgalloc_32.h b/arch/sparc/include/asm/pgalloc_32.h
index 282be50a4adf..51dea1c004d4 100644
--- a/arch/sparc/include/asm/pgalloc_32.h
+++ b/arch/sparc/include/asm/pgalloc_32.h
@@ -38,7 +38,8 @@ static inline void pgd_set(pgd_t * pgdp, pmd_t * pmdp)
#define pgd_populate(MM, PGD, PMD) pgd_set(PGD, PMD)
static inline pmd_t *pmd_alloc_one(struct mm_struct *mm,
- unsigned long address)
+ unsigned long address,
+ gfp_t gfp)
{
return srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE,
SRMMU_PMD_TABLE_SIZE);
@@ -60,12 +61,11 @@ void pmd_set(pmd_t *pmdp, pte_t *ptep);
pgtable_t pte_alloc_one(struct mm_struct *mm);
-static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm)
+static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, gfp_t gfp)
{
return srmmu_get_nocache(PTE_SIZE, PTE_SIZE);
}
-
static inline void free_pte_fast(pte_t *pte)
{
srmmu_free_nocache(pte, PTE_SIZE);
diff --git a/arch/sparc/include/asm/pgalloc_64.h b/arch/sparc/include/asm/pgalloc_64.h
index 48abccba4991..e772ee60ee57 100644
--- a/arch/sparc/include/asm/pgalloc_64.h
+++ b/arch/sparc/include/asm/pgalloc_64.h
@@ -40,9 +40,10 @@ static inline void __pud_populate(pud_t *pud, pmd_t *pmd)
#define pud_populate(MM, PUD, PMD) __pud_populate(PUD, PMD)
-static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
+static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr,
+ gfp_t gfp)
{
- return kmem_cache_alloc(pgtable_cache, GFP_KERNEL);
+ return kmem_cache_alloc(pgtable_cache, gfp);
}
static inline void pud_free(struct mm_struct *mm, pud_t *pud)
@@ -50,9 +51,10 @@ static inline void pud_free(struct mm_struct *mm, pud_t *pud)
kmem_cache_free(pgtable_cache, pud);
}
-static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
+static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr,
+ gfp_t gfp)
{
- return kmem_cache_alloc(pgtable_cache, GFP_KERNEL);
+ return kmem_cache_alloc(pgtable_cache, gfp);
}
static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
@@ -60,7 +62,7 @@ static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
kmem_cache_free(pgtable_cache, pmd);
}
-pte_t *pte_alloc_one_kernel(struct mm_struct *mm);
+pte_t *pte_alloc_one_kernel(struct mm_struct *mm, gfp_t gfp);
pgtable_t pte_alloc_one(struct mm_struct *mm);
void pte_free_kernel(struct mm_struct *mm, pte_t *pte);
void pte_free(struct mm_struct *mm, pgtable_t ptepage);
diff --git a/arch/sparc/mm/hugetlbpage.c b/arch/sparc/mm/hugetlbpage.c
index f78793a06bbd..aeacfb0aabbe 100644
--- a/arch/sparc/mm/hugetlbpage.c
+++ b/arch/sparc/mm/hugetlbpage.c
@@ -281,12 +281,12 @@ pte_t *huge_pte_alloc(struct mm_struct *mm,
pmd_t *pmd;
pgd = pgd_offset(mm, addr);
- pud = pud_alloc(mm, pgd, addr);
+ pud = pud_alloc(mm, pgd, addr, GFP_KERNEL);
if (!pud)
return NULL;
if (sz >= PUD_SIZE)
return (pte_t *)pud;
- pmd = pmd_alloc(mm, pud, addr);
+ pmd = pmd_alloc(mm, pud, addr, GFP_KERNEL);
if (!pmd)
return NULL;
if (sz >= PMD_SIZE)
diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c
index b4221d3727d0..73cf28b0e35b 100644
--- a/arch/sparc/mm/init_64.c
+++ b/arch/sparc/mm/init_64.c
@@ -2925,15 +2925,9 @@ void __flush_tlb_all(void)
: : "r" (pstate));
}
-pte_t *pte_alloc_one_kernel(struct mm_struct *mm)
+pte_t *pte_alloc_one_kernel(struct mm_struct *mm, gfp_t gfp)
{
- struct page *page = alloc_page(GFP_KERNEL | __GFP_ZERO);
- pte_t *pte = NULL;
-
- if (page)
- pte = (pte_t *) page_address(page);
-
- return pte;
+ return (pte_t *) get_zeroed_page(gfp);
}
pgtable_t pte_alloc_one(struct mm_struct *mm)
diff --git a/arch/sparc/mm/srmmu.c b/arch/sparc/mm/srmmu.c
index b609362e846f..e9cb40c6e4ec 100644
--- a/arch/sparc/mm/srmmu.c
+++ b/arch/sparc/mm/srmmu.c
@@ -369,7 +369,7 @@ pgtable_t pte_alloc_one(struct mm_struct *mm)
unsigned long pte;
struct page *page;
- if ((pte = (unsigned long)pte_alloc_one_kernel(mm)) == 0)
+ if ((pte = (unsigned long)pte_alloc_one_kernel(mm, GFP_KERNEL)) == 0)
return NULL;
page = pfn_to_page(__nocache_pa(pte) >> PAGE_SHIFT);
if (!pgtable_page_ctor(page)) {
diff --git a/arch/um/include/asm/pgalloc.h b/arch/um/include/asm/pgalloc.h
index 99eb5682792a..71090e43d031 100644
--- a/arch/um/include/asm/pgalloc.h
+++ b/arch/um/include/asm/pgalloc.h
@@ -25,7 +25,7 @@
extern pgd_t *pgd_alloc(struct mm_struct *);
extern void pgd_free(struct mm_struct *mm, pgd_t *pgd);
-extern pte_t *pte_alloc_one_kernel(struct mm_struct *);
+extern pte_t *pte_alloc_one_kernel(struct mm_struct *, gfp_t);
extern pgtable_t pte_alloc_one(struct mm_struct *);
static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
diff --git a/arch/um/include/asm/pgtable-3level.h b/arch/um/include/asm/pgtable-3level.h
index c4d876dfb9ac..7f5fd792348f 100644
--- a/arch/um/include/asm/pgtable-3level.h
+++ b/arch/um/include/asm/pgtable-3level.h
@@ -80,7 +80,8 @@ static inline void pgd_mkuptodate(pgd_t pgd) { pgd_val(pgd) &= ~_PAGE_NEWPAGE; }
#endif
struct mm_struct;
-extern pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long address);
+extern pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long address,
+ gfp_t gfp);
static inline void pud_clear (pud_t *pud)
{
diff --git a/arch/um/kernel/mem.c b/arch/um/kernel/mem.c
index 799b571a8f88..84804df746c3 100644
--- a/arch/um/kernel/mem.c
+++ b/arch/um/kernel/mem.c
@@ -199,12 +199,9 @@ void pgd_free(struct mm_struct *mm, pgd_t *pgd)
free_page((unsigned long) pgd);
}
-pte_t *pte_alloc_one_kernel(struct mm_struct *mm)
+pte_t *pte_alloc_one_kernel(struct mm_struct *mm, gfp_t gfp)
{
- pte_t *pte;
-
- pte = (pte_t *)__get_free_page(GFP_KERNEL|__GFP_ZERO);
- return pte;
+ return (pte_t *)get_zeroed_page(gfp);
}
pgtable_t pte_alloc_one(struct mm_struct *mm)
@@ -222,14 +219,10 @@ pgtable_t pte_alloc_one(struct mm_struct *mm)
}
#ifdef CONFIG_3_LEVEL_PGTABLES
-pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long address)
+pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long address,
+ gfp_t gfp)
{
- pmd_t *pmd = (pmd_t *) __get_free_page(GFP_KERNEL);
-
- if (pmd)
- memset(pmd, 0, PAGE_SIZE);
-
- return pmd;
+ return (pmd_t *) get_zeroed_page(gfp);
}
#endif
diff --git a/arch/um/kernel/skas/mmu.c b/arch/um/kernel/skas/mmu.c
index 7a1f2a936fd1..b677b615d691 100644
--- a/arch/um/kernel/skas/mmu.c
+++ b/arch/um/kernel/skas/mmu.c
@@ -24,11 +24,11 @@ static int init_stub_pte(struct mm_struct *mm, unsigned long proc,
pte_t *pte;
pgd = pgd_offset(mm, proc);
- pud = pud_alloc(mm, pgd, proc);
+ pud = pud_alloc(mm, pgd, proc, GFP_KERNEL);
if (!pud)
goto out;
- pmd = pmd_alloc(mm, pud, proc);
+ pmd = pmd_alloc(mm, pud, proc, GFP_KERNEL);
if (!pmd)
goto out_pmd;
diff --git a/arch/unicore32/include/asm/pgalloc.h b/arch/unicore32/include/asm/pgalloc.h
index 7cceabecf4e3..e5f6c1ae6414 100644
--- a/arch/unicore32/include/asm/pgalloc.h
+++ b/arch/unicore32/include/asm/pgalloc.h
@@ -28,17 +28,15 @@ extern void free_pgd_slow(struct mm_struct *mm, pgd_t *pgd);
#define pgd_alloc(mm) get_pgd_slow(mm)
#define pgd_free(mm, pgd) free_pgd_slow(mm, pgd)
-#define PGALLOC_GFP (GFP_KERNEL | __GFP_ZERO)
-
/*
* Allocate one PTE table.
*/
static inline pte_t *
-pte_alloc_one_kernel(struct mm_struct *mm)
+pte_alloc_one_kernel(struct mm_struct *mm, gfp_t gfp)
{
pte_t *pte;
- pte = (pte_t *)__get_free_page(PGALLOC_GFP);
+ pte = (pte_t *)get_zeroed_page(gfp);
if (pte)
clean_dcache_area(pte, PTRS_PER_PTE * sizeof(pte_t));
@@ -50,7 +48,7 @@ pte_alloc_one(struct mm_struct *mm)
{
struct page *pte;
- pte = alloc_pages(PGALLOC_GFP, 0);
+ pte = alloc_pages(GFP_KERNEL|__GFP_ZERO, 0);
if (!pte)
return NULL;
if (!PageHighMem(pte)) {
diff --git a/arch/unicore32/mm/pgd.c b/arch/unicore32/mm/pgd.c
index a830a300aaa1..b9c628a55f21 100644
--- a/arch/unicore32/mm/pgd.c
+++ b/arch/unicore32/mm/pgd.c
@@ -50,7 +50,7 @@ pgd_t *get_pgd_slow(struct mm_struct *mm)
* On UniCore, first page must always be allocated since it
* contains the machine vectors.
*/
- new_pmd = pmd_alloc(mm, (pud_t *)new_pgd, 0);
+ new_pmd = pmd_alloc(mm, (pud_t *)new_pgd, 0, GFP_KERNEL);
if (!new_pmd)
goto no_pmd;
diff --git a/arch/x86/include/asm/pgalloc.h b/arch/x86/include/asm/pgalloc.h
index a281e61ec60c..1909a8dfaf00 100644
--- a/arch/x86/include/asm/pgalloc.h
+++ b/arch/x86/include/asm/pgalloc.h
@@ -47,7 +47,7 @@ extern gfp_t __userpte_alloc_gfp;
extern pgd_t *pgd_alloc(struct mm_struct *);
extern void pgd_free(struct mm_struct *mm, pgd_t *pgd);
-extern pte_t *pte_alloc_one_kernel(struct mm_struct *);
+extern pte_t *pte_alloc_one_kernel(struct mm_struct *, gfp_t);
extern pgtable_t pte_alloc_one(struct mm_struct *);
/* Should really implement gc for free page table pages. This could be
@@ -99,14 +99,14 @@ static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd,
#define pmd_pgtable(pmd) pmd_page(pmd)
#if CONFIG_PGTABLE_LEVELS > 2
-static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
+static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr,
+ gfp_t gfp)
{
struct page *page;
- gfp_t gfp = GFP_KERNEL_ACCOUNT | __GFP_ZERO;
- if (mm == &init_mm)
- gfp &= ~__GFP_ACCOUNT;
- page = alloc_pages(gfp, 0);
+ if (mm != &init_mm)
+ gfp |= __GFP_ACCOUNT;
+ page = alloc_pages(gfp|__GFP_ZERO, 0);
if (!page)
return NULL;
if (!pgtable_pmd_page_ctor(page)) {
@@ -160,12 +160,11 @@ static inline void p4d_populate_safe(struct mm_struct *mm, p4d_t *p4d, pud_t *pu
set_p4d_safe(p4d, __p4d(_PAGE_TABLE | __pa(pud)));
}
-static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
+static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr,
+ gfp_t gfp)
{
- gfp_t gfp = GFP_KERNEL_ACCOUNT;
-
- if (mm == &init_mm)
- gfp &= ~__GFP_ACCOUNT;
+ if (mm != &init_mm)
+ gfp |= __GFP_ACCOUNT;
return (pud_t *)get_zeroed_page(gfp);
}
@@ -200,12 +199,11 @@ static inline void pgd_populate_safe(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4
set_pgd_safe(pgd, __pgd(_PAGE_TABLE | __pa(p4d)));
}
-static inline p4d_t *p4d_alloc_one(struct mm_struct *mm, unsigned long addr)
+static inline p4d_t *p4d_alloc_one(struct mm_struct *mm, unsigned long addr,
+ gfp_t gfp)
{
- gfp_t gfp = GFP_KERNEL_ACCOUNT;
-
- if (mm == &init_mm)
- gfp &= ~__GFP_ACCOUNT;
+ if (mm != &init_mm)
+ gfp |= __GFP_ACCOUNT;
return (p4d_t *)get_zeroed_page(gfp);
}
diff --git a/arch/x86/kernel/espfix_64.c b/arch/x86/kernel/espfix_64.c
index aebd0d5bc086..46df9bb51f93 100644
--- a/arch/x86/kernel/espfix_64.c
+++ b/arch/x86/kernel/espfix_64.c
@@ -126,7 +126,7 @@ void __init init_espfix_bsp(void)
/* Install the espfix pud into the kernel page directory */
pgd = &init_top_pgt[pgd_index(ESPFIX_BASE_ADDR)];
- p4d = p4d_alloc(&init_mm, pgd, ESPFIX_BASE_ADDR);
+ p4d = p4d_alloc(&init_mm, pgd, ESPFIX_BASE_ADDR, GFP_KERNEL);
p4d_populate(&init_mm, p4d, espfix_pud_page);
/* Randomize the locations */
diff --git a/arch/x86/kernel/tboot.c b/arch/x86/kernel/tboot.c
index 6e5ef8fb8a02..9a4f0fa6d60f 100644
--- a/arch/x86/kernel/tboot.c
+++ b/arch/x86/kernel/tboot.c
@@ -124,13 +124,13 @@ static int map_tboot_page(unsigned long vaddr, unsigned long pfn,
pte_t *pte;
pgd = pgd_offset(&tboot_mm, vaddr);
- p4d = p4d_alloc(&tboot_mm, pgd, vaddr);
+ p4d = p4d_alloc(&tboot_mm, pgd, vaddr, GFP_KERNEL);
if (!p4d)
return -1;
- pud = pud_alloc(&tboot_mm, p4d, vaddr);
+ pud = pud_alloc(&tboot_mm, p4d, vaddr, GFP_KERNEL);
if (!pud)
return -1;
- pmd = pmd_alloc(&tboot_mm, pud, vaddr);
+ pmd = pmd_alloc(&tboot_mm, pud, vaddr, GFP_KERNEL);
if (!pmd)
return -1;
pte = pte_alloc_map(&tboot_mm, pmd, vaddr);
diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c
index 7bd01709a091..04cb5aec7b2a 100644
--- a/arch/x86/mm/pgtable.c
+++ b/arch/x86/mm/pgtable.c
@@ -23,9 +23,9 @@ EXPORT_SYMBOL(physical_mask);
gfp_t __userpte_alloc_gfp = PGALLOC_GFP | PGALLOC_USER_GFP;
-pte_t *pte_alloc_one_kernel(struct mm_struct *mm)
+pte_t *pte_alloc_one_kernel(struct mm_struct *mm, gfp_t gfp)
{
- return (pte_t *)__get_free_page(PGALLOC_GFP & ~__GFP_ACCOUNT);
+ return (pte_t *) get_zeroed_page(gfp);
}
pgtable_t pte_alloc_one(struct mm_struct *mm)
diff --git a/arch/x86/platform/efi/efi_64.c b/arch/x86/platform/efi/efi_64.c
index cf0347f61b21..9cb455ba2864 100644
--- a/arch/x86/platform/efi/efi_64.c
+++ b/arch/x86/platform/efi/efi_64.c
@@ -106,7 +106,7 @@ pgd_t * __init efi_call_phys_prolog(void)
pgd_efi = pgd_offset_k(addr_pgd);
save_pgd[pgd] = *pgd_efi;
- p4d = p4d_alloc(&init_mm, pgd_efi, addr_pgd);
+ p4d = p4d_alloc(&init_mm, pgd_efi, addr_pgd, GFP_KERNEL);
if (!p4d) {
pr_err("Failed to allocate p4d table!\n");
goto out;
@@ -116,7 +116,8 @@ pgd_t * __init efi_call_phys_prolog(void)
addr_p4d = addr_pgd + i * P4D_SIZE;
p4d_efi = p4d + p4d_index(addr_p4d);
- pud = pud_alloc(&init_mm, p4d_efi, addr_p4d);
+ pud = pud_alloc(&init_mm, p4d_efi, addr_p4d,
+ GFP_KERNEL);
if (!pud) {
pr_err("Failed to allocate pud table!\n");
goto out;
@@ -217,13 +218,13 @@ int __init efi_alloc_page_tables(void)
return -ENOMEM;
pgd = efi_pgd + pgd_index(EFI_VA_END);
- p4d = p4d_alloc(&init_mm, pgd, EFI_VA_END);
+ p4d = p4d_alloc(&init_mm, pgd, EFI_VA_END, GFP_KERNEL);
if (!p4d) {
free_page((unsigned long)efi_pgd);
return -ENOMEM;
}
- pud = pud_alloc(&init_mm, p4d, EFI_VA_END);
+ pud = pud_alloc(&init_mm, p4d, EFI_VA_END, GFP_KERNEL);
if (!pud) {
if (pgtable_l5_enabled())
free_page((unsigned long) pgd_page_vaddr(*pgd));
diff --git a/arch/xtensa/include/asm/pgalloc.h b/arch/xtensa/include/asm/pgalloc.h
index b3b388ff2f01..cc7ec6dd0941 100644
--- a/arch/xtensa/include/asm/pgalloc.h
+++ b/arch/xtensa/include/asm/pgalloc.h
@@ -38,12 +38,12 @@ static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)
free_page((unsigned long)pgd);
}
-static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm)
+static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, gfp_t gfp)
{
pte_t *ptep;
int i;
- ptep = (pte_t *)__get_free_page(GFP_KERNEL);
+ ptep = (pte_t *)__get_free_page(gfp);
if (!ptep)
return NULL;
for (i = 0; i < 1024; i++)
diff --git a/include/asm-generic/4level-fixup.h b/include/asm-generic/4level-fixup.h
index e3667c9a33a5..652b68f47555 100644
--- a/include/asm-generic/4level-fixup.h
+++ b/include/asm-generic/4level-fixup.h
@@ -12,9 +12,9 @@
#define pud_t pgd_t
-#define pmd_alloc(mm, pud, address) \
- ((unlikely(pgd_none(*(pud))) && __pmd_alloc(mm, pud, address))? \
- NULL: pmd_offset(pud, address))
+#define pmd_alloc(mm, pud, address, gfp) \
+ ((unlikely(pgd_none(*(pud))) && __pmd_alloc(mm, pud, address, gfp)) \
+ ? NULL : pmd_offset(pud, address))
#define pud_offset(pgd, start) (pgd)
#define pud_none(pud) 0
diff --git a/include/asm-generic/5level-fixup.h b/include/asm-generic/5level-fixup.h
index bb6cb347018c..c6f68f6a9fcf 100644
--- a/include/asm-generic/5level-fixup.h
+++ b/include/asm-generic/5level-fixup.h
@@ -13,11 +13,11 @@
#define p4d_t pgd_t
-#define pud_alloc(mm, p4d, address) \
- ((unlikely(pgd_none(*(p4d))) && __pud_alloc(mm, p4d, address)) ? \
+#define pud_alloc(mm, p4d, address, gfp) \
+ ((unlikely(pgd_none(*(p4d))) && __pud_alloc(mm, p4d, address, gfp)) ? \
NULL : pud_offset(p4d, address))
-#define p4d_alloc(mm, pgd, address) (pgd)
+#define p4d_alloc(mm, pgd, address, gfp) (pgd)
#define p4d_offset(pgd, start) (pgd)
#define p4d_none(p4d) 0
#define p4d_bad(p4d) 0
diff --git a/include/asm-generic/pgtable-nop4d-hack.h b/include/asm-generic/pgtable-nop4d-hack.h
index 829bdb0d6327..3ba3c7e4b99e 100644
--- a/include/asm-generic/pgtable-nop4d-hack.h
+++ b/include/asm-generic/pgtable-nop4d-hack.h
@@ -53,7 +53,7 @@ static inline pud_t *pud_offset(pgd_t *pgd, unsigned long address)
* allocating and freeing a pud is trivial: the 1-entry pud is
* inside the pgd, so has no extra memory associated with it.
*/
-#define pud_alloc_one(mm, address) NULL
+#define pud_alloc_one(mm, address, gfp) NULL
#define pud_free(mm, x) do { } while (0)
#define __pud_free_tlb(tlb, x, a) do { } while (0)
diff --git a/include/asm-generic/pgtable-nop4d.h b/include/asm-generic/pgtable-nop4d.h
index aebab905e6cd..7c9e00e44d8b 100644
--- a/include/asm-generic/pgtable-nop4d.h
+++ b/include/asm-generic/pgtable-nop4d.h
@@ -48,7 +48,7 @@ static inline p4d_t *p4d_offset(pgd_t *pgd, unsigned long address)
* allocating and freeing a p4d is trivial: the 1-entry p4d is
* inside the pgd, so has no extra memory associated with it.
*/
-#define p4d_alloc_one(mm, address) NULL
+#define p4d_alloc_one(mm, address, gfp) NULL
#define p4d_free(mm, x) do { } while (0)
#define __p4d_free_tlb(tlb, x, a) do { } while (0)
diff --git a/include/asm-generic/pgtable-nopmd.h b/include/asm-generic/pgtable-nopmd.h
index b85b8271a73d..e4a51cbdefbb 100644
--- a/include/asm-generic/pgtable-nopmd.h
+++ b/include/asm-generic/pgtable-nopmd.h
@@ -56,7 +56,7 @@ static inline pmd_t * pmd_offset(pud_t * pud, unsigned long address)
* allocating and freeing a pmd is trivial: the 1-entry pmd is
* inside the pud, so has no extra memory associated with it.
*/
-#define pmd_alloc_one(mm, address) NULL
+#define pmd_alloc_one(mm, address, gfp) NULL
static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
{
}
diff --git a/include/asm-generic/pgtable-nopud.h b/include/asm-generic/pgtable-nopud.h
index c77a1d301155..e7aacf134c02 100644
--- a/include/asm-generic/pgtable-nopud.h
+++ b/include/asm-generic/pgtable-nopud.h
@@ -57,7 +57,7 @@ static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address)
* allocating and freeing a pud is trivial: the 1-entry pud is
* inside the p4d, so has no extra memory associated with it.
*/
-#define pud_alloc_one(mm, address) NULL
+#define pud_alloc_one(mm, address, gfp) NULL
#define pud_free(mm, x) do { } while (0)
#define __pud_free_tlb(tlb, x, a) do { } while (0)
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 80bb6408fe73..8604ae2e1c05 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -1780,17 +1780,18 @@ static inline pte_t *get_locked_pte(struct mm_struct *mm, unsigned long addr,
#ifdef __PAGETABLE_P4D_FOLDED
static inline int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd,
- unsigned long address)
+ unsigned long address, gfp_t gfp)
{
return 0;
}
#else
-int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address);
+int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd,
+ unsigned long address, gfp_t gfp);
#endif
#if defined(__PAGETABLE_PUD_FOLDED) || !defined(CONFIG_MMU)
static inline int __pud_alloc(struct mm_struct *mm, p4d_t *p4d,
- unsigned long address)
+ unsigned long address, gfp_t gfp)
{
return 0;
}
@@ -1798,7 +1799,8 @@ static inline void mm_inc_nr_puds(struct mm_struct *mm) {}
static inline void mm_dec_nr_puds(struct mm_struct *mm) {}
#else
-int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address);
+int __pud_alloc(struct mm_struct *mm, p4d_t *p4d,
+ unsigned long address, gfp_t gfp);
static inline void mm_inc_nr_puds(struct mm_struct *mm)
{
@@ -1817,7 +1819,7 @@ static inline void mm_dec_nr_puds(struct mm_struct *mm)
#if defined(__PAGETABLE_PMD_FOLDED) || !defined(CONFIG_MMU)
static inline int __pmd_alloc(struct mm_struct *mm, pud_t *pud,
- unsigned long address)
+ unsigned long address, gfp_t gfp)
{
return 0;
}
@@ -1826,7 +1828,8 @@ static inline void mm_inc_nr_pmds(struct mm_struct *mm) {}
static inline void mm_dec_nr_pmds(struct mm_struct *mm) {}
#else
-int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address);
+int __pmd_alloc(struct mm_struct *mm, pud_t *pud,
+ unsigned long address, gfp_t gfp);
static inline void mm_inc_nr_pmds(struct mm_struct *mm)
{
@@ -1876,7 +1879,7 @@ static inline void mm_dec_nr_ptes(struct mm_struct *mm) {}
#endif
int __pte_alloc(struct mm_struct *mm, pmd_t *pmd);
-int __pte_alloc_kernel(pmd_t *pmd);
+int __pte_alloc_kernel(pmd_t *pmd, gfp_t gfp);
/*
* The following ifdef needed to get the 4level-fixup.h header to work.
@@ -1886,24 +1889,25 @@ int __pte_alloc_kernel(pmd_t *pmd);
#ifndef __ARCH_HAS_5LEVEL_HACK
static inline p4d_t *p4d_alloc(struct mm_struct *mm, pgd_t *pgd,
- unsigned long address)
+ unsigned long address, gfp_t gfp)
{
- return (unlikely(pgd_none(*pgd)) && __p4d_alloc(mm, pgd, address)) ?
- NULL : p4d_offset(pgd, address);
+ return (unlikely(pgd_none(*pgd)) && __p4d_alloc(mm, pgd, address, gfp))
+ ? NULL : p4d_offset(pgd, address);
}
static inline pud_t *pud_alloc(struct mm_struct *mm, p4d_t *p4d,
- unsigned long address)
+ unsigned long address, gfp_t gfp)
{
- return (unlikely(p4d_none(*p4d)) && __pud_alloc(mm, p4d, address)) ?
- NULL : pud_offset(p4d, address);
+ return (unlikely(p4d_none(*p4d)) && __pud_alloc(mm, p4d, address, gfp))
+ ? NULL : pud_offset(p4d, address);
}
#endif /* !__ARCH_HAS_5LEVEL_HACK */
-static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
+static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud,
+ unsigned long address, gfp_t gfp)
{
- return (unlikely(pud_none(*pud)) && __pmd_alloc(mm, pud, address))?
- NULL: pmd_offset(pud, address);
+ return (unlikely(pud_none(*pud)) && __pmd_alloc(mm, pud, address, gfp))
+ ? NULL : pmd_offset(pud, address);
}
#endif /* CONFIG_MMU && !__ARCH_HAS_4LEVEL_HACK */
@@ -2016,8 +2020,8 @@ static inline void pgtable_page_dtor(struct page *page)
(pte_alloc(mm, pmd) ? \
NULL : pte_offset_map_lock(mm, pmd, address, ptlp))
-#define pte_alloc_kernel(pmd, address) \
- ((unlikely(pmd_none(*(pmd))) && __pte_alloc_kernel(pmd))? \
+#define pte_alloc_kernel(pmd, address, gfp) \
+ ((unlikely(pmd_none(*(pmd))) && __pte_alloc_kernel(pmd, gfp))? \
NULL: pte_offset_kernel(pmd, address))
#if USE_SPLIT_PMD_PTLOCKS
diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h
index 398e9c95cd61..11788d5ba379 100644
--- a/include/linux/vmalloc.h
+++ b/include/linux/vmalloc.h
@@ -135,7 +135,7 @@ extern struct vm_struct *__get_vm_area_caller(unsigned long size,
extern struct vm_struct *remove_vm_area(const void *addr);
extern struct vm_struct *find_vm_area(const void *addr);
-extern int map_vm_area(struct vm_struct *area, pgprot_t prot,
+extern int map_vm_area(struct vm_struct *area, gfp_t gfp, pgprot_t prot,
struct page **pages);
#ifdef CONFIG_MMU
extern int map_kernel_range_noflush(unsigned long start, unsigned long size,
diff --git a/lib/ioremap.c b/lib/ioremap.c
index 063213685563..a4e21ef50c7e 100644
--- a/lib/ioremap.c
+++ b/lib/ioremap.c
@@ -65,7 +65,7 @@ static int ioremap_pte_range(pmd_t *pmd, unsigned long addr,
u64 pfn;
pfn = phys_addr >> PAGE_SHIFT;
- pte = pte_alloc_kernel(pmd, addr);
+ pte = pte_alloc_kernel(pmd, addr, GFP_KERNEL);
if (!pte)
return -ENOMEM;
do {
@@ -101,7 +101,7 @@ static inline int ioremap_pmd_range(pud_t *pud, unsigned long addr,
pmd_t *pmd;
unsigned long next;
- pmd = pmd_alloc(&init_mm, pud, addr);
+ pmd = pmd_alloc(&init_mm, pud, addr, GFP_KERNEL);
if (!pmd)
return -ENOMEM;
do {
@@ -141,7 +141,7 @@ static inline int ioremap_pud_range(p4d_t *p4d, unsigned long addr,
pud_t *pud;
unsigned long next;
- pud = pud_alloc(&init_mm, p4d, addr);
+ pud = pud_alloc(&init_mm, p4d, addr, GFP_KERNEL);
if (!pud)
return -ENOMEM;
do {
@@ -181,7 +181,7 @@ static inline int ioremap_p4d_range(pgd_t *pgd, unsigned long addr,
p4d_t *p4d;
unsigned long next;
- p4d = p4d_alloc(&init_mm, pgd, addr);
+ p4d = p4d_alloc(&init_mm, pgd, addr, GFP_KERNEL);
if (!p4d)
return -ENOMEM;
do {
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 8dfdffc34a99..1eecd0a2e8ce 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -4667,7 +4667,7 @@ pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud)
spinlock_t *ptl;
if (!vma_shareable(vma, addr))
- return (pte_t *)pmd_alloc(mm, pud, addr);
+ return (pte_t *)pmd_alloc(mm, pud, addr, GFP_KERNEL);
i_mmap_lock_write(mapping);
vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) {
@@ -4698,7 +4698,7 @@ pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud)
}
spin_unlock(ptl);
out:
- pte = (pte_t *)pmd_alloc(mm, pud, addr);
+ pte = (pte_t *)pmd_alloc(mm, pud, addr, GFP_KERNEL);
i_mmap_unlock_write(mapping);
return pte;
}
@@ -4760,10 +4760,10 @@ pte_t *huge_pte_alloc(struct mm_struct *mm,
pte_t *pte = NULL;
pgd = pgd_offset(mm, addr);
- p4d = p4d_alloc(mm, pgd, addr);
+ p4d = p4d_alloc(mm, pgd, addr, GFP_KERNEL);
if (!p4d)
return NULL;
- pud = pud_alloc(mm, p4d, addr);
+ pud = pud_alloc(mm, p4d, addr, GFP_KERNEL);
if (pud) {
if (sz == PUD_SIZE) {
pte = (pte_t *)pud;
@@ -4772,7 +4772,8 @@ pte_t *huge_pte_alloc(struct mm_struct *mm,
if (want_pmd_share() && pud_none(*pud))
pte = huge_pmd_share(mm, addr, pud);
else
- pte = (pte_t *)pmd_alloc(mm, pud, addr);
+ pte = (pte_t *)pmd_alloc(mm, pud, addr,
+ GFP_KERNEL);
}
}
BUG_ON(pte && pte_present(*pte) && !pte_huge(*pte));
diff --git a/mm/kasan/init.c b/mm/kasan/init.c
index 45a1b5e38e1e..e6f4ed529cae 100644
--- a/mm/kasan/init.c
+++ b/mm/kasan/init.c
@@ -123,7 +123,7 @@ static int __ref zero_pmd_populate(pud_t *pud, unsigned long addr,
pte_t *p;
if (slab_is_available())
- p = pte_alloc_one_kernel(&init_mm);
+ p = pte_alloc_one_kernel(&init_mm, GFP_KERNEL);
else
p = early_alloc(PAGE_SIZE, NUMA_NO_NODE);
if (!p)
@@ -160,7 +160,7 @@ static int __ref zero_pud_populate(p4d_t *p4d, unsigned long addr,
pmd_t *p;
if (slab_is_available()) {
- p = pmd_alloc(&init_mm, pud, addr);
+ p = pmd_alloc(&init_mm, pud, addr, GFP_KERNEL);
if (!p)
return -ENOMEM;
} else {
@@ -201,7 +201,7 @@ static int __ref zero_p4d_populate(pgd_t *pgd, unsigned long addr,
pud_t *p;
if (slab_is_available()) {
- p = pud_alloc(&init_mm, p4d, addr);
+ p = pud_alloc(&init_mm, p4d, addr, GFP_KERNEL);
if (!p)
return -ENOMEM;
} else {
@@ -274,7 +274,7 @@ int __ref kasan_populate_early_shadow(const void *shadow_start,
p4d_t *p;
if (slab_is_available()) {
- p = p4d_alloc(&init_mm, pgd, addr);
+ p = p4d_alloc(&init_mm, pgd, addr, GFP_KERNEL);
if (!p)
return -ENOMEM;
} else {
diff --git a/mm/memory.c b/mm/memory.c
index e11ca9dd823f..2931d3e418cb 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -434,9 +434,9 @@ int __pte_alloc(struct mm_struct *mm, pmd_t *pmd)
return 0;
}
-int __pte_alloc_kernel(pmd_t *pmd)
+int __pte_alloc_kernel(pmd_t *pmd, gfp_t gfp)
{
- pte_t *new = pte_alloc_one_kernel(&init_mm);
+ pte_t *new = pte_alloc_one_kernel(&init_mm, gfp);
if (!new)
return -ENOMEM;
@@ -883,7 +883,7 @@ static inline int copy_pmd_range(struct mm_struct *dst_mm, struct mm_struct *src
pmd_t *src_pmd, *dst_pmd;
unsigned long next;
- dst_pmd = pmd_alloc(dst_mm, dst_pud, addr);
+ dst_pmd = pmd_alloc(dst_mm, dst_pud, addr, GFP_KERNEL);
if (!dst_pmd)
return -ENOMEM;
src_pmd = pmd_offset(src_pud, addr);
@@ -917,7 +917,7 @@ static inline int copy_pud_range(struct mm_struct *dst_mm, struct mm_struct *src
pud_t *src_pud, *dst_pud;
unsigned long next;
- dst_pud = pud_alloc(dst_mm, dst_p4d, addr);
+ dst_pud = pud_alloc(dst_mm, dst_p4d, addr, GFP_KERNEL);
if (!dst_pud)
return -ENOMEM;
src_pud = pud_offset(src_p4d, addr);
@@ -951,7 +951,7 @@ static inline int copy_p4d_range(struct mm_struct *dst_mm, struct mm_struct *src
p4d_t *src_p4d, *dst_p4d;
unsigned long next;
- dst_p4d = p4d_alloc(dst_mm, dst_pgd, addr);
+ dst_p4d = p4d_alloc(dst_mm, dst_pgd, addr, GFP_KERNEL);
if (!dst_p4d)
return -ENOMEM;
src_p4d = p4d_offset(src_pgd, addr);
@@ -1421,13 +1421,13 @@ pte_t *__get_locked_pte(struct mm_struct *mm, unsigned long addr,
pmd_t *pmd;
pgd = pgd_offset(mm, addr);
- p4d = p4d_alloc(mm, pgd, addr);
+ p4d = p4d_alloc(mm, pgd, addr, GFP_KERNEL);
if (!p4d)
return NULL;
- pud = pud_alloc(mm, p4d, addr);
+ pud = pud_alloc(mm, p4d, addr, GFP_KERNEL);
if (!pud)
return NULL;
- pmd = pmd_alloc(mm, pud, addr);
+ pmd = pmd_alloc(mm, pud, addr, GFP_KERNEL);
if (!pmd)
return NULL;
@@ -1764,7 +1764,7 @@ static inline int remap_pmd_range(struct mm_struct *mm, pud_t *pud,
int err;
pfn -= addr >> PAGE_SHIFT;
- pmd = pmd_alloc(mm, pud, addr);
+ pmd = pmd_alloc(mm, pud, addr, GFP_KERNEL);
if (!pmd)
return -ENOMEM;
VM_BUG_ON(pmd_trans_huge(*pmd));
@@ -1787,7 +1787,7 @@ static inline int remap_pud_range(struct mm_struct *mm, p4d_t *p4d,
int err;
pfn -= addr >> PAGE_SHIFT;
- pud = pud_alloc(mm, p4d, addr);
+ pud = pud_alloc(mm, p4d, addr, GFP_KERNEL);
if (!pud)
return -ENOMEM;
do {
@@ -1809,7 +1809,7 @@ static inline int remap_p4d_range(struct mm_struct *mm, pgd_t *pgd,
int err;
pfn -= addr >> PAGE_SHIFT;
- p4d = p4d_alloc(mm, pgd, addr);
+ p4d = p4d_alloc(mm, pgd, addr, GFP_KERNEL);
if (!p4d)
return -ENOMEM;
do {
@@ -1948,7 +1948,7 @@ static int apply_to_pte_range(struct mm_struct *mm, pmd_t *pmd,
spinlock_t *uninitialized_var(ptl);
pte = (mm == &init_mm) ?
- pte_alloc_kernel(pmd, addr) :
+ pte_alloc_kernel(pmd, addr, GFP_KERNEL) :
pte_alloc_map_lock(mm, pmd, addr, &ptl);
if (!pte)
return -ENOMEM;
@@ -1982,7 +1982,7 @@ static int apply_to_pmd_range(struct mm_struct *mm, pud_t *pud,
BUG_ON(pud_huge(*pud));
- pmd = pmd_alloc(mm, pud, addr);
+ pmd = pmd_alloc(mm, pud, addr, GFP_KERNEL);
if (!pmd)
return -ENOMEM;
do {
@@ -2002,7 +2002,7 @@ static int apply_to_pud_range(struct mm_struct *mm, p4d_t *p4d,
unsigned long next;
int err;
- pud = pud_alloc(mm, p4d, addr);
+ pud = pud_alloc(mm, p4d, addr, GFP_KERNEL);
if (!pud)
return -ENOMEM;
do {
@@ -2022,7 +2022,7 @@ static int apply_to_p4d_range(struct mm_struct *mm, pgd_t *pgd,
unsigned long next;
int err;
- p4d = p4d_alloc(mm, pgd, addr);
+ p4d = p4d_alloc(mm, pgd, addr, GFP_KERNEL);
if (!p4d)
return -ENOMEM;
do {
@@ -3845,11 +3845,11 @@ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma,
vm_fault_t ret;
pgd = pgd_offset(mm, address);
- p4d = p4d_alloc(mm, pgd, address);
+ p4d = p4d_alloc(mm, pgd, address, GFP_KERNEL);
if (!p4d)
return VM_FAULT_OOM;
- vmf.pud = pud_alloc(mm, p4d, address);
+ vmf.pud = pud_alloc(mm, p4d, address, GFP_KERNEL);
if (!vmf.pud)
return VM_FAULT_OOM;
if (pud_none(*vmf.pud) && __transparent_hugepage_enabled(vma)) {
@@ -3875,7 +3875,7 @@ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma,
}
}
- vmf.pmd = pmd_alloc(mm, vmf.pud, address);
+ vmf.pmd = pmd_alloc(mm, vmf.pud, address, GFP_KERNEL);
if (!vmf.pmd)
return VM_FAULT_OOM;
if (pmd_none(*vmf.pmd) && __transparent_hugepage_enabled(vma)) {
@@ -3968,9 +3968,10 @@ EXPORT_SYMBOL_GPL(handle_mm_fault);
* Allocate p4d page table.
* We've already handled the fast-path in-line.
*/
-int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
+int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address,
+ gfp_t gfp)
{
- p4d_t *new = p4d_alloc_one(mm, address);
+ p4d_t *new = p4d_alloc_one(mm, address, gfp);
if (!new)
return -ENOMEM;
@@ -3991,9 +3992,10 @@ int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
* Allocate page upper directory.
* We've already handled the fast-path in-line.
*/
-int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address)
+int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address,
+ gfp_t gfp)
{
- pud_t *new = pud_alloc_one(mm, address);
+ pud_t *new = pud_alloc_one(mm, address, gfp);
if (!new)
return -ENOMEM;
@@ -4023,10 +4025,11 @@ int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address)
* Allocate page middle directory.
* We've already handled the fast-path in-line.
*/
-int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
+int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address,
+ gfp_t gfp)
{
spinlock_t *ptl;
- pmd_t *new = pmd_alloc_one(mm, address);
+ pmd_t *new = pmd_alloc_one(mm, address, gfp);
if (!new)
return -ENOMEM;
diff --git a/mm/migrate.c b/mm/migrate.c
index 181f5d2718a9..5ebc71f10213 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -2611,13 +2611,13 @@ static void migrate_vma_insert_page(struct migrate_vma *migrate,
goto abort;
pgdp = pgd_offset(mm, addr);
- p4dp = p4d_alloc(mm, pgdp, addr);
+ p4dp = p4d_alloc(mm, pgdp, addr, GFP_KERNEL);
if (!p4dp)
goto abort;
- pudp = pud_alloc(mm, p4dp, addr);
+ pudp = pud_alloc(mm, p4dp, addr, GFP_KERNEL);
if (!pudp)
goto abort;
- pmdp = pmd_alloc(mm, pudp, addr);
+ pmdp = pmd_alloc(mm, pudp, addr, GFP_KERNEL);
if (!pmdp)
goto abort;
diff --git a/mm/mremap.c b/mm/mremap.c
index 3320616ed93f..4be0866a6f2d 100644
--- a/mm/mremap.c
+++ b/mm/mremap.c
@@ -65,14 +65,14 @@ static pmd_t *alloc_new_pmd(struct mm_struct *mm, struct vm_area_struct *vma,
pmd_t *pmd;
pgd = pgd_offset(mm, addr);
- p4d = p4d_alloc(mm, pgd, addr);
+ p4d = p4d_alloc(mm, pgd, addr, GFP_KERNEL);
if (!p4d)
return NULL;
- pud = pud_alloc(mm, p4d, addr);
+ pud = pud_alloc(mm, p4d, addr, GFP_KERNEL);
if (!pud)
return NULL;
- pmd = pmd_alloc(mm, pud, addr);
+ pmd = pmd_alloc(mm, pud, addr, GFP_KERNEL);
if (!pmd)
return NULL;
diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c
index d59b5a73dfb3..9bb9d448342b 100644
--- a/mm/userfaultfd.c
+++ b/mm/userfaultfd.c
@@ -153,10 +153,10 @@ static pmd_t *mm_alloc_pmd(struct mm_struct *mm, unsigned long address)
pud_t *pud;
pgd = pgd_offset(mm, address);
- p4d = p4d_alloc(mm, pgd, address);
+ p4d = p4d_alloc(mm, pgd, address, GFP_KERNEL);
if (!p4d)
return NULL;
- pud = pud_alloc(mm, p4d, address);
+ pud = pud_alloc(mm, p4d, address, GFP_KERNEL);
if (!pud)
return NULL;
/*
@@ -164,7 +164,7 @@ static pmd_t *mm_alloc_pmd(struct mm_struct *mm, unsigned long address)
* missing, the *pmd may be already established and in
* turn it may also be a trans_huge_pmd.
*/
- return pmd_alloc(mm, pud, address);
+ return pmd_alloc(mm, pud, address, GFP_KERNEL);
}
#ifdef CONFIG_HUGETLB_PAGE
diff --git a/mm/vmalloc.c b/mm/vmalloc.c
index 871e41c55e23..40b9465cb394 100644
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -132,7 +132,8 @@ static void vunmap_page_range(unsigned long addr, unsigned long end)
}
static int vmap_pte_range(pmd_t *pmd, unsigned long addr,
- unsigned long end, pgprot_t prot, struct page **pages, int *nr)
+ unsigned long end, gfp_t gfp, pgprot_t prot,
+ struct page **pages, int *nr)
{
pte_t *pte;
@@ -141,7 +142,7 @@ static int vmap_pte_range(pmd_t *pmd, unsigned long addr,
* callers keep track of where we're up to.
*/
- pte = pte_alloc_kernel(pmd, addr);
+ pte = pte_alloc_kernel(pmd, addr, gfp);
if (!pte)
return -ENOMEM;
do {
@@ -158,51 +159,54 @@ static int vmap_pte_range(pmd_t *pmd, unsigned long addr,
}
static int vmap_pmd_range(pud_t *pud, unsigned long addr,
- unsigned long end, pgprot_t prot, struct page **pages, int *nr)
+ unsigned long end, gfp_t gfp, pgprot_t prot,
+ struct page **pages, int *nr)
{
pmd_t *pmd;
unsigned long next;
- pmd = pmd_alloc(&init_mm, pud, addr);
+ pmd = pmd_alloc(&init_mm, pud, addr, gfp);
if (!pmd)
return -ENOMEM;
do {
next = pmd_addr_end(addr, end);
- if (vmap_pte_range(pmd, addr, next, prot, pages, nr))
+ if (vmap_pte_range(pmd, addr, next, gfp, prot, pages, nr))
return -ENOMEM;
} while (pmd++, addr = next, addr != end);
return 0;
}
static int vmap_pud_range(p4d_t *p4d, unsigned long addr,
- unsigned long end, pgprot_t prot, struct page **pages, int *nr)
+ unsigned long end, gfp_t gfp, pgprot_t prot,
+ struct page **pages, int *nr)
{
pud_t *pud;
unsigned long next;
- pud = pud_alloc(&init_mm, p4d, addr);
+ pud = pud_alloc(&init_mm, p4d, addr, gfp);
if (!pud)
return -ENOMEM;
do {
next = pud_addr_end(addr, end);
- if (vmap_pmd_range(pud, addr, next, prot, pages, nr))
+ if (vmap_pmd_range(pud, addr, next, gfp, prot, pages, nr))
return -ENOMEM;
} while (pud++, addr = next, addr != end);
return 0;
}
static int vmap_p4d_range(pgd_t *pgd, unsigned long addr,
- unsigned long end, pgprot_t prot, struct page **pages, int *nr)
+ unsigned long end, gfp_t gfp, pgprot_t prot,
+ struct page **pages, int *nr)
{
p4d_t *p4d;
unsigned long next;
- p4d = p4d_alloc(&init_mm, pgd, addr);
+ p4d = p4d_alloc(&init_mm, pgd, addr, gfp);
if (!p4d)
return -ENOMEM;
do {
next = p4d_addr_end(addr, end);
- if (vmap_pud_range(p4d, addr, next, prot, pages, nr))
+ if (vmap_pud_range(p4d, addr, next, gfp, prot, pages, nr))
return -ENOMEM;
} while (p4d++, addr = next, addr != end);
return 0;
@@ -215,7 +219,8 @@ static int vmap_p4d_range(pgd_t *pgd, unsigned long addr,
* Ie. pte at addr+N*PAGE_SIZE shall point to pfn corresponding to pages[N]
*/
static int vmap_page_range_noflush(unsigned long start, unsigned long end,
- pgprot_t prot, struct page **pages)
+ gfp_t gfp, pgprot_t prot,
+ struct page **pages)
{
pgd_t *pgd;
unsigned long next;
@@ -227,7 +232,7 @@ static int vmap_page_range_noflush(unsigned long start, unsigned long end,
pgd = pgd_offset_k(addr);
do {
next = pgd_addr_end(addr, end);
- err = vmap_p4d_range(pgd, addr, next, prot, pages, &nr);
+ err = vmap_p4d_range(pgd, addr, next, gfp, prot, pages, &nr);
if (err)
return err;
} while (pgd++, addr = next, addr != end);
@@ -236,11 +241,11 @@ static int vmap_page_range_noflush(unsigned long start, unsigned long end,
}
static int vmap_page_range(unsigned long start, unsigned long end,
- pgprot_t prot, struct page **pages)
+ gfp_t gfp, pgprot_t prot, struct page **pages)
{
int ret;
- ret = vmap_page_range_noflush(start, end, prot, pages);
+ ret = vmap_page_range_noflush(start, end, gfp, prot, pages);
flush_cache_vmap(start, end);
return ret;
}
@@ -1178,7 +1183,7 @@ void *vm_map_ram(struct page **pages, unsigned int count, int node, pgprot_t pro
addr = va->va_start;
mem = (void *)addr;
}
- if (vmap_page_range(addr, addr + size, prot, pages) < 0) {
+ if (vmap_page_range(addr, addr + size, GFP_KERNEL, prot, pages) < 0) {
vm_unmap_ram(mem, count);
return NULL;
}
@@ -1293,7 +1298,8 @@ void __init vmalloc_init(void)
int map_kernel_range_noflush(unsigned long addr, unsigned long size,
pgprot_t prot, struct page **pages)
{
- return vmap_page_range_noflush(addr, addr + size, prot, pages);
+ return vmap_page_range_noflush(addr, addr + size, GFP_KERNEL, prot,
+ pages);
}
/**
@@ -1334,13 +1340,14 @@ void unmap_kernel_range(unsigned long addr, unsigned long size)
}
EXPORT_SYMBOL_GPL(unmap_kernel_range);
-int map_vm_area(struct vm_struct *area, pgprot_t prot, struct page **pages)
+int map_vm_area(struct vm_struct *area, gfp_t gfp,
+ pgprot_t prot, struct page **pages)
{
unsigned long addr = (unsigned long)area->addr;
unsigned long end = addr + get_vm_area_size(area);
int err;
- err = vmap_page_range(addr, end, prot, pages);
+ err = vmap_page_range(addr, end, gfp, prot, pages);
return err > 0 ? 0 : err;
}
@@ -1642,7 +1649,7 @@ void *vmap(struct page **pages, unsigned int count,
if (!area)
return NULL;
- if (map_vm_area(area, prot, pages)) {
+ if (map_vm_area(area, GFP_KERNEL, prot, pages)) {
vunmap(area->addr);
return NULL;
}
@@ -1701,7 +1708,7 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
cond_resched();
}
- if (map_vm_area(area, prot, pages))
+ if (map_vm_area(area, gfp_mask, prot, pages))
goto fail;
return area->addr;
diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c
index 0787d33b80d8..d369e5bf2711 100644
--- a/mm/zsmalloc.c
+++ b/mm/zsmalloc.c
@@ -1151,7 +1151,7 @@ static inline void __zs_cpu_down(struct mapping_area *area)
static inline void *__zs_map_object(struct mapping_area *area,
struct page *pages[2], int off, int size)
{
- BUG_ON(map_vm_area(area->vm, PAGE_KERNEL, pages));
+ BUG_ON(map_vm_area(area->vm, GFP_KERNEL, PAGE_KERNEL, pages));
area->vm_addr = area->vm->addr;
return area->vm_addr + off;
}
diff --git a/virt/kvm/arm/mmu.c b/virt/kvm/arm/mmu.c
index 30251e288629..470d027cf8a4 100644
--- a/virt/kvm/arm/mmu.c
+++ b/virt/kvm/arm/mmu.c
@@ -647,7 +647,7 @@ static int create_hyp_pmd_mappings(pud_t *pud, unsigned long start,
BUG_ON(pmd_sect(*pmd));
if (pmd_none(*pmd)) {
- pte = pte_alloc_one_kernel(NULL);
+ pte = pte_alloc_one_kernel(NULL, GFP_KERNEL);
if (!pte) {
kvm_err("Cannot allocate Hyp pte\n");
return -ENOMEM;
@@ -679,7 +679,7 @@ static int create_hyp_pud_mappings(pgd_t *pgd, unsigned long start,
pud = pud_offset(pgd, addr);
if (pud_none_or_clear_bad(pud)) {
- pmd = pmd_alloc_one(NULL, addr);
+ pmd = pmd_alloc_one(NULL, addr, GFP_KERNEL);
if (!pmd) {
kvm_err("Cannot allocate Hyp pmd\n");
return -ENOMEM;
@@ -714,7 +714,7 @@ static int __create_hyp_mappings(pgd_t *pgdp, unsigned long ptrs_per_pgd,
pgd = pgdp + kvm_pgd_index(addr, ptrs_per_pgd);
if (pgd_none(*pgd)) {
- pud = pud_alloc_one(NULL, addr);
+ pud = pud_alloc_one(NULL, addr, GFP_KERNEL);
if (!pud) {
kvm_err("Cannot allocate Hyp pud\n");
err = -ENOMEM;