summaryrefslogtreecommitdiff
path: root/mm/hugetlb.c
diff options
context:
space:
mode:
Diffstat (limited to 'mm/hugetlb.c')
-rw-r--r--mm/hugetlb.c91
1 files changed, 37 insertions, 54 deletions
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 59d91c36770c..311c5d601310 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -4846,32 +4846,6 @@ static void set_huge_ptep_maybe_writable(struct vm_area_struct *vma,
set_huge_ptep_writable(vma, address, ptep);
}
-bool is_hugetlb_entry_migration(pte_t pte)
-{
- swp_entry_t swp;
-
- if (huge_pte_none(pte) || pte_present(pte))
- return false;
- swp = pte_to_swp_entry(pte);
- if (is_migration_entry(swp))
- return true;
- else
- return false;
-}
-
-bool is_hugetlb_entry_hwpoisoned(pte_t pte)
-{
- swp_entry_t swp;
-
- if (huge_pte_none(pte) || pte_present(pte))
- return false;
- swp = pte_to_swp_entry(pte);
- if (is_hwpoison_entry(swp))
- return true;
- else
- return false;
-}
-
static void
hugetlb_install_folio(struct vm_area_struct *vma, pte_t *ptep, unsigned long addr,
struct folio *new_folio, pte_t old, unsigned long sz)
@@ -4900,6 +4874,7 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
unsigned long npages = pages_per_huge_page(h);
struct mmu_notifier_range range;
unsigned long last_addr_mask;
+ softleaf_t softleaf;
int ret = 0;
if (cow) {
@@ -4947,16 +4922,16 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
entry = huge_ptep_get(src_vma->vm_mm, addr, src_pte);
again:
if (huge_pte_none(entry)) {
- /*
- * Skip if src entry none.
- */
- ;
- } else if (unlikely(is_hugetlb_entry_hwpoisoned(entry))) {
+ /* Skip if src entry none. */
+ goto next;
+ }
+
+ softleaf = softleaf_from_pte(entry);
+ if (unlikely(softleaf_is_hwpoison(softleaf))) {
if (!userfaultfd_wp(dst_vma))
entry = huge_pte_clear_uffd_wp(entry);
set_huge_pte_at(dst, addr, dst_pte, entry, sz);
- } else if (unlikely(is_hugetlb_entry_migration(entry))) {
- softleaf_t softleaf = softleaf_from_pte(entry);
+ } else if (unlikely(softleaf_is_migration(softleaf))) {
bool uffd_wp = pte_swp_uffd_wp(entry);
if (!is_readable_migration_entry(softleaf) && cow) {
@@ -4975,7 +4950,6 @@ again:
entry = huge_pte_clear_uffd_wp(entry);
set_huge_pte_at(dst, addr, dst_pte, entry, sz);
} else if (unlikely(pte_is_marker(entry))) {
- const softleaf_t softleaf = softleaf_from_pte(entry);
const pte_marker marker = copy_pte_marker(softleaf, dst_vma);
if (marker)
@@ -5033,9 +5007,7 @@ again:
}
hugetlb_install_folio(dst_vma, dst_pte, addr,
new_folio, src_pte_old, sz);
- spin_unlock(src_ptl);
- spin_unlock(dst_ptl);
- continue;
+ goto next;
}
if (cow) {
@@ -5056,6 +5028,8 @@ again:
set_huge_pte_at(dst, addr, dst_pte, entry, sz);
hugetlb_count_add(npages, dst);
}
+
+next:
spin_unlock(src_ptl);
spin_unlock(dst_ptl);
}
@@ -6064,8 +6038,10 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
ret = 0;
/* Not present, either a migration or a hwpoisoned entry */
- if (!pte_present(vmf.orig_pte)) {
- if (is_hugetlb_entry_migration(vmf.orig_pte)) {
+ if (!pte_present(vmf.orig_pte) && !huge_pte_none(vmf.orig_pte)) {
+ const softleaf_t softleaf = softleaf_from_pte(vmf.orig_pte);
+
+ if (softleaf_is_migration(softleaf)) {
/*
* Release the hugetlb fault lock now, but retain
* the vma lock, because it is needed to guard the
@@ -6076,9 +6052,12 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
mutex_unlock(&hugetlb_fault_mutex_table[hash]);
migration_entry_wait_huge(vma, vmf.address, vmf.pte);
return 0;
- } else if (is_hugetlb_entry_hwpoisoned(vmf.orig_pte))
+ }
+ if (softleaf_is_hwpoison(softleaf)) {
ret = VM_FAULT_HWPOISON_LARGE |
VM_FAULT_SET_HINDEX(hstate_index(h));
+ }
+
goto out_mutex;
}
@@ -6460,7 +6439,9 @@ long hugetlb_change_protection(struct vm_area_struct *vma,
i_mmap_lock_write(vma->vm_file->f_mapping);
last_addr_mask = hugetlb_mask_last_page(h);
for (; address < end; address += psize) {
+ softleaf_t entry;
spinlock_t *ptl;
+
ptep = hugetlb_walk(vma, address, psize);
if (!ptep) {
if (!uffd_wp) {
@@ -6492,15 +6473,23 @@ long hugetlb_change_protection(struct vm_area_struct *vma,
continue;
}
pte = huge_ptep_get(mm, address, ptep);
- if (unlikely(is_hugetlb_entry_hwpoisoned(pte))) {
- /* Nothing to do. */
- } else if (unlikely(is_hugetlb_entry_migration(pte))) {
- softleaf_t entry = softleaf_from_pte(pte);
+ if (huge_pte_none(pte)) {
+ if (unlikely(uffd_wp))
+ /* Safe to modify directly (none->non-present). */
+ set_huge_pte_at(mm, address, ptep,
+ make_pte_marker(PTE_MARKER_UFFD_WP),
+ psize);
+ goto next;
+ }
+ entry = softleaf_from_pte(pte);
+ if (unlikely(softleaf_is_hwpoison(entry))) {
+ /* Nothing to do. */
+ } else if (unlikely(softleaf_is_migration(entry))) {
struct folio *folio = softleaf_to_folio(entry);
pte_t newpte = pte;
- if (is_writable_migration_entry(entry)) {
+ if (softleaf_is_migration_write(entry)) {
if (folio_test_anon(folio))
entry = make_readable_exclusive_migration_entry(
swp_offset(entry));
@@ -6527,7 +6516,7 @@ long hugetlb_change_protection(struct vm_area_struct *vma,
if (pte_is_uffd_wp_marker(pte) && uffd_wp_resolve)
/* Safe to modify directly (non-present->none). */
huge_pte_clear(mm, address, ptep, psize);
- } else if (!huge_pte_none(pte)) {
+ } else {
pte_t old_pte;
unsigned int shift = huge_page_shift(hstate_vma(vma));
@@ -6540,16 +6529,10 @@ long hugetlb_change_protection(struct vm_area_struct *vma,
pte = huge_pte_clear_uffd_wp(pte);
huge_ptep_modify_prot_commit(vma, address, ptep, old_pte, pte);
pages++;
- } else {
- /* None pte */
- if (unlikely(uffd_wp))
- /* Safe to modify directly (none->non-present). */
- set_huge_pte_at(mm, address, ptep,
- make_pte_marker(PTE_MARKER_UFFD_WP),
- psize);
}
- spin_unlock(ptl);
+next:
+ spin_unlock(ptl);
cond_resched();
}
/*
generated by cgit v1.2.3 (git 2.46.0) at 2025-12-08 23:42:55 +0000