diff options
| author | Kent Overstreet <kent.overstreet@linux.dev> | 2022-11-04 18:23:15 -0400 |
|---|---|---|
| committer | Suren Baghdasaryan <surenb@google.com> | 2023-03-13 15:11:35 +0000 |
| commit | e4459fa41e09cc1be119b08a94c7fc369b6d0dff (patch) | |
| tree | 9f0088925c41c750145b037bcb7dbcd7398b7ae5 | |
| parent | 53e51876a2aa25b1150519436bc5f8b5e768413f (diff) | |
mm: percpu: Add codetag reference into pcpuobj_ext
To store codetag for every per-cpu allocation, a codetag reference is
embedded into pcpuobj_ext when CONFIG_MEM_ALLOC_PROFILING=y. Hooks to
use the newly introduced codetag are added.
Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>
Signed-off-by: Suren Baghdasaryan <surenb@google.com>
| -rw-r--r-- | mm/percpu-internal.h | 11 | ||||
| -rw-r--r-- | mm/percpu.c | 25 |
2 files changed, 34 insertions, 2 deletions
diff --git a/mm/percpu-internal.h b/mm/percpu-internal.h index b15f3f24ec71..622ec0ee2460 100644 --- a/mm/percpu-internal.h +++ b/mm/percpu-internal.h @@ -36,9 +36,12 @@ struct pcpuobj_ext { #ifdef CONFIG_MEMCG_KMEM struct obj_cgroup *cgroup; #endif +#ifdef CONFIG_MEM_ALLOC_PROFILING + union codetag_ref tag; +#endif }; -#ifdef CONFIG_MEMCG_KMEM +#if defined(CONFIG_MEMCG_KMEM) || defined(CONFIG_MEM_ALLOC_PROFILING) #define NEED_PCPUOBJ_EXT #endif @@ -79,7 +82,11 @@ struct pcpu_chunk { static inline bool need_pcpuobj_ext(void) { - return !mem_cgroup_kmem_disabled(); + if (IS_ENABLED(CONFIG_MEM_ALLOC_PROFILING)) + return true; + if (!mem_cgroup_kmem_disabled()) + return true; + return false; } extern spinlock_t pcpu_lock; diff --git a/mm/percpu.c b/mm/percpu.c index 578202671656..87a92988ad88 100644 --- a/mm/percpu.c +++ b/mm/percpu.c @@ -1701,6 +1701,31 @@ static void pcpu_memcg_free_hook(struct pcpu_chunk *chunk, int off, size_t size) } #endif /* CONFIG_MEMCG_KMEM */ +#ifdef CONFIG_MEM_ALLOC_PROFILING +static void pcpu_alloc_tag_alloc_hook(struct pcpu_chunk *chunk, int off, + size_t size) +{ + if (likely(chunk->obj_exts)) + alloc_tag_add(&chunk->obj_exts[off >> PCPU_MIN_ALLOC_SHIFT].tag, + current->alloc_tag, size); +} + +static void pcpu_alloc_tag_free_hook(struct pcpu_chunk *chunk, int off, size_t size) +{ + if (likely(chunk->obj_exts)) + alloc_tag_sub_noalloc(&chunk->obj_exts[off >> PCPU_MIN_ALLOC_SHIFT].tag, size); +} +#else +static void pcpu_alloc_tag_alloc_hook(struct pcpu_chunk *chunk, int off, + size_t size) +{ +} + +static void pcpu_alloc_tag_free_hook(struct pcpu_chunk *chunk, int off, size_t size) +{ +} +#endif + /** * pcpu_alloc - the percpu allocator * @size: size of area to allocate in bytes |