From 6faa68337b0c90923a1405ae9c196cee64921b7e Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Wed, 9 May 2012 10:09:51 -0500 Subject: slub: Use freelist instead of "object" in __slab_alloc The variable "object" really refers to a list of objects that we are handling. Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg diff --git a/mm/slub.c b/mm/slub.c index 80848cd..83f2582 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -2127,7 +2127,7 @@ slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid) static inline void *new_slab_objects(struct kmem_cache *s, gfp_t flags, int node, struct kmem_cache_cpu **pc) { - void *object; + void *freelist; struct kmem_cache_cpu *c; struct page *page = new_slab(s, flags, node); @@ -2140,7 +2140,7 @@ static inline void *new_slab_objects(struct kmem_cache *s, gfp_t flags, * No other reference to the page yet so we can * muck around with it freely without cmpxchg */ - object = page->freelist; + freelist = page->freelist; page->freelist = NULL; stat(s, ALLOC_SLAB); @@ -2148,9 +2148,9 @@ static inline void *new_slab_objects(struct kmem_cache *s, gfp_t flags, c->page = page; *pc = c; } else - object = NULL; + freelist = NULL; - return object; + return freelist; } /* @@ -2170,6 +2170,7 @@ static inline void *get_freelist(struct kmem_cache *s, struct page *page) do { freelist = page->freelist; counters = page->counters; + new.counters = counters; VM_BUG_ON(!new.frozen); @@ -2203,7 +2204,7 @@ static inline void *get_freelist(struct kmem_cache *s, struct page *page) static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, unsigned long addr, struct kmem_cache_cpu *c) { - void **object; + void *freelist; unsigned long flags; local_irq_save(flags); @@ -2219,6 +2220,7 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, if (!c->page) goto new_slab; redo: + if (unlikely(!node_match(c, node))) { stat(s, ALLOC_NODE_MISMATCH); deactivate_slab(s, c); @@ -2226,15 +2228,15 @@ redo: } /* must check again c->freelist in case of cpu migration or IRQ */ - object = c->freelist; - if (object) + freelist = c->freelist; + if (freelist) goto load_freelist; stat(s, ALLOC_SLOWPATH); - object = get_freelist(s, c->page); + freelist = get_freelist(s, c->page); - if (!object) { + if (!freelist) { c->page = NULL; stat(s, DEACTIVATE_BYPASS); goto new_slab; @@ -2243,10 +2245,10 @@ redo: stat(s, ALLOC_REFILL); load_freelist: - c->freelist = get_freepointer(s, object); + c->freelist = get_freepointer(s, freelist); c->tid = next_tid(c->tid); local_irq_restore(flags); - return object; + return freelist; new_slab: @@ -2260,13 +2262,13 @@ new_slab: } /* Then do expensive stuff like retrieving pages from the partial lists */ - object = get_partial(s, gfpflags, node, c); + freelist = get_partial(s, gfpflags, node, c); - if (unlikely(!object)) { + if (unlikely(!freelist)) { - object = new_slab_objects(s, gfpflags, node, &c); + freelist = new_slab_objects(s, gfpflags, node, &c); - if (unlikely(!object)) { + if (unlikely(!freelist)) { if (!(gfpflags & __GFP_NOWARN) && printk_ratelimit()) slab_out_of_memory(s, gfpflags, node); @@ -2279,14 +2281,14 @@ new_slab: goto load_freelist; /* Only entered in the debug case */ - if (!alloc_debug_processing(s, c->page, object, addr)) + if (!alloc_debug_processing(s, c->page, freelist, addr)) goto new_slab; /* Slab failed checks. Next slab needed */ - c->freelist = get_freepointer(s, object); + c->freelist = get_freepointer(s, freelist); deactivate_slab(s, c); c->node = NUMA_NO_NODE; local_irq_restore(flags); - return object; + return freelist; } /* -- cgit v0.10.2 From 507effeaba29bf724dfe38317fbd11d0fe25fa40 Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Wed, 9 May 2012 10:09:52 -0500 Subject: slub: Add frozen check in __slab_alloc Verify that objects returned from __slab_alloc come from slab pages in the correct state. Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg diff --git a/mm/slub.c b/mm/slub.c index 83f2582..a3395c2 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -2245,6 +2245,12 @@ redo: stat(s, ALLOC_REFILL); load_freelist: + /* + * freelist is pointing to the list of objects to be used. + * page is pointing to the page from which the objects are obtained. + * That page must be frozen for per cpu allocations to work. + */ + VM_BUG_ON(!c->page->frozen); c->freelist = get_freepointer(s, freelist); c->tid = next_tid(c->tid); local_irq_restore(flags); -- cgit v0.10.2 From 7ced3719719669ad6bd279b45fa3c1a517b2e057 Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Wed, 9 May 2012 10:09:53 -0500 Subject: slub: Acquire_slab() avoid loop Avoid the loop in acquire slab and simply fail if there is a conflict. This will cause the next page on the list to be considered. Acked-by: David Rientjes Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg diff --git a/mm/slub.c b/mm/slub.c index a3395c2..9892775 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -1490,12 +1490,12 @@ static inline void remove_partial(struct kmem_cache_node *n, } /* - * Lock slab, remove from the partial list and put the object into the - * per cpu freelist. + * Remove slab from the partial list, freeze it and + * return the pointer to the freelist. * * Returns a list of objects or NULL if it fails. * - * Must hold list_lock. + * Must hold list_lock since we modify the partial list. */ static inline void *acquire_slab(struct kmem_cache *s, struct kmem_cache_node *n, struct page *page, @@ -1510,22 +1510,24 @@ static inline void *acquire_slab(struct kmem_cache *s, * The old freelist is the list of objects for the * per cpu allocation list. */ - do { - freelist = page->freelist; - counters = page->counters; - new.counters = counters; - if (mode) - new.inuse = page->objects; + freelist = page->freelist; + counters = page->counters; + new.counters = counters; + if (mode) + new.inuse = page->objects; - VM_BUG_ON(new.frozen); - new.frozen = 1; + VM_BUG_ON(new.frozen); + new.frozen = 1; - } while (!__cmpxchg_double_slab(s, page, + if (!__cmpxchg_double_slab(s, page, freelist, counters, NULL, new.counters, - "lock and freeze")); + "acquire_slab")) + + return NULL; remove_partial(n, page); + WARN_ON(!freelist); return freelist; } -- cgit v0.10.2 From f469743673ceda5181970eb6b8090ba728c956fb Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Wed, 9 May 2012 10:09:54 -0500 Subject: slub: Simplify control flow in __slab_alloc() Simplify control flow a bit avoiding nesting. Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg diff --git a/mm/slub.c b/mm/slub.c index 9892775..5aacd43 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -2272,17 +2272,15 @@ new_slab: /* Then do expensive stuff like retrieving pages from the partial lists */ freelist = get_partial(s, gfpflags, node, c); - if (unlikely(!freelist)) { - + if (!freelist) freelist = new_slab_objects(s, gfpflags, node, &c); - if (unlikely(!freelist)) { - if (!(gfpflags & __GFP_NOWARN) && printk_ratelimit()) - slab_out_of_memory(s, gfpflags, node); + if (unlikely(!freelist)) { + if (!(gfpflags & __GFP_NOWARN) && printk_ratelimit()) + slab_out_of_memory(s, gfpflags, node); - local_irq_restore(flags); - return NULL; - } + local_irq_restore(flags); + return NULL; } if (likely(!kmem_cache_debug(s))) -- cgit v0.10.2 From 188fd063208942a4681d8e8a4484ad0d4ae0fda1 Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Wed, 9 May 2012 10:09:55 -0500 Subject: slub: new_slab_objects() can also get objects from partial list Moving the attempt to get a slab page from the partial lists simplifies __slab_alloc which is rather complicated. Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg diff --git a/mm/slub.c b/mm/slub.c index 5aacd43..b29246b 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -2130,9 +2130,15 @@ static inline void *new_slab_objects(struct kmem_cache *s, gfp_t flags, int node, struct kmem_cache_cpu **pc) { void *freelist; - struct kmem_cache_cpu *c; - struct page *page = new_slab(s, flags, node); + struct kmem_cache_cpu *c = *pc; + struct page *page; + + freelist = get_partial(s, flags, node, c); + if (freelist) + return freelist; + + page = new_slab(s, flags, node); if (page) { c = __this_cpu_ptr(s->cpu_slab); if (c->page) @@ -2269,11 +2275,7 @@ new_slab: goto redo; } - /* Then do expensive stuff like retrieving pages from the partial lists */ - freelist = get_partial(s, gfpflags, node, c); - - if (!freelist) - freelist = new_slab_objects(s, gfpflags, node, &c); + freelist = new_slab_objects(s, gfpflags, node, &c); if (unlikely(!freelist)) { if (!(gfpflags & __GFP_NOWARN) && printk_ratelimit()) -- cgit v0.10.2 From ec3ab083a7a004282ee374bdaeb0aa603521b8eb Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Wed, 9 May 2012 10:09:56 -0500 Subject: slub: Get rid of the node field The node field is always page_to_nid(c->page). So its rather easy to replace. Note that there maybe slightly more overhead in various hot paths due to the need to shift the bits from page->flags. However, that is mostly compensated for by a smaller footprint of the kmem_cache_cpu structure (this patch reduces that to 3 words per cache) which allows better caching. Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h index c2f8c8b..ebdcf4b 100644 --- a/include/linux/slub_def.h +++ b/include/linux/slub_def.h @@ -48,7 +48,6 @@ struct kmem_cache_cpu { unsigned long tid; /* Globally unique transaction id */ struct page *page; /* The slab from which we are allocating */ struct page *partial; /* Partially allocated frozen slabs */ - int node; /* The node of the page (or -1 for debug) */ #ifdef CONFIG_SLUB_STATS unsigned stat[NR_SLUB_STAT_ITEMS]; #endif diff --git a/mm/slub.c b/mm/slub.c index b29246b..aed8792 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -1561,7 +1561,6 @@ static void *get_partial_node(struct kmem_cache *s, if (!object) { c->page = page; - c->node = page_to_nid(page); stat(s, ALLOC_FROM_PARTIAL); object = t; available = page->objects - page->inuse; @@ -2057,7 +2056,7 @@ static void flush_all(struct kmem_cache *s) static inline int node_match(struct kmem_cache_cpu *c, int node) { #ifdef CONFIG_NUMA - if (node != NUMA_NO_NODE && c->node != node) + if (node != NUMA_NO_NODE && page_to_nid(c->page) != node) return 0; #endif return 1; @@ -2152,7 +2151,6 @@ static inline void *new_slab_objects(struct kmem_cache *s, gfp_t flags, page->freelist = NULL; stat(s, ALLOC_SLAB); - c->node = page_to_nid(page); c->page = page; *pc = c; } else @@ -2269,7 +2267,6 @@ new_slab: if (c->partial) { c->page = c->partial; c->partial = c->page->next; - c->node = page_to_nid(c->page); stat(s, CPU_PARTIAL_ALLOC); c->freelist = NULL; goto redo; @@ -2294,7 +2291,6 @@ new_slab: c->freelist = get_freepointer(s, freelist); deactivate_slab(s, c); - c->node = NUMA_NO_NODE; local_irq_restore(flags); return freelist; } @@ -4507,30 +4503,31 @@ static ssize_t show_slab_objects(struct kmem_cache *s, for_each_possible_cpu(cpu) { struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu); - int node = ACCESS_ONCE(c->node); + int node; struct page *page; - if (node < 0) - continue; page = ACCESS_ONCE(c->page); - if (page) { - if (flags & SO_TOTAL) - x = page->objects; - else if (flags & SO_OBJECTS) - x = page->inuse; - else - x = 1; + if (!page) + continue; - total += x; - nodes[node] += x; - } - page = c->partial; + node = page_to_nid(page); + if (flags & SO_TOTAL) + x = page->objects; + else if (flags & SO_OBJECTS) + x = page->inuse; + else + x = 1; + + total += x; + nodes[node] += x; + page = ACCESS_ONCE(c->partial); if (page) { x = page->pobjects; total += x; nodes[node] += x; } + per_cpu[node]++; } } -- cgit v0.10.2 From c17dda40a6a4ed95f035db38b7ba4fab0d99da44 Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Wed, 9 May 2012 10:09:57 -0500 Subject: slub: Separate out kmem_cache_cpu processing from deactivate_slab Processing on fields of kmem_cache_cpu is cleaner if code working on fields of this struct is taken out of deactivate_slab(). Acked-by: David Rientjes Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg diff --git a/mm/slub.c b/mm/slub.c index aed8792..2389a01 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -1729,14 +1729,12 @@ void init_kmem_cache_cpus(struct kmem_cache *s) /* * Remove the cpu slab */ -static void deactivate_slab(struct kmem_cache *s, struct kmem_cache_cpu *c) +static void deactivate_slab(struct kmem_cache *s, struct page *page, void *freelist) { enum slab_modes { M_NONE, M_PARTIAL, M_FULL, M_FREE }; - struct page *page = c->page; struct kmem_cache_node *n = get_node(s, page_to_nid(page)); int lock = 0; enum slab_modes l = M_NONE, m = M_NONE; - void *freelist; void *nextfree; int tail = DEACTIVATE_TO_HEAD; struct page new; @@ -1747,11 +1745,6 @@ static void deactivate_slab(struct kmem_cache *s, struct kmem_cache_cpu *c) tail = DEACTIVATE_TO_TAIL; } - c->tid = next_tid(c->tid); - c->page = NULL; - freelist = c->freelist; - c->freelist = NULL; - /* * Stage one: Free all available per cpu objects back * to the page freelist while it is still frozen. Leave the @@ -2009,7 +2002,11 @@ int put_cpu_partial(struct kmem_cache *s, struct page *page, int drain) static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c) { stat(s, CPUSLAB_FLUSH); - deactivate_slab(s, c); + deactivate_slab(s, c->page, c->freelist); + + c->tid = next_tid(c->tid); + c->page = NULL; + c->freelist = NULL; } /* @@ -2229,7 +2226,9 @@ redo: if (unlikely(!node_match(c, node))) { stat(s, ALLOC_NODE_MISMATCH); - deactivate_slab(s, c); + deactivate_slab(s, c->page, c->freelist); + c->page = NULL; + c->freelist = NULL; goto new_slab; } @@ -2289,8 +2288,9 @@ new_slab: if (!alloc_debug_processing(s, c->page, freelist, addr)) goto new_slab; /* Slab failed checks. Next slab needed */ - c->freelist = get_freepointer(s, freelist); - deactivate_slab(s, c); + deactivate_slab(s, c->page, get_freepointer(s, freelist)); + c->page = NULL; + c->freelist = NULL; local_irq_restore(flags); return freelist; } -- cgit v0.10.2 From f6e7def7f7d749759e4bf36dcc25ae289a20d868 Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Wed, 9 May 2012 10:09:58 -0500 Subject: slub: Use page variable instead of c->page. Store the value of c->page to avoid additional fetches from per cpu data. Acked-by: David Rientjes Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg diff --git a/mm/slub.c b/mm/slub.c index 2389a01..6b60fc9 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -2208,6 +2208,7 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, unsigned long addr, struct kmem_cache_cpu *c) { void *freelist; + struct page *page; unsigned long flags; local_irq_save(flags); @@ -2220,13 +2221,14 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, c = this_cpu_ptr(s->cpu_slab); #endif - if (!c->page) + page = c->page; + if (!page) goto new_slab; redo: if (unlikely(!node_match(c, node))) { stat(s, ALLOC_NODE_MISMATCH); - deactivate_slab(s, c->page, c->freelist); + deactivate_slab(s, page, c->freelist); c->page = NULL; c->freelist = NULL; goto new_slab; @@ -2239,7 +2241,7 @@ redo: stat(s, ALLOC_SLOWPATH); - freelist = get_freelist(s, c->page); + freelist = get_freelist(s, page); if (!freelist) { c->page = NULL; @@ -2264,8 +2266,8 @@ load_freelist: new_slab: if (c->partial) { - c->page = c->partial; - c->partial = c->page->next; + page = c->page = c->partial; + c->partial = page->next; stat(s, CPU_PARTIAL_ALLOC); c->freelist = NULL; goto redo; @@ -2281,14 +2283,15 @@ new_slab: return NULL; } + page = c->page; if (likely(!kmem_cache_debug(s))) goto load_freelist; /* Only entered in the debug case */ - if (!alloc_debug_processing(s, c->page, freelist, addr)) + if (!alloc_debug_processing(s, page, freelist, addr)) goto new_slab; /* Slab failed checks. Next slab needed */ - deactivate_slab(s, c->page, get_freepointer(s, freelist)); + deactivate_slab(s, page, get_freepointer(s, freelist)); c->page = NULL; c->freelist = NULL; local_irq_restore(flags); -- cgit v0.10.2 From 57d437d2aa680f42d75cef45205834d5f605550a Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Wed, 9 May 2012 10:09:59 -0500 Subject: slub: pass page to node_match() instead of kmem_cache_cpu structure Avoid passing the kmem_cache_cpu pointer to node_match. This makes the node_match function more generic and easier to understand. Acked-by: David Rientjes Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg diff --git a/mm/slub.c b/mm/slub.c index 6b60fc9..719509e 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -2050,10 +2050,10 @@ static void flush_all(struct kmem_cache *s) * Check if the objects in a per cpu structure fit numa * locality expectations. */ -static inline int node_match(struct kmem_cache_cpu *c, int node) +static inline int node_match(struct page *page, int node) { #ifdef CONFIG_NUMA - if (node != NUMA_NO_NODE && page_to_nid(c->page) != node) + if (node != NUMA_NO_NODE && page_to_nid(page) != node) return 0; #endif return 1; @@ -2226,7 +2226,7 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, goto new_slab; redo: - if (unlikely(!node_match(c, node))) { + if (unlikely(!node_match(page, node))) { stat(s, ALLOC_NODE_MISMATCH); deactivate_slab(s, page, c->freelist); c->page = NULL; @@ -2313,6 +2313,7 @@ static __always_inline void *slab_alloc(struct kmem_cache *s, { void **object; struct kmem_cache_cpu *c; + struct page *page; unsigned long tid; if (slab_pre_alloc_hook(s, gfpflags)) @@ -2338,7 +2339,8 @@ redo: barrier(); object = c->freelist; - if (unlikely(!object || !node_match(c, node))) + page = c->page; + if (unlikely(!object || !node_match(page, node))) object = __slab_alloc(s, gfpflags, node, addr, c); -- cgit v0.10.2 From b8c24c4aef94b1f0daafb450363fef13a1163780 Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Wed, 13 Jun 2012 10:24:52 -0500 Subject: slob: Define page struct fields used in mm_types.h Define the fields used by slob in mm_types.h and use struct page instead of struct slob_page in slob. This cleans up numerous of typecasts in slob.c and makes readers aware of slob's use of page struct fields. [Also cleans up some bitrot in slob.c. The page struct field layout in slob.c is an old layout and does not match the one in mm_types.h] Reviewed-by: Glauber Costa Acked-by: David Rientjes Reviewed-by: Joonsoo Kim Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index dad95bd..5922c34 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -53,7 +53,7 @@ struct page { struct { union { pgoff_t index; /* Our offset within mapping. */ - void *freelist; /* slub first free object */ + void *freelist; /* slub/slob first free object */ }; union { @@ -81,11 +81,12 @@ struct page { */ atomic_t _mapcount; - struct { + struct { /* SLUB */ unsigned inuse:16; unsigned objects:15; unsigned frozen:1; }; + int units; /* SLOB */ }; atomic_t _count; /* Usage count, see below. */ }; @@ -107,6 +108,8 @@ struct page { short int pobjects; #endif }; + + struct list_head list; /* slobs list of pages */ }; /* Remainder is not double word aligned */ diff --git a/mm/slob.c b/mm/slob.c index 8105be4..30862a2 100644 --- a/mm/slob.c +++ b/mm/slob.c @@ -92,33 +92,12 @@ struct slob_block { typedef struct slob_block slob_t; /* - * We use struct page fields to manage some slob allocation aspects, - * however to avoid the horrible mess in include/linux/mm_types.h, we'll - * just define our own struct page type variant here. - */ -struct slob_page { - union { - struct { - unsigned long flags; /* mandatory */ - atomic_t _count; /* mandatory */ - slobidx_t units; /* free units left in page */ - unsigned long pad[2]; - slob_t *free; /* first free slob_t in page */ - struct list_head list; /* linked list of free pages */ - }; - struct page page; - }; -}; -static inline void struct_slob_page_wrong_size(void) -{ BUILD_BUG_ON(sizeof(struct slob_page) != sizeof(struct page)); } - -/* * free_slob_page: call before a slob_page is returned to the page allocator. */ -static inline void free_slob_page(struct slob_page *sp) +static inline void free_slob_page(struct page *sp) { - reset_page_mapcount(&sp->page); - sp->page.mapping = NULL; + reset_page_mapcount(sp); + sp->mapping = NULL; } /* @@ -133,44 +112,44 @@ static LIST_HEAD(free_slob_large); /* * is_slob_page: True for all slob pages (false for bigblock pages) */ -static inline int is_slob_page(struct slob_page *sp) +static inline int is_slob_page(struct page *sp) { - return PageSlab((struct page *)sp); + return PageSlab(sp); } -static inline void set_slob_page(struct slob_page *sp) +static inline void set_slob_page(struct page *sp) { - __SetPageSlab((struct page *)sp); + __SetPageSlab(sp); } -static inline void clear_slob_page(struct slob_page *sp) +static inline void clear_slob_page(struct page *sp) { - __ClearPageSlab((struct page *)sp); + __ClearPageSlab(sp); } -static inline struct slob_page *slob_page(const void *addr) +static inline struct page *slob_page(const void *addr) { - return (struct slob_page *)virt_to_page(addr); + return virt_to_page(addr); } /* * slob_page_free: true for pages on free_slob_pages list. */ -static inline int slob_page_free(struct slob_page *sp) +static inline int slob_page_free(struct page *sp) { - return PageSlobFree((struct page *)sp); + return PageSlobFree(sp); } -static void set_slob_page_free(struct slob_page *sp, struct list_head *list) +static void set_slob_page_free(struct page *sp, struct list_head *list) { list_add(&sp->list, list); - __SetPageSlobFree((struct page *)sp); + __SetPageSlobFree(sp); } -static inline void clear_slob_page_free(struct slob_page *sp) +static inline void clear_slob_page_free(struct page *sp) { list_del(&sp->list); - __ClearPageSlobFree((struct page *)sp); + __ClearPageSlobFree(sp); } #define SLOB_UNIT sizeof(slob_t) @@ -267,12 +246,12 @@ static void slob_free_pages(void *b, int order) /* * Allocate a slob block within a given slob_page sp. */ -static void *slob_page_alloc(struct slob_page *sp, size_t size, int align) +static void *slob_page_alloc(struct page *sp, size_t size, int align) { slob_t *prev, *cur, *aligned = NULL; int delta = 0, units = SLOB_UNITS(size); - for (prev = NULL, cur = sp->free; ; prev = cur, cur = slob_next(cur)) { + for (prev = NULL, cur = sp->freelist; ; prev = cur, cur = slob_next(cur)) { slobidx_t avail = slob_units(cur); if (align) { @@ -296,12 +275,12 @@ static void *slob_page_alloc(struct slob_page *sp, size_t size, int align) if (prev) set_slob(prev, slob_units(prev), next); else - sp->free = next; + sp->freelist = next; } else { /* fragment */ if (prev) set_slob(prev, slob_units(prev), cur + units); else - sp->free = cur + units; + sp->freelist = cur + units; set_slob(cur + units, avail - units, next); } @@ -320,7 +299,7 @@ static void *slob_page_alloc(struct slob_page *sp, size_t size, int align) */ static void *slob_alloc(size_t size, gfp_t gfp, int align, int node) { - struct slob_page *sp; + struct page *sp; struct list_head *prev; struct list_head *slob_list; slob_t *b = NULL; @@ -341,7 +320,7 @@ static void *slob_alloc(size_t size, gfp_t gfp, int align, int node) * If there's a node specification, search for a partial * page with a matching node id in the freelist. */ - if (node != -1 && page_to_nid(&sp->page) != node) + if (node != -1 && page_to_nid(sp) != node) continue; #endif /* Enough room on this page? */ @@ -374,7 +353,7 @@ static void *slob_alloc(size_t size, gfp_t gfp, int align, int node) spin_lock_irqsave(&slob_lock, flags); sp->units = SLOB_UNITS(PAGE_SIZE); - sp->free = b; + sp->freelist = b; INIT_LIST_HEAD(&sp->list); set_slob(b, SLOB_UNITS(PAGE_SIZE), b + SLOB_UNITS(PAGE_SIZE)); set_slob_page_free(sp, slob_list); @@ -392,7 +371,7 @@ static void *slob_alloc(size_t size, gfp_t gfp, int align, int node) */ static void slob_free(void *block, int size) { - struct slob_page *sp; + struct page *sp; slob_t *prev, *next, *b = (slob_t *)block; slobidx_t units; unsigned long flags; @@ -421,7 +400,7 @@ static void slob_free(void *block, int size) if (!slob_page_free(sp)) { /* This slob page is about to become partially free. Easy! */ sp->units = units; - sp->free = b; + sp->freelist = b; set_slob(b, units, (void *)((unsigned long)(b + SLOB_UNITS(PAGE_SIZE)) & PAGE_MASK)); @@ -441,15 +420,15 @@ static void slob_free(void *block, int size) */ sp->units += units; - if (b < sp->free) { - if (b + units == sp->free) { - units += slob_units(sp->free); - sp->free = slob_next(sp->free); + if (b < (slob_t *)sp->freelist) { + if (b + units == sp->freelist) { + units += slob_units(sp->freelist); + sp->freelist = slob_next(sp->freelist); } - set_slob(b, units, sp->free); - sp->free = b; + set_slob(b, units, sp->freelist); + sp->freelist = b; } else { - prev = sp->free; + prev = sp->freelist; next = slob_next(prev); while (b > next) { prev = next; @@ -522,7 +501,7 @@ EXPORT_SYMBOL(__kmalloc_node); void kfree(const void *block) { - struct slob_page *sp; + struct page *sp; trace_kfree(_RET_IP_, block); @@ -536,14 +515,14 @@ void kfree(const void *block) unsigned int *m = (unsigned int *)(block - align); slob_free(m, *m + align); } else - put_page(&sp->page); + put_page(sp); } EXPORT_SYMBOL(kfree); /* can't use ksize for kmem_cache_alloc memory, only kmalloc */ size_t ksize(const void *block) { - struct slob_page *sp; + struct page *sp; BUG_ON(!block); if (unlikely(block == ZERO_SIZE_PTR)) @@ -555,7 +534,7 @@ size_t ksize(const void *block) unsigned int *m = (unsigned int *)(block - align); return SLOB_UNITS(*m) * SLOB_UNIT; } else - return sp->page.private; + return sp->private; } EXPORT_SYMBOL(ksize); -- cgit v0.10.2 From 690d5777392180fdc05a82c0c7979e50e8d93de8 Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Wed, 13 Jun 2012 10:24:53 -0500 Subject: slob: No need to zero mapping since it is no longer in use Reviewed-by: Joonsoo Kim Acked-by: David Rientjes Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg diff --git a/mm/slob.c b/mm/slob.c index 30862a2..74c3bb2 100644 --- a/mm/slob.c +++ b/mm/slob.c @@ -97,7 +97,6 @@ typedef struct slob_block slob_t; static inline void free_slob_page(struct page *sp) { reset_page_mapcount(sp); - sp->mapping = NULL; } /* -- cgit v0.10.2 From b5568280c9b9162b384be9d447013b74d682d4b3 Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Wed, 13 Jun 2012 10:24:54 -0500 Subject: slob: Remove various small accessors Those have become so simple that they are no longer needed. Reviewed-by: Joonsoo Kim Acked-by: David Rientjes signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg diff --git a/mm/slob.c b/mm/slob.c index 74c3bb2..c85265d 100644 --- a/mm/slob.c +++ b/mm/slob.c @@ -92,14 +92,6 @@ struct slob_block { typedef struct slob_block slob_t; /* - * free_slob_page: call before a slob_page is returned to the page allocator. - */ -static inline void free_slob_page(struct page *sp) -{ - reset_page_mapcount(sp); -} - -/* * All partially free slob pages go on these lists. */ #define SLOB_BREAK1 256 @@ -109,29 +101,6 @@ static LIST_HEAD(free_slob_medium); static LIST_HEAD(free_slob_large); /* - * is_slob_page: True for all slob pages (false for bigblock pages) - */ -static inline int is_slob_page(struct page *sp) -{ - return PageSlab(sp); -} - -static inline void set_slob_page(struct page *sp) -{ - __SetPageSlab(sp); -} - -static inline void clear_slob_page(struct page *sp) -{ - __ClearPageSlab(sp); -} - -static inline struct page *slob_page(const void *addr) -{ - return virt_to_page(addr); -} - -/* * slob_page_free: true for pages on free_slob_pages list. */ static inline int slob_page_free(struct page *sp) @@ -347,8 +316,8 @@ static void *slob_alloc(size_t size, gfp_t gfp, int align, int node) b = slob_new_pages(gfp & ~__GFP_ZERO, 0, node); if (!b) return NULL; - sp = slob_page(b); - set_slob_page(sp); + sp = virt_to_page(b); + __SetPageSlab(sp); spin_lock_irqsave(&slob_lock, flags); sp->units = SLOB_UNITS(PAGE_SIZE); @@ -380,7 +349,7 @@ static void slob_free(void *block, int size) return; BUG_ON(!size); - sp = slob_page(block); + sp = virt_to_page(block); units = SLOB_UNITS(size); spin_lock_irqsave(&slob_lock, flags); @@ -390,8 +359,8 @@ static void slob_free(void *block, int size) if (slob_page_free(sp)) clear_slob_page_free(sp); spin_unlock_irqrestore(&slob_lock, flags); - clear_slob_page(sp); - free_slob_page(sp); + __ClearPageSlab(sp); + reset_page_mapcount(sp); slob_free_pages(b, 0); return; } @@ -508,8 +477,8 @@ void kfree(const void *block) return; kmemleak_free(block); - sp = slob_page(block); - if (is_slob_page(sp)) { + sp = virt_to_page(block); + if (PageSlab(sp)) { int align = max(ARCH_KMALLOC_MINALIGN, ARCH_SLAB_MINALIGN); unsigned int *m = (unsigned int *)(block - align); slob_free(m, *m + align); @@ -527,8 +496,8 @@ size_t ksize(const void *block) if (unlikely(block == ZERO_SIZE_PTR)) return 0; - sp = slob_page(block); - if (is_slob_page(sp)) { + sp = virt_to_page(block); + if (PageSlab(sp)) { int align = max(ARCH_KMALLOC_MINALIGN, ARCH_SLAB_MINALIGN); unsigned int *m = (unsigned int *)(block - align); return SLOB_UNITS(*m) * SLOB_UNIT; -- cgit v0.10.2 From e571b0ad3495be5793e54e21cd244c4545c49d88 Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Wed, 13 Jun 2012 10:24:55 -0500 Subject: slab: Use page struct fields instead of casting Add fields to the page struct so that it is properly documented that slab overlays the lru fields. This cleans up some casts in slab. Reviewed-by: Glauber Costa Reviewed-by: Joonsoo Kim Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 5922c34..680a5e4 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -110,6 +110,10 @@ struct page { }; struct list_head list; /* slobs list of pages */ + struct { /* slab fields */ + struct kmem_cache *slab_cache; + struct slab *slab_page; + }; }; /* Remainder is not double word aligned */ diff --git a/mm/slab.c b/mm/slab.c index e901a36..af05147 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -496,25 +496,25 @@ static bool slab_max_order_set __initdata; */ static inline void page_set_cache(struct page *page, struct kmem_cache *cache) { - page->lru.next = (struct list_head *)cache; + page->slab_cache = cache; } static inline struct kmem_cache *page_get_cache(struct page *page) { page = compound_head(page); BUG_ON(!PageSlab(page)); - return (struct kmem_cache *)page->lru.next; + return page->slab_cache; } static inline void page_set_slab(struct page *page, struct slab *slab) { - page->lru.prev = (struct list_head *)slab; + page->slab_page = slab; } static inline struct slab *page_get_slab(struct page *page) { BUG_ON(!PageSlab(page)); - return (struct slab *)page->lru.prev; + return page->slab_page; } static inline struct kmem_cache *virt_to_cache(const void *obj) -- cgit v0.10.2 From 350260889b251821e770573dfd65cd851b4ef781 Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Wed, 13 Jun 2012 10:24:56 -0500 Subject: slab: Remove some accessors Those are rather trivial now and its better to see inline what is really going on. Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg diff --git a/mm/slab.c b/mm/slab.c index af05147..28a8f7d 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -489,16 +489,6 @@ EXPORT_SYMBOL(slab_buffer_size); static int slab_max_order = SLAB_MAX_ORDER_LO; static bool slab_max_order_set __initdata; -/* - * Functions for storing/retrieving the cachep and or slab from the page - * allocator. These are used to find the slab an obj belongs to. With kfree(), - * these are used to find the cache which an obj belongs to. - */ -static inline void page_set_cache(struct page *page, struct kmem_cache *cache) -{ - page->slab_cache = cache; -} - static inline struct kmem_cache *page_get_cache(struct page *page) { page = compound_head(page); @@ -506,27 +496,18 @@ static inline struct kmem_cache *page_get_cache(struct page *page) return page->slab_cache; } -static inline void page_set_slab(struct page *page, struct slab *slab) -{ - page->slab_page = slab; -} - -static inline struct slab *page_get_slab(struct page *page) -{ - BUG_ON(!PageSlab(page)); - return page->slab_page; -} - static inline struct kmem_cache *virt_to_cache(const void *obj) { struct page *page = virt_to_head_page(obj); - return page_get_cache(page); + return page->slab_cache; } static inline struct slab *virt_to_slab(const void *obj) { struct page *page = virt_to_head_page(obj); - return page_get_slab(page); + + VM_BUG_ON(!PageSlab(page)); + return page->slab_page; } static inline void *index_to_obj(struct kmem_cache *cache, struct slab *slab, @@ -2918,8 +2899,8 @@ static void slab_map_pages(struct kmem_cache *cache, struct slab *slab, nr_pages <<= cache->gfporder; do { - page_set_cache(page, cache); - page_set_slab(page, slab); + page->slab_cache = cache; + page->slab_page = slab; page++; } while (--nr_pages); } @@ -3057,7 +3038,7 @@ static void *cache_free_debugcheck(struct kmem_cache *cachep, void *objp, kfree_debugcheck(objp); page = virt_to_head_page(objp); - slabp = page_get_slab(page); + slabp = page->slab_page; if (cachep->flags & SLAB_RED_ZONE) { verify_redzone_free(cachep, objp); @@ -3261,7 +3242,7 @@ static void *cache_alloc_debugcheck_after(struct kmem_cache *cachep, struct slab *slabp; unsigned objnr; - slabp = page_get_slab(virt_to_head_page(objp)); + slabp = virt_to_head_page(objp)->slab_page; objnr = (unsigned)(objp - slabp->s_mem) / cachep->buffer_size; slab_bufctl(slabp)[objnr] = BUFCTL_ACTIVE; } -- cgit v0.10.2 From 3b0efdfa1e719303536c04d9abca43abeb40f80a Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Wed, 13 Jun 2012 10:24:57 -0500 Subject: mm, sl[aou]b: Extract common fields from struct kmem_cache Define a struct that describes common fields used in all slab allocators. A slab allocator either uses the common definition (like SLOB) or is required to provide members of kmem_cache with the definition given. After that it will be possible to share code that only operates on those fields of kmem_cache. The patch basically takes the slob definition of kmem cache and uses the field namees for the other allocators. It also standardizes the names used for basic object lengths in allocators: object_size Struct size specified at kmem_cache_create. Basically the payload expected to be used by the subsystem. size The size of memory allocator for each object. This size is larger than object_size and includes padding, alignment and extra metadata for each object (f.e. for debugging and rcu). Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg diff --git a/include/linux/slab.h b/include/linux/slab.h index 67d5d94..0dd2dfa 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -93,6 +93,30 @@ (unsigned long)ZERO_SIZE_PTR) /* + * Common fields provided in kmem_cache by all slab allocators + * This struct is either used directly by the allocator (SLOB) + * or the allocator must include definitions for all fields + * provided in kmem_cache_common in their definition of kmem_cache. + * + * Once we can do anonymous structs (C11 standard) we could put a + * anonymous struct definition in these allocators so that the + * separate allocations in the kmem_cache structure of SLAB and + * SLUB is no longer needed. + */ +#ifdef CONFIG_SLOB +struct kmem_cache { + unsigned int object_size;/* The original size of the object */ + unsigned int size; /* The aligned/padded/added on size */ + unsigned int align; /* Alignment as calculated */ + unsigned long flags; /* Active flags on the slab */ + const char *name; /* Slab name for sysfs */ + int refcount; /* Use counter */ + void (*ctor)(void *); /* Called on object slot creation */ + struct list_head list; /* List of all slab caches on the system */ +}; +#endif + +/* * struct kmem_cache related prototypes */ void __init kmem_cache_init(void); diff --git a/include/linux/slab_def.h b/include/linux/slab_def.h index fbd1117..1d93f27 100644 --- a/include/linux/slab_def.h +++ b/include/linux/slab_def.h @@ -27,7 +27,7 @@ struct kmem_cache { unsigned int limit; unsigned int shared; - unsigned int buffer_size; + unsigned int size; u32 reciprocal_buffer_size; /* 2) touched by every alloc & free from the backend */ @@ -52,7 +52,10 @@ struct kmem_cache { /* 4) cache creation/removal */ const char *name; - struct list_head next; + struct list_head list; + int refcount; + int object_size; + int align; /* 5) statistics */ #ifdef CONFIG_DEBUG_SLAB @@ -73,12 +76,11 @@ struct kmem_cache { /* * If debugging is enabled, then the allocator can add additional - * fields and/or padding to every object. buffer_size contains the total + * fields and/or padding to every object. size contains the total * object size including these internal fields, the following two * variables contain the offset to the user object and its size. */ int obj_offset; - int obj_size; #endif /* CONFIG_DEBUG_SLAB */ /* 6) per-cpu/per-node data, touched during every alloc/free */ diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h index ebdcf4b..df448ad 100644 --- a/include/linux/slub_def.h +++ b/include/linux/slub_def.h @@ -82,7 +82,7 @@ struct kmem_cache { unsigned long flags; unsigned long min_partial; int size; /* The size of an object including meta data */ - int objsize; /* The size of an object without meta data */ + int object_size; /* The size of an object without meta data */ int offset; /* Free pointer offset. */ int cpu_partial; /* Number of per cpu partial objects to keep around */ struct kmem_cache_order_objects oo; diff --git a/mm/slab.c b/mm/slab.c index 28a8f7d..e2b3907 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -424,8 +424,8 @@ static void kmem_list3_init(struct kmem_list3 *parent) * cachep->obj_offset - BYTES_PER_WORD .. cachep->obj_offset - 1: * redzone word. * cachep->obj_offset: The real object. - * cachep->buffer_size - 2* BYTES_PER_WORD: redzone word [BYTES_PER_WORD long] - * cachep->buffer_size - 1* BYTES_PER_WORD: last caller address + * cachep->size - 2* BYTES_PER_WORD: redzone word [BYTES_PER_WORD long] + * cachep->size - 1* BYTES_PER_WORD: last caller address * [BYTES_PER_WORD long] */ static int obj_offset(struct kmem_cache *cachep) @@ -435,7 +435,7 @@ static int obj_offset(struct kmem_cache *cachep) static int obj_size(struct kmem_cache *cachep) { - return cachep->obj_size; + return cachep->object_size; } static unsigned long long *dbg_redzone1(struct kmem_cache *cachep, void *objp) @@ -449,23 +449,23 @@ static unsigned long long *dbg_redzone2(struct kmem_cache *cachep, void *objp) { BUG_ON(!(cachep->flags & SLAB_RED_ZONE)); if (cachep->flags & SLAB_STORE_USER) - return (unsigned long long *)(objp + cachep->buffer_size - + return (unsigned long long *)(objp + cachep->size - sizeof(unsigned long long) - REDZONE_ALIGN); - return (unsigned long long *) (objp + cachep->buffer_size - + return (unsigned long long *) (objp + cachep->size - sizeof(unsigned long long)); } static void **dbg_userword(struct kmem_cache *cachep, void *objp) { BUG_ON(!(cachep->flags & SLAB_STORE_USER)); - return (void **)(objp + cachep->buffer_size - BYTES_PER_WORD); + return (void **)(objp + cachep->size - BYTES_PER_WORD); } #else #define obj_offset(x) 0 -#define obj_size(cachep) (cachep->buffer_size) +#define obj_size(cachep) (cachep->size) #define dbg_redzone1(cachep, objp) ({BUG(); (unsigned long long *)NULL;}) #define dbg_redzone2(cachep, objp) ({BUG(); (unsigned long long *)NULL;}) #define dbg_userword(cachep, objp) ({BUG(); (void **)NULL;}) @@ -475,7 +475,7 @@ static void **dbg_userword(struct kmem_cache *cachep, void *objp) #ifdef CONFIG_TRACING size_t slab_buffer_size(struct kmem_cache *cachep) { - return cachep->buffer_size; + return cachep->size; } EXPORT_SYMBOL(slab_buffer_size); #endif @@ -513,13 +513,13 @@ static inline struct slab *virt_to_slab(const void *obj) static inline void *index_to_obj(struct kmem_cache *cache, struct slab *slab, unsigned int idx) { - return slab->s_mem + cache->buffer_size * idx; + return slab->s_mem + cache->size * idx; } /* - * We want to avoid an expensive divide : (offset / cache->buffer_size) - * Using the fact that buffer_size is a constant for a particular cache, - * we can replace (offset / cache->buffer_size) by + * We want to avoid an expensive divide : (offset / cache->size) + * Using the fact that size is a constant for a particular cache, + * we can replace (offset / cache->size) by * reciprocal_divide(offset, cache->reciprocal_buffer_size) */ static inline unsigned int obj_to_index(const struct kmem_cache *cache, @@ -565,7 +565,7 @@ static struct kmem_cache cache_cache = { .batchcount = 1, .limit = BOOT_CPUCACHE_ENTRIES, .shared = 1, - .buffer_size = sizeof(struct kmem_cache), + .size = sizeof(struct kmem_cache), .name = "kmem_cache", }; @@ -1134,7 +1134,7 @@ static int init_cache_nodelists_node(int node) struct kmem_list3 *l3; const int memsize = sizeof(struct kmem_list3); - list_for_each_entry(cachep, &cache_chain, next) { + list_for_each_entry(cachep, &cache_chain, list) { /* * Set up the size64 kmemlist for cpu before we can * begin anything. Make sure some other cpu on this @@ -1172,7 +1172,7 @@ static void __cpuinit cpuup_canceled(long cpu) int node = cpu_to_mem(cpu); const struct cpumask *mask = cpumask_of_node(node); - list_for_each_entry(cachep, &cache_chain, next) { + list_for_each_entry(cachep, &cache_chain, list) { struct array_cache *nc; struct array_cache *shared; struct array_cache **alien; @@ -1222,7 +1222,7 @@ free_array_cache: * the respective cache's slabs, now we can go ahead and * shrink each nodelist to its limit. */ - list_for_each_entry(cachep, &cache_chain, next) { + list_for_each_entry(cachep, &cache_chain, list) { l3 = cachep->nodelists[node]; if (!l3) continue; @@ -1251,7 +1251,7 @@ static int __cpuinit cpuup_prepare(long cpu) * Now we can go ahead with allocating the shared arrays and * array caches */ - list_for_each_entry(cachep, &cache_chain, next) { + list_for_each_entry(cachep, &cache_chain, list) { struct array_cache *nc; struct array_cache *shared = NULL; struct array_cache **alien = NULL; @@ -1383,7 +1383,7 @@ static int __meminit drain_cache_nodelists_node(int node) struct kmem_cache *cachep; int ret = 0; - list_for_each_entry(cachep, &cache_chain, next) { + list_for_each_entry(cachep, &cache_chain, list) { struct kmem_list3 *l3; l3 = cachep->nodelists[node]; @@ -1526,7 +1526,7 @@ void __init kmem_cache_init(void) /* 1) create the cache_cache */ INIT_LIST_HEAD(&cache_chain); - list_add(&cache_cache.next, &cache_chain); + list_add(&cache_cache.list, &cache_chain); cache_cache.colour_off = cache_line_size(); cache_cache.array[smp_processor_id()] = &initarray_cache.cache; cache_cache.nodelists[node] = &initkmem_list3[CACHE_CACHE + node]; @@ -1534,18 +1534,16 @@ void __init kmem_cache_init(void) /* * struct kmem_cache size depends on nr_node_ids & nr_cpu_ids */ - cache_cache.buffer_size = offsetof(struct kmem_cache, array[nr_cpu_ids]) + + cache_cache.size = offsetof(struct kmem_cache, array[nr_cpu_ids]) + nr_node_ids * sizeof(struct kmem_list3 *); -#if DEBUG - cache_cache.obj_size = cache_cache.buffer_size; -#endif - cache_cache.buffer_size = ALIGN(cache_cache.buffer_size, + cache_cache.object_size = cache_cache.size; + cache_cache.size = ALIGN(cache_cache.size, cache_line_size()); cache_cache.reciprocal_buffer_size = - reciprocal_value(cache_cache.buffer_size); + reciprocal_value(cache_cache.size); for (order = 0; order < MAX_ORDER; order++) { - cache_estimate(order, cache_cache.buffer_size, + cache_estimate(order, cache_cache.size, cache_line_size(), 0, &left_over, &cache_cache.num); if (cache_cache.num) break; @@ -1671,7 +1669,7 @@ void __init kmem_cache_init_late(void) /* 6) resize the head arrays to their final sizes */ mutex_lock(&cache_chain_mutex); - list_for_each_entry(cachep, &cache_chain, next) + list_for_each_entry(cachep, &cache_chain, list) if (enable_cpucache(cachep, GFP_NOWAIT)) BUG(); mutex_unlock(&cache_chain_mutex); @@ -1724,7 +1722,7 @@ slab_out_of_memory(struct kmem_cache *cachep, gfp_t gfpflags, int nodeid) "SLAB: Unable to allocate memory on node %d (gfp=0x%x)\n", nodeid, gfpflags); printk(KERN_WARNING " cache: %s, object size: %d, order: %d\n", - cachep->name, cachep->buffer_size, cachep->gfporder); + cachep->name, cachep->size, cachep->gfporder); for_each_online_node(node) { unsigned long active_objs = 0, num_objs = 0, free_objects = 0; @@ -2028,10 +2026,10 @@ static void slab_destroy_debugcheck(struct kmem_cache *cachep, struct slab *slab if (cachep->flags & SLAB_POISON) { #ifdef CONFIG_DEBUG_PAGEALLOC - if (cachep->buffer_size % PAGE_SIZE == 0 && + if (cachep->size % PAGE_SIZE == 0 && OFF_SLAB(cachep)) kernel_map_pages(virt_to_page(objp), - cachep->buffer_size / PAGE_SIZE, 1); + cachep->size / PAGE_SIZE, 1); else check_poison_obj(cachep, objp); #else @@ -2281,7 +2279,7 @@ kmem_cache_create (const char *name, size_t size, size_t align, mutex_lock(&cache_chain_mutex); } - list_for_each_entry(pc, &cache_chain, next) { + list_for_each_entry(pc, &cache_chain, list) { char tmp; int res; @@ -2294,7 +2292,7 @@ kmem_cache_create (const char *name, size_t size, size_t align, if (res) { printk(KERN_ERR "SLAB: cache with size %d has lost its name\n", - pc->buffer_size); + pc->size); continue; } @@ -2399,8 +2397,9 @@ kmem_cache_create (const char *name, size_t size, size_t align, goto oops; cachep->nodelists = (struct kmem_list3 **)&cachep->array[nr_cpu_ids]; + cachep->object_size = size; + cachep->align = align; #if DEBUG - cachep->obj_size = size; /* * Both debugging options require word-alignment which is calculated @@ -2423,7 +2422,7 @@ kmem_cache_create (const char *name, size_t size, size_t align, } #if FORCED_DEBUG && defined(CONFIG_DEBUG_PAGEALLOC) if (size >= malloc_sizes[INDEX_L3 + 1].cs_size - && cachep->obj_size > cache_line_size() && ALIGN(size, align) < PAGE_SIZE) { + && cachep->object_size > cache_line_size() && ALIGN(size, align) < PAGE_SIZE) { cachep->obj_offset += PAGE_SIZE - ALIGN(size, align); size = PAGE_SIZE; } @@ -2492,7 +2491,7 @@ kmem_cache_create (const char *name, size_t size, size_t align, cachep->gfpflags = 0; if (CONFIG_ZONE_DMA_FLAG && (flags & SLAB_CACHE_DMA)) cachep->gfpflags |= GFP_DMA; - cachep->buffer_size = size; + cachep->size = size; cachep->reciprocal_buffer_size = reciprocal_value(size); if (flags & CFLGS_OFF_SLAB) { @@ -2526,7 +2525,7 @@ kmem_cache_create (const char *name, size_t size, size_t align, } /* cache setup completed, link it into the list */ - list_add(&cachep->next, &cache_chain); + list_add(&cachep->list, &cache_chain); oops: if (!cachep && (flags & SLAB_PANIC)) panic("kmem_cache_create(): failed to create slab `%s'\n", @@ -2721,10 +2720,10 @@ void kmem_cache_destroy(struct kmem_cache *cachep) /* * the chain is never empty, cache_cache is never destroyed */ - list_del(&cachep->next); + list_del(&cachep->list); if (__cache_shrink(cachep)) { slab_error(cachep, "Can't free all objects"); - list_add(&cachep->next, &cache_chain); + list_add(&cachep->list, &cache_chain); mutex_unlock(&cache_chain_mutex); put_online_cpus(); return; @@ -2821,10 +2820,10 @@ static void cache_init_objs(struct kmem_cache *cachep, slab_error(cachep, "constructor overwrote the" " start of an object"); } - if ((cachep->buffer_size % PAGE_SIZE) == 0 && + if ((cachep->size % PAGE_SIZE) == 0 && OFF_SLAB(cachep) && cachep->flags & SLAB_POISON) kernel_map_pages(virt_to_page(objp), - cachep->buffer_size / PAGE_SIZE, 0); + cachep->size / PAGE_SIZE, 0); #else if (cachep->ctor) cachep->ctor(objp); @@ -3058,10 +3057,10 @@ static void *cache_free_debugcheck(struct kmem_cache *cachep, void *objp, #endif if (cachep->flags & SLAB_POISON) { #ifdef CONFIG_DEBUG_PAGEALLOC - if ((cachep->buffer_size % PAGE_SIZE)==0 && OFF_SLAB(cachep)) { + if ((cachep->size % PAGE_SIZE)==0 && OFF_SLAB(cachep)) { store_stackinfo(cachep, objp, (unsigned long)caller); kernel_map_pages(virt_to_page(objp), - cachep->buffer_size / PAGE_SIZE, 0); + cachep->size / PAGE_SIZE, 0); } else { poison_obj(cachep, objp, POISON_FREE); } @@ -3211,9 +3210,9 @@ static void *cache_alloc_debugcheck_after(struct kmem_cache *cachep, return objp; if (cachep->flags & SLAB_POISON) { #ifdef CONFIG_DEBUG_PAGEALLOC - if ((cachep->buffer_size % PAGE_SIZE) == 0 && OFF_SLAB(cachep)) + if ((cachep->size % PAGE_SIZE) == 0 && OFF_SLAB(cachep)) kernel_map_pages(virt_to_page(objp), - cachep->buffer_size / PAGE_SIZE, 1); + cachep->size / PAGE_SIZE, 1); else check_poison_obj(cachep, objp); #else @@ -3243,7 +3242,7 @@ static void *cache_alloc_debugcheck_after(struct kmem_cache *cachep, unsigned objnr; slabp = virt_to_head_page(objp)->slab_page; - objnr = (unsigned)(objp - slabp->s_mem) / cachep->buffer_size; + objnr = (unsigned)(objp - slabp->s_mem) / cachep->size; slab_bufctl(slabp)[objnr] = BUFCTL_ACTIVE; } #endif @@ -3747,7 +3746,7 @@ void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags) void *ret = __cache_alloc(cachep, flags, __builtin_return_address(0)); trace_kmem_cache_alloc(_RET_IP_, ret, - obj_size(cachep), cachep->buffer_size, flags); + obj_size(cachep), cachep->size, flags); return ret; } @@ -3775,7 +3774,7 @@ void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid) __builtin_return_address(0)); trace_kmem_cache_alloc_node(_RET_IP_, ret, - obj_size(cachep), cachep->buffer_size, + obj_size(cachep), cachep->size, flags, nodeid); return ret; @@ -3857,7 +3856,7 @@ static __always_inline void *__do_kmalloc(size_t size, gfp_t flags, ret = __cache_alloc(cachep, flags, caller); trace_kmalloc((unsigned long) caller, ret, - size, cachep->buffer_size, flags); + size, cachep->size, flags); return ret; } @@ -4011,7 +4010,7 @@ static int alloc_kmemlist(struct kmem_cache *cachep, gfp_t gfp) return 0; fail: - if (!cachep->next.next) { + if (!cachep->list.next) { /* Cache is not active yet. Roll back what we did */ node--; while (node >= 0) { @@ -4105,13 +4104,13 @@ static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp) * The numbers are guessed, we should auto-tune as described by * Bonwick. */ - if (cachep->buffer_size > 131072) + if (cachep->size > 131072) limit = 1; - else if (cachep->buffer_size > PAGE_SIZE) + else if (cachep->size > PAGE_SIZE) limit = 8; - else if (cachep->buffer_size > 1024) + else if (cachep->size > 1024) limit = 24; - else if (cachep->buffer_size > 256) + else if (cachep->size > 256) limit = 54; else limit = 120; @@ -4126,7 +4125,7 @@ static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp) * to a larger limit. Thus disabled by default. */ shared = 0; - if (cachep->buffer_size <= PAGE_SIZE && num_possible_cpus() > 1) + if (cachep->size <= PAGE_SIZE && num_possible_cpus() > 1) shared = 8; #if DEBUG @@ -4196,7 +4195,7 @@ static void cache_reap(struct work_struct *w) /* Give up. Setup the next iteration. */ goto out; - list_for_each_entry(searchp, &cache_chain, next) { + list_for_each_entry(searchp, &cache_chain, list) { check_irq_on(); /* @@ -4289,7 +4288,7 @@ static void s_stop(struct seq_file *m, void *p) static int s_show(struct seq_file *m, void *p) { - struct kmem_cache *cachep = list_entry(p, struct kmem_cache, next); + struct kmem_cache *cachep = list_entry(p, struct kmem_cache, list); struct slab *slabp; unsigned long active_objs; unsigned long num_objs; @@ -4345,7 +4344,7 @@ static int s_show(struct seq_file *m, void *p) printk(KERN_ERR "slab: cache %s error: %s\n", name, error); seq_printf(m, "%-17s %6lu %6lu %6u %4u %4d", - name, active_objs, num_objs, cachep->buffer_size, + name, active_objs, num_objs, cachep->size, cachep->num, (1 << cachep->gfporder)); seq_printf(m, " : tunables %4u %4u %4u", cachep->limit, cachep->batchcount, cachep->shared); @@ -4437,7 +4436,7 @@ static ssize_t slabinfo_write(struct file *file, const char __user *buffer, /* Find the cache in the chain of caches. */ mutex_lock(&cache_chain_mutex); res = -EINVAL; - list_for_each_entry(cachep, &cache_chain, next) { + list_for_each_entry(cachep, &cache_chain, list) { if (!strcmp(cachep->name, kbuf)) { if (limit < 1 || batchcount < 1 || batchcount > limit || shared < 0) { @@ -4513,7 +4512,7 @@ static void handle_slab(unsigned long *n, struct kmem_cache *c, struct slab *s) int i; if (n[0] == n[1]) return; - for (i = 0, p = s->s_mem; i < c->num; i++, p += c->buffer_size) { + for (i = 0, p = s->s_mem; i < c->num; i++, p += c->size) { if (slab_bufctl(s)[i] != BUFCTL_ACTIVE) continue; if (!add_caller(n, (unsigned long)*dbg_userword(c, p))) diff --git a/mm/slob.c b/mm/slob.c index c85265d..95d1c7d 100644 --- a/mm/slob.c +++ b/mm/slob.c @@ -506,13 +506,6 @@ size_t ksize(const void *block) } EXPORT_SYMBOL(ksize); -struct kmem_cache { - unsigned int size, align; - unsigned long flags; - const char *name; - void (*ctor)(void *); -}; - struct kmem_cache *kmem_cache_create(const char *name, size_t size, size_t align, unsigned long flags, void (*ctor)(void *)) { @@ -523,7 +516,7 @@ struct kmem_cache *kmem_cache_create(const char *name, size_t size, if (c) { c->name = name; - c->size = size; + c->size = c->object_size; if (flags & SLAB_DESTROY_BY_RCU) { /* leave room for rcu footer at the end of object */ c->size += sizeof(struct slob_rcu); diff --git a/mm/slub.c b/mm/slub.c index 2de3c99..797271f 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -311,7 +311,7 @@ static inline size_t slab_ksize(const struct kmem_cache *s) * and whatever may come after it. */ if (s->flags & (SLAB_RED_ZONE | SLAB_POISON)) - return s->objsize; + return s->object_size; #endif /* @@ -609,11 +609,11 @@ static void print_trailer(struct kmem_cache *s, struct page *page, u8 *p) if (p > addr + 16) print_section("Bytes b4 ", p - 16, 16); - print_section("Object ", p, min_t(unsigned long, s->objsize, + print_section("Object ", p, min_t(unsigned long, s->object_size, PAGE_SIZE)); if (s->flags & SLAB_RED_ZONE) - print_section("Redzone ", p + s->objsize, - s->inuse - s->objsize); + print_section("Redzone ", p + s->object_size, + s->inuse - s->object_size); if (s->offset) off = s->offset + sizeof(void *); @@ -655,12 +655,12 @@ static void init_object(struct kmem_cache *s, void *object, u8 val) u8 *p = object; if (s->flags & __OBJECT_POISON) { - memset(p, POISON_FREE, s->objsize - 1); - p[s->objsize - 1] = POISON_END; + memset(p, POISON_FREE, s->object_size - 1); + p[s->object_size - 1] = POISON_END; } if (s->flags & SLAB_RED_ZONE) - memset(p + s->objsize, val, s->inuse - s->objsize); + memset(p + s->object_size, val, s->inuse - s->object_size); } static void restore_bytes(struct kmem_cache *s, char *message, u8 data, @@ -705,10 +705,10 @@ static int check_bytes_and_report(struct kmem_cache *s, struct page *page, * Poisoning uses 0x6b (POISON_FREE) and the last byte is * 0xa5 (POISON_END) * - * object + s->objsize + * object + s->object_size * Padding to reach word boundary. This is also used for Redzoning. * Padding is extended by another word if Redzoning is enabled and - * objsize == inuse. + * object_size == inuse. * * We fill with 0xbb (RED_INACTIVE) for inactive objects and with * 0xcc (RED_ACTIVE) for objects in use. @@ -727,7 +727,7 @@ static int check_bytes_and_report(struct kmem_cache *s, struct page *page, * object + s->size * Nothing is used beyond s->size. * - * If slabcaches are merged then the objsize and inuse boundaries are mostly + * If slabcaches are merged then the object_size and inuse boundaries are mostly * ignored. And therefore no slab options that rely on these boundaries * may be used with merged slabcaches. */ @@ -787,25 +787,25 @@ static int check_object(struct kmem_cache *s, struct page *page, void *object, u8 val) { u8 *p = object; - u8 *endobject = object + s->objsize; + u8 *endobject = object + s->object_size; if (s->flags & SLAB_RED_ZONE) { if (!check_bytes_and_report(s, page, object, "Redzone", - endobject, val, s->inuse - s->objsize)) + endobject, val, s->inuse - s->object_size)) return 0; } else { - if ((s->flags & SLAB_POISON) && s->objsize < s->inuse) { + if ((s->flags & SLAB_POISON) && s->object_size < s->inuse) { check_bytes_and_report(s, page, p, "Alignment padding", - endobject, POISON_INUSE, s->inuse - s->objsize); + endobject, POISON_INUSE, s->inuse - s->object_size); } } if (s->flags & SLAB_POISON) { if (val != SLUB_RED_ACTIVE && (s->flags & __OBJECT_POISON) && (!check_bytes_and_report(s, page, p, "Poison", p, - POISON_FREE, s->objsize - 1) || + POISON_FREE, s->object_size - 1) || !check_bytes_and_report(s, page, p, "Poison", - p + s->objsize - 1, POISON_END, 1))) + p + s->object_size - 1, POISON_END, 1))) return 0; /* * check_pad_bytes cleans up on its own. @@ -926,7 +926,7 @@ static void trace(struct kmem_cache *s, struct page *page, void *object, page->freelist); if (!alloc) - print_section("Object ", (void *)object, s->objsize); + print_section("Object ", (void *)object, s->object_size); dump_stack(); } @@ -942,14 +942,14 @@ static inline int slab_pre_alloc_hook(struct kmem_cache *s, gfp_t flags) lockdep_trace_alloc(flags); might_sleep_if(flags & __GFP_WAIT); - return should_failslab(s->objsize, flags, s->flags); + return should_failslab(s->object_size, flags, s->flags); } static inline void slab_post_alloc_hook(struct kmem_cache *s, gfp_t flags, void *object) { flags &= gfp_allowed_mask; kmemcheck_slab_alloc(s, flags, object, slab_ksize(s)); - kmemleak_alloc_recursive(object, s->objsize, 1, s->flags, flags); + kmemleak_alloc_recursive(object, s->object_size, 1, s->flags, flags); } static inline void slab_free_hook(struct kmem_cache *s, void *x) @@ -966,13 +966,13 @@ static inline void slab_free_hook(struct kmem_cache *s, void *x) unsigned long flags; local_irq_save(flags); - kmemcheck_slab_free(s, x, s->objsize); - debug_check_no_locks_freed(x, s->objsize); + kmemcheck_slab_free(s, x, s->object_size); + debug_check_no_locks_freed(x, s->object_size); local_irq_restore(flags); } #endif if (!(s->flags & SLAB_DEBUG_OBJECTS)) - debug_check_no_obj_freed(x, s->objsize); + debug_check_no_obj_freed(x, s->object_size); } /* @@ -1207,7 +1207,7 @@ out: __setup("slub_debug", setup_slub_debug); -static unsigned long kmem_cache_flags(unsigned long objsize, +static unsigned long kmem_cache_flags(unsigned long object_size, unsigned long flags, const char *name, void (*ctor)(void *)) { @@ -1237,7 +1237,7 @@ static inline int check_object(struct kmem_cache *s, struct page *page, static inline void add_full(struct kmem_cache *s, struct kmem_cache_node *n, struct page *page) {} static inline void remove_full(struct kmem_cache *s, struct page *page) {} -static inline unsigned long kmem_cache_flags(unsigned long objsize, +static inline unsigned long kmem_cache_flags(unsigned long object_size, unsigned long flags, const char *name, void (*ctor)(void *)) { @@ -2098,10 +2098,10 @@ slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid) "SLUB: Unable to allocate memory on node %d (gfp=0x%x)\n", nid, gfpflags); printk(KERN_WARNING " cache: %s, object size: %d, buffer size: %d, " - "default order: %d, min order: %d\n", s->name, s->objsize, + "default order: %d, min order: %d\n", s->name, s->object_size, s->size, oo_order(s->oo), oo_order(s->min)); - if (oo_order(s->min) > get_order(s->objsize)) + if (oo_order(s->min) > get_order(s->object_size)) printk(KERN_WARNING " %s debugging increased min order, use " "slub_debug=O to disable.\n", s->name); @@ -2374,7 +2374,7 @@ redo: } if (unlikely(gfpflags & __GFP_ZERO) && object) - memset(object, 0, s->objsize); + memset(object, 0, s->object_size); slab_post_alloc_hook(s, gfpflags, object); @@ -2385,7 +2385,7 @@ void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags) { void *ret = slab_alloc(s, gfpflags, NUMA_NO_NODE, _RET_IP_); - trace_kmem_cache_alloc(_RET_IP_, ret, s->objsize, s->size, gfpflags); + trace_kmem_cache_alloc(_RET_IP_, ret, s->object_size, s->size, gfpflags); return ret; } @@ -2415,7 +2415,7 @@ void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node) void *ret = slab_alloc(s, gfpflags, node, _RET_IP_); trace_kmem_cache_alloc_node(_RET_IP_, ret, - s->objsize, s->size, gfpflags, node); + s->object_size, s->size, gfpflags, node); return ret; } @@ -2910,7 +2910,7 @@ static void set_min_partial(struct kmem_cache *s, unsigned long min) static int calculate_sizes(struct kmem_cache *s, int forced_order) { unsigned long flags = s->flags; - unsigned long size = s->objsize; + unsigned long size = s->object_size; unsigned long align = s->align; int order; @@ -2939,7 +2939,7 @@ static int calculate_sizes(struct kmem_cache *s, int forced_order) * end of the object and the free pointer. If not then add an * additional word to have some bytes to store Redzone information. */ - if ((flags & SLAB_RED_ZONE) && size == s->objsize) + if ((flags & SLAB_RED_ZONE) && size == s->object_size) size += sizeof(void *); #endif @@ -2987,7 +2987,7 @@ static int calculate_sizes(struct kmem_cache *s, int forced_order) * user specified and the dynamic determination of cache line size * on bootup. */ - align = calculate_alignment(flags, align, s->objsize); + align = calculate_alignment(flags, align, s->object_size); s->align = align; /* @@ -3035,7 +3035,7 @@ static int kmem_cache_open(struct kmem_cache *s, memset(s, 0, kmem_size); s->name = name; s->ctor = ctor; - s->objsize = size; + s->object_size = size; s->align = align; s->flags = kmem_cache_flags(size, flags, name, ctor); s->reserved = 0; @@ -3050,7 +3050,7 @@ static int kmem_cache_open(struct kmem_cache *s, * Disable debugging flags that store metadata if the min slab * order increased. */ - if (get_order(s->size) > get_order(s->objsize)) { + if (get_order(s->size) > get_order(s->object_size)) { s->flags &= ~DEBUG_METADATA_FLAGS; s->offset = 0; if (!calculate_sizes(s, -1)) @@ -3124,7 +3124,7 @@ error: */ unsigned int kmem_cache_size(struct kmem_cache *s) { - return s->objsize; + return s->object_size; } EXPORT_SYMBOL(kmem_cache_size); @@ -3853,11 +3853,11 @@ void __init kmem_cache_init(void) if (s && s->size) { char *name = kasprintf(GFP_NOWAIT, - "dma-kmalloc-%d", s->objsize); + "dma-kmalloc-%d", s->object_size); BUG_ON(!name); kmalloc_dma_caches[i] = create_kmalloc_cache(name, - s->objsize, SLAB_CACHE_DMA); + s->object_size, SLAB_CACHE_DMA); } } #endif @@ -3951,7 +3951,7 @@ struct kmem_cache *kmem_cache_create(const char *name, size_t size, * Adjust the object sizes so that we clear * the complete object on kzalloc. */ - s->objsize = max(s->objsize, (int)size); + s->object_size = max(s->object_size, (int)size); s->inuse = max_t(int, s->inuse, ALIGN(size, sizeof(void *))); if (sysfs_slab_alias(s, name)) { @@ -4634,7 +4634,7 @@ SLAB_ATTR_RO(align); static ssize_t object_size_show(struct kmem_cache *s, char *buf) { - return sprintf(buf, "%d\n", s->objsize); + return sprintf(buf, "%d\n", s->object_size); } SLAB_ATTR_RO(object_size); @@ -5438,7 +5438,7 @@ __initcall(slab_sysfs_init); static void print_slabinfo_header(struct seq_file *m) { seq_puts(m, "slabinfo - version: 2.1\n"); - seq_puts(m, "# name " + seq_puts(m, "# name " " "); seq_puts(m, " : tunables "); seq_puts(m, " : slabdata "); -- cgit v0.10.2 From 8c138bc00925521c4e764269db3a903bd2a51592 Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Wed, 13 Jun 2012 10:24:58 -0500 Subject: slab: Get rid of obj_size macro The size of the slab object is frequently needed. Since we now have a size field directly in the kmem_cache structure there is no need anymore of the obj_size macro/function. Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg diff --git a/mm/slab.c b/mm/slab.c index e2b3907..fc4a774 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -433,11 +433,6 @@ static int obj_offset(struct kmem_cache *cachep) return cachep->obj_offset; } -static int obj_size(struct kmem_cache *cachep) -{ - return cachep->object_size; -} - static unsigned long long *dbg_redzone1(struct kmem_cache *cachep, void *objp) { BUG_ON(!(cachep->flags & SLAB_RED_ZONE)); @@ -465,7 +460,6 @@ static void **dbg_userword(struct kmem_cache *cachep, void *objp) #else #define obj_offset(x) 0 -#define obj_size(cachep) (cachep->size) #define dbg_redzone1(cachep, objp) ({BUG(); (unsigned long long *)NULL;}) #define dbg_redzone2(cachep, objp) ({BUG(); (unsigned long long *)NULL;}) #define dbg_userword(cachep, objp) ({BUG(); (void **)NULL;}) @@ -1853,7 +1847,7 @@ static void kmem_rcu_free(struct rcu_head *head) static void store_stackinfo(struct kmem_cache *cachep, unsigned long *addr, unsigned long caller) { - int size = obj_size(cachep); + int size = cachep->object_size; addr = (unsigned long *)&((char *)addr)[obj_offset(cachep)]; @@ -1885,7 +1879,7 @@ static void store_stackinfo(struct kmem_cache *cachep, unsigned long *addr, static void poison_obj(struct kmem_cache *cachep, void *addr, unsigned char val) { - int size = obj_size(cachep); + int size = cachep->object_size; addr = &((char *)addr)[obj_offset(cachep)]; memset(addr, val, size); @@ -1945,7 +1939,7 @@ static void print_objinfo(struct kmem_cache *cachep, void *objp, int lines) printk("\n"); } realobj = (char *)objp + obj_offset(cachep); - size = obj_size(cachep); + size = cachep->object_size; for (i = 0; i < size && lines; i += 16, lines--) { int limit; limit = 16; @@ -1962,7 +1956,7 @@ static void check_poison_obj(struct kmem_cache *cachep, void *objp) int lines = 0; realobj = (char *)objp + obj_offset(cachep); - size = obj_size(cachep); + size = cachep->object_size; for (i = 0; i < size; i++) { char exp = POISON_FREE; @@ -3265,7 +3259,7 @@ static bool slab_should_failslab(struct kmem_cache *cachep, gfp_t flags) if (cachep == &cache_cache) return false; - return should_failslab(obj_size(cachep), flags, cachep->flags); + return should_failslab(cachep->object_size, flags, cachep->flags); } static inline void *____cache_alloc(struct kmem_cache *cachep, gfp_t flags) @@ -3525,14 +3519,14 @@ __cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid, out: local_irq_restore(save_flags); ptr = cache_alloc_debugcheck_after(cachep, flags, ptr, caller); - kmemleak_alloc_recursive(ptr, obj_size(cachep), 1, cachep->flags, + kmemleak_alloc_recursive(ptr, cachep->object_size, 1, cachep->flags, flags); if (likely(ptr)) - kmemcheck_slab_alloc(cachep, flags, ptr, obj_size(cachep)); + kmemcheck_slab_alloc(cachep, flags, ptr, cachep->object_size); if (unlikely((flags & __GFP_ZERO) && ptr)) - memset(ptr, 0, obj_size(cachep)); + memset(ptr, 0, cachep->object_size); return ptr; } @@ -3587,15 +3581,15 @@ __cache_alloc(struct kmem_cache *cachep, gfp_t flags, void *caller) objp = __do_cache_alloc(cachep, flags); local_irq_restore(save_flags); objp = cache_alloc_debugcheck_after(cachep, flags, objp, caller); - kmemleak_alloc_recursive(objp, obj_size(cachep), 1, cachep->flags, + kmemleak_alloc_recursive(objp, cachep->object_size, 1, cachep->flags, flags); prefetchw(objp); if (likely(objp)) - kmemcheck_slab_alloc(cachep, flags, objp, obj_size(cachep)); + kmemcheck_slab_alloc(cachep, flags, objp, cachep->object_size); if (unlikely((flags & __GFP_ZERO) && objp)) - memset(objp, 0, obj_size(cachep)); + memset(objp, 0, cachep->object_size); return objp; } @@ -3711,7 +3705,7 @@ static inline void __cache_free(struct kmem_cache *cachep, void *objp, kmemleak_free_recursive(objp, cachep->flags); objp = cache_free_debugcheck(cachep, objp, caller); - kmemcheck_slab_free(cachep, objp, obj_size(cachep)); + kmemcheck_slab_free(cachep, objp, cachep->object_size); /* * Skip calling cache_free_alien() when the platform is not numa. @@ -3746,7 +3740,7 @@ void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags) void *ret = __cache_alloc(cachep, flags, __builtin_return_address(0)); trace_kmem_cache_alloc(_RET_IP_, ret, - obj_size(cachep), cachep->size, flags); + cachep->object_size, cachep->size, flags); return ret; } @@ -3774,7 +3768,7 @@ void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid) __builtin_return_address(0)); trace_kmem_cache_alloc_node(_RET_IP_, ret, - obj_size(cachep), cachep->size, + cachep->object_size, cachep->size, flags, nodeid); return ret; @@ -3896,9 +3890,9 @@ void kmem_cache_free(struct kmem_cache *cachep, void *objp) unsigned long flags; local_irq_save(flags); - debug_check_no_locks_freed(objp, obj_size(cachep)); + debug_check_no_locks_freed(objp, cachep->size); if (!(cachep->flags & SLAB_DEBUG_OBJECTS)) - debug_check_no_obj_freed(objp, obj_size(cachep)); + debug_check_no_obj_freed(objp, cachep->object_size); __cache_free(cachep, objp, __builtin_return_address(0)); local_irq_restore(flags); @@ -3927,8 +3921,9 @@ void kfree(const void *objp) local_irq_save(flags); kfree_debugcheck(objp); c = virt_to_cache(objp); - debug_check_no_locks_freed(objp, obj_size(c)); - debug_check_no_obj_freed(objp, obj_size(c)); + debug_check_no_locks_freed(objp, c->object_size); + + debug_check_no_obj_freed(objp, c->object_size); __cache_free(c, (void *)objp, __builtin_return_address(0)); local_irq_restore(flags); } @@ -3936,7 +3931,7 @@ EXPORT_SYMBOL(kfree); unsigned int kmem_cache_size(struct kmem_cache *cachep) { - return obj_size(cachep); + return cachep->object_size; } EXPORT_SYMBOL(kmem_cache_size); @@ -4657,6 +4652,6 @@ size_t ksize(const void *objp) if (unlikely(objp == ZERO_SIZE_PTR)) return 0; - return obj_size(virt_to_cache(objp)); + return virt_to_cache(objp)->object_size; } EXPORT_SYMBOL(ksize); -- cgit v0.10.2 From e7b691b085fda913830e5280ae6f724b2a63c824 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Sat, 9 Jun 2012 02:40:03 -0700 Subject: slab/mempolicy: always use local policy from interrupt context slab_node() could access current->mempolicy from interrupt context. However there's a race condition during exit where the mempolicy is first freed and then the pointer zeroed. Using this from interrupts seems bogus anyways. The interrupt will interrupt a random process and therefore get a random mempolicy. Many times, this will be idle's, which noone can change. Just disable this here and always use local for slab from interrupts. I also cleaned up the callers of slab_node a bit which always passed the same argument. I believe the original mempolicy code did that in fact, so it's likely a regression. v2: send version with correct logic v3: simplify. fix typo. Reported-by: Arun Sharma Cc: penberg@kernel.org Cc: cl@linux.com Signed-off-by: Andi Kleen [tdmackey@twitter.com: Rework control flow based on feedback from cl@linux.com, fix logic, and cleanup current task_struct reference] Acked-by: David Rientjes Acked-by: Christoph Lameter Acked-by: KOSAKI Motohiro Signed-off-by: David Mackey Signed-off-by: Pekka Enberg diff --git a/include/linux/mempolicy.h b/include/linux/mempolicy.h index 4aa4273..95b738c 100644 --- a/include/linux/mempolicy.h +++ b/include/linux/mempolicy.h @@ -215,7 +215,7 @@ extern struct zonelist *huge_zonelist(struct vm_area_struct *vma, extern bool init_nodemask_of_mempolicy(nodemask_t *mask); extern bool mempolicy_nodemask_intersects(struct task_struct *tsk, const nodemask_t *mask); -extern unsigned slab_node(struct mempolicy *policy); +extern unsigned slab_node(void); extern enum zone_type policy_zone; diff --git a/mm/mempolicy.c b/mm/mempolicy.c index f15c1b2..cb0b230 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -1602,8 +1602,14 @@ static unsigned interleave_nodes(struct mempolicy *policy) * task can change it's policy. The system default policy requires no * such protection. */ -unsigned slab_node(struct mempolicy *policy) +unsigned slab_node(void) { + struct mempolicy *policy; + + if (in_interrupt()) + return numa_node_id(); + + policy = current->mempolicy; if (!policy || policy->flags & MPOL_F_LOCAL) return numa_node_id(); diff --git a/mm/slab.c b/mm/slab.c index fc4a774..dd607a8 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -3310,7 +3310,7 @@ static void *alternate_node_alloc(struct kmem_cache *cachep, gfp_t flags) if (cpuset_do_slab_mem_spread() && (cachep->flags & SLAB_MEM_SPREAD)) nid_alloc = cpuset_slab_spread_node(); else if (current->mempolicy) - nid_alloc = slab_node(current->mempolicy); + nid_alloc = slab_node(); if (nid_alloc != nid_here) return ____cache_alloc_node(cachep, flags, nid_alloc); return NULL; @@ -3342,7 +3342,7 @@ static void *fallback_alloc(struct kmem_cache *cache, gfp_t flags) retry_cpuset: cpuset_mems_cookie = get_mems_allowed(); - zonelist = node_zonelist(slab_node(current->mempolicy), flags); + zonelist = node_zonelist(slab_node(), flags); retry: /* diff --git a/mm/slub.c b/mm/slub.c index 797271f..348fed1 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -1617,7 +1617,7 @@ static void *get_any_partial(struct kmem_cache *s, gfp_t flags, do { cpuset_mems_cookie = get_mems_allowed(); - zonelist = node_zonelist(slab_node(current->mempolicy), flags); + zonelist = node_zonelist(slab_node(), flags); for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) { struct kmem_cache_node *n; -- cgit v0.10.2 From d24ac77f71ded6a013bacb09f359eac0b0f29a80 Mon Sep 17 00:00:00 2001 From: Joonsoo Kim Date: Fri, 18 May 2012 22:01:17 +0900 Subject: slub: use __cmpxchg_double_slab() at interrupt disabled place get_freelist(), unfreeze_partials() are only called with interrupt disabled, so __cmpxchg_double_slab() is suitable. Acked-by: Christoph Lameter Signed-off-by: Joonsoo Kim Signed-off-by: Pekka Enberg diff --git a/mm/slub.c b/mm/slub.c index 348fed1..4f406cd 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -1872,7 +1872,11 @@ redo: } } -/* Unfreeze all the cpu partial slabs */ +/* + * Unfreeze all the cpu partial slabs. + * + * This function must be called with interrupt disabled. + */ static void unfreeze_partials(struct kmem_cache *s) { struct kmem_cache_node *n = NULL; @@ -1928,7 +1932,7 @@ static void unfreeze_partials(struct kmem_cache *s) l = m; } - } while (!cmpxchg_double_slab(s, page, + } while (!__cmpxchg_double_slab(s, page, old.freelist, old.counters, new.freelist, new.counters, "unfreezing slab")); @@ -2165,6 +2169,8 @@ static inline void *new_slab_objects(struct kmem_cache *s, gfp_t flags, * The page is still frozen if the return value is not NULL. * * If this function returns NULL then the page has been unfrozen. + * + * This function must be called with interrupt disabled. */ static inline void *get_freelist(struct kmem_cache *s, struct page *page) { @@ -2182,7 +2188,7 @@ static inline void *get_freelist(struct kmem_cache *s, struct page *page) new.inuse = page->objects; new.frozen = freelist != NULL; - } while (!cmpxchg_double_slab(s, page, + } while (!__cmpxchg_double_slab(s, page, freelist, counters, NULL, new.counters, "get_freelist")); -- cgit v0.10.2 From 43d77867a4f333de4e4189114c480dd365133c09 Mon Sep 17 00:00:00 2001 From: Joonsoo Kim Date: Sat, 9 Jun 2012 02:23:16 +0900 Subject: slub: refactoring unfreeze_partials() Current implementation of unfreeze_partials() is so complicated, but benefit from it is insignificant. In addition many code in do {} while loop have a bad influence to a fail rate of cmpxchg_double_slab. Under current implementation which test status of cpu partial slab and acquire list_lock in do {} while loop, we don't need to acquire a list_lock and gain a little benefit when front of the cpu partial slab is to be discarded, but this is a rare case. In case that add_partial is performed and cmpxchg_double_slab is failed, remove_partial should be called case by case. I think that these are disadvantages of current implementation, so I do refactoring unfreeze_partials(). Minimizing code in do {} while loop introduce a reduced fail rate of cmpxchg_double_slab. Below is output of 'slabinfo -r kmalloc-256' when './perf stat -r 33 hackbench 50 process 4000 > /dev/null' is done. ** before ** Cmpxchg_double Looping ------------------------ Locked Cmpxchg Double redos 182685 Unlocked Cmpxchg Double redos 0 ** after ** Cmpxchg_double Looping ------------------------ Locked Cmpxchg Double redos 177995 Unlocked Cmpxchg Double redos 1 We can see cmpxchg_double_slab fail rate is improved slightly. Bolow is output of './perf stat -r 30 hackbench 50 process 4000 > /dev/null'. ** before ** Performance counter stats for './hackbench 50 process 4000' (30 runs): 108517.190463 task-clock # 7.926 CPUs utilized ( +- 0.24% ) 2,919,550 context-switches # 0.027 M/sec ( +- 3.07% ) 100,774 CPU-migrations # 0.929 K/sec ( +- 4.72% ) 124,201 page-faults # 0.001 M/sec ( +- 0.15% ) 401,500,234,387 cycles # 3.700 GHz ( +- 0.24% ) stalled-cycles-frontend stalled-cycles-backend 250,576,913,354 instructions # 0.62 insns per cycle ( +- 0.13% ) 45,934,956,860 branches # 423.297 M/sec ( +- 0.14% ) 188,219,787 branch-misses # 0.41% of all branches ( +- 0.56% ) 13.691837307 seconds time elapsed ( +- 0.24% ) ** after ** Performance counter stats for './hackbench 50 process 4000' (30 runs): 107784.479767 task-clock # 7.928 CPUs utilized ( +- 0.22% ) 2,834,781 context-switches # 0.026 M/sec ( +- 2.33% ) 93,083 CPU-migrations # 0.864 K/sec ( +- 3.45% ) 123,967 page-faults # 0.001 M/sec ( +- 0.15% ) 398,781,421,836 cycles # 3.700 GHz ( +- 0.22% ) stalled-cycles-frontend stalled-cycles-backend 250,189,160,419 instructions # 0.63 insns per cycle ( +- 0.09% ) 45,855,370,128 branches # 425.436 M/sec ( +- 0.10% ) 169,881,248 branch-misses # 0.37% of all branches ( +- 0.43% ) 13.596272341 seconds time elapsed ( +- 0.22% ) No regression is found, but rather we can see slightly better result. Acked-by: Christoph Lameter Signed-off-by: Joonsoo Kim Signed-off-by: Pekka Enberg diff --git a/mm/slub.c b/mm/slub.c index 4f406cd..f96d8bc 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -1879,18 +1879,24 @@ redo: */ static void unfreeze_partials(struct kmem_cache *s) { - struct kmem_cache_node *n = NULL; + struct kmem_cache_node *n = NULL, *n2 = NULL; struct kmem_cache_cpu *c = this_cpu_ptr(s->cpu_slab); struct page *page, *discard_page = NULL; while ((page = c->partial)) { - enum slab_modes { M_PARTIAL, M_FREE }; - enum slab_modes l, m; struct page new; struct page old; c->partial = page->next; - l = M_FREE; + + n2 = get_node(s, page_to_nid(page)); + if (n != n2) { + if (n) + spin_unlock(&n->list_lock); + + n = n2; + spin_lock(&n->list_lock); + } do { @@ -1903,43 +1909,17 @@ static void unfreeze_partials(struct kmem_cache *s) new.frozen = 0; - if (!new.inuse && (!n || n->nr_partial > s->min_partial)) - m = M_FREE; - else { - struct kmem_cache_node *n2 = get_node(s, - page_to_nid(page)); - - m = M_PARTIAL; - if (n != n2) { - if (n) - spin_unlock(&n->list_lock); - - n = n2; - spin_lock(&n->list_lock); - } - } - - if (l != m) { - if (l == M_PARTIAL) { - remove_partial(n, page); - stat(s, FREE_REMOVE_PARTIAL); - } else { - add_partial(n, page, - DEACTIVATE_TO_TAIL); - stat(s, FREE_ADD_PARTIAL); - } - - l = m; - } - } while (!__cmpxchg_double_slab(s, page, old.freelist, old.counters, new.freelist, new.counters, "unfreezing slab")); - if (m == M_FREE) { + if (unlikely(!new.inuse && n->nr_partial > s->min_partial)) { page->next = discard_page; discard_page = page; + } else { + add_partial(n, page, DEACTIVATE_TO_TAIL); + stat(s, FREE_ADD_PARTIAL); } } -- cgit v0.10.2 From a618e89f1e6fb3cdfc8ef0ad54a0d57830bf8881 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Thu, 14 Jun 2012 16:17:21 +0400 Subject: slab: rename gfpflags to allocflags A consistent name with slub saves us an acessor function. In both caches, this field represents the same thing. We would like to use it from the mem_cgroup code. Signed-off-by: Glauber Costa Acked-by: Christoph Lameter CC: Pekka Enberg Signed-off-by: Pekka Enberg diff --git a/include/linux/slab_def.h b/include/linux/slab_def.h index 1d93f27..0c634fa 100644 --- a/include/linux/slab_def.h +++ b/include/linux/slab_def.h @@ -39,7 +39,7 @@ struct kmem_cache { unsigned int gfporder; /* force GFP flags, e.g. GFP_DMA */ - gfp_t gfpflags; + gfp_t allocflags; size_t colour; /* cache colouring range */ unsigned int colour_off; /* colour offset */ diff --git a/mm/slab.c b/mm/slab.c index dd607a8..bb79652 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -1771,7 +1771,7 @@ static void *kmem_getpages(struct kmem_cache *cachep, gfp_t flags, int nodeid) flags |= __GFP_COMP; #endif - flags |= cachep->gfpflags; + flags |= cachep->allocflags; if (cachep->flags & SLAB_RECLAIM_ACCOUNT) flags |= __GFP_RECLAIMABLE; @@ -2482,9 +2482,9 @@ kmem_cache_create (const char *name, size_t size, size_t align, cachep->colour = left_over / cachep->colour_off; cachep->slab_size = slab_size; cachep->flags = flags; - cachep->gfpflags = 0; + cachep->allocflags = 0; if (CONFIG_ZONE_DMA_FLAG && (flags & SLAB_CACHE_DMA)) - cachep->gfpflags |= GFP_DMA; + cachep->allocflags |= GFP_DMA; cachep->size = size; cachep->reciprocal_buffer_size = reciprocal_value(size); @@ -2831,9 +2831,9 @@ static void kmem_flagcheck(struct kmem_cache *cachep, gfp_t flags) { if (CONFIG_ZONE_DMA_FLAG) { if (flags & GFP_DMA) - BUG_ON(!(cachep->gfpflags & GFP_DMA)); + BUG_ON(!(cachep->allocflags & GFP_DMA)); else - BUG_ON(cachep->gfpflags & GFP_DMA); + BUG_ON(cachep->allocflags & GFP_DMA); } } -- cgit v0.10.2 From 0672aa7c236ada6c636e68b2ac2aa135169e6e18 Mon Sep 17 00:00:00 2001 From: Thierry Reding Date: Fri, 22 Jun 2012 19:42:49 +0200 Subject: mm, slab: Build fix for recent kmem_cache changes Commit 3b0efdf ("mm, sl[aou]b: Extract common fields from struct kmem_cache") renamed the kmem_cache structure's "next" field to "list" but forgot to update one instance in leaks_show(). Signed-off-by: Thierry Reding Signed-off-by: Pekka Enberg diff --git a/mm/slab.c b/mm/slab.c index bb79652..d95ad4c 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -4533,7 +4533,7 @@ static void show_symbol(struct seq_file *m, unsigned long address) static int leaks_show(struct seq_file *m, void *p) { - struct kmem_cache *cachep = list_entry(p, struct kmem_cache, next); + struct kmem_cache *cachep = list_entry(p, struct kmem_cache, list); struct slab *slabp; struct kmem_list3 *l3; const char *name; -- cgit v0.10.2 From d97d476b1bb11e24268a6bac8214f9bc58716b45 Mon Sep 17 00:00:00 2001 From: Feng Tang Date: Mon, 2 Jul 2012 14:29:10 +0800 Subject: slab: Fix a typo in commit 8c138b "slab: Get rid of obj_size macro" Commit 8c138b only sits in Pekka's and linux-next tree now, which tries to replace obj_size(cachep) with cachep->object_size, but has a typo in kmem_cache_free() by using "size" instead of "object_size", which casues some regressions. Reported-and-tested-by: Fengguang Wu Signed-off-by: Feng Tang Cc: Christoph Lameter Acked-by: Glauber Costa Signed-off-by: Pekka Enberg diff --git a/mm/slab.c b/mm/slab.c index d95ad4c..8b7cb80 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -3890,7 +3890,7 @@ void kmem_cache_free(struct kmem_cache *cachep, void *objp) unsigned long flags; local_irq_save(flags); - debug_check_no_locks_freed(objp, cachep->size); + debug_check_no_locks_freed(objp, cachep->object_size); if (!(cachep->flags & SLAB_DEBUG_OBJECTS)) debug_check_no_obj_freed(objp, cachep->object_size); __cache_free(cachep, objp, __builtin_return_address(0)); -- cgit v0.10.2 From a164f89628fa813a2b012ec033625e9e507c29bb Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Thu, 21 Jun 2012 00:59:18 +0400 Subject: slab: move FULL state transition to an initcall During kmem_cache_init_late(), we transition to the LATE state, and after some more work, to the FULL state, its last state This is quite different from slub, that will only transition to its last state (previously SYSFS), in a (late)initcall, after a lot more of the kernel is ready. This means that in slab, we have no way to taking actions dependent on the initialization of other pieces of the kernel that are supposed to start way after kmem_init_late(), such as cgroups initialization. To achieve more consistency in this behavior, that patch only transitions to the UP state in kmem_init_late. In my analysis, setup_cpu_cache() should be happy to test for >= UP, instead of == FULL. It also has passed some tests I've made. We then only mark FULL state after the reap timers are in place, meaning that no further setup is expected. Signed-off-by: Glauber Costa Acked-by: Christoph Lameter Acked-by: David Rientjes Signed-off-by: Pekka Enberg diff --git a/mm/slab.c b/mm/slab.c index 8b7cb80..105f188 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -1668,9 +1668,6 @@ void __init kmem_cache_init_late(void) BUG(); mutex_unlock(&cache_chain_mutex); - /* Done! */ - g_cpucache_up = FULL; - /* * Register a cpu startup notifier callback that initializes * cpu_cache_get for all new cpus @@ -1700,6 +1697,9 @@ static int __init cpucache_init(void) */ for_each_online_cpu(cpu) start_cpu_timer(cpu); + + /* Done! */ + g_cpucache_up = FULL; return 0; } __initcall(cpucache_init); @@ -2167,7 +2167,7 @@ static size_t calculate_slab_order(struct kmem_cache *cachep, static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp) { - if (g_cpucache_up == FULL) + if (g_cpucache_up >= LATE) return enable_cpucache(cachep, gfp); if (g_cpucache_up == NONE) { -- cgit v0.10.2 From 4b57ad939263935e4e3aec4d74a11dd02a3421e4 Mon Sep 17 00:00:00 2001 From: majianpeng Date: Tue, 26 Jun 2012 09:30:31 +0800 Subject: mm: Fix signal SIGFPE in slabinfo.c. In function slab_stats(), if total_free is equal zero, it will error. So fix it. Acked-by: Christoph Lameter Signed-off-by: majianpeng Signed-off-by: Pekka Enberg diff --git a/tools/vm/slabinfo.c b/tools/vm/slabinfo.c index 164cbcf..808d5a9 100644 --- a/tools/vm/slabinfo.c +++ b/tools/vm/slabinfo.c @@ -437,34 +437,34 @@ static void slab_stats(struct slabinfo *s) printf("Fastpath %8lu %8lu %3lu %3lu\n", s->alloc_fastpath, s->free_fastpath, s->alloc_fastpath * 100 / total_alloc, - s->free_fastpath * 100 / total_free); + total_free ? s->free_fastpath * 100 / total_free : 0); printf("Slowpath %8lu %8lu %3lu %3lu\n", total_alloc - s->alloc_fastpath, s->free_slowpath, (total_alloc - s->alloc_fastpath) * 100 / total_alloc, - s->free_slowpath * 100 / total_free); + total_free ? s->free_slowpath * 100 / total_free : 0); printf("Page Alloc %8lu %8lu %3lu %3lu\n", s->alloc_slab, s->free_slab, s->alloc_slab * 100 / total_alloc, - s->free_slab * 100 / total_free); + total_free ? s->free_slab * 100 / total_free : 0); printf("Add partial %8lu %8lu %3lu %3lu\n", s->deactivate_to_head + s->deactivate_to_tail, s->free_add_partial, (s->deactivate_to_head + s->deactivate_to_tail) * 100 / total_alloc, - s->free_add_partial * 100 / total_free); + total_free ? s->free_add_partial * 100 / total_free : 0); printf("Remove partial %8lu %8lu %3lu %3lu\n", s->alloc_from_partial, s->free_remove_partial, s->alloc_from_partial * 100 / total_alloc, - s->free_remove_partial * 100 / total_free); + total_free ? s->free_remove_partial * 100 / total_free : 0); printf("Cpu partial list %8lu %8lu %3lu %3lu\n", s->cpu_partial_alloc, s->cpu_partial_free, s->cpu_partial_alloc * 100 / total_alloc, - s->cpu_partial_free * 100 / total_free); + total_free ? s->cpu_partial_free * 100 / total_free : 0); printf("RemoteObj/SlabFrozen %8lu %8lu %3lu %3lu\n", s->deactivate_remote_frees, s->free_frozen, s->deactivate_remote_frees * 100 / total_alloc, - s->free_frozen * 100 / total_free); + total_free ? s->free_frozen * 100 / total_free : 0); printf("Total %8lu %8lu\n\n", total_alloc, total_free); -- cgit v0.10.2 From 068ce415bea9e2b96bde76dc1bf6e672a89903ee Mon Sep 17 00:00:00 2001 From: Julia Lawall Date: Sun, 8 Jul 2012 13:37:40 +0200 Subject: slub: remove invalid reference to list iterator variable If list_for_each_entry, etc complete a traversal of the list, the iterator variable ends up pointing to an address at an offset from the list head, and not a meaningful structure. Thus this value should not be used after the end of the iterator. The patch replaces s->name by al->name, which is referenced nearby. This problem was found using Coccinelle (http://coccinelle.lip6.fr/). Signed-off-by: Julia Lawall Signed-off-by: Pekka Enberg diff --git a/mm/slub.c b/mm/slub.c index f96d8bc..79fe9c6 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -5405,7 +5405,7 @@ static int __init slab_sysfs_init(void) err = sysfs_slab_alias(al->s, al->name); if (err) printk(KERN_ERR "SLUB: Unable to add boot slab alias" - " %s to sysfs\n", s->name); + " %s to sysfs\n", al->name); kfree(al); } -- cgit v0.10.2 From 039363f38bfe5f6281e9eae5e0518b11577d9d50 Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Fri, 6 Jul 2012 15:25:10 -0500 Subject: mm, sl[aou]b: Extract common code for kmem_cache_create() Kmem_cache_create() does a variety of sanity checks but those vary depending on the allocator. Use the strictest tests and put them into a slab_common file. Make the tests conditional on CONFIG_DEBUG_VM. This patch has the effect of adding sanity checks for SLUB and SLOB under CONFIG_DEBUG_VM and removes the checks in SLAB for !CONFIG_DEBUG_VM. Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg diff --git a/include/linux/slab.h b/include/linux/slab.h index 0dd2dfa..0cb7c7e 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -130,6 +130,10 @@ int kmem_cache_shrink(struct kmem_cache *); void kmem_cache_free(struct kmem_cache *, void *); unsigned int kmem_cache_size(struct kmem_cache *); +/* Slab internal function */ +struct kmem_cache *__kmem_cache_create(const char *, size_t, size_t, + unsigned long, + void (*)(void *)); /* * Please use this macro to create slab caches. Simply specify the * name of the structure and maybe some flags that are listed above. diff --git a/mm/Makefile b/mm/Makefile index a156285..ae37078 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -16,7 +16,8 @@ obj-y := filemap.o mempool.o oom_kill.o fadvise.o \ readahead.o swap.o truncate.o vmscan.o shmem.o \ prio_tree.o util.o mmzone.o vmstat.o backing-dev.o \ page_isolation.o mm_init.o mmu_context.o percpu.o \ - compaction.o $(mmu-y) + compaction.o slab_common.o $(mmu-y) + obj-y += init-mm.o ifdef CONFIG_NO_BOOTMEM diff --git a/mm/slab.c b/mm/slab.c index 105f188..10c821e 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -1558,7 +1558,7 @@ void __init kmem_cache_init(void) * bug. */ - sizes[INDEX_AC].cs_cachep = kmem_cache_create(names[INDEX_AC].name, + sizes[INDEX_AC].cs_cachep = __kmem_cache_create(names[INDEX_AC].name, sizes[INDEX_AC].cs_size, ARCH_KMALLOC_MINALIGN, ARCH_KMALLOC_FLAGS|SLAB_PANIC, @@ -1566,7 +1566,7 @@ void __init kmem_cache_init(void) if (INDEX_AC != INDEX_L3) { sizes[INDEX_L3].cs_cachep = - kmem_cache_create(names[INDEX_L3].name, + __kmem_cache_create(names[INDEX_L3].name, sizes[INDEX_L3].cs_size, ARCH_KMALLOC_MINALIGN, ARCH_KMALLOC_FLAGS|SLAB_PANIC, @@ -1584,14 +1584,14 @@ void __init kmem_cache_init(void) * allow tighter packing of the smaller caches. */ if (!sizes->cs_cachep) { - sizes->cs_cachep = kmem_cache_create(names->name, + sizes->cs_cachep = __kmem_cache_create(names->name, sizes->cs_size, ARCH_KMALLOC_MINALIGN, ARCH_KMALLOC_FLAGS|SLAB_PANIC, NULL); } #ifdef CONFIG_ZONE_DMA - sizes->cs_dmacachep = kmem_cache_create( + sizes->cs_dmacachep = __kmem_cache_create( names->name_dma, sizes->cs_size, ARCH_KMALLOC_MINALIGN, @@ -2220,7 +2220,7 @@ static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp) } /** - * kmem_cache_create - Create a cache. + * __kmem_cache_create - Create a cache. * @name: A string which is used in /proc/slabinfo to identify this cache. * @size: The size of objects to be created in this cache. * @align: The required alignment for the objects. @@ -2247,7 +2247,7 @@ static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp) * as davem. */ struct kmem_cache * -kmem_cache_create (const char *name, size_t size, size_t align, +__kmem_cache_create (const char *name, size_t size, size_t align, unsigned long flags, void (*ctor)(void *)) { size_t left_over, slab_size, ralign; @@ -2388,7 +2388,7 @@ kmem_cache_create (const char *name, size_t size, size_t align, /* Get cache's description obj. */ cachep = kmem_cache_zalloc(&cache_cache, gfp); if (!cachep) - goto oops; + return NULL; cachep->nodelists = (struct kmem_list3 **)&cachep->array[nr_cpu_ids]; cachep->object_size = size; @@ -2445,8 +2445,7 @@ kmem_cache_create (const char *name, size_t size, size_t align, printk(KERN_ERR "kmem_cache_create: couldn't create cache %s.\n", name); kmem_cache_free(&cache_cache, cachep); - cachep = NULL; - goto oops; + return NULL; } slab_size = ALIGN(cachep->num * sizeof(kmem_bufctl_t) + sizeof(struct slab), align); @@ -2504,8 +2503,7 @@ kmem_cache_create (const char *name, size_t size, size_t align, if (setup_cpu_cache(cachep, gfp)) { __kmem_cache_destroy(cachep); - cachep = NULL; - goto oops; + return NULL; } if (flags & SLAB_DEBUG_OBJECTS) { @@ -2521,16 +2519,12 @@ kmem_cache_create (const char *name, size_t size, size_t align, /* cache setup completed, link it into the list */ list_add(&cachep->list, &cache_chain); oops: - if (!cachep && (flags & SLAB_PANIC)) - panic("kmem_cache_create(): failed to create slab `%s'\n", - name); if (slab_is_available()) { mutex_unlock(&cache_chain_mutex); put_online_cpus(); } return cachep; } -EXPORT_SYMBOL(kmem_cache_create); #if DEBUG static void check_irq_off(void) diff --git a/mm/slab_common.c b/mm/slab_common.c new file mode 100644 index 0000000..80412be --- /dev/null +++ b/mm/slab_common.c @@ -0,0 +1,68 @@ +/* + * Slab allocator functions that are independent of the allocator strategy + * + * (C) 2012 Christoph Lameter + */ +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +/* + * kmem_cache_create - Create a cache. + * @name: A string which is used in /proc/slabinfo to identify this cache. + * @size: The size of objects to be created in this cache. + * @align: The required alignment for the objects. + * @flags: SLAB flags + * @ctor: A constructor for the objects. + * + * Returns a ptr to the cache on success, NULL on failure. + * Cannot be called within a interrupt, but can be interrupted. + * The @ctor is run when new pages are allocated by the cache. + * + * The flags are + * + * %SLAB_POISON - Poison the slab with a known test pattern (a5a5a5a5) + * to catch references to uninitialised memory. + * + * %SLAB_RED_ZONE - Insert `Red' zones around the allocated memory to check + * for buffer overruns. + * + * %SLAB_HWCACHE_ALIGN - Align the objects in this cache to a hardware + * cacheline. This can be beneficial if you're counting cycles as closely + * as davem. + */ + +struct kmem_cache *kmem_cache_create(const char *name, size_t size, size_t align, + unsigned long flags, void (*ctor)(void *)) +{ + struct kmem_cache *s = NULL; + +#ifdef CONFIG_DEBUG_VM + if (!name || in_interrupt() || size < sizeof(void *) || + size > KMALLOC_MAX_SIZE) { + printk(KERN_ERR "kmem_cache_create(%s) integrity check" + " failed\n", name); + goto out; + } +#endif + + s = __kmem_cache_create(name, size, align, flags, ctor); + +#ifdef CONFIG_DEBUG_VM +out: +#endif + if (!s && (flags & SLAB_PANIC)) + panic("kmem_cache_create: Failed to create slab '%s'\n", name); + + return s; +} +EXPORT_SYMBOL(kmem_cache_create); diff --git a/mm/slob.c b/mm/slob.c index 95d1c7d..d63923d 100644 --- a/mm/slob.c +++ b/mm/slob.c @@ -506,7 +506,7 @@ size_t ksize(const void *block) } EXPORT_SYMBOL(ksize); -struct kmem_cache *kmem_cache_create(const char *name, size_t size, +struct kmem_cache *__kmem_cache_create(const char *name, size_t size, size_t align, unsigned long flags, void (*ctor)(void *)) { struct kmem_cache *c; @@ -529,13 +529,11 @@ struct kmem_cache *kmem_cache_create(const char *name, size_t size, c->align = ARCH_SLAB_MINALIGN; if (c->align < align) c->align = align; - } else if (flags & SLAB_PANIC) - panic("Cannot create slab cache %s\n", name); - kmemleak_alloc(c, sizeof(struct kmem_cache), 1, GFP_KERNEL); + kmemleak_alloc(c, sizeof(struct kmem_cache), 1, GFP_KERNEL); + } return c; } -EXPORT_SYMBOL(kmem_cache_create); void kmem_cache_destroy(struct kmem_cache *c) { diff --git a/mm/slub.c b/mm/slub.c index 79fe9c6..6551cc9 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -3920,15 +3920,12 @@ static struct kmem_cache *find_mergeable(size_t size, return NULL; } -struct kmem_cache *kmem_cache_create(const char *name, size_t size, +struct kmem_cache *__kmem_cache_create(const char *name, size_t size, size_t align, unsigned long flags, void (*ctor)(void *)) { struct kmem_cache *s; char *n; - if (WARN_ON(!name)) - return NULL; - down_write(&slub_lock); s = find_mergeable(size, align, flags, name, ctor); if (s) { @@ -3972,14 +3969,8 @@ struct kmem_cache *kmem_cache_create(const char *name, size_t size, kfree(n); err: up_write(&slub_lock); - - if (flags & SLAB_PANIC) - panic("Cannot create slabcache %s\n", name); - else - s = NULL; return s; } -EXPORT_SYMBOL(kmem_cache_create); #ifdef CONFIG_SMP /* -- cgit v0.10.2 From 97d06609158e61f6bdf538c4a6788e2de492236f Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Fri, 6 Jul 2012 15:25:11 -0500 Subject: mm, sl[aou]b: Common definition for boot state of the slab allocators All allocators have some sort of support for the bootstrap status. Setup a common definition for the boot states and make all slab allocators use that definition. Reviewed-by: Glauber Costa Reviewed-by: Joonsoo Kim Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg diff --git a/include/linux/slab.h b/include/linux/slab.h index 0cb7c7e..0dd2dfa 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -130,10 +130,6 @@ int kmem_cache_shrink(struct kmem_cache *); void kmem_cache_free(struct kmem_cache *, void *); unsigned int kmem_cache_size(struct kmem_cache *); -/* Slab internal function */ -struct kmem_cache *__kmem_cache_create(const char *, size_t, size_t, - unsigned long, - void (*)(void *)); /* * Please use this macro to create slab caches. Simply specify the * name of the structure and maybe some flags that are listed above. diff --git a/mm/slab.c b/mm/slab.c index 10c821e..59a466b 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -87,6 +87,7 @@ */ #include +#include "slab.h" #include #include #include @@ -565,27 +566,6 @@ static struct kmem_cache cache_cache = { #define BAD_ALIEN_MAGIC 0x01020304ul -/* - * chicken and egg problem: delay the per-cpu array allocation - * until the general caches are up. - */ -static enum { - NONE, - PARTIAL_AC, - PARTIAL_L3, - EARLY, - LATE, - FULL -} g_cpucache_up; - -/* - * used by boot code to determine if it can use slab based allocator - */ -int slab_is_available(void) -{ - return g_cpucache_up >= EARLY; -} - #ifdef CONFIG_LOCKDEP /* @@ -651,7 +631,7 @@ static void init_node_lock_keys(int q) { struct cache_sizes *s = malloc_sizes; - if (g_cpucache_up < LATE) + if (slab_state < UP) return; for (s = malloc_sizes; s->cs_size != ULONG_MAX; s++) { @@ -1649,14 +1629,14 @@ void __init kmem_cache_init(void) } } - g_cpucache_up = EARLY; + slab_state = UP; } void __init kmem_cache_init_late(void) { struct kmem_cache *cachep; - g_cpucache_up = LATE; + slab_state = UP; /* Annotate slab for lockdep -- annotate the malloc caches */ init_lock_keys(); @@ -1668,6 +1648,9 @@ void __init kmem_cache_init_late(void) BUG(); mutex_unlock(&cache_chain_mutex); + /* Done! */ + slab_state = FULL; + /* * Register a cpu startup notifier callback that initializes * cpu_cache_get for all new cpus @@ -1699,7 +1682,7 @@ static int __init cpucache_init(void) start_cpu_timer(cpu); /* Done! */ - g_cpucache_up = FULL; + slab_state = FULL; return 0; } __initcall(cpucache_init); @@ -2167,10 +2150,10 @@ static size_t calculate_slab_order(struct kmem_cache *cachep, static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp) { - if (g_cpucache_up >= LATE) + if (slab_state >= FULL) return enable_cpucache(cachep, gfp); - if (g_cpucache_up == NONE) { + if (slab_state == DOWN) { /* * Note: the first kmem_cache_create must create the cache * that's used by kmalloc(24), otherwise the creation of @@ -2185,16 +2168,16 @@ static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp) */ set_up_list3s(cachep, SIZE_AC); if (INDEX_AC == INDEX_L3) - g_cpucache_up = PARTIAL_L3; + slab_state = PARTIAL_L3; else - g_cpucache_up = PARTIAL_AC; + slab_state = PARTIAL_ARRAYCACHE; } else { cachep->array[smp_processor_id()] = kmalloc(sizeof(struct arraycache_init), gfp); - if (g_cpucache_up == PARTIAL_AC) { + if (slab_state == PARTIAL_ARRAYCACHE) { set_up_list3s(cachep, SIZE_L3); - g_cpucache_up = PARTIAL_L3; + slab_state = PARTIAL_L3; } else { int node; for_each_online_node(node) { diff --git a/mm/slab.h b/mm/slab.h new file mode 100644 index 0000000..f9a9815 --- /dev/null +++ b/mm/slab.h @@ -0,0 +1,29 @@ +#ifndef MM_SLAB_H +#define MM_SLAB_H +/* + * Internal slab definitions + */ + +/* + * State of the slab allocator. + * + * This is used to describe the states of the allocator during bootup. + * Allocators use this to gradually bootstrap themselves. Most allocators + * have the problem that the structures used for managing slab caches are + * allocated from slab caches themselves. + */ +enum slab_state { + DOWN, /* No slab functionality yet */ + PARTIAL, /* SLUB: kmem_cache_node available */ + PARTIAL_ARRAYCACHE, /* SLAB: kmalloc size for arraycache available */ + PARTIAL_L3, /* SLAB: kmalloc size for l3 struct available */ + UP, /* Slab caches usable but not all extras yet */ + FULL /* Everything is working */ +}; + +extern enum slab_state slab_state; + +struct kmem_cache *__kmem_cache_create(const char *name, size_t size, + size_t align, unsigned long flags, void (*ctor)(void *)); + +#endif diff --git a/mm/slab_common.c b/mm/slab_common.c index 80412be..ca1aaf6 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -16,6 +16,10 @@ #include #include +#include "slab.h" + +enum slab_state slab_state; + /* * kmem_cache_create - Create a cache. * @name: A string which is used in /proc/slabinfo to identify this cache. @@ -66,3 +70,8 @@ out: return s; } EXPORT_SYMBOL(kmem_cache_create); + +int slab_is_available(void) +{ + return slab_state >= UP; +} diff --git a/mm/slob.c b/mm/slob.c index d63923d..0111e0d 100644 --- a/mm/slob.c +++ b/mm/slob.c @@ -59,6 +59,8 @@ #include #include +#include "slab.h" + #include #include /* struct reclaim_state */ #include @@ -531,6 +533,7 @@ struct kmem_cache *__kmem_cache_create(const char *name, size_t size, c->align = align; kmemleak_alloc(c, sizeof(struct kmem_cache), 1, GFP_KERNEL); + c->refcount = 1; } return c; } @@ -616,19 +619,12 @@ int kmem_cache_shrink(struct kmem_cache *d) } EXPORT_SYMBOL(kmem_cache_shrink); -static unsigned int slob_ready __read_mostly; - -int slab_is_available(void) -{ - return slob_ready; -} - void __init kmem_cache_init(void) { - slob_ready = 1; + slab_state = UP; } void __init kmem_cache_init_late(void) { - /* Nothing to do */ + slab_state = FULL; } diff --git a/mm/slub.c b/mm/slub.c index 6551cc9..4c385164 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -16,6 +16,7 @@ #include #include #include +#include "slab.h" #include #include #include @@ -182,13 +183,6 @@ static int kmem_size = sizeof(struct kmem_cache); static struct notifier_block slab_notifier; #endif -static enum { - DOWN, /* No slab functionality available */ - PARTIAL, /* Kmem_cache_node works */ - UP, /* Everything works but does not show up in sysfs */ - SYSFS /* Sysfs up */ -} slab_state = DOWN; - /* A list of all slab caches on the system */ static DECLARE_RWSEM(slub_lock); static LIST_HEAD(slab_caches); @@ -237,11 +231,6 @@ static inline void stat(const struct kmem_cache *s, enum stat_item si) * Core slab cache functions *******************************************************************/ -int slab_is_available(void) -{ - return slab_state >= UP; -} - static inline struct kmem_cache_node *get_node(struct kmem_cache *s, int node) { return s->node[node]; @@ -5274,7 +5263,7 @@ static int sysfs_slab_add(struct kmem_cache *s) const char *name; int unmergeable; - if (slab_state < SYSFS) + if (slab_state < FULL) /* Defer until later */ return 0; @@ -5319,7 +5308,7 @@ static int sysfs_slab_add(struct kmem_cache *s) static void sysfs_slab_remove(struct kmem_cache *s) { - if (slab_state < SYSFS) + if (slab_state < FULL) /* * Sysfs has not been setup yet so no need to remove the * cache from sysfs. @@ -5347,7 +5336,7 @@ static int sysfs_slab_alias(struct kmem_cache *s, const char *name) { struct saved_alias *al; - if (slab_state == SYSFS) { + if (slab_state == FULL) { /* * If we have a leftover link then remove it. */ @@ -5380,7 +5369,7 @@ static int __init slab_sysfs_init(void) return -ENOSYS; } - slab_state = SYSFS; + slab_state = FULL; list_for_each_entry(s, &slab_caches, list) { err = sysfs_slab_add(s); -- cgit v0.10.2 From 18004c5d4084d965aa1396392706b8688306427a Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Fri, 6 Jul 2012 15:25:12 -0500 Subject: mm, sl[aou]b: Use a common mutex definition Use the mutex definition from SLAB and make it the common way to take a sleeping lock. This has the effect of using a mutex instead of a rw semaphore for SLUB. SLOB gains the use of a mutex for kmem_cache_create serialization. Not needed now but SLOB may acquire some more features later (like slabinfo / sysfs support) through the expansion of the common code that will need this. Reviewed-by: Glauber Costa Reviewed-by: Joonsoo Kim Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg diff --git a/mm/slab.c b/mm/slab.c index 59a466b..fd7dac6 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -68,7 +68,7 @@ * Further notes from the original documentation: * * 11 April '97. Started multi-threading - markhe - * The global cache-chain is protected by the mutex 'cache_chain_mutex'. + * The global cache-chain is protected by the mutex 'slab_mutex'. * The sem is only needed when accessing/extending the cache-chain, which * can never happen inside an interrupt (kmem_cache_create(), * kmem_cache_shrink() and kmem_cache_reap()). @@ -671,12 +671,6 @@ static void slab_set_debugobj_lock_classes(struct kmem_cache *cachep) } #endif -/* - * Guard access to the cache-chain. - */ -static DEFINE_MUTEX(cache_chain_mutex); -static struct list_head cache_chain; - static DEFINE_PER_CPU(struct delayed_work, slab_reap_work); static inline struct array_cache *cpu_cache_get(struct kmem_cache *cachep) @@ -1100,7 +1094,7 @@ static inline int cache_free_alien(struct kmem_cache *cachep, void *objp) * When hotplugging memory or a cpu, existing nodelists are not replaced if * already in use. * - * Must hold cache_chain_mutex. + * Must hold slab_mutex. */ static int init_cache_nodelists_node(int node) { @@ -1108,7 +1102,7 @@ static int init_cache_nodelists_node(int node) struct kmem_list3 *l3; const int memsize = sizeof(struct kmem_list3); - list_for_each_entry(cachep, &cache_chain, list) { + list_for_each_entry(cachep, &slab_caches, list) { /* * Set up the size64 kmemlist for cpu before we can * begin anything. Make sure some other cpu on this @@ -1124,7 +1118,7 @@ static int init_cache_nodelists_node(int node) /* * The l3s don't come and go as CPUs come and - * go. cache_chain_mutex is sufficient + * go. slab_mutex is sufficient * protection here. */ cachep->nodelists[node] = l3; @@ -1146,7 +1140,7 @@ static void __cpuinit cpuup_canceled(long cpu) int node = cpu_to_mem(cpu); const struct cpumask *mask = cpumask_of_node(node); - list_for_each_entry(cachep, &cache_chain, list) { + list_for_each_entry(cachep, &slab_caches, list) { struct array_cache *nc; struct array_cache *shared; struct array_cache **alien; @@ -1196,7 +1190,7 @@ free_array_cache: * the respective cache's slabs, now we can go ahead and * shrink each nodelist to its limit. */ - list_for_each_entry(cachep, &cache_chain, list) { + list_for_each_entry(cachep, &slab_caches, list) { l3 = cachep->nodelists[node]; if (!l3) continue; @@ -1225,7 +1219,7 @@ static int __cpuinit cpuup_prepare(long cpu) * Now we can go ahead with allocating the shared arrays and * array caches */ - list_for_each_entry(cachep, &cache_chain, list) { + list_for_each_entry(cachep, &slab_caches, list) { struct array_cache *nc; struct array_cache *shared = NULL; struct array_cache **alien = NULL; @@ -1293,9 +1287,9 @@ static int __cpuinit cpuup_callback(struct notifier_block *nfb, switch (action) { case CPU_UP_PREPARE: case CPU_UP_PREPARE_FROZEN: - mutex_lock(&cache_chain_mutex); + mutex_lock(&slab_mutex); err = cpuup_prepare(cpu); - mutex_unlock(&cache_chain_mutex); + mutex_unlock(&slab_mutex); break; case CPU_ONLINE: case CPU_ONLINE_FROZEN: @@ -1305,7 +1299,7 @@ static int __cpuinit cpuup_callback(struct notifier_block *nfb, case CPU_DOWN_PREPARE: case CPU_DOWN_PREPARE_FROZEN: /* - * Shutdown cache reaper. Note that the cache_chain_mutex is + * Shutdown cache reaper. Note that the slab_mutex is * held so that if cache_reap() is invoked it cannot do * anything expensive but will only modify reap_work * and reschedule the timer. @@ -1332,9 +1326,9 @@ static int __cpuinit cpuup_callback(struct notifier_block *nfb, #endif case CPU_UP_CANCELED: case CPU_UP_CANCELED_FROZEN: - mutex_lock(&cache_chain_mutex); + mutex_lock(&slab_mutex); cpuup_canceled(cpu); - mutex_unlock(&cache_chain_mutex); + mutex_unlock(&slab_mutex); break; } return notifier_from_errno(err); @@ -1350,14 +1344,14 @@ static struct notifier_block __cpuinitdata cpucache_notifier = { * Returns -EBUSY if all objects cannot be drained so that the node is not * removed. * - * Must hold cache_chain_mutex. + * Must hold slab_mutex. */ static int __meminit drain_cache_nodelists_node(int node) { struct kmem_cache *cachep; int ret = 0; - list_for_each_entry(cachep, &cache_chain, list) { + list_for_each_entry(cachep, &slab_caches, list) { struct kmem_list3 *l3; l3 = cachep->nodelists[node]; @@ -1388,14 +1382,14 @@ static int __meminit slab_memory_callback(struct notifier_block *self, switch (action) { case MEM_GOING_ONLINE: - mutex_lock(&cache_chain_mutex); + mutex_lock(&slab_mutex); ret = init_cache_nodelists_node(nid); - mutex_unlock(&cache_chain_mutex); + mutex_unlock(&slab_mutex); break; case MEM_GOING_OFFLINE: - mutex_lock(&cache_chain_mutex); + mutex_lock(&slab_mutex); ret = drain_cache_nodelists_node(nid); - mutex_unlock(&cache_chain_mutex); + mutex_unlock(&slab_mutex); break; case MEM_ONLINE: case MEM_OFFLINE: @@ -1499,8 +1493,8 @@ void __init kmem_cache_init(void) node = numa_mem_id(); /* 1) create the cache_cache */ - INIT_LIST_HEAD(&cache_chain); - list_add(&cache_cache.list, &cache_chain); + INIT_LIST_HEAD(&slab_caches); + list_add(&cache_cache.list, &slab_caches); cache_cache.colour_off = cache_line_size(); cache_cache.array[smp_processor_id()] = &initarray_cache.cache; cache_cache.nodelists[node] = &initkmem_list3[CACHE_CACHE + node]; @@ -1642,11 +1636,11 @@ void __init kmem_cache_init_late(void) init_lock_keys(); /* 6) resize the head arrays to their final sizes */ - mutex_lock(&cache_chain_mutex); - list_for_each_entry(cachep, &cache_chain, list) + mutex_lock(&slab_mutex); + list_for_each_entry(cachep, &slab_caches, list) if (enable_cpucache(cachep, GFP_NOWAIT)) BUG(); - mutex_unlock(&cache_chain_mutex); + mutex_unlock(&slab_mutex); /* Done! */ slab_state = FULL; @@ -2253,10 +2247,10 @@ __kmem_cache_create (const char *name, size_t size, size_t align, */ if (slab_is_available()) { get_online_cpus(); - mutex_lock(&cache_chain_mutex); + mutex_lock(&slab_mutex); } - list_for_each_entry(pc, &cache_chain, list) { + list_for_each_entry(pc, &slab_caches, list) { char tmp; int res; @@ -2500,10 +2494,10 @@ __kmem_cache_create (const char *name, size_t size, size_t align, } /* cache setup completed, link it into the list */ - list_add(&cachep->list, &cache_chain); + list_add(&cachep->list, &slab_caches); oops: if (slab_is_available()) { - mutex_unlock(&cache_chain_mutex); + mutex_unlock(&slab_mutex); put_online_cpus(); } return cachep; @@ -2622,7 +2616,7 @@ out: return nr_freed; } -/* Called with cache_chain_mutex held to protect against cpu hotplug */ +/* Called with slab_mutex held to protect against cpu hotplug */ static int __cache_shrink(struct kmem_cache *cachep) { int ret = 0, i = 0; @@ -2657,9 +2651,9 @@ int kmem_cache_shrink(struct kmem_cache *cachep) BUG_ON(!cachep || in_interrupt()); get_online_cpus(); - mutex_lock(&cache_chain_mutex); + mutex_lock(&slab_mutex); ret = __cache_shrink(cachep); - mutex_unlock(&cache_chain_mutex); + mutex_unlock(&slab_mutex); put_online_cpus(); return ret; } @@ -2687,15 +2681,15 @@ void kmem_cache_destroy(struct kmem_cache *cachep) /* Find the cache in the chain of caches. */ get_online_cpus(); - mutex_lock(&cache_chain_mutex); + mutex_lock(&slab_mutex); /* * the chain is never empty, cache_cache is never destroyed */ list_del(&cachep->list); if (__cache_shrink(cachep)) { slab_error(cachep, "Can't free all objects"); - list_add(&cachep->list, &cache_chain); - mutex_unlock(&cache_chain_mutex); + list_add(&cachep->list, &slab_caches); + mutex_unlock(&slab_mutex); put_online_cpus(); return; } @@ -2704,7 +2698,7 @@ void kmem_cache_destroy(struct kmem_cache *cachep) rcu_barrier(); __kmem_cache_destroy(cachep); - mutex_unlock(&cache_chain_mutex); + mutex_unlock(&slab_mutex); put_online_cpus(); } EXPORT_SYMBOL(kmem_cache_destroy); @@ -4017,7 +4011,7 @@ static void do_ccupdate_local(void *info) new->new[smp_processor_id()] = old; } -/* Always called with the cache_chain_mutex held */ +/* Always called with the slab_mutex held */ static int do_tune_cpucache(struct kmem_cache *cachep, int limit, int batchcount, int shared, gfp_t gfp) { @@ -4061,7 +4055,7 @@ static int do_tune_cpucache(struct kmem_cache *cachep, int limit, return alloc_kmemlist(cachep, gfp); } -/* Called with cache_chain_mutex held always */ +/* Called with slab_mutex held always */ static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp) { int err; @@ -4163,11 +4157,11 @@ static void cache_reap(struct work_struct *w) int node = numa_mem_id(); struct delayed_work *work = to_delayed_work(w); - if (!mutex_trylock(&cache_chain_mutex)) + if (!mutex_trylock(&slab_mutex)) /* Give up. Setup the next iteration. */ goto out; - list_for_each_entry(searchp, &cache_chain, list) { + list_for_each_entry(searchp, &slab_caches, list) { check_irq_on(); /* @@ -4205,7 +4199,7 @@ next: cond_resched(); } check_irq_on(); - mutex_unlock(&cache_chain_mutex); + mutex_unlock(&slab_mutex); next_reap_node(); out: /* Set up the next iteration */ @@ -4241,21 +4235,21 @@ static void *s_start(struct seq_file *m, loff_t *pos) { loff_t n = *pos; - mutex_lock(&cache_chain_mutex); + mutex_lock(&slab_mutex); if (!n) print_slabinfo_header(m); - return seq_list_start(&cache_chain, *pos); + return seq_list_start(&slab_caches, *pos); } static void *s_next(struct seq_file *m, void *p, loff_t *pos) { - return seq_list_next(p, &cache_chain, pos); + return seq_list_next(p, &slab_caches, pos); } static void s_stop(struct seq_file *m, void *p) { - mutex_unlock(&cache_chain_mutex); + mutex_unlock(&slab_mutex); } static int s_show(struct seq_file *m, void *p) @@ -4406,9 +4400,9 @@ static ssize_t slabinfo_write(struct file *file, const char __user *buffer, return -EINVAL; /* Find the cache in the chain of caches. */ - mutex_lock(&cache_chain_mutex); + mutex_lock(&slab_mutex); res = -EINVAL; - list_for_each_entry(cachep, &cache_chain, list) { + list_for_each_entry(cachep, &slab_caches, list) { if (!strcmp(cachep->name, kbuf)) { if (limit < 1 || batchcount < 1 || batchcount > limit || shared < 0) { @@ -4421,7 +4415,7 @@ static ssize_t slabinfo_write(struct file *file, const char __user *buffer, break; } } - mutex_unlock(&cache_chain_mutex); + mutex_unlock(&slab_mutex); if (res >= 0) res = count; return res; @@ -4444,8 +4438,8 @@ static const struct file_operations proc_slabinfo_operations = { static void *leaks_start(struct seq_file *m, loff_t *pos) { - mutex_lock(&cache_chain_mutex); - return seq_list_start(&cache_chain, *pos); + mutex_lock(&slab_mutex); + return seq_list_start(&slab_caches, *pos); } static inline int add_caller(unsigned long *n, unsigned long v) @@ -4544,17 +4538,17 @@ static int leaks_show(struct seq_file *m, void *p) name = cachep->name; if (n[0] == n[1]) { /* Increase the buffer size */ - mutex_unlock(&cache_chain_mutex); + mutex_unlock(&slab_mutex); m->private = kzalloc(n[0] * 4 * sizeof(unsigned long), GFP_KERNEL); if (!m->private) { /* Too bad, we are really out */ m->private = n; - mutex_lock(&cache_chain_mutex); + mutex_lock(&slab_mutex); return -ENOMEM; } *(unsigned long *)m->private = n[0] * 2; kfree(n); - mutex_lock(&cache_chain_mutex); + mutex_lock(&slab_mutex); /* Now make sure this entry will be retried */ m->count = m->size; return 0; diff --git a/mm/slab.h b/mm/slab.h index f9a9815..db7848c 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -23,6 +23,10 @@ enum slab_state { extern enum slab_state slab_state; +/* The slab cache mutex protects the management structures during changes */ +extern struct mutex slab_mutex; +extern struct list_head slab_caches; + struct kmem_cache *__kmem_cache_create(const char *name, size_t size, size_t align, unsigned long flags, void (*ctor)(void *)); diff --git a/mm/slab_common.c b/mm/slab_common.c index ca1aaf6..50e1ff1 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -19,6 +19,8 @@ #include "slab.h" enum slab_state slab_state; +LIST_HEAD(slab_caches); +DEFINE_MUTEX(slab_mutex); /* * kmem_cache_create - Create a cache. diff --git a/mm/slub.c b/mm/slub.c index 4c385164..8c4fd37 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -36,13 +36,13 @@ /* * Lock order: - * 1. slub_lock (Global Semaphore) + * 1. slab_mutex (Global Mutex) * 2. node->list_lock * 3. slab_lock(page) (Only on some arches and for debugging) * - * slub_lock + * slab_mutex * - * The role of the slub_lock is to protect the list of all the slabs + * The role of the slab_mutex is to protect the list of all the slabs * and to synchronize major metadata changes to slab cache structures. * * The slab_lock is only used for debugging and on arches that do not @@ -183,10 +183,6 @@ static int kmem_size = sizeof(struct kmem_cache); static struct notifier_block slab_notifier; #endif -/* A list of all slab caches on the system */ -static DECLARE_RWSEM(slub_lock); -static LIST_HEAD(slab_caches); - /* * Tracking user of a slab. */ @@ -3177,11 +3173,11 @@ static inline int kmem_cache_close(struct kmem_cache *s) */ void kmem_cache_destroy(struct kmem_cache *s) { - down_write(&slub_lock); + mutex_lock(&slab_mutex); s->refcount--; if (!s->refcount) { list_del(&s->list); - up_write(&slub_lock); + mutex_unlock(&slab_mutex); if (kmem_cache_close(s)) { printk(KERN_ERR "SLUB %s: %s called for cache that " "still has objects.\n", s->name, __func__); @@ -3191,7 +3187,7 @@ void kmem_cache_destroy(struct kmem_cache *s) rcu_barrier(); sysfs_slab_remove(s); } else - up_write(&slub_lock); + mutex_unlock(&slab_mutex); } EXPORT_SYMBOL(kmem_cache_destroy); @@ -3253,7 +3249,7 @@ static struct kmem_cache *__init create_kmalloc_cache(const char *name, /* * This function is called with IRQs disabled during early-boot on - * single CPU so there's no need to take slub_lock here. + * single CPU so there's no need to take slab_mutex here. */ if (!kmem_cache_open(s, name, size, ARCH_KMALLOC_MINALIGN, flags, NULL)) @@ -3538,10 +3534,10 @@ static int slab_mem_going_offline_callback(void *arg) { struct kmem_cache *s; - down_read(&slub_lock); + mutex_lock(&slab_mutex); list_for_each_entry(s, &slab_caches, list) kmem_cache_shrink(s); - up_read(&slub_lock); + mutex_unlock(&slab_mutex); return 0; } @@ -3562,7 +3558,7 @@ static void slab_mem_offline_callback(void *arg) if (offline_node < 0) return; - down_read(&slub_lock); + mutex_lock(&slab_mutex); list_for_each_entry(s, &slab_caches, list) { n = get_node(s, offline_node); if (n) { @@ -3578,7 +3574,7 @@ static void slab_mem_offline_callback(void *arg) kmem_cache_free(kmem_cache_node, n); } } - up_read(&slub_lock); + mutex_unlock(&slab_mutex); } static int slab_mem_going_online_callback(void *arg) @@ -3601,7 +3597,7 @@ static int slab_mem_going_online_callback(void *arg) * allocate a kmem_cache_node structure in order to bring the node * online. */ - down_read(&slub_lock); + mutex_lock(&slab_mutex); list_for_each_entry(s, &slab_caches, list) { /* * XXX: kmem_cache_alloc_node will fallback to other nodes @@ -3617,7 +3613,7 @@ static int slab_mem_going_online_callback(void *arg) s->node[nid] = n; } out: - up_read(&slub_lock); + mutex_unlock(&slab_mutex); return ret; } @@ -3915,7 +3911,7 @@ struct kmem_cache *__kmem_cache_create(const char *name, size_t size, struct kmem_cache *s; char *n; - down_write(&slub_lock); + mutex_lock(&slab_mutex); s = find_mergeable(size, align, flags, name, ctor); if (s) { s->refcount++; @@ -3930,7 +3926,7 @@ struct kmem_cache *__kmem_cache_create(const char *name, size_t size, s->refcount--; goto err; } - up_write(&slub_lock); + mutex_unlock(&slab_mutex); return s; } @@ -3943,9 +3939,9 @@ struct kmem_cache *__kmem_cache_create(const char *name, size_t size, if (kmem_cache_open(s, n, size, align, flags, ctor)) { list_add(&s->list, &slab_caches); - up_write(&slub_lock); + mutex_unlock(&slab_mutex); if (sysfs_slab_add(s)) { - down_write(&slub_lock); + mutex_lock(&slab_mutex); list_del(&s->list); kfree(n); kfree(s); @@ -3957,7 +3953,7 @@ struct kmem_cache *__kmem_cache_create(const char *name, size_t size, } kfree(n); err: - up_write(&slub_lock); + mutex_unlock(&slab_mutex); return s; } @@ -3978,13 +3974,13 @@ static int __cpuinit slab_cpuup_callback(struct notifier_block *nfb, case CPU_UP_CANCELED_FROZEN: case CPU_DEAD: case CPU_DEAD_FROZEN: - down_read(&slub_lock); + mutex_lock(&slab_mutex); list_for_each_entry(s, &slab_caches, list) { local_irq_save(flags); __flush_cpu_slab(s, cpu); local_irq_restore(flags); } - up_read(&slub_lock); + mutex_unlock(&slab_mutex); break; default: break; @@ -5360,11 +5356,11 @@ static int __init slab_sysfs_init(void) struct kmem_cache *s; int err; - down_write(&slub_lock); + mutex_lock(&slab_mutex); slab_kset = kset_create_and_add("slab", &slab_uevent_ops, kernel_kobj); if (!slab_kset) { - up_write(&slub_lock); + mutex_unlock(&slab_mutex); printk(KERN_ERR "Cannot register slab subsystem.\n"); return -ENOSYS; } @@ -5389,7 +5385,7 @@ static int __init slab_sysfs_init(void) kfree(al); } - up_write(&slub_lock); + mutex_unlock(&slab_mutex); resiliency_test(); return 0; } @@ -5415,7 +5411,7 @@ static void *s_start(struct seq_file *m, loff_t *pos) { loff_t n = *pos; - down_read(&slub_lock); + mutex_lock(&slab_mutex); if (!n) print_slabinfo_header(m); @@ -5429,7 +5425,7 @@ static void *s_next(struct seq_file *m, void *p, loff_t *pos) static void s_stop(struct seq_file *m, void *p) { - up_read(&slub_lock); + mutex_unlock(&slab_mutex); } static int s_show(struct seq_file *m, void *p) -- cgit v0.10.2 From 20cea9683ecc6dd75a80c0dd02dc69c64e95be75 Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Fri, 6 Jul 2012 15:25:13 -0500 Subject: mm, sl[aou]b: Move kmem_cache_create mutex handling to common code Move the mutex handling into the common kmem_cache_create() function. Then we can also move more checks out of SLAB's kmem_cache_create() into the common code. Reviewed-by: Glauber Costa Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg diff --git a/mm/slab.c b/mm/slab.c index fd7dac6..1fcf3ac 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -2228,55 +2228,10 @@ __kmem_cache_create (const char *name, size_t size, size_t align, unsigned long flags, void (*ctor)(void *)) { size_t left_over, slab_size, ralign; - struct kmem_cache *cachep = NULL, *pc; + struct kmem_cache *cachep = NULL; gfp_t gfp; - /* - * Sanity checks... these are all serious usage bugs. - */ - if (!name || in_interrupt() || (size < BYTES_PER_WORD) || - size > KMALLOC_MAX_SIZE) { - printk(KERN_ERR "%s: Early error in slab %s\n", __func__, - name); - BUG(); - } - - /* - * We use cache_chain_mutex to ensure a consistent view of - * cpu_online_mask as well. Please see cpuup_callback - */ - if (slab_is_available()) { - get_online_cpus(); - mutex_lock(&slab_mutex); - } - - list_for_each_entry(pc, &slab_caches, list) { - char tmp; - int res; - - /* - * This happens when the module gets unloaded and doesn't - * destroy its slab cache and no-one else reuses the vmalloc - * area of the module. Print a warning. - */ - res = probe_kernel_address(pc->name, tmp); - if (res) { - printk(KERN_ERR - "SLAB: cache with size %d has lost its name\n", - pc->size); - continue; - } - - if (!strcmp(pc->name, name)) { - printk(KERN_ERR - "kmem_cache_create: duplicate cache %s\n", name); - dump_stack(); - goto oops; - } - } - #if DEBUG - WARN_ON(strchr(name, ' ')); /* It confuses parsers */ #if FORCED_DEBUG /* * Enable redzoning and last user accounting, except for caches with @@ -2495,11 +2450,6 @@ __kmem_cache_create (const char *name, size_t size, size_t align, /* cache setup completed, link it into the list */ list_add(&cachep->list, &slab_caches); -oops: - if (slab_is_available()) { - mutex_unlock(&slab_mutex); - put_online_cpus(); - } return cachep; } diff --git a/mm/slab_common.c b/mm/slab_common.c index 50e1ff1..12637ce 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -11,7 +11,8 @@ #include #include #include - +#include +#include #include #include #include @@ -61,8 +62,46 @@ struct kmem_cache *kmem_cache_create(const char *name, size_t size, size_t align } #endif + get_online_cpus(); + mutex_lock(&slab_mutex); + +#ifdef CONFIG_DEBUG_VM + list_for_each_entry(s, &slab_caches, list) { + char tmp; + int res; + + /* + * This happens when the module gets unloaded and doesn't + * destroy its slab cache and no-one else reuses the vmalloc + * area of the module. Print a warning. + */ + res = probe_kernel_address(s->name, tmp); + if (res) { + printk(KERN_ERR + "Slab cache with size %d has lost its name\n", + s->object_size); + continue; + } + + if (!strcmp(s->name, name)) { + printk(KERN_ERR "kmem_cache_create(%s): Cache name" + " already exists.\n", + name); + dump_stack(); + s = NULL; + goto oops; + } + } + + WARN_ON(strchr(name, ' ')); /* It confuses parsers */ +#endif + s = __kmem_cache_create(name, size, align, flags, ctor); +oops: + mutex_unlock(&slab_mutex); + put_online_cpus(); + #ifdef CONFIG_DEBUG_VM out: #endif diff --git a/mm/slub.c b/mm/slub.c index 8c4fd37..0e0504e 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -3911,7 +3911,6 @@ struct kmem_cache *__kmem_cache_create(const char *name, size_t size, struct kmem_cache *s; char *n; - mutex_lock(&slab_mutex); s = find_mergeable(size, align, flags, name, ctor); if (s) { s->refcount++; @@ -3924,37 +3923,36 @@ struct kmem_cache *__kmem_cache_create(const char *name, size_t size, if (sysfs_slab_alias(s, name)) { s->refcount--; - goto err; + return NULL; } - mutex_unlock(&slab_mutex); return s; } n = kstrdup(name, GFP_KERNEL); if (!n) - goto err; + return NULL; s = kmalloc(kmem_size, GFP_KERNEL); if (s) { if (kmem_cache_open(s, n, size, align, flags, ctor)) { + int r; + list_add(&s->list, &slab_caches); mutex_unlock(&slab_mutex); - if (sysfs_slab_add(s)) { - mutex_lock(&slab_mutex); - list_del(&s->list); - kfree(n); - kfree(s); - goto err; - } - return s; + r = sysfs_slab_add(s); + mutex_lock(&slab_mutex); + + if (!r) + return s; + + list_del(&s->list); + kmem_cache_close(s); } kfree(s); } kfree(n); -err: - mutex_unlock(&slab_mutex); - return s; + return NULL; } #ifdef CONFIG_SMP -- cgit v0.10.2 From 737b719ed6569ffcd015fcdf3039dc7af47af877 Mon Sep 17 00:00:00 2001 From: David Rientjes Date: Mon, 9 Jul 2012 14:00:38 -0700 Subject: mm, slub: ensure irqs are enabled for kmemcheck kmemcheck_alloc_shadow() requires irqs to be enabled, so wait to disable them until after its called for __GFP_WAIT allocations. This fixes a warning for such allocations: WARNING: at kernel/lockdep.c:2739 lockdep_trace_alloc+0x14e/0x1c0() Acked-by: Fengguang Wu Acked-by: Steven Rostedt Tested-by: Fengguang Wu Signed-off-by: David Rientjes Signed-off-by: Pekka Enberg diff --git a/mm/slub.c b/mm/slub.c index 0e0504e..e517d43 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -1299,13 +1299,7 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) stat(s, ORDER_FALLBACK); } - if (flags & __GFP_WAIT) - local_irq_disable(); - - if (!page) - return NULL; - - if (kmemcheck_enabled + if (kmemcheck_enabled && page && !(s->flags & (SLAB_NOTRACK | DEBUG_DEFAULT_FLAGS))) { int pages = 1 << oo_order(oo); @@ -1321,6 +1315,11 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) kmemcheck_mark_unallocated_pages(page, pages); } + if (flags & __GFP_WAIT) + local_irq_disable(); + if (!page) + return NULL; + page->objects = oo_objects(oo); mod_zone_page_state(page_zone(page), (s->flags & SLAB_RECLAIM_ACCOUNT) ? -- cgit v0.10.2 From 44a8bdea19dff1abcdc4528e5f7e038b18ee5255 Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Tue, 10 Jul 2012 18:31:05 -0500 Subject: slob: Fix early boot kernel crash Commit fd3142a59af2012a7c5dc72ec97a4935ff1c5fc6 broke slob since a piece of a change for a later patch slipped into it. Fengguang Wu writes: The commit crashes the kernel w/o any dmesg output (the attached one is created by the script as a summary for that run). This is very reproducible in kvm for the attached config. Reported-by: Fengguang Wu Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg diff --git a/mm/slob.c b/mm/slob.c index 0111e0d..45d4ca7 100644 --- a/mm/slob.c +++ b/mm/slob.c @@ -518,7 +518,7 @@ struct kmem_cache *__kmem_cache_create(const char *name, size_t size, if (c) { c->name = name; - c->size = c->object_size; + c->size = size; if (flags & SLAB_DESTROY_BY_RCU) { /* leave room for rcu footer at the end of object */ c->size += sizeof(struct slob_rcu); -- cgit v0.10.2 From 73a1180e140d45cb9ef5fbab103d3bbfc4c84606 Mon Sep 17 00:00:00 2001 From: Shuah Khan Date: Fri, 13 Jul 2012 17:12:05 -0600 Subject: mm: Fix build warning in kmem_cache_create() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The label oops is used in CONFIG_DEBUG_VM ifdef block and is defined outside ifdef CONFIG_DEBUG_VM block. This results in the following build warning when built with CONFIG_DEBUG_VM disabled. Fix to move label oops definition to inside a CONFIG_DEBUG_VM block. mm/slab_common.c: In function ‘kmem_cache_create’: mm/slab_common.c:101:1: warning: label ‘oops’ defined but not used [-Wunused-label] Signed-off-by: Shuah Khan Signed-off-by: Pekka Enberg diff --git a/mm/slab_common.c b/mm/slab_common.c index 12637ce..aa3ca5b 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -98,7 +98,9 @@ struct kmem_cache *kmem_cache_create(const char *name, size_t size, size_t align s = __kmem_cache_create(name, size, align, flags, ctor); +#ifdef CONFIG_DEBUG_VM oops: +#endif mutex_unlock(&slab_mutex); put_online_cpus(); -- cgit v0.10.2