Change memory reclaim

 - Spin up a background thread to call the shrinkers every 1 second
 - Memory allocations will only call reclaim after a failed allocation,
   not every single time

This will be a major performance boost on allocation intensive
workloads.

Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>

diff --git a/include/linux/slab.h b/include/linux/slab.h
index 17fe235eef1786c603cba131ebd9e83480db4b27..cf48570c1580e0ea092a4380bbc2a650829b496c 100644 (file)
--- a/include/linux/slab.h
+++ b/include/linux/slab.h
@@ -20,12 +20,10 @@
 
 static inline void *kmalloc(size_t size, gfp_t flags)
 {
-       unsigned i = 0;
+       unsigned i;
        void *p;
 
-       do {
-               run_shrinkers(flags, i != 0);
-
+       for (i = 0; i < 10; i++) {
                if (size) {
                        size_t alignment = min(rounddown_pow_of_two(size), (size_t)PAGE_SIZE);
                        alignment = max(sizeof(void *), alignment);
@@ -34,9 +32,15 @@ static inline void *kmalloc(size_t size, gfp_t flags)
                } else {
                        p = malloc(0);
                }
-               if (p && (flags & __GFP_ZERO))
-                       memset(p, 0, size);
-       } while (!p && i++ < 10);
+
+               if (p) {
+                       if (flags & __GFP_ZERO)
+                               memset(p, 0, size);
+                       break;
+               }
+
+               run_shrinkers(flags, true);
+       }
 
        return p;
 }
@@ -93,16 +97,20 @@ static inline void *krealloc_array(void *p, size_t new_n, size_t new_size, gfp_t
 static inline struct page *alloc_pages(gfp_t flags, unsigned int order)
 {
        size_t size = PAGE_SIZE << order;
-       unsigned i = 0;
+       unsigned i;
        void *p;
 
-       do {
-               run_shrinkers(flags, i != 0);
-
+       for (i = 0; i < 10; i++) {
                p = aligned_alloc(PAGE_SIZE, size);
-               if (p && (flags & __GFP_ZERO))
-                       memset(p, 0, size);
-       } while (!p && i++ < 10);
+
+               if (p) {
+                       if (flags & __GFP_ZERO)
+                               memset(p, 0, size);
+                       break;
+               }
+
+               run_shrinkers(flags, true);
+       }
 
        return p;
 }
@@ -193,20 +201,24 @@ static inline struct kmem_cache *kmem_cache_create(size_t obj_size)
 
 #define vfree(p)               free(p)
 
-static inline void *__vmalloc(unsigned long size, gfp_t gfp_mask)
+static inline void *__vmalloc(unsigned long size, gfp_t flags)
 {
-       unsigned i = 0;
+       unsigned i;
        void *p;
 
        size = round_up(size, PAGE_SIZE);
 
-       do {
-               run_shrinkers(gfp_mask, i != 0);
-
+       for (i = 0; i < 10; i++) {
                p = aligned_alloc(PAGE_SIZE, size);
-               if (p && gfp_mask & __GFP_ZERO)
-                       memset(p, 0, size);
-       } while (!p && i++ < 10);
+
+               if (p) {
+                       if (flags & __GFP_ZERO)
+                               memset(p, 0, size);
+                       break;
+               }
+
+               run_shrinkers(flags, true);
+       }
 
        return p;
 }

diff --git a/linux/shrinker.c b/linux/shrinker.c
index 23e288d845cb67da8030672e58038cbe5b49d2df..0b5715b36a00ee210dfcd6b53c3e53d6e706b196 100644 (file)
--- a/linux/shrinker.c
+++ b/linux/shrinker.c
@@ -1,6 +1,7 @@
 
 #include <stdio.h>
 
+#include <linux/kthread.h>
 #include <linux/list.h>
 #include <linux/mm.h>
 #include <linux/mutex.h>
@@ -126,3 +127,31 @@ void run_shrinkers(gfp_t gfp_mask, bool allocation_failed)
        }
        mutex_unlock(&shrinker_lock);
 }
+
+static int shrinker_thread(void *arg)
+{
+       while (!kthread_should_stop()) {
+               sleep(1);
+               run_shrinkers(GFP_KERNEL, false);
+       }
+
+       return 0;
+}
+
+struct task_struct *shrinker_task;
+
+__attribute__((constructor(103)))
+static void shrinker_thread_init(void)
+{
+       shrinker_task = kthread_run(shrinker_thread, NULL, "shrinkers");
+       BUG_ON(IS_ERR(shrinker_task));
+}
+
+__attribute__((destructor(103)))
+static void shrinker_thread_exit(void)
+{
+       int ret = kthread_stop(shrinker_task);
+       BUG_ON(ret);
+
+       shrinker_task = NULL;
+}