1 files changed, 419 insertions, 0 deletions
diff --git a/debian/patches-rt/0002-mm-swap-Add-static-key-dependent-pagevec-locking.patch b/debian/patches-rt/0002-mm-swap-Add-static-key-dependent-pagevec-locking.patch
new file mode 100644
index 000000000..c4f2d5a93
--- /dev/null
+++ b/debian/patches-rt/0002-mm-swap-Add-static-key-dependent-pagevec-locking.patch
@@ -0,0 +1,419 @@
+From: Thomas Gleixner <tglx@linutronix.de>
+Date: Thu, 18 Apr 2019 11:09:05 +0200
+Subject: [PATCH 2/4] mm/swap: Add static key dependent pagevec locking
+Origin: https://www.kernel.org/pub/linux/kernel/projects/rt/5.4/older/patches-5.4.3-rt1.tar.xz
+
+The locking of struct pagevec is done by disabling preemption. In case the
+struct has be accessed form interrupt context then interrupts are
+disabled. This means the struct can only be accessed locally from the
+CPU. There is also no lockdep coverage which would scream during if it
+accessed from wrong context.
+
+Create struct swap_pagevec which contains of a pagevec member and a
+spin_lock_t. Introduce a static key, which changes the locking behavior
+only if the key is set in the following way: Before the struct is accessed
+the spin_lock has to be acquired instead of using preempt_disable(). Since
+the struct is used CPU-locally there is no spinning on the lock but the
+lock is acquired immediately. If the struct is accessed from interrupt
+context, spin_lock_irqsave() is used.
+
+No functional change yet because static key is not enabled.
+
+[anna-maria: introduce static key]
+Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
+Signed-off-by: Anna-Maria Gleixner <anna-maria@linutronix.de>
+Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
+---
+ mm/compaction.c |   14 ++-
+ mm/internal.h   |    2 
+ mm/swap.c       |  202 +++++++++++++++++++++++++++++++++++++++++++++-----------
+ 3 files changed, 176 insertions(+), 42 deletions(-)
+
+--- a/mm/compaction.c
++++ b/mm/compaction.c
+@@ -2244,10 +2244,16 @@ compact_zone(struct compact_control *cc,
+ 				block_start_pfn(cc->migrate_pfn, cc->order);
+ 
+ 			if (last_migrated_pfn < current_block_start) {
+-				cpu = get_cpu();
+-				lru_add_drain_cpu(cpu);
+-				drain_local_pages(cc->zone);
+-				put_cpu();
++				if (static_branch_likely(&use_pvec_lock)) {
++					cpu = raw_smp_processor_id();
++					lru_add_drain_cpu(cpu);
++					drain_cpu_pages(cpu, cc->zone);
++				} else {
++					cpu = get_cpu();
++					lru_add_drain_cpu(cpu);
++					drain_local_pages(cc->zone);
++					put_cpu();
++				}
+ 				/* No more flushing until we migrate again */
+ 				last_migrated_pfn = 0;
+ 			}
+--- a/mm/internal.h
++++ b/mm/internal.h
+@@ -32,6 +32,8 @@
+ /* Do not use these with a slab allocator */
+ #define GFP_SLAB_BUG_MASK (__GFP_DMA32|__GFP_HIGHMEM|~__GFP_BITS_MASK)
+ 
++extern struct static_key_false use_pvec_lock;
++
+ void page_writeback_init(void);
+ 
+ vm_fault_t do_swap_page(struct vm_fault *vmf);
+--- a/mm/swap.c
++++ b/mm/swap.c
+@@ -44,15 +44,108 @@
+ /* How many pages do we try to swap or page in/out together? */
+ int page_cluster;
+ 
+-static DEFINE_PER_CPU(struct pagevec, lru_add_pvec);
+-static DEFINE_PER_CPU(struct pagevec, lru_rotate_pvecs);
+-static DEFINE_PER_CPU(struct pagevec, lru_deactivate_file_pvecs);
+-static DEFINE_PER_CPU(struct pagevec, lru_deactivate_pvecs);
+-static DEFINE_PER_CPU(struct pagevec, lru_lazyfree_pvecs);
++DEFINE_STATIC_KEY_FALSE(use_pvec_lock);
++
++struct swap_pagevec {
++	spinlock_t	lock;
++	struct pagevec	pvec;
++};
++
++#define DEFINE_PER_CPU_PAGEVEC(lvar)				\
++	DEFINE_PER_CPU(struct swap_pagevec, lvar) = {		\
++		.lock = __SPIN_LOCK_UNLOCKED((lvar).lock) }
++
++static DEFINE_PER_CPU_PAGEVEC(lru_add_pvec);
++static DEFINE_PER_CPU_PAGEVEC(lru_rotate_pvecs);
++static DEFINE_PER_CPU_PAGEVEC(lru_deactivate_file_pvecs);
++static DEFINE_PER_CPU_PAGEVEC(lru_deactivate_pvecs);
++static DEFINE_PER_CPU_PAGEVEC(lru_lazyfree_pvecs);
+ #ifdef CONFIG_SMP
+-static DEFINE_PER_CPU(struct pagevec, activate_page_pvecs);
++static DEFINE_PER_CPU_PAGEVEC(activate_page_pvecs);
+ #endif
+ 
++static inline
++struct swap_pagevec *lock_swap_pvec(struct swap_pagevec __percpu *p)
++{
++	struct swap_pagevec *swpvec;
++
++	if (static_branch_likely(&use_pvec_lock)) {
++		swpvec = raw_cpu_ptr(p);
++
++		spin_lock(&swpvec->lock);
++	} else {
++		swpvec = &get_cpu_var(*p);
++	}
++	return swpvec;
++}
++
++static inline struct swap_pagevec *
++lock_swap_pvec_cpu(struct swap_pagevec __percpu *p, int cpu)
++{
++	struct swap_pagevec *swpvec = per_cpu_ptr(p, cpu);
++
++	if (static_branch_likely(&use_pvec_lock))
++		spin_lock(&swpvec->lock);
++
++	return swpvec;
++}
++
++static inline struct swap_pagevec *
++lock_swap_pvec_irqsave(struct swap_pagevec __percpu *p, unsigned long *flags)
++{
++	struct swap_pagevec *swpvec;
++
++	if (static_branch_likely(&use_pvec_lock)) {
++		swpvec = raw_cpu_ptr(p);
++
++		spin_lock_irqsave(&swpvec->lock, (*flags));
++	} else {
++		local_irq_save(*flags);
++
++		swpvec = this_cpu_ptr(p);
++	}
++	return swpvec;
++}
++
++static inline struct swap_pagevec *
++lock_swap_pvec_cpu_irqsave(struct swap_pagevec __percpu *p, int cpu,
++			   unsigned long *flags)
++{
++	struct swap_pagevec *swpvec = per_cpu_ptr(p, cpu);
++
++	if (static_branch_likely(&use_pvec_lock))
++		spin_lock_irqsave(&swpvec->lock, *flags);
++	else
++		local_irq_save(*flags);
++
++	return swpvec;
++}
++
++static inline void unlock_swap_pvec(struct swap_pagevec *swpvec,
++				    struct swap_pagevec __percpu *p)
++{
++	if (static_branch_likely(&use_pvec_lock))
++		spin_unlock(&swpvec->lock);
++	else
++		put_cpu_var(*p);
++
++}
++
++static inline void unlock_swap_pvec_cpu(struct swap_pagevec *swpvec)
++{
++	if (static_branch_likely(&use_pvec_lock))
++		spin_unlock(&swpvec->lock);
++}
++
++static inline void
++unlock_swap_pvec_irqrestore(struct swap_pagevec *swpvec, unsigned long flags)
++{
++	if (static_branch_likely(&use_pvec_lock))
++		spin_unlock_irqrestore(&swpvec->lock, flags);
++	else
++		local_irq_restore(flags);
++}
++
+ /*
+  * This path almost never happens for VM activity - pages are normally
+  * freed via pagevecs.  But it gets used by networking.
+@@ -250,15 +343,17 @@ void rotate_reclaimable_page(struct page
+ {
+ 	if (!PageLocked(page) && !PageDirty(page) &&
+ 	    !PageUnevictable(page) && PageLRU(page)) {
++		struct swap_pagevec *swpvec;
+ 		struct pagevec *pvec;
+ 		unsigned long flags;
+ 
+ 		get_page(page);
+-		local_irq_save(flags);
+-		pvec = this_cpu_ptr(&lru_rotate_pvecs);
++
++		swpvec = lock_swap_pvec_irqsave(&lru_rotate_pvecs, &flags);
++		pvec = &swpvec->pvec;
+ 		if (!pagevec_add(pvec, page) || PageCompound(page))
+ 			pagevec_move_tail(pvec);
+-		local_irq_restore(flags);
++		unlock_swap_pvec_irqrestore(swpvec, flags);
+ 	}
+ }
+ 
+@@ -293,27 +388,32 @@ static void __activate_page(struct page
+ #ifdef CONFIG_SMP
+ static void activate_page_drain(int cpu)
+ {
+-	struct pagevec *pvec = &per_cpu(activate_page_pvecs, cpu);
++	struct swap_pagevec *swpvec = lock_swap_pvec_cpu(&activate_page_pvecs, cpu);
++	struct pagevec *pvec = &swpvec->pvec;
+ 
+ 	if (pagevec_count(pvec))
+ 		pagevec_lru_move_fn(pvec, __activate_page, NULL);
++	unlock_swap_pvec_cpu(swpvec);
+ }
+ 
+ static bool need_activate_page_drain(int cpu)
+ {
+-	return pagevec_count(&per_cpu(activate_page_pvecs, cpu)) != 0;
++	return pagevec_count(per_cpu_ptr(&activate_page_pvecs.pvec, cpu)) != 0;
+ }
+ 
+ void activate_page(struct page *page)
+ {
+ 	page = compound_head(page);
+ 	if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {
+-		struct pagevec *pvec = &get_cpu_var(activate_page_pvecs);
++		struct swap_pagevec *swpvec;
++		struct pagevec *pvec;
+ 
+ 		get_page(page);
++		swpvec = lock_swap_pvec(&activate_page_pvecs);
++		pvec = &swpvec->pvec;
+ 		if (!pagevec_add(pvec, page) || PageCompound(page))
+ 			pagevec_lru_move_fn(pvec, __activate_page, NULL);
+-		put_cpu_var(activate_page_pvecs);
++		unlock_swap_pvec(swpvec, &activate_page_pvecs);
+ 	}
+ }
+ 
+@@ -335,7 +435,8 @@ void activate_page(struct page *page)
+ 
+ static void __lru_cache_activate_page(struct page *page)
+ {
+-	struct pagevec *pvec = &get_cpu_var(lru_add_pvec);
++	struct swap_pagevec *swpvec = lock_swap_pvec(&lru_add_pvec);
++	struct pagevec *pvec = &swpvec->pvec;
+ 	int i;
+ 
+ 	/*
+@@ -357,7 +458,7 @@ static void __lru_cache_activate_page(st
+ 		}
+ 	}
+ 
+-	put_cpu_var(lru_add_pvec);
++	unlock_swap_pvec(swpvec, &lru_add_pvec);
+ }
+ 
+ /*
+@@ -399,12 +500,13 @@ EXPORT_SYMBOL(mark_page_accessed);
+ 
+ static void __lru_cache_add(struct page *page)
+ {
+-	struct pagevec *pvec = &get_cpu_var(lru_add_pvec);
++	struct swap_pagevec *swpvec = lock_swap_pvec(&lru_add_pvec);
++	struct pagevec *pvec = &swpvec->pvec;
+ 
+ 	get_page(page);
+ 	if (!pagevec_add(pvec, page) || PageCompound(page))
+ 		__pagevec_lru_add(pvec);
+-	put_cpu_var(lru_add_pvec);
++	unlock_swap_pvec(swpvec, &lru_add_pvec);
+ }
+ 
+ /**
+@@ -588,32 +690,40 @@ static void lru_lazyfree_fn(struct page
+  */
+ void lru_add_drain_cpu(int cpu)
+ {
+-	struct pagevec *pvec = &per_cpu(lru_add_pvec, cpu);
++	struct swap_pagevec *swpvec = lock_swap_pvec_cpu(&lru_add_pvec, cpu);
++	struct pagevec *pvec = &swpvec->pvec;
++	unsigned long flags;
+ 
+ 	if (pagevec_count(pvec))
+ 		__pagevec_lru_add(pvec);
++	unlock_swap_pvec_cpu(swpvec);
+ 
+-	pvec = &per_cpu(lru_rotate_pvecs, cpu);
++	swpvec = lock_swap_pvec_cpu_irqsave(&lru_rotate_pvecs, cpu, &flags);
++	pvec = &swpvec->pvec;
+ 	if (pagevec_count(pvec)) {
+-		unsigned long flags;
+ 
+ 		/* No harm done if a racing interrupt already did this */
+-		local_irq_save(flags);
+ 		pagevec_move_tail(pvec);
+-		local_irq_restore(flags);
+ 	}
++	unlock_swap_pvec_irqrestore(swpvec, flags);
+ 
+-	pvec = &per_cpu(lru_deactivate_file_pvecs, cpu);
++	swpvec = lock_swap_pvec_cpu(&lru_deactivate_file_pvecs, cpu);
++	pvec = &swpvec->pvec;
+ 	if (pagevec_count(pvec))
+ 		pagevec_lru_move_fn(pvec, lru_deactivate_file_fn, NULL);
++	unlock_swap_pvec_cpu(swpvec);
+ 
+-	pvec = &per_cpu(lru_deactivate_pvecs, cpu);
++	swpvec = lock_swap_pvec_cpu(&lru_deactivate_pvecs, cpu);
++	pvec = &swpvec->pvec;
+ 	if (pagevec_count(pvec))
+ 		pagevec_lru_move_fn(pvec, lru_deactivate_fn, NULL);
++	unlock_swap_pvec_cpu(swpvec);
+ 
+-	pvec = &per_cpu(lru_lazyfree_pvecs, cpu);
++	swpvec = lock_swap_pvec_cpu(&lru_lazyfree_pvecs, cpu);
++	pvec = &swpvec->pvec;
+ 	if (pagevec_count(pvec))
+ 		pagevec_lru_move_fn(pvec, lru_lazyfree_fn, NULL);
++	unlock_swap_pvec_cpu(swpvec);
+ 
+ 	activate_page_drain(cpu);
+ }
+@@ -628,6 +738,9 @@ void lru_add_drain_cpu(int cpu)
+  */
+ void deactivate_file_page(struct page *page)
+ {
++	struct swap_pagevec *swpvec;
++	struct pagevec *pvec;
++
+ 	/*
+ 	 * In a workload with many unevictable page such as mprotect,
+ 	 * unevictable page deactivation for accelerating reclaim is pointless.
+@@ -636,11 +749,12 @@ void deactivate_file_page(struct page *p
+ 		return;
+ 
+ 	if (likely(get_page_unless_zero(page))) {
+-		struct pagevec *pvec = &get_cpu_var(lru_deactivate_file_pvecs);
++		swpvec = lock_swap_pvec(&lru_deactivate_file_pvecs);
++		pvec = &swpvec->pvec;
+ 
+ 		if (!pagevec_add(pvec, page) || PageCompound(page))
+ 			pagevec_lru_move_fn(pvec, lru_deactivate_file_fn, NULL);
+-		put_cpu_var(lru_deactivate_file_pvecs);
++		unlock_swap_pvec(swpvec, &lru_deactivate_file_pvecs);
+ 	}
+ }
+ 
+@@ -655,12 +769,16 @@ void deactivate_file_page(struct page *p
+ void deactivate_page(struct page *page)
+ {
+ 	if (PageLRU(page) && PageActive(page) && !PageUnevictable(page)) {
+-		struct pagevec *pvec = &get_cpu_var(lru_deactivate_pvecs);
++		struct swap_pagevec *swpvec;
++		struct pagevec *pvec;
++
++		swpvec = lock_swap_pvec(&lru_deactivate_pvecs);
++		pvec = &swpvec->pvec;
+ 
+ 		get_page(page);
+ 		if (!pagevec_add(pvec, page) || PageCompound(page))
+ 			pagevec_lru_move_fn(pvec, lru_deactivate_fn, NULL);
+-		put_cpu_var(lru_deactivate_pvecs);
++		unlock_swap_pvec(swpvec, &lru_deactivate_pvecs);
+ 	}
+ }
+ 
+@@ -673,21 +791,29 @@ void deactivate_page(struct page *page)
+  */
+ void mark_page_lazyfree(struct page *page)
+ {
++	struct swap_pagevec *swpvec;
++	struct pagevec *pvec;
++
+ 	if (PageLRU(page) && PageAnon(page) && PageSwapBacked(page) &&
+ 	    !PageSwapCache(page) && !PageUnevictable(page)) {
+-		struct pagevec *pvec = &get_cpu_var(lru_lazyfree_pvecs);
++		swpvec = lock_swap_pvec(&lru_lazyfree_pvecs);
++		pvec = &swpvec->pvec;
+ 
+ 		get_page(page);
+ 		if (!pagevec_add(pvec, page) || PageCompound(page))
+ 			pagevec_lru_move_fn(pvec, lru_lazyfree_fn, NULL);
+-		put_cpu_var(lru_lazyfree_pvecs);
++		unlock_swap_pvec(swpvec, &lru_lazyfree_pvecs);
+ 	}
+ }
+ 
+ void lru_add_drain(void)
+ {
+-	lru_add_drain_cpu(get_cpu());
+-	put_cpu();
++	if (static_branch_likely(&use_pvec_lock)) {
++		lru_add_drain_cpu(raw_smp_processor_id());
++	} else {
++		lru_add_drain_cpu(get_cpu());
++		put_cpu();
++	}
+ }
+ 
+ #ifdef CONFIG_SMP
+@@ -725,11 +851,11 @@ void lru_add_drain_all(void)
+ 	for_each_online_cpu(cpu) {
+ 		struct work_struct *work = &per_cpu(lru_add_drain_work, cpu);
+ 
+-		if (pagevec_count(&per_cpu(lru_add_pvec, cpu)) ||
+-		    pagevec_count(&per_cpu(lru_rotate_pvecs, cpu)) ||
+-		    pagevec_count(&per_cpu(lru_deactivate_file_pvecs, cpu)) ||
+-		    pagevec_count(&per_cpu(lru_deactivate_pvecs, cpu)) ||
+-		    pagevec_count(&per_cpu(lru_lazyfree_pvecs, cpu)) ||
++		if (pagevec_count(&per_cpu(lru_add_pvec.pvec, cpu)) ||
++		    pagevec_count(&per_cpu(lru_rotate_pvecs.pvec, cpu)) ||
++		    pagevec_count(&per_cpu(lru_deactivate_file_pvecs.pvec, cpu)) ||
++		    pagevec_count(&per_cpu(lru_deactivate_pvecs.pvec, cpu)) ||
++		    pagevec_count(&per_cpu(lru_lazyfree_pvecs.pvec, cpu)) ||
+ 		    need_activate_page_drain(cpu)) {
+ 			INIT_WORK(work, lru_add_drain_per_cpu);
+ 			queue_work_on(cpu, mm_percpu_wq, work);