Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
| 2 | * linux/mm/vmscan.c |
| 3 | * |
| 4 | * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds |
| 5 | * |
| 6 | * Swap reorganised 29.12.95, Stephen Tweedie. |
| 7 | * kswapd added: 7.1.96 sct |
| 8 | * Removed kswapd_ctl limits, and swap out as many pages as needed |
| 9 | * to bring the system back to freepages.high: 2.4.97, Rik van Riel. |
| 10 | * Zone aware kswapd started 02/00, Kanoj Sarcar ([email protected]). |
| 11 | * Multiqueue VM started 5.8.00, Rik van Riel. |
| 12 | */ |
| 13 | |
Mitchel Humpherys | b1de0d1 | 2014-06-06 14:38:30 -0700 | [diff] [blame] | 14 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt |
| 15 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 16 | #include <linux/mm.h> |
| 17 | #include <linux/module.h> |
Tejun Heo | 5a0e3ad | 2010-03-24 17:04:11 +0900 | [diff] [blame] | 18 | #include <linux/gfp.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 19 | #include <linux/kernel_stat.h> |
| 20 | #include <linux/swap.h> |
| 21 | #include <linux/pagemap.h> |
| 22 | #include <linux/init.h> |
| 23 | #include <linux/highmem.h> |
Anton Vorontsov | 70ddf63 | 2013-04-29 15:08:31 -0700 | [diff] [blame] | 24 | #include <linux/vmpressure.h> |
Andrew Morton | e129b5c | 2006-09-27 01:50:00 -0700 | [diff] [blame] | 25 | #include <linux/vmstat.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 26 | #include <linux/file.h> |
| 27 | #include <linux/writeback.h> |
| 28 | #include <linux/blkdev.h> |
| 29 | #include <linux/buffer_head.h> /* for try_to_release_page(), |
| 30 | buffer_heads_over_limit */ |
| 31 | #include <linux/mm_inline.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 32 | #include <linux/backing-dev.h> |
| 33 | #include <linux/rmap.h> |
| 34 | #include <linux/topology.h> |
| 35 | #include <linux/cpu.h> |
| 36 | #include <linux/cpuset.h> |
Mel Gorman | 3e7d344 | 2011-01-13 15:45:56 -0800 | [diff] [blame] | 37 | #include <linux/compaction.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 38 | #include <linux/notifier.h> |
| 39 | #include <linux/rwsem.h> |
Rafael J. Wysocki | 248a030 | 2006-03-22 00:09:04 -0800 | [diff] [blame] | 40 | #include <linux/delay.h> |
Yasunori Goto | 3218ae1 | 2006-06-27 02:53:33 -0700 | [diff] [blame] | 41 | #include <linux/kthread.h> |
Nigel Cunningham | 7dfb710 | 2006-12-06 20:34:23 -0800 | [diff] [blame] | 42 | #include <linux/freezer.h> |
Balbir Singh | 66e1707 | 2008-02-07 00:13:56 -0800 | [diff] [blame] | 43 | #include <linux/memcontrol.h> |
Keika Kobayashi | 873b477 | 2008-07-25 01:48:52 -0700 | [diff] [blame] | 44 | #include <linux/delayacct.h> |
Lee Schermerhorn | af936a1 | 2008-10-18 20:26:53 -0700 | [diff] [blame] | 45 | #include <linux/sysctl.h> |
KOSAKI Motohiro | 929bea7 | 2011-04-14 15:22:12 -0700 | [diff] [blame] | 46 | #include <linux/oom.h> |
Linus Torvalds | 268bb0c | 2011-05-20 12:50:29 -0700 | [diff] [blame] | 47 | #include <linux/prefetch.h> |
Mitchel Humpherys | b1de0d1 | 2014-06-06 14:38:30 -0700 | [diff] [blame] | 48 | #include <linux/printk.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 49 | |
| 50 | #include <asm/tlbflush.h> |
| 51 | #include <asm/div64.h> |
| 52 | |
| 53 | #include <linux/swapops.h> |
Rafael Aquini | 117aad1 | 2013-09-30 13:45:16 -0700 | [diff] [blame] | 54 | #include <linux/balloon_compaction.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 55 | |
Nick Piggin | 0f8053a | 2006-03-22 00:08:33 -0800 | [diff] [blame] | 56 | #include "internal.h" |
| 57 | |
Mel Gorman | 33906bc | 2010-08-09 17:19:16 -0700 | [diff] [blame] | 58 | #define CREATE_TRACE_POINTS |
| 59 | #include <trace/events/vmscan.h> |
| 60 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 61 | struct scan_control { |
KOSAKI Motohiro | 22fba33 | 2009-12-14 17:59:10 -0800 | [diff] [blame] | 62 | /* How many pages shrink_list() should reclaim */ |
| 63 | unsigned long nr_to_reclaim; |
| 64 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 65 | /* This context's GFP mask */ |
Al Viro | 6daa0e2 | 2005-10-21 03:18:50 -0400 | [diff] [blame] | 66 | gfp_t gfp_mask; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 67 | |
Johannes Weiner | ee814fe | 2014-08-06 16:06:19 -0700 | [diff] [blame] | 68 | /* Allocation order */ |
Andy Whitcroft | 5ad333e | 2007-07-17 04:03:16 -0700 | [diff] [blame] | 69 | int order; |
Balbir Singh | 66e1707 | 2008-02-07 00:13:56 -0800 | [diff] [blame] | 70 | |
Johannes Weiner | ee814fe | 2014-08-06 16:06:19 -0700 | [diff] [blame] | 71 | /* |
| 72 | * Nodemask of nodes allowed by the caller. If NULL, all nodes |
| 73 | * are scanned. |
| 74 | */ |
| 75 | nodemask_t *nodemask; |
Konstantin Khlebnikov | 9e3b2f8 | 2012-05-29 15:06:57 -0700 | [diff] [blame] | 76 | |
KOSAKI Motohiro | 5f53e76 | 2010-05-24 14:32:37 -0700 | [diff] [blame] | 77 | /* |
Johannes Weiner | f16015f | 2012-01-12 17:17:52 -0800 | [diff] [blame] | 78 | * The memory cgroup that hit its limit and as a result is the |
| 79 | * primary target of this reclaim invocation. |
| 80 | */ |
| 81 | struct mem_cgroup *target_mem_cgroup; |
Balbir Singh | 66e1707 | 2008-02-07 00:13:56 -0800 | [diff] [blame] | 82 | |
Johannes Weiner | ee814fe | 2014-08-06 16:06:19 -0700 | [diff] [blame] | 83 | /* Scan (total_size >> priority) pages at once */ |
| 84 | int priority; |
| 85 | |
| 86 | unsigned int may_writepage:1; |
| 87 | |
| 88 | /* Can mapped pages be reclaimed? */ |
| 89 | unsigned int may_unmap:1; |
| 90 | |
| 91 | /* Can pages be swapped as part of reclaim? */ |
| 92 | unsigned int may_swap:1; |
| 93 | |
| 94 | unsigned int hibernation_mode:1; |
| 95 | |
| 96 | /* One of the zones is ready for compaction */ |
| 97 | unsigned int compaction_ready:1; |
| 98 | |
| 99 | /* Incremented by the number of inactive pages that were scanned */ |
| 100 | unsigned long nr_scanned; |
| 101 | |
| 102 | /* Number of pages freed so far during a call to shrink_zones() */ |
| 103 | unsigned long nr_reclaimed; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 104 | }; |
| 105 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 106 | #define lru_to_page(_head) (list_entry((_head)->prev, struct page, lru)) |
| 107 | |
| 108 | #ifdef ARCH_HAS_PREFETCH |
| 109 | #define prefetch_prev_lru_page(_page, _base, _field) \ |
| 110 | do { \ |
| 111 | if ((_page)->lru.prev != _base) { \ |
| 112 | struct page *prev; \ |
| 113 | \ |
| 114 | prev = lru_to_page(&(_page->lru)); \ |
| 115 | prefetch(&prev->_field); \ |
| 116 | } \ |
| 117 | } while (0) |
| 118 | #else |
| 119 | #define prefetch_prev_lru_page(_page, _base, _field) do { } while (0) |
| 120 | #endif |
| 121 | |
| 122 | #ifdef ARCH_HAS_PREFETCHW |
| 123 | #define prefetchw_prev_lru_page(_page, _base, _field) \ |
| 124 | do { \ |
| 125 | if ((_page)->lru.prev != _base) { \ |
| 126 | struct page *prev; \ |
| 127 | \ |
| 128 | prev = lru_to_page(&(_page |