diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c index 651768313b421e..9a42a11c7319cd 100644 --- a/arch/x86/xen/enlighten.c +++ b/arch/x86/xen/enlighten.c @@ -10,8 +10,6 @@ #include #include -#include -#include #include #include #include @@ -442,66 +440,22 @@ void __init xen_add_extra_mem(unsigned long start_pfn, unsigned long n_pfns) } #ifdef CONFIG_XEN_UNPOPULATED_ALLOC -/* - * How much of the extra memory dom0 keeps on the balloon, to grow into up - * to the ceiling Xen set for it (dom0_mem=...,max:). Without a max: Xen - * reports UINT_MAX and dom0 keeps nothing, so growing stays opt-in. The - * last memory section always stays with the mapping pool, so that the - * grant table and other early users never fall back to holes in the - * physical map before PCI devices have claimed their MMIO windows. - */ -static unsigned long __init xen_dom0_growth_pages(unsigned long extra_pages) -{ - domid_t domid = DOMID_SELF; - long max_pages, cur_pages; - - if (!xen_initial_domain() || extra_pages <= PAGES_PER_SECTION) - return 0; - - max_pages = HYPERVISOR_memory_op(XENMEM_maximum_reservation, &domid); - cur_pages = HYPERVISOR_memory_op(XENMEM_current_reservation, &domid); - if (cur_pages <= 0 || max_pages <= cur_pages || max_pages >= UINT_MAX) - return 0; - - return min_t(unsigned long, max_pages - cur_pages, - extra_pages - PAGES_PER_SECTION); -} - int __init arch_xen_unpopulated_init(struct resource **res) { - unsigned long extra_pages = 0, growth_pages; unsigned int i; if (!xen_domain()) return -ENODEV; - /* Must be set strictly before calling xen_free_unpopulated_pages(). */ *res = &iomem_resource; - for (i = 0; i < XEN_EXTRA_MEM_MAX_REGIONS; i++) - extra_pages += xen_extra_mem[i].n_pfns; - growth_pages = xen_dom0_growth_pages(extra_pages); - /* - * Initialize with pages from the extra memory regions (see - * arch/x86/xen/setup.c), less what dom0 keeps to grow into. Every - * region gives up the same fraction: on PVH that spreads dom0's - * growth over the NUMA nodes the regions belong to, and leaves each - * node's pool its share of the rest. + * Initialize with the extra memory regions (see + * arch/x86/xen/setup.c). */ for (i = 0; i < XEN_EXTRA_MEM_MAX_REGIONS; i++) { - struct xen_memory_region *region = &xen_extra_mem[i]; - unsigned long keep = 0, j; - - if (growth_pages) - keep = mult_frac(region->n_pfns, growth_pages, - extra_pages); - - for (j = keep; j < region->n_pfns; j++) { - struct page *pg = pfn_to_page(region->start_pfn + j); - - xen_free_unpopulated_pages(1, &pg); - } + xen_unpopulated_add_range(xen_extra_mem[i].start_pfn, + xen_extra_mem[i].n_pfns); /* * Account for the region being in the physmap but unpopulated. @@ -509,9 +463,9 @@ int __init arch_xen_unpopulated_init(struct resource **res) * driver to know how much of the physmap is unpopulated and * set an accurate initial memory target. */ - xen_unpopulated_pages += region->n_pfns - keep; - /* Only the part dom0 keeps is added to the balloon driver. */ - region->n_pfns = keep; + xen_unpopulated_pages += xen_extra_mem[i].n_pfns; + /* Zero so region is not also added to the balloon driver. */ + xen_extra_mem[i].n_pfns = 0; } return 0; diff --git a/drivers/xen/balloon.c b/drivers/xen/balloon.c index f4ec5e7c2786bb..538d2000e73197 100644 --- a/drivers/xen/balloon.c +++ b/drivers/xen/balloon.c @@ -632,6 +632,48 @@ static enum bp_state balloon_deflate(unsigned long nr_pages) return increase_reservation(nr_pages); } +/* + * Grow dom0 into 2 MiB blocks of the unpopulated allocator's address space, + * rather than hotplugging memory at the first free physical address, which + * can belong to a PCI device that has not claimed it yet. The blocks are + * parked on the balloon only for this call and deflated like any others, and + * whatever Xen did not populate goes straight back, so the balloon never + * holds address space grant mappings could use. + */ +static enum bp_state balloon_grow_dom0(unsigned long nr_pages) +{ + unsigned long before = balloon_stats.current_pages; + struct page *heads[BALLOON_BLOCK_BATCH], *page; + unsigned int nr = 1, i; + enum bp_state state; + + BUILD_BUG_ON(XEN_UNPOPULATED_BLOCK_PAGES != BALLOON_BLOCK_NR_PAGES); + + /* + * Take as many blocks as balloon_deflate() populates whole, or one for + * it to populate a page at a time. + */ + if (balloon_blocks_usable()) + nr = clamp(nr_pages / BALLOON_BLOCK_NR_PAGES, 1UL, + ARRAY_SIZE(heads)); + nr = xen_alloc_unpopulated_blocks(heads, nr); + if (!nr) { + pr_warn_once("no room left for dom0 to grow into\n"); + return BP_ECANCELED; + } + + for (i = 0; i < nr; i++) + balloon_append_block(heads[i]); + + state = balloon_deflate(nr_pages); + balloon_stats.total_pages += balloon_stats.current_pages - before; + + while ((page = balloon_retrieve(false))) + xen_free_unpopulated_pages(1, &page); + + return state; +} + static enum bp_state decrease_reservation(unsigned long nr_pages, gfp_t gfp) { enum bp_state state = BP_DONE; @@ -830,6 +872,8 @@ static int balloon_thread(void *unused) if (credit > 0) { if (balloon_is_inflated()) balloon_state = balloon_deflate(credit); + else if (xen_initial_domain()) + balloon_state = balloon_grow_dom0(credit); else balloon_state = reserve_additional_memory(); } @@ -968,6 +1012,31 @@ void xen_free_ballooned_pages(unsigned int nr_pages, struct page **pages) } EXPORT_SYMBOL(xen_free_ballooned_pages); +/* + * How many pages dom0 can still grow by: the 2 MiB blocks the unpopulated + * allocator has to spare, within Xen's ceiling for dom0 (dom0_mem=...,max:) + * when one is set. Without a max: Xen reports UINT_MAX. + */ +unsigned long balloon_room_pages(void) +{ + domid_t domid = DOMID_SELF; + long max_pages, cur_pages; + unsigned long room; + + if (!xen_initial_domain()) + return 0; + + room = xen_unpopulated_spare_blocks() * XEN_UNPOPULATED_BLOCK_PAGES; + + max_pages = HYPERVISOR_memory_op(XENMEM_maximum_reservation, &domid); + cur_pages = HYPERVISOR_memory_op(XENMEM_current_reservation, &domid); + if (max_pages > 0 && max_pages < UINT_MAX && cur_pages > 0) + room = min_t(unsigned long, room, + max(max_pages - cur_pages, 0L)); + + return room; +} + #define BALLOON_OOM_DEFLATE_BATCH 512UL /* pages per OOM event (~2 MiB) */ static int balloon_oom_notify(struct notifier_block *nb, unsigned long dummy, @@ -1024,26 +1093,10 @@ static struct notifier_block balloon_oom_nb = { .notifier_call = balloon_oom_notify, }; -static void __init balloon_append_range(unsigned long pfn, - unsigned long end_pfn) -{ - while (pfn < end_pfn) { - if (balloon_blocks_usable() && - IS_ALIGNED(pfn, BALLOON_BLOCK_NR_PAGES) && - end_pfn - pfn >= BALLOON_BLOCK_NR_PAGES) { - balloon_append_block(pfn_to_page(pfn)); - pfn += BALLOON_BLOCK_NR_PAGES; - } else { - balloon_append(pfn_to_page(pfn)); - pfn++; - } - } -} - static int __init balloon_add_regions(bool append) { unsigned long start_pfn, pages; - unsigned long extra_pfn_end; + unsigned long pfn, extra_pfn_end; unsigned int i; for (i = 0; i < XEN_EXTRA_MEM_MAX_REGIONS; i++) { @@ -1060,7 +1113,8 @@ static int __init balloon_add_regions(bool append) */ extra_pfn_end = min(max_pfn, start_pfn + pages); - balloon_append_range(start_pfn, extra_pfn_end); + for (pfn = start_pfn; pfn < extra_pfn_end; pfn++) + balloon_append(pfn_to_page(pfn)); /* * There are two different use-cases depending on how the diff --git a/drivers/xen/unpopulated-alloc.c b/drivers/xen/unpopulated-alloc.c index 73ea5adf786615..7dacc13447b609 100644 --- a/drivers/xen/unpopulated-alloc.c +++ b/drivers/xen/unpopulated-alloc.c @@ -24,10 +24,31 @@ * * A single mutex covers all per-node lists. Alloc/free are * connect-time events on the I/O backends and not contention-sensitive. + * + * The boot-time unbacked address space, which arch_xen_unpopulated_init() + * hands over with xen_unpopulated_add_range(), is not ZONE_DEVICE but RAM + * pfns with nothing behind them. Its aligned 2 MiB blocks go on block_list[] + * of their node, linked through their first page as page_list[] links pages, + * and the pages around them go on page_list[]. + * + * dom0's balloon takes blocks off block_list[], which Xen can populate as one + * extent each, and leaves each node at least MAPPING_FLOOR_PAGES free pages. + * + * Mappings take pages from page_list[] only, and give them back to it. When + * page_list[] runs empty, a block moves onto it from block_list[]. */ static DEFINE_MUTEX(list_lock); static struct page *page_list[MAX_NUMNODES]; static unsigned int list_count[MAX_NUMNODES]; +static struct page *block_list[MAX_NUMNODES]; +static unsigned long block_count[MAX_NUMNODES]; + +/* + * Free pages of each node that dom0's balloon leaves to mappings, so that + * they rarely need fill_list() to carve a region out of address space that + * may belong to a device. + */ +#define MAPPING_FLOOR_PAGES PAGES_PER_SECTION static struct resource *target_resource; @@ -180,6 +201,74 @@ static int fill_list(unsigned int nr_pages, int node) return ret; } +/* Put @pg on page_list[] of its own node. Called with list_lock held. */ +static void push_page(struct page *pg) +{ + int node = xen_unpopulated_clamp_node(page_to_nid(pg)); + + pg->zone_device_data = page_list[node]; + page_list[node] = pg; + list_count[node]++; +} + +/* + * Put the 2 MiB block that starts at @head on block_list[] of its node. + * Called with list_lock held. + */ +static void push_block(struct page *head) +{ + int node = page_to_nid(head); + + head->zone_device_data = block_list[node]; + block_list[node] = head; + block_count[node]++; +} + +/* + * Take a 2 MiB block off block_list[@node]. Returns its first page, or NULL + * when none is left. Called with list_lock held. + */ +static struct page *pop_block(int node) +{ + struct page *head = block_list[node]; + + if (head) { + block_list[node] = head->zone_device_data; + block_count[node]--; + } + return head; +} + +/* + * Take a page off page_list[@node], which a block refills when it is empty. + * Returns NULL when the pool is empty. + */ +static struct page *take_page(int node) +{ + struct page *pg; + unsigned long pfn, i; + + if (!page_list[node] && block_list[node]) { + pfn = page_to_pfn(pop_block(node)); + for (i = 0; i < XEN_UNPOPULATED_BLOCK_PAGES; i++) + push_page(pfn_to_page(pfn + i)); + } + + pg = page_list[node]; + if (pg) { + page_list[node] = pg->zone_device_data; + list_count[node]--; + } + return pg; +} + +/* Free pages of @node, those in the blocks on block_list[] included. */ +static unsigned long pool_pages(int node) +{ + return list_count[node] + + block_count[node] * XEN_UNPOPULATED_BLOCK_PAGES; +} + /* * Take @nr_pages from the pool of @node, or from any node's pool when @any is * set. Called with list_lock held. Never carves a new region. @@ -187,23 +276,21 @@ static int fill_list(unsigned int nr_pages, int node) static int pop_pages(unsigned int nr_pages, struct page **pages, int node, bool any) { - unsigned int i, total = 0; + unsigned long total = 0; + unsigned int i; for (i = 0; i < MAX_NUMNODES; i++) - total += list_count[i]; - if ((any ? total : list_count[node]) < nr_pages) + total += pool_pages(i); + if ((any ? total : pool_pages(node)) < nr_pages) return -ENOMEM; for (i = 0; i < nr_pages; i++) { int src = node; struct page *pg; - while (!page_list[src]) + while (!(pg = take_page(src))) src = (src + 1) % MAX_NUMNODES; - pg = page_list[src]; - page_list[src] = pg->zone_device_data; - list_count[src]--; pages[i] = pg; #ifdef CONFIG_XEN_HAVE_PVMMU @@ -213,14 +300,8 @@ static int pop_pages(unsigned int nr_pages, struct page **pages, int node, if (ret < 0) { unsigned int j; - for (j = 0; j <= i; j++) { - int dst = xen_unpopulated_clamp_node( - page_to_nid(pages[j])); - - pages[j]->zone_device_data = page_list[dst]; - page_list[dst] = pages[j]; - list_count[dst]++; - } + for (j = 0; j <= i; j++) + push_page(pages[j]); return ret; } } @@ -283,8 +364,9 @@ EXPORT_SYMBOL(xen_alloc_unpopulated_pages_node); */ int xen_alloc_unpopulated_pages(unsigned int nr_pages, struct page **pages) { - unsigned int i, total = 0; + unsigned long total = 0; int node = numa_node_id(); + unsigned int i; int ret; if (!target_resource) @@ -292,7 +374,7 @@ int xen_alloc_unpopulated_pages(unsigned int nr_pages, struct page **pages) mutex_lock(&list_lock); for (i = 0; i < MAX_NUMNODES; i++) - total += list_count[i]; + total += pool_pages(i); ret = 0; if (total < nr_pages) ret = fill_list(nr_pages - total, node); @@ -324,16 +406,111 @@ void xen_free_unpopulated_pages(unsigned int nr_pages, struct page **pages) } mutex_lock(&list_lock); - for (i = 0; i < nr_pages; i++) { - int node = xen_unpopulated_clamp_node(page_to_nid(pages[i])); + for (i = 0; i < nr_pages; i++) + push_page(pages[i]); + mutex_unlock(&list_lock); +} +EXPORT_SYMBOL(xen_free_unpopulated_pages); - pages[i]->zone_device_data = page_list[node]; - page_list[node] = pages[i]; - list_count[node]++; +/** + * xen_unpopulated_add_range - hand over boot-time unbacked address space + * @start_pfn: first pfn of the range + * @nr_pages: number of pages in the range + * + * The pages must have a struct page and nothing behind them, as the extra + * memory regions of a Xen guest do. Each aligned 2 MiB block goes on + * block_list[] of its node, and the other pages on page_list[]. + */ +void __init xen_unpopulated_add_range(unsigned long start_pfn, + unsigned long nr_pages) +{ + unsigned long pfn = start_pfn, end_pfn = start_pfn + nr_pages, last; + struct page *pg; + + mutex_lock(&list_lock); + while (pfn < end_pfn) { + pg = pfn_to_page(pfn); + last = pfn + XEN_UNPOPULATED_BLOCK_PAGES - 1; + + /* + * As with __pageblock_pfn_to_page we assume no A B A + * alternations of numa node assignments within a block, thus + * check only the first and last entries. + */ + if (IS_ALIGNED(pfn, XEN_UNPOPULATED_BLOCK_PAGES) && + last < end_pfn && + page_to_nid(pg) == page_to_nid(pfn_to_page(last))) { + push_block(pg); + pfn += XEN_UNPOPULATED_BLOCK_PAGES; + } else { + push_page(pg); + pfn++; + } } mutex_unlock(&list_lock); } -EXPORT_SYMBOL(xen_free_unpopulated_pages); + +/* + * Blocks dom0's balloon may take from @node. Pages on page_list[] count + * toward MAPPING_FLOOR_PAGES, and @node keeps enough blocks to make up the + * rest. + */ +static unsigned long spare_blocks(int node) +{ + unsigned long keep = 0; + + if (list_count[node] < MAPPING_FLOOR_PAGES) + keep = DIV_ROUND_UP(MAPPING_FLOOR_PAGES - list_count[node], + XEN_UNPOPULATED_BLOCK_PAGES); + + return block_count[node] > keep ? block_count[node] - keep : 0; +} + +/** + * xen_alloc_unpopulated_blocks - take 2 MiB blocks for dom0's balloon + * @heads: first page of each block taken + * @nr_blocks: number of blocks wanted + * + * Blocks come from the boot-time address space of the lowest node that can + * spare one and still keep MAPPING_FLOOR_PAGES free pages. Give back the + * pages Xen did not populate with xen_free_unpopulated_pages(), after which + * they serve mappings and do not form a block again. + * + * Returns the number of blocks taken, which may be less than @nr_blocks. + */ +unsigned int xen_alloc_unpopulated_blocks(struct page **heads, + unsigned int nr_blocks) +{ + unsigned int nr = 0; + int node; + + mutex_lock(&list_lock); + for (node = 0; node < MAX_NUMNODES; node++) + while (nr < nr_blocks && spare_blocks(node)) + heads[nr++] = pop_block(node); + mutex_unlock(&list_lock); + + return nr; +} + +/** + * xen_unpopulated_spare_blocks - 2 MiB blocks dom0's balloon could still take + * + * Each node spares the blocks on its block_list[] beyond those it keeps to make + * up MAPPING_FLOOR_PAGES free pages. + */ +unsigned long xen_unpopulated_spare_blocks(void) +{ + unsigned long nr = 0; + int node; + + mutex_lock(&list_lock); + for (node = 0; node < MAX_NUMNODES; node++) + nr += spare_blocks(node); + mutex_unlock(&list_lock); + + return nr; +} static int __init unpopulated_init(void) { diff --git a/drivers/xen/xen-balloon.c b/drivers/xen/xen-balloon.c index bf538f12781261..07e58db1fe26f1 100644 --- a/drivers/xen/xen-balloon.c +++ b/drivers/xen/xen-balloon.c @@ -146,6 +146,7 @@ BALLOON_SHOW(current_kb, "%lu\n", PAGES2KB(balloon_stats.current_pages)); BALLOON_SHOW(low_kb, "%lu\n", PAGES2KB(balloon_stats.balloon_low)); BALLOON_SHOW(high_kb, "%lu\n", PAGES2KB(balloon_stats.balloon_high)); BALLOON_SHOW(blocks, "%lu\n", balloon_stats.balloon_blocks); +BALLOON_SHOW(room_kb, "%lu\n", PAGES2KB(balloon_room_pages())); static DEVICE_ULONG_ATTR(schedule_delay, 0444, balloon_stats.schedule_delay); static DEVICE_ULONG_ATTR(max_schedule_delay, 0644, balloon_stats.max_schedule_delay); @@ -225,6 +226,7 @@ static struct attribute *balloon_info_attrs[] = { &dev_attr_low_kb.attr, &dev_attr_high_kb.attr, &dev_attr_blocks.attr, + &dev_attr_room_kb.attr, NULL }; diff --git a/include/xen/balloon.h b/include/xen/balloon.h index 4702026337e554..00a44b9d75162d 100644 --- a/include/xen/balloon.h +++ b/include/xen/balloon.h @@ -27,6 +27,7 @@ struct balloon_stats { extern struct balloon_stats balloon_stats; void balloon_set_new_target(unsigned long target); +unsigned long balloon_room_pages(void); int xen_alloc_ballooned_pages(unsigned int nr_pages, struct page **pages); void xen_free_ballooned_pages(unsigned int nr_pages, struct page **pages); diff --git a/include/xen/xen.h b/include/xen/xen.h index ccf9bfa338131c..c1eec9f7329558 100644 --- a/include/xen/xen.h +++ b/include/xen/xen.h @@ -68,12 +68,19 @@ bool xen_biovec_phys_mergeable(const struct bio_vec *vec1, extern u64 xen_saved_max_mem_size; #endif +/* Pages in a 2 MiB block of unbacked address space, one order-9 Xen extent. */ +#define XEN_UNPOPULATED_BLOCK_PAGES (1UL << (21 - PAGE_SHIFT)) + #ifdef CONFIG_XEN_UNPOPULATED_ALLOC extern unsigned long xen_unpopulated_pages; int xen_alloc_unpopulated_pages_node(unsigned int nr_pages, struct page **pages, int node); int xen_alloc_unpopulated_pages(unsigned int nr_pages, struct page **pages); void xen_free_unpopulated_pages(unsigned int nr_pages, struct page **pages); +void xen_unpopulated_add_range(unsigned long start_pfn, unsigned long nr_pages); +unsigned int xen_alloc_unpopulated_blocks(struct page **heads, + unsigned int nr_blocks); +unsigned long xen_unpopulated_spare_blocks(void); #include int arch_xen_unpopulated_init(struct resource **res); #else @@ -94,6 +101,15 @@ static inline void xen_free_unpopulated_pages(unsigned int nr_pages, { xen_free_ballooned_pages(nr_pages, pages); } +static inline unsigned int xen_alloc_unpopulated_blocks(struct page **heads, + unsigned int nr_blocks) +{ + return 0; +} +static inline unsigned long xen_unpopulated_spare_blocks(void) +{ + return 0; +} #endif /*