diff --git a/drivers/staging/android/ion/ion.c b/drivers/staging/android/ion/ion.c index b2dcee5ceae5..ddf8fde9ae93 100644 --- a/drivers/staging/android/ion/ion.c +++ b/drivers/staging/android/ion/ion.c @@ -28,8 +28,6 @@ #include #include #include -#include -#include #include #include #include @@ -143,7 +141,6 @@ static void ion_buffer_add(struct ion_device *dev, static int ion_buffer_alloc_dirty(struct ion_buffer *buffer); -static bool ion_heap_drain_freelist(struct ion_heap *heap); /* this function should only be called while dev->lock is held */ static struct ion_buffer *ion_buffer_create(struct ion_heap *heap, struct ion_device *dev, @@ -170,7 +167,7 @@ static struct ion_buffer *ion_buffer_create(struct ion_heap *heap, if (!(heap->flags & ION_HEAP_FLAG_DEFER_FREE)) goto err2; - ion_heap_drain_freelist(heap); + ion_heap_freelist_drain(heap, 0); ret = heap->ops->allocate(heap, buffer, len, align, flags); if (ret) @@ -230,7 +227,7 @@ err2: return ERR_PTR(ret); } -static void _ion_buffer_destroy(struct ion_buffer *buffer) +void ion_buffer_destroy(struct ion_buffer *buffer) { if (WARN_ON(buffer->kmap_cnt > 0)) buffer->heap->ops->unmap_kernel(buffer->heap, buffer); @@ -241,7 +238,7 @@ static void _ion_buffer_destroy(struct ion_buffer *buffer) kfree(buffer); } -static void ion_buffer_destroy(struct kref *kref) +static void _ion_buffer_destroy(struct kref *kref) { struct ion_buffer *buffer = container_of(kref, struct ion_buffer, ref); struct ion_heap *heap = buffer->heap; @@ -251,14 +248,10 @@ static void ion_buffer_destroy(struct kref *kref) rb_erase(&buffer->node, &dev->buffers); mutex_unlock(&dev->buffer_lock); - if (heap->flags & ION_HEAP_FLAG_DEFER_FREE) { - rt_mutex_lock(&heap->lock); - list_add(&buffer->list, &heap->free_list); - rt_mutex_unlock(&heap->lock); - wake_up(&heap->waitqueue); - return; - } - _ion_buffer_destroy(buffer); + if (heap->flags & ION_HEAP_FLAG_DEFER_FREE) + ion_heap_freelist_add(heap, buffer); + else + ion_buffer_destroy(buffer); } static void ion_buffer_get(struct ion_buffer *buffer) @@ -268,7 +261,7 @@ static void ion_buffer_get(struct ion_buffer *buffer) static int ion_buffer_put(struct ion_buffer *buffer) { - return kref_put(&buffer->ref, ion_buffer_destroy); + return kref_put(&buffer->ref, _ion_buffer_destroy); } static void ion_buffer_add_to_handle(struct ion_buffer *buffer) @@ -1298,80 +1291,53 @@ static const struct file_operations debug_heap_fops = { .release = single_release, }; -static size_t ion_heap_free_list_is_empty(struct ion_heap *heap) +#ifdef DEBUG_HEAP_SHRINKER +static int debug_shrink_set(void *data, u64 val) { - bool is_empty; + struct ion_heap *heap = data; + struct shrink_control sc; + int objs; - rt_mutex_lock(&heap->lock); - is_empty = list_empty(&heap->free_list); - rt_mutex_unlock(&heap->lock); + sc.gfp_mask = -1; + sc.nr_to_scan = 0; - return is_empty; + if (!val) + return 0; + + objs = heap->shrinker.shrink(&heap->shrinker, &sc); + sc.nr_to_scan = objs; + + heap->shrinker.shrink(&heap->shrinker, &sc); + return 0; } -static int ion_heap_deferred_free(void *data) +static int debug_shrink_get(void *data, u64 *val) { - struct ion_heap *heap = data; + struct ion_heap *heap = data; + struct shrink_control sc; + int objs; - while (true) { - struct ion_buffer *buffer; + sc.gfp_mask = -1; + sc.nr_to_scan = 0; - wait_event_freezable(heap->waitqueue, - !ion_heap_free_list_is_empty(heap)); - - rt_mutex_lock(&heap->lock); - if (list_empty(&heap->free_list)) { - rt_mutex_unlock(&heap->lock); - continue; - } - buffer = list_first_entry(&heap->free_list, struct ion_buffer, - list); - list_del(&buffer->list); - rt_mutex_unlock(&heap->lock); - _ion_buffer_destroy(buffer); - } - - return 0; + objs = heap->shrinker.shrink(&heap->shrinker, &sc); + *val = objs; + return 0; } -static bool ion_heap_drain_freelist(struct ion_heap *heap) -{ - struct ion_buffer *buffer, *tmp; - - if (ion_heap_free_list_is_empty(heap)) - return false; - rt_mutex_lock(&heap->lock); - list_for_each_entry_safe(buffer, tmp, &heap->free_list, list) { - list_del(&buffer->list); - _ion_buffer_destroy(buffer); - } - BUG_ON(!list_empty(&heap->free_list)); - rt_mutex_unlock(&heap->lock); - - - return true; -} +DEFINE_SIMPLE_ATTRIBUTE(debug_shrink_fops, debug_shrink_get, + debug_shrink_set, "%llu\n"); +#endif void ion_device_add_heap(struct ion_device *dev, struct ion_heap *heap) { - struct sched_param param = { .sched_priority = 0 }; - if (!heap->ops->allocate || !heap->ops->free || !heap->ops->map_dma || !heap->ops->unmap_dma) pr_err("%s: can not add heap with invalid ops struct.\n", __func__); - if (heap->flags & ION_HEAP_FLAG_DEFER_FREE) { - INIT_LIST_HEAD(&heap->free_list); - rt_mutex_init(&heap->lock); - init_waitqueue_head(&heap->waitqueue); - heap->task = kthread_run(ion_heap_deferred_free, heap, - "%s", heap->name); - sched_setscheduler(heap->task, SCHED_IDLE, ¶m); - if (IS_ERR(heap->task)) - pr_err("%s: creating thread for deferred free failed\n", - __func__); - } + if (heap->flags & ION_HEAP_FLAG_DEFER_FREE) + ion_heap_init_deferred_free(heap); heap->dev = dev; down_write(&dev->lock); @@ -1381,6 +1347,15 @@ void ion_device_add_heap(struct ion_device *dev, struct ion_heap *heap) plist_add(&heap->node, &dev->heaps); debugfs_create_file(heap->name, 0664, dev->debug_root, heap, &debug_heap_fops); +#ifdef DEBUG_HEAP_SHRINKER + if (heap->shrinker.shrink) { + char debug_name[64]; + + snprintf(debug_name, 64, "%s_shrink", heap->name); + debugfs_create_file(debug_name, 0644, dev->debug_root, heap, + &debug_shrink_fops); + } +#endif up_write(&dev->lock); } diff --git a/drivers/staging/android/ion/ion_heap.c b/drivers/staging/android/ion/ion_heap.c index 11066a2abb4b..f541bff26435 100644 --- a/drivers/staging/android/ion/ion_heap.c +++ b/drivers/staging/android/ion/ion_heap.c @@ -15,7 +15,11 @@ */ #include +#include +#include #include +#include +#include #include #include #include "ion.h" @@ -130,6 +134,109 @@ end: return ret; } +void ion_heap_free_page(struct ion_buffer *buffer, struct page *page, + unsigned int order) +{ + int i; + + if (!ion_buffer_fault_user_mappings(buffer)) { + __free_pages(page, order); + return; + } + for (i = 0; i < (1 << order); i++) + __free_page(page + i); +} + +void ion_heap_freelist_add(struct ion_heap *heap, struct ion_buffer * buffer) +{ + rt_mutex_lock(&heap->lock); + list_add(&buffer->list, &heap->free_list); + heap->free_list_size += buffer->size; + rt_mutex_unlock(&heap->lock); + wake_up(&heap->waitqueue); +} + +size_t ion_heap_freelist_size(struct ion_heap *heap) +{ + size_t size; + + rt_mutex_lock(&heap->lock); + size = heap->free_list_size; + rt_mutex_unlock(&heap->lock); + + return size; +} + +size_t ion_heap_freelist_drain(struct ion_heap *heap, size_t size) +{ + struct ion_buffer *buffer, *tmp; + size_t total_drained = 0; + + if (ion_heap_freelist_size(heap) == 0) + return 0; + + rt_mutex_lock(&heap->lock); + if (size == 0) + size = heap->free_list_size; + + list_for_each_entry_safe(buffer, tmp, &heap->free_list, list) { + if (total_drained >= size) + break; + list_del(&buffer->list); + ion_buffer_destroy(buffer); + heap->free_list_size -= buffer->size; + total_drained += buffer->size; + } + rt_mutex_unlock(&heap->lock); + + return total_drained; +} + +int ion_heap_deferred_free(void *data) +{ + struct ion_heap *heap = data; + + while (true) { + struct ion_buffer *buffer; + + wait_event_freezable(heap->waitqueue, + ion_heap_freelist_size(heap) > 0); + + rt_mutex_lock(&heap->lock); + if (list_empty(&heap->free_list)) { + rt_mutex_unlock(&heap->lock); + continue; + } + buffer = list_first_entry(&heap->free_list, struct ion_buffer, + list); + list_del(&buffer->list); + heap->free_list_size -= buffer->size; + rt_mutex_unlock(&heap->lock); + ion_buffer_destroy(buffer); + } + + return 0; +} + +int ion_heap_init_deferred_free(struct ion_heap *heap) +{ + struct sched_param param = { .sched_priority = 0 }; + + INIT_LIST_HEAD(&heap->free_list); + heap->free_list_size = 0; + rt_mutex_init(&heap->lock); + init_waitqueue_head(&heap->waitqueue); + heap->task = kthread_run(ion_heap_deferred_free, heap, + "%s", heap->name); + sched_setscheduler(heap->task, SCHED_IDLE, ¶m); + if (IS_ERR(heap->task)) { + pr_err("%s: creating thread for deferred free failed\n", + __func__); + return PTR_RET(heap->task); + } + return 0; +} + struct ion_heap *ion_heap_create(struct ion_platform_heap *heap_data) { struct ion_heap *heap = NULL; diff --git a/drivers/staging/android/ion/ion_page_pool.c b/drivers/staging/android/ion/ion_page_pool.c index df6e29c1854e..d624dee170ba 100644 --- a/drivers/staging/android/ion/ion_page_pool.c +++ b/drivers/staging/android/ion/ion_page_pool.c @@ -21,14 +21,8 @@ #include #include #include -#include #include "ion_priv.h" -/* #define DEBUG_PAGE_POOL_SHRINKER */ - -static struct plist_head pools = PLIST_HEAD_INIT(pools); -static struct shrinker shrinker; - struct ion_page_pool_item { struct page *page; struct list_head list; @@ -126,109 +120,46 @@ void ion_page_pool_free(struct ion_page_pool *pool, struct page* page) ion_page_pool_free_pages(pool, page); } -#ifdef DEBUG_PAGE_POOL_SHRINKER -static int debug_drop_pools_set(void *data, u64 val) +static int ion_page_pool_total(struct ion_page_pool *pool, bool high) { - struct shrink_control sc; - int objs; - - sc.gfp_mask = -1; - sc.nr_to_scan = 0; - - if (!val) - return 0; - - objs = shrinker.shrink(&shrinker, &sc); - sc.nr_to_scan = objs; - - shrinker.shrink(&shrinker, &sc); - return 0; -} - -static int debug_drop_pools_get(void *data, u64 *val) -{ - struct shrink_control sc; - int objs; - - sc.gfp_mask = -1; - sc.nr_to_scan = 0; - - objs = shrinker.shrink(&shrinker, &sc); - *val = objs; - return 0; -} - -DEFINE_SIMPLE_ATTRIBUTE(debug_drop_pools_fops, debug_drop_pools_get, - debug_drop_pools_set, "%llu\n"); - -static int debug_grow_pools_set(void *data, u64 val) -{ - struct ion_page_pool *pool; - struct page *page; - - plist_for_each_entry(pool, &pools, list) { - if (val != pool->list.prio) - continue; - page = ion_page_pool_alloc_pages(pool); - if (page) - ion_page_pool_add(pool, page); - } - - return 0; -} - -DEFINE_SIMPLE_ATTRIBUTE(debug_grow_pools_fops, debug_drop_pools_get, - debug_grow_pools_set, "%llu\n"); -#endif - -static int ion_page_pool_total(bool high) -{ - struct ion_page_pool *pool; int total = 0; - plist_for_each_entry(pool, &pools, list) { - total += high ? (pool->high_count + pool->low_count) * - (1 << pool->order) : + total += high ? (pool->high_count + pool->low_count) * + (1 << pool->order) : pool->low_count * (1 << pool->order); - } return total; } -static int ion_page_pool_shrink(struct shrinker *shrinker, - struct shrink_control *sc) +int ion_page_pool_shrink(struct ion_page_pool *pool, gfp_t gfp_mask, + int nr_to_scan) { - struct ion_page_pool *pool; int nr_freed = 0; int i; bool high; - int nr_to_scan = sc->nr_to_scan; - high = sc->gfp_mask & __GFP_HIGHMEM; + high = gfp_mask & __GFP_HIGHMEM; if (nr_to_scan == 0) - return ion_page_pool_total(high); + return ion_page_pool_total(pool, high); - plist_for_each_entry(pool, &pools, list) { - for (i = 0; i < nr_to_scan; i++) { - struct page *page; + for (i = 0; i < nr_to_scan; i++) { + struct page *page; - mutex_lock(&pool->mutex); - if (high && pool->high_count) { - page = ion_page_pool_remove(pool, true); - } else if (pool->low_count) { - page = ion_page_pool_remove(pool, false); - } else { - mutex_unlock(&pool->mutex); - break; - } + mutex_lock(&pool->mutex); + if (high && pool->high_count) { + page = ion_page_pool_remove(pool, true); + } else if (pool->low_count) { + page = ion_page_pool_remove(pool, false); + } else { mutex_unlock(&pool->mutex); - ion_page_pool_free_pages(pool, page); - nr_freed += (1 << pool->order); + break; } - nr_to_scan -= i; + mutex_unlock(&pool->mutex); + ion_page_pool_free_pages(pool, page); + nr_freed += (1 << pool->order); } - return ion_page_pool_total(high); + return nr_freed; } struct ion_page_pool *ion_page_pool_create(gfp_t gfp_mask, unsigned int order) @@ -245,35 +176,22 @@ struct ion_page_pool *ion_page_pool_create(gfp_t gfp_mask, unsigned int order) pool->order = order; mutex_init(&pool->mutex); plist_node_init(&pool->list, order); - plist_add(&pool->list, &pools); return pool; } void ion_page_pool_destroy(struct ion_page_pool *pool) { - plist_del(&pool->list, &pools); kfree(pool); } static int __init ion_page_pool_init(void) { - shrinker.shrink = ion_page_pool_shrink; - shrinker.seeks = DEFAULT_SEEKS; - shrinker.batch = 0; - register_shrinker(&shrinker); -#ifdef DEBUG_PAGE_POOL_SHRINKER - debugfs_create_file("ion_pools_shrink", 0644, NULL, NULL, - &debug_drop_pools_fops); - debugfs_create_file("ion_pools_grow", 0644, NULL, NULL, - &debug_grow_pools_fops); -#endif return 0; } static void __exit ion_page_pool_exit(void) { - unregister_shrinker(&shrinker); } module_init(ion_page_pool_init); diff --git a/drivers/staging/android/ion/ion_priv.h b/drivers/staging/android/ion/ion_priv.h index ab1a8d956ada..bdc1e344d731 100644 --- a/drivers/staging/android/ion/ion_priv.h +++ b/drivers/staging/android/ion/ion_priv.h @@ -82,6 +82,7 @@ struct ion_buffer { char task_comm[TASK_COMM_LEN]; pid_t pid; }; +void ion_buffer_destroy(struct ion_buffer *buffer); /** * struct ion_heap_ops - ops to operate on a given heap @@ -127,7 +128,12 @@ struct ion_heap_ops { * allocating. These are specified by platform data and * MUST be unique * @name: used for debugging + * @shrinker: a shrinker for the heap, if the heap caches system + * memory, it must define a shrinker to return it on low + * memory conditions, this includes system memory cached + * in the deferred free lists for heaps that support it * @free_list: free list head if deferred free is used + * @free_list_size size of the deferred free list in bytes * @lock: protects the free list * @waitqueue: queue to wait on from deferred free thread * @task: task struct of deferred free thread @@ -147,7 +153,9 @@ struct ion_heap { unsigned long flags; unsigned int id; const char *name; + struct shrinker shrinker; struct list_head free_list; + size_t free_list_size; struct rt_mutex lock; wait_queue_head_t waitqueue; struct task_struct *task; @@ -205,6 +213,43 @@ int ion_heap_map_user(struct ion_heap *, struct ion_buffer *, struct vm_area_struct *); int ion_heap_buffer_zero(struct ion_buffer *buffer); +/** + * ion_heap_init_deferred_free -- initialize deferred free functionality + * @heap: the heap + * + * If a heap sets the ION_HEAP_FLAG_DEFER_FREE flag this function will + * be called to setup deferred frees. Calls to free the buffer will + * return immediately and the actual free will occur some time later + */ +int ion_heap_init_deferred_free(struct ion_heap *heap); + +/** + * ion_heap_freelist_add - add a buffer to the deferred free list + * @heap: the heap + * @buffer: the buffer + * + * Adds an item to the deferred freelist. + */ +void ion_heap_freelist_add(struct ion_heap *heap, struct ion_buffer *buffer); + +/** + * ion_heap_freelist_drain - drain the deferred free list + * @heap: the heap + * @size: ammount of memory to drain in bytes + * + * Drains the indicated amount of memory from the deferred freelist immediately. + * Returns the total amount freed. The total freed may be higher depending + * on the size of the items in the list, or lower if there is insufficient + * total memory on the freelist. + */ +size_t ion_heap_freelist_drain(struct ion_heap *heap, size_t size); + +/** + * ion_heap_freelist_size - returns the size of the freelist in bytes + * @heap: the heap + */ +size_t ion_heap_freelist_size(struct ion_heap *heap); + /** * functions for creating and destroying the built in ion heaps. @@ -274,8 +319,6 @@ struct ion_page_pool { struct list_head high_items; struct list_head low_items; struct mutex mutex; - void *(*alloc)(struct ion_page_pool *pool); - void (*free)(struct ion_page_pool *pool, struct page *page); gfp_t gfp_mask; unsigned int order; struct plist_node list; @@ -286,4 +329,14 @@ void ion_page_pool_destroy(struct ion_page_pool *); void *ion_page_pool_alloc(struct ion_page_pool *); void ion_page_pool_free(struct ion_page_pool *, struct page *); +/** ion_page_pool_shrink - shrinks the size of the memory cached in the pool + * @pool: the pool + * @gfp_mask: the memory type to reclaim + * @nr_to_scan: number of items to shrink in pages + * + * returns the number of items freed in pages + */ +int ion_page_pool_shrink(struct ion_page_pool *pool, gfp_t gfp_mask, + int nr_to_scan); + #endif /* _ION_PRIV_H */ diff --git a/drivers/staging/android/ion/ion_system_heap.c b/drivers/staging/android/ion/ion_system_heap.c index 6665797f5370..fedbc0d5fa7d 100644 --- a/drivers/staging/android/ion/ion_system_heap.c +++ b/drivers/staging/android/ion/ion_system_heap.c @@ -253,6 +253,50 @@ static struct ion_heap_ops system_heap_ops = { .map_user = ion_heap_map_user, }; +static int ion_system_heap_shrink(struct shrinker *shrinker, + struct shrink_control *sc) { + + struct ion_heap *heap = container_of(shrinker, struct ion_heap, + shrinker); + struct ion_system_heap *sys_heap = container_of(heap, + struct ion_system_heap, + heap); + int nr_total = 0; + int nr_freed = 0; + int i; + + if (sc->nr_to_scan == 0) + goto end; + + /* shrink the free list first, no point in zeroing the memory if + we're just going to reclaim it */ + nr_freed += ion_heap_freelist_drain(heap, sc->nr_to_scan * PAGE_SIZE) / + PAGE_SIZE; + + if (nr_freed >= sc->nr_to_scan) + goto end; + + for (i = 0; i < num_orders; i++) { + struct ion_page_pool *pool = sys_heap->pools[i]; + + nr_freed += ion_page_pool_shrink(pool, sc->gfp_mask, + sc->nr_to_scan); + if (nr_freed >= sc->nr_to_scan) + break; + } + +end: + /* total number of items is whatever the page pools are holding + plus whatever's in the freelist */ + for (i = 0; i < num_orders; i++) { + struct ion_page_pool *pool = sys_heap->pools[i]; + nr_total += ion_page_pool_shrink(pool, sc->gfp_mask, 0); + } + nr_total += ion_heap_freelist_size(heap) / PAGE_SIZE; + return nr_total; + +} + static int ion_system_heap_debug_show(struct ion_heap *heap, struct seq_file *s, void *unused) { @@ -299,6 +343,11 @@ struct ion_heap *ion_system_heap_create(struct ion_platform_heap *unused) goto err_create_pool; heap->pools[i] = pool; } + + heap->heap.shrinker.shrink = ion_system_heap_shrink; + heap->heap.shrinker.seeks = DEFAULT_SEEKS; + heap->heap.shrinker.batch = 0; + register_shrinker(&heap->heap.shrinker); heap->heap.debug_show = ion_system_heap_debug_show; return &heap->heap; err_create_pool: