diff options
Diffstat (limited to 'drivers/staging/zcache/zcache-main.c')
| -rw-r--r-- | drivers/staging/zcache/zcache-main.c | 1943 |
1 files changed, 1943 insertions, 0 deletions
diff --git a/drivers/staging/zcache/zcache-main.c b/drivers/staging/zcache/zcache-main.c new file mode 100644 index 00000000..81972fa4 --- /dev/null +++ b/drivers/staging/zcache/zcache-main.c @@ -0,0 +1,1943 @@ +/* + * zcache.c + * + * Copyright (c) 2010-2012, Dan Magenheimer, Oracle Corp. + * Copyright (c) 2010,2011, Nitin Gupta + * + * Zcache provides an in-kernel "host implementation" for transcendent memory + * ("tmem") and, thus indirectly, for cleancache and frontswap. Zcache uses + * lzo1x compression to improve density and an embedded allocator called + * "zbud" which "buddies" two compressed pages semi-optimally in each physical + * pageframe. Zbud is integrally tied into tmem to allow pageframes to + * be "reclaimed" efficiently. + */ + +#include <linux/module.h> +#include <linux/cpu.h> +#include <linux/highmem.h> +#include <linux/list.h> +#include <linux/slab.h> +#include <linux/spinlock.h> +#include <linux/types.h> +#include <linux/string.h> +#include <linux/atomic.h> +#include <linux/math64.h> +#include <linux/crypto.h> +#include <linux/swap.h> +#include <linux/swapops.h> +#include <linux/pagemap.h> +#include <linux/writeback.h> + +#include <linux/cleancache.h> +#include <linux/frontswap.h> +#include "tmem.h" +#include "zcache.h" +#include "zbud.h" +#include "ramster.h" +#include "debug.h" +#ifdef CONFIG_RAMSTER +static bool ramster_enabled __read_mostly; +static int disable_frontswap_selfshrink; +#else +#define ramster_enabled false +#define disable_frontswap_selfshrink 0 +#endif + +#ifndef __PG_WAS_ACTIVE +static inline bool PageWasActive(struct page *page) +{ + return true; +} + +static inline void SetPageWasActive(struct page *page) +{ +} +#endif + +#ifdef FRONTSWAP_HAS_EXCLUSIVE_GETS +static bool frontswap_has_exclusive_gets __read_mostly = true; +#else +static bool frontswap_has_exclusive_gets __read_mostly; +static inline void frontswap_tmem_exclusive_gets(bool b) +{ +} +#endif + +/* + * mark pampd to special value in order that later + * retrieve will identify zero-filled pages + */ +#define ZERO_FILLED 0x2 + +/* enable (or fix code) when Seth's patches are accepted upstream */ +#define zcache_writeback_enabled 0 + +static bool zcache_enabled __read_mostly; +static bool disable_cleancache __read_mostly; +static bool disable_frontswap __read_mostly; +static bool disable_frontswap_ignore_nonactive __read_mostly; +static bool disable_cleancache_ignore_nonactive __read_mostly; +static char *namestr __read_mostly = "zcache"; + +#define ZCACHE_GFP_MASK \ + (__GFP_FS | __GFP_NORETRY | __GFP_NOWARN | __GFP_NOMEMALLOC) + +/* crypto API for zcache */ +#ifdef CONFIG_ZCACHE_MODULE +static char *zcache_comp_name = "lzo"; +#else +#define ZCACHE_COMP_NAME_SZ CRYPTO_MAX_ALG_NAME +static char zcache_comp_name[ZCACHE_COMP_NAME_SZ] __read_mostly; +#endif +static struct crypto_comp * __percpu *zcache_comp_pcpu_tfms __read_mostly; + +enum comp_op { + ZCACHE_COMPOP_COMPRESS, + ZCACHE_COMPOP_DECOMPRESS +}; + +static inline int zcache_comp_op(enum comp_op op, + const u8 *src, unsigned int slen, + u8 *dst, unsigned int *dlen) +{ + struct crypto_comp *tfm; + int ret = -1; + + BUG_ON(!zcache_comp_pcpu_tfms); + tfm = *per_cpu_ptr(zcache_comp_pcpu_tfms, get_cpu()); + BUG_ON(!tfm); + switch (op) { + case ZCACHE_COMPOP_COMPRESS: + ret = crypto_comp_compress(tfm, src, slen, dst, dlen); + break; + case ZCACHE_COMPOP_DECOMPRESS: + ret = crypto_comp_decompress(tfm, src, slen, dst, dlen); + break; + default: + ret = -EINVAL; + } + put_cpu(); + return ret; +} + +/* + * policy parameters + */ + +/* + * byte count defining poor compression; pages with greater zsize will be + * rejected + */ +static unsigned int zbud_max_zsize __read_mostly = (PAGE_SIZE / 8) * 7; +/* + * byte count defining poor *mean* compression; pages with greater zsize + * will be rejected until sufficient better-compressed pages are accepted + * driving the mean below this threshold + */ +static unsigned int zbud_max_mean_zsize __read_mostly = (PAGE_SIZE / 8) * 5; + +/* + * for now, used named slabs so can easily track usage; later can + * either just use kmalloc, or perhaps add a slab-like allocator + * to more carefully manage total memory utilization + */ +static struct kmem_cache *zcache_objnode_cache; +static struct kmem_cache *zcache_obj_cache; + +static DEFINE_PER_CPU(struct zcache_preload, zcache_preloads) = { 0, }; + +/* Used by debug.c */ +ssize_t zcache_pers_zpages; +u64 zcache_pers_zbytes; +ssize_t zcache_eph_pageframes; +ssize_t zcache_pers_pageframes; + +/* Used by this code. */ +ssize_t zcache_last_active_file_pageframes; +ssize_t zcache_last_inactive_file_pageframes; +ssize_t zcache_last_active_anon_pageframes; +ssize_t zcache_last_inactive_anon_pageframes; +#ifdef CONFIG_ZCACHE_WRITEBACK +ssize_t zcache_writtenback_pages; +ssize_t zcache_outstanding_writeback_pages; +#endif +/* + * zcache core code starts here + */ + +static struct zcache_client zcache_host; +static struct zcache_client zcache_clients[MAX_CLIENTS]; + +static inline bool is_local_client(struct zcache_client *cli) +{ + return cli == &zcache_host; +} + +static struct zcache_client *zcache_get_client_by_id(uint16_t cli_id) +{ + struct zcache_client *cli = &zcache_host; + + if (cli_id != LOCAL_CLIENT) { + if (cli_id >= MAX_CLIENTS) + goto out; + cli = &zcache_clients[cli_id]; + } +out: + return cli; +} + +/* + * Tmem operations assume the poolid implies the invoking client. + * Zcache only has one client (the kernel itself): LOCAL_CLIENT. + * RAMster has each client numbered by cluster node, and a KVM version + * of zcache would have one client per guest and each client might + * have a poolid==N. + */ +struct tmem_pool *zcache_get_pool_by_id(uint16_t cli_id, uint16_t poolid) +{ + struct tmem_pool *pool = NULL; + struct zcache_client *cli = NULL; + + cli = zcache_get_client_by_id(cli_id); + if (cli == NULL) + goto out; + if (!is_local_client(cli)) + atomic_inc(&cli->refcount); + if (poolid < MAX_POOLS_PER_CLIENT) { + pool = cli->tmem_pools[poolid]; + if (pool != NULL) + atomic_inc(&pool->refcount); + } +out: + return pool; +} + +void zcache_put_pool(struct tmem_pool *pool) +{ + struct zcache_client *cli = NULL; + + if (pool == NULL) + BUG(); + cli = pool->client; + atomic_dec(&pool->refcount); + if (!is_local_client(cli)) + atomic_dec(&cli->refcount); +} + +int zcache_new_client(uint16_t cli_id) +{ + struct zcache_client *cli; + int ret = -1; + + cli = zcache_get_client_by_id(cli_id); + if (cli == NULL) + goto out; + if (cli->allocated) + goto out; + cli->allocated = 1; + ret = 0; +out: + return ret; +} + +/* + * zcache implementation for tmem host ops + */ + +static struct tmem_objnode *zcache_objnode_alloc(struct tmem_pool *pool) +{ + struct tmem_objnode *objnode = NULL; + struct zcache_preload *kp; + int i; + + kp = &__get_cpu_var(zcache_preloads); + for (i = 0; i < ARRAY_SIZE(kp->objnodes); i++) { + objnode = kp->objnodes[i]; + if (objnode != NULL) { + kp->objnodes[i] = NULL; + break; + } + } + BUG_ON(objnode == NULL); + inc_zcache_objnode_count(); + return objnode; +} + +static void zcache_objnode_free(struct tmem_objnode *objnode, + struct tmem_pool *pool) +{ + dec_zcache_objnode_count(); + kmem_cache_free(zcache_objnode_cache, objnode); +} + +static struct tmem_obj *zcache_obj_alloc(struct tmem_pool *pool) +{ + struct tmem_obj *obj = NULL; + struct zcache_preload *kp; + + kp = &__get_cpu_var(zcache_preloads); + obj = kp->obj; + BUG_ON(obj == NULL); + kp->obj = NULL; + inc_zcache_obj_count(); + return obj; +} + +static void zcache_obj_free(struct tmem_obj *obj, struct tmem_pool *pool) +{ + dec_zcache_obj_count(); + kmem_cache_free(zcache_obj_cache, obj); +} + +/* + * Compressing zero-filled pages will waste memory and introduce + * serious fragmentation, skip it to avoid overhead. + */ +static bool page_is_zero_filled(struct page *p) +{ + unsigned int pos; + char *page; + + page = kmap_atomic(p); + for (pos = 0; pos < PAGE_SIZE / sizeof(*page); pos++) { + if (page[pos]) { + kunmap_atomic(page); + return false; + } + } + kunmap_atomic(page); + + return true; +} + +static void handle_zero_filled_page(void *p) +{ + void *user_mem; + struct page *page = (struct page *)p; + + user_mem = kmap_atomic(page); + memset(user_mem, 0, PAGE_SIZE); + kunmap_atomic(user_mem); + + flush_dcache_page(page); +} + +static struct tmem_hostops zcache_hostops = { + .obj_alloc = zcache_obj_alloc, + .obj_free = zcache_obj_free, + .objnode_alloc = zcache_objnode_alloc, + .objnode_free = zcache_objnode_free, +}; + +static struct page *zcache_alloc_page(void) +{ + struct page *page = alloc_page(ZCACHE_GFP_MASK); + + if (page != NULL) + inc_zcache_pageframes_alloced(); + return page; +} + +static void zcache_free_page(struct page *page) +{ + long curr_pageframes; + static long max_pageframes, min_pageframes; + + if (page == NULL) + BUG(); + __free_page(page); + inc_zcache_pageframes_freed(); + curr_pageframes = curr_pageframes_count(); + if (curr_pageframes > max_pageframes) + max_pageframes = curr_pageframes; + if (curr_pageframes < min_pageframes) + min_pageframes = curr_pageframes; +#ifdef CONFIG_ZCACHE_DEBUG + if (curr_pageframes > 2L || curr_pageframes < -2L) { + /* pr_info here */ + } +#endif +} + +/* + * zcache implementations for PAM page descriptor ops + */ + +/* forward reference */ +static void zcache_compress(struct page *from, + void **out_va, unsigned *out_len); + +static struct page *zcache_evict_eph_pageframe(void); + +static void *zcache_pampd_eph_create(char *data, size_t size, bool raw, + struct tmem_handle *th) +{ + void *pampd = NULL, *cdata = data; + unsigned clen = size; + bool zero_filled = false; + struct page *page = (struct page *)(data), *newpage; + + if (page_is_zero_filled(page)) { + clen = 0; + zero_filled = true; + inc_zcache_zero_filled_pages(); + goto got_pampd; + } + + if (!raw) { + zcache_compress(page, &cdata, &clen); + if (clen > zbud_max_buddy_size()) { + inc_zcache_compress_poor(); + goto out; + } + } else { + BUG_ON(clen > zbud_max_buddy_size()); + } + + /* look for space via an existing match first */ + pampd = (void *)zbud_match_prep(th, true, cdata, clen); + if (pampd != NULL) + goto got_pampd; + + /* no match, now we need to find (or free up) a full page */ + newpage = zcache_alloc_page(); + if (newpage != NULL) + goto create_in_new_page; + + inc_zcache_failed_getfreepages(); + /* can't allocate a page, evict an ephemeral page via LRU */ + newpage = zcache_evict_eph_pageframe(); + if (newpage == NULL) { + inc_zcache_eph_ate_tail_failed(); + goto out; + } + inc_zcache_eph_ate_tail(); + +create_in_new_page: + pampd = (void *)zbud_create_prep(th, true, cdata, clen, newpage); + BUG_ON(pampd == NULL); + inc_zcache_eph_pageframes(); + +got_pampd: + inc_zcache_eph_zbytes(clen); + inc_zcache_eph_zpages(); + if (ramster_enabled && raw && !zero_filled) + ramster_count_foreign_pages(true, 1); + if (zero_filled) + pampd = (void *)ZERO_FILLED; +out: + return pampd; +} + +static void *zcache_pampd_pers_create(char *data, size_t size, bool raw, + struct tmem_handle *th) +{ + void *pampd = NULL, *cdata = data; + unsigned clen = size; + bool zero_filled = false; + struct page *page = (struct page *)(data), *newpage; + unsigned long zbud_mean_zsize; + unsigned long curr_pers_zpages, total_zsize; + + if (data == NULL) { + BUG_ON(!ramster_enabled); + goto create_pampd; + } + + if (page_is_zero_filled(page)) { + clen = 0; + zero_filled = true; + inc_zcache_zero_filled_pages(); + goto got_pampd; + } + + curr_pers_zpages = zcache_pers_zpages; +/* FIXME CONFIG_RAMSTER... subtract atomic remote_pers_pages here? */ + if (!raw) + zcache_compress(page, &cdata, &clen); + /* reject if compression is too poor */ + if (clen > zbud_max_zsize) { + inc_zcache_compress_poor(); + goto out; + } + /* reject if mean compression is too poor */ + if ((clen > zbud_max_mean_zsize) && (curr_pers_zpages > 0)) { + total_zsize = zcache_pers_zbytes; + if ((long)total_zsize < 0) + total_zsize = 0; + zbud_mean_zsize = div_u64(total_zsize, + curr_pers_zpages); + if (zbud_mean_zsize > zbud_max_mean_zsize) { + inc_zcache_mean_compress_poor(); + goto out; + } + } + +create_pampd: + /* look for space via an existing match first */ + pampd = (void *)zbud_match_prep(th, false, cdata, clen); + if (pampd != NULL) + goto got_pampd; + + /* no match, now we need to find (or free up) a full page */ + newpage = zcache_alloc_page(); + if (newpage != NULL) + goto create_in_new_page; + /* + * FIXME do the following only if eph is oversized? + * if (zcache_eph_pageframes > + * (global_page_state(NR_LRU_BASE + LRU_ACTIVE_FILE) + + * global_page_state(NR_LRU_BASE + LRU_INACTIVE_FILE))) + */ + inc_zcache_failed_getfreepages(); + /* can't allocate a page, evict an ephemeral page via LRU */ + newpage = zcache_evict_eph_pageframe(); + if (newpage == NULL) { + inc_zcache_pers_ate_eph_failed(); + goto out; + } + inc_zcache_pers_ate_eph(); + +create_in_new_page: + pampd = (void *)zbud_create_prep(th, false, cdata, clen, newpage); + BUG_ON(pampd == NULL); + inc_zcache_pers_pageframes(); + +got_pampd: + inc_zcache_pers_zpages(); + inc_zcache_pers_zbytes(clen); + if (ramster_enabled && raw && !zero_filled) + ramster_count_foreign_pages(false, 1); + if (zero_filled) + pampd = (void *)ZERO_FILLED; +out: + return pampd; +} + +/* + * This is called directly from zcache_put_page to pre-allocate space + * to store a zpage. + */ +void *zcache_pampd_create(char *data, unsigned int size, bool raw, + int eph, struct tmem_handle *th) +{ + void *pampd = NULL; + struct zcache_preload *kp; + struct tmem_objnode *objnode; + struct tmem_obj *obj; + int i; + + BUG_ON(!irqs_disabled()); + /* pre-allocate per-cpu metadata */ + BUG_ON(zcache_objnode_cache == NULL); + BUG_ON(zcache_obj_cache == NULL); + kp = &__get_cpu_var(zcache_preloads); + for (i = 0; i < ARRAY_SIZE(kp->objnodes); i++) { + objnode = kp->objnodes[i]; + if (objnode == NULL) { + objnode = kmem_cache_alloc(zcache_objnode_cache, + ZCACHE_GFP_MASK); + if (unlikely(objnode == NULL)) { + inc_zcache_failed_alloc(); + goto out; + } + kp->objnodes[i] = objnode; + } + } + if (kp->obj == NULL) { + obj = kmem_cache_alloc(zcache_obj_cache, ZCACHE_GFP_MASK); + kp->obj = obj; + } + if (unlikely(kp->obj == NULL)) { + inc_zcache_failed_alloc(); + goto out; + } + /* + * ok, have all the metadata pre-allocated, now do the data + * but since how we allocate the data is dependent on ephemeral + * or persistent, we split the call here to different sub-functions + */ + if (eph) + pampd = zcache_pampd_eph_create(data, size, raw, th); + else + pampd = zcache_pampd_pers_create(data, size, raw, th); +out: + return pampd; +} + +/* + * This is a pamops called via tmem_put and is necessary to "finish" + * a pampd creation. + */ +void zcache_pampd_create_finish(void *pampd, bool eph) +{ + if (pampd != (void *)ZERO_FILLED) + zbud_create_finish((struct zbudref *)pampd, eph); +} + +/* + * This is passed as a function parameter to zbud_decompress so that + * zbud need not be familiar with the details of crypto. It assumes that + * the bytes from_va and to_va through from_va+size-1 and to_va+size-1 are + * kmapped. It must be successful, else there is a logic bug somewhere. + */ +static void zcache_decompress(char *from_va, unsigned int size, char *to_va) +{ + int ret; + unsigned int outlen = PAGE_SIZE; + + ret = zcache_comp_op(ZCACHE_COMPOP_DECOMPRESS, from_va, size, + to_va, &outlen); + BUG_ON(ret); + BUG_ON(outlen != PAGE_SIZE); +} + +/* + * Decompress from the kernel va to a pageframe + */ +void zcache_decompress_to_page(char *from_va, unsigned int size, + struct page *to_page) +{ + char *to_va = kmap_atomic(to_page); + zcache_decompress(from_va, size, to_va); + kunmap_atomic(to_va); +} + +/* + * fill the pageframe corresponding to the struct page with the data + * from the passed pampd + */ +static int zcache_pampd_get_data(char *data, size_t *sizep, bool raw, + void *pampd, struct tmem_pool *pool, + struct tmem_oid *oid, uint32_t index) +{ + int ret; + bool eph = !is_persistent(pool); + + BUG_ON(preemptible()); + BUG_ON(eph); /* fix later if shared pools get implemented */ + BUG_ON(pampd_is_remote(pampd)); + + if (pampd == (void *)ZERO_FILLED) { + handle_zero_filled_page(data); + if (!raw) + *sizep = PAGE_SIZE; + return 0; + } + + if (raw) + ret = zbud_copy_from_zbud(data, (struct zbudref *)pampd, + sizep, eph); + else { + ret = zbud_decompress((struct page *)(data), + (struct zbudref *)pampd, false, + zcache_decompress); + *sizep = PAGE_SIZE; + } + return ret; +} + +/* + * fill the pageframe corresponding to the struct page with the data + * from the passed pampd + */ +static int zcache_pampd_get_data_and_free(char *data, size_t *sizep, bool raw, + void *pampd, struct tmem_pool *pool, + struct tmem_oid *oid, uint32_t index) +{ + int ret = 0; + bool eph = !is_persistent(pool), zero_filled = false; + struct page *page = NULL; + unsigned int zsize, zpages; + + BUG_ON(preemptible()); + BUG_ON(pampd_is_remote(pampd)); + + if (pampd == (void *)ZERO_FILLED) { + handle_zero_filled_page(data); + zero_filled = true; + zsize = 0; + zpages = 1; + if (!raw) + *sizep = PAGE_SIZE; + dec_zcache_zero_filled_pages(); + goto zero_fill; + } + + if (raw) + ret = zbud_copy_from_zbud(data, (struct zbudref *)pampd, + sizep, eph); + else { + ret = zbud_decompress((struct page *)(data), + (struct zbudref *)pampd, eph, + zcache_decompress); + *sizep = PAGE_SIZE; + } + page = zbud_free_and_delist((struct zbudref *)pampd, eph, + &zsize, &zpages); +zero_fill: + if (eph) { + if (page) + dec_zcache_eph_pageframes(); + dec_zcache_eph_zpages(zpages); + dec_zcache_eph_zbytes(zsize); + } else { + if (page) + dec_zcache_pers_pageframes(); + dec_zcache_pers_zpages(zpages); + dec_zcache_pers_zbytes(zsize); + } + if (!is_local_client(pool->client) && !zero_filled) + ramster_count_foreign_pages(eph, -1); + if (page && !zero_filled) + zcache_free_page(page); + return ret; +} + +/* + * free the pampd and remove it from any zcache lists + * pampd must no longer be pointed to from any tmem data structures! + */ +static void zcache_pampd_free(void *pampd, struct tmem_pool *pool, + struct tmem_oid *oid, uint32_t index, bool acct) +{ + struct page *page = NULL; + unsigned int zsize, zpages; + bool zero_filled = false; + + BUG_ON(preemptible()); + + if (pampd == (void *)ZERO_FILLED) { + zero_filled = true; + zsize = 0; + zpages = 1; + dec_zcache_zero_filled_pages(); + } + + if (pampd_is_remote(pampd) && !zero_filled) { + BUG_ON(!ramster_enabled); + pampd = ramster_pampd_free(pampd, pool, oid, index, acct); + if (pampd == NULL) + return; + } + if (is_ephemeral(pool)) { + if (!zero_filled) + page = zbud_free_and_delist((struct zbudref *)pampd, + true, &zsize, &zpages); + if (page) + dec_zcache_eph_pageframes(); + dec_zcache_eph_zpages(zpages); + dec_zcache_eph_zbytes(zsize); + /* FIXME CONFIG_RAMSTER... check acct parameter? */ + } else { + if (!zero_filled) + page = zbud_free_and_delist((struct zbudref *)pampd, + false, &zsize, &zpages); + if (page) + dec_zcache_pers_pageframes(); + dec_zcache_pers_zpages(zpages); + dec_zcache_pers_zbytes(zsize); + } + if (!is_local_client(pool->client) && !zero_filled) + ramster_count_foreign_pages(is_ephemeral(pool), -1); + if (page && !zero_filled) + zcache_free_page(page); +} + +static struct tmem_pamops zcache_pamops = { + .create_finish = zcache_pampd_create_finish, + .get_data = zcache_pampd_get_data, + .get_data_and_free = zcache_pampd_get_data_and_free, + .free = zcache_pampd_free, +}; + +/* + * zcache compression/decompression and related per-cpu stuff + */ + +static DEFINE_PER_CPU(unsigned char *, zcache_dstmem); +#define ZCACHE_DSTMEM_ORDER 1 + +static void zcache_compress(struct page *from, void **out_va, unsigned *out_len) +{ + int ret; + unsigned char *dmem = __get_cpu_var(zcache_dstmem); + char *from_va; + + BUG_ON(!irqs_disabled()); + /* no buffer or no compressor so can't compress */ + BUG_ON(dmem == NULL); + *out_len = PAGE_SIZE << ZCACHE_DSTMEM_ORDER; + from_va = kmap_atomic(from); + mb(); + ret = zcache_comp_op(ZCACHE_COMPOP_COMPRESS, from_va, PAGE_SIZE, dmem, + out_len); + BUG_ON(ret); + *out_va = dmem; + kunmap_atomic(from_va); +} + +static int zcache_comp_cpu_up(int cpu) +{ + struct crypto_comp *tfm; + + tfm = crypto_alloc_comp(zcache_comp_name, 0, 0); + if (IS_ERR(tfm)) + return NOTIFY_BAD; + *per_cpu_ptr(zcache_comp_pcpu_tfms, cpu) = tfm; + return NOTIFY_OK; +} + +static void zcache_comp_cpu_down(int cpu) +{ + struct crypto_comp *tfm; + + tfm = *per_cpu_ptr(zcache_comp_pcpu_tfms, cpu); + crypto_free_comp(tfm); + *per_cpu_ptr(zcache_comp_pcpu_tfms, cpu) = NULL; +} + +static int zcache_cpu_notifier(struct notifier_block *nb, + unsigned long action, void *pcpu) +{ + int ret, i, cpu = (long)pcpu; + struct zcache_preload *kp; + + switch (action) { + case CPU_UP_PREPARE: + ret = zcache_comp_cpu_up(cpu); + if (ret != NOTIFY_OK) { + pr_err("%s: can't allocate compressor xform\n", + namestr); + return ret; + } + per_cpu(zcache_dstmem, cpu) = (void *)__get_free_pages( + GFP_KERNEL | __GFP_REPEAT, ZCACHE_DSTMEM_ORDER); + if (ramster_enabled) + ramster_cpu_up(cpu); + break; + case CPU_DEAD: + case CPU_UP_CANCELED: + zcache_comp_cpu_down(cpu); + free_pages((unsigned long)per_cpu(zcache_dstmem, cpu), + ZCACHE_DSTMEM_ORDER); + per_cpu(zcache_dstmem, cpu) = NULL; + kp = &per_cpu(zcache_preloads, cpu); + for (i = 0; i < ARRAY_SIZE(kp->objnodes); i++) { + if (kp->objnodes[i]) + kmem_cache_free(zcache_objnode_cache, + kp->objnodes[i]); + } + if (kp->obj) { + kmem_cache_free(zcache_obj_cache, kp->obj); + kp->obj = NULL; + } + if (ramster_enabled) + ramster_cpu_down(cpu); + break; + default: + break; + } + return NOTIFY_OK; +} + +static struct notifier_block zcache_cpu_notifier_block = { + .notifier_call = zcache_cpu_notifier +}; + +/* + * The following code interacts with the zbud eviction and zbud + * zombify code to access LRU pages + */ + +static struct page *zcache_evict_eph_pageframe(void) +{ + struct page *page; + unsigned int zsize = 0, zpages = 0; + + page = zbud_evict_pageframe_lru(&zsize, &zpages); + if (page == NULL) + goto out; + dec_zcache_eph_zbytes(zsize); + dec_zcache_eph_zpages(zpages); + inc_zcache_evicted_eph_zpages(zpages); + dec_zcache_eph_pageframes(); + inc_zcache_evicted_eph_pageframes(); +out: + return page; +} + +#ifdef CONFIG_ZCACHE_WRITEBACK + +static atomic_t zcache_outstanding_writeback_pages_atomic = ATOMIC_INIT(0); + +static inline void inc_zcache_outstanding_writeback_pages(void) +{ + zcache_outstanding_writeback_pages = + atomic_inc_return(&zcache_outstanding_writeback_pages_atomic); +} +static inline void dec_zcache_outstanding_writeback_pages(void) +{ + zcache_outstanding_writeback_pages = + atomic_dec_return(&zcache_outstanding_writeback_pages_atomic); +}; +static void unswiz(struct tmem_oid oid, u32 index, + unsigned *type, pgoff_t *offset); + +/* + * Choose an LRU persistent pageframe and attempt to write it back to + * the backing swap disk by calling frontswap_writeback on both zpages. + * + * This is work-in-progress. + */ + +static void zcache_end_swap_write(struct bio *bio, int err) +{ + end_swap_bio_write(bio, err); + dec_zcache_outstanding_writeback_pages(); + zcache_writtenback_pages++; +} + +/* + * zcache_get_swap_cache_page + * + * This is an adaption of read_swap_cache_async() + * + * If success, page is returned in retpage + * Returns 0 if page was already in the swap cache, page is not locked + * Returns 1 if the new page needs to be populated, page is locked + */ +static int zcache_get_swap_cache_page(int type, pgoff_t offset, + struct page *new_page) +{ + struct page *found_page; + swp_entry_t entry = swp_entry(type, offset); + int err; + + BUG_ON(new_page == NULL); + do { + /* + * First check the swap cache. Since this is normally + * called after lookup_swap_cache() failed, re-calling + * that would confuse statistics. + */ + found_page = find_get_page(&swapper_space, entry.val); + if (found_page) + return 0; + + /* + * call radix_tree_preload() while we can wait. + */ + err = radix_tree_preload(GFP_KERNEL); + if (err) + break; + + /* + * Swap entry may have been freed since our caller observed it. + */ + err = swapcache_prepare(entry); + if (err == -EEXIST) { /* seems racy */ + radix_tree_preload_end(); + continue; + } + if (err) { /* swp entry is obsolete ? */ + radix_tree_preload_end(); + break; + } + + /* May fail (-ENOMEM) if radix-tree node allocation failed. */ + __set_page_locked(new_page); + SetPageSwapBacked(new_page); + err = __add_to_swap_cache(new_page, entry); + if (likely(!err)) { + radix_tree_preload_end(); + lru_cache_add_anon(new_page); + return 1; + } + radix_tree_preload_end(); + ClearPageSwapBacked(new_page); + __clear_page_locked(new_page); + /* + * add_to_swap_cache() doesn't return -EEXIST, so we can safely + * clear SWAP_HAS_CACHE flag. + */ + swapcache_free(entry, NULL); + /* FIXME: is it possible to get here without err==-ENOMEM? + * If not, we can dispense with the do loop, use goto retry */ + } while (err != -ENOMEM); + + return -ENOMEM; +} + +/* + * Given a frontswap zpage in zcache (identified by type/offset) and + * an empty page, put the page into the swap cache, use frontswap + * to get the page from zcache into the empty page, then give it + * to the swap subsystem to send to disk (carefully avoiding the + * possibility that frontswap might snatch it back). + * Returns < 0 if error, 0 if successful, and 1 if successful but + * the newpage passed in not needed and should be freed. + */ +static int zcache_frontswap_writeback_zpage(int type, pgoff_t offset, + struct page *newpage) +{ + struct page *page = newpage; + int ret; + struct writeback_control wbc = { + .sync_mode = WB_SYNC_NONE, + }; + + ret = zcache_get_swap_cache_page(type, offset, page); + if (ret < 0) + return ret; + else if (ret == 0) { + /* more uptodate page is already in swapcache */ + __frontswap_invalidate_page(type, offset); + return 1; + } + + BUG_ON(!frontswap_has_exclusive_gets); /* load must also invalidate */ + /* FIXME: how is it possible to get here when page is unlocked? */ + __frontswap_load(page); + SetPageUptodate(page); /* above does SetPageDirty, is that enough? */ + + /* start writeback */ + SetPageReclaim(page); + /* + * Return value is ignored here because it doesn't change anything + * for us. Page is returned unlocked. + */ + (void)__swap_writepage(page, &wbc, zcache_end_swap_write); + page_cache_release(page); + inc_zcache_outstanding_writeback_pages(); + + return 0; +} + +/* + * The following is still a magic number... we want to allow forward progress + * for writeback because it clears out needed RAM when under pressure, but + * we don't want to allow writeback to absorb and queue too many GFP_KERNEL + * pages if the swap device is very slow. + */ +#define ZCACHE_MAX_OUTSTANDING_WRITEBACK_PAGES 6400 + +/* + * Try to allocate two free pages, first using a non-aggressive alloc, + * then by evicting zcache ephemeral (clean pagecache) pages, and last + * by aggressive GFP_KERNEL alloc. We allow zbud to choose a pageframe + * consisting of 1-2 zbuds/zpages, then call the writeback_zpage helper + * function above for each. + */ +static int zcache_frontswap_writeback(void) +{ + struct tmem_handle th[2]; + int ret = 0; + int nzbuds, writeback_ret; + unsigned type; + struct page *znewpage1 = NULL, *znewpage2 = NULL; + struct page *evictpage1 = NULL, *evictpage2 = NULL; + struct page *newpage1 = NULL, *newpage2 = NULL; + struct page *page1 = NULL, *page2 = NULL; + pgoff_t offset; + + znewpage1 = alloc_page(ZCACHE_GFP_MASK); + znewpage2 = alloc_page(ZCACHE_GFP_MASK); + if (znewpage1 == NULL) + evictpage1 = zcache_evict_eph_pageframe(); + if (znewpage2 == NULL) + evictpage2 = zcache_evict_eph_pageframe(); + + if ((evictpage1 == NULL || evictpage2 == NULL) && + atomic_read(&zcache_outstanding_writeback_pages_atomic) > + ZCACHE_MAX_OUTSTANDING_WRITEBACK_PAGES) { + goto free_and_out; + } + if (znewpage1 == NULL && evictpage1 == NULL) + newpage1 = alloc_page(GFP_KERNEL); + if (znewpage2 == NULL && evictpage2 == NULL) + newpage2 = alloc_page(GFP_KERNEL); + if (newpage1 == NULL || newpage2 == NULL) + goto free_and_out; + + /* ok, we have two pageframes pre-allocated, get a pair of zbuds */ + nzbuds = zbud_make_zombie_lru(&th[0], NULL, NULL, false); + if (nzbuds == 0) { + ret = -ENOENT; + goto free_and_out; + } + + /* process the first zbud */ + unswiz(th[0].oid, th[0].index, &type, &offset); + page1 = (znewpage1 != NULL) ? znewpage1 : + ((newpage1 != NULL) ? newpage1 : evictpage1); + writeback_ret = zcache_frontswap_writeback_zpage(type, offset, page1); + if (writeback_ret < 0) { + ret = -ENOMEM; + goto free_and_out; + } + if (evictpage1 != NULL) + zcache_pageframes_freed = + atomic_inc_return(&zcache_pageframes_freed_atomic); + if (writeback_ret == 0) { + /* zcache_get_swap_cache_page will free, don't double free */ + znewpage1 = NULL; + newpage1 = NULL; + evictpage1 = NULL; + } + if (nzbuds < 2) + goto free_and_out; + + /* if there is a second zbud, process it */ + unswiz(th[1].oid, th[1].index, &type, &offset); + page2 = (znewpage2 != NULL) ? znewpage2 : + ((newpage2 != NULL) ? newpage2 : evictpage2); + writeback_ret = zcache_frontswap_writeback_zpage(type, offset, page2); + if (writeback_ret < 0) { + ret = -ENOMEM; + goto free_and_out; + } + if (evictpage2 != NULL) + zcache_pageframes_freed = + atomic_inc_return(&zcache_pageframes_freed_atomic); + if (writeback_ret == 0) { + znewpage2 = NULL; + newpage2 = NULL; + evictpage2 = NULL; + } + +free_and_out: + if (znewpage1 != NULL) + page_cache_release(znewpage1); + if (znewpage2 != NULL) + page_cache_release(znewpage2); + if (newpage1 != NULL) + page_cache_release(newpage1); + if (newpage2 != NULL) + page_cache_release(newpage2); + if (evictpage1 != NULL) + zcache_free_page(evictpage1); + if (evictpage2 != NULL) + zcache_free_page(evictpage2); + return ret; +} +#endif /* CONFIG_ZCACHE_WRITEBACK */ + +/* + * When zcache is disabled ("frozen"), pools can be created and destroyed, + * but all puts (and thus all other operations that require memory allocation) + * must fail. If zcache is unfrozen, accepts puts, then frozen again, + * data consistency requires all puts while frozen to be converted into + * flushes. + */ +static bool zcache_freeze; + +/* + * This zcache shrinker interface reduces the number of ephemeral pageframes + * used by zcache to approximately the same as the total number of LRU_FILE + * pageframes in use, and now also reduces the number of persistent pageframes + * used by zcache to approximately the same as the total number of LRU_ANON + * pageframes in use. FIXME POLICY: Probably the writeback should only occur + * if the eviction doesn't free enough pages. + */ +static int shrink_zcache_memory(struct shrinker *shrink, + struct shrink_control *sc) +{ + static bool in_progress; + int ret = -1; + int nr = sc->nr_to_scan; + int nr_evict = 0; + int nr_writeback = 0; + struct page *page; + int file_pageframes_inuse, anon_pageframes_inuse; + + if (nr <= 0) + goto skip_evict; + + /* don't allow more than one eviction thread at a time */ + if (in_progress) + goto skip_evict; + + in_progress = true; + + /* we are going to ignore nr, and target a different value */ + zcache_last_active_file_pageframes = + global_page_state(NR_LRU_BASE + LRU_ACTIVE_FILE); + zcache_last_inactive_file_pageframes = + global_page_state(NR_LRU_BASE + LRU_INACTIVE_FILE); + file_pageframes_inuse = zcache_last_active_file_pageframes + + zcache_last_inactive_file_pageframes; + if (zcache_eph_pageframes > file_pageframes_inuse) + nr_evict = zcache_eph_pageframes - file_pageframes_inuse; + else + nr_evict = 0; + while (nr_evict-- > 0) { + page = zcache_evict_eph_pageframe(); + if (page == NULL) + break; + zcache_free_page(page); + } + + zcache_last_active_anon_pageframes = + global_page_state(NR_LRU_BASE + LRU_ACTIVE_ANON); + zcache_last_inactive_anon_pageframes = + global_page_state(NR_LRU_BASE + LRU_INACTIVE_ANON); + anon_pageframes_inuse = zcache_last_active_anon_pageframes + + zcache_last_inactive_anon_pageframes; + if (zcache_pers_pageframes > anon_pageframes_inuse) + nr_writeback = zcache_pers_pageframes - anon_pageframes_inuse; + else + nr_writeback = 0; + while (nr_writeback-- > 0) { +#ifdef CONFIG_ZCACHE_WRITEBACK + int writeback_ret; + writeback_ret = zcache_frontswap_writeback(); + if (writeback_ret == -ENOMEM) +#endif + break; + } + in_progress = false; + +skip_evict: + /* resample: has changed, but maybe not all the way yet */ + zcache_last_active_file_pageframes = + global_page_state(NR_LRU_BASE + LRU_ACTIVE_FILE); + zcache_last_inactive_file_pageframes = + global_page_state(NR_LRU_BASE + LRU_INACTIVE_FILE); + ret = zcache_eph_pageframes - zcache_last_active_file_pageframes + + zcache_last_inactive_file_pageframes; + if (ret < 0) + ret = 0; + return ret; +} + +static struct shrinker zcache_shrinker = { + .shrink = shrink_zcache_memory, + .seeks = DEFAULT_SEEKS, +}; + +/* + * zcache shims between cleancache/frontswap ops and tmem + */ + +/* FIXME rename these core routines to zcache_tmemput etc? */ +int zcache_put_page(int cli_id, int pool_id, struct tmem_oid *oidp, + uint32_t index, void *page, + unsigned int size, bool raw, int ephemeral) +{ + struct tmem_pool *pool; + struct tmem_handle th; + int ret = -1; + void *pampd = NULL; + + BUG_ON(!irqs_disabled()); + pool = zcache_get_pool_by_id(cli_id, pool_id); + if (unlikely(pool == NULL)) + goto out; + if (!zcache_freeze) { + ret = 0; + th.client_id = cli_id; + th.pool_id = pool_id; + th.oid = *oidp; + th.index = index; + pampd = zcache_pampd_create((char *)page, size, raw, + ephemeral, &th); + if (pampd == NULL) { + ret = -ENOMEM; + if (ephemeral) + inc_zcache_failed_eph_puts(); + else + inc_zcache_failed_pers_puts(); + } else { + if (ramster_enabled) + ramster_do_preload_flnode(pool); + ret = tmem_put(pool, oidp, index, 0, pampd); + if (ret < 0) + BUG(); + } + zcache_put_pool(pool); + } else { + inc_zcache_put_to_flush(); + if (ramster_enabled) + ramster_do_preload_flnode(pool); + if (atomic_read(&pool->obj_count) > 0) + /* the put fails whether the flush succeeds or not */ + (void)tmem_flush_page(pool, oidp, index); + zcache_put_pool(pool); + } +out: + return ret; +} + +int zcache_get_page(int cli_id, int pool_id, struct tmem_oid *oidp, + uint32_t index, void *page, + size_t *sizep, bool raw, int get_and_free) +{ + struct tmem_pool *pool; + int ret = -1; + bool eph; + + if (!raw) { + BUG_ON(irqs_disabled()); + BUG_ON(in_softirq()); + } + pool = zcache_get_pool_by_id(cli_id, pool_id); + eph = is_ephemeral(pool); + if (likely(pool != NULL)) { + if (atomic_read(&pool->obj_count) > 0) + ret = tmem_get(pool, oidp, index, (char *)(page), + sizep, raw, get_and_free); + zcache_put_pool(pool); + } + WARN_ONCE((!is_ephemeral(pool) && (ret != 0)), + "zcache_get fails on persistent pool, " + "bad things are very likely to happen soon\n"); +#ifdef RAMSTER_TESTING + if (ret != 0 && ret != -1 && !(ret == -EINVAL && is_ephemeral(pool))) + pr_err("TESTING zcache_get tmem_get returns ret=%d\n", ret); +#endif + return ret; +} + +int zcache_flush_page(int cli_id, int pool_id, + struct tmem_oid *oidp, uint32_t index) +{ + struct tmem_pool *pool; + int ret = -1; + unsigned long flags; + + local_irq_save(flags); + inc_zcache_flush_total(); + pool = zcache_get_pool_by_id(cli_id, pool_id); + if (ramster_enabled) + ramster_do_preload_flnode(pool); + if (likely(pool != NULL)) { + if (atomic_read(&pool->obj_count) > 0) + ret = tmem_flush_page(pool, oidp, index); + zcache_put_pool(pool); + } + if (ret >= 0) + inc_zcache_flush_found(); + local_irq_restore(flags); + return ret; +} + +int zcache_flush_object(int cli_id, int pool_id, + struct tmem_oid *oidp) +{ + struct tmem_pool *pool; + int ret = -1; + unsigned long flags; + + local_irq_save(flags); + inc_zcache_flobj_total(); + pool = zcache_get_pool_by_id(cli_id, pool_id); + if (ramster_enabled) + ramster_do_preload_flnode(pool); + if (likely(pool != NULL)) { + if (atomic_read(&pool->obj_count) > 0) + ret = tmem_flush_object(pool, oidp); + zcache_put_pool(pool); + } + if (ret >= 0) + inc_zcache_flobj_found(); + local_irq_restore(flags); + return ret; +} + +static int zcache_client_destroy_pool(int cli_id, int pool_id) +{ + struct tmem_pool *pool = NULL; + struct zcache_client *cli = NULL; + int ret = -1; + + if (pool_id < 0) + goto out; + if (cli_id == LOCAL_CLIENT) + cli = &zcache_host; + else if ((unsigned int)cli_id < MAX_CLIENTS) + cli = &zcache_clients[cli_id]; + if (cli == NULL) + goto out; + atomic_inc(&cli->refcount); + pool = cli->tmem_pools[pool_id]; + if (pool == NULL) + goto out; + cli->tmem_pools[pool_id] = NULL; + /* wait for pool activity on other cpus to quiesce */ + while (atomic_read(&pool->refcount) != 0) + ; + atomic_dec(&cli->refcount); + local_bh_disable(); + ret = tmem_destroy_pool(pool); + local_bh_enable(); + kfree(pool); + if (cli_id == LOCAL_CLIENT) + pr_info("%s: destroyed local pool id=%d\n", namestr, pool_id); + else + pr_info("%s: destroyed pool id=%d, client=%d\n", + namestr, pool_id, cli_id); +out: + return ret; +} + +int zcache_new_pool(uint16_t cli_id, uint32_t flags) +{ + int poolid = -1; + struct tmem_pool *pool; + struct zcache_client *cli = NULL; + + if (cli_id == LOCAL_CLIENT) + cli = &zcache_host; + else if ((unsigned int)cli_id < MAX_CLIENTS) + cli = &zcache_clients[cli_id]; + if (cli == NULL) + goto out; + atomic_inc(&cli->refcount); + pool = kmalloc(sizeof(struct tmem_pool), GFP_ATOMIC); + if (pool == NULL) + goto out; + + for (poolid = 0; poolid < MAX_POOLS_PER_CLIENT; poolid++) + if (cli->tmem_pools[poolid] == NULL) + break; + if (poolid >= MAX_POOLS_PER_CLIENT) { + pr_info("%s: pool creation failed: max exceeded\n", namestr); + kfree(pool); + poolid = -1; + goto out; + } + atomic_set(&pool->refcount, 0); + pool->client = cli; + pool->pool_id = poolid; + tmem_new_pool(pool, flags); + cli->tmem_pools[poolid] = pool; + if (cli_id == LOCAL_CLIENT) + pr_info("%s: created %s local tmem pool, id=%d\n", namestr, + flags & TMEM_POOL_PERSIST ? "persistent" : "ephemeral", + poolid); + else + pr_info("%s: created %s tmem pool, id=%d, client=%d\n", namestr, + flags & TMEM_POOL_PERSIST ? "persistent" : "ephemeral", + poolid, cli_id); +out: + if (cli != NULL) + atomic_dec(&cli->refcount); + return poolid; +} + +static int zcache_local_new_pool(uint32_t flags) +{ + return zcache_new_pool(LOCAL_CLIENT, flags); +} + +int zcache_autocreate_pool(unsigned int cli_id, unsigned int pool_id, bool eph) +{ + struct tmem_pool *pool; + struct zcache_client *cli = NULL; + uint32_t flags = eph ? 0 : TMEM_POOL_PERSIST; + int ret = -1; + + BUG_ON(!ramster_enabled); + if (cli_id == LOCAL_CLIENT) + goto out; + if (pool_id >= MAX_POOLS_PER_CLIENT) + goto out; + if (cli_id >= MAX_CLIENTS) + goto out; + + cli = &zcache_clients[cli_id]; + if ((eph && disable_cleancache) || (!eph && disable_frontswap)) { + pr_err("zcache_autocreate_pool: pool type disabled\n"); + goto out; + } + if (!cli->allocated) { + if (zcache_new_client(cli_id)) { + pr_err("zcache_autocreate_pool: can't create client\n"); + goto out; + } + cli = &zcache_clients[cli_id]; + } + atomic_inc(&cli->refcount); + pool = cli->tmem_pools[pool_id]; + if (pool != NULL) { + if (pool->persistent && eph) { + pr_err("zcache_autocreate_pool: type mismatch\n"); + goto out; + } + ret = 0; + goto out; + } + pool = kmalloc(sizeof(struct tmem_pool), GFP_KERNEL); + if (pool == NULL) + goto out; + + atomic_set(&pool->refcount, 0); + pool->client = cli; + pool->pool_id = pool_id; + tmem_new_pool(pool, flags); + cli->tmem_pools[pool_id] = pool; + pr_info("%s: AUTOcreated %s tmem poolid=%d, for remote client=%d\n", + namestr, flags & TMEM_POOL_PERSIST ? "persistent" : "ephemeral", + pool_id, cli_id); + ret = 0; +out: + if (cli != NULL) + atomic_dec(&cli->refcount); + return ret; +} + +/********** + * Two kernel functionalities currently can be layered on top of tmem. + * These are "cleancache" which is used as a second-chance cache for clean + * page cache pages; and "frontswap" which is used for swap pages + * to avoid writes to disk. A generic "shim" is provided here for each + * to translate in-kernel semantics to zcache semantics. + */ + +static void zcache_cleancache_put_page(int pool_id, + struct cleancache_filekey key, + pgoff_t index, struct page *page) +{ + u32 ind = (u32) index; + struct tmem_oid oid = *(struct tmem_oid *)&key; + + if (!disable_cleancache_ignore_nonactive && !PageWasActive(page)) { + inc_zcache_eph_nonactive_puts_ignored(); + return; + } + if (likely(ind == index)) + (void)zcache_put_page(LOCAL_CLIENT, pool_id, &oid, index, + page, PAGE_SIZE, false, 1); +} + +static int zcache_cleancache_get_page(int pool_id, + struct cleancache_filekey key, + pgoff_t index, struct page *page) +{ + u32 ind = (u32) index; + struct tmem_oid oid = *(struct tmem_oid *)&key; + size_t size; + int ret = -1; + + if (likely(ind == index)) { + ret = zcache_get_page(LOCAL_CLIENT, pool_id, &oid, index, + page, &size, false, 0); + BUG_ON(ret >= 0 && size != PAGE_SIZE); + if (ret == 0) + SetPageWasActive(page); + } + return ret; +} + +static void zcache_cleancache_flush_page(int pool_id, + struct cleancache_filekey key, + pgoff_t index) +{ + u32 ind = (u32) index; + struct tmem_oid oid = *(struct tmem_oid *)&key; + + if (likely(ind == index)) + (void)zcache_flush_page(LOCAL_CLIENT, pool_id, &oid, ind); +} + +static void zcache_cleancache_flush_inode(int pool_id, + struct cleancache_filekey key) +{ + struct tmem_oid oid = *(struct tmem_oid *)&key; + + (void)zcache_flush_object(LOCAL_CLIENT, pool_id, &oid); +} + +static void zcache_cleancache_flush_fs(int pool_id) +{ + if (pool_id >= 0) + (void)zcache_client_destroy_pool(LOCAL_CLIENT, pool_id); +} + +static int zcache_cleancache_init_fs(size_t pagesize) +{ + BUG_ON(sizeof(struct cleancache_filekey) != + sizeof(struct tmem_oid)); + BUG_ON(pagesize != PAGE_SIZE); + return zcache_local_new_pool(0); +} + +static int zcache_cleancache_init_shared_fs(char *uuid, size_t pagesize) +{ + /* shared pools are unsupported and map to private */ + BUG_ON(sizeof(struct cleancache_filekey) != + sizeof(struct tmem_oid)); + BUG_ON(pagesize != PAGE_SIZE); + return zcache_local_new_pool(0); +} + +static struct cleancache_ops zcache_cleancache_ops = { + .put_page = zcache_cleancache_put_page, + .get_page = zcache_cleancache_get_page, + .invalidate_page = zcache_cleancache_flush_page, + .invalidate_inode = zcache_cleancache_flush_inode, + .invalidate_fs = zcache_cleancache_flush_fs, + .init_shared_fs = zcache_cleancache_init_shared_fs, + .init_fs = zcache_cleancache_init_fs +}; + +struct cleancache_ops *zcache_cleancache_register_ops(void) +{ + struct cleancache_ops *old_ops = + cleancache_register_ops(&zcache_cleancache_ops); + + return old_ops; +} + +/* a single tmem poolid is used for all frontswap "types" (swapfiles) */ +static int zcache_frontswap_poolid __read_mostly = -1; + +/* + * Swizzling increases objects per swaptype, increasing tmem concurrency + * for heavy swaploads. Later, larger nr_cpus -> larger SWIZ_BITS + * Setting SWIZ_BITS to 27 basically reconstructs the swap entry from + * frontswap_get_page(), but has side-effects. Hence using 8. + */ +#define SWIZ_BITS 8 +#define SWIZ_MASK ((1 << SWIZ_BITS) - 1) +#define _oswiz(_type, _ind) ((_type << SWIZ_BITS) | (_ind & SWIZ_MASK)) +#define iswiz(_ind) (_ind >> SWIZ_BITS) + +static inline struct tmem_oid oswiz(unsigned type, u32 ind) +{ + struct tmem_oid oid = { .oid = { 0 } }; + oid.oid[0] = _oswiz(type, ind); + return oid; +} + +#ifdef CONFIG_ZCACHE_WRITEBACK +static void unswiz(struct tmem_oid oid, u32 index, + unsigned *type, pgoff_t *offset) +{ + *type = (unsigned)(oid.oid[0] >> SWIZ_BITS); + *offset = (pgoff_t)((index << SWIZ_BITS) | + (oid.oid[0] & SWIZ_MASK)); +} +#endif + +static int zcache_frontswap_put_page(unsigned type, pgoff_t offset, + struct page *page) +{ + u64 ind64 = (u64)offset; + u32 ind = (u32)offset; + struct tmem_oid oid = oswiz(type, ind); + int ret = -1; + unsigned long flags; + + BUG_ON(!PageLocked(page)); + if (!disable_frontswap_ignore_nonactive && !PageWasActive(page)) { + inc_zcache_pers_nonactive_puts_ignored(); + ret = -ERANGE; + goto out; + } + if (likely(ind64 == ind)) { + local_irq_save(flags); + ret = zcache_put_page(LOCAL_CLIENT, zcache_frontswap_poolid, + &oid, iswiz(ind), + page, PAGE_SIZE, false, 0); + local_irq_restore(flags); + } +out: + return ret; +} + +/* returns 0 if the page was successfully gotten from frontswap, -1 if + * was not present (should never happen!) */ +static int zcache_frontswap_get_page(unsigned type, pgoff_t offset, + struct page *page) +{ + u64 ind64 = (u64)offset; + u32 ind = (u32)offset; + struct tmem_oid oid = oswiz(type, ind); + size_t size; + int ret = -1, get_and_free; + + if (frontswap_has_exclusive_gets) + get_and_free = 1; + else + get_and_free = -1; + BUG_ON(!PageLocked(page)); + if (likely(ind64 == ind)) { + ret = zcache_get_page(LOCAL_CLIENT, zcache_frontswap_poolid, + &oid, iswiz(ind), + page, &size, false, get_and_free); + BUG_ON(ret >= 0 && size != PAGE_SIZE); + } + return ret; +} + +/* flush a single page from frontswap */ +static void zcache_frontswap_flush_page(unsigned type, pgoff_t offset) +{ + u64 ind64 = (u64)offset; + u32 ind = (u32)offset; + struct tmem_oid oid = oswiz(type, ind); + + if (likely(ind64 == ind)) + (void)zcache_flush_page(LOCAL_CLIENT, zcache_frontswap_poolid, + &oid, iswiz(ind)); +} + +/* flush all pages from the passed swaptype */ +static void zcache_frontswap_flush_area(unsigned type) +{ + struct tmem_oid oid; + int ind; + + for (ind = SWIZ_MASK; ind >= 0; ind--) { + oid = oswiz(type, ind); + (void)zcache_flush_object(LOCAL_CLIENT, + zcache_frontswap_poolid, &oid); + } +} + +static void zcache_frontswap_init(unsigned ignored) +{ + /* a single tmem poolid is used for all frontswap "types" (swapfiles) */ + if (zcache_frontswap_poolid < 0) + zcache_frontswap_poolid = + zcache_local_new_pool(TMEM_POOL_PERSIST); +} + +static struct frontswap_ops zcache_frontswap_ops = { + .store = zcache_frontswap_put_page, + .load = zcache_frontswap_get_page, + .invalidate_page = zcache_frontswap_flush_page, + .invalidate_area = zcache_frontswap_flush_area, + .init = zcache_frontswap_init +}; + +struct frontswap_ops *zcache_frontswap_register_ops(void) +{ + struct frontswap_ops *old_ops = + frontswap_register_ops(&zcache_frontswap_ops); + + return old_ops; +} + +/* + * zcache initialization + * NOTE FOR NOW zcache or ramster MUST BE PROVIDED AS A KERNEL BOOT PARAMETER + * OR NOTHING HAPPENS! + */ + +#ifndef CONFIG_ZCACHE_MODULE +static int __init enable_zcache(char *s) +{ + zcache_enabled = true; + return 1; +} +__setup("zcache", enable_zcache); + +static int __init enable_ramster(char *s) +{ + zcache_enabled = true; +#ifdef CONFIG_RAMSTER + ramster_enabled = true; +#endif + return 1; +} +__setup("ramster", enable_ramster); + +/* allow independent dynamic disabling of cleancache and frontswap */ + +static int __init no_cleancache(char *s) +{ + disable_cleancache = true; + return 1; +} + +__setup("nocleancache", no_cleancache); + +static int __init no_frontswap(char *s) +{ + disable_frontswap = true; + return 1; +} + +__setup("nofrontswap", no_frontswap); + +static int __init no_frontswap_exclusive_gets(char *s) +{ + frontswap_has_exclusive_gets = false; + return 1; +} + +__setup("nofrontswapexclusivegets", no_frontswap_exclusive_gets); + +static int __init no_frontswap_ignore_nonactive(char *s) +{ + disable_frontswap_ignore_nonactive = true; + return 1; +} + +__setup("nofrontswapignorenonactive", no_frontswap_ignore_nonactive); + +static int __init no_cleancache_ignore_nonactive(char *s) +{ + disable_cleancache_ignore_nonactive = true; + return 1; +} + +__setup("nocleancacheignorenonactive", no_cleancache_ignore_nonactive); + +static int __init enable_zcache_compressor(char *s) +{ + strlcpy(zcache_comp_name, s, sizeof(zcache_comp_name)); + zcache_enabled = true; + return 1; +} +__setup("zcache=", enable_zcache_compressor); +#endif + + +static int zcache_comp_init(void) +{ + int ret = 0; + + /* check crypto algorithm */ +#ifdef CONFIG_ZCACHE_MODULE + ret = crypto_has_comp(zcache_comp_name, 0, 0); + if (!ret) { + ret = -1; + goto out; + } +#else + if (*zcache_comp_name != '\0') { + ret = crypto_has_comp(zcache_comp_name, 0, 0); + if (!ret) { + pr_info("zcache: %s not supported\n", + zcache_comp_name); + ret = 1; + goto out; + } + } + if (!ret) + strcpy(zcache_comp_name, "lzo"); + ret = crypto_has_comp(zcache_comp_name, 0, 0); + if (!ret) { + ret = 1; + goto out; + } +#endif + pr_info("zcache: using %s compressor\n", zcache_comp_name); + + /* alloc percpu transforms */ + ret = 0; + zcache_comp_pcpu_tfms = alloc_percpu(struct crypto_comp *); + if (!zcache_comp_pcpu_tfms) + ret = 1; +out: + return ret; +} + +static int zcache_init(void) +{ + int ret = 0; + +#ifdef CONFIG_ZCACHE_MODULE + zcache_enabled = 1; +#endif + if (ramster_enabled) { + namestr = "ramster"; + ramster_register_pamops(&zcache_pamops); + } + zcache_debugfs_init(); + if (zcache_enabled) { + unsigned int cpu; + + tmem_register_hostops(&zcache_hostops); + tmem_register_pamops(&zcache_pamops); + ret = register_cpu_notifier(&zcache_cpu_notifier_block); + if (ret) { + pr_err("%s: can't register cpu notifier\n", namestr); + goto out; + } + ret = zcache_comp_init(); + if (ret) { + pr_err("%s: compressor initialization failed\n", + namestr); + goto out; + } + for_each_online_cpu(cpu) { + void *pcpu = (void *)(long)cpu; + zcache_cpu_notifier(&zcache_cpu_notifier_block, + CPU_UP_PREPARE, pcpu); + } + } + zcache_objnode_cache = kmem_cache_create("zcache_objnode", + sizeof(struct tmem_objnode), 0, 0, NULL); + zcache_obj_cache = kmem_cache_create("zcache_obj", + sizeof(struct tmem_obj), 0, 0, NULL); + ret = zcache_new_client(LOCAL_CLIENT); + if (ret) { + pr_err("%s: can't create client\n", namestr); + goto out; + } + zbud_init(); + if (zcache_enabled && !disable_cleancache) { + struct cleancache_ops *old_ops; + + register_shrinker(&zcache_shrinker); + old_ops = zcache_cleancache_register_ops(); + pr_info("%s: cleancache enabled using kernel transcendent " + "memory and compression buddies\n", namestr); +#ifdef CONFIG_ZCACHE_DEBUG + pr_info("%s: cleancache: ignorenonactive = %d\n", + namestr, !disable_cleancache_ignore_nonactive); +#endif + if (old_ops != NULL) + pr_warn("%s: cleancache_ops overridden\n", namestr); + } + if (zcache_enabled && !disable_frontswap) { + struct frontswap_ops *old_ops; + + old_ops = zcache_frontswap_register_ops(); + if (frontswap_has_exclusive_gets) + frontswap_tmem_exclusive_gets(true); + pr_info("%s: frontswap enabled using kernel transcendent " + "memory and compression buddies\n", namestr); +#ifdef CONFIG_ZCACHE_DEBUG + pr_info("%s: frontswap: excl gets = %d active only = %d\n", + namestr, frontswap_has_exclusive_gets, + !disable_frontswap_ignore_nonactive); +#endif + if (IS_ERR(old_ops) || old_ops) { + if (IS_ERR(old_ops)) + return PTR_RET(old_ops); + pr_warn("%s: frontswap_ops overridden\n", namestr); + } + } + if (ramster_enabled) + ramster_init(!disable_cleancache, !disable_frontswap, + frontswap_has_exclusive_gets, + !disable_frontswap_selfshrink); +out: + return ret; +} + +#ifdef CONFIG_ZCACHE_MODULE +#ifdef CONFIG_RAMSTER +module_param(ramster_enabled, bool, S_IRUGO); +module_param(disable_frontswap_selfshrink, int, S_IRUGO); +#endif +module_param(disable_cleancache, bool, S_IRUGO); +module_param(disable_frontswap, bool, S_IRUGO); +#ifdef FRONTSWAP_HAS_EXCLUSIVE_GETS +module_param(frontswap_has_exclusive_gets, bool, S_IRUGO); +#endif +module_param(disable_frontswap_ignore_nonactive, bool, S_IRUGO); +module_param(zcache_comp_name, charp, S_IRUGO); +module_init(zcache_init); +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Dan Magenheimer <dan.magenheimer@oracle.com>"); +MODULE_DESCRIPTION("In-kernel compression of cleancache/frontswap pages"); +#else +late_initcall(zcache_init); +#endif |