fs/fscache/cookie.c

   1 /* netfs cookie management
   2  *
   3  * Copyright (C) 2004-2007 Red Hat, Inc. All Rights Reserved.
   4  * Written by David Howells (dhowells@redhat.com)
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License
   8  * as published by the Free Software Foundation; either version
   9  * 2 of the License, or (at your option) any later version.
  10  *
  11  * See Documentation/filesystems/caching/netfs-api.txt for more information on
  12  * the netfs API.
  13  */
  14
  15 #define FSCACHE_DEBUG_LEVEL COOKIE
  16 #include <linux/module.h>
  17 #include <linux/slab.h>
  18 #include "internal.h"
  19
  20 struct kmem_cache *fscache_cookie_jar;
  21
  22 static atomic_t fscache_object_debug_id = ATOMIC_INIT(0);
  23
  24 #define fscache_cookie_hash_shift 15
  25 static struct hlist_bl_head fscache_cookie_hash[1 << fscache_cookie_hash_shift];
  26
  27 static int fscache_acquire_non_index_cookie(struct fscache_cookie *cookie,
  28                                             loff_t object_size);
  29 static int fscache_alloc_object(struct fscache_cache *cache,
  30                                 struct fscache_cookie *cookie);
  31 static int fscache_attach_object(struct fscache_cookie *cookie,
  32                                  struct fscache_object *object);
  33
  34 static void fscache_print_cookie(struct fscache_cookie *cookie, char prefix)
  35 {
  36         struct hlist_node *object;
  37         const u8 *k;
  38         unsigned loop;
  39
  40         pr_err("%c-cookie c=%p [p=%p fl=%lx nc=%u na=%u]\n",
  41                prefix, cookie, cookie->parent, cookie->flags,
  42                atomic_read(&cookie->n_children),
  43                atomic_read(&cookie->n_active));
  44         pr_err("%c-cookie d=%p n=%p\n",
  45                prefix, cookie->def, cookie->netfs_data);
  46
  47         object = READ_ONCE(cookie->backing_objects.first);
  48         if (object)
  49                 pr_err("%c-cookie o=%p\n",
  50                        prefix, hlist_entry(object, struct fscache_object, cookie_link));
  51
  52         pr_err("%c-key=[%u] '", prefix, cookie->key_len);
  53         k = (cookie->key_len <= sizeof(cookie->inline_key)) ?
  54                 cookie->inline_key : cookie->key;
  55         for (loop = 0; loop < cookie->key_len; loop++)
  56                 pr_cont("%02x", k[loop]);
  57         pr_cont("'\n");
  58 }
  59
  60 void fscache_free_cookie(struct fscache_cookie *cookie)
  61 {
  62         if (cookie) {
  63                 BUG_ON(!hlist_empty(&cookie->backing_objects));
  64                 if (cookie->aux_len > sizeof(cookie->inline_aux))
  65                         kfree(cookie->aux);
  66                 if (cookie->key_len > sizeof(cookie->inline_key))
  67                         kfree(cookie->key);
  68                 kmem_cache_free(fscache_cookie_jar, cookie);
  69         }
  70 }
  71
  72 /*
  73  * Set the index key in a cookie.  The cookie struct has space for a 16-byte
  74  * key plus length and hash, but if that's not big enough, it's instead a
  75  * pointer to a buffer containing 3 bytes of hash, 1 byte of length and then
  76  * the key data.
  77  */
  78 static int fscache_set_key(struct fscache_cookie *cookie,
  79                            const void *index_key, size_t index_key_len)
  80 {
  81         u32 *buf;
  82         int bufs;
  83
  84         bufs = DIV_ROUND_UP(index_key_len, sizeof(*buf));
  85
  86         if (index_key_len > sizeof(cookie->inline_key)) {
  87                 buf = kcalloc(bufs, sizeof(*buf), GFP_KERNEL);
  88                 if (!buf)
  89                         return -ENOMEM;
  90                 cookie->key = buf;
  91         } else {
  92                 buf = (u32 *)cookie->inline_key;
  93         }
  94
  95         memcpy(buf, index_key, index_key_len);
  96         cookie->key_hash = fscache_hash(0, buf, bufs);
  97         return 0;
  98 }
  99
 100 static long fscache_compare_cookie(const struct fscache_cookie *a,
 101                                    const struct fscache_cookie *b)
 102 {
 103         const void *ka, *kb;
 104
 105         if (a->key_hash != b->key_hash)
 106                 return (long)a->key_hash - (long)b->key_hash;
 107         if (a->parent != b->parent)
 108                 return (long)a->parent - (long)b->parent;
 109         if (a->key_len != b->key_len)
 110                 return (long)a->key_len - (long)b->key_len;
 111         if (a->type != b->type)
 112                 return (long)a->type - (long)b->type;
 113
 114         if (a->key_len <= sizeof(a->inline_key)) {
 115                 ka = &a->inline_key;
 116                 kb = &b->inline_key;
 117         } else {
 118                 ka = a->key;
 119                 kb = b->key;
 120         }
 121         return memcmp(ka, kb, a->key_len);
 122 }
 123
 124 /*
 125  * Allocate a cookie.
 126  */
 127 struct fscache_cookie *fscache_alloc_cookie(
 128         struct fscache_cookie *parent,
 129         const struct fscache_cookie_def *def,
 130         const void *index_key, size_t index_key_len,
 131         const void *aux_data, size_t aux_data_len,
 132         void *netfs_data,
 133         loff_t object_size)
 134 {
 135         struct fscache_cookie *cookie;
 136
 137         /* allocate and initialise a cookie */
 138         cookie = kmem_cache_zalloc(fscache_cookie_jar, GFP_KERNEL);
 139         if (!cookie)
 140                 return NULL;
 141
 142         cookie->key_len = index_key_len;
 143         cookie->aux_len = aux_data_len;
 144
 145         if (fscache_set_key(cookie, index_key, index_key_len) < 0)
 146                 goto nomem;
 147
 148         if (cookie->aux_len <= sizeof(cookie->inline_aux)) {
 149                 memcpy(cookie->inline_aux, aux_data, cookie->aux_len);
 150         } else {
 151                 cookie->aux = kmemdup(aux_data, cookie->aux_len, GFP_KERNEL);
 152                 if (!cookie->aux)
 153                         goto nomem;
 154         }
 155
 156         atomic_set(&cookie->usage, 1);
 157         atomic_set(&cookie->n_children, 0);
 158
 159         /* We keep the active count elevated until relinquishment to prevent an
 160          * attempt to wake up every time the object operations queue quiesces.
 161          */
 162         atomic_set(&cookie->n_active, 1);
 163
 164         cookie->def             = def;
 165         cookie->parent          = parent;
 166         cookie->netfs_data      = netfs_data;
 167         cookie->flags           = (1 << FSCACHE_COOKIE_NO_DATA_YET);
 168         cookie->type            = def->type;
 169         spin_lock_init(&cookie->lock);
 170         spin_lock_init(&cookie->stores_lock);
 171         INIT_HLIST_HEAD(&cookie->backing_objects);
 172
 173         /* radix tree insertion won't use the preallocation pool unless it's
 174          * told it may not wait */
 175         INIT_RADIX_TREE(&cookie->stores, GFP_NOFS & ~__GFP_DIRECT_RECLAIM);
 176         return cookie;
 177
 178 nomem:
 179         fscache_free_cookie(cookie);
 180         return NULL;
 181 }
 182
 183 /*
 184  * Attempt to insert the new cookie into the hash.  If there's a collision, we
 185  * return the old cookie if it's not in use and an error otherwise.
 186  */
 187 struct fscache_cookie *fscache_hash_cookie(struct fscache_cookie *candidate)
 188 {
 189         struct fscache_cookie *cursor;
 190         struct hlist_bl_head *h;
 191         struct hlist_bl_node *p;
 192         unsigned int bucket;
 193
 194         bucket = candidate->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1);
 195         h = &fscache_cookie_hash[bucket];
 196
 197         hlist_bl_lock(h);
 198         hlist_bl_for_each_entry(cursor, p, h, hash_link) {
 199                 if (fscache_compare_cookie(candidate, cursor) == 0)
 200                         goto collision;
 201         }
 202
 203         __set_bit(FSCACHE_COOKIE_ACQUIRED, &candidate->flags);
 204         fscache_cookie_get(candidate->parent, fscache_cookie_get_acquire_parent);
 205         atomic_inc(&candidate->parent->n_children);
 206         hlist_bl_add_head(&candidate->hash_link, h);
 207         hlist_bl_unlock(h);
 208         return candidate;
 209
 210 collision:
 211         if (test_and_set_bit(FSCACHE_COOKIE_ACQUIRED, &cursor->flags)) {
 212                 trace_fscache_cookie(cursor, fscache_cookie_collision,
 213                                      atomic_read(&cursor->usage));
 214                 pr_err("Duplicate cookie detected\n");
 215                 fscache_print_cookie(cursor, 'O');
 216                 fscache_print_cookie(candidate, 'N');
 217                 hlist_bl_unlock(h);
 218                 return NULL;
 219         }
 220
 221         fscache_cookie_get(cursor, fscache_cookie_get_reacquire);
 222         hlist_bl_unlock(h);
 223         return cursor;
 224 }
 225
 226 /*
 227  * request a cookie to represent an object (index, datafile, xattr, etc)
 228  * - parent specifies the parent object
 229  *   - the top level index cookie for each netfs is stored in the fscache_netfs
 230  *     struct upon registration
 231  * - def points to the definition
 232  * - the netfs_data will be passed to the functions pointed to in *def
 233  * - all attached caches will be searched to see if they contain this object
 234  * - index objects aren't stored on disk until there's a dependent file that
 235  *   needs storing
 236  * - other objects are stored in a selected cache immediately, and all the
 237  *   indices forming the path to it are instantiated if necessary
 238  * - we never let on to the netfs about errors
 239  *   - we may set a negative cookie pointer, but that's okay
 240  */
 241 struct fscache_cookie *__fscache_acquire_cookie(
 242         struct fscache_cookie *parent,
 243         const struct fscache_cookie_def *def,
 244         const void *index_key, size_t index_key_len,
 245         const void *aux_data, size_t aux_data_len,
 246         void *netfs_data,
 247         loff_t object_size,
 248         bool enable)
 249 {
 250         struct fscache_cookie *candidate, *cookie;
 251
 252         BUG_ON(!def);
 253
 254         _enter("{%s},{%s},%p,%u",
 255                parent ? (char *) parent->def->name : "<no-parent>",
 256                def->name, netfs_data, enable);
 257
 258         if (!index_key || !index_key_len || index_key_len > 255 || aux_data_len > 255)
 259                 return NULL;
 260         if (!aux_data || !aux_data_len) {
 261                 aux_data = NULL;
 262                 aux_data_len = 0;
 263         }
 264
 265         fscache_stat(&fscache_n_acquires);
 266
 267         /* if there's no parent cookie, then we don't create one here either */
 268         if (!parent) {
 269                 fscache_stat(&fscache_n_acquires_null);
 270                 _leave(" [no parent]");
 271                 return NULL;
 272         }
 273
 274         /* validate the definition */
 275         BUG_ON(!def->name[0]);
 276
 277         BUG_ON(def->type == FSCACHE_COOKIE_TYPE_INDEX &&
 278                parent->type != FSCACHE_COOKIE_TYPE_INDEX);
 279
 280         candidate = fscache_alloc_cookie(parent, def,
 281                                          index_key, index_key_len,
 282                                          aux_data, aux_data_len,
 283                                          netfs_data, object_size);
 284         if (!candidate) {
 285                 fscache_stat(&fscache_n_acquires_oom);
 286                 _leave(" [ENOMEM]");
 287                 return NULL;
 288         }
 289
 290         cookie = fscache_hash_cookie(candidate);
 291         if (!cookie) {
 292                 trace_fscache_cookie(candidate, fscache_cookie_discard, 1);
 293                 goto out;
 294         }
 295
 296         if (cookie == candidate)
 297                 candidate = NULL;
 298
 299         switch (cookie->type) {
 300         case FSCACHE_COOKIE_TYPE_INDEX:
 301                 fscache_stat(&fscache_n_cookie_index);
 302                 break;
 303         case FSCACHE_COOKIE_TYPE_DATAFILE:
 304                 fscache_stat(&fscache_n_cookie_data);
 305                 break;
 306         default:
 307                 fscache_stat(&fscache_n_cookie_special);
 308                 break;
 309         }
 310
 311         trace_fscache_acquire(cookie);
 312
 313         if (enable) {
 314                 /* if the object is an index then we need do nothing more here
 315                  * - we create indices on disk when we need them as an index
 316                  * may exist in multiple caches */
 317                 if (cookie->type != FSCACHE_COOKIE_TYPE_INDEX) {
 318                         if (fscache_acquire_non_index_cookie(cookie, object_size) == 0) {
 319                                 set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags);
 320                         } else {
 321                                 atomic_dec(&parent->n_children);
 322                                 fscache_cookie_put(cookie,
 323                                                    fscache_cookie_put_acquire_nobufs);
 324                                 fscache_stat(&fscache_n_acquires_nobufs);
 325                                 _leave(" = NULL");
 326                                 return NULL;
 327                         }
 328                 } else {
 329                         set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags);
 330                 }
 331         }
 332
 333         fscache_stat(&fscache_n_acquires_ok);
 334
 335 out:
 336         fscache_free_cookie(candidate);
 337         return cookie;
 338 }
 339 EXPORT_SYMBOL(__fscache_acquire_cookie);
 340
 341 /*
 342  * Enable a cookie to permit it to accept new operations.
 343  */
 344 void __fscache_enable_cookie(struct fscache_cookie *cookie,
 345                              const void *aux_data,
 346                              loff_t object_size,
 347                              bool (*can_enable)(void *data),
 348                              void *data)
 349 {
 350         _enter("%p", cookie);
 351
 352         trace_fscache_enable(cookie);
 353
 354         wait_on_bit_lock(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK,
 355                          TASK_UNINTERRUPTIBLE);
 356
 357         fscache_update_aux(cookie, aux_data);
 358
 359         if (test_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags))
 360                 goto out_unlock;
 361
 362         if (can_enable && !can_enable(data)) {
 363                 /* The netfs decided it didn't want to enable after all */
 364         } else if (cookie->type != FSCACHE_COOKIE_TYPE_INDEX) {
 365                 /* Wait for outstanding disablement to complete */
 366                 __fscache_wait_on_invalidate(cookie);
 367
 368                 if (fscache_acquire_non_index_cookie(cookie, object_size) == 0)
 369                         set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags);
 370         } else {
 371                 set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags);
 372         }
 373
 374 out_unlock:
 375         clear_bit_unlock(FSCACHE_COOKIE_ENABLEMENT_LOCK, &cookie->flags);
 376         wake_up_bit(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK);
 377 }
 378 EXPORT_SYMBOL(__fscache_enable_cookie);
 379
 380 /*
 381  * acquire a non-index cookie
 382  * - this must make sure the index chain is instantiated and instantiate the
 383  *   object representation too
 384  */
 385 static int fscache_acquire_non_index_cookie(struct fscache_cookie *cookie,
 386                                             loff_t object_size)
 387 {
 388         struct fscache_object *object;
 389         struct fscache_cache *cache;
 390         int ret;
 391
 392         _enter("");
 393
 394         set_bit(FSCACHE_COOKIE_UNAVAILABLE, &cookie->flags);
 395
 396         /* now we need to see whether the backing objects for this cookie yet
 397          * exist, if not there'll be nothing to search */
 398         down_read(&fscache_addremove_sem);
 399
 400         if (list_empty(&fscache_cache_list)) {
 401                 up_read(&fscache_addremove_sem);
 402                 _leave(" = 0 [no caches]");
 403                 return 0;
 404         }
 405
 406         /* select a cache in which to store the object */
 407         cache = fscache_select_cache_for_object(cookie->parent);
 408         if (!cache) {
 409                 up_read(&fscache_addremove_sem);
 410                 fscache_stat(&fscache_n_acquires_no_cache);
 411                 _leave(" = -ENOMEDIUM [no cache]");
 412                 return -ENOMEDIUM;
 413         }
 414
 415         _debug("cache %s", cache->tag->name);
 416
 417         set_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags);
 418
 419         /* ask the cache to allocate objects for this cookie and its parent
 420          * chain */
 421         ret = fscache_alloc_object(cache, cookie);
 422         if (ret < 0) {
 423                 up_read(&fscache_addremove_sem);
 424                 _leave(" = %d", ret);
 425                 return ret;
 426         }
 427
 428         spin_lock(&cookie->lock);
 429         if (hlist_empty(&cookie->backing_objects)) {
 430                 spin_unlock(&cookie->lock);
 431                 goto unavailable;
 432         }
 433
 434         object = hlist_entry(cookie->backing_objects.first,
 435                              struct fscache_object, cookie_link);
 436
 437         fscache_set_store_limit(object, object_size);
 438
 439         /* initiate the process of looking up all the objects in the chain
 440          * (done by fscache_initialise_object()) */
 441         fscache_raise_event(object, FSCACHE_OBJECT_EV_NEW_CHILD);
 442
 443         spin_unlock(&cookie->lock);
 444
 445         /* we may be required to wait for lookup to complete at this point */
 446         if (!fscache_defer_lookup) {
 447                 _debug("non-deferred lookup %p", &cookie->flags);
 448                 wait_on_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP,
 449                             TASK_UNINTERRUPTIBLE);
 450                 _debug("complete");
 451                 if (test_bit(FSCACHE_COOKIE_UNAVAILABLE, &cookie->flags))
 452                         goto unavailable;
 453         }
 454
 455         up_read(&fscache_addremove_sem);
 456         _leave(" = 0 [deferred]");
 457         return 0;
 458
 459 unavailable:
 460         up_read(&fscache_addremove_sem);
 461         _leave(" = -ENOBUFS");
 462         return -ENOBUFS;
 463 }
 464
 465 /*
 466  * recursively allocate cache object records for a cookie/cache combination
 467  * - caller must be holding the addremove sem
 468  */
 469 static int fscache_alloc_object(struct fscache_cache *cache,
 470                                 struct fscache_cookie *cookie)
 471 {
 472         struct fscache_object *object;
 473         int ret;
 474
 475         _enter("%p,%p{%s}", cache, cookie, cookie->def->name);
 476
 477         spin_lock(&cookie->lock);
 478         hlist_for_each_entry(object, &cookie->backing_objects,
 479                              cookie_link) {
 480                 if (object->cache == cache)
 481                         goto object_already_extant;
 482         }
 483         spin_unlock(&cookie->lock);
 484
 485         /* ask the cache to allocate an object (we may end up with duplicate
 486          * objects at this stage, but we sort that out later) */
 487         fscache_stat(&fscache_n_cop_alloc_object);
 488         object = cache->ops->alloc_object(cache, cookie);
 489         fscache_stat_d(&fscache_n_cop_alloc_object);
 490         if (IS_ERR(object)) {
 491                 fscache_stat(&fscache_n_object_no_alloc);
 492                 ret = PTR_ERR(object);
 493                 goto error;
 494         }
 495
 496         ASSERTCMP(object->cookie, ==, cookie);
 497         fscache_stat(&fscache_n_object_alloc);
 498
 499         object->debug_id = atomic_inc_return(&fscache_object_debug_id);
 500
 501         _debug("ALLOC OBJ%x: %s {%lx}",
 502                object->debug_id, cookie->def->name, object->events);
 503
 504         ret = fscache_alloc_object(cache, cookie->parent);
 505         if (ret < 0)
 506                 goto error_put;
 507
 508         /* only attach if we managed to allocate all we needed, otherwise
 509          * discard the object we just allocated and instead use the one
 510          * attached to the cookie */
 511         if (fscache_attach_object(cookie, object) < 0) {
 512                 fscache_stat(&fscache_n_cop_put_object);
 513                 cache->ops->put_object(object, fscache_obj_put_attach_fail);
 514                 fscache_stat_d(&fscache_n_cop_put_object);
 515         }
 516
 517         _leave(" = 0");
 518         return 0;
 519
 520 object_already_extant:
 521         ret = -ENOBUFS;
 522         if (fscache_object_is_dying(object) ||
 523             fscache_cache_is_broken(object)) {
 524                 spin_unlock(&cookie->lock);
 525                 goto error;
 526         }
 527         spin_unlock(&cookie->lock);
 528         _leave(" = 0 [found]");
 529         return 0;
 530
 531 error_put:
 532         fscache_stat(&fscache_n_cop_put_object);
 533         cache->ops->put_object(object, fscache_obj_put_alloc_fail);
 534         fscache_stat_d(&fscache_n_cop_put_object);
 535 error:
 536         _leave(" = %d", ret);
 537         return ret;
 538 }
 539
 540 /*
 541  * attach a cache object to a cookie
 542  */
 543 static int fscache_attach_object(struct fscache_cookie *cookie,
 544                                  struct fscache_object *object)
 545 {
 546         struct fscache_object *p;
 547         struct fscache_cache *cache = object->cache;
 548         int ret;
 549
 550         _enter("{%s},{OBJ%x}", cookie->def->name, object->debug_id);
 551
 552         ASSERTCMP(object->cookie, ==, cookie);
 553
 554         spin_lock(&cookie->lock);
 555
 556         /* there may be multiple initial creations of this object, but we only
 557          * want one */
 558         ret = -EEXIST;
 559         hlist_for_each_entry(p, &cookie->backing_objects, cookie_link) {
 560                 if (p->cache == object->cache) {
 561                         if (fscache_object_is_dying(p))
 562                                 ret = -ENOBUFS;
 563                         goto cant_attach_object;
 564                 }
 565         }
 566
 567         /* pin the parent object */
 568         spin_lock_nested(&cookie->parent->lock, 1);
 569         hlist_for_each_entry(p, &cookie->parent->backing_objects,
 570                              cookie_link) {
 571                 if (p->cache == object->cache) {
 572                         if (fscache_object_is_dying(p)) {
 573                                 ret = -ENOBUFS;
 574                                 spin_unlock(&cookie->parent->lock);
 575                                 goto cant_attach_object;
 576                         }
 577                         object->parent = p;
 578                         spin_lock(&p->lock);
 579                         p->n_children++;
 580                         spin_unlock(&p->lock);
 581                         break;
 582                 }
 583         }
 584         spin_unlock(&cookie->parent->lock);
 585
 586         /* attach to the cache's object list */
 587         if (list_empty(&object->cache_link)) {
 588                 spin_lock(&cache->object_list_lock);
 589                 list_add(&object->cache_link, &cache->object_list);
 590                 spin_unlock(&cache->object_list_lock);
 591         }
 592
 593         /* Attach to the cookie.  The object already has a ref on it. */
 594         hlist_add_head(&object->cookie_link, &cookie->backing_objects);
 595
 596         fscache_objlist_add(object);
 597         ret = 0;
 598
 599 cant_attach_object:
 600         spin_unlock(&cookie->lock);
 601         _leave(" = %d", ret);
 602         return ret;
 603 }
 604
 605 /*
 606  * Invalidate an object.  Callable with spinlocks held.
 607  */
 608 void __fscache_invalidate(struct fscache_cookie *cookie)
 609 {
 610         struct fscache_object *object;
 611
 612         _enter("{%s}", cookie->def->name);
 613
 614         fscache_stat(&fscache_n_invalidates);
 615
 616         /* Only permit invalidation of data files.  Invalidating an index will
 617          * require the caller to release all its attachments to the tree rooted
 618          * there, and if it's doing that, it may as well just retire the
 619          * cookie.
 620          */
 621         ASSERTCMP(cookie->type, ==, FSCACHE_COOKIE_TYPE_DATAFILE);
 622
 623         /* If there's an object, we tell the object state machine to handle the
 624          * invalidation on our behalf, otherwise there's nothing to do.
 625          */
 626         if (!hlist_empty(&cookie->backing_objects)) {
 627                 spin_lock(&cookie->lock);
 628
 629                 if (fscache_cookie_enabled(cookie) &&
 630                     !hlist_empty(&cookie->backing_objects) &&
 631                     !test_and_set_bit(FSCACHE_COOKIE_INVALIDATING,
 632                                       &cookie->flags)) {
 633                         object = hlist_entry(cookie->backing_objects.first,
 634                                              struct fscache_object,
 635                                              cookie_link);
 636                         if (fscache_object_is_live(object))
 637                                 fscache_raise_event(
 638                                         object, FSCACHE_OBJECT_EV_INVALIDATE);
 639                 }
 640
 641                 spin_unlock(&cookie->lock);
 642         }
 643
 644         _leave("");
 645 }
 646 EXPORT_SYMBOL(__fscache_invalidate);
 647
 648 /*
 649  * Wait for object invalidation to complete.
 650  */
 651 void __fscache_wait_on_invalidate(struct fscache_cookie *cookie)
 652 {
 653         _enter("%p", cookie);
 654
 655         wait_on_bit(&cookie->flags, FSCACHE_COOKIE_INVALIDATING,
 656                     TASK_UNINTERRUPTIBLE);
 657
 658         _leave("");
 659 }
 660 EXPORT_SYMBOL(__fscache_wait_on_invalidate);
 661
 662 /*
 663  * update the index entries backing a cookie
 664  */
 665 void __fscache_update_cookie(struct fscache_cookie *cookie, const void *aux_data)
 666 {
 667         struct fscache_object *object;
 668
 669         fscache_stat(&fscache_n_updates);
 670
 671         if (!cookie) {
 672                 fscache_stat(&fscache_n_updates_null);
 673                 _leave(" [no cookie]");
 674                 return;
 675         }
 676
 677         _enter("{%s}", cookie->def->name);
 678
 679         spin_lock(&cookie->lock);
 680
 681         fscache_update_aux(cookie, aux_data);
 682
 683         if (fscache_cookie_enabled(cookie)) {
 684                 /* update the index entry on disk in each cache backing this
 685                  * cookie.
 686                  */
 687                 hlist_for_each_entry(object,
 688                                      &cookie->backing_objects, cookie_link) {
 689                         fscache_raise_event(object, FSCACHE_OBJECT_EV_UPDATE);
 690                 }
 691         }
 692
 693         spin_unlock(&cookie->lock);
 694         _leave("");
 695 }
 696 EXPORT_SYMBOL(__fscache_update_cookie);
 697
 698 /*
 699  * Disable a cookie to stop it from accepting new requests from the netfs.
 700  */
 701 void __fscache_disable_cookie(struct fscache_cookie *cookie,
 702                               const void *aux_data,
 703                               bool invalidate)
 704 {
 705         struct fscache_object *object;
 706         bool awaken = false;
 707
 708         _enter("%p,%u", cookie, invalidate);
 709
 710         trace_fscache_disable(cookie);
 711
 712         ASSERTCMP(atomic_read(&cookie->n_active), >, 0);
 713
 714         if (atomic_read(&cookie->n_children) != 0) {
 715                 pr_err("Cookie '%s' still has children\n",
 716                        cookie->def->name);
 717                 BUG();
 718         }
 719
 720         wait_on_bit_lock(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK,
 721                          TASK_UNINTERRUPTIBLE);
 722
 723         fscache_update_aux(cookie, aux_data);
 724
 725         if (!test_and_clear_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags))
 726                 goto out_unlock_enable;
 727
 728         /* If the cookie is being invalidated, wait for that to complete first
 729          * so that we can reuse the flag.
 730          */
 731         __fscache_wait_on_invalidate(cookie);
 732
 733         /* Dispose of the backing objects */
 734         set_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags);
 735
 736         spin_lock(&cookie->lock);
 737         if (!hlist_empty(&cookie->backing_objects)) {
 738                 hlist_for_each_entry(object, &cookie->backing_objects, cookie_link) {
 739                         if (invalidate)
 740                                 set_bit(FSCACHE_OBJECT_RETIRED, &object->flags);
 741                         clear_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags);
 742                         fscache_raise_event(object, FSCACHE_OBJECT_EV_KILL);
 743                 }
 744         } else {
 745                 if (test_and_clear_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags))
 746                         awaken = true;
 747         }
 748         spin_unlock(&cookie->lock);
 749         if (awaken)
 750                 wake_up_bit(&cookie->flags, FSCACHE_COOKIE_INVALIDATING);
 751
 752         /* Wait for cessation of activity requiring access to the netfs (when
 753          * n_active reaches 0).  This makes sure outstanding reads and writes
 754          * have completed.
 755          */
 756         if (!atomic_dec_and_test(&cookie->n_active)) {
 757                 wait_var_event(&cookie->n_active,
 758                                !atomic_read(&cookie->n_active));
 759         }
 760
 761         /* Make sure any pending writes are cancelled. */
 762         if (cookie->type != FSCACHE_COOKIE_TYPE_INDEX)
 763                 fscache_invalidate_writes(cookie);
 764
 765         /* Reset the cookie state if it wasn't relinquished */
 766         if (!test_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags)) {
 767                 atomic_inc(&cookie->n_active);
 768                 set_bit(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags);
 769         }
 770
 771 out_unlock_enable:
 772         clear_bit_unlock(FSCACHE_COOKIE_ENABLEMENT_LOCK, &cookie->flags);
 773         wake_up_bit(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK);
 774         _leave("");
 775 }
 776 EXPORT_SYMBOL(__fscache_disable_cookie);
 777
 778 /*
 779  * release a cookie back to the cache
 780  * - the object will be marked as recyclable on disk if retire is true
 781  * - all dependents of this cookie must have already been unregistered
 782  *   (indices/files/pages)
 783  */
 784 void __fscache_relinquish_cookie(struct fscache_cookie *cookie,
 785                                  const void *aux_data,
 786                                  bool retire)
 787 {
 788         fscache_stat(&fscache_n_relinquishes);
 789         if (retire)
 790                 fscache_stat(&fscache_n_relinquishes_retire);
 791
 792         if (!cookie) {
 793                 fscache_stat(&fscache_n_relinquishes_null);
 794                 _leave(" [no cookie]");
 795                 return;
 796         }
 797
 798         _enter("%p{%s,%p,%d},%d",
 799                cookie, cookie->def->name, cookie->netfs_data,
 800                atomic_read(&cookie->n_active), retire);
 801
 802         trace_fscache_relinquish(cookie, retire);
 803
 804         /* No further netfs-accessing operations on this cookie permitted */
 805         if (test_and_set_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags))
 806                 BUG();
 807
 808         __fscache_disable_cookie(cookie, aux_data, retire);
 809
 810         /* Clear pointers back to the netfs */
 811         cookie->netfs_data      = NULL;
 812         cookie->def             = NULL;
 813         BUG_ON(!radix_tree_empty(&cookie->stores));
 814
 815         if (cookie->parent) {
 816                 ASSERTCMP(atomic_read(&cookie->parent->usage), >, 0);
 817                 ASSERTCMP(atomic_read(&cookie->parent->n_children), >, 0);
 818                 atomic_dec(&cookie->parent->n_children);
 819         }
 820
 821         /* Dispose of the netfs's link to the cookie */
 822         ASSERTCMP(atomic_read(&cookie->usage), >, 0);
 823         fscache_cookie_put(cookie, fscache_cookie_put_relinquish);
 824
 825         _leave("");
 826 }
 827 EXPORT_SYMBOL(__fscache_relinquish_cookie);
 828
 829 /*
 830  * Remove a cookie from the hash table.
 831  */
 832 static void fscache_unhash_cookie(struct fscache_cookie *cookie)
 833 {
 834         struct hlist_bl_head *h;
 835         unsigned int bucket;
 836
 837         bucket = cookie->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1);
 838         h = &fscache_cookie_hash[bucket];
 839
 840         hlist_bl_lock(h);
 841         hlist_bl_del(&cookie->hash_link);
 842         hlist_bl_unlock(h);
 843 }
 844
 845 /*
 846  * Drop a reference to a cookie.
 847  */
 848 void fscache_cookie_put(struct fscache_cookie *cookie,
 849                         enum fscache_cookie_trace where)
 850 {
 851         struct fscache_cookie *parent;
 852         int usage;
 853
 854         _enter("%p", cookie);
 855
 856         do {
 857                 usage = atomic_dec_return(&cookie->usage);
 858                 trace_fscache_cookie(cookie, where, usage);
 859
 860                 if (usage > 0)
 861                         return;
 862                 BUG_ON(usage < 0);
 863
 864                 parent = cookie->parent;
 865                 fscache_unhash_cookie(cookie);
 866                 fscache_free_cookie(cookie);
 867
 868                 cookie = parent;
 869                 where = fscache_cookie_put_parent;
 870         } while (cookie);
 871
 872         _leave("");
 873 }
 874
 875 /*
 876  * check the consistency between the netfs inode and the backing cache
 877  *
 878  * NOTE: it only serves no-index type
 879  */
 880 int __fscache_check_consistency(struct fscache_cookie *cookie,
 881                                 const void *aux_data)
 882 {
 883         struct fscache_operation *op;
 884         struct fscache_object *object;
 885         bool wake_cookie = false;
 886         int ret;
 887
 888         _enter("%p,", cookie);
 889
 890         ASSERTCMP(cookie->type, ==, FSCACHE_COOKIE_TYPE_DATAFILE);
 891
 892         if (fscache_wait_for_deferred_lookup(cookie) < 0)
 893                 return -ERESTARTSYS;
 894
 895         if (hlist_empty(&cookie->backing_objects))
 896                 return 0;
 897
 898         op = kzalloc(sizeof(*op), GFP_NOIO | __GFP_NOMEMALLOC | __GFP_NORETRY);
 899         if (!op)
 900                 return -ENOMEM;
 901
 902         fscache_operation_init(cookie, op, NULL, NULL, NULL);
 903         op->flags = FSCACHE_OP_MYTHREAD |
 904                 (1 << FSCACHE_OP_WAITING) |
 905                 (1 << FSCACHE_OP_UNUSE_COOKIE);
 906         trace_fscache_page_op(cookie, NULL, op, fscache_page_op_check_consistency);
 907
 908         spin_lock(&cookie->lock);
 909
 910         fscache_update_aux(cookie, aux_data);
 911
 912         if (!fscache_cookie_enabled(cookie) ||
 913             hlist_empty(&cookie->backing_objects))
 914                 goto inconsistent;
 915         object = hlist_entry(cookie->backing_objects.first,
 916                              struct fscache_object, cookie_link);
 917         if (test_bit(FSCACHE_IOERROR, &object->cache->flags))
 918                 goto inconsistent;
 919
 920         op->debug_id = atomic_inc_return(&fscache_op_debug_id);
 921
 922         __fscache_use_cookie(cookie);
 923         if (fscache_submit_op(object, op) < 0)
 924                 goto submit_failed;
 925
 926         /* the work queue now carries its own ref on the object */
 927         spin_unlock(&cookie->lock);
 928
 929         ret = fscache_wait_for_operation_activation(object, op, NULL, NULL);
 930         if (ret == 0) {
 931                 /* ask the cache to honour the operation */
 932                 ret = object->cache->ops->check_consistency(op);
 933                 fscache_op_complete(op, false);
 934         } else if (ret == -ENOBUFS) {
 935                 ret = 0;
 936         }
 937
 938         fscache_put_operation(op);
 939         _leave(" = %d", ret);
 940         return ret;
 941
 942 submit_failed:
 943         wake_cookie = __fscache_unuse_cookie(cookie);
 944 inconsistent:
 945         spin_unlock(&cookie->lock);
 946         if (wake_cookie)
 947                 __fscache_wake_unused_cookie(cookie);
 948         kfree(op);
 949         _leave(" = -ESTALE");
 950         return -ESTALE;
 951 }
 952 EXPORT_SYMBOL(__fscache_check_consistency);