drivers/staging/erofs/unzip_vle_lz4.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * linux/drivers/staging/erofs/unzip_vle_lz4.c
   4  *
   5  * Copyright (C) 2018 HUAWEI, Inc.
   6  *             http://www.huawei.com/
   7  * Created by Gao Xiang <gaoxiang25@huawei.com>
   8  *
   9  * This file is subject to the terms and conditions of the GNU General Public
  10  * License.  See the file COPYING in the main directory of the Linux
  11  * distribution for more details.
  12  */
  13 #include "unzip_vle.h"
  14
  15 #if Z_EROFS_CLUSTER_MAX_PAGES > Z_EROFS_VLE_INLINE_PAGEVECS
  16 #define EROFS_PERCPU_NR_PAGES   Z_EROFS_CLUSTER_MAX_PAGES
  17 #else
  18 #define EROFS_PERCPU_NR_PAGES   Z_EROFS_VLE_INLINE_PAGEVECS
  19 #endif
  20
  21 static struct {
  22         char data[PAGE_SIZE * EROFS_PERCPU_NR_PAGES];
  23 } erofs_pcpubuf[NR_CPUS];
  24
  25 int z_erofs_vle_plain_copy(struct page **compressed_pages,
  26                            unsigned clusterpages,
  27                            struct page **pages,
  28                            unsigned nr_pages,
  29                            unsigned short pageofs)
  30 {
  31         unsigned i, j;
  32         void *src = NULL;
  33         const unsigned righthalf = PAGE_SIZE - pageofs;
  34         char *percpu_data;
  35         bool mirrored[Z_EROFS_CLUSTER_MAX_PAGES] = { 0 };
  36
  37         preempt_disable();
  38         percpu_data = erofs_pcpubuf[smp_processor_id()].data;
  39
  40         j = 0;
  41         for (i = 0; i < nr_pages; j = i++) {
  42                 struct page *page = pages[i];
  43                 void *dst;
  44
  45                 if (page == NULL) {
  46                         if (src != NULL) {
  47                                 if (!mirrored[j])
  48                                         kunmap_atomic(src);
  49                                 src = NULL;
  50                         }
  51                         continue;
  52                 }
  53
  54                 dst = kmap_atomic(page);
  55
  56                 for (; j < clusterpages; ++j) {
  57                         if (compressed_pages[j] != page)
  58                                 continue;
  59
  60                         DBG_BUGON(mirrored[j]);
  61                         memcpy(percpu_data + j * PAGE_SIZE, dst, PAGE_SIZE);
  62                         mirrored[j] = true;
  63                         break;
  64                 }
  65
  66                 if (i) {
  67                         if (src == NULL)
  68                                 src = mirrored[i-1] ?
  69                                         percpu_data + (i-1) * PAGE_SIZE :
  70                                         kmap_atomic(compressed_pages[i-1]);
  71
  72                         memcpy(dst, src + righthalf, pageofs);
  73
  74                         if (!mirrored[i-1])
  75                                 kunmap_atomic(src);
  76
  77                         if (unlikely(i >= clusterpages)) {
  78                                 kunmap_atomic(dst);
  79                                 break;
  80                         }
  81                 }
  82
  83                 if (!righthalf)
  84                         src = NULL;
  85                 else {
  86                         src = mirrored[i] ? percpu_data + i * PAGE_SIZE :
  87                                 kmap_atomic(compressed_pages[i]);
  88
  89                         memcpy(dst + pageofs, src, righthalf);
  90                 }
  91
  92                 kunmap_atomic(dst);
  93         }
  94
  95         if (src != NULL && !mirrored[j])
  96                 kunmap_atomic(src);
  97
  98         preempt_enable();
  99         return 0;
 100 }
 101
 102 extern int z_erofs_unzip_lz4(void *in, void *out, size_t inlen, size_t outlen);
 103
 104 int z_erofs_vle_unzip_fast_percpu(struct page **compressed_pages,
 105                                   unsigned clusterpages,
 106                                   struct page **pages,
 107                                   unsigned outlen,
 108                                   unsigned short pageofs)
 109 {
 110         void *vin, *vout;
 111         unsigned nr_pages, i, j;
 112         int ret;
 113
 114         if (outlen + pageofs > EROFS_PERCPU_NR_PAGES * PAGE_SIZE)
 115                 return -ENOTSUPP;
 116
 117         nr_pages = DIV_ROUND_UP(outlen + pageofs, PAGE_SIZE);
 118
 119         if (clusterpages == 1) {
 120                 vin = kmap_atomic(compressed_pages[0]);
 121         } else {
 122                 vin = erofs_vmap(compressed_pages, clusterpages);
 123                 if (!vin)
 124                         return -ENOMEM;
 125         }
 126
 127         preempt_disable();
 128         vout = erofs_pcpubuf[smp_processor_id()].data;
 129
 130         ret = z_erofs_unzip_lz4(vin, vout + pageofs,
 131                 clusterpages * PAGE_SIZE, outlen);
 132
 133         if (ret < 0)
 134                 goto out;
 135         ret = 0;
 136
 137         for (i = 0; i < nr_pages; ++i) {
 138                 j = min((unsigned)PAGE_SIZE - pageofs, outlen);
 139
 140                 if (pages[i] != NULL) {
 141                         if (clusterpages == 1 &&
 142                             pages[i] == compressed_pages[0]) {
 143                                 memcpy(vin + pageofs, vout + pageofs, j);
 144                         } else {
 145                                 void *dst = kmap_atomic(pages[i]);
 146
 147                                 memcpy(dst + pageofs, vout + pageofs, j);
 148                                 kunmap_atomic(dst);
 149                         }
 150                 }
 151                 vout += PAGE_SIZE;
 152                 outlen -= j;
 153                 pageofs = 0;
 154         }
 155
 156 out:
 157         preempt_enable();
 158
 159         if (clusterpages == 1)
 160                 kunmap_atomic(vin);
 161         else
 162                 erofs_vunmap(vin, clusterpages);
 163
 164         return ret;
 165 }
 166
 167 int z_erofs_vle_unzip_vmap(struct page **compressed_pages,
 168                            unsigned clusterpages,
 169                            void *vout,
 170                            unsigned llen,
 171                            unsigned short pageofs,
 172                            bool overlapped)
 173 {
 174         void *vin;
 175         unsigned i;
 176         int ret;
 177
 178         if (overlapped) {
 179                 preempt_disable();
 180                 vin = erofs_pcpubuf[smp_processor_id()].data;
 181
 182                 for (i = 0; i < clusterpages; ++i) {
 183                         void *t = kmap_atomic(compressed_pages[i]);
 184
 185                         memcpy(vin + PAGE_SIZE *i, t, PAGE_SIZE);
 186                         kunmap_atomic(t);
 187                 }
 188         } else if (clusterpages == 1)
 189                 vin = kmap_atomic(compressed_pages[0]);
 190         else {
 191                 vin = erofs_vmap(compressed_pages, clusterpages);
 192         }
 193
 194         ret = z_erofs_unzip_lz4(vin, vout + pageofs,
 195                 clusterpages * PAGE_SIZE, llen);
 196         if (ret > 0)
 197                 ret = 0;
 198
 199         if (!overlapped) {
 200                 if (clusterpages == 1)
 201                         kunmap_atomic(vin);
 202                 else {
 203                         erofs_vunmap(vin, clusterpages);
 204                 }
 205         } else
 206                 preempt_enable();
 207
 208         return ret;
 209 }
 210