arch/xtensa/kernel/pci-dma.c

   1 /*
   2  * DMA coherent memory allocation.
   3  *
   4  * This program is free software; you can redistribute  it and/or modify it
   5  * under  the terms of  the GNU General  Public License as published by the
   6  * Free Software Foundation;  either version 2 of the  License, or (at your
   7  * option) any later version.
   8  *
   9  * Copyright (C) 2002 - 2005 Tensilica Inc.
  10  * Copyright (C) 2015 Cadence Design Systems Inc.
  11  *
  12  * Based on version for i386.
  13  *
  14  * Chris Zankel <chris@zankel.net>
  15  * Joe Taylor <joe@tensilica.com, joetylr@yahoo.com>
  16  */
  17
  18 #include <linux/dma-contiguous.h>
  19 #include <linux/gfp.h>
  20 #include <linux/highmem.h>
  21 #include <linux/mm.h>
  22 #include <linux/module.h>
  23 #include <linux/pci.h>
  24 #include <linux/string.h>
  25 #include <linux/types.h>
  26 #include <asm/cacheflush.h>
  27 #include <asm/io.h>
  28
  29 void dma_cache_sync(struct device *dev, void *vaddr, size_t size,
  30                     enum dma_data_direction dir)
  31 {
  32         switch (dir) {
  33         case DMA_BIDIRECTIONAL:
  34                 __flush_invalidate_dcache_range((unsigned long)vaddr, size);
  35                 break;
  36
  37         case DMA_FROM_DEVICE:
  38                 __invalidate_dcache_range((unsigned long)vaddr, size);
  39                 break;
  40
  41         case DMA_TO_DEVICE:
  42                 __flush_dcache_range((unsigned long)vaddr, size);
  43                 break;
  44
  45         case DMA_NONE:
  46                 BUG();
  47                 break;
  48         }
  49 }
  50 EXPORT_SYMBOL(dma_cache_sync);
  51
  52 static void do_cache_op(dma_addr_t dma_handle, size_t size,
  53                         void (*fn)(unsigned long, unsigned long))
  54 {
  55         unsigned long off = dma_handle & (PAGE_SIZE - 1);
  56         unsigned long pfn = PFN_DOWN(dma_handle);
  57         struct page *page = pfn_to_page(pfn);
  58
  59         if (!PageHighMem(page))
  60                 fn((unsigned long)bus_to_virt(dma_handle), size);
  61         else
  62                 while (size > 0) {
  63                         size_t sz = min_t(size_t, size, PAGE_SIZE - off);
  64                         void *vaddr = kmap_atomic(page);
  65
  66                         fn((unsigned long)vaddr + off, sz);
  67                         kunmap_atomic(vaddr);
  68                         off = 0;
  69                         ++page;
  70                         size -= sz;
  71                 }
  72 }
  73
  74 static void xtensa_sync_single_for_cpu(struct device *dev,
  75                                        dma_addr_t dma_handle, size_t size,
  76                                        enum dma_data_direction dir)
  77 {
  78         switch (dir) {
  79         case DMA_BIDIRECTIONAL:
  80         case DMA_FROM_DEVICE:
  81                 do_cache_op(dma_handle, size, __invalidate_dcache_range);
  82                 break;
  83
  84         case DMA_NONE:
  85                 BUG();
  86                 break;
  87
  88         default:
  89                 break;
  90         }
  91 }
  92
  93 static void xtensa_sync_single_for_device(struct device *dev,
  94                                           dma_addr_t dma_handle, size_t size,
  95                                           enum dma_data_direction dir)
  96 {
  97         switch (dir) {
  98         case DMA_BIDIRECTIONAL:
  99         case DMA_TO_DEVICE:
 100                 if (XCHAL_DCACHE_IS_WRITEBACK)
 101                         do_cache_op(dma_handle, size, __flush_dcache_range);
 102                 break;
 103
 104         case DMA_NONE:
 105                 BUG();
 106                 break;
 107
 108         default:
 109                 break;
 110         }
 111 }
 112
 113 static void xtensa_sync_sg_for_cpu(struct device *dev,
 114                                    struct scatterlist *sg, int nents,
 115                                    enum dma_data_direction dir)
 116 {
 117         struct scatterlist *s;
 118         int i;
 119
 120         for_each_sg(sg, s, nents, i) {
 121                 xtensa_sync_single_for_cpu(dev, sg_dma_address(s),
 122                                            sg_dma_len(s), dir);
 123         }
 124 }
 125
 126 static void xtensa_sync_sg_for_device(struct device *dev,
 127                                       struct scatterlist *sg, int nents,
 128                                       enum dma_data_direction dir)
 129 {
 130         struct scatterlist *s;
 131         int i;
 132
 133         for_each_sg(sg, s, nents, i) {
 134                 xtensa_sync_single_for_device(dev, sg_dma_address(s),
 135                                               sg_dma_len(s), dir);
 136         }
 137 }
 138
 139 /*
 140  * Note: We assume that the full memory space is always mapped to 'kseg'
 141  *       Otherwise we have to use page attributes (not implemented).
 142  */
 143
 144 static void *xtensa_dma_alloc(struct device *dev, size_t size,
 145                               dma_addr_t *handle, gfp_t flag,
 146                               unsigned long attrs)
 147 {
 148         unsigned long ret;
 149         unsigned long uncached = 0;
 150         unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT;
 151         struct page *page = NULL;
 152
 153         /* ignore region speicifiers */
 154
 155         flag &= ~(__GFP_DMA | __GFP_HIGHMEM);
 156
 157         if (dev == NULL || (dev->coherent_dma_mask < 0xffffffff))
 158                 flag |= GFP_DMA;
 159
 160         if (gfpflags_allow_blocking(flag))
 161                 page = dma_alloc_from_contiguous(dev, count, get_order(size),
 162                                                  flag);
 163
 164         if (!page)
 165                 page = alloc_pages(flag, get_order(size));
 166
 167         if (!page)
 168                 return NULL;
 169
 170         ret = (unsigned long)page_address(page);
 171
 172         /* We currently don't support coherent memory outside KSEG */
 173
 174         BUG_ON(ret < XCHAL_KSEG_CACHED_VADDR ||
 175                ret > XCHAL_KSEG_CACHED_VADDR + XCHAL_KSEG_SIZE - 1);
 176
 177         uncached = ret + XCHAL_KSEG_BYPASS_VADDR - XCHAL_KSEG_CACHED_VADDR;
 178         *handle = virt_to_bus((void *)ret);
 179         __invalidate_dcache_range(ret, size);
 180
 181         return (void *)uncached;
 182 }
 183
 184 static void xtensa_dma_free(struct device *dev, size_t size, void *vaddr,
 185                             dma_addr_t dma_handle, unsigned long attrs)
 186 {
 187         unsigned long addr = (unsigned long)vaddr +
 188                 XCHAL_KSEG_CACHED_VADDR - XCHAL_KSEG_BYPASS_VADDR;
 189         struct page *page = virt_to_page(addr);
 190         unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT;
 191
 192         BUG_ON(addr < XCHAL_KSEG_CACHED_VADDR ||
 193                addr > XCHAL_KSEG_CACHED_VADDR + XCHAL_KSEG_SIZE - 1);
 194
 195         if (!dma_release_from_contiguous(dev, page, count))
 196                 __free_pages(page, get_order(size));
 197 }
 198
 199 static dma_addr_t xtensa_map_page(struct device *dev, struct page *page,
 200                                   unsigned long offset, size_t size,
 201                                   enum dma_data_direction dir,
 202                                   unsigned long attrs)
 203 {
 204         dma_addr_t dma_handle = page_to_phys(page) + offset;
 205
 206         if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
 207                 xtensa_sync_single_for_device(dev, dma_handle, size, dir);
 208
 209         return dma_handle;
 210 }
 211
 212 static void xtensa_unmap_page(struct device *dev, dma_addr_t dma_handle,
 213                               size_t size, enum dma_data_direction dir,
 214                               unsigned long attrs)
 215 {
 216         if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
 217                 xtensa_sync_single_for_cpu(dev, dma_handle, size, dir);
 218 }
 219
 220 static int xtensa_map_sg(struct device *dev, struct scatterlist *sg,
 221                          int nents, enum dma_data_direction dir,
 222                          unsigned long attrs)
 223 {
 224         struct scatterlist *s;
 225         int i;
 226
 227         for_each_sg(sg, s, nents, i) {
 228                 s->dma_address = xtensa_map_page(dev, sg_page(s), s->offset,
 229                                                  s->length, dir, attrs);
 230         }
 231         return nents;
 232 }
 233
 234 static void xtensa_unmap_sg(struct device *dev,
 235                             struct scatterlist *sg, int nents,
 236                             enum dma_data_direction dir,
 237                             unsigned long attrs)
 238 {
 239         struct scatterlist *s;
 240         int i;
 241
 242         for_each_sg(sg, s, nents, i) {
 243                 xtensa_unmap_page(dev, sg_dma_address(s),
 244                                   sg_dma_len(s), dir, attrs);
 245         }
 246 }
 247
 248 int xtensa_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
 249 {
 250         return 0;
 251 }
 252
 253 const struct dma_map_ops xtensa_dma_map_ops = {
 254         .alloc = xtensa_dma_alloc,
 255         .free = xtensa_dma_free,
 256         .map_page = xtensa_map_page,
 257         .unmap_page = xtensa_unmap_page,
 258         .map_sg = xtensa_map_sg,
 259         .unmap_sg = xtensa_unmap_sg,
 260         .sync_single_for_cpu = xtensa_sync_single_for_cpu,
 261         .sync_single_for_device = xtensa_sync_single_for_device,
 262         .sync_sg_for_cpu = xtensa_sync_sg_for_cpu,
 263         .sync_sg_for_device = xtensa_sync_sg_for_device,
 264         .mapping_error = xtensa_dma_mapping_error,
 265 };
 266 EXPORT_SYMBOL(xtensa_dma_map_ops);
 267
 268 #define PREALLOC_DMA_DEBUG_ENTRIES (1 << 16)
 269
 270 static int __init xtensa_dma_init(void)
 271 {
 272         dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
 273         return 0;
 274 }
 275 fs_initcall(xtensa_dma_init);