kernel/dma/noncoherent.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2018 Christoph Hellwig.
   4  *
   5  * DMA operations that map physical memory directly without providing cache
   6  * coherence.
   7  */
   8 #include <linux/export.h>
   9 #include <linux/mm.h>
  10 #include <linux/dma-direct.h>
  11 #include <linux/dma-noncoherent.h>
  12 #include <linux/scatterlist.h>
  13
  14 static void dma_noncoherent_sync_single_for_device(struct device *dev,
  15                 dma_addr_t addr, size_t size, enum dma_data_direction dir)
  16 {
  17         arch_sync_dma_for_device(dev, dma_to_phys(dev, addr), size, dir);
  18 }
  19
  20 static void dma_noncoherent_sync_sg_for_device(struct device *dev,
  21                 struct scatterlist *sgl, int nents, enum dma_data_direction dir)
  22 {
  23         struct scatterlist *sg;
  24         int i;
  25
  26         for_each_sg(sgl, sg, nents, i)
  27                 arch_sync_dma_for_device(dev, sg_phys(sg), sg->length, dir);
  28 }
  29
  30 static dma_addr_t dma_noncoherent_map_page(struct device *dev, struct page *page,
  31                 unsigned long offset, size_t size, enum dma_data_direction dir,
  32                 unsigned long attrs)
  33 {
  34         dma_addr_t addr;
  35
  36         addr = dma_direct_map_page(dev, page, offset, size, dir, attrs);
  37         if (!dma_mapping_error(dev, addr) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
  38                 arch_sync_dma_for_device(dev, page_to_phys(page) + offset,
  39                                 size, dir);
  40         return addr;
  41 }
  42
  43 static int dma_noncoherent_map_sg(struct device *dev, struct scatterlist *sgl,
  44                 int nents, enum dma_data_direction dir, unsigned long attrs)
  45 {
  46         nents = dma_direct_map_sg(dev, sgl, nents, dir, attrs);
  47         if (nents > 0 && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
  48                 dma_noncoherent_sync_sg_for_device(dev, sgl, nents, dir);
  49         return nents;
  50 }
  51
  52 #if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
  53     defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL)
  54 static void dma_noncoherent_sync_single_for_cpu(struct device *dev,
  55                 dma_addr_t addr, size_t size, enum dma_data_direction dir)
  56 {
  57         arch_sync_dma_for_cpu(dev, dma_to_phys(dev, addr), size, dir);
  58         arch_sync_dma_for_cpu_all(dev);
  59 }
  60
  61 static void dma_noncoherent_sync_sg_for_cpu(struct device *dev,
  62                 struct scatterlist *sgl, int nents, enum dma_data_direction dir)
  63 {
  64         struct scatterlist *sg;
  65         int i;
  66
  67         for_each_sg(sgl, sg, nents, i)
  68                 arch_sync_dma_for_cpu(dev, sg_phys(sg), sg->length, dir);
  69         arch_sync_dma_for_cpu_all(dev);
  70 }
  71
  72 static void dma_noncoherent_unmap_page(struct device *dev, dma_addr_t addr,
  73                 size_t size, enum dma_data_direction dir, unsigned long attrs)
  74 {
  75         if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
  76                 dma_noncoherent_sync_single_for_cpu(dev, addr, size, dir);
  77 }
  78
  79 static void dma_noncoherent_unmap_sg(struct device *dev, struct scatterlist *sgl,
  80                 int nents, enum dma_data_direction dir, unsigned long attrs)
  81 {
  82         if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
  83                 dma_noncoherent_sync_sg_for_cpu(dev, sgl, nents, dir);
  84 }
  85 #endif
  86
  87 const struct dma_map_ops dma_noncoherent_ops = {
  88         .alloc                  = arch_dma_alloc,
  89         .free                   = arch_dma_free,
  90         .mmap                   = arch_dma_mmap,
  91         .sync_single_for_device = dma_noncoherent_sync_single_for_device,
  92         .sync_sg_for_device     = dma_noncoherent_sync_sg_for_device,
  93         .map_page               = dma_noncoherent_map_page,
  94         .map_sg                 = dma_noncoherent_map_sg,
  95 #if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
  96     defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL)
  97         .sync_single_for_cpu    = dma_noncoherent_sync_single_for_cpu,
  98         .sync_sg_for_cpu        = dma_noncoherent_sync_sg_for_cpu,
  99         .unmap_page             = dma_noncoherent_unmap_page,
 100         .unmap_sg               = dma_noncoherent_unmap_sg,
 101 #endif
 102         .dma_supported          = dma_direct_supported,
 103         .mapping_error          = dma_direct_mapping_error,
 104         .cache_sync             = arch_dma_cache_sync,
 105 };
 106 EXPORT_SYMBOL(dma_noncoherent_ops);