4 * Copyright (C) 2012 VMware, Inc. All rights reserved.
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License as published by the
8 * Free Software Foundation version 2 and no later version.
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
12 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 #include <linux/vmw_vmci_defs.h>
17 #include <linux/vmw_vmci_api.h>
18 #include <linux/highmem.h>
19 #include <linux/kernel.h>
21 #include <linux/module.h>
22 #include <linux/mutex.h>
23 #include <linux/pagemap.h>
24 #include <linux/pci.h>
25 #include <linux/sched.h>
26 #include <linux/slab.h>
27 #include <linux/uio.h>
28 #include <linux/wait.h>
29 #include <linux/vmalloc.h>
30 #include <linux/skbuff.h>
32 #include "vmci_handle_array.h"
33 #include "vmci_queue_pair.h"
34 #include "vmci_datagram.h"
35 #include "vmci_resource.h"
36 #include "vmci_context.h"
37 #include "vmci_driver.h"
38 #include "vmci_event.h"
39 #include "vmci_route.h"
42 * In the following, we will distinguish between two kinds of VMX processes -
43 * the ones with versions lower than VMCI_VERSION_NOVMVM that use specialized
44 * VMCI page files in the VMX and supporting VM to VM communication and the
45 * newer ones that use the guest memory directly. We will in the following
46 * refer to the older VMX versions as old-style VMX'en, and the newer ones as
49 * The state transition datagram is as follows (the VMCIQPB_ prefix has been
50 * removed for readability) - see below for more details on the transtions:
52 * -------------- NEW -------------
55 * CREATED_NO_MEM <-----------------> CREATED_MEM
57 * | o-----------------------o |
60 * ATTACHED_NO_MEM <----------------> ATTACHED_MEM
62 * | o----------------------o |
65 * SHUTDOWN_NO_MEM <----------------> SHUTDOWN_MEM
68 * -------------> gone <-------------
70 * In more detail. When a VMCI queue pair is first created, it will be in the
71 * VMCIQPB_NEW state. It will then move into one of the following states:
73 * - VMCIQPB_CREATED_NO_MEM: this state indicates that either:
75 * - the created was performed by a host endpoint, in which case there is
76 * no backing memory yet.
78 * - the create was initiated by an old-style VMX, that uses
79 * vmci_qp_broker_set_page_store to specify the UVAs of the queue pair at
80 * a later point in time. This state can be distinguished from the one
81 * above by the context ID of the creator. A host side is not allowed to
82 * attach until the page store has been set.
84 * - VMCIQPB_CREATED_MEM: this state is the result when the queue pair
85 * is created by a VMX using the queue pair device backend that
86 * sets the UVAs of the queue pair immediately and stores the
87 * information for later attachers. At this point, it is ready for
88 * the host side to attach to it.
90 * Once the queue pair is in one of the created states (with the exception of
91 * the case mentioned for older VMX'en above), it is possible to attach to the
92 * queue pair. Again we have two new states possible:
94 * - VMCIQPB_ATTACHED_MEM: this state can be reached through the following
97 * - from VMCIQPB_CREATED_NO_MEM when a new-style VMX allocates a queue
98 * pair, and attaches to a queue pair previously created by the host side.
100 * - from VMCIQPB_CREATED_MEM when the host side attaches to a queue pair
101 * already created by a guest.
103 * - from VMCIQPB_ATTACHED_NO_MEM, when an old-style VMX calls
104 * vmci_qp_broker_set_page_store (see below).
106 * - VMCIQPB_ATTACHED_NO_MEM: If the queue pair already was in the
107 * VMCIQPB_CREATED_NO_MEM due to a host side create, an old-style VMX will
108 * bring the queue pair into this state. Once vmci_qp_broker_set_page_store
109 * is called to register the user memory, the VMCIQPB_ATTACH_MEM state
112 * From the attached queue pair, the queue pair can enter the shutdown states
113 * when either side of the queue pair detaches. If the guest side detaches
114 * first, the queue pair will enter the VMCIQPB_SHUTDOWN_NO_MEM state, where
115 * the content of the queue pair will no longer be available. If the host
116 * side detaches first, the queue pair will either enter the
117 * VMCIQPB_SHUTDOWN_MEM, if the guest memory is currently mapped, or
118 * VMCIQPB_SHUTDOWN_NO_MEM, if the guest memory is not mapped
119 * (e.g., the host detaches while a guest is stunned).
121 * New-style VMX'en will also unmap guest memory, if the guest is
122 * quiesced, e.g., during a snapshot operation. In that case, the guest
123 * memory will no longer be available, and the queue pair will transition from
124 * *_MEM state to a *_NO_MEM state. The VMX may later map the memory once more,
125 * in which case the queue pair will transition from the *_NO_MEM state at that
126 * point back to the *_MEM state. Note that the *_NO_MEM state may have changed,
127 * since the peer may have either attached or detached in the meantime. The
128 * values are laid out such that ++ on a state will move from a *_NO_MEM to a
129 * *_MEM state, and vice versa.
133 * VMCIMemcpy{To,From}QueueFunc() prototypes. Functions of these
134 * types are passed around to enqueue and dequeue routines. Note that
135 * often the functions passed are simply wrappers around memcpy
138 * Note: In order for the memcpy typedefs to be compatible with the VMKernel,
139 * there's an unused last parameter for the hosted side. In
140 * ESX, that parameter holds a buffer type.
142 typedef int vmci_memcpy_to_queue_func(struct vmci_queue *queue,
143 u64 queue_offset, const void *src,
144 size_t src_offset, size_t size);
145 typedef int vmci_memcpy_from_queue_func(void *dest, size_t dest_offset,
146 const struct vmci_queue *queue,
147 u64 queue_offset, size_t size);
149 /* The Kernel specific component of the struct vmci_queue structure. */
150 struct vmci_queue_kern_if {
151 struct mutex __mutex; /* Protects the queue. */
152 struct mutex *mutex; /* Shared by producer and consumer queues. */
153 size_t num_pages; /* Number of pages incl. header. */
154 bool host; /* Host or guest? */
159 } g; /* Used by the guest. */
162 struct page **header_page;
163 } h; /* Used by the host. */
168 * This structure is opaque to the clients.
171 struct vmci_handle handle;
172 struct vmci_queue *produce_q;
173 struct vmci_queue *consume_q;
180 unsigned int blocked;
181 unsigned int generation;
182 wait_queue_head_t event;
185 enum qp_broker_state {
187 VMCIQPB_CREATED_NO_MEM,
189 VMCIQPB_ATTACHED_NO_MEM,
190 VMCIQPB_ATTACHED_MEM,
191 VMCIQPB_SHUTDOWN_NO_MEM,
192 VMCIQPB_SHUTDOWN_MEM,
196 #define QPBROKERSTATE_HAS_MEM(_qpb) (_qpb->state == VMCIQPB_CREATED_MEM || \
197 _qpb->state == VMCIQPB_ATTACHED_MEM || \
198 _qpb->state == VMCIQPB_SHUTDOWN_MEM)
201 * In the queue pair broker, we always use the guest point of view for
202 * the produce and consume queue values and references, e.g., the
203 * produce queue size stored is the guests produce queue size. The
204 * host endpoint will need to swap these around. The only exception is
205 * the local queue pairs on the host, in which case the host endpoint
206 * that creates the queue pair will have the right orientation, and
207 * the attaching host endpoint will need to swap.
210 struct list_head list_item;
211 struct vmci_handle handle;
219 struct qp_broker_entry {
220 struct vmci_resource resource;
224 enum qp_broker_state state;
225 bool require_trusted_attach;
226 bool created_by_trusted;
227 bool vmci_page_files; /* Created by VMX using VMCI page files */
228 struct vmci_queue *produce_q;
229 struct vmci_queue *consume_q;
230 struct vmci_queue_header saved_produce_q;
231 struct vmci_queue_header saved_consume_q;
232 vmci_event_release_cb wakeup_cb;
234 void *local_mem; /* Kernel memory for local queue pair */
237 struct qp_guest_endpoint {
238 struct vmci_resource resource;
243 struct ppn_set ppn_set;
247 struct list_head head;
248 struct mutex mutex; /* Protect queue list. */
251 static struct qp_list qp_broker_list = {
252 .head = LIST_HEAD_INIT(qp_broker_list.head),
253 .mutex = __MUTEX_INITIALIZER(qp_broker_list.mutex),
256 static struct qp_list qp_guest_endpoints = {
257 .head = LIST_HEAD_INIT(qp_guest_endpoints.head),
258 .mutex = __MUTEX_INITIALIZER(qp_guest_endpoints.mutex),
261 #define INVALID_VMCI_GUEST_MEM_ID 0
262 #define QPE_NUM_PAGES(_QPE) ((u32) \
263 (DIV_ROUND_UP(_QPE.produce_size, PAGE_SIZE) + \
264 DIV_ROUND_UP(_QPE.consume_size, PAGE_SIZE) + 2))
268 * Frees kernel VA space for a given queue and its queue header, and
269 * frees physical data pages.
271 static void qp_free_queue(void *q, u64 size)
273 struct vmci_queue *queue = q;
278 /* Given size does not include header, so add in a page here. */
279 for (i = 0; i < DIV_ROUND_UP(size, PAGE_SIZE) + 1; i++) {
280 dma_free_coherent(&vmci_pdev->dev, PAGE_SIZE,
281 queue->kernel_if->u.g.vas[i],
282 queue->kernel_if->u.g.pas[i]);
290 * Allocates kernel queue pages of specified size with IOMMU mappings,
291 * plus space for the queue structure/kernel interface and the queue
294 static void *qp_alloc_queue(u64 size, u32 flags)
297 struct vmci_queue *queue;
300 size_t queue_size = sizeof(*queue) + sizeof(*queue->kernel_if);
303 if (size > SIZE_MAX - PAGE_SIZE)
305 num_pages = DIV_ROUND_UP(size, PAGE_SIZE) + 1;
307 (SIZE_MAX - queue_size) /
308 (sizeof(*queue->kernel_if->u.g.pas) +
309 sizeof(*queue->kernel_if->u.g.vas)))
312 pas_size = num_pages * sizeof(*queue->kernel_if->u.g.pas);
313 vas_size = num_pages * sizeof(*queue->kernel_if->u.g.vas);
314 queue_size += pas_size + vas_size;
316 queue = vmalloc(queue_size);
320 queue->q_header = NULL;
321 queue->saved_header = NULL;
322 queue->kernel_if = (struct vmci_queue_kern_if *)(queue + 1);
323 queue->kernel_if->mutex = NULL;
324 queue->kernel_if->num_pages = num_pages;
325 queue->kernel_if->u.g.pas = (dma_addr_t *)(queue->kernel_if + 1);
326 queue->kernel_if->u.g.vas =
327 (void **)((u8 *)queue->kernel_if->u.g.pas + pas_size);
328 queue->kernel_if->host = false;
330 for (i = 0; i < num_pages; i++) {
331 queue->kernel_if->u.g.vas[i] =
332 dma_alloc_coherent(&vmci_pdev->dev, PAGE_SIZE,
333 &queue->kernel_if->u.g.pas[i],
335 if (!queue->kernel_if->u.g.vas[i]) {
336 /* Size excl. the header. */
337 qp_free_queue(queue, i * PAGE_SIZE);
342 /* Queue header is the first page. */
343 queue->q_header = queue->kernel_if->u.g.vas[0];
349 * Copies from a given buffer or iovector to a VMCI Queue. Uses
350 * kmap()/kunmap() to dynamically map/unmap required portions of the queue
351 * by traversing the offset -> page translation structure for the queue.
352 * Assumes that offset + size does not wrap around in the queue.
354 static int __qp_memcpy_to_queue(struct vmci_queue *queue,
360 struct vmci_queue_kern_if *kernel_if = queue->kernel_if;
361 size_t bytes_copied = 0;
363 while (bytes_copied < size) {
364 const u64 page_index =
365 (queue_offset + bytes_copied) / PAGE_SIZE;
366 const size_t page_offset =
367 (queue_offset + bytes_copied) & (PAGE_SIZE - 1);
372 va = kmap(kernel_if->u.h.page[page_index]);
374 va = kernel_if->u.g.vas[page_index + 1];
377 if (size - bytes_copied > PAGE_SIZE - page_offset)
378 /* Enough payload to fill up from this page. */
379 to_copy = PAGE_SIZE - page_offset;
381 to_copy = size - bytes_copied;
384 struct msghdr *msg = (struct msghdr *)src;
387 /* The iovec will track bytes_copied internally. */
388 err = memcpy_from_msg((u8 *)va + page_offset,
392 kunmap(kernel_if->u.h.page[page_index]);
393 return VMCI_ERROR_INVALID_ARGS;
396 memcpy((u8 *)va + page_offset,
397 (u8 *)src + bytes_copied, to_copy);
400 bytes_copied += to_copy;
402 kunmap(kernel_if->u.h.page[page_index]);
409 * Copies to a given buffer or iovector from a VMCI Queue. Uses
410 * kmap()/kunmap() to dynamically map/unmap required portions of the queue
411 * by traversing the offset -> page translation structure for the queue.
412 * Assumes that offset + size does not wrap around in the queue.
414 static int __qp_memcpy_from_queue(void *dest,
415 const struct vmci_queue *queue,
420 struct vmci_queue_kern_if *kernel_if = queue->kernel_if;
421 size_t bytes_copied = 0;
423 while (bytes_copied < size) {
424 const u64 page_index =
425 (queue_offset + bytes_copied) / PAGE_SIZE;
426 const size_t page_offset =
427 (queue_offset + bytes_copied) & (PAGE_SIZE - 1);
432 va = kmap(kernel_if->u.h.page[page_index]);
434 va = kernel_if->u.g.vas[page_index + 1];
437 if (size - bytes_copied > PAGE_SIZE - page_offset)
438 /* Enough payload to fill up this page. */
439 to_copy = PAGE_SIZE - page_offset;
441 to_copy = size - bytes_copied;
444 struct msghdr *msg = dest;
447 /* The iovec will track bytes_copied internally. */
448 err = memcpy_to_msg(msg, (u8 *)va + page_offset,
452 kunmap(kernel_if->u.h.page[page_index]);
453 return VMCI_ERROR_INVALID_ARGS;
456 memcpy((u8 *)dest + bytes_copied,
457 (u8 *)va + page_offset, to_copy);
460 bytes_copied += to_copy;
462 kunmap(kernel_if->u.h.page[page_index]);
469 * Allocates two list of PPNs --- one for the pages in the produce queue,
470 * and the other for the pages in the consume queue. Intializes the list
471 * of PPNs with the page frame numbers of the KVA for the two queues (and
472 * the queue headers).
474 static int qp_alloc_ppn_set(void *prod_q,
475 u64 num_produce_pages,
477 u64 num_consume_pages, struct ppn_set *ppn_set)
481 struct vmci_queue *produce_q = prod_q;
482 struct vmci_queue *consume_q = cons_q;
485 if (!produce_q || !num_produce_pages || !consume_q ||
486 !num_consume_pages || !ppn_set)
487 return VMCI_ERROR_INVALID_ARGS;
489 if (ppn_set->initialized)
490 return VMCI_ERROR_ALREADY_EXISTS;
493 kmalloc(num_produce_pages * sizeof(*produce_ppns), GFP_KERNEL);
495 return VMCI_ERROR_NO_MEM;
498 kmalloc(num_consume_pages * sizeof(*consume_ppns), GFP_KERNEL);
501 return VMCI_ERROR_NO_MEM;
504 for (i = 0; i < num_produce_pages; i++) {
508 produce_q->kernel_if->u.g.pas[i] >> PAGE_SHIFT;
509 pfn = produce_ppns[i];
511 /* Fail allocation if PFN isn't supported by hypervisor. */
512 if (sizeof(pfn) > sizeof(*produce_ppns)
513 && pfn != produce_ppns[i])
517 for (i = 0; i < num_consume_pages; i++) {
521 consume_q->kernel_if->u.g.pas[i] >> PAGE_SHIFT;
522 pfn = consume_ppns[i];
524 /* Fail allocation if PFN isn't supported by hypervisor. */
525 if (sizeof(pfn) > sizeof(*consume_ppns)
526 && pfn != consume_ppns[i])
530 ppn_set->num_produce_pages = num_produce_pages;
531 ppn_set->num_consume_pages = num_consume_pages;
532 ppn_set->produce_ppns = produce_ppns;
533 ppn_set->consume_ppns = consume_ppns;
534 ppn_set->initialized = true;
540 return VMCI_ERROR_INVALID_ARGS;
544 * Frees the two list of PPNs for a queue pair.
546 static void qp_free_ppn_set(struct ppn_set *ppn_set)
548 if (ppn_set->initialized) {
549 /* Do not call these functions on NULL inputs. */
550 kfree(ppn_set->produce_ppns);
551 kfree(ppn_set->consume_ppns);
553 memset(ppn_set, 0, sizeof(*ppn_set));
557 * Populates the list of PPNs in the hypercall structure with the PPNS
558 * of the produce queue and the consume queue.
560 static int qp_populate_ppn_set(u8 *call_buf, const struct ppn_set *ppn_set)
562 memcpy(call_buf, ppn_set->produce_ppns,
563 ppn_set->num_produce_pages * sizeof(*ppn_set->produce_ppns));
565 ppn_set->num_produce_pages * sizeof(*ppn_set->produce_ppns),
566 ppn_set->consume_ppns,
567 ppn_set->num_consume_pages * sizeof(*ppn_set->consume_ppns));
572 static int qp_memcpy_to_queue(struct vmci_queue *queue,
574 const void *src, size_t src_offset, size_t size)
576 return __qp_memcpy_to_queue(queue, queue_offset,
577 (u8 *)src + src_offset, size, false);
580 static int qp_memcpy_from_queue(void *dest,
582 const struct vmci_queue *queue,
583 u64 queue_offset, size_t size)
585 return __qp_memcpy_from_queue((u8 *)dest + dest_offset,
586 queue, queue_offset, size, false);
590 * Copies from a given iovec from a VMCI Queue.
592 static int qp_memcpy_to_queue_iov(struct vmci_queue *queue,
595 size_t src_offset, size_t size)
599 * We ignore src_offset because src is really a struct iovec * and will
600 * maintain offset internally.
602 return __qp_memcpy_to_queue(queue, queue_offset, msg, size, true);
606 * Copies to a given iovec from a VMCI Queue.
608 static int qp_memcpy_from_queue_iov(void *dest,
610 const struct vmci_queue *queue,
611 u64 queue_offset, size_t size)
614 * We ignore dest_offset because dest is really a struct iovec * and
615 * will maintain offset internally.
617 return __qp_memcpy_from_queue(dest, queue, queue_offset, size, true);
621 * Allocates kernel VA space of specified size plus space for the queue
622 * and kernel interface. This is different from the guest queue allocator,
623 * because we do not allocate our own queue header/data pages here but
624 * share those of the guest.
626 static struct vmci_queue *qp_host_alloc_queue(u64 size)
628 struct vmci_queue *queue;
629 size_t queue_page_size;
631 const size_t queue_size = sizeof(*queue) + sizeof(*(queue->kernel_if));
633 if (size > SIZE_MAX - PAGE_SIZE)
635 num_pages = DIV_ROUND_UP(size, PAGE_SIZE) + 1;
636 if (num_pages > (SIZE_MAX - queue_size) /
637 sizeof(*queue->kernel_if->u.h.page))
640 queue_page_size = num_pages * sizeof(*queue->kernel_if->u.h.page);
642 if (queue_size + queue_page_size > KMALLOC_MAX_SIZE)
645 queue = kzalloc(queue_size + queue_page_size, GFP_KERNEL);
647 queue->q_header = NULL;
648 queue->saved_header = NULL;
649 queue->kernel_if = (struct vmci_queue_kern_if *)(queue + 1);
650 queue->kernel_if->host = true;
651 queue->kernel_if->mutex = NULL;
652 queue->kernel_if->num_pages = num_pages;
653 queue->kernel_if->u.h.header_page =
654 (struct page **)((u8 *)queue + queue_size);
655 queue->kernel_if->u.h.page =
656 &queue->kernel_if->u.h.header_page[1];
663 * Frees kernel memory for a given queue (header plus translation
666 static void qp_host_free_queue(struct vmci_queue *queue, u64 queue_size)
672 * Initialize the mutex for the pair of queues. This mutex is used to
673 * protect the q_header and the buffer from changing out from under any
674 * users of either queue. Of course, it's only any good if the mutexes
675 * are actually acquired. Queue structure must lie on non-paged memory
676 * or we cannot guarantee access to the mutex.
678 static void qp_init_queue_mutex(struct vmci_queue *produce_q,
679 struct vmci_queue *consume_q)
682 * Only the host queue has shared state - the guest queues do not
683 * need to synchronize access using a queue mutex.
686 if (produce_q->kernel_if->host) {
687 produce_q->kernel_if->mutex = &produce_q->kernel_if->__mutex;
688 consume_q->kernel_if->mutex = &produce_q->kernel_if->__mutex;
689 mutex_init(produce_q->kernel_if->mutex);
694 * Cleans up the mutex for the pair of queues.
696 static void qp_cleanup_queue_mutex(struct vmci_queue *produce_q,
697 struct vmci_queue *consume_q)
699 if (produce_q->kernel_if->host) {
700 produce_q->kernel_if->mutex = NULL;
701 consume_q->kernel_if->mutex = NULL;
706 * Acquire the mutex for the queue. Note that the produce_q and
707 * the consume_q share a mutex. So, only one of the two need to
708 * be passed in to this routine. Either will work just fine.
710 static void qp_acquire_queue_mutex(struct vmci_queue *queue)
712 if (queue->kernel_if->host)
713 mutex_lock(queue->kernel_if->mutex);
717 * Release the mutex for the queue. Note that the produce_q and
718 * the consume_q share a mutex. So, only one of the two need to
719 * be passed in to this routine. Either will work just fine.
721 static void qp_release_queue_mutex(struct vmci_queue *queue)
723 if (queue->kernel_if->host)
724 mutex_unlock(queue->kernel_if->mutex);
728 * Helper function to release pages in the PageStoreAttachInfo
729 * previously obtained using get_user_pages.
731 static void qp_release_pages(struct page **pages,
732 u64 num_pages, bool dirty)
736 for (i = 0; i < num_pages; i++) {
738 set_page_dirty_lock(pages[i]);
746 * Lock the user pages referenced by the {produce,consume}Buffer
747 * struct into memory and populate the {produce,consume}Pages
748 * arrays in the attach structure with them.
750 static int qp_host_get_user_memory(u64 produce_uva,
752 struct vmci_queue *produce_q,
753 struct vmci_queue *consume_q)
756 int err = VMCI_SUCCESS;
758 retval = get_user_pages_fast((uintptr_t) produce_uva,
759 produce_q->kernel_if->num_pages, 1,
760 produce_q->kernel_if->u.h.header_page);
761 if (retval < (int)produce_q->kernel_if->num_pages) {
762 pr_debug("get_user_pages_fast(produce) failed (retval=%d)",
765 qp_release_pages(produce_q->kernel_if->u.h.header_page,
767 err = VMCI_ERROR_NO_MEM;
771 retval = get_user_pages_fast((uintptr_t) consume_uva,
772 consume_q->kernel_if->num_pages, 1,
773 consume_q->kernel_if->u.h.header_page);
774 if (retval < (int)consume_q->kernel_if->num_pages) {
775 pr_debug("get_user_pages_fast(consume) failed (retval=%d)",
778 qp_release_pages(consume_q->kernel_if->u.h.header_page,
780 qp_release_pages(produce_q->kernel_if->u.h.header_page,
781 produce_q->kernel_if->num_pages, false);
782 err = VMCI_ERROR_NO_MEM;
790 * Registers the specification of the user pages used for backing a queue
791 * pair. Enough information to map in pages is stored in the OS specific
792 * part of the struct vmci_queue structure.
794 static int qp_host_register_user_memory(struct vmci_qp_page_store *page_store,
795 struct vmci_queue *produce_q,
796 struct vmci_queue *consume_q)
802 * The new style and the old style mapping only differs in
803 * that we either get a single or two UVAs, so we split the
804 * single UVA range at the appropriate spot.
806 produce_uva = page_store->pages;
807 consume_uva = page_store->pages +
808 produce_q->kernel_if->num_pages * PAGE_SIZE;
809 return qp_host_get_user_memory(produce_uva, consume_uva, produce_q,
814 * Releases and removes the references to user pages stored in the attach
815 * struct. Pages are released from the page cache and may become
818 static void qp_host_unregister_user_memory(struct vmci_queue *produce_q,
819 struct vmci_queue *consume_q)
821 qp_release_pages(produce_q->kernel_if->u.h.header_page,
822 produce_q->kernel_if->num_pages, true);
823 memset(produce_q->kernel_if->u.h.header_page, 0,
824 sizeof(*produce_q->kernel_if->u.h.header_page) *
825 produce_q->kernel_if->num_pages);
826 qp_release_pages(consume_q->kernel_if->u.h.header_page,
827 consume_q->kernel_if->num_pages, true);
828 memset(consume_q->kernel_if->u.h.header_page, 0,
829 sizeof(*consume_q->kernel_if->u.h.header_page) *
830 consume_q->kernel_if->num_pages);
834 * Once qp_host_register_user_memory has been performed on a
835 * queue, the queue pair headers can be mapped into the
836 * kernel. Once mapped, they must be unmapped with
837 * qp_host_unmap_queues prior to calling
838 * qp_host_unregister_user_memory.
841 static int qp_host_map_queues(struct vmci_queue *produce_q,
842 struct vmci_queue *consume_q)
846 if (!produce_q->q_header || !consume_q->q_header) {
847 struct page *headers[2];
849 if (produce_q->q_header != consume_q->q_header)
850 return VMCI_ERROR_QUEUEPAIR_MISMATCH;
852 if (produce_q->kernel_if->u.h.header_page == NULL ||
853 *produce_q->kernel_if->u.h.header_page == NULL)
854 return VMCI_ERROR_UNAVAILABLE;
856 headers[0] = *produce_q->kernel_if->u.h.header_page;
857 headers[1] = *consume_q->kernel_if->u.h.header_page;
859 produce_q->q_header = vmap(headers, 2, VM_MAP, PAGE_KERNEL);
860 if (produce_q->q_header != NULL) {
861 consume_q->q_header =
862 (struct vmci_queue_header *)((u8 *)
863 produce_q->q_header +
865 result = VMCI_SUCCESS;
867 pr_warn("vmap failed\n");
868 result = VMCI_ERROR_NO_MEM;
871 result = VMCI_SUCCESS;
878 * Unmaps previously mapped queue pair headers from the kernel.
879 * Pages are unpinned.
881 static int qp_host_unmap_queues(u32 gid,
882 struct vmci_queue *produce_q,
883 struct vmci_queue *consume_q)
885 if (produce_q->q_header) {
886 if (produce_q->q_header < consume_q->q_header)
887 vunmap(produce_q->q_header);
889 vunmap(consume_q->q_header);
891 produce_q->q_header = NULL;
892 consume_q->q_header = NULL;
899 * Finds the entry in the list corresponding to a given handle. Assumes
900 * that the list is locked.
902 static struct qp_entry *qp_list_find(struct qp_list *qp_list,
903 struct vmci_handle handle)
905 struct qp_entry *entry;
907 if (vmci_handle_is_invalid(handle))
910 list_for_each_entry(entry, &qp_list->head, list_item) {
911 if (vmci_handle_is_equal(entry->handle, handle))
919 * Finds the entry in the list corresponding to a given handle.
921 static struct qp_guest_endpoint *
922 qp_guest_handle_to_entry(struct vmci_handle handle)
924 struct qp_guest_endpoint *entry;
925 struct qp_entry *qp = qp_list_find(&qp_guest_endpoints, handle);
927 entry = qp ? container_of(
928 qp, struct qp_guest_endpoint, qp) : NULL;
933 * Finds the entry in the list corresponding to a given handle.
935 static struct qp_broker_entry *
936 qp_broker_handle_to_entry(struct vmci_handle handle)
938 struct qp_broker_entry *entry;
939 struct qp_entry *qp = qp_list_find(&qp_broker_list, handle);
941 entry = qp ? container_of(
942 qp, struct qp_broker_entry, qp) : NULL;
947 * Dispatches a queue pair event message directly into the local event
950 static int qp_notify_peer_local(bool attach, struct vmci_handle handle)
952 u32 context_id = vmci_get_context_id();
953 struct vmci_event_qp ev;
955 memset(&ev, 0, sizeof(ev));
956 ev.msg.hdr.dst = vmci_make_handle(context_id, VMCI_EVENT_HANDLER);
957 ev.msg.hdr.src = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
958 VMCI_CONTEXT_RESOURCE_ID);
959 ev.msg.hdr.payload_size = sizeof(ev) - sizeof(ev.msg.hdr);
960 ev.msg.event_data.event =
961 attach ? VMCI_EVENT_QP_PEER_ATTACH : VMCI_EVENT_QP_PEER_DETACH;
962 ev.payload.peer_id = context_id;
963 ev.payload.handle = handle;
965 return vmci_event_dispatch(&ev.msg.hdr);
969 * Allocates and initializes a qp_guest_endpoint structure.
970 * Allocates a queue_pair rid (and handle) iff the given entry has
971 * an invalid handle. 0 through VMCI_RESERVED_RESOURCE_ID_MAX
972 * are reserved handles. Assumes that the QP list mutex is held
975 static struct qp_guest_endpoint *
976 qp_guest_endpoint_create(struct vmci_handle handle,
985 struct qp_guest_endpoint *entry;
986 /* One page each for the queue headers. */
987 const u64 num_ppns = DIV_ROUND_UP(produce_size, PAGE_SIZE) +
988 DIV_ROUND_UP(consume_size, PAGE_SIZE) + 2;
990 if (vmci_handle_is_invalid(handle)) {
991 u32 context_id = vmci_get_context_id();
993 handle = vmci_make_handle(context_id, VMCI_INVALID_ID);
996 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
998 entry->qp.peer = peer;
999 entry->qp.flags = flags;
1000 entry->qp.produce_size = produce_size;
1001 entry->qp.consume_size = consume_size;
1002 entry->qp.ref_count = 0;
1003 entry->num_ppns = num_ppns;
1004 entry->produce_q = produce_q;
1005 entry->consume_q = consume_q;
1006 INIT_LIST_HEAD(&entry->qp.list_item);
1008 /* Add resource obj */
1009 result = vmci_resource_add(&entry->resource,
1010 VMCI_RESOURCE_TYPE_QPAIR_GUEST,
1012 entry->qp.handle = vmci_resource_handle(&entry->resource);
1013 if ((result != VMCI_SUCCESS) ||
1014 qp_list_find(&qp_guest_endpoints, entry->qp.handle)) {
1015 pr_warn("Failed to add new resource (handle=0x%x:0x%x), error: %d",
1016 handle.context, handle.resource, result);
1025 * Frees a qp_guest_endpoint structure.
1027 static void qp_guest_endpoint_destroy(struct qp_guest_endpoint *entry)
1029 qp_free_ppn_set(&entry->ppn_set);
1030 qp_cleanup_queue_mutex(entry->produce_q, entry->consume_q);
1031 qp_free_queue(entry->produce_q, entry->qp.produce_size);
1032 qp_free_queue(entry->consume_q, entry->qp.consume_size);
1033 /* Unlink from resource hash table and free callback */
1034 vmci_resource_remove(&entry->resource);
1040 * Helper to make a queue_pairAlloc hypercall when the driver is
1041 * supporting a guest device.
1043 static int qp_alloc_hypercall(const struct qp_guest_endpoint *entry)
1045 struct vmci_qp_alloc_msg *alloc_msg;
1049 if (!entry || entry->num_ppns <= 2)
1050 return VMCI_ERROR_INVALID_ARGS;
1052 msg_size = sizeof(*alloc_msg) +
1053 (size_t) entry->num_ppns * sizeof(u32);
1054 alloc_msg = kmalloc(msg_size, GFP_KERNEL);
1056 return VMCI_ERROR_NO_MEM;
1058 alloc_msg->hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
1059 VMCI_QUEUEPAIR_ALLOC);
1060 alloc_msg->hdr.src = VMCI_ANON_SRC_HANDLE;
1061 alloc_msg->hdr.payload_size = msg_size - VMCI_DG_HEADERSIZE;
1062 alloc_msg->handle = entry->qp.handle;
1063 alloc_msg->peer = entry->qp.peer;
1064 alloc_msg->flags = entry->qp.flags;
1065 alloc_msg->produce_size = entry->qp.produce_size;
1066 alloc_msg->consume_size = entry->qp.consume_size;
1067 alloc_msg->num_ppns = entry->num_ppns;
1069 result = qp_populate_ppn_set((u8 *)alloc_msg + sizeof(*alloc_msg),
1071 if (result == VMCI_SUCCESS)
1072 result = vmci_send_datagram(&alloc_msg->hdr);
1080 * Helper to make a queue_pairDetach hypercall when the driver is
1081 * supporting a guest device.
1083 static int qp_detatch_hypercall(struct vmci_handle handle)
1085 struct vmci_qp_detach_msg detach_msg;
1087 detach_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
1088 VMCI_QUEUEPAIR_DETACH);
1089 detach_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
1090 detach_msg.hdr.payload_size = sizeof(handle);
1091 detach_msg.handle = handle;
1093 return vmci_send_datagram(&detach_msg.hdr);
1097 * Adds the given entry to the list. Assumes that the list is locked.
1099 static void qp_list_add_entry(struct qp_list *qp_list, struct qp_entry *entry)
1102 list_add(&entry->list_item, &qp_list->head);
1106 * Removes the given entry from the list. Assumes that the list is locked.
1108 static void qp_list_remove_entry(struct qp_list *qp_list,
1109 struct qp_entry *entry)
1112 list_del(&entry->list_item);
1116 * Helper for VMCI queue_pair detach interface. Frees the physical
1117 * pages for the queue pair.
1119 static int qp_detatch_guest_work(struct vmci_handle handle)
1122 struct qp_guest_endpoint *entry;
1123 u32 ref_count = ~0; /* To avoid compiler warning below */
1125 mutex_lock(&qp_guest_endpoints.mutex);
1127 entry = qp_guest_handle_to_entry(handle);
1129 mutex_unlock(&qp_guest_endpoints.mutex);
1130 return VMCI_ERROR_NOT_FOUND;
1133 if (entry->qp.flags & VMCI_QPFLAG_LOCAL) {
1134 result = VMCI_SUCCESS;
1136 if (entry->qp.ref_count > 1) {
1137 result = qp_notify_peer_local(false, handle);
1139 * We can fail to notify a local queuepair
1140 * because we can't allocate. We still want
1141 * to release the entry if that happens, so
1142 * don't bail out yet.
1146 result = qp_detatch_hypercall(handle);
1147 if (result < VMCI_SUCCESS) {
1149 * We failed to notify a non-local queuepair.
1150 * That other queuepair might still be
1151 * accessing the shared memory, so don't
1152 * release the entry yet. It will get cleaned
1153 * up by VMCIqueue_pair_Exit() if necessary
1154 * (assuming we are going away, otherwise why
1158 mutex_unlock(&qp_guest_endpoints.mutex);
1164 * If we get here then we either failed to notify a local queuepair, or
1165 * we succeeded in all cases. Release the entry if required.
1168 entry->qp.ref_count--;
1169 if (entry->qp.ref_count == 0)
1170 qp_list_remove_entry(&qp_guest_endpoints, &entry->qp);
1172 /* If we didn't remove the entry, this could change once we unlock. */
1174 ref_count = entry->qp.ref_count;
1176 mutex_unlock(&qp_guest_endpoints.mutex);
1179 qp_guest_endpoint_destroy(entry);
1185 * This functions handles the actual allocation of a VMCI queue
1186 * pair guest endpoint. Allocates physical pages for the queue
1187 * pair. It makes OS dependent calls through generic wrappers.
1189 static int qp_alloc_guest_work(struct vmci_handle *handle,
1190 struct vmci_queue **produce_q,
1192 struct vmci_queue **consume_q,
1198 const u64 num_produce_pages =
1199 DIV_ROUND_UP(produce_size, PAGE_SIZE) + 1;
1200 const u64 num_consume_pages =
1201 DIV_ROUND_UP(consume_size, PAGE_SIZE) + 1;
1202 void *my_produce_q = NULL;
1203 void *my_consume_q = NULL;
1205 struct qp_guest_endpoint *queue_pair_entry = NULL;
1207 if (priv_flags != VMCI_NO_PRIVILEGE_FLAGS)
1208 return VMCI_ERROR_NO_ACCESS;
1210 mutex_lock(&qp_guest_endpoints.mutex);
1212 queue_pair_entry = qp_guest_handle_to_entry(*handle);
1213 if (queue_pair_entry) {
1214 if (queue_pair_entry->qp.flags & VMCI_QPFLAG_LOCAL) {
1215 /* Local attach case. */
1216 if (queue_pair_entry->qp.ref_count > 1) {
1217 pr_devel("Error attempting to attach more than once\n");
1218 result = VMCI_ERROR_UNAVAILABLE;
1219 goto error_keep_entry;
1222 if (queue_pair_entry->qp.produce_size != consume_size ||
1223 queue_pair_entry->qp.consume_size !=
1225 queue_pair_entry->qp.flags !=
1226 (flags & ~VMCI_QPFLAG_ATTACH_ONLY)) {
1227 pr_devel("Error mismatched queue pair in local attach\n");
1228 result = VMCI_ERROR_QUEUEPAIR_MISMATCH;
1229 goto error_keep_entry;
1233 * Do a local attach. We swap the consume and
1234 * produce queues for the attacher and deliver
1237 result = qp_notify_peer_local(true, *handle);
1238 if (result < VMCI_SUCCESS)
1239 goto error_keep_entry;
1241 my_produce_q = queue_pair_entry->consume_q;
1242 my_consume_q = queue_pair_entry->produce_q;
1246 result = VMCI_ERROR_ALREADY_EXISTS;
1247 goto error_keep_entry;
1250 my_produce_q = qp_alloc_queue(produce_size, flags);
1251 if (!my_produce_q) {
1252 pr_warn("Error allocating pages for produce queue\n");
1253 result = VMCI_ERROR_NO_MEM;
1257 my_consume_q = qp_alloc_queue(consume_size, flags);
1258 if (!my_consume_q) {
1259 pr_warn("Error allocating pages for consume queue\n");
1260 result = VMCI_ERROR_NO_MEM;
1264 queue_pair_entry = qp_guest_endpoint_create(*handle, peer, flags,
1265 produce_size, consume_size,
1266 my_produce_q, my_consume_q);
1267 if (!queue_pair_entry) {
1268 pr_warn("Error allocating memory in %s\n", __func__);
1269 result = VMCI_ERROR_NO_MEM;
1273 result = qp_alloc_ppn_set(my_produce_q, num_produce_pages, my_consume_q,
1275 &queue_pair_entry->ppn_set);
1276 if (result < VMCI_SUCCESS) {
1277 pr_warn("qp_alloc_ppn_set failed\n");
1282 * It's only necessary to notify the host if this queue pair will be
1283 * attached to from another context.
1285 if (queue_pair_entry->qp.flags & VMCI_QPFLAG_LOCAL) {
1286 /* Local create case. */
1287 u32 context_id = vmci_get_context_id();
1290 * Enforce similar checks on local queue pairs as we
1291 * do for regular ones. The handle's context must
1292 * match the creator or attacher context id (here they
1293 * are both the current context id) and the
1294 * attach-only flag cannot exist during create. We
1295 * also ensure specified peer is this context or an
1298 if (queue_pair_entry->qp.handle.context != context_id ||
1299 (queue_pair_entry->qp.peer != VMCI_INVALID_ID &&
1300 queue_pair_entry->qp.peer != context_id)) {
1301 result = VMCI_ERROR_NO_ACCESS;
1305 if (queue_pair_entry->qp.flags & VMCI_QPFLAG_ATTACH_ONLY) {
1306 result = VMCI_ERROR_NOT_FOUND;
1310 result = qp_alloc_hypercall(queue_pair_entry);
1311 if (result < VMCI_SUCCESS) {
1312 pr_warn("qp_alloc_hypercall result = %d\n", result);
1317 qp_init_queue_mutex((struct vmci_queue *)my_produce_q,
1318 (struct vmci_queue *)my_consume_q);
1320 qp_list_add_entry(&qp_guest_endpoints, &queue_pair_entry->qp);
1323 queue_pair_entry->qp.ref_count++;
1324 *handle = queue_pair_entry->qp.handle;
1325 *produce_q = (struct vmci_queue *)my_produce_q;
1326 *consume_q = (struct vmci_queue *)my_consume_q;
1329 * We should initialize the queue pair header pages on a local
1330 * queue pair create. For non-local queue pairs, the
1331 * hypervisor initializes the header pages in the create step.
1333 if ((queue_pair_entry->qp.flags & VMCI_QPFLAG_LOCAL) &&
1334 queue_pair_entry->qp.ref_count == 1) {
1335 vmci_q_header_init((*produce_q)->q_header, *handle);
1336 vmci_q_header_init((*consume_q)->q_header, *handle);
1339 mutex_unlock(&qp_guest_endpoints.mutex);
1341 return VMCI_SUCCESS;
1344 mutex_unlock(&qp_guest_endpoints.mutex);
1345 if (queue_pair_entry) {
1346 /* The queues will be freed inside the destroy routine. */
1347 qp_guest_endpoint_destroy(queue_pair_entry);
1349 qp_free_queue(my_produce_q, produce_size);
1350 qp_free_queue(my_consume_q, consume_size);
1355 /* This path should only be used when an existing entry was found. */
1356 mutex_unlock(&qp_guest_endpoints.mutex);
1361 * The first endpoint issuing a queue pair allocation will create the state
1362 * of the queue pair in the queue pair broker.
1364 * If the creator is a guest, it will associate a VMX virtual address range
1365 * with the queue pair as specified by the page_store. For compatibility with
1366 * older VMX'en, that would use a separate step to set the VMX virtual
1367 * address range, the virtual address range can be registered later using
1368 * vmci_qp_broker_set_page_store. In that case, a page_store of NULL should be
1371 * If the creator is the host, a page_store of NULL should be used as well,
1372 * since the host is not able to supply a page store for the queue pair.
1374 * For older VMX and host callers, the queue pair will be created in the
1375 * VMCIQPB_CREATED_NO_MEM state, and for current VMX callers, it will be
1376 * created in VMCOQPB_CREATED_MEM state.
1378 static int qp_broker_create(struct vmci_handle handle,
1384 struct vmci_qp_page_store *page_store,
1385 struct vmci_ctx *context,
1386 vmci_event_release_cb wakeup_cb,
1387 void *client_data, struct qp_broker_entry **ent)
1389 struct qp_broker_entry *entry = NULL;
1390 const u32 context_id = vmci_ctx_get_id(context);
1391 bool is_local = flags & VMCI_QPFLAG_LOCAL;
1393 u64 guest_produce_size;
1394 u64 guest_consume_size;
1396 /* Do not create if the caller asked not to. */
1397 if (flags & VMCI_QPFLAG_ATTACH_ONLY)
1398 return VMCI_ERROR_NOT_FOUND;
1401 * Creator's context ID should match handle's context ID or the creator
1402 * must allow the context in handle's context ID as the "peer".
1404 if (handle.context != context_id && handle.context != peer)
1405 return VMCI_ERROR_NO_ACCESS;
1407 if (VMCI_CONTEXT_IS_VM(context_id) && VMCI_CONTEXT_IS_VM(peer))
1408 return VMCI_ERROR_DST_UNREACHABLE;
1411 * Creator's context ID for local queue pairs should match the
1412 * peer, if a peer is specified.
1414 if (is_local && peer != VMCI_INVALID_ID && context_id != peer)
1415 return VMCI_ERROR_NO_ACCESS;
1417 entry = kzalloc(sizeof(*entry), GFP_ATOMIC);
1419 return VMCI_ERROR_NO_MEM;
1421 if (vmci_ctx_get_id(context) == VMCI_HOST_CONTEXT_ID && !is_local) {
1423 * The queue pair broker entry stores values from the guest
1424 * point of view, so a creating host side endpoint should swap
1425 * produce and consume values -- unless it is a local queue
1426 * pair, in which case no swapping is necessary, since the local
1427 * attacher will swap queues.
1430 guest_produce_size = consume_size;
1431 guest_consume_size = produce_size;
1433 guest_produce_size = produce_size;
1434 guest_consume_size = consume_size;
1437 entry->qp.handle = handle;
1438 entry->qp.peer = peer;
1439 entry->qp.flags = flags;
1440 entry->qp.produce_size = guest_produce_size;
1441 entry->qp.consume_size = guest_consume_size;
1442 entry->qp.ref_count = 1;
1443 entry->create_id = context_id;
1444 entry->attach_id = VMCI_INVALID_ID;
1445 entry->state = VMCIQPB_NEW;
1446 entry->require_trusted_attach =
1447 !!(context->priv_flags & VMCI_PRIVILEGE_FLAG_RESTRICTED);
1448 entry->created_by_trusted =
1449 !!(priv_flags & VMCI_PRIVILEGE_FLAG_TRUSTED);
1450 entry->vmci_page_files = false;
1451 entry->wakeup_cb = wakeup_cb;
1452 entry->client_data = client_data;
1453 entry->produce_q = qp_host_alloc_queue(guest_produce_size);
1454 if (entry->produce_q == NULL) {
1455 result = VMCI_ERROR_NO_MEM;
1458 entry->consume_q = qp_host_alloc_queue(guest_consume_size);
1459 if (entry->consume_q == NULL) {
1460 result = VMCI_ERROR_NO_MEM;
1464 qp_init_queue_mutex(entry->produce_q, entry->consume_q);
1466 INIT_LIST_HEAD(&entry->qp.list_item);
1471 entry->local_mem = kcalloc(QPE_NUM_PAGES(entry->qp),
1472 PAGE_SIZE, GFP_KERNEL);
1473 if (entry->local_mem == NULL) {
1474 result = VMCI_ERROR_NO_MEM;
1477 entry->state = VMCIQPB_CREATED_MEM;
1478 entry->produce_q->q_header = entry->local_mem;
1479 tmp = (u8 *)entry->local_mem + PAGE_SIZE *
1480 (DIV_ROUND_UP(entry->qp.produce_size, PAGE_SIZE) + 1);
1481 entry->consume_q->q_header = (struct vmci_queue_header *)tmp;
1482 } else if (page_store) {
1484 * The VMX already initialized the queue pair headers, so no
1485 * need for the kernel side to do that.
1487 result = qp_host_register_user_memory(page_store,
1490 if (result < VMCI_SUCCESS)
1493 entry->state = VMCIQPB_CREATED_MEM;
1496 * A create without a page_store may be either a host
1497 * side create (in which case we are waiting for the
1498 * guest side to supply the memory) or an old style
1499 * queue pair create (in which case we will expect a
1500 * set page store call as the next step).
1502 entry->state = VMCIQPB_CREATED_NO_MEM;
1505 qp_list_add_entry(&qp_broker_list, &entry->qp);
1509 /* Add to resource obj */
1510 result = vmci_resource_add(&entry->resource,
1511 VMCI_RESOURCE_TYPE_QPAIR_HOST,
1513 if (result != VMCI_SUCCESS) {
1514 pr_warn("Failed to add new resource (handle=0x%x:0x%x), error: %d",
1515 handle.context, handle.resource, result);
1519 entry->qp.handle = vmci_resource_handle(&entry->resource);
1521 vmci_q_header_init(entry->produce_q->q_header,
1523 vmci_q_header_init(entry->consume_q->q_header,
1527 vmci_ctx_qp_create(context, entry->qp.handle);
1529 return VMCI_SUCCESS;
1532 if (entry != NULL) {
1533 qp_host_free_queue(entry->produce_q, guest_produce_size);
1534 qp_host_free_queue(entry->consume_q, guest_consume_size);
1542 * Enqueues an event datagram to notify the peer VM attached to
1543 * the given queue pair handle about attach/detach event by the
1544 * given VM. Returns Payload size of datagram enqueued on
1545 * success, error code otherwise.
1547 static int qp_notify_peer(bool attach,
1548 struct vmci_handle handle,
1553 struct vmci_event_qp ev;
1555 if (vmci_handle_is_invalid(handle) || my_id == VMCI_INVALID_ID ||
1556 peer_id == VMCI_INVALID_ID)
1557 return VMCI_ERROR_INVALID_ARGS;
1560 * In vmci_ctx_enqueue_datagram() we enforce the upper limit on
1561 * number of pending events from the hypervisor to a given VM
1562 * otherwise a rogue VM could do an arbitrary number of attach
1563 * and detach operations causing memory pressure in the host
1567 memset(&ev, 0, sizeof(ev));
1568 ev.msg.hdr.dst = vmci_make_handle(peer_id, VMCI_EVENT_HANDLER);
1569 ev.msg.hdr.src = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
1570 VMCI_CONTEXT_RESOURCE_ID);
1571 ev.msg.hdr.payload_size = sizeof(ev) - sizeof(ev.msg.hdr);
1572 ev.msg.event_data.event = attach ?
1573 VMCI_EVENT_QP_PEER_ATTACH : VMCI_EVENT_QP_PEER_DETACH;
1574 ev.payload.handle = handle;
1575 ev.payload.peer_id = my_id;
1577 rv = vmci_datagram_dispatch(VMCI_HYPERVISOR_CONTEXT_ID,
1578 &ev.msg.hdr, false);
1579 if (rv < VMCI_SUCCESS)
1580 pr_warn("Failed to enqueue queue_pair %s event datagram for context (ID=0x%x)\n",
1581 attach ? "ATTACH" : "DETACH", peer_id);
1587 * The second endpoint issuing a queue pair allocation will attach to
1588 * the queue pair registered with the queue pair broker.
1590 * If the attacher is a guest, it will associate a VMX virtual address
1591 * range with the queue pair as specified by the page_store. At this
1592 * point, the already attach host endpoint may start using the queue
1593 * pair, and an attach event is sent to it. For compatibility with
1594 * older VMX'en, that used a separate step to set the VMX virtual
1595 * address range, the virtual address range can be registered later
1596 * using vmci_qp_broker_set_page_store. In that case, a page_store of
1597 * NULL should be used, and the attach event will be generated once
1598 * the actual page store has been set.
1600 * If the attacher is the host, a page_store of NULL should be used as
1601 * well, since the page store information is already set by the guest.
1603 * For new VMX and host callers, the queue pair will be moved to the
1604 * VMCIQPB_ATTACHED_MEM state, and for older VMX callers, it will be
1605 * moved to the VMCOQPB_ATTACHED_NO_MEM state.
1607 static int qp_broker_attach(struct qp_broker_entry *entry,
1613 struct vmci_qp_page_store *page_store,
1614 struct vmci_ctx *context,
1615 vmci_event_release_cb wakeup_cb,
1617 struct qp_broker_entry **ent)
1619 const u32 context_id = vmci_ctx_get_id(context);
1620 bool is_local = flags & VMCI_QPFLAG_LOCAL;
1623 if (entry->state != VMCIQPB_CREATED_NO_MEM &&
1624 entry->state != VMCIQPB_CREATED_MEM)
1625 return VMCI_ERROR_UNAVAILABLE;
1628 if (!(entry->qp.flags & VMCI_QPFLAG_LOCAL) ||
1629 context_id != entry->create_id) {
1630 return VMCI_ERROR_INVALID_ARGS;
1632 } else if (context_id == entry->create_id ||
1633 context_id == entry->attach_id) {
1634 return VMCI_ERROR_ALREADY_EXISTS;
1637 if (VMCI_CONTEXT_IS_VM(context_id) &&
1638 VMCI_CONTEXT_IS_VM(entry->create_id))
1639 return VMCI_ERROR_DST_UNREACHABLE;
1642 * If we are attaching from a restricted context then the queuepair
1643 * must have been created by a trusted endpoint.
1645 if ((context->priv_flags & VMCI_PRIVILEGE_FLAG_RESTRICTED) &&
1646 !entry->created_by_trusted)
1647 return VMCI_ERROR_NO_ACCESS;
1650 * If we are attaching to a queuepair that was created by a restricted
1651 * context then we must be trusted.
1653 if (entry->require_trusted_attach &&
1654 (!(priv_flags & VMCI_PRIVILEGE_FLAG_TRUSTED)))
1655 return VMCI_ERROR_NO_ACCESS;
1658 * If the creator specifies VMCI_INVALID_ID in "peer" field, access
1659 * control check is not performed.
1661 if (entry->qp.peer != VMCI_INVALID_ID && entry->qp.peer != context_id)
1662 return VMCI_ERROR_NO_ACCESS;
1664 if (entry->create_id == VMCI_HOST_CONTEXT_ID) {
1666 * Do not attach if the caller doesn't support Host Queue Pairs
1667 * and a host created this queue pair.
1670 if (!vmci_ctx_supports_host_qp(context))
1671 return VMCI_ERROR_INVALID_RESOURCE;
1673 } else if (context_id == VMCI_HOST_CONTEXT_ID) {
1674 struct vmci_ctx *create_context;
1675 bool supports_host_qp;
1678 * Do not attach a host to a user created queue pair if that
1679 * user doesn't support host queue pair end points.
1682 create_context = vmci_ctx_get(entry->create_id);
1683 supports_host_qp = vmci_ctx_supports_host_qp(create_context);
1684 vmci_ctx_put(create_context);
1686 if (!supports_host_qp)
1687 return VMCI_ERROR_INVALID_RESOURCE;
1690 if ((entry->qp.flags & ~VMCI_QP_ASYMM) != (flags & ~VMCI_QP_ASYMM_PEER))
1691 return VMCI_ERROR_QUEUEPAIR_MISMATCH;
1693 if (context_id != VMCI_HOST_CONTEXT_ID) {
1695 * The queue pair broker entry stores values from the guest
1696 * point of view, so an attaching guest should match the values
1697 * stored in the entry.
1700 if (entry->qp.produce_size != produce_size ||
1701 entry->qp.consume_size != consume_size) {
1702 return VMCI_ERROR_QUEUEPAIR_MISMATCH;
1704 } else if (entry->qp.produce_size != consume_size ||
1705 entry->qp.consume_size != produce_size) {
1706 return VMCI_ERROR_QUEUEPAIR_MISMATCH;
1709 if (context_id != VMCI_HOST_CONTEXT_ID) {
1711 * If a guest attached to a queue pair, it will supply
1712 * the backing memory. If this is a pre NOVMVM vmx,
1713 * the backing memory will be supplied by calling
1714 * vmci_qp_broker_set_page_store() following the
1715 * return of the vmci_qp_broker_alloc() call. If it is
1716 * a vmx of version NOVMVM or later, the page store
1717 * must be supplied as part of the
1718 * vmci_qp_broker_alloc call. Under all circumstances
1719 * must the initially created queue pair not have any
1720 * memory associated with it already.
1723 if (entry->state != VMCIQPB_CREATED_NO_MEM)
1724 return VMCI_ERROR_INVALID_ARGS;
1726 if (page_store != NULL) {
1728 * Patch up host state to point to guest
1729 * supplied memory. The VMX already
1730 * initialized the queue pair headers, so no
1731 * need for the kernel side to do that.
1734 result = qp_host_register_user_memory(page_store,
1737 if (result < VMCI_SUCCESS)
1740 entry->state = VMCIQPB_ATTACHED_MEM;
1742 entry->state = VMCIQPB_ATTACHED_NO_MEM;
1744 } else if (entry->state == VMCIQPB_CREATED_NO_MEM) {
1746 * The host side is attempting to attach to a queue
1747 * pair that doesn't have any memory associated with
1748 * it. This must be a pre NOVMVM vmx that hasn't set
1749 * the page store information yet, or a quiesced VM.
1752 return VMCI_ERROR_UNAVAILABLE;
1754 /* The host side has successfully attached to a queue pair. */
1755 entry->state = VMCIQPB_ATTACHED_MEM;
1758 if (entry->state == VMCIQPB_ATTACHED_MEM) {
1760 qp_notify_peer(true, entry->qp.handle, context_id,
1762 if (result < VMCI_SUCCESS)
1763 pr_warn("Failed to notify peer (ID=0x%x) of attach to queue pair (handle=0x%x:0x%x)\n",
1764 entry->create_id, entry->qp.handle.context,
1765 entry->qp.handle.resource);
1768 entry->attach_id = context_id;
1769 entry->qp.ref_count++;
1771 entry->wakeup_cb = wakeup_cb;
1772 entry->client_data = client_data;
1776 * When attaching to local queue pairs, the context already has
1777 * an entry tracking the queue pair, so don't add another one.
1780 vmci_ctx_qp_create(context, entry->qp.handle);
1785 return VMCI_SUCCESS;
1789 * queue_pair_Alloc for use when setting up queue pair endpoints
1792 static int qp_broker_alloc(struct vmci_handle handle,
1798 struct vmci_qp_page_store *page_store,
1799 struct vmci_ctx *context,
1800 vmci_event_release_cb wakeup_cb,
1802 struct qp_broker_entry **ent,
1805 const u32 context_id = vmci_ctx_get_id(context);
1807 struct qp_broker_entry *entry = NULL;
1808 bool is_local = flags & VMCI_QPFLAG_LOCAL;
1811 if (vmci_handle_is_invalid(handle) ||
1812 (flags & ~VMCI_QP_ALL_FLAGS) || is_local ||
1813 !(produce_size || consume_size) ||
1814 !context || context_id == VMCI_INVALID_ID ||
1815 handle.context == VMCI_INVALID_ID) {
1816 return VMCI_ERROR_INVALID_ARGS;
1819 if (page_store && !VMCI_QP_PAGESTORE_IS_WELLFORMED(page_store))
1820 return VMCI_ERROR_INVALID_ARGS;
1823 * In the initial argument check, we ensure that non-vmkernel hosts
1824 * are not allowed to create local queue pairs.
1827 mutex_lock(&qp_broker_list.mutex);
1829 if (!is_local && vmci_ctx_qp_exists(context, handle)) {
1830 pr_devel("Context (ID=0x%x) already attached to queue pair (handle=0x%x:0x%x)\n",
1831 context_id, handle.context, handle.resource);
1832 mutex_unlock(&qp_broker_list.mutex);
1833 return VMCI_ERROR_ALREADY_EXISTS;
1836 if (handle.resource != VMCI_INVALID_ID)
1837 entry = qp_broker_handle_to_entry(handle);
1842 qp_broker_create(handle, peer, flags, priv_flags,
1843 produce_size, consume_size, page_store,
1844 context, wakeup_cb, client_data, ent);
1848 qp_broker_attach(entry, peer, flags, priv_flags,
1849 produce_size, consume_size, page_store,
1850 context, wakeup_cb, client_data, ent);
1853 mutex_unlock(&qp_broker_list.mutex);
1856 *swap = (context_id == VMCI_HOST_CONTEXT_ID) &&
1857 !(create && is_local);
1863 * This function implements the kernel API for allocating a queue
1866 static int qp_alloc_host_work(struct vmci_handle *handle,
1867 struct vmci_queue **produce_q,
1869 struct vmci_queue **consume_q,
1874 vmci_event_release_cb wakeup_cb,
1877 struct vmci_handle new_handle;
1878 struct vmci_ctx *context;
1879 struct qp_broker_entry *entry;
1883 if (vmci_handle_is_invalid(*handle)) {
1884 new_handle = vmci_make_handle(
1885 VMCI_HOST_CONTEXT_ID, VMCI_INVALID_ID);
1887 new_handle = *handle;
1889 context = vmci_ctx_get(VMCI_HOST_CONTEXT_ID);
1892 qp_broker_alloc(new_handle, peer, flags, priv_flags,
1893 produce_size, consume_size, NULL, context,
1894 wakeup_cb, client_data, &entry, &swap);
1895 if (result == VMCI_SUCCESS) {
1898 * If this is a local queue pair, the attacher
1899 * will swap around produce and consume
1903 *produce_q = entry->consume_q;
1904 *consume_q = entry->produce_q;
1906 *produce_q = entry->produce_q;
1907 *consume_q = entry->consume_q;
1910 *handle = vmci_resource_handle(&entry->resource);
1912 *handle = VMCI_INVALID_HANDLE;
1913 pr_devel("queue pair broker failed to alloc (result=%d)\n",
1916 vmci_ctx_put(context);
1921 * Allocates a VMCI queue_pair. Only checks validity of input
1922 * arguments. The real work is done in the host or guest
1923 * specific function.
1925 int vmci_qp_alloc(struct vmci_handle *handle,
1926 struct vmci_queue **produce_q,
1928 struct vmci_queue **consume_q,
1933 bool guest_endpoint,
1934 vmci_event_release_cb wakeup_cb,
1937 if (!handle || !produce_q || !consume_q ||
1938 (!produce_size && !consume_size) || (flags & ~VMCI_QP_ALL_FLAGS))
1939 return VMCI_ERROR_INVALID_ARGS;
1941 if (guest_endpoint) {
1942 return qp_alloc_guest_work(handle, produce_q,
1943 produce_size, consume_q,
1947 return qp_alloc_host_work(handle, produce_q,
1948 produce_size, consume_q,
1949 consume_size, peer, flags,
1950 priv_flags, wakeup_cb, client_data);
1955 * This function implements the host kernel API for detaching from
1958 static int qp_detatch_host_work(struct vmci_handle handle)
1961 struct vmci_ctx *context;
1963 context = vmci_ctx_get(VMCI_HOST_CONTEXT_ID);
1965 result = vmci_qp_broker_detach(handle, context);
1967 vmci_ctx_put(context);
1972 * Detaches from a VMCI queue_pair. Only checks validity of input argument.
1973 * Real work is done in the host or guest specific function.
1975 static int qp_detatch(struct vmci_handle handle, bool guest_endpoint)
1977 if (vmci_handle_is_invalid(handle))
1978 return VMCI_ERROR_INVALID_ARGS;
1981 return qp_detatch_guest_work(handle);
1983 return qp_detatch_host_work(handle);
1987 * Returns the entry from the head of the list. Assumes that the list is
1990 static struct qp_entry *qp_list_get_head(struct qp_list *qp_list)
1992 if (!list_empty(&qp_list->head)) {
1993 struct qp_entry *entry =
1994 list_first_entry(&qp_list->head, struct qp_entry,
2002 void vmci_qp_broker_exit(void)
2004 struct qp_entry *entry;
2005 struct qp_broker_entry *be;
2007 mutex_lock(&qp_broker_list.mutex);
2009 while ((entry = qp_list_get_head(&qp_broker_list))) {
2010 be = (struct qp_broker_entry *)entry;
2012 qp_list_remove_entry(&qp_broker_list, entry);
2016 mutex_unlock(&qp_broker_list.mutex);
2020 * Requests that a queue pair be allocated with the VMCI queue
2021 * pair broker. Allocates a queue pair entry if one does not
2022 * exist. Attaches to one if it exists, and retrieves the page
2023 * files backing that queue_pair. Assumes that the queue pair
2024 * broker lock is held.
2026 int vmci_qp_broker_alloc(struct vmci_handle handle,
2032 struct vmci_qp_page_store *page_store,
2033 struct vmci_ctx *context)
2035 return qp_broker_alloc(handle, peer, flags, priv_flags,
2036 produce_size, consume_size,
2037 page_store, context, NULL, NULL, NULL, NULL);
2041 * VMX'en with versions lower than VMCI_VERSION_NOVMVM use a separate
2042 * step to add the UVAs of the VMX mapping of the queue pair. This function
2043 * provides backwards compatibility with such VMX'en, and takes care of
2044 * registering the page store for a queue pair previously allocated by the
2045 * VMX during create or attach. This function will move the queue pair state
2046 * to either from VMCIQBP_CREATED_NO_MEM to VMCIQBP_CREATED_MEM or
2047 * VMCIQBP_ATTACHED_NO_MEM to VMCIQBP_ATTACHED_MEM. If moving to the
2048 * attached state with memory, the queue pair is ready to be used by the
2049 * host peer, and an attached event will be generated.
2051 * Assumes that the queue pair broker lock is held.
2053 * This function is only used by the hosted platform, since there is no
2054 * issue with backwards compatibility for vmkernel.
2056 int vmci_qp_broker_set_page_store(struct vmci_handle handle,
2059 struct vmci_ctx *context)
2061 struct qp_broker_entry *entry;
2063 const u32 context_id = vmci_ctx_get_id(context);
2065 if (vmci_handle_is_invalid(handle) || !context ||
2066 context_id == VMCI_INVALID_ID)
2067 return VMCI_ERROR_INVALID_ARGS;
2070 * We only support guest to host queue pairs, so the VMX must
2071 * supply UVAs for the mapped page files.
2074 if (produce_uva == 0 || consume_uva == 0)
2075 return VMCI_ERROR_INVALID_ARGS;
2077 mutex_lock(&qp_broker_list.mutex);
2079 if (!vmci_ctx_qp_exists(context, handle)) {
2080 pr_warn("Context (ID=0x%x) not attached to queue pair (handle=0x%x:0x%x)\n",
2081 context_id, handle.context, handle.resource);
2082 result = VMCI_ERROR_NOT_FOUND;
2086 entry = qp_broker_handle_to_entry(handle);
2088 result = VMCI_ERROR_NOT_FOUND;
2093 * If I'm the owner then I can set the page store.
2095 * Or, if a host created the queue_pair and I'm the attached peer
2096 * then I can set the page store.
2098 if (entry->create_id != context_id &&
2099 (entry->create_id != VMCI_HOST_CONTEXT_ID ||
2100 entry->attach_id != context_id)) {
2101 result = VMCI_ERROR_QUEUEPAIR_NOTOWNER;
2105 if (entry->state != VMCIQPB_CREATED_NO_MEM &&
2106 entry->state != VMCIQPB_ATTACHED_NO_MEM) {
2107 result = VMCI_ERROR_UNAVAILABLE;
2111 result = qp_host_get_user_memory(produce_uva, consume_uva,
2112 entry->produce_q, entry->consume_q);
2113 if (result < VMCI_SUCCESS)
2116 result = qp_host_map_queues(entry->produce_q, entry->consume_q);
2117 if (result < VMCI_SUCCESS) {
2118 qp_host_unregister_user_memory(entry->produce_q,
2123 if (entry->state == VMCIQPB_CREATED_NO_MEM)
2124 entry->state = VMCIQPB_CREATED_MEM;
2126 entry->state = VMCIQPB_ATTACHED_MEM;
2128 entry->vmci_page_files = true;
2130 if (entry->state == VMCIQPB_ATTACHED_MEM) {
2132 qp_notify_peer(true, handle, context_id, entry->create_id);
2133 if (result < VMCI_SUCCESS) {
2134 pr_warn("Failed to notify peer (ID=0x%x) of attach to queue pair (handle=0x%x:0x%x)\n",
2135 entry->create_id, entry->qp.handle.context,
2136 entry->qp.handle.resource);
2140 result = VMCI_SUCCESS;
2142 mutex_unlock(&qp_broker_list.mutex);
2147 * Resets saved queue headers for the given QP broker
2148 * entry. Should be used when guest memory becomes available
2149 * again, or the guest detaches.
2151 static void qp_reset_saved_headers(struct qp_broker_entry *entry)
2153 entry->produce_q->saved_header = NULL;
2154 entry->consume_q->saved_header = NULL;
2158 * The main entry point for detaching from a queue pair registered with the
2159 * queue pair broker. If more than one endpoint is attached to the queue
2160 * pair, the first endpoint will mainly decrement a reference count and
2161 * generate a notification to its peer. The last endpoint will clean up
2162 * the queue pair state registered with the broker.
2164 * When a guest endpoint detaches, it will unmap and unregister the guest
2165 * memory backing the queue pair. If the host is still attached, it will
2166 * no longer be able to access the queue pair content.
2168 * If the queue pair is already in a state where there is no memory
2169 * registered for the queue pair (any *_NO_MEM state), it will transition to
2170 * the VMCIQPB_SHUTDOWN_NO_MEM state. This will also happen, if a guest
2171 * endpoint is the first of two endpoints to detach. If the host endpoint is
2172 * the first out of two to detach, the queue pair will move to the
2173 * VMCIQPB_SHUTDOWN_MEM state.
2175 int vmci_qp_broker_detach(struct vmci_handle handle, struct vmci_ctx *context)
2177 struct qp_broker_entry *entry;
2178 const u32 context_id = vmci_ctx_get_id(context);
2180 bool is_local = false;
2183 if (vmci_handle_is_invalid(handle) || !context ||
2184 context_id == VMCI_INVALID_ID) {
2185 return VMCI_ERROR_INVALID_ARGS;
2188 mutex_lock(&qp_broker_list.mutex);
2190 if (!vmci_ctx_qp_exists(context, handle)) {
2191 pr_devel("Context (ID=0x%x) not attached to queue pair (handle=0x%x:0x%x)\n",
2192 context_id, handle.context, handle.resource);
2193 result = VMCI_ERROR_NOT_FOUND;
2197 entry = qp_broker_handle_to_entry(handle);
2199 pr_devel("Context (ID=0x%x) reports being attached to queue pair(handle=0x%x:0x%x) that isn't present in broker\n",
2200 context_id, handle.context, handle.resource);
2201 result = VMCI_ERROR_NOT_FOUND;
2205 if (context_id != entry->create_id && context_id != entry->attach_id) {
2206 result = VMCI_ERROR_QUEUEPAIR_NOTATTACHED;
2210 if (context_id == entry->create_id) {
2211 peer_id = entry->attach_id;
2212 entry->create_id = VMCI_INVALID_ID;
2214 peer_id = entry->create_id;
2215 entry->attach_id = VMCI_INVALID_ID;
2217 entry->qp.ref_count--;
2219 is_local = entry->qp.flags & VMCI_QPFLAG_LOCAL;
2221 if (context_id != VMCI_HOST_CONTEXT_ID) {
2222 bool headers_mapped;
2225 * Pre NOVMVM vmx'en may detach from a queue pair
2226 * before setting the page store, and in that case
2227 * there is no user memory to detach from. Also, more
2228 * recent VMX'en may detach from a queue pair in the
2232 qp_acquire_queue_mutex(entry->produce_q);
2233 headers_mapped = entry->produce_q->q_header ||
2234 entry->consume_q->q_header;
2235 if (QPBROKERSTATE_HAS_MEM(entry)) {
2237 qp_host_unmap_queues(INVALID_VMCI_GUEST_MEM_ID,
2240 if (result < VMCI_SUCCESS)
2241 pr_warn("Failed to unmap queue headers for queue pair (handle=0x%x:0x%x,result=%d)\n",
2242 handle.context, handle.resource,
2245 if (entry->vmci_page_files)
2246 qp_host_unregister_user_memory(entry->produce_q,
2250 qp_host_unregister_user_memory(entry->produce_q,
2256 if (!headers_mapped)
2257 qp_reset_saved_headers(entry);
2259 qp_release_queue_mutex(entry->produce_q);
2261 if (!headers_mapped && entry->wakeup_cb)
2262 entry->wakeup_cb(entry->client_data);
2265 if (entry->wakeup_cb) {
2266 entry->wakeup_cb = NULL;
2267 entry->client_data = NULL;
2271 if (entry->qp.ref_count == 0) {
2272 qp_list_remove_entry(&qp_broker_list, &entry->qp);
2275 kfree(entry->local_mem);
2277 qp_cleanup_queue_mutex(entry->produce_q, entry->consume_q);
2278 qp_host_free_queue(entry->produce_q, entry->qp.produce_size);
2279 qp_host_free_queue(entry->consume_q, entry->qp.consume_size);
2280 /* Unlink from resource hash table and free callback */
2281 vmci_resource_remove(&entry->resource);
2285 vmci_ctx_qp_destroy(context, handle);
2287 qp_notify_peer(false, handle, context_id, peer_id);
2288 if (context_id == VMCI_HOST_CONTEXT_ID &&
2289 QPBROKERSTATE_HAS_MEM(entry)) {
2290 entry->state = VMCIQPB_SHUTDOWN_MEM;
2292 entry->state = VMCIQPB_SHUTDOWN_NO_MEM;
2296 vmci_ctx_qp_destroy(context, handle);
2299 result = VMCI_SUCCESS;
2301 mutex_unlock(&qp_broker_list.mutex);
2306 * Establishes the necessary mappings for a queue pair given a
2307 * reference to the queue pair guest memory. This is usually
2308 * called when a guest is unquiesced and the VMX is allowed to
2309 * map guest memory once again.
2311 int vmci_qp_broker_map(struct vmci_handle handle,
2312 struct vmci_ctx *context,
2315 struct qp_broker_entry *entry;
2316 const u32 context_id = vmci_ctx_get_id(context);
2317 bool is_local = false;
2320 if (vmci_handle_is_invalid(handle) || !context ||
2321 context_id == VMCI_INVALID_ID)
2322 return VMCI_ERROR_INVALID_ARGS;
2324 mutex_lock(&qp_broker_list.mutex);
2326 if (!vmci_ctx_qp_exists(context, handle)) {
2327 pr_devel("Context (ID=0x%x) not attached to queue pair (handle=0x%x:0x%x)\n",
2328 context_id, handle.context, handle.resource);
2329 result = VMCI_ERROR_NOT_FOUND;
2333 entry = qp_broker_handle_to_entry(handle);
2335 pr_devel("Context (ID=0x%x) reports being attached to queue pair (handle=0x%x:0x%x) that isn't present in broker\n",
2336 context_id, handle.context, handle.resource);
2337 result = VMCI_ERROR_NOT_FOUND;
2341 if (context_id != entry->create_id && context_id != entry->attach_id) {
2342 result = VMCI_ERROR_QUEUEPAIR_NOTATTACHED;
2346 is_local = entry->qp.flags & VMCI_QPFLAG_LOCAL;
2347 result = VMCI_SUCCESS;
2349 if (context_id != VMCI_HOST_CONTEXT_ID &&
2350 !QPBROKERSTATE_HAS_MEM(entry)) {
2351 struct vmci_qp_page_store page_store;
2353 page_store.pages = guest_mem;
2354 page_store.len = QPE_NUM_PAGES(entry->qp);
2356 qp_acquire_queue_mutex(entry->produce_q);
2357 qp_reset_saved_headers(entry);
2359 qp_host_register_user_memory(&page_store,
2362 qp_release_queue_mutex(entry->produce_q);
2363 if (result == VMCI_SUCCESS) {
2364 /* Move state from *_NO_MEM to *_MEM */
2368 if (entry->wakeup_cb)
2369 entry->wakeup_cb(entry->client_data);
2374 mutex_unlock(&qp_broker_list.mutex);
2379 * Saves a snapshot of the queue headers for the given QP broker
2380 * entry. Should be used when guest memory is unmapped.
2382 * VMCI_SUCCESS on success, appropriate error code if guest memory
2383 * can't be accessed..
2385 static int qp_save_headers(struct qp_broker_entry *entry)
2389 if (entry->produce_q->saved_header != NULL &&
2390 entry->consume_q->saved_header != NULL) {
2392 * If the headers have already been saved, we don't need to do
2393 * it again, and we don't want to map in the headers
2397 return VMCI_SUCCESS;
2400 if (NULL == entry->produce_q->q_header ||
2401 NULL == entry->consume_q->q_header) {
2402 result = qp_host_map_queues(entry->produce_q, entry->consume_q);
2403 if (result < VMCI_SUCCESS)
2407 memcpy(&entry->saved_produce_q, entry->produce_q->q_header,
2408 sizeof(entry->saved_produce_q));
2409 entry->produce_q->saved_header = &entry->saved_produce_q;
2410 memcpy(&entry->saved_consume_q, entry->consume_q->q_header,
2411 sizeof(entry->saved_consume_q));
2412 entry->consume_q->saved_header = &entry->saved_consume_q;
2414 return VMCI_SUCCESS;
2418 * Removes all references to the guest memory of a given queue pair, and
2419 * will move the queue pair from state *_MEM to *_NO_MEM. It is usually
2420 * called when a VM is being quiesced where access to guest memory should
2423 int vmci_qp_broker_unmap(struct vmci_handle handle,
2424 struct vmci_ctx *context,
2427 struct qp_broker_entry *entry;
2428 const u32 context_id = vmci_ctx_get_id(context);
2429 bool is_local = false;
2432 if (vmci_handle_is_invalid(handle) || !context ||
2433 context_id == VMCI_INVALID_ID)
2434 return VMCI_ERROR_INVALID_ARGS;
2436 mutex_lock(&qp_broker_list.mutex);
2438 if (!vmci_ctx_qp_exists(context, handle)) {
2439 pr_devel("Context (ID=0x%x) not attached to queue pair (handle=0x%x:0x%x)\n",
2440 context_id, handle.context, handle.resource);
2441 result = VMCI_ERROR_NOT_FOUND;
2445 entry = qp_broker_handle_to_entry(handle);
2447 pr_devel("Context (ID=0x%x) reports being attached to queue pair (handle=0x%x:0x%x) that isn't present in broker\n",
2448 context_id, handle.context, handle.resource);
2449 result = VMCI_ERROR_NOT_FOUND;
2453 if (context_id != entry->create_id && context_id != entry->attach_id) {
2454 result = VMCI_ERROR_QUEUEPAIR_NOTATTACHED;
2458 is_local = entry->qp.flags & VMCI_QPFLAG_LOCAL;
2460 if (context_id != VMCI_HOST_CONTEXT_ID &&
2461 QPBROKERSTATE_HAS_MEM(entry)) {
2462 qp_acquire_queue_mutex(entry->produce_q);
2463 result = qp_save_headers(entry);
2464 if (result < VMCI_SUCCESS)
2465 pr_warn("Failed to save queue headers for queue pair (handle=0x%x:0x%x,result=%d)\n",
2466 handle.context, handle.resource, result);
2468 qp_host_unmap_queues(gid, entry->produce_q, entry->consume_q);
2471 * On hosted, when we unmap queue pairs, the VMX will also
2472 * unmap the guest memory, so we invalidate the previously
2473 * registered memory. If the queue pair is mapped again at a
2474 * later point in time, we will need to reregister the user
2475 * memory with a possibly new user VA.
2477 qp_host_unregister_user_memory(entry->produce_q,
2481 * Move state from *_MEM to *_NO_MEM.
2485 qp_release_queue_mutex(entry->produce_q);
2488 result = VMCI_SUCCESS;
2491 mutex_unlock(&qp_broker_list.mutex);
2496 * Destroys all guest queue pair endpoints. If active guest queue
2497 * pairs still exist, hypercalls to attempt detach from these
2498 * queue pairs will be made. Any failure to detach is silently
2501 void vmci_qp_guest_endpoints_exit(void)
2503 struct qp_entry *entry;
2504 struct qp_guest_endpoint *ep;
2506 mutex_lock(&qp_guest_endpoints.mutex);
2508 while ((entry = qp_list_get_head(&qp_guest_endpoints))) {
2509 ep = (struct qp_guest_endpoint *)entry;
2511 /* Don't make a hypercall for local queue_pairs. */
2512 if (!(entry->flags & VMCI_QPFLAG_LOCAL))
2513 qp_detatch_hypercall(entry->handle);
2515 /* We cannot fail the exit, so let's reset ref_count. */
2516 entry->ref_count = 0;
2517 qp_list_remove_entry(&qp_guest_endpoints, entry);
2519 qp_guest_endpoint_destroy(ep);
2522 mutex_unlock(&qp_guest_endpoints.mutex);
2526 * Helper routine that will lock the queue pair before subsequent
2528 * Note: Non-blocking on the host side is currently only implemented in ESX.
2529 * Since non-blocking isn't yet implemented on the host personality we
2530 * have no reason to acquire a spin lock. So to avoid the use of an
2531 * unnecessary lock only acquire the mutex if we can block.
2533 static void qp_lock(const struct vmci_qp *qpair)
2535 qp_acquire_queue_mutex(qpair->produce_q);
2539 * Helper routine that unlocks the queue pair after calling
2542 static void qp_unlock(const struct vmci_qp *qpair)
2544 qp_release_queue_mutex(qpair->produce_q);
2548 * The queue headers may not be mapped at all times. If a queue is
2549 * currently not mapped, it will be attempted to do so.
2551 static int qp_map_queue_headers(struct vmci_queue *produce_q,
2552 struct vmci_queue *consume_q)
2556 if (NULL == produce_q->q_header || NULL == consume_q->q_header) {
2557 result = qp_host_map_queues(produce_q, consume_q);
2558 if (result < VMCI_SUCCESS)
2559 return (produce_q->saved_header &&
2560 consume_q->saved_header) ?
2561 VMCI_ERROR_QUEUEPAIR_NOT_READY :
2562 VMCI_ERROR_QUEUEPAIR_NOTATTACHED;
2565 return VMCI_SUCCESS;
2569 * Helper routine that will retrieve the produce and consume
2570 * headers of a given queue pair. If the guest memory of the
2571 * queue pair is currently not available, the saved queue headers
2572 * will be returned, if these are available.
2574 static int qp_get_queue_headers(const struct vmci_qp *qpair,
2575 struct vmci_queue_header **produce_q_header,
2576 struct vmci_queue_header **consume_q_header)
2580 result = qp_map_queue_headers(qpair->produce_q, qpair->consume_q);
2581 if (result == VMCI_SUCCESS) {
2582 *produce_q_header = qpair->produce_q->q_header;
2583 *consume_q_header = qpair->consume_q->q_header;
2584 } else if (qpair->produce_q->saved_header &&
2585 qpair->consume_q->saved_header) {
2586 *produce_q_header = qpair->produce_q->saved_header;
2587 *consume_q_header = qpair->consume_q->saved_header;
2588 result = VMCI_SUCCESS;
2595 * Callback from VMCI queue pair broker indicating that a queue
2596 * pair that was previously not ready, now either is ready or
2599 static int qp_wakeup_cb(void *client_data)
2601 struct vmci_qp *qpair = (struct vmci_qp *)client_data;
2604 while (qpair->blocked > 0) {
2606 qpair->generation++;
2607 wake_up(&qpair->event);
2611 return VMCI_SUCCESS;
2615 * Makes the calling thread wait for the queue pair to become
2616 * ready for host side access. Returns true when thread is
2617 * woken up after queue pair state change, false otherwise.
2619 static bool qp_wait_for_ready_queue(struct vmci_qp *qpair)
2621 unsigned int generation;
2624 generation = qpair->generation;
2626 wait_event(qpair->event, generation != qpair->generation);
2633 * Enqueues a given buffer to the produce queue using the provided
2634 * function. As many bytes as possible (space available in the queue)
2635 * are enqueued. Assumes the queue->mutex has been acquired. Returns
2636 * VMCI_ERROR_QUEUEPAIR_NOSPACE if no space was available to enqueue
2637 * data, VMCI_ERROR_INVALID_SIZE, if any queue pointer is outside the
2638 * queue (as defined by the queue size), VMCI_ERROR_INVALID_ARGS, if
2639 * an error occured when accessing the buffer,
2640 * VMCI_ERROR_QUEUEPAIR_NOTATTACHED, if the queue pair pages aren't
2641 * available. Otherwise, the number of bytes written to the queue is
2642 * returned. Updates the tail pointer of the produce queue.
2644 static ssize_t qp_enqueue_locked(struct vmci_queue *produce_q,
2645 struct vmci_queue *consume_q,
2646 const u64 produce_q_size,
2649 vmci_memcpy_to_queue_func memcpy_to_queue)
2656 result = qp_map_queue_headers(produce_q, consume_q);
2657 if (unlikely(result != VMCI_SUCCESS))
2660 free_space = vmci_q_header_free_space(produce_q->q_header,
2661 consume_q->q_header,
2663 if (free_space == 0)
2664 return VMCI_ERROR_QUEUEPAIR_NOSPACE;
2666 if (free_space < VMCI_SUCCESS)
2667 return (ssize_t) free_space;
2669 written = (size_t) (free_space > buf_size ? buf_size : free_space);
2670 tail = vmci_q_header_producer_tail(produce_q->q_header);
2671 if (likely(tail + written < produce_q_size)) {
2672 result = memcpy_to_queue(produce_q, tail, buf, 0, written);
2674 /* Tail pointer wraps around. */
2676 const size_t tmp = (size_t) (produce_q_size - tail);
2678 result = memcpy_to_queue(produce_q, tail, buf, 0, tmp);
2679 if (result >= VMCI_SUCCESS)
2680 result = memcpy_to_queue(produce_q, 0, buf, tmp,
2684 if (result < VMCI_SUCCESS)
2687 vmci_q_header_add_producer_tail(produce_q->q_header, written,
2693 * Dequeues data (if available) from the given consume queue. Writes data
2694 * to the user provided buffer using the provided function.
2695 * Assumes the queue->mutex has been acquired.
2697 * VMCI_ERROR_QUEUEPAIR_NODATA if no data was available to dequeue.
2698 * VMCI_ERROR_INVALID_SIZE, if any queue pointer is outside the queue
2699 * (as defined by the queue size).
2700 * VMCI_ERROR_INVALID_ARGS, if an error occured when accessing the buffer.
2701 * Otherwise the number of bytes dequeued is returned.
2703 * Updates the head pointer of the consume queue.
2705 static ssize_t qp_dequeue_locked(struct vmci_queue *produce_q,
2706 struct vmci_queue *consume_q,
2707 const u64 consume_q_size,
2710 vmci_memcpy_from_queue_func memcpy_from_queue,
2711 bool update_consumer)
2718 result = qp_map_queue_headers(produce_q, consume_q);
2719 if (unlikely(result != VMCI_SUCCESS))
2722 buf_ready = vmci_q_header_buf_ready(consume_q->q_header,
2723 produce_q->q_header,
2726 return VMCI_ERROR_QUEUEPAIR_NODATA;
2728 if (buf_ready < VMCI_SUCCESS)
2729 return (ssize_t) buf_ready;
2731 read = (size_t) (buf_ready > buf_size ? buf_size : buf_ready);
2732 head = vmci_q_header_consumer_head(produce_q->q_header);
2733 if (likely(head + read < consume_q_size)) {
2734 result = memcpy_from_queue(buf, 0, consume_q, head, read);
2736 /* Head pointer wraps around. */
2738 const size_t tmp = (size_t) (consume_q_size - head);
2740 result = memcpy_from_queue(buf, 0, consume_q, head, tmp);
2741 if (result >= VMCI_SUCCESS)
2742 result = memcpy_from_queue(buf, tmp, consume_q, 0,
2747 if (result < VMCI_SUCCESS)
2750 if (update_consumer)
2751 vmci_q_header_add_consumer_head(produce_q->q_header,
2752 read, consume_q_size);
2758 * vmci_qpair_alloc() - Allocates a queue pair.
2759 * @qpair: Pointer for the new vmci_qp struct.
2760 * @handle: Handle to track the resource.
2761 * @produce_qsize: Desired size of the producer queue.
2762 * @consume_qsize: Desired size of the consumer queue.
2763 * @peer: ContextID of the peer.
2764 * @flags: VMCI flags.
2765 * @priv_flags: VMCI priviledge flags.
2767 * This is the client interface for allocating the memory for a
2768 * vmci_qp structure and then attaching to the underlying
2769 * queue. If an error occurs allocating the memory for the
2770 * vmci_qp structure no attempt is made to attach. If an
2771 * error occurs attaching, then the structure is freed.
2773 int vmci_qpair_alloc(struct vmci_qp **qpair,
2774 struct vmci_handle *handle,
2781 struct vmci_qp *my_qpair;
2783 struct vmci_handle src = VMCI_INVALID_HANDLE;
2784 struct vmci_handle dst = vmci_make_handle(peer, VMCI_INVALID_ID);
2785 enum vmci_route route;
2786 vmci_event_release_cb wakeup_cb;
2790 * Restrict the size of a queuepair. The device already
2791 * enforces a limit on the total amount of memory that can be
2792 * allocated to queuepairs for a guest. However, we try to
2793 * allocate this memory before we make the queuepair
2794 * allocation hypercall. On Linux, we allocate each page
2795 * separately, which means rather than fail, the guest will
2796 * thrash while it tries to allocate, and will become
2797 * increasingly unresponsive to the point where it appears to
2798 * be hung. So we place a limit on the size of an individual
2799 * queuepair here, and leave the device to enforce the
2800 * restriction on total queuepair memory. (Note that this
2801 * doesn't prevent all cases; a user with only this much
2802 * physical memory could still get into trouble.) The error
2803 * used by the device is NO_RESOURCES, so use that here too.
2806 if (produce_qsize + consume_qsize < max(produce_qsize, consume_qsize) ||
2807 produce_qsize + consume_qsize > VMCI_MAX_GUEST_QP_MEMORY)
2808 return VMCI_ERROR_NO_RESOURCES;
2810 retval = vmci_route(&src, &dst, false, &route);
2811 if (retval < VMCI_SUCCESS)
2812 route = vmci_guest_code_active() ?
2813 VMCI_ROUTE_AS_GUEST : VMCI_ROUTE_AS_HOST;
2815 if (flags & (VMCI_QPFLAG_NONBLOCK | VMCI_QPFLAG_PINNED)) {
2816 pr_devel("NONBLOCK OR PINNED set");
2817 return VMCI_ERROR_INVALID_ARGS;
2820 my_qpair = kzalloc(sizeof(*my_qpair), GFP_KERNEL);
2822 return VMCI_ERROR_NO_MEM;
2824 my_qpair->produce_q_size = produce_qsize;
2825 my_qpair->consume_q_size = consume_qsize;
2826 my_qpair->peer = peer;
2827 my_qpair->flags = flags;
2828 my_qpair->priv_flags = priv_flags;
2833 if (VMCI_ROUTE_AS_HOST == route) {
2834 my_qpair->guest_endpoint = false;
2835 if (!(flags & VMCI_QPFLAG_LOCAL)) {
2836 my_qpair->blocked = 0;
2837 my_qpair->generation = 0;
2838 init_waitqueue_head(&my_qpair->event);
2839 wakeup_cb = qp_wakeup_cb;
2840 client_data = (void *)my_qpair;
2843 my_qpair->guest_endpoint = true;
2846 retval = vmci_qp_alloc(handle,
2847 &my_qpair->produce_q,
2848 my_qpair->produce_q_size,
2849 &my_qpair->consume_q,
2850 my_qpair->consume_q_size,
2853 my_qpair->priv_flags,
2854 my_qpair->guest_endpoint,
2855 wakeup_cb, client_data);
2857 if (retval < VMCI_SUCCESS) {
2863 my_qpair->handle = *handle;
2867 EXPORT_SYMBOL_GPL(vmci_qpair_alloc);
2870 * vmci_qpair_detach() - Detatches the client from a queue pair.
2871 * @qpair: Reference of a pointer to the qpair struct.
2873 * This is the client interface for detaching from a VMCIQPair.
2874 * Note that this routine will free the memory allocated for the
2875 * vmci_qp structure too.
2877 int vmci_qpair_detach(struct vmci_qp **qpair)
2880 struct vmci_qp *old_qpair;
2882 if (!qpair || !(*qpair))
2883 return VMCI_ERROR_INVALID_ARGS;
2886 result = qp_detatch(old_qpair->handle, old_qpair->guest_endpoint);
2889 * The guest can fail to detach for a number of reasons, and
2890 * if it does so, it will cleanup the entry (if there is one).
2891 * The host can fail too, but it won't cleanup the entry
2892 * immediately, it will do that later when the context is
2893 * freed. Either way, we need to release the qpair struct
2894 * here; there isn't much the caller can do, and we don't want
2898 memset(old_qpair, 0, sizeof(*old_qpair));
2899 old_qpair->handle = VMCI_INVALID_HANDLE;
2900 old_qpair->peer = VMCI_INVALID_ID;
2906 EXPORT_SYMBOL_GPL(vmci_qpair_detach);
2909 * vmci_qpair_get_produce_indexes() - Retrieves the indexes of the producer.
2910 * @qpair: Pointer to the queue pair struct.
2911 * @producer_tail: Reference used for storing producer tail index.
2912 * @consumer_head: Reference used for storing the consumer head index.
2914 * This is the client interface for getting the current indexes of the
2915 * QPair from the point of the view of the caller as the producer.
2917 int vmci_qpair_get_produce_indexes(const struct vmci_qp *qpair,
2921 struct vmci_queue_header *produce_q_header;
2922 struct vmci_queue_header *consume_q_header;
2926 return VMCI_ERROR_INVALID_ARGS;
2930 qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
2931 if (result == VMCI_SUCCESS)
2932 vmci_q_header_get_pointers(produce_q_header, consume_q_header,
2933 producer_tail, consumer_head);
2936 if (result == VMCI_SUCCESS &&
2937 ((producer_tail && *producer_tail >= qpair->produce_q_size) ||
2938 (consumer_head && *consumer_head >= qpair->produce_q_size)))
2939 return VMCI_ERROR_INVALID_SIZE;
2943 EXPORT_SYMBOL_GPL(vmci_qpair_get_produce_indexes);
2946 * vmci_qpair_get_consume_indexes() - Retrieves the indexes of the comsumer.
2947 * @qpair: Pointer to the queue pair struct.
2948 * @consumer_tail: Reference used for storing consumer tail index.
2949 * @producer_head: Reference used for storing the producer head index.
2951 * This is the client interface for getting the current indexes of the
2952 * QPair from the point of the view of the caller as the consumer.
2954 int vmci_qpair_get_consume_indexes(const struct vmci_qp *qpair,
2958 struct vmci_queue_header *produce_q_header;
2959 struct vmci_queue_header *consume_q_header;
2963 return VMCI_ERROR_INVALID_ARGS;
2967 qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
2968 if (result == VMCI_SUCCESS)
2969 vmci_q_header_get_pointers(consume_q_header, produce_q_header,
2970 consumer_tail, producer_head);
2973 if (result == VMCI_SUCCESS &&
2974 ((consumer_tail && *consumer_tail >= qpair->consume_q_size) ||
2975 (producer_head && *producer_head >= qpair->consume_q_size)))
2976 return VMCI_ERROR_INVALID_SIZE;
2980 EXPORT_SYMBOL_GPL(vmci_qpair_get_consume_indexes);
2983 * vmci_qpair_produce_free_space() - Retrieves free space in producer queue.
2984 * @qpair: Pointer to the queue pair struct.
2986 * This is the client interface for getting the amount of free
2987 * space in the QPair from the point of the view of the caller as
2988 * the producer which is the common case. Returns < 0 if err, else
2989 * available bytes into which data can be enqueued if > 0.
2991 s64 vmci_qpair_produce_free_space(const struct vmci_qp *qpair)
2993 struct vmci_queue_header *produce_q_header;
2994 struct vmci_queue_header *consume_q_header;
2998 return VMCI_ERROR_INVALID_ARGS;
3002 qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
3003 if (result == VMCI_SUCCESS)
3004 result = vmci_q_header_free_space(produce_q_header,
3006 qpair->produce_q_size);
3014 EXPORT_SYMBOL_GPL(vmci_qpair_produce_free_space);
3017 * vmci_qpair_consume_free_space() - Retrieves free space in consumer queue.
3018 * @qpair: Pointer to the queue pair struct.
3020 * This is the client interface for getting the amount of free
3021 * space in the QPair from the point of the view of the caller as
3022 * the consumer which is not the common case. Returns < 0 if err, else
3023 * available bytes into which data can be enqueued if > 0.
3025 s64 vmci_qpair_consume_free_space(const struct vmci_qp *qpair)
3027 struct vmci_queue_header *produce_q_header;
3028 struct vmci_queue_header *consume_q_header;
3032 return VMCI_ERROR_INVALID_ARGS;
3036 qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
3037 if (result == VMCI_SUCCESS)
3038 result = vmci_q_header_free_space(consume_q_header,
3040 qpair->consume_q_size);
3048 EXPORT_SYMBOL_GPL(vmci_qpair_consume_free_space);
3051 * vmci_qpair_produce_buf_ready() - Gets bytes ready to read from
3053 * @qpair: Pointer to the queue pair struct.
3055 * This is the client interface for getting the amount of
3056 * enqueued data in the QPair from the point of the view of the
3057 * caller as the producer which is not the common case. Returns < 0 if err,
3058 * else available bytes that may be read.
3060 s64 vmci_qpair_produce_buf_ready(const struct vmci_qp *qpair)
3062 struct vmci_queue_header *produce_q_header;
3063 struct vmci_queue_header *consume_q_header;
3067 return VMCI_ERROR_INVALID_ARGS;
3071 qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
3072 if (result == VMCI_SUCCESS)
3073 result = vmci_q_header_buf_ready(produce_q_header,
3075 qpair->produce_q_size);
3083 EXPORT_SYMBOL_GPL(vmci_qpair_produce_buf_ready);
3086 * vmci_qpair_consume_buf_ready() - Gets bytes ready to read from
3088 * @qpair: Pointer to the queue pair struct.
3090 * This is the client interface for getting the amount of
3091 * enqueued data in the QPair from the point of the view of the
3092 * caller as the consumer which is the normal case. Returns < 0 if err,
3093 * else available bytes that may be read.
3095 s64 vmci_qpair_consume_buf_ready(const struct vmci_qp *qpair)
3097 struct vmci_queue_header *produce_q_header;
3098 struct vmci_queue_header *consume_q_header;
3102 return VMCI_ERROR_INVALID_ARGS;
3106 qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
3107 if (result == VMCI_SUCCESS)
3108 result = vmci_q_header_buf_ready(consume_q_header,
3110 qpair->consume_q_size);
3118 EXPORT_SYMBOL_GPL(vmci_qpair_consume_buf_ready);
3121 * vmci_qpair_enqueue() - Throw data on the queue.
3122 * @qpair: Pointer to the queue pair struct.
3123 * @buf: Pointer to buffer containing data
3124 * @buf_size: Length of buffer.
3125 * @buf_type: Buffer type (Unused).
3127 * This is the client interface for enqueueing data into the queue.
3128 * Returns number of bytes enqueued or < 0 on error.
3130 ssize_t vmci_qpair_enqueue(struct vmci_qp *qpair,
3138 return VMCI_ERROR_INVALID_ARGS;
3143 result = qp_enqueue_locked(qpair->produce_q,
3145 qpair->produce_q_size,
3147 qp_memcpy_to_queue);
3149 if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
3150 !qp_wait_for_ready_queue(qpair))
3151 result = VMCI_ERROR_WOULD_BLOCK;
3153 } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
3159 EXPORT_SYMBOL_GPL(vmci_qpair_enqueue);
3162 * vmci_qpair_dequeue() - Get data from the queue.
3163 * @qpair: Pointer to the queue pair struct.
3164 * @buf: Pointer to buffer for the data
3165 * @buf_size: Length of buffer.
3166 * @buf_type: Buffer type (Unused).
3168 * This is the client interface for dequeueing data from the queue.
3169 * Returns number of bytes dequeued or < 0 on error.
3171 ssize_t vmci_qpair_dequeue(struct vmci_qp *qpair,
3179 return VMCI_ERROR_INVALID_ARGS;
3184 result = qp_dequeue_locked(qpair->produce_q,
3186 qpair->consume_q_size,
3188 qp_memcpy_from_queue, true);
3190 if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
3191 !qp_wait_for_ready_queue(qpair))
3192 result = VMCI_ERROR_WOULD_BLOCK;
3194 } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
3200 EXPORT_SYMBOL_GPL(vmci_qpair_dequeue);
3203 * vmci_qpair_peek() - Peek at the data in the queue.
3204 * @qpair: Pointer to the queue pair struct.
3205 * @buf: Pointer to buffer for the data
3206 * @buf_size: Length of buffer.
3207 * @buf_type: Buffer type (Unused on Linux).
3209 * This is the client interface for peeking into a queue. (I.e.,
3210 * copy data from the queue without updating the head pointer.)
3211 * Returns number of bytes dequeued or < 0 on error.
3213 ssize_t vmci_qpair_peek(struct vmci_qp *qpair,
3221 return VMCI_ERROR_INVALID_ARGS;
3226 result = qp_dequeue_locked(qpair->produce_q,
3228 qpair->consume_q_size,
3230 qp_memcpy_from_queue, false);
3232 if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
3233 !qp_wait_for_ready_queue(qpair))
3234 result = VMCI_ERROR_WOULD_BLOCK;
3236 } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
3242 EXPORT_SYMBOL_GPL(vmci_qpair_peek);
3245 * vmci_qpair_enquev() - Throw data on the queue using iov.
3246 * @qpair: Pointer to the queue pair struct.
3247 * @iov: Pointer to buffer containing data
3248 * @iov_size: Length of buffer.
3249 * @buf_type: Buffer type (Unused).
3251 * This is the client interface for enqueueing data into the queue.
3252 * This function uses IO vectors to handle the work. Returns number
3253 * of bytes enqueued or < 0 on error.
3255 ssize_t vmci_qpair_enquev(struct vmci_qp *qpair,
3263 return VMCI_ERROR_INVALID_ARGS;
3268 result = qp_enqueue_locked(qpair->produce_q,
3270 qpair->produce_q_size,
3272 qp_memcpy_to_queue_iov);
3274 if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
3275 !qp_wait_for_ready_queue(qpair))
3276 result = VMCI_ERROR_WOULD_BLOCK;
3278 } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
3284 EXPORT_SYMBOL_GPL(vmci_qpair_enquev);
3287 * vmci_qpair_dequev() - Get data from the queue using iov.
3288 * @qpair: Pointer to the queue pair struct.
3289 * @iov: Pointer to buffer for the data
3290 * @iov_size: Length of buffer.
3291 * @buf_type: Buffer type (Unused).
3293 * This is the client interface for dequeueing data from the queue.
3294 * This function uses IO vectors to handle the work. Returns number
3295 * of bytes dequeued or < 0 on error.
3297 ssize_t vmci_qpair_dequev(struct vmci_qp *qpair,
3305 return VMCI_ERROR_INVALID_ARGS;
3310 result = qp_dequeue_locked(qpair->produce_q,
3312 qpair->consume_q_size,
3314 qp_memcpy_from_queue_iov,
3317 if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
3318 !qp_wait_for_ready_queue(qpair))
3319 result = VMCI_ERROR_WOULD_BLOCK;
3321 } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
3327 EXPORT_SYMBOL_GPL(vmci_qpair_dequev);
3330 * vmci_qpair_peekv() - Peek at the data in the queue using iov.
3331 * @qpair: Pointer to the queue pair struct.
3332 * @iov: Pointer to buffer for the data
3333 * @iov_size: Length of buffer.
3334 * @buf_type: Buffer type (Unused on Linux).
3336 * This is the client interface for peeking into a queue. (I.e.,
3337 * copy data from the queue without updating the head pointer.)
3338 * This function uses IO vectors to handle the work. Returns number
3339 * of bytes peeked or < 0 on error.
3341 ssize_t vmci_qpair_peekv(struct vmci_qp *qpair,
3349 return VMCI_ERROR_INVALID_ARGS;
3354 result = qp_dequeue_locked(qpair->produce_q,
3356 qpair->consume_q_size,
3358 qp_memcpy_from_queue_iov,
3361 if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
3362 !qp_wait_for_ready_queue(qpair))
3363 result = VMCI_ERROR_WOULD_BLOCK;
3365 } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
3370 EXPORT_SYMBOL_GPL(vmci_qpair_peekv);