4 * Copyright (C) 2012 VMware, Inc. All rights reserved.
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License as published by the
8 * Free Software Foundation version 2 and no later version.
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
12 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 #include <linux/vmw_vmci_defs.h>
17 #include <linux/vmw_vmci_api.h>
18 #include <linux/highmem.h>
19 #include <linux/kernel.h>
21 #include <linux/module.h>
22 #include <linux/mutex.h>
23 #include <linux/pagemap.h>
24 #include <linux/pci.h>
25 #include <linux/sched.h>
26 #include <linux/slab.h>
27 #include <linux/uio.h>
28 #include <linux/wait.h>
29 #include <linux/vmalloc.h>
30 #include <linux/skbuff.h>
32 #include "vmci_handle_array.h"
33 #include "vmci_queue_pair.h"
34 #include "vmci_datagram.h"
35 #include "vmci_resource.h"
36 #include "vmci_context.h"
37 #include "vmci_driver.h"
38 #include "vmci_event.h"
39 #include "vmci_route.h"
42 * In the following, we will distinguish between two kinds of VMX processes -
43 * the ones with versions lower than VMCI_VERSION_NOVMVM that use specialized
44 * VMCI page files in the VMX and supporting VM to VM communication and the
45 * newer ones that use the guest memory directly. We will in the following
46 * refer to the older VMX versions as old-style VMX'en, and the newer ones as
49 * The state transition datagram is as follows (the VMCIQPB_ prefix has been
50 * removed for readability) - see below for more details on the transtions:
52 * -------------- NEW -------------
55 * CREATED_NO_MEM <-----------------> CREATED_MEM
57 * | o-----------------------o |
60 * ATTACHED_NO_MEM <----------------> ATTACHED_MEM
62 * | o----------------------o |
65 * SHUTDOWN_NO_MEM <----------------> SHUTDOWN_MEM
68 * -------------> gone <-------------
70 * In more detail. When a VMCI queue pair is first created, it will be in the
71 * VMCIQPB_NEW state. It will then move into one of the following states:
73 * - VMCIQPB_CREATED_NO_MEM: this state indicates that either:
75 * - the created was performed by a host endpoint, in which case there is
76 * no backing memory yet.
78 * - the create was initiated by an old-style VMX, that uses
79 * vmci_qp_broker_set_page_store to specify the UVAs of the queue pair at
80 * a later point in time. This state can be distinguished from the one
81 * above by the context ID of the creator. A host side is not allowed to
82 * attach until the page store has been set.
84 * - VMCIQPB_CREATED_MEM: this state is the result when the queue pair
85 * is created by a VMX using the queue pair device backend that
86 * sets the UVAs of the queue pair immediately and stores the
87 * information for later attachers. At this point, it is ready for
88 * the host side to attach to it.
90 * Once the queue pair is in one of the created states (with the exception of
91 * the case mentioned for older VMX'en above), it is possible to attach to the
92 * queue pair. Again we have two new states possible:
94 * - VMCIQPB_ATTACHED_MEM: this state can be reached through the following
97 * - from VMCIQPB_CREATED_NO_MEM when a new-style VMX allocates a queue
98 * pair, and attaches to a queue pair previously created by the host side.
100 * - from VMCIQPB_CREATED_MEM when the host side attaches to a queue pair
101 * already created by a guest.
103 * - from VMCIQPB_ATTACHED_NO_MEM, when an old-style VMX calls
104 * vmci_qp_broker_set_page_store (see below).
106 * - VMCIQPB_ATTACHED_NO_MEM: If the queue pair already was in the
107 * VMCIQPB_CREATED_NO_MEM due to a host side create, an old-style VMX will
108 * bring the queue pair into this state. Once vmci_qp_broker_set_page_store
109 * is called to register the user memory, the VMCIQPB_ATTACH_MEM state
112 * From the attached queue pair, the queue pair can enter the shutdown states
113 * when either side of the queue pair detaches. If the guest side detaches
114 * first, the queue pair will enter the VMCIQPB_SHUTDOWN_NO_MEM state, where
115 * the content of the queue pair will no longer be available. If the host
116 * side detaches first, the queue pair will either enter the
117 * VMCIQPB_SHUTDOWN_MEM, if the guest memory is currently mapped, or
118 * VMCIQPB_SHUTDOWN_NO_MEM, if the guest memory is not mapped
119 * (e.g., the host detaches while a guest is stunned).
121 * New-style VMX'en will also unmap guest memory, if the guest is
122 * quiesced, e.g., during a snapshot operation. In that case, the guest
123 * memory will no longer be available, and the queue pair will transition from
124 * *_MEM state to a *_NO_MEM state. The VMX may later map the memory once more,
125 * in which case the queue pair will transition from the *_NO_MEM state at that
126 * point back to the *_MEM state. Note that the *_NO_MEM state may have changed,
127 * since the peer may have either attached or detached in the meantime. The
128 * values are laid out such that ++ on a state will move from a *_NO_MEM to a
129 * *_MEM state, and vice versa.
132 /* The Kernel specific component of the struct vmci_queue structure. */
133 struct vmci_queue_kern_if {
134 struct mutex __mutex; /* Protects the queue. */
135 struct mutex *mutex; /* Shared by producer and consumer queues. */
136 size_t num_pages; /* Number of pages incl. header. */
137 bool host; /* Host or guest? */
142 } g; /* Used by the guest. */
145 struct page **header_page;
146 } h; /* Used by the host. */
151 * This structure is opaque to the clients.
154 struct vmci_handle handle;
155 struct vmci_queue *produce_q;
156 struct vmci_queue *consume_q;
163 unsigned int blocked;
164 unsigned int generation;
165 wait_queue_head_t event;
168 enum qp_broker_state {
170 VMCIQPB_CREATED_NO_MEM,
172 VMCIQPB_ATTACHED_NO_MEM,
173 VMCIQPB_ATTACHED_MEM,
174 VMCIQPB_SHUTDOWN_NO_MEM,
175 VMCIQPB_SHUTDOWN_MEM,
179 #define QPBROKERSTATE_HAS_MEM(_qpb) (_qpb->state == VMCIQPB_CREATED_MEM || \
180 _qpb->state == VMCIQPB_ATTACHED_MEM || \
181 _qpb->state == VMCIQPB_SHUTDOWN_MEM)
184 * In the queue pair broker, we always use the guest point of view for
185 * the produce and consume queue values and references, e.g., the
186 * produce queue size stored is the guests produce queue size. The
187 * host endpoint will need to swap these around. The only exception is
188 * the local queue pairs on the host, in which case the host endpoint
189 * that creates the queue pair will have the right orientation, and
190 * the attaching host endpoint will need to swap.
193 struct list_head list_item;
194 struct vmci_handle handle;
202 struct qp_broker_entry {
203 struct vmci_resource resource;
207 enum qp_broker_state state;
208 bool require_trusted_attach;
209 bool created_by_trusted;
210 bool vmci_page_files; /* Created by VMX using VMCI page files */
211 struct vmci_queue *produce_q;
212 struct vmci_queue *consume_q;
213 struct vmci_queue_header saved_produce_q;
214 struct vmci_queue_header saved_consume_q;
215 vmci_event_release_cb wakeup_cb;
217 void *local_mem; /* Kernel memory for local queue pair */
220 struct qp_guest_endpoint {
221 struct vmci_resource resource;
226 struct ppn_set ppn_set;
230 struct list_head head;
231 struct mutex mutex; /* Protect queue list. */
234 static struct qp_list qp_broker_list = {
235 .head = LIST_HEAD_INIT(qp_broker_list.head),
236 .mutex = __MUTEX_INITIALIZER(qp_broker_list.mutex),
239 static struct qp_list qp_guest_endpoints = {
240 .head = LIST_HEAD_INIT(qp_guest_endpoints.head),
241 .mutex = __MUTEX_INITIALIZER(qp_guest_endpoints.mutex),
244 #define INVALID_VMCI_GUEST_MEM_ID 0
245 #define QPE_NUM_PAGES(_QPE) ((u32) \
246 (DIV_ROUND_UP(_QPE.produce_size, PAGE_SIZE) + \
247 DIV_ROUND_UP(_QPE.consume_size, PAGE_SIZE) + 2))
251 * Frees kernel VA space for a given queue and its queue header, and
252 * frees physical data pages.
254 static void qp_free_queue(void *q, u64 size)
256 struct vmci_queue *queue = q;
261 /* Given size does not include header, so add in a page here. */
262 for (i = 0; i < DIV_ROUND_UP(size, PAGE_SIZE) + 1; i++) {
263 dma_free_coherent(&vmci_pdev->dev, PAGE_SIZE,
264 queue->kernel_if->u.g.vas[i],
265 queue->kernel_if->u.g.pas[i]);
273 * Allocates kernel queue pages of specified size with IOMMU mappings,
274 * plus space for the queue structure/kernel interface and the queue
277 static void *qp_alloc_queue(u64 size, u32 flags)
280 struct vmci_queue *queue;
283 size_t queue_size = sizeof(*queue) + sizeof(*queue->kernel_if);
286 if (size > SIZE_MAX - PAGE_SIZE)
288 num_pages = DIV_ROUND_UP(size, PAGE_SIZE) + 1;
290 (SIZE_MAX - queue_size) /
291 (sizeof(*queue->kernel_if->u.g.pas) +
292 sizeof(*queue->kernel_if->u.g.vas)))
295 pas_size = num_pages * sizeof(*queue->kernel_if->u.g.pas);
296 vas_size = num_pages * sizeof(*queue->kernel_if->u.g.vas);
297 queue_size += pas_size + vas_size;
299 queue = vmalloc(queue_size);
303 queue->q_header = NULL;
304 queue->saved_header = NULL;
305 queue->kernel_if = (struct vmci_queue_kern_if *)(queue + 1);
306 queue->kernel_if->mutex = NULL;
307 queue->kernel_if->num_pages = num_pages;
308 queue->kernel_if->u.g.pas = (dma_addr_t *)(queue->kernel_if + 1);
309 queue->kernel_if->u.g.vas =
310 (void **)((u8 *)queue->kernel_if->u.g.pas + pas_size);
311 queue->kernel_if->host = false;
313 for (i = 0; i < num_pages; i++) {
314 queue->kernel_if->u.g.vas[i] =
315 dma_alloc_coherent(&vmci_pdev->dev, PAGE_SIZE,
316 &queue->kernel_if->u.g.pas[i],
318 if (!queue->kernel_if->u.g.vas[i]) {
319 /* Size excl. the header. */
320 qp_free_queue(queue, i * PAGE_SIZE);
325 /* Queue header is the first page. */
326 queue->q_header = queue->kernel_if->u.g.vas[0];
332 * Copies from a given buffer or iovector to a VMCI Queue. Uses
333 * kmap()/kunmap() to dynamically map/unmap required portions of the queue
334 * by traversing the offset -> page translation structure for the queue.
335 * Assumes that offset + size does not wrap around in the queue.
337 static int qp_memcpy_to_queue_iter(struct vmci_queue *queue,
339 struct iov_iter *from,
342 struct vmci_queue_kern_if *kernel_if = queue->kernel_if;
343 size_t bytes_copied = 0;
345 while (bytes_copied < size) {
346 const u64 page_index =
347 (queue_offset + bytes_copied) / PAGE_SIZE;
348 const size_t page_offset =
349 (queue_offset + bytes_copied) & (PAGE_SIZE - 1);
354 va = kmap(kernel_if->u.h.page[page_index]);
356 va = kernel_if->u.g.vas[page_index + 1];
359 if (size - bytes_copied > PAGE_SIZE - page_offset)
360 /* Enough payload to fill up from this page. */
361 to_copy = PAGE_SIZE - page_offset;
363 to_copy = size - bytes_copied;
365 if (!copy_from_iter_full((u8 *)va + page_offset, to_copy,
368 kunmap(kernel_if->u.h.page[page_index]);
369 return VMCI_ERROR_INVALID_ARGS;
371 bytes_copied += to_copy;
373 kunmap(kernel_if->u.h.page[page_index]);
380 * Copies to a given buffer or iovector from a VMCI Queue. Uses
381 * kmap()/kunmap() to dynamically map/unmap required portions of the queue
382 * by traversing the offset -> page translation structure for the queue.
383 * Assumes that offset + size does not wrap around in the queue.
385 static int qp_memcpy_from_queue_iter(struct iov_iter *to,
386 const struct vmci_queue *queue,
387 u64 queue_offset, size_t size)
389 struct vmci_queue_kern_if *kernel_if = queue->kernel_if;
390 size_t bytes_copied = 0;
392 while (bytes_copied < size) {
393 const u64 page_index =
394 (queue_offset + bytes_copied) / PAGE_SIZE;
395 const size_t page_offset =
396 (queue_offset + bytes_copied) & (PAGE_SIZE - 1);
402 va = kmap(kernel_if->u.h.page[page_index]);
404 va = kernel_if->u.g.vas[page_index + 1];
407 if (size - bytes_copied > PAGE_SIZE - page_offset)
408 /* Enough payload to fill up this page. */
409 to_copy = PAGE_SIZE - page_offset;
411 to_copy = size - bytes_copied;
413 err = copy_to_iter((u8 *)va + page_offset, to_copy, to);
414 if (err != to_copy) {
416 kunmap(kernel_if->u.h.page[page_index]);
417 return VMCI_ERROR_INVALID_ARGS;
419 bytes_copied += to_copy;
421 kunmap(kernel_if->u.h.page[page_index]);
428 * Allocates two list of PPNs --- one for the pages in the produce queue,
429 * and the other for the pages in the consume queue. Intializes the list
430 * of PPNs with the page frame numbers of the KVA for the two queues (and
431 * the queue headers).
433 static int qp_alloc_ppn_set(void *prod_q,
434 u64 num_produce_pages,
436 u64 num_consume_pages, struct ppn_set *ppn_set)
440 struct vmci_queue *produce_q = prod_q;
441 struct vmci_queue *consume_q = cons_q;
444 if (!produce_q || !num_produce_pages || !consume_q ||
445 !num_consume_pages || !ppn_set)
446 return VMCI_ERROR_INVALID_ARGS;
448 if (ppn_set->initialized)
449 return VMCI_ERROR_ALREADY_EXISTS;
452 kmalloc_array(num_produce_pages, sizeof(*produce_ppns),
455 return VMCI_ERROR_NO_MEM;
458 kmalloc_array(num_consume_pages, sizeof(*consume_ppns),
462 return VMCI_ERROR_NO_MEM;
465 for (i = 0; i < num_produce_pages; i++) {
469 produce_q->kernel_if->u.g.pas[i] >> PAGE_SHIFT;
470 pfn = produce_ppns[i];
472 /* Fail allocation if PFN isn't supported by hypervisor. */
473 if (sizeof(pfn) > sizeof(*produce_ppns)
474 && pfn != produce_ppns[i])
478 for (i = 0; i < num_consume_pages; i++) {
482 consume_q->kernel_if->u.g.pas[i] >> PAGE_SHIFT;
483 pfn = consume_ppns[i];
485 /* Fail allocation if PFN isn't supported by hypervisor. */
486 if (sizeof(pfn) > sizeof(*consume_ppns)
487 && pfn != consume_ppns[i])
491 ppn_set->num_produce_pages = num_produce_pages;
492 ppn_set->num_consume_pages = num_consume_pages;
493 ppn_set->produce_ppns = produce_ppns;
494 ppn_set->consume_ppns = consume_ppns;
495 ppn_set->initialized = true;
501 return VMCI_ERROR_INVALID_ARGS;
505 * Frees the two list of PPNs for a queue pair.
507 static void qp_free_ppn_set(struct ppn_set *ppn_set)
509 if (ppn_set->initialized) {
510 /* Do not call these functions on NULL inputs. */
511 kfree(ppn_set->produce_ppns);
512 kfree(ppn_set->consume_ppns);
514 memset(ppn_set, 0, sizeof(*ppn_set));
518 * Populates the list of PPNs in the hypercall structure with the PPNS
519 * of the produce queue and the consume queue.
521 static int qp_populate_ppn_set(u8 *call_buf, const struct ppn_set *ppn_set)
523 memcpy(call_buf, ppn_set->produce_ppns,
524 ppn_set->num_produce_pages * sizeof(*ppn_set->produce_ppns));
526 ppn_set->num_produce_pages * sizeof(*ppn_set->produce_ppns),
527 ppn_set->consume_ppns,
528 ppn_set->num_consume_pages * sizeof(*ppn_set->consume_ppns));
534 * Allocates kernel VA space of specified size plus space for the queue
535 * and kernel interface. This is different from the guest queue allocator,
536 * because we do not allocate our own queue header/data pages here but
537 * share those of the guest.
539 static struct vmci_queue *qp_host_alloc_queue(u64 size)
541 struct vmci_queue *queue;
542 size_t queue_page_size;
544 const size_t queue_size = sizeof(*queue) + sizeof(*(queue->kernel_if));
546 if (size > SIZE_MAX - PAGE_SIZE)
548 num_pages = DIV_ROUND_UP(size, PAGE_SIZE) + 1;
549 if (num_pages > (SIZE_MAX - queue_size) /
550 sizeof(*queue->kernel_if->u.h.page))
553 queue_page_size = num_pages * sizeof(*queue->kernel_if->u.h.page);
555 if (queue_size + queue_page_size > KMALLOC_MAX_SIZE)
558 queue = kzalloc(queue_size + queue_page_size, GFP_KERNEL);
560 queue->q_header = NULL;
561 queue->saved_header = NULL;
562 queue->kernel_if = (struct vmci_queue_kern_if *)(queue + 1);
563 queue->kernel_if->host = true;
564 queue->kernel_if->mutex = NULL;
565 queue->kernel_if->num_pages = num_pages;
566 queue->kernel_if->u.h.header_page =
567 (struct page **)((u8 *)queue + queue_size);
568 queue->kernel_if->u.h.page =
569 &queue->kernel_if->u.h.header_page[1];
576 * Frees kernel memory for a given queue (header plus translation
579 static void qp_host_free_queue(struct vmci_queue *queue, u64 queue_size)
585 * Initialize the mutex for the pair of queues. This mutex is used to
586 * protect the q_header and the buffer from changing out from under any
587 * users of either queue. Of course, it's only any good if the mutexes
588 * are actually acquired. Queue structure must lie on non-paged memory
589 * or we cannot guarantee access to the mutex.
591 static void qp_init_queue_mutex(struct vmci_queue *produce_q,
592 struct vmci_queue *consume_q)
595 * Only the host queue has shared state - the guest queues do not
596 * need to synchronize access using a queue mutex.
599 if (produce_q->kernel_if->host) {
600 produce_q->kernel_if->mutex = &produce_q->kernel_if->__mutex;
601 consume_q->kernel_if->mutex = &produce_q->kernel_if->__mutex;
602 mutex_init(produce_q->kernel_if->mutex);
607 * Cleans up the mutex for the pair of queues.
609 static void qp_cleanup_queue_mutex(struct vmci_queue *produce_q,
610 struct vmci_queue *consume_q)
612 if (produce_q->kernel_if->host) {
613 produce_q->kernel_if->mutex = NULL;
614 consume_q->kernel_if->mutex = NULL;
619 * Acquire the mutex for the queue. Note that the produce_q and
620 * the consume_q share a mutex. So, only one of the two need to
621 * be passed in to this routine. Either will work just fine.
623 static void qp_acquire_queue_mutex(struct vmci_queue *queue)
625 if (queue->kernel_if->host)
626 mutex_lock(queue->kernel_if->mutex);
630 * Release the mutex for the queue. Note that the produce_q and
631 * the consume_q share a mutex. So, only one of the two need to
632 * be passed in to this routine. Either will work just fine.
634 static void qp_release_queue_mutex(struct vmci_queue *queue)
636 if (queue->kernel_if->host)
637 mutex_unlock(queue->kernel_if->mutex);
641 * Helper function to release pages in the PageStoreAttachInfo
642 * previously obtained using get_user_pages.
644 static void qp_release_pages(struct page **pages,
645 u64 num_pages, bool dirty)
649 for (i = 0; i < num_pages; i++) {
651 set_page_dirty_lock(pages[i]);
659 * Lock the user pages referenced by the {produce,consume}Buffer
660 * struct into memory and populate the {produce,consume}Pages
661 * arrays in the attach structure with them.
663 static int qp_host_get_user_memory(u64 produce_uva,
665 struct vmci_queue *produce_q,
666 struct vmci_queue *consume_q)
669 int err = VMCI_SUCCESS;
671 retval = get_user_pages_fast((uintptr_t) produce_uva,
672 produce_q->kernel_if->num_pages, 1,
673 produce_q->kernel_if->u.h.header_page);
674 if (retval < (int)produce_q->kernel_if->num_pages) {
675 pr_debug("get_user_pages_fast(produce) failed (retval=%d)",
678 qp_release_pages(produce_q->kernel_if->u.h.header_page,
680 err = VMCI_ERROR_NO_MEM;
684 retval = get_user_pages_fast((uintptr_t) consume_uva,
685 consume_q->kernel_if->num_pages, 1,
686 consume_q->kernel_if->u.h.header_page);
687 if (retval < (int)consume_q->kernel_if->num_pages) {
688 pr_debug("get_user_pages_fast(consume) failed (retval=%d)",
691 qp_release_pages(consume_q->kernel_if->u.h.header_page,
693 qp_release_pages(produce_q->kernel_if->u.h.header_page,
694 produce_q->kernel_if->num_pages, false);
695 err = VMCI_ERROR_NO_MEM;
703 * Registers the specification of the user pages used for backing a queue
704 * pair. Enough information to map in pages is stored in the OS specific
705 * part of the struct vmci_queue structure.
707 static int qp_host_register_user_memory(struct vmci_qp_page_store *page_store,
708 struct vmci_queue *produce_q,
709 struct vmci_queue *consume_q)
715 * The new style and the old style mapping only differs in
716 * that we either get a single or two UVAs, so we split the
717 * single UVA range at the appropriate spot.
719 produce_uva = page_store->pages;
720 consume_uva = page_store->pages +
721 produce_q->kernel_if->num_pages * PAGE_SIZE;
722 return qp_host_get_user_memory(produce_uva, consume_uva, produce_q,
727 * Releases and removes the references to user pages stored in the attach
728 * struct. Pages are released from the page cache and may become
731 static void qp_host_unregister_user_memory(struct vmci_queue *produce_q,
732 struct vmci_queue *consume_q)
734 qp_release_pages(produce_q->kernel_if->u.h.header_page,
735 produce_q->kernel_if->num_pages, true);
736 memset(produce_q->kernel_if->u.h.header_page, 0,
737 sizeof(*produce_q->kernel_if->u.h.header_page) *
738 produce_q->kernel_if->num_pages);
739 qp_release_pages(consume_q->kernel_if->u.h.header_page,
740 consume_q->kernel_if->num_pages, true);
741 memset(consume_q->kernel_if->u.h.header_page, 0,
742 sizeof(*consume_q->kernel_if->u.h.header_page) *
743 consume_q->kernel_if->num_pages);
747 * Once qp_host_register_user_memory has been performed on a
748 * queue, the queue pair headers can be mapped into the
749 * kernel. Once mapped, they must be unmapped with
750 * qp_host_unmap_queues prior to calling
751 * qp_host_unregister_user_memory.
754 static int qp_host_map_queues(struct vmci_queue *produce_q,
755 struct vmci_queue *consume_q)
759 if (!produce_q->q_header || !consume_q->q_header) {
760 struct page *headers[2];
762 if (produce_q->q_header != consume_q->q_header)
763 return VMCI_ERROR_QUEUEPAIR_MISMATCH;
765 if (produce_q->kernel_if->u.h.header_page == NULL ||
766 *produce_q->kernel_if->u.h.header_page == NULL)
767 return VMCI_ERROR_UNAVAILABLE;
769 headers[0] = *produce_q->kernel_if->u.h.header_page;
770 headers[1] = *consume_q->kernel_if->u.h.header_page;
772 produce_q->q_header = vmap(headers, 2, VM_MAP, PAGE_KERNEL);
773 if (produce_q->q_header != NULL) {
774 consume_q->q_header =
775 (struct vmci_queue_header *)((u8 *)
776 produce_q->q_header +
778 result = VMCI_SUCCESS;
780 pr_warn("vmap failed\n");
781 result = VMCI_ERROR_NO_MEM;
784 result = VMCI_SUCCESS;
791 * Unmaps previously mapped queue pair headers from the kernel.
792 * Pages are unpinned.
794 static int qp_host_unmap_queues(u32 gid,
795 struct vmci_queue *produce_q,
796 struct vmci_queue *consume_q)
798 if (produce_q->q_header) {
799 if (produce_q->q_header < consume_q->q_header)
800 vunmap(produce_q->q_header);
802 vunmap(consume_q->q_header);
804 produce_q->q_header = NULL;
805 consume_q->q_header = NULL;
812 * Finds the entry in the list corresponding to a given handle. Assumes
813 * that the list is locked.
815 static struct qp_entry *qp_list_find(struct qp_list *qp_list,
816 struct vmci_handle handle)
818 struct qp_entry *entry;
820 if (vmci_handle_is_invalid(handle))
823 list_for_each_entry(entry, &qp_list->head, list_item) {
824 if (vmci_handle_is_equal(entry->handle, handle))
832 * Finds the entry in the list corresponding to a given handle.
834 static struct qp_guest_endpoint *
835 qp_guest_handle_to_entry(struct vmci_handle handle)
837 struct qp_guest_endpoint *entry;
838 struct qp_entry *qp = qp_list_find(&qp_guest_endpoints, handle);
840 entry = qp ? container_of(
841 qp, struct qp_guest_endpoint, qp) : NULL;
846 * Finds the entry in the list corresponding to a given handle.
848 static struct qp_broker_entry *
849 qp_broker_handle_to_entry(struct vmci_handle handle)
851 struct qp_broker_entry *entry;
852 struct qp_entry *qp = qp_list_find(&qp_broker_list, handle);
854 entry = qp ? container_of(
855 qp, struct qp_broker_entry, qp) : NULL;
860 * Dispatches a queue pair event message directly into the local event
863 static int qp_notify_peer_local(bool attach, struct vmci_handle handle)
865 u32 context_id = vmci_get_context_id();
866 struct vmci_event_qp ev;
868 memset(&ev, 0, sizeof(ev));
869 ev.msg.hdr.dst = vmci_make_handle(context_id, VMCI_EVENT_HANDLER);
870 ev.msg.hdr.src = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
871 VMCI_CONTEXT_RESOURCE_ID);
872 ev.msg.hdr.payload_size = sizeof(ev) - sizeof(ev.msg.hdr);
873 ev.msg.event_data.event =
874 attach ? VMCI_EVENT_QP_PEER_ATTACH : VMCI_EVENT_QP_PEER_DETACH;
875 ev.payload.peer_id = context_id;
876 ev.payload.handle = handle;
878 return vmci_event_dispatch(&ev.msg.hdr);
882 * Allocates and initializes a qp_guest_endpoint structure.
883 * Allocates a queue_pair rid (and handle) iff the given entry has
884 * an invalid handle. 0 through VMCI_RESERVED_RESOURCE_ID_MAX
885 * are reserved handles. Assumes that the QP list mutex is held
888 static struct qp_guest_endpoint *
889 qp_guest_endpoint_create(struct vmci_handle handle,
898 struct qp_guest_endpoint *entry;
899 /* One page each for the queue headers. */
900 const u64 num_ppns = DIV_ROUND_UP(produce_size, PAGE_SIZE) +
901 DIV_ROUND_UP(consume_size, PAGE_SIZE) + 2;
903 if (vmci_handle_is_invalid(handle)) {
904 u32 context_id = vmci_get_context_id();
906 handle = vmci_make_handle(context_id, VMCI_INVALID_ID);
909 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
911 entry->qp.peer = peer;
912 entry->qp.flags = flags;
913 entry->qp.produce_size = produce_size;
914 entry->qp.consume_size = consume_size;
915 entry->qp.ref_count = 0;
916 entry->num_ppns = num_ppns;
917 entry->produce_q = produce_q;
918 entry->consume_q = consume_q;
919 INIT_LIST_HEAD(&entry->qp.list_item);
921 /* Add resource obj */
922 result = vmci_resource_add(&entry->resource,
923 VMCI_RESOURCE_TYPE_QPAIR_GUEST,
925 entry->qp.handle = vmci_resource_handle(&entry->resource);
926 if ((result != VMCI_SUCCESS) ||
927 qp_list_find(&qp_guest_endpoints, entry->qp.handle)) {
928 pr_warn("Failed to add new resource (handle=0x%x:0x%x), error: %d",
929 handle.context, handle.resource, result);
938 * Frees a qp_guest_endpoint structure.
940 static void qp_guest_endpoint_destroy(struct qp_guest_endpoint *entry)
942 qp_free_ppn_set(&entry->ppn_set);
943 qp_cleanup_queue_mutex(entry->produce_q, entry->consume_q);
944 qp_free_queue(entry->produce_q, entry->qp.produce_size);
945 qp_free_queue(entry->consume_q, entry->qp.consume_size);
946 /* Unlink from resource hash table and free callback */
947 vmci_resource_remove(&entry->resource);
953 * Helper to make a queue_pairAlloc hypercall when the driver is
954 * supporting a guest device.
956 static int qp_alloc_hypercall(const struct qp_guest_endpoint *entry)
958 struct vmci_qp_alloc_msg *alloc_msg;
962 if (!entry || entry->num_ppns <= 2)
963 return VMCI_ERROR_INVALID_ARGS;
965 msg_size = sizeof(*alloc_msg) +
966 (size_t) entry->num_ppns * sizeof(u32);
967 alloc_msg = kmalloc(msg_size, GFP_KERNEL);
969 return VMCI_ERROR_NO_MEM;
971 alloc_msg->hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
972 VMCI_QUEUEPAIR_ALLOC);
973 alloc_msg->hdr.src = VMCI_ANON_SRC_HANDLE;
974 alloc_msg->hdr.payload_size = msg_size - VMCI_DG_HEADERSIZE;
975 alloc_msg->handle = entry->qp.handle;
976 alloc_msg->peer = entry->qp.peer;
977 alloc_msg->flags = entry->qp.flags;
978 alloc_msg->produce_size = entry->qp.produce_size;
979 alloc_msg->consume_size = entry->qp.consume_size;
980 alloc_msg->num_ppns = entry->num_ppns;
982 result = qp_populate_ppn_set((u8 *)alloc_msg + sizeof(*alloc_msg),
984 if (result == VMCI_SUCCESS)
985 result = vmci_send_datagram(&alloc_msg->hdr);
993 * Helper to make a queue_pairDetach hypercall when the driver is
994 * supporting a guest device.
996 static int qp_detatch_hypercall(struct vmci_handle handle)
998 struct vmci_qp_detach_msg detach_msg;
1000 detach_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
1001 VMCI_QUEUEPAIR_DETACH);
1002 detach_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
1003 detach_msg.hdr.payload_size = sizeof(handle);
1004 detach_msg.handle = handle;
1006 return vmci_send_datagram(&detach_msg.hdr);
1010 * Adds the given entry to the list. Assumes that the list is locked.
1012 static void qp_list_add_entry(struct qp_list *qp_list, struct qp_entry *entry)
1015 list_add(&entry->list_item, &qp_list->head);
1019 * Removes the given entry from the list. Assumes that the list is locked.
1021 static void qp_list_remove_entry(struct qp_list *qp_list,
1022 struct qp_entry *entry)
1025 list_del(&entry->list_item);
1029 * Helper for VMCI queue_pair detach interface. Frees the physical
1030 * pages for the queue pair.
1032 static int qp_detatch_guest_work(struct vmci_handle handle)
1035 struct qp_guest_endpoint *entry;
1036 u32 ref_count = ~0; /* To avoid compiler warning below */
1038 mutex_lock(&qp_guest_endpoints.mutex);
1040 entry = qp_guest_handle_to_entry(handle);
1042 mutex_unlock(&qp_guest_endpoints.mutex);
1043 return VMCI_ERROR_NOT_FOUND;
1046 if (entry->qp.flags & VMCI_QPFLAG_LOCAL) {
1047 result = VMCI_SUCCESS;
1049 if (entry->qp.ref_count > 1) {
1050 result = qp_notify_peer_local(false, handle);
1052 * We can fail to notify a local queuepair
1053 * because we can't allocate. We still want
1054 * to release the entry if that happens, so
1055 * don't bail out yet.
1059 result = qp_detatch_hypercall(handle);
1060 if (result < VMCI_SUCCESS) {
1062 * We failed to notify a non-local queuepair.
1063 * That other queuepair might still be
1064 * accessing the shared memory, so don't
1065 * release the entry yet. It will get cleaned
1066 * up by VMCIqueue_pair_Exit() if necessary
1067 * (assuming we are going away, otherwise why
1071 mutex_unlock(&qp_guest_endpoints.mutex);
1077 * If we get here then we either failed to notify a local queuepair, or
1078 * we succeeded in all cases. Release the entry if required.
1081 entry->qp.ref_count--;
1082 if (entry->qp.ref_count == 0)
1083 qp_list_remove_entry(&qp_guest_endpoints, &entry->qp);
1085 /* If we didn't remove the entry, this could change once we unlock. */
1087 ref_count = entry->qp.ref_count;
1089 mutex_unlock(&qp_guest_endpoints.mutex);
1092 qp_guest_endpoint_destroy(entry);
1098 * This functions handles the actual allocation of a VMCI queue
1099 * pair guest endpoint. Allocates physical pages for the queue
1100 * pair. It makes OS dependent calls through generic wrappers.
1102 static int qp_alloc_guest_work(struct vmci_handle *handle,
1103 struct vmci_queue **produce_q,
1105 struct vmci_queue **consume_q,
1111 const u64 num_produce_pages =
1112 DIV_ROUND_UP(produce_size, PAGE_SIZE) + 1;
1113 const u64 num_consume_pages =
1114 DIV_ROUND_UP(consume_size, PAGE_SIZE) + 1;
1115 void *my_produce_q = NULL;
1116 void *my_consume_q = NULL;
1118 struct qp_guest_endpoint *queue_pair_entry = NULL;
1120 if (priv_flags != VMCI_NO_PRIVILEGE_FLAGS)
1121 return VMCI_ERROR_NO_ACCESS;
1123 mutex_lock(&qp_guest_endpoints.mutex);
1125 queue_pair_entry = qp_guest_handle_to_entry(*handle);
1126 if (queue_pair_entry) {
1127 if (queue_pair_entry->qp.flags & VMCI_QPFLAG_LOCAL) {
1128 /* Local attach case. */
1129 if (queue_pair_entry->qp.ref_count > 1) {
1130 pr_devel("Error attempting to attach more than once\n");
1131 result = VMCI_ERROR_UNAVAILABLE;
1132 goto error_keep_entry;
1135 if (queue_pair_entry->qp.produce_size != consume_size ||
1136 queue_pair_entry->qp.consume_size !=
1138 queue_pair_entry->qp.flags !=
1139 (flags & ~VMCI_QPFLAG_ATTACH_ONLY)) {
1140 pr_devel("Error mismatched queue pair in local attach\n");
1141 result = VMCI_ERROR_QUEUEPAIR_MISMATCH;
1142 goto error_keep_entry;
1146 * Do a local attach. We swap the consume and
1147 * produce queues for the attacher and deliver
1150 result = qp_notify_peer_local(true, *handle);
1151 if (result < VMCI_SUCCESS)
1152 goto error_keep_entry;
1154 my_produce_q = queue_pair_entry->consume_q;
1155 my_consume_q = queue_pair_entry->produce_q;
1159 result = VMCI_ERROR_ALREADY_EXISTS;
1160 goto error_keep_entry;
1163 my_produce_q = qp_alloc_queue(produce_size, flags);
1164 if (!my_produce_q) {
1165 pr_warn("Error allocating pages for produce queue\n");
1166 result = VMCI_ERROR_NO_MEM;
1170 my_consume_q = qp_alloc_queue(consume_size, flags);
1171 if (!my_consume_q) {
1172 pr_warn("Error allocating pages for consume queue\n");
1173 result = VMCI_ERROR_NO_MEM;
1177 queue_pair_entry = qp_guest_endpoint_create(*handle, peer, flags,
1178 produce_size, consume_size,
1179 my_produce_q, my_consume_q);
1180 if (!queue_pair_entry) {
1181 pr_warn("Error allocating memory in %s\n", __func__);
1182 result = VMCI_ERROR_NO_MEM;
1186 result = qp_alloc_ppn_set(my_produce_q, num_produce_pages, my_consume_q,
1188 &queue_pair_entry->ppn_set);
1189 if (result < VMCI_SUCCESS) {
1190 pr_warn("qp_alloc_ppn_set failed\n");
1195 * It's only necessary to notify the host if this queue pair will be
1196 * attached to from another context.
1198 if (queue_pair_entry->qp.flags & VMCI_QPFLAG_LOCAL) {
1199 /* Local create case. */
1200 u32 context_id = vmci_get_context_id();
1203 * Enforce similar checks on local queue pairs as we
1204 * do for regular ones. The handle's context must
1205 * match the creator or attacher context id (here they
1206 * are both the current context id) and the
1207 * attach-only flag cannot exist during create. We
1208 * also ensure specified peer is this context or an
1211 if (queue_pair_entry->qp.handle.context != context_id ||
1212 (queue_pair_entry->qp.peer != VMCI_INVALID_ID &&
1213 queue_pair_entry->qp.peer != context_id)) {
1214 result = VMCI_ERROR_NO_ACCESS;
1218 if (queue_pair_entry->qp.flags & VMCI_QPFLAG_ATTACH_ONLY) {
1219 result = VMCI_ERROR_NOT_FOUND;
1223 result = qp_alloc_hypercall(queue_pair_entry);
1224 if (result < VMCI_SUCCESS) {
1225 pr_warn("qp_alloc_hypercall result = %d\n", result);
1230 qp_init_queue_mutex((struct vmci_queue *)my_produce_q,
1231 (struct vmci_queue *)my_consume_q);
1233 qp_list_add_entry(&qp_guest_endpoints, &queue_pair_entry->qp);
1236 queue_pair_entry->qp.ref_count++;
1237 *handle = queue_pair_entry->qp.handle;
1238 *produce_q = (struct vmci_queue *)my_produce_q;
1239 *consume_q = (struct vmci_queue *)my_consume_q;
1242 * We should initialize the queue pair header pages on a local
1243 * queue pair create. For non-local queue pairs, the
1244 * hypervisor initializes the header pages in the create step.
1246 if ((queue_pair_entry->qp.flags & VMCI_QPFLAG_LOCAL) &&
1247 queue_pair_entry->qp.ref_count == 1) {
1248 vmci_q_header_init((*produce_q)->q_header, *handle);
1249 vmci_q_header_init((*consume_q)->q_header, *handle);
1252 mutex_unlock(&qp_guest_endpoints.mutex);
1254 return VMCI_SUCCESS;
1257 mutex_unlock(&qp_guest_endpoints.mutex);
1258 if (queue_pair_entry) {
1259 /* The queues will be freed inside the destroy routine. */
1260 qp_guest_endpoint_destroy(queue_pair_entry);
1262 qp_free_queue(my_produce_q, produce_size);
1263 qp_free_queue(my_consume_q, consume_size);
1268 /* This path should only be used when an existing entry was found. */
1269 mutex_unlock(&qp_guest_endpoints.mutex);
1274 * The first endpoint issuing a queue pair allocation will create the state
1275 * of the queue pair in the queue pair broker.
1277 * If the creator is a guest, it will associate a VMX virtual address range
1278 * with the queue pair as specified by the page_store. For compatibility with
1279 * older VMX'en, that would use a separate step to set the VMX virtual
1280 * address range, the virtual address range can be registered later using
1281 * vmci_qp_broker_set_page_store. In that case, a page_store of NULL should be
1284 * If the creator is the host, a page_store of NULL should be used as well,
1285 * since the host is not able to supply a page store for the queue pair.
1287 * For older VMX and host callers, the queue pair will be created in the
1288 * VMCIQPB_CREATED_NO_MEM state, and for current VMX callers, it will be
1289 * created in VMCOQPB_CREATED_MEM state.
1291 static int qp_broker_create(struct vmci_handle handle,
1297 struct vmci_qp_page_store *page_store,
1298 struct vmci_ctx *context,
1299 vmci_event_release_cb wakeup_cb,
1300 void *client_data, struct qp_broker_entry **ent)
1302 struct qp_broker_entry *entry = NULL;
1303 const u32 context_id = vmci_ctx_get_id(context);
1304 bool is_local = flags & VMCI_QPFLAG_LOCAL;
1306 u64 guest_produce_size;
1307 u64 guest_consume_size;
1309 /* Do not create if the caller asked not to. */
1310 if (flags & VMCI_QPFLAG_ATTACH_ONLY)
1311 return VMCI_ERROR_NOT_FOUND;
1314 * Creator's context ID should match handle's context ID or the creator
1315 * must allow the context in handle's context ID as the "peer".
1317 if (handle.context != context_id && handle.context != peer)
1318 return VMCI_ERROR_NO_ACCESS;
1320 if (VMCI_CONTEXT_IS_VM(context_id) && VMCI_CONTEXT_IS_VM(peer))
1321 return VMCI_ERROR_DST_UNREACHABLE;
1324 * Creator's context ID for local queue pairs should match the
1325 * peer, if a peer is specified.
1327 if (is_local && peer != VMCI_INVALID_ID && context_id != peer)
1328 return VMCI_ERROR_NO_ACCESS;
1330 entry = kzalloc(sizeof(*entry), GFP_ATOMIC);
1332 return VMCI_ERROR_NO_MEM;
1334 if (vmci_ctx_get_id(context) == VMCI_HOST_CONTEXT_ID && !is_local) {
1336 * The queue pair broker entry stores values from the guest
1337 * point of view, so a creating host side endpoint should swap
1338 * produce and consume values -- unless it is a local queue
1339 * pair, in which case no swapping is necessary, since the local
1340 * attacher will swap queues.
1343 guest_produce_size = consume_size;
1344 guest_consume_size = produce_size;
1346 guest_produce_size = produce_size;
1347 guest_consume_size = consume_size;
1350 entry->qp.handle = handle;
1351 entry->qp.peer = peer;
1352 entry->qp.flags = flags;
1353 entry->qp.produce_size = guest_produce_size;
1354 entry->qp.consume_size = guest_consume_size;
1355 entry->qp.ref_count = 1;
1356 entry->create_id = context_id;
1357 entry->attach_id = VMCI_INVALID_ID;
1358 entry->state = VMCIQPB_NEW;
1359 entry->require_trusted_attach =
1360 !!(context->priv_flags & VMCI_PRIVILEGE_FLAG_RESTRICTED);
1361 entry->created_by_trusted =
1362 !!(priv_flags & VMCI_PRIVILEGE_FLAG_TRUSTED);
1363 entry->vmci_page_files = false;
1364 entry->wakeup_cb = wakeup_cb;
1365 entry->client_data = client_data;
1366 entry->produce_q = qp_host_alloc_queue(guest_produce_size);
1367 if (entry->produce_q == NULL) {
1368 result = VMCI_ERROR_NO_MEM;
1371 entry->consume_q = qp_host_alloc_queue(guest_consume_size);
1372 if (entry->consume_q == NULL) {
1373 result = VMCI_ERROR_NO_MEM;
1377 qp_init_queue_mutex(entry->produce_q, entry->consume_q);
1379 INIT_LIST_HEAD(&entry->qp.list_item);
1384 entry->local_mem = kcalloc(QPE_NUM_PAGES(entry->qp),
1385 PAGE_SIZE, GFP_KERNEL);
1386 if (entry->local_mem == NULL) {
1387 result = VMCI_ERROR_NO_MEM;
1390 entry->state = VMCIQPB_CREATED_MEM;
1391 entry->produce_q->q_header = entry->local_mem;
1392 tmp = (u8 *)entry->local_mem + PAGE_SIZE *
1393 (DIV_ROUND_UP(entry->qp.produce_size, PAGE_SIZE) + 1);
1394 entry->consume_q->q_header = (struct vmci_queue_header *)tmp;
1395 } else if (page_store) {
1397 * The VMX already initialized the queue pair headers, so no
1398 * need for the kernel side to do that.
1400 result = qp_host_register_user_memory(page_store,
1403 if (result < VMCI_SUCCESS)
1406 entry->state = VMCIQPB_CREATED_MEM;
1409 * A create without a page_store may be either a host
1410 * side create (in which case we are waiting for the
1411 * guest side to supply the memory) or an old style
1412 * queue pair create (in which case we will expect a
1413 * set page store call as the next step).
1415 entry->state = VMCIQPB_CREATED_NO_MEM;
1418 qp_list_add_entry(&qp_broker_list, &entry->qp);
1422 /* Add to resource obj */
1423 result = vmci_resource_add(&entry->resource,
1424 VMCI_RESOURCE_TYPE_QPAIR_HOST,
1426 if (result != VMCI_SUCCESS) {
1427 pr_warn("Failed to add new resource (handle=0x%x:0x%x), error: %d",
1428 handle.context, handle.resource, result);
1432 entry->qp.handle = vmci_resource_handle(&entry->resource);
1434 vmci_q_header_init(entry->produce_q->q_header,
1436 vmci_q_header_init(entry->consume_q->q_header,
1440 vmci_ctx_qp_create(context, entry->qp.handle);
1442 return VMCI_SUCCESS;
1445 if (entry != NULL) {
1446 qp_host_free_queue(entry->produce_q, guest_produce_size);
1447 qp_host_free_queue(entry->consume_q, guest_consume_size);
1455 * Enqueues an event datagram to notify the peer VM attached to
1456 * the given queue pair handle about attach/detach event by the
1457 * given VM. Returns Payload size of datagram enqueued on
1458 * success, error code otherwise.
1460 static int qp_notify_peer(bool attach,
1461 struct vmci_handle handle,
1466 struct vmci_event_qp ev;
1468 if (vmci_handle_is_invalid(handle) || my_id == VMCI_INVALID_ID ||
1469 peer_id == VMCI_INVALID_ID)
1470 return VMCI_ERROR_INVALID_ARGS;
1473 * In vmci_ctx_enqueue_datagram() we enforce the upper limit on
1474 * number of pending events from the hypervisor to a given VM
1475 * otherwise a rogue VM could do an arbitrary number of attach
1476 * and detach operations causing memory pressure in the host
1480 memset(&ev, 0, sizeof(ev));
1481 ev.msg.hdr.dst = vmci_make_handle(peer_id, VMCI_EVENT_HANDLER);
1482 ev.msg.hdr.src = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
1483 VMCI_CONTEXT_RESOURCE_ID);
1484 ev.msg.hdr.payload_size = sizeof(ev) - sizeof(ev.msg.hdr);
1485 ev.msg.event_data.event = attach ?
1486 VMCI_EVENT_QP_PEER_ATTACH : VMCI_EVENT_QP_PEER_DETACH;
1487 ev.payload.handle = handle;
1488 ev.payload.peer_id = my_id;
1490 rv = vmci_datagram_dispatch(VMCI_HYPERVISOR_CONTEXT_ID,
1491 &ev.msg.hdr, false);
1492 if (rv < VMCI_SUCCESS)
1493 pr_warn("Failed to enqueue queue_pair %s event datagram for context (ID=0x%x)\n",
1494 attach ? "ATTACH" : "DETACH", peer_id);
1500 * The second endpoint issuing a queue pair allocation will attach to
1501 * the queue pair registered with the queue pair broker.
1503 * If the attacher is a guest, it will associate a VMX virtual address
1504 * range with the queue pair as specified by the page_store. At this
1505 * point, the already attach host endpoint may start using the queue
1506 * pair, and an attach event is sent to it. For compatibility with
1507 * older VMX'en, that used a separate step to set the VMX virtual
1508 * address range, the virtual address range can be registered later
1509 * using vmci_qp_broker_set_page_store. In that case, a page_store of
1510 * NULL should be used, and the attach event will be generated once
1511 * the actual page store has been set.
1513 * If the attacher is the host, a page_store of NULL should be used as
1514 * well, since the page store information is already set by the guest.
1516 * For new VMX and host callers, the queue pair will be moved to the
1517 * VMCIQPB_ATTACHED_MEM state, and for older VMX callers, it will be
1518 * moved to the VMCOQPB_ATTACHED_NO_MEM state.
1520 static int qp_broker_attach(struct qp_broker_entry *entry,
1526 struct vmci_qp_page_store *page_store,
1527 struct vmci_ctx *context,
1528 vmci_event_release_cb wakeup_cb,
1530 struct qp_broker_entry **ent)
1532 const u32 context_id = vmci_ctx_get_id(context);
1533 bool is_local = flags & VMCI_QPFLAG_LOCAL;
1536 if (entry->state != VMCIQPB_CREATED_NO_MEM &&
1537 entry->state != VMCIQPB_CREATED_MEM)
1538 return VMCI_ERROR_UNAVAILABLE;
1541 if (!(entry->qp.flags & VMCI_QPFLAG_LOCAL) ||
1542 context_id != entry->create_id) {
1543 return VMCI_ERROR_INVALID_ARGS;
1545 } else if (context_id == entry->create_id ||
1546 context_id == entry->attach_id) {
1547 return VMCI_ERROR_ALREADY_EXISTS;
1550 if (VMCI_CONTEXT_IS_VM(context_id) &&
1551 VMCI_CONTEXT_IS_VM(entry->create_id))
1552 return VMCI_ERROR_DST_UNREACHABLE;
1555 * If we are attaching from a restricted context then the queuepair
1556 * must have been created by a trusted endpoint.
1558 if ((context->priv_flags & VMCI_PRIVILEGE_FLAG_RESTRICTED) &&
1559 !entry->created_by_trusted)
1560 return VMCI_ERROR_NO_ACCESS;
1563 * If we are attaching to a queuepair that was created by a restricted
1564 * context then we must be trusted.
1566 if (entry->require_trusted_attach &&
1567 (!(priv_flags & VMCI_PRIVILEGE_FLAG_TRUSTED)))
1568 return VMCI_ERROR_NO_ACCESS;
1571 * If the creator specifies VMCI_INVALID_ID in "peer" field, access
1572 * control check is not performed.
1574 if (entry->qp.peer != VMCI_INVALID_ID && entry->qp.peer != context_id)
1575 return VMCI_ERROR_NO_ACCESS;
1577 if (entry->create_id == VMCI_HOST_CONTEXT_ID) {
1579 * Do not attach if the caller doesn't support Host Queue Pairs
1580 * and a host created this queue pair.
1583 if (!vmci_ctx_supports_host_qp(context))
1584 return VMCI_ERROR_INVALID_RESOURCE;
1586 } else if (context_id == VMCI_HOST_CONTEXT_ID) {
1587 struct vmci_ctx *create_context;
1588 bool supports_host_qp;
1591 * Do not attach a host to a user created queue pair if that
1592 * user doesn't support host queue pair end points.
1595 create_context = vmci_ctx_get(entry->create_id);
1596 supports_host_qp = vmci_ctx_supports_host_qp(create_context);
1597 vmci_ctx_put(create_context);
1599 if (!supports_host_qp)
1600 return VMCI_ERROR_INVALID_RESOURCE;
1603 if ((entry->qp.flags & ~VMCI_QP_ASYMM) != (flags & ~VMCI_QP_ASYMM_PEER))
1604 return VMCI_ERROR_QUEUEPAIR_MISMATCH;
1606 if (context_id != VMCI_HOST_CONTEXT_ID) {
1608 * The queue pair broker entry stores values from the guest
1609 * point of view, so an attaching guest should match the values
1610 * stored in the entry.
1613 if (entry->qp.produce_size != produce_size ||
1614 entry->qp.consume_size != consume_size) {
1615 return VMCI_ERROR_QUEUEPAIR_MISMATCH;
1617 } else if (entry->qp.produce_size != consume_size ||
1618 entry->qp.consume_size != produce_size) {
1619 return VMCI_ERROR_QUEUEPAIR_MISMATCH;
1622 if (context_id != VMCI_HOST_CONTEXT_ID) {
1624 * If a guest attached to a queue pair, it will supply
1625 * the backing memory. If this is a pre NOVMVM vmx,
1626 * the backing memory will be supplied by calling
1627 * vmci_qp_broker_set_page_store() following the
1628 * return of the vmci_qp_broker_alloc() call. If it is
1629 * a vmx of version NOVMVM or later, the page store
1630 * must be supplied as part of the
1631 * vmci_qp_broker_alloc call. Under all circumstances
1632 * must the initially created queue pair not have any
1633 * memory associated with it already.
1636 if (entry->state != VMCIQPB_CREATED_NO_MEM)
1637 return VMCI_ERROR_INVALID_ARGS;
1639 if (page_store != NULL) {
1641 * Patch up host state to point to guest
1642 * supplied memory. The VMX already
1643 * initialized the queue pair headers, so no
1644 * need for the kernel side to do that.
1647 result = qp_host_register_user_memory(page_store,
1650 if (result < VMCI_SUCCESS)
1653 entry->state = VMCIQPB_ATTACHED_MEM;
1655 entry->state = VMCIQPB_ATTACHED_NO_MEM;
1657 } else if (entry->state == VMCIQPB_CREATED_NO_MEM) {
1659 * The host side is attempting to attach to a queue
1660 * pair that doesn't have any memory associated with
1661 * it. This must be a pre NOVMVM vmx that hasn't set
1662 * the page store information yet, or a quiesced VM.
1665 return VMCI_ERROR_UNAVAILABLE;
1667 /* The host side has successfully attached to a queue pair. */
1668 entry->state = VMCIQPB_ATTACHED_MEM;
1671 if (entry->state == VMCIQPB_ATTACHED_MEM) {
1673 qp_notify_peer(true, entry->qp.handle, context_id,
1675 if (result < VMCI_SUCCESS)
1676 pr_warn("Failed to notify peer (ID=0x%x) of attach to queue pair (handle=0x%x:0x%x)\n",
1677 entry->create_id, entry->qp.handle.context,
1678 entry->qp.handle.resource);
1681 entry->attach_id = context_id;
1682 entry->qp.ref_count++;
1684 entry->wakeup_cb = wakeup_cb;
1685 entry->client_data = client_data;
1689 * When attaching to local queue pairs, the context already has
1690 * an entry tracking the queue pair, so don't add another one.
1693 vmci_ctx_qp_create(context, entry->qp.handle);
1698 return VMCI_SUCCESS;
1702 * queue_pair_Alloc for use when setting up queue pair endpoints
1705 static int qp_broker_alloc(struct vmci_handle handle,
1711 struct vmci_qp_page_store *page_store,
1712 struct vmci_ctx *context,
1713 vmci_event_release_cb wakeup_cb,
1715 struct qp_broker_entry **ent,
1718 const u32 context_id = vmci_ctx_get_id(context);
1720 struct qp_broker_entry *entry = NULL;
1721 bool is_local = flags & VMCI_QPFLAG_LOCAL;
1724 if (vmci_handle_is_invalid(handle) ||
1725 (flags & ~VMCI_QP_ALL_FLAGS) || is_local ||
1726 !(produce_size || consume_size) ||
1727 !context || context_id == VMCI_INVALID_ID ||
1728 handle.context == VMCI_INVALID_ID) {
1729 return VMCI_ERROR_INVALID_ARGS;
1732 if (page_store && !VMCI_QP_PAGESTORE_IS_WELLFORMED(page_store))
1733 return VMCI_ERROR_INVALID_ARGS;
1736 * In the initial argument check, we ensure that non-vmkernel hosts
1737 * are not allowed to create local queue pairs.
1740 mutex_lock(&qp_broker_list.mutex);
1742 if (!is_local && vmci_ctx_qp_exists(context, handle)) {
1743 pr_devel("Context (ID=0x%x) already attached to queue pair (handle=0x%x:0x%x)\n",
1744 context_id, handle.context, handle.resource);
1745 mutex_unlock(&qp_broker_list.mutex);
1746 return VMCI_ERROR_ALREADY_EXISTS;
1749 if (handle.resource != VMCI_INVALID_ID)
1750 entry = qp_broker_handle_to_entry(handle);
1755 qp_broker_create(handle, peer, flags, priv_flags,
1756 produce_size, consume_size, page_store,
1757 context, wakeup_cb, client_data, ent);
1761 qp_broker_attach(entry, peer, flags, priv_flags,
1762 produce_size, consume_size, page_store,
1763 context, wakeup_cb, client_data, ent);
1766 mutex_unlock(&qp_broker_list.mutex);
1769 *swap = (context_id == VMCI_HOST_CONTEXT_ID) &&
1770 !(create && is_local);
1776 * This function implements the kernel API for allocating a queue
1779 static int qp_alloc_host_work(struct vmci_handle *handle,
1780 struct vmci_queue **produce_q,
1782 struct vmci_queue **consume_q,
1787 vmci_event_release_cb wakeup_cb,
1790 struct vmci_handle new_handle;
1791 struct vmci_ctx *context;
1792 struct qp_broker_entry *entry;
1796 if (vmci_handle_is_invalid(*handle)) {
1797 new_handle = vmci_make_handle(
1798 VMCI_HOST_CONTEXT_ID, VMCI_INVALID_ID);
1800 new_handle = *handle;
1802 context = vmci_ctx_get(VMCI_HOST_CONTEXT_ID);
1805 qp_broker_alloc(new_handle, peer, flags, priv_flags,
1806 produce_size, consume_size, NULL, context,
1807 wakeup_cb, client_data, &entry, &swap);
1808 if (result == VMCI_SUCCESS) {
1811 * If this is a local queue pair, the attacher
1812 * will swap around produce and consume
1816 *produce_q = entry->consume_q;
1817 *consume_q = entry->produce_q;
1819 *produce_q = entry->produce_q;
1820 *consume_q = entry->consume_q;
1823 *handle = vmci_resource_handle(&entry->resource);
1825 *handle = VMCI_INVALID_HANDLE;
1826 pr_devel("queue pair broker failed to alloc (result=%d)\n",
1829 vmci_ctx_put(context);
1834 * Allocates a VMCI queue_pair. Only checks validity of input
1835 * arguments. The real work is done in the host or guest
1836 * specific function.
1838 int vmci_qp_alloc(struct vmci_handle *handle,
1839 struct vmci_queue **produce_q,
1841 struct vmci_queue **consume_q,
1846 bool guest_endpoint,
1847 vmci_event_release_cb wakeup_cb,
1850 if (!handle || !produce_q || !consume_q ||
1851 (!produce_size && !consume_size) || (flags & ~VMCI_QP_ALL_FLAGS))
1852 return VMCI_ERROR_INVALID_ARGS;
1854 if (guest_endpoint) {
1855 return qp_alloc_guest_work(handle, produce_q,
1856 produce_size, consume_q,
1860 return qp_alloc_host_work(handle, produce_q,
1861 produce_size, consume_q,
1862 consume_size, peer, flags,
1863 priv_flags, wakeup_cb, client_data);
1868 * This function implements the host kernel API for detaching from
1871 static int qp_detatch_host_work(struct vmci_handle handle)
1874 struct vmci_ctx *context;
1876 context = vmci_ctx_get(VMCI_HOST_CONTEXT_ID);
1878 result = vmci_qp_broker_detach(handle, context);
1880 vmci_ctx_put(context);
1885 * Detaches from a VMCI queue_pair. Only checks validity of input argument.
1886 * Real work is done in the host or guest specific function.
1888 static int qp_detatch(struct vmci_handle handle, bool guest_endpoint)
1890 if (vmci_handle_is_invalid(handle))
1891 return VMCI_ERROR_INVALID_ARGS;
1894 return qp_detatch_guest_work(handle);
1896 return qp_detatch_host_work(handle);
1900 * Returns the entry from the head of the list. Assumes that the list is
1903 static struct qp_entry *qp_list_get_head(struct qp_list *qp_list)
1905 if (!list_empty(&qp_list->head)) {
1906 struct qp_entry *entry =
1907 list_first_entry(&qp_list->head, struct qp_entry,
1915 void vmci_qp_broker_exit(void)
1917 struct qp_entry *entry;
1918 struct qp_broker_entry *be;
1920 mutex_lock(&qp_broker_list.mutex);
1922 while ((entry = qp_list_get_head(&qp_broker_list))) {
1923 be = (struct qp_broker_entry *)entry;
1925 qp_list_remove_entry(&qp_broker_list, entry);
1929 mutex_unlock(&qp_broker_list.mutex);
1933 * Requests that a queue pair be allocated with the VMCI queue
1934 * pair broker. Allocates a queue pair entry if one does not
1935 * exist. Attaches to one if it exists, and retrieves the page
1936 * files backing that queue_pair. Assumes that the queue pair
1937 * broker lock is held.
1939 int vmci_qp_broker_alloc(struct vmci_handle handle,
1945 struct vmci_qp_page_store *page_store,
1946 struct vmci_ctx *context)
1948 return qp_broker_alloc(handle, peer, flags, priv_flags,
1949 produce_size, consume_size,
1950 page_store, context, NULL, NULL, NULL, NULL);
1954 * VMX'en with versions lower than VMCI_VERSION_NOVMVM use a separate
1955 * step to add the UVAs of the VMX mapping of the queue pair. This function
1956 * provides backwards compatibility with such VMX'en, and takes care of
1957 * registering the page store for a queue pair previously allocated by the
1958 * VMX during create or attach. This function will move the queue pair state
1959 * to either from VMCIQBP_CREATED_NO_MEM to VMCIQBP_CREATED_MEM or
1960 * VMCIQBP_ATTACHED_NO_MEM to VMCIQBP_ATTACHED_MEM. If moving to the
1961 * attached state with memory, the queue pair is ready to be used by the
1962 * host peer, and an attached event will be generated.
1964 * Assumes that the queue pair broker lock is held.
1966 * This function is only used by the hosted platform, since there is no
1967 * issue with backwards compatibility for vmkernel.
1969 int vmci_qp_broker_set_page_store(struct vmci_handle handle,
1972 struct vmci_ctx *context)
1974 struct qp_broker_entry *entry;
1976 const u32 context_id = vmci_ctx_get_id(context);
1978 if (vmci_handle_is_invalid(handle) || !context ||
1979 context_id == VMCI_INVALID_ID)
1980 return VMCI_ERROR_INVALID_ARGS;
1983 * We only support guest to host queue pairs, so the VMX must
1984 * supply UVAs for the mapped page files.
1987 if (produce_uva == 0 || consume_uva == 0)
1988 return VMCI_ERROR_INVALID_ARGS;
1990 mutex_lock(&qp_broker_list.mutex);
1992 if (!vmci_ctx_qp_exists(context, handle)) {
1993 pr_warn("Context (ID=0x%x) not attached to queue pair (handle=0x%x:0x%x)\n",
1994 context_id, handle.context, handle.resource);
1995 result = VMCI_ERROR_NOT_FOUND;
1999 entry = qp_broker_handle_to_entry(handle);
2001 result = VMCI_ERROR_NOT_FOUND;
2006 * If I'm the owner then I can set the page store.
2008 * Or, if a host created the queue_pair and I'm the attached peer
2009 * then I can set the page store.
2011 if (entry->create_id != context_id &&
2012 (entry->create_id != VMCI_HOST_CONTEXT_ID ||
2013 entry->attach_id != context_id)) {
2014 result = VMCI_ERROR_QUEUEPAIR_NOTOWNER;
2018 if (entry->state != VMCIQPB_CREATED_NO_MEM &&
2019 entry->state != VMCIQPB_ATTACHED_NO_MEM) {
2020 result = VMCI_ERROR_UNAVAILABLE;
2024 result = qp_host_get_user_memory(produce_uva, consume_uva,
2025 entry->produce_q, entry->consume_q);
2026 if (result < VMCI_SUCCESS)
2029 result = qp_host_map_queues(entry->produce_q, entry->consume_q);
2030 if (result < VMCI_SUCCESS) {
2031 qp_host_unregister_user_memory(entry->produce_q,
2036 if (entry->state == VMCIQPB_CREATED_NO_MEM)
2037 entry->state = VMCIQPB_CREATED_MEM;
2039 entry->state = VMCIQPB_ATTACHED_MEM;
2041 entry->vmci_page_files = true;
2043 if (entry->state == VMCIQPB_ATTACHED_MEM) {
2045 qp_notify_peer(true, handle, context_id, entry->create_id);
2046 if (result < VMCI_SUCCESS) {
2047 pr_warn("Failed to notify peer (ID=0x%x) of attach to queue pair (handle=0x%x:0x%x)\n",
2048 entry->create_id, entry->qp.handle.context,
2049 entry->qp.handle.resource);
2053 result = VMCI_SUCCESS;
2055 mutex_unlock(&qp_broker_list.mutex);
2060 * Resets saved queue headers for the given QP broker
2061 * entry. Should be used when guest memory becomes available
2062 * again, or the guest detaches.
2064 static void qp_reset_saved_headers(struct qp_broker_entry *entry)
2066 entry->produce_q->saved_header = NULL;
2067 entry->consume_q->saved_header = NULL;
2071 * The main entry point for detaching from a queue pair registered with the
2072 * queue pair broker. If more than one endpoint is attached to the queue
2073 * pair, the first endpoint will mainly decrement a reference count and
2074 * generate a notification to its peer. The last endpoint will clean up
2075 * the queue pair state registered with the broker.
2077 * When a guest endpoint detaches, it will unmap and unregister the guest
2078 * memory backing the queue pair. If the host is still attached, it will
2079 * no longer be able to access the queue pair content.
2081 * If the queue pair is already in a state where there is no memory
2082 * registered for the queue pair (any *_NO_MEM state), it will transition to
2083 * the VMCIQPB_SHUTDOWN_NO_MEM state. This will also happen, if a guest
2084 * endpoint is the first of two endpoints to detach. If the host endpoint is
2085 * the first out of two to detach, the queue pair will move to the
2086 * VMCIQPB_SHUTDOWN_MEM state.
2088 int vmci_qp_broker_detach(struct vmci_handle handle, struct vmci_ctx *context)
2090 struct qp_broker_entry *entry;
2091 const u32 context_id = vmci_ctx_get_id(context);
2093 bool is_local = false;
2096 if (vmci_handle_is_invalid(handle) || !context ||
2097 context_id == VMCI_INVALID_ID) {
2098 return VMCI_ERROR_INVALID_ARGS;
2101 mutex_lock(&qp_broker_list.mutex);
2103 if (!vmci_ctx_qp_exists(context, handle)) {
2104 pr_devel("Context (ID=0x%x) not attached to queue pair (handle=0x%x:0x%x)\n",
2105 context_id, handle.context, handle.resource);
2106 result = VMCI_ERROR_NOT_FOUND;
2110 entry = qp_broker_handle_to_entry(handle);
2112 pr_devel("Context (ID=0x%x) reports being attached to queue pair(handle=0x%x:0x%x) that isn't present in broker\n",
2113 context_id, handle.context, handle.resource);
2114 result = VMCI_ERROR_NOT_FOUND;
2118 if (context_id != entry->create_id && context_id != entry->attach_id) {
2119 result = VMCI_ERROR_QUEUEPAIR_NOTATTACHED;
2123 if (context_id == entry->create_id) {
2124 peer_id = entry->attach_id;
2125 entry->create_id = VMCI_INVALID_ID;
2127 peer_id = entry->create_id;
2128 entry->attach_id = VMCI_INVALID_ID;
2130 entry->qp.ref_count--;
2132 is_local = entry->qp.flags & VMCI_QPFLAG_LOCAL;
2134 if (context_id != VMCI_HOST_CONTEXT_ID) {
2135 bool headers_mapped;
2138 * Pre NOVMVM vmx'en may detach from a queue pair
2139 * before setting the page store, and in that case
2140 * there is no user memory to detach from. Also, more
2141 * recent VMX'en may detach from a queue pair in the
2145 qp_acquire_queue_mutex(entry->produce_q);
2146 headers_mapped = entry->produce_q->q_header ||
2147 entry->consume_q->q_header;
2148 if (QPBROKERSTATE_HAS_MEM(entry)) {
2150 qp_host_unmap_queues(INVALID_VMCI_GUEST_MEM_ID,
2153 if (result < VMCI_SUCCESS)
2154 pr_warn("Failed to unmap queue headers for queue pair (handle=0x%x:0x%x,result=%d)\n",
2155 handle.context, handle.resource,
2158 qp_host_unregister_user_memory(entry->produce_q,
2163 if (!headers_mapped)
2164 qp_reset_saved_headers(entry);
2166 qp_release_queue_mutex(entry->produce_q);
2168 if (!headers_mapped && entry->wakeup_cb)
2169 entry->wakeup_cb(entry->client_data);
2172 if (entry->wakeup_cb) {
2173 entry->wakeup_cb = NULL;
2174 entry->client_data = NULL;
2178 if (entry->qp.ref_count == 0) {
2179 qp_list_remove_entry(&qp_broker_list, &entry->qp);
2182 kfree(entry->local_mem);
2184 qp_cleanup_queue_mutex(entry->produce_q, entry->consume_q);
2185 qp_host_free_queue(entry->produce_q, entry->qp.produce_size);
2186 qp_host_free_queue(entry->consume_q, entry->qp.consume_size);
2187 /* Unlink from resource hash table and free callback */
2188 vmci_resource_remove(&entry->resource);
2192 vmci_ctx_qp_destroy(context, handle);
2194 qp_notify_peer(false, handle, context_id, peer_id);
2195 if (context_id == VMCI_HOST_CONTEXT_ID &&
2196 QPBROKERSTATE_HAS_MEM(entry)) {
2197 entry->state = VMCIQPB_SHUTDOWN_MEM;
2199 entry->state = VMCIQPB_SHUTDOWN_NO_MEM;
2203 vmci_ctx_qp_destroy(context, handle);
2206 result = VMCI_SUCCESS;
2208 mutex_unlock(&qp_broker_list.mutex);
2213 * Establishes the necessary mappings for a queue pair given a
2214 * reference to the queue pair guest memory. This is usually
2215 * called when a guest is unquiesced and the VMX is allowed to
2216 * map guest memory once again.
2218 int vmci_qp_broker_map(struct vmci_handle handle,
2219 struct vmci_ctx *context,
2222 struct qp_broker_entry *entry;
2223 const u32 context_id = vmci_ctx_get_id(context);
2226 if (vmci_handle_is_invalid(handle) || !context ||
2227 context_id == VMCI_INVALID_ID)
2228 return VMCI_ERROR_INVALID_ARGS;
2230 mutex_lock(&qp_broker_list.mutex);
2232 if (!vmci_ctx_qp_exists(context, handle)) {
2233 pr_devel("Context (ID=0x%x) not attached to queue pair (handle=0x%x:0x%x)\n",
2234 context_id, handle.context, handle.resource);
2235 result = VMCI_ERROR_NOT_FOUND;
2239 entry = qp_broker_handle_to_entry(handle);
2241 pr_devel("Context (ID=0x%x) reports being attached to queue pair (handle=0x%x:0x%x) that isn't present in broker\n",
2242 context_id, handle.context, handle.resource);
2243 result = VMCI_ERROR_NOT_FOUND;
2247 if (context_id != entry->create_id && context_id != entry->attach_id) {
2248 result = VMCI_ERROR_QUEUEPAIR_NOTATTACHED;
2252 result = VMCI_SUCCESS;
2254 if (context_id != VMCI_HOST_CONTEXT_ID &&
2255 !QPBROKERSTATE_HAS_MEM(entry)) {
2256 struct vmci_qp_page_store page_store;
2258 page_store.pages = guest_mem;
2259 page_store.len = QPE_NUM_PAGES(entry->qp);
2261 qp_acquire_queue_mutex(entry->produce_q);
2262 qp_reset_saved_headers(entry);
2264 qp_host_register_user_memory(&page_store,
2267 qp_release_queue_mutex(entry->produce_q);
2268 if (result == VMCI_SUCCESS) {
2269 /* Move state from *_NO_MEM to *_MEM */
2273 if (entry->wakeup_cb)
2274 entry->wakeup_cb(entry->client_data);
2279 mutex_unlock(&qp_broker_list.mutex);
2284 * Saves a snapshot of the queue headers for the given QP broker
2285 * entry. Should be used when guest memory is unmapped.
2287 * VMCI_SUCCESS on success, appropriate error code if guest memory
2288 * can't be accessed..
2290 static int qp_save_headers(struct qp_broker_entry *entry)
2294 if (entry->produce_q->saved_header != NULL &&
2295 entry->consume_q->saved_header != NULL) {
2297 * If the headers have already been saved, we don't need to do
2298 * it again, and we don't want to map in the headers
2302 return VMCI_SUCCESS;
2305 if (NULL == entry->produce_q->q_header ||
2306 NULL == entry->consume_q->q_header) {
2307 result = qp_host_map_queues(entry->produce_q, entry->consume_q);
2308 if (result < VMCI_SUCCESS)
2312 memcpy(&entry->saved_produce_q, entry->produce_q->q_header,
2313 sizeof(entry->saved_produce_q));
2314 entry->produce_q->saved_header = &entry->saved_produce_q;
2315 memcpy(&entry->saved_consume_q, entry->consume_q->q_header,
2316 sizeof(entry->saved_consume_q));
2317 entry->consume_q->saved_header = &entry->saved_consume_q;
2319 return VMCI_SUCCESS;
2323 * Removes all references to the guest memory of a given queue pair, and
2324 * will move the queue pair from state *_MEM to *_NO_MEM. It is usually
2325 * called when a VM is being quiesced where access to guest memory should
2328 int vmci_qp_broker_unmap(struct vmci_handle handle,
2329 struct vmci_ctx *context,
2332 struct qp_broker_entry *entry;
2333 const u32 context_id = vmci_ctx_get_id(context);
2336 if (vmci_handle_is_invalid(handle) || !context ||
2337 context_id == VMCI_INVALID_ID)
2338 return VMCI_ERROR_INVALID_ARGS;
2340 mutex_lock(&qp_broker_list.mutex);
2342 if (!vmci_ctx_qp_exists(context, handle)) {
2343 pr_devel("Context (ID=0x%x) not attached to queue pair (handle=0x%x:0x%x)\n",
2344 context_id, handle.context, handle.resource);
2345 result = VMCI_ERROR_NOT_FOUND;
2349 entry = qp_broker_handle_to_entry(handle);
2351 pr_devel("Context (ID=0x%x) reports being attached to queue pair (handle=0x%x:0x%x) that isn't present in broker\n",
2352 context_id, handle.context, handle.resource);
2353 result = VMCI_ERROR_NOT_FOUND;
2357 if (context_id != entry->create_id && context_id != entry->attach_id) {
2358 result = VMCI_ERROR_QUEUEPAIR_NOTATTACHED;
2362 if (context_id != VMCI_HOST_CONTEXT_ID &&
2363 QPBROKERSTATE_HAS_MEM(entry)) {
2364 qp_acquire_queue_mutex(entry->produce_q);
2365 result = qp_save_headers(entry);
2366 if (result < VMCI_SUCCESS)
2367 pr_warn("Failed to save queue headers for queue pair (handle=0x%x:0x%x,result=%d)\n",
2368 handle.context, handle.resource, result);
2370 qp_host_unmap_queues(gid, entry->produce_q, entry->consume_q);
2373 * On hosted, when we unmap queue pairs, the VMX will also
2374 * unmap the guest memory, so we invalidate the previously
2375 * registered memory. If the queue pair is mapped again at a
2376 * later point in time, we will need to reregister the user
2377 * memory with a possibly new user VA.
2379 qp_host_unregister_user_memory(entry->produce_q,
2383 * Move state from *_MEM to *_NO_MEM.
2387 qp_release_queue_mutex(entry->produce_q);
2390 result = VMCI_SUCCESS;
2393 mutex_unlock(&qp_broker_list.mutex);
2398 * Destroys all guest queue pair endpoints. If active guest queue
2399 * pairs still exist, hypercalls to attempt detach from these
2400 * queue pairs will be made. Any failure to detach is silently
2403 void vmci_qp_guest_endpoints_exit(void)
2405 struct qp_entry *entry;
2406 struct qp_guest_endpoint *ep;
2408 mutex_lock(&qp_guest_endpoints.mutex);
2410 while ((entry = qp_list_get_head(&qp_guest_endpoints))) {
2411 ep = (struct qp_guest_endpoint *)entry;
2413 /* Don't make a hypercall for local queue_pairs. */
2414 if (!(entry->flags & VMCI_QPFLAG_LOCAL))
2415 qp_detatch_hypercall(entry->handle);
2417 /* We cannot fail the exit, so let's reset ref_count. */
2418 entry->ref_count = 0;
2419 qp_list_remove_entry(&qp_guest_endpoints, entry);
2421 qp_guest_endpoint_destroy(ep);
2424 mutex_unlock(&qp_guest_endpoints.mutex);
2428 * Helper routine that will lock the queue pair before subsequent
2430 * Note: Non-blocking on the host side is currently only implemented in ESX.
2431 * Since non-blocking isn't yet implemented on the host personality we
2432 * have no reason to acquire a spin lock. So to avoid the use of an
2433 * unnecessary lock only acquire the mutex if we can block.
2435 static void qp_lock(const struct vmci_qp *qpair)
2437 qp_acquire_queue_mutex(qpair->produce_q);
2441 * Helper routine that unlocks the queue pair after calling
2444 static void qp_unlock(const struct vmci_qp *qpair)
2446 qp_release_queue_mutex(qpair->produce_q);
2450 * The queue headers may not be mapped at all times. If a queue is
2451 * currently not mapped, it will be attempted to do so.
2453 static int qp_map_queue_headers(struct vmci_queue *produce_q,
2454 struct vmci_queue *consume_q)
2458 if (NULL == produce_q->q_header || NULL == consume_q->q_header) {
2459 result = qp_host_map_queues(produce_q, consume_q);
2460 if (result < VMCI_SUCCESS)
2461 return (produce_q->saved_header &&
2462 consume_q->saved_header) ?
2463 VMCI_ERROR_QUEUEPAIR_NOT_READY :
2464 VMCI_ERROR_QUEUEPAIR_NOTATTACHED;
2467 return VMCI_SUCCESS;
2471 * Helper routine that will retrieve the produce and consume
2472 * headers of a given queue pair. If the guest memory of the
2473 * queue pair is currently not available, the saved queue headers
2474 * will be returned, if these are available.
2476 static int qp_get_queue_headers(const struct vmci_qp *qpair,
2477 struct vmci_queue_header **produce_q_header,
2478 struct vmci_queue_header **consume_q_header)
2482 result = qp_map_queue_headers(qpair->produce_q, qpair->consume_q);
2483 if (result == VMCI_SUCCESS) {
2484 *produce_q_header = qpair->produce_q->q_header;
2485 *consume_q_header = qpair->consume_q->q_header;
2486 } else if (qpair->produce_q->saved_header &&
2487 qpair->consume_q->saved_header) {
2488 *produce_q_header = qpair->produce_q->saved_header;
2489 *consume_q_header = qpair->consume_q->saved_header;
2490 result = VMCI_SUCCESS;
2497 * Callback from VMCI queue pair broker indicating that a queue
2498 * pair that was previously not ready, now either is ready or
2501 static int qp_wakeup_cb(void *client_data)
2503 struct vmci_qp *qpair = (struct vmci_qp *)client_data;
2506 while (qpair->blocked > 0) {
2508 qpair->generation++;
2509 wake_up(&qpair->event);
2513 return VMCI_SUCCESS;
2517 * Makes the calling thread wait for the queue pair to become
2518 * ready for host side access. Returns true when thread is
2519 * woken up after queue pair state change, false otherwise.
2521 static bool qp_wait_for_ready_queue(struct vmci_qp *qpair)
2523 unsigned int generation;
2526 generation = qpair->generation;
2528 wait_event(qpair->event, generation != qpair->generation);
2535 * Enqueues a given buffer to the produce queue using the provided
2536 * function. As many bytes as possible (space available in the queue)
2537 * are enqueued. Assumes the queue->mutex has been acquired. Returns
2538 * VMCI_ERROR_QUEUEPAIR_NOSPACE if no space was available to enqueue
2539 * data, VMCI_ERROR_INVALID_SIZE, if any queue pointer is outside the
2540 * queue (as defined by the queue size), VMCI_ERROR_INVALID_ARGS, if
2541 * an error occured when accessing the buffer,
2542 * VMCI_ERROR_QUEUEPAIR_NOTATTACHED, if the queue pair pages aren't
2543 * available. Otherwise, the number of bytes written to the queue is
2544 * returned. Updates the tail pointer of the produce queue.
2546 static ssize_t qp_enqueue_locked(struct vmci_queue *produce_q,
2547 struct vmci_queue *consume_q,
2548 const u64 produce_q_size,
2549 struct iov_iter *from)
2553 size_t buf_size = iov_iter_count(from);
2557 result = qp_map_queue_headers(produce_q, consume_q);
2558 if (unlikely(result != VMCI_SUCCESS))
2561 free_space = vmci_q_header_free_space(produce_q->q_header,
2562 consume_q->q_header,
2564 if (free_space == 0)
2565 return VMCI_ERROR_QUEUEPAIR_NOSPACE;
2567 if (free_space < VMCI_SUCCESS)
2568 return (ssize_t) free_space;
2570 written = (size_t) (free_space > buf_size ? buf_size : free_space);
2571 tail = vmci_q_header_producer_tail(produce_q->q_header);
2572 if (likely(tail + written < produce_q_size)) {
2573 result = qp_memcpy_to_queue_iter(produce_q, tail, from, written);
2575 /* Tail pointer wraps around. */
2577 const size_t tmp = (size_t) (produce_q_size - tail);
2579 result = qp_memcpy_to_queue_iter(produce_q, tail, from, tmp);
2580 if (result >= VMCI_SUCCESS)
2581 result = qp_memcpy_to_queue_iter(produce_q, 0, from,
2585 if (result < VMCI_SUCCESS)
2588 vmci_q_header_add_producer_tail(produce_q->q_header, written,
2594 * Dequeues data (if available) from the given consume queue. Writes data
2595 * to the user provided buffer using the provided function.
2596 * Assumes the queue->mutex has been acquired.
2598 * VMCI_ERROR_QUEUEPAIR_NODATA if no data was available to dequeue.
2599 * VMCI_ERROR_INVALID_SIZE, if any queue pointer is outside the queue
2600 * (as defined by the queue size).
2601 * VMCI_ERROR_INVALID_ARGS, if an error occured when accessing the buffer.
2602 * Otherwise the number of bytes dequeued is returned.
2604 * Updates the head pointer of the consume queue.
2606 static ssize_t qp_dequeue_locked(struct vmci_queue *produce_q,
2607 struct vmci_queue *consume_q,
2608 const u64 consume_q_size,
2609 struct iov_iter *to,
2610 bool update_consumer)
2612 size_t buf_size = iov_iter_count(to);
2618 result = qp_map_queue_headers(produce_q, consume_q);
2619 if (unlikely(result != VMCI_SUCCESS))
2622 buf_ready = vmci_q_header_buf_ready(consume_q->q_header,
2623 produce_q->q_header,
2626 return VMCI_ERROR_QUEUEPAIR_NODATA;
2628 if (buf_ready < VMCI_SUCCESS)
2629 return (ssize_t) buf_ready;
2631 read = (size_t) (buf_ready > buf_size ? buf_size : buf_ready);
2632 head = vmci_q_header_consumer_head(produce_q->q_header);
2633 if (likely(head + read < consume_q_size)) {
2634 result = qp_memcpy_from_queue_iter(to, consume_q, head, read);
2636 /* Head pointer wraps around. */
2638 const size_t tmp = (size_t) (consume_q_size - head);
2640 result = qp_memcpy_from_queue_iter(to, consume_q, head, tmp);
2641 if (result >= VMCI_SUCCESS)
2642 result = qp_memcpy_from_queue_iter(to, consume_q, 0,
2647 if (result < VMCI_SUCCESS)
2650 if (update_consumer)
2651 vmci_q_header_add_consumer_head(produce_q->q_header,
2652 read, consume_q_size);
2658 * vmci_qpair_alloc() - Allocates a queue pair.
2659 * @qpair: Pointer for the new vmci_qp struct.
2660 * @handle: Handle to track the resource.
2661 * @produce_qsize: Desired size of the producer queue.
2662 * @consume_qsize: Desired size of the consumer queue.
2663 * @peer: ContextID of the peer.
2664 * @flags: VMCI flags.
2665 * @priv_flags: VMCI priviledge flags.
2667 * This is the client interface for allocating the memory for a
2668 * vmci_qp structure and then attaching to the underlying
2669 * queue. If an error occurs allocating the memory for the
2670 * vmci_qp structure no attempt is made to attach. If an
2671 * error occurs attaching, then the structure is freed.
2673 int vmci_qpair_alloc(struct vmci_qp **qpair,
2674 struct vmci_handle *handle,
2681 struct vmci_qp *my_qpair;
2683 struct vmci_handle src = VMCI_INVALID_HANDLE;
2684 struct vmci_handle dst = vmci_make_handle(peer, VMCI_INVALID_ID);
2685 enum vmci_route route;
2686 vmci_event_release_cb wakeup_cb;
2690 * Restrict the size of a queuepair. The device already
2691 * enforces a limit on the total amount of memory that can be
2692 * allocated to queuepairs for a guest. However, we try to
2693 * allocate this memory before we make the queuepair
2694 * allocation hypercall. On Linux, we allocate each page
2695 * separately, which means rather than fail, the guest will
2696 * thrash while it tries to allocate, and will become
2697 * increasingly unresponsive to the point where it appears to
2698 * be hung. So we place a limit on the size of an individual
2699 * queuepair here, and leave the device to enforce the
2700 * restriction on total queuepair memory. (Note that this
2701 * doesn't prevent all cases; a user with only this much
2702 * physical memory could still get into trouble.) The error
2703 * used by the device is NO_RESOURCES, so use that here too.
2706 if (produce_qsize + consume_qsize < max(produce_qsize, consume_qsize) ||
2707 produce_qsize + consume_qsize > VMCI_MAX_GUEST_QP_MEMORY)
2708 return VMCI_ERROR_NO_RESOURCES;
2710 retval = vmci_route(&src, &dst, false, &route);
2711 if (retval < VMCI_SUCCESS)
2712 route = vmci_guest_code_active() ?
2713 VMCI_ROUTE_AS_GUEST : VMCI_ROUTE_AS_HOST;
2715 if (flags & (VMCI_QPFLAG_NONBLOCK | VMCI_QPFLAG_PINNED)) {
2716 pr_devel("NONBLOCK OR PINNED set");
2717 return VMCI_ERROR_INVALID_ARGS;
2720 my_qpair = kzalloc(sizeof(*my_qpair), GFP_KERNEL);
2722 return VMCI_ERROR_NO_MEM;
2724 my_qpair->produce_q_size = produce_qsize;
2725 my_qpair->consume_q_size = consume_qsize;
2726 my_qpair->peer = peer;
2727 my_qpair->flags = flags;
2728 my_qpair->priv_flags = priv_flags;
2733 if (VMCI_ROUTE_AS_HOST == route) {
2734 my_qpair->guest_endpoint = false;
2735 if (!(flags & VMCI_QPFLAG_LOCAL)) {
2736 my_qpair->blocked = 0;
2737 my_qpair->generation = 0;
2738 init_waitqueue_head(&my_qpair->event);
2739 wakeup_cb = qp_wakeup_cb;
2740 client_data = (void *)my_qpair;
2743 my_qpair->guest_endpoint = true;
2746 retval = vmci_qp_alloc(handle,
2747 &my_qpair->produce_q,
2748 my_qpair->produce_q_size,
2749 &my_qpair->consume_q,
2750 my_qpair->consume_q_size,
2753 my_qpair->priv_flags,
2754 my_qpair->guest_endpoint,
2755 wakeup_cb, client_data);
2757 if (retval < VMCI_SUCCESS) {
2763 my_qpair->handle = *handle;
2767 EXPORT_SYMBOL_GPL(vmci_qpair_alloc);
2770 * vmci_qpair_detach() - Detatches the client from a queue pair.
2771 * @qpair: Reference of a pointer to the qpair struct.
2773 * This is the client interface for detaching from a VMCIQPair.
2774 * Note that this routine will free the memory allocated for the
2775 * vmci_qp structure too.
2777 int vmci_qpair_detach(struct vmci_qp **qpair)
2780 struct vmci_qp *old_qpair;
2782 if (!qpair || !(*qpair))
2783 return VMCI_ERROR_INVALID_ARGS;
2786 result = qp_detatch(old_qpair->handle, old_qpair->guest_endpoint);
2789 * The guest can fail to detach for a number of reasons, and
2790 * if it does so, it will cleanup the entry (if there is one).
2791 * The host can fail too, but it won't cleanup the entry
2792 * immediately, it will do that later when the context is
2793 * freed. Either way, we need to release the qpair struct
2794 * here; there isn't much the caller can do, and we don't want
2798 memset(old_qpair, 0, sizeof(*old_qpair));
2799 old_qpair->handle = VMCI_INVALID_HANDLE;
2800 old_qpair->peer = VMCI_INVALID_ID;
2806 EXPORT_SYMBOL_GPL(vmci_qpair_detach);
2809 * vmci_qpair_get_produce_indexes() - Retrieves the indexes of the producer.
2810 * @qpair: Pointer to the queue pair struct.
2811 * @producer_tail: Reference used for storing producer tail index.
2812 * @consumer_head: Reference used for storing the consumer head index.
2814 * This is the client interface for getting the current indexes of the
2815 * QPair from the point of the view of the caller as the producer.
2817 int vmci_qpair_get_produce_indexes(const struct vmci_qp *qpair,
2821 struct vmci_queue_header *produce_q_header;
2822 struct vmci_queue_header *consume_q_header;
2826 return VMCI_ERROR_INVALID_ARGS;
2830 qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
2831 if (result == VMCI_SUCCESS)
2832 vmci_q_header_get_pointers(produce_q_header, consume_q_header,
2833 producer_tail, consumer_head);
2836 if (result == VMCI_SUCCESS &&
2837 ((producer_tail && *producer_tail >= qpair->produce_q_size) ||
2838 (consumer_head && *consumer_head >= qpair->produce_q_size)))
2839 return VMCI_ERROR_INVALID_SIZE;
2843 EXPORT_SYMBOL_GPL(vmci_qpair_get_produce_indexes);
2846 * vmci_qpair_get_consume_indexes() - Retrieves the indexes of the consumer.
2847 * @qpair: Pointer to the queue pair struct.
2848 * @consumer_tail: Reference used for storing consumer tail index.
2849 * @producer_head: Reference used for storing the producer head index.
2851 * This is the client interface for getting the current indexes of the
2852 * QPair from the point of the view of the caller as the consumer.
2854 int vmci_qpair_get_consume_indexes(const struct vmci_qp *qpair,
2858 struct vmci_queue_header *produce_q_header;
2859 struct vmci_queue_header *consume_q_header;
2863 return VMCI_ERROR_INVALID_ARGS;
2867 qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
2868 if (result == VMCI_SUCCESS)
2869 vmci_q_header_get_pointers(consume_q_header, produce_q_header,
2870 consumer_tail, producer_head);
2873 if (result == VMCI_SUCCESS &&
2874 ((consumer_tail && *consumer_tail >= qpair->consume_q_size) ||
2875 (producer_head && *producer_head >= qpair->consume_q_size)))
2876 return VMCI_ERROR_INVALID_SIZE;
2880 EXPORT_SYMBOL_GPL(vmci_qpair_get_consume_indexes);
2883 * vmci_qpair_produce_free_space() - Retrieves free space in producer queue.
2884 * @qpair: Pointer to the queue pair struct.
2886 * This is the client interface for getting the amount of free
2887 * space in the QPair from the point of the view of the caller as
2888 * the producer which is the common case. Returns < 0 if err, else
2889 * available bytes into which data can be enqueued if > 0.
2891 s64 vmci_qpair_produce_free_space(const struct vmci_qp *qpair)
2893 struct vmci_queue_header *produce_q_header;
2894 struct vmci_queue_header *consume_q_header;
2898 return VMCI_ERROR_INVALID_ARGS;
2902 qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
2903 if (result == VMCI_SUCCESS)
2904 result = vmci_q_header_free_space(produce_q_header,
2906 qpair->produce_q_size);
2914 EXPORT_SYMBOL_GPL(vmci_qpair_produce_free_space);
2917 * vmci_qpair_consume_free_space() - Retrieves free space in consumer queue.
2918 * @qpair: Pointer to the queue pair struct.
2920 * This is the client interface for getting the amount of free
2921 * space in the QPair from the point of the view of the caller as
2922 * the consumer which is not the common case. Returns < 0 if err, else
2923 * available bytes into which data can be enqueued if > 0.
2925 s64 vmci_qpair_consume_free_space(const struct vmci_qp *qpair)
2927 struct vmci_queue_header *produce_q_header;
2928 struct vmci_queue_header *consume_q_header;
2932 return VMCI_ERROR_INVALID_ARGS;
2936 qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
2937 if (result == VMCI_SUCCESS)
2938 result = vmci_q_header_free_space(consume_q_header,
2940 qpair->consume_q_size);
2948 EXPORT_SYMBOL_GPL(vmci_qpair_consume_free_space);
2951 * vmci_qpair_produce_buf_ready() - Gets bytes ready to read from
2953 * @qpair: Pointer to the queue pair struct.
2955 * This is the client interface for getting the amount of
2956 * enqueued data in the QPair from the point of the view of the
2957 * caller as the producer which is not the common case. Returns < 0 if err,
2958 * else available bytes that may be read.
2960 s64 vmci_qpair_produce_buf_ready(const struct vmci_qp *qpair)
2962 struct vmci_queue_header *produce_q_header;
2963 struct vmci_queue_header *consume_q_header;
2967 return VMCI_ERROR_INVALID_ARGS;
2971 qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
2972 if (result == VMCI_SUCCESS)
2973 result = vmci_q_header_buf_ready(produce_q_header,
2975 qpair->produce_q_size);
2983 EXPORT_SYMBOL_GPL(vmci_qpair_produce_buf_ready);
2986 * vmci_qpair_consume_buf_ready() - Gets bytes ready to read from
2988 * @qpair: Pointer to the queue pair struct.
2990 * This is the client interface for getting the amount of
2991 * enqueued data in the QPair from the point of the view of the
2992 * caller as the consumer which is the normal case. Returns < 0 if err,
2993 * else available bytes that may be read.
2995 s64 vmci_qpair_consume_buf_ready(const struct vmci_qp *qpair)
2997 struct vmci_queue_header *produce_q_header;
2998 struct vmci_queue_header *consume_q_header;
3002 return VMCI_ERROR_INVALID_ARGS;
3006 qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
3007 if (result == VMCI_SUCCESS)
3008 result = vmci_q_header_buf_ready(consume_q_header,
3010 qpair->consume_q_size);
3018 EXPORT_SYMBOL_GPL(vmci_qpair_consume_buf_ready);
3021 * vmci_qpair_enqueue() - Throw data on the queue.
3022 * @qpair: Pointer to the queue pair struct.
3023 * @buf: Pointer to buffer containing data
3024 * @buf_size: Length of buffer.
3025 * @buf_type: Buffer type (Unused).
3027 * This is the client interface for enqueueing data into the queue.
3028 * Returns number of bytes enqueued or < 0 on error.
3030 ssize_t vmci_qpair_enqueue(struct vmci_qp *qpair,
3036 struct iov_iter from;
3037 struct kvec v = {.iov_base = (void *)buf, .iov_len = buf_size};
3040 return VMCI_ERROR_INVALID_ARGS;
3042 iov_iter_kvec(&from, WRITE | ITER_KVEC, &v, 1, buf_size);
3047 result = qp_enqueue_locked(qpair->produce_q,
3049 qpair->produce_q_size,
3052 if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
3053 !qp_wait_for_ready_queue(qpair))
3054 result = VMCI_ERROR_WOULD_BLOCK;
3056 } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
3062 EXPORT_SYMBOL_GPL(vmci_qpair_enqueue);
3065 * vmci_qpair_dequeue() - Get data from the queue.
3066 * @qpair: Pointer to the queue pair struct.
3067 * @buf: Pointer to buffer for the data
3068 * @buf_size: Length of buffer.
3069 * @buf_type: Buffer type (Unused).
3071 * This is the client interface for dequeueing data from the queue.
3072 * Returns number of bytes dequeued or < 0 on error.
3074 ssize_t vmci_qpair_dequeue(struct vmci_qp *qpair,
3081 struct kvec v = {.iov_base = buf, .iov_len = buf_size};
3084 return VMCI_ERROR_INVALID_ARGS;
3086 iov_iter_kvec(&to, READ | ITER_KVEC, &v, 1, buf_size);
3091 result = qp_dequeue_locked(qpair->produce_q,
3093 qpair->consume_q_size,
3096 if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
3097 !qp_wait_for_ready_queue(qpair))
3098 result = VMCI_ERROR_WOULD_BLOCK;
3100 } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
3106 EXPORT_SYMBOL_GPL(vmci_qpair_dequeue);
3109 * vmci_qpair_peek() - Peek at the data in the queue.
3110 * @qpair: Pointer to the queue pair struct.
3111 * @buf: Pointer to buffer for the data
3112 * @buf_size: Length of buffer.
3113 * @buf_type: Buffer type (Unused on Linux).
3115 * This is the client interface for peeking into a queue. (I.e.,
3116 * copy data from the queue without updating the head pointer.)
3117 * Returns number of bytes dequeued or < 0 on error.
3119 ssize_t vmci_qpair_peek(struct vmci_qp *qpair,
3125 struct kvec v = {.iov_base = buf, .iov_len = buf_size};
3129 return VMCI_ERROR_INVALID_ARGS;
3131 iov_iter_kvec(&to, READ | ITER_KVEC, &v, 1, buf_size);
3136 result = qp_dequeue_locked(qpair->produce_q,
3138 qpair->consume_q_size,
3141 if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
3142 !qp_wait_for_ready_queue(qpair))
3143 result = VMCI_ERROR_WOULD_BLOCK;
3145 } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
3151 EXPORT_SYMBOL_GPL(vmci_qpair_peek);
3154 * vmci_qpair_enquev() - Throw data on the queue using iov.
3155 * @qpair: Pointer to the queue pair struct.
3156 * @iov: Pointer to buffer containing data
3157 * @iov_size: Length of buffer.
3158 * @buf_type: Buffer type (Unused).
3160 * This is the client interface for enqueueing data into the queue.
3161 * This function uses IO vectors to handle the work. Returns number
3162 * of bytes enqueued or < 0 on error.
3164 ssize_t vmci_qpair_enquev(struct vmci_qp *qpair,
3172 return VMCI_ERROR_INVALID_ARGS;
3177 result = qp_enqueue_locked(qpair->produce_q,
3179 qpair->produce_q_size,
3182 if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
3183 !qp_wait_for_ready_queue(qpair))
3184 result = VMCI_ERROR_WOULD_BLOCK;
3186 } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
3192 EXPORT_SYMBOL_GPL(vmci_qpair_enquev);
3195 * vmci_qpair_dequev() - Get data from the queue using iov.
3196 * @qpair: Pointer to the queue pair struct.
3197 * @iov: Pointer to buffer for the data
3198 * @iov_size: Length of buffer.
3199 * @buf_type: Buffer type (Unused).
3201 * This is the client interface for dequeueing data from the queue.
3202 * This function uses IO vectors to handle the work. Returns number
3203 * of bytes dequeued or < 0 on error.
3205 ssize_t vmci_qpair_dequev(struct vmci_qp *qpair,
3213 return VMCI_ERROR_INVALID_ARGS;
3218 result = qp_dequeue_locked(qpair->produce_q,
3220 qpair->consume_q_size,
3221 &msg->msg_iter, true);
3223 if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
3224 !qp_wait_for_ready_queue(qpair))
3225 result = VMCI_ERROR_WOULD_BLOCK;
3227 } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
3233 EXPORT_SYMBOL_GPL(vmci_qpair_dequev);
3236 * vmci_qpair_peekv() - Peek at the data in the queue using iov.
3237 * @qpair: Pointer to the queue pair struct.
3238 * @iov: Pointer to buffer for the data
3239 * @iov_size: Length of buffer.
3240 * @buf_type: Buffer type (Unused on Linux).
3242 * This is the client interface for peeking into a queue. (I.e.,
3243 * copy data from the queue without updating the head pointer.)
3244 * This function uses IO vectors to handle the work. Returns number
3245 * of bytes peeked or < 0 on error.
3247 ssize_t vmci_qpair_peekv(struct vmci_qp *qpair,
3255 return VMCI_ERROR_INVALID_ARGS;
3260 result = qp_dequeue_locked(qpair->produce_q,
3262 qpair->consume_q_size,
3263 &msg->msg_iter, false);
3265 if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
3266 !qp_wait_for_ready_queue(qpair))
3267 result = VMCI_ERROR_WOULD_BLOCK;
3269 } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
3274 EXPORT_SYMBOL_GPL(vmci_qpair_peekv);