drivers/staging/vc04_services/interface/vchiq_arm/vchiq_arm.c

   1 /**
   2  * Copyright (c) 2014 Raspberry Pi (Trading) Ltd. All rights reserved.
   3  * Copyright (c) 2010-2012 Broadcom. All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions, and the following disclaimer,
  10  *    without modification.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  * 3. The names of the above-listed copyright holders may not be used
  15  *    to endorse or promote products derived from this software without
  16  *    specific prior written permission.
  17  *
  18  * ALTERNATIVELY, this software may be distributed under the terms of the
  19  * GNU General Public License ("GPL") version 2, as published by the Free
  20  * Software Foundation.
  21  *
  22  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
  23  * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  24  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  25  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  26  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  27  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  28  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  29  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
  30  * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
  31  * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  32  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  33  */
  34
  35 #include <linux/kernel.h>
  36 #include <linux/module.h>
  37 #include <linux/sched/signal.h>
  38 #include <linux/types.h>
  39 #include <linux/errno.h>
  40 #include <linux/cdev.h>
  41 #include <linux/fs.h>
  42 #include <linux/device.h>
  43 #include <linux/mm.h>
  44 #include <linux/highmem.h>
  45 #include <linux/pagemap.h>
  46 #include <linux/bug.h>
  47 #include <linux/semaphore.h>
  48 #include <linux/list.h>
  49 #include <linux/of.h>
  50 #include <linux/platform_device.h>
  51 #include <linux/compat.h>
  52 #include <soc/bcm2835/raspberrypi-firmware.h>
  53
  54 #include "vchiq_core.h"
  55 #include "vchiq_ioctl.h"
  56 #include "vchiq_arm.h"
  57 #include "vchiq_debugfs.h"
  58 #include "vchiq_killable.h"
  59
  60 #define DEVICE_NAME "vchiq"
  61
  62 /* Override the default prefix, which would be vchiq_arm (from the filename) */
  63 #undef MODULE_PARAM_PREFIX
  64 #define MODULE_PARAM_PREFIX DEVICE_NAME "."
  65
  66 #define VCHIQ_MINOR 0
  67
  68 /* Some per-instance constants */
  69 #define MAX_COMPLETIONS 128
  70 #define MAX_SERVICES 64
  71 #define MAX_ELEMENTS 8
  72 #define MSG_QUEUE_SIZE 128
  73
  74 #define KEEPALIVE_VER 1
  75 #define KEEPALIVE_VER_MIN KEEPALIVE_VER
  76
  77 /* Run time control of log level, based on KERN_XXX level. */
  78 int vchiq_arm_log_level = VCHIQ_LOG_DEFAULT;
  79 int vchiq_susp_log_level = VCHIQ_LOG_ERROR;
  80
  81 #define SUSPEND_TIMER_TIMEOUT_MS 100
  82 #define SUSPEND_RETRY_TIMER_TIMEOUT_MS 1000
  83
  84 #define VC_SUSPEND_NUM_OFFSET 3 /* number of values before idle which are -ve */
  85 static const char *const suspend_state_names[] = {
  86         "VC_SUSPEND_FORCE_CANCELED",
  87         "VC_SUSPEND_REJECTED",
  88         "VC_SUSPEND_FAILED",
  89         "VC_SUSPEND_IDLE",
  90         "VC_SUSPEND_REQUESTED",
  91         "VC_SUSPEND_IN_PROGRESS",
  92         "VC_SUSPEND_SUSPENDED"
  93 };
  94 #define VC_RESUME_NUM_OFFSET 1 /* number of values before idle which are -ve */
  95 static const char *const resume_state_names[] = {
  96         "VC_RESUME_FAILED",
  97         "VC_RESUME_IDLE",
  98         "VC_RESUME_REQUESTED",
  99         "VC_RESUME_IN_PROGRESS",
 100         "VC_RESUME_RESUMED"
 101 };
 102 /* The number of times we allow force suspend to timeout before actually
 103 ** _forcing_ suspend.  This is to cater for SW which fails to release vchiq
 104 ** correctly - we don't want to prevent ARM suspend indefinitely in this case.
 105 */
 106 #define FORCE_SUSPEND_FAIL_MAX 8
 107
 108 /* The time in ms allowed for videocore to go idle when force suspend has been
 109  * requested */
 110 #define FORCE_SUSPEND_TIMEOUT_MS 200
 111
 112 static void suspend_timer_callback(struct timer_list *t);
 113
 114 typedef struct user_service_struct {
 115         VCHIQ_SERVICE_T *service;
 116         void *userdata;
 117         VCHIQ_INSTANCE_T instance;
 118         char is_vchi;
 119         char dequeue_pending;
 120         char close_pending;
 121         int message_available_pos;
 122         int msg_insert;
 123         int msg_remove;
 124         struct semaphore insert_event;
 125         struct semaphore remove_event;
 126         struct semaphore close_event;
 127         VCHIQ_HEADER_T * msg_queue[MSG_QUEUE_SIZE];
 128 } USER_SERVICE_T;
 129
 130 struct bulk_waiter_node {
 131         struct bulk_waiter bulk_waiter;
 132         int pid;
 133         struct list_head list;
 134 };
 135
 136 struct vchiq_instance_struct {
 137         VCHIQ_STATE_T *state;
 138         VCHIQ_COMPLETION_DATA_T completions[MAX_COMPLETIONS];
 139         int completion_insert;
 140         int completion_remove;
 141         struct semaphore insert_event;
 142         struct semaphore remove_event;
 143         struct mutex completion_mutex;
 144
 145         int connected;
 146         int closing;
 147         int pid;
 148         int mark;
 149         int use_close_delivered;
 150         int trace;
 151
 152         struct list_head bulk_waiter_list;
 153         struct mutex bulk_waiter_list_mutex;
 154
 155         VCHIQ_DEBUGFS_NODE_T debugfs_node;
 156 };
 157
 158 typedef struct dump_context_struct {
 159         char __user *buf;
 160         size_t actual;
 161         size_t space;
 162         loff_t offset;
 163 } DUMP_CONTEXT_T;
 164
 165 static struct cdev    vchiq_cdev;
 166 static dev_t          vchiq_devid;
 167 static VCHIQ_STATE_T g_state;
 168 static struct class  *vchiq_class;
 169 static struct device *vchiq_dev;
 170 static DEFINE_SPINLOCK(msg_queue_spinlock);
 171 static struct platform_device *bcm2835_camera;
 172
 173 static const char *const ioctl_names[] = {
 174         "CONNECT",
 175         "SHUTDOWN",
 176         "CREATE_SERVICE",
 177         "REMOVE_SERVICE",
 178         "QUEUE_MESSAGE",
 179         "QUEUE_BULK_TRANSMIT",
 180         "QUEUE_BULK_RECEIVE",
 181         "AWAIT_COMPLETION",
 182         "DEQUEUE_MESSAGE",
 183         "GET_CLIENT_ID",
 184         "GET_CONFIG",
 185         "CLOSE_SERVICE",
 186         "USE_SERVICE",
 187         "RELEASE_SERVICE",
 188         "SET_SERVICE_OPTION",
 189         "DUMP_PHYS_MEM",
 190         "LIB_VERSION",
 191         "CLOSE_DELIVERED"
 192 };
 193
 194 vchiq_static_assert(ARRAY_SIZE(ioctl_names) ==
 195                     (VCHIQ_IOC_MAX + 1));
 196
 197 static VCHIQ_STATUS_T
 198 vchiq_blocking_bulk_transfer(VCHIQ_SERVICE_HANDLE_T handle, void *data,
 199         unsigned int size, VCHIQ_BULK_DIR_T dir);
 200
 201 #define VCHIQ_INIT_RETRIES 10
 202 VCHIQ_STATUS_T vchiq_initialise(VCHIQ_INSTANCE_T *instance_out)
 203 {
 204         VCHIQ_STATUS_T status = VCHIQ_ERROR;
 205         VCHIQ_STATE_T *state;
 206         VCHIQ_INSTANCE_T instance = NULL;
 207         int i;
 208
 209         vchiq_log_trace(vchiq_core_log_level, "%s called", __func__);
 210
 211         /* VideoCore may not be ready due to boot up timing.
 212          * It may never be ready if kernel and firmware are mismatched,so don't
 213          * block forever.
 214          */
 215         for (i = 0; i < VCHIQ_INIT_RETRIES; i++) {
 216                 state = vchiq_get_state();
 217                 if (state)
 218                         break;
 219                 usleep_range(500, 600);
 220         }
 221         if (i == VCHIQ_INIT_RETRIES) {
 222                 vchiq_log_error(vchiq_core_log_level,
 223                         "%s: videocore not initialized\n", __func__);
 224                 goto failed;
 225         } else if (i > 0) {
 226                 vchiq_log_warning(vchiq_core_log_level,
 227                         "%s: videocore initialized after %d retries\n",
 228                         __func__, i);
 229         }
 230
 231         instance = kzalloc(sizeof(*instance), GFP_KERNEL);
 232         if (!instance) {
 233                 vchiq_log_error(vchiq_core_log_level,
 234                         "%s: error allocating vchiq instance\n", __func__);
 235                 goto failed;
 236         }
 237
 238         instance->connected = 0;
 239         instance->state = state;
 240         mutex_init(&instance->bulk_waiter_list_mutex);
 241         INIT_LIST_HEAD(&instance->bulk_waiter_list);
 242
 243         *instance_out = instance;
 244
 245         status = VCHIQ_SUCCESS;
 246
 247 failed:
 248         vchiq_log_trace(vchiq_core_log_level,
 249                 "%s(%p): returning %d", __func__, instance, status);
 250
 251         return status;
 252 }
 253 EXPORT_SYMBOL(vchiq_initialise);
 254
 255 VCHIQ_STATUS_T vchiq_shutdown(VCHIQ_INSTANCE_T instance)
 256 {
 257         VCHIQ_STATUS_T status;
 258         VCHIQ_STATE_T *state = instance->state;
 259
 260         vchiq_log_trace(vchiq_core_log_level,
 261                 "%s(%p) called", __func__, instance);
 262
 263         if (mutex_lock_killable(&state->mutex) != 0)
 264                 return VCHIQ_RETRY;
 265
 266         /* Remove all services */
 267         status = vchiq_shutdown_internal(state, instance);
 268
 269         mutex_unlock(&state->mutex);
 270
 271         vchiq_log_trace(vchiq_core_log_level,
 272                 "%s(%p): returning %d", __func__, instance, status);
 273
 274         if (status == VCHIQ_SUCCESS) {
 275                 struct list_head *pos, *next;
 276
 277                 list_for_each_safe(pos, next,
 278                                 &instance->bulk_waiter_list) {
 279                         struct bulk_waiter_node *waiter;
 280
 281                         waiter = list_entry(pos,
 282                                         struct bulk_waiter_node,
 283                                         list);
 284                         list_del(pos);
 285                         vchiq_log_info(vchiq_arm_log_level,
 286                                         "bulk_waiter - cleaned up %pK for pid %d",
 287                                         waiter, waiter->pid);
 288                         kfree(waiter);
 289                 }
 290                 kfree(instance);
 291         }
 292
 293         return status;
 294 }
 295 EXPORT_SYMBOL(vchiq_shutdown);
 296
 297 static int vchiq_is_connected(VCHIQ_INSTANCE_T instance)
 298 {
 299         return instance->connected;
 300 }
 301
 302 VCHIQ_STATUS_T vchiq_connect(VCHIQ_INSTANCE_T instance)
 303 {
 304         VCHIQ_STATUS_T status;
 305         VCHIQ_STATE_T *state = instance->state;
 306
 307         vchiq_log_trace(vchiq_core_log_level,
 308                 "%s(%p) called", __func__, instance);
 309
 310         if (mutex_lock_killable(&state->mutex) != 0) {
 311                 vchiq_log_trace(vchiq_core_log_level,
 312                         "%s: call to mutex_lock failed", __func__);
 313                 status = VCHIQ_RETRY;
 314                 goto failed;
 315         }
 316         status = vchiq_connect_internal(state, instance);
 317
 318         if (status == VCHIQ_SUCCESS)
 319                 instance->connected = 1;
 320
 321         mutex_unlock(&state->mutex);
 322
 323 failed:
 324         vchiq_log_trace(vchiq_core_log_level,
 325                 "%s(%p): returning %d", __func__, instance, status);
 326
 327         return status;
 328 }
 329 EXPORT_SYMBOL(vchiq_connect);
 330
 331 VCHIQ_STATUS_T vchiq_add_service(
 332         VCHIQ_INSTANCE_T              instance,
 333         const VCHIQ_SERVICE_PARAMS_T *params,
 334         VCHIQ_SERVICE_HANDLE_T       *phandle)
 335 {
 336         VCHIQ_STATUS_T status;
 337         VCHIQ_STATE_T *state = instance->state;
 338         VCHIQ_SERVICE_T *service = NULL;
 339         int srvstate;
 340
 341         vchiq_log_trace(vchiq_core_log_level,
 342                 "%s(%p) called", __func__, instance);
 343
 344         *phandle = VCHIQ_SERVICE_HANDLE_INVALID;
 345
 346         srvstate = vchiq_is_connected(instance)
 347                 ? VCHIQ_SRVSTATE_LISTENING
 348                 : VCHIQ_SRVSTATE_HIDDEN;
 349
 350         service = vchiq_add_service_internal(
 351                 state,
 352                 params,
 353                 srvstate,
 354                 instance,
 355                 NULL);
 356
 357         if (service) {
 358                 *phandle = service->handle;
 359                 status = VCHIQ_SUCCESS;
 360         } else
 361                 status = VCHIQ_ERROR;
 362
 363         vchiq_log_trace(vchiq_core_log_level,
 364                 "%s(%p): returning %d", __func__, instance, status);
 365
 366         return status;
 367 }
 368 EXPORT_SYMBOL(vchiq_add_service);
 369
 370 VCHIQ_STATUS_T vchiq_open_service(
 371         VCHIQ_INSTANCE_T              instance,
 372         const VCHIQ_SERVICE_PARAMS_T *params,
 373         VCHIQ_SERVICE_HANDLE_T       *phandle)
 374 {
 375         VCHIQ_STATUS_T   status = VCHIQ_ERROR;
 376         VCHIQ_STATE_T   *state = instance->state;
 377         VCHIQ_SERVICE_T *service = NULL;
 378
 379         vchiq_log_trace(vchiq_core_log_level,
 380                 "%s(%p) called", __func__, instance);
 381
 382         *phandle = VCHIQ_SERVICE_HANDLE_INVALID;
 383
 384         if (!vchiq_is_connected(instance))
 385                 goto failed;
 386
 387         service = vchiq_add_service_internal(state,
 388                 params,
 389                 VCHIQ_SRVSTATE_OPENING,
 390                 instance,
 391                 NULL);
 392
 393         if (service) {
 394                 *phandle = service->handle;
 395                 status = vchiq_open_service_internal(service, current->pid);
 396                 if (status != VCHIQ_SUCCESS) {
 397                         vchiq_remove_service(service->handle);
 398                         *phandle = VCHIQ_SERVICE_HANDLE_INVALID;
 399                 }
 400         }
 401
 402 failed:
 403         vchiq_log_trace(vchiq_core_log_level,
 404                 "%s(%p): returning %d", __func__, instance, status);
 405
 406         return status;
 407 }
 408 EXPORT_SYMBOL(vchiq_open_service);
 409
 410 VCHIQ_STATUS_T
 411 vchiq_bulk_transmit(VCHIQ_SERVICE_HANDLE_T handle, const void *data,
 412         unsigned int size, void *userdata, VCHIQ_BULK_MODE_T mode)
 413 {
 414         VCHIQ_STATUS_T status;
 415
 416         switch (mode) {
 417         case VCHIQ_BULK_MODE_NOCALLBACK:
 418         case VCHIQ_BULK_MODE_CALLBACK:
 419                 status = vchiq_bulk_transfer(handle,
 420                         VCHI_MEM_HANDLE_INVALID, (void *)data, size, userdata,
 421                         mode, VCHIQ_BULK_TRANSMIT);
 422                 break;
 423         case VCHIQ_BULK_MODE_BLOCKING:
 424                 status = vchiq_blocking_bulk_transfer(handle,
 425                         (void *)data, size, VCHIQ_BULK_TRANSMIT);
 426                 break;
 427         default:
 428                 return VCHIQ_ERROR;
 429         }
 430
 431         return status;
 432 }
 433 EXPORT_SYMBOL(vchiq_bulk_transmit);
 434
 435 VCHIQ_STATUS_T
 436 vchiq_bulk_receive(VCHIQ_SERVICE_HANDLE_T handle, void *data,
 437         unsigned int size, void *userdata, VCHIQ_BULK_MODE_T mode)
 438 {
 439         VCHIQ_STATUS_T status;
 440
 441         switch (mode) {
 442         case VCHIQ_BULK_MODE_NOCALLBACK:
 443         case VCHIQ_BULK_MODE_CALLBACK:
 444                 status = vchiq_bulk_transfer(handle,
 445                         VCHI_MEM_HANDLE_INVALID, data, size, userdata,
 446                         mode, VCHIQ_BULK_RECEIVE);
 447                 break;
 448         case VCHIQ_BULK_MODE_BLOCKING:
 449                 status = vchiq_blocking_bulk_transfer(handle,
 450                         (void *)data, size, VCHIQ_BULK_RECEIVE);
 451                 break;
 452         default:
 453                 return VCHIQ_ERROR;
 454         }
 455
 456         return status;
 457 }
 458 EXPORT_SYMBOL(vchiq_bulk_receive);
 459
 460 static VCHIQ_STATUS_T
 461 vchiq_blocking_bulk_transfer(VCHIQ_SERVICE_HANDLE_T handle, void *data,
 462         unsigned int size, VCHIQ_BULK_DIR_T dir)
 463 {
 464         VCHIQ_INSTANCE_T instance;
 465         VCHIQ_SERVICE_T *service;
 466         VCHIQ_STATUS_T status;
 467         struct bulk_waiter_node *waiter = NULL;
 468         struct list_head *pos;
 469
 470         service = find_service_by_handle(handle);
 471         if (!service)
 472                 return VCHIQ_ERROR;
 473
 474         instance = service->instance;
 475
 476         unlock_service(service);
 477
 478         mutex_lock(&instance->bulk_waiter_list_mutex);
 479         list_for_each(pos, &instance->bulk_waiter_list) {
 480                 if (list_entry(pos, struct bulk_waiter_node,
 481                                 list)->pid == current->pid) {
 482                         waiter = list_entry(pos,
 483                                 struct bulk_waiter_node,
 484                                 list);
 485                         list_del(pos);
 486                         break;
 487                 }
 488         }
 489         mutex_unlock(&instance->bulk_waiter_list_mutex);
 490
 491         if (waiter) {
 492                 VCHIQ_BULK_T *bulk = waiter->bulk_waiter.bulk;
 493
 494                 if (bulk) {
 495                         /* This thread has an outstanding bulk transfer. */
 496                         if ((bulk->data != data) ||
 497                                 (bulk->size != size)) {
 498                                 /* This is not a retry of the previous one.
 499                                  * Cancel the signal when the transfer
 500                                  * completes.
 501                                  */
 502                                 spin_lock(&bulk_waiter_spinlock);
 503                                 bulk->userdata = NULL;
 504                                 spin_unlock(&bulk_waiter_spinlock);
 505                         }
 506                 }
 507         }
 508
 509         if (!waiter) {
 510                 waiter = kzalloc(sizeof(struct bulk_waiter_node), GFP_KERNEL);
 511                 if (!waiter) {
 512                         vchiq_log_error(vchiq_core_log_level,
 513                                 "%s - out of memory", __func__);
 514                         return VCHIQ_ERROR;
 515                 }
 516         }
 517
 518         status = vchiq_bulk_transfer(handle, VCHI_MEM_HANDLE_INVALID,
 519                 data, size, &waiter->bulk_waiter, VCHIQ_BULK_MODE_BLOCKING,
 520                 dir);
 521         if ((status != VCHIQ_RETRY) || fatal_signal_pending(current) ||
 522                 !waiter->bulk_waiter.bulk) {
 523                 VCHIQ_BULK_T *bulk = waiter->bulk_waiter.bulk;
 524
 525                 if (bulk) {
 526                         /* Cancel the signal when the transfer
 527                          * completes.
 528                          */
 529                         spin_lock(&bulk_waiter_spinlock);
 530                         bulk->userdata = NULL;
 531                         spin_unlock(&bulk_waiter_spinlock);
 532                 }
 533                 kfree(waiter);
 534         } else {
 535                 waiter->pid = current->pid;
 536                 mutex_lock(&instance->bulk_waiter_list_mutex);
 537                 list_add(&waiter->list, &instance->bulk_waiter_list);
 538                 mutex_unlock(&instance->bulk_waiter_list_mutex);
 539                 vchiq_log_info(vchiq_arm_log_level,
 540                                 "saved bulk_waiter %pK for pid %d",
 541                                 waiter, current->pid);
 542         }
 543
 544         return status;
 545 }
 546 /****************************************************************************
 547 *
 548 *   add_completion
 549 *
 550 ***************************************************************************/
 551
 552 static VCHIQ_STATUS_T
 553 add_completion(VCHIQ_INSTANCE_T instance, VCHIQ_REASON_T reason,
 554         VCHIQ_HEADER_T *header, USER_SERVICE_T *user_service,
 555         void *bulk_userdata)
 556 {
 557         VCHIQ_COMPLETION_DATA_T *completion;
 558         int insert;
 559
 560         DEBUG_INITIALISE(g_state.local)
 561
 562         insert = instance->completion_insert;
 563         while ((insert - instance->completion_remove) >= MAX_COMPLETIONS) {
 564                 /* Out of space - wait for the client */
 565                 DEBUG_TRACE(SERVICE_CALLBACK_LINE);
 566                 vchiq_log_trace(vchiq_arm_log_level,
 567                         "%s - completion queue full", __func__);
 568                 DEBUG_COUNT(COMPLETION_QUEUE_FULL_COUNT);
 569                 if (down_interruptible(&instance->remove_event) != 0) {
 570                         vchiq_log_info(vchiq_arm_log_level,
 571                                 "service_callback interrupted");
 572                         return VCHIQ_RETRY;
 573                 } else if (instance->closing) {
 574                         vchiq_log_info(vchiq_arm_log_level,
 575                                 "service_callback closing");
 576                         return VCHIQ_SUCCESS;
 577                 }
 578                 DEBUG_TRACE(SERVICE_CALLBACK_LINE);
 579         }
 580
 581         completion = &instance->completions[insert & (MAX_COMPLETIONS - 1)];
 582
 583         completion->header = header;
 584         completion->reason = reason;
 585         /* N.B. service_userdata is updated while processing AWAIT_COMPLETION */
 586         completion->service_userdata = user_service->service;
 587         completion->bulk_userdata = bulk_userdata;
 588
 589         if (reason == VCHIQ_SERVICE_CLOSED) {
 590                 /* Take an extra reference, to be held until
 591                    this CLOSED notification is delivered. */
 592                 lock_service(user_service->service);
 593                 if (instance->use_close_delivered)
 594                         user_service->close_pending = 1;
 595         }
 596
 597         /* A write barrier is needed here to ensure that the entire completion
 598                 record is written out before the insert point. */
 599         wmb();
 600
 601         if (reason == VCHIQ_MESSAGE_AVAILABLE)
 602                 user_service->message_available_pos = insert;
 603
 604         insert++;
 605         instance->completion_insert = insert;
 606
 607         up(&instance->insert_event);
 608
 609         return VCHIQ_SUCCESS;
 610 }
 611
 612 /****************************************************************************
 613 *
 614 *   service_callback
 615 *
 616 ***************************************************************************/
 617
 618 static VCHIQ_STATUS_T
 619 service_callback(VCHIQ_REASON_T reason, VCHIQ_HEADER_T *header,
 620         VCHIQ_SERVICE_HANDLE_T handle, void *bulk_userdata)
 621 {
 622         /* How do we ensure the callback goes to the right client?
 623         ** The service_user data points to a USER_SERVICE_T record containing
 624         ** the original callback and the user state structure, which contains a
 625         ** circular buffer for completion records.
 626         */
 627         USER_SERVICE_T *user_service;
 628         VCHIQ_SERVICE_T *service;
 629         VCHIQ_INSTANCE_T instance;
 630         bool skip_completion = false;
 631
 632         DEBUG_INITIALISE(g_state.local)
 633
 634         DEBUG_TRACE(SERVICE_CALLBACK_LINE);
 635
 636         service = handle_to_service(handle);
 637         BUG_ON(!service);
 638         user_service = (USER_SERVICE_T *)service->base.userdata;
 639         instance = user_service->instance;
 640
 641         if (!instance || instance->closing)
 642                 return VCHIQ_SUCCESS;
 643
 644         vchiq_log_trace(vchiq_arm_log_level,
 645                 "%s - service %lx(%d,%p), reason %d, header %lx, "
 646                 "instance %lx, bulk_userdata %lx",
 647                 __func__, (unsigned long)user_service,
 648                 service->localport, user_service->userdata,
 649                 reason, (unsigned long)header,
 650                 (unsigned long)instance, (unsigned long)bulk_userdata);
 651
 652         if (header && user_service->is_vchi) {
 653                 spin_lock(&msg_queue_spinlock);
 654                 while (user_service->msg_insert ==
 655                         (user_service->msg_remove + MSG_QUEUE_SIZE)) {
 656                         spin_unlock(&msg_queue_spinlock);
 657                         DEBUG_TRACE(SERVICE_CALLBACK_LINE);
 658                         DEBUG_COUNT(MSG_QUEUE_FULL_COUNT);
 659                         vchiq_log_trace(vchiq_arm_log_level,
 660                                 "service_callback - msg queue full");
 661                         /* If there is no MESSAGE_AVAILABLE in the completion
 662                         ** queue, add one
 663                         */
 664                         if ((user_service->message_available_pos -
 665                                 instance->completion_remove) < 0) {
 666                                 VCHIQ_STATUS_T status;
 667
 668                                 vchiq_log_info(vchiq_arm_log_level,
 669                                         "Inserting extra MESSAGE_AVAILABLE");
 670                                 DEBUG_TRACE(SERVICE_CALLBACK_LINE);
 671                                 status = add_completion(instance, reason,
 672                                         NULL, user_service, bulk_userdata);
 673                                 if (status != VCHIQ_SUCCESS) {
 674                                         DEBUG_TRACE(SERVICE_CALLBACK_LINE);
 675                                         return status;
 676                                 }
 677                         }
 678
 679                         DEBUG_TRACE(SERVICE_CALLBACK_LINE);
 680                         if (down_interruptible(&user_service->remove_event)
 681                                 != 0) {
 682                                 vchiq_log_info(vchiq_arm_log_level,
 683                                         "%s interrupted", __func__);
 684                                 DEBUG_TRACE(SERVICE_CALLBACK_LINE);
 685                                 return VCHIQ_RETRY;
 686                         } else if (instance->closing) {
 687                                 vchiq_log_info(vchiq_arm_log_level,
 688                                         "%s closing", __func__);
 689                                 DEBUG_TRACE(SERVICE_CALLBACK_LINE);
 690                                 return VCHIQ_ERROR;
 691                         }
 692                         DEBUG_TRACE(SERVICE_CALLBACK_LINE);
 693                         spin_lock(&msg_queue_spinlock);
 694                 }
 695
 696                 user_service->msg_queue[user_service->msg_insert &
 697                         (MSG_QUEUE_SIZE - 1)] = header;
 698                 user_service->msg_insert++;
 699
 700                 /* If there is a thread waiting in DEQUEUE_MESSAGE, or if
 701                 ** there is a MESSAGE_AVAILABLE in the completion queue then
 702                 ** bypass the completion queue.
 703                 */
 704                 if (((user_service->message_available_pos -
 705                         instance->completion_remove) >= 0) ||
 706                         user_service->dequeue_pending) {
 707                         user_service->dequeue_pending = 0;
 708                         skip_completion = true;
 709                 }
 710
 711                 spin_unlock(&msg_queue_spinlock);
 712                 up(&user_service->insert_event);
 713
 714                 header = NULL;
 715         }
 716         DEBUG_TRACE(SERVICE_CALLBACK_LINE);
 717
 718         if (skip_completion)
 719                 return VCHIQ_SUCCESS;
 720
 721         return add_completion(instance, reason, header, user_service,
 722                 bulk_userdata);
 723 }
 724
 725 /****************************************************************************
 726 *
 727 *   user_service_free
 728 *
 729 ***************************************************************************/
 730 static void
 731 user_service_free(void *userdata)
 732 {
 733         kfree(userdata);
 734 }
 735
 736 /****************************************************************************
 737 *
 738 *   close_delivered
 739 *
 740 ***************************************************************************/
 741 static void close_delivered(USER_SERVICE_T *user_service)
 742 {
 743         vchiq_log_info(vchiq_arm_log_level,
 744                 "%s(handle=%x)",
 745                 __func__, user_service->service->handle);
 746
 747         if (user_service->close_pending) {
 748                 /* Allow the underlying service to be culled */
 749                 unlock_service(user_service->service);
 750
 751                 /* Wake the user-thread blocked in close_ or remove_service */
 752                 up(&user_service->close_event);
 753
 754                 user_service->close_pending = 0;
 755         }
 756 }
 757
 758 struct vchiq_io_copy_callback_context {
 759         struct vchiq_element *current_element;
 760         size_t current_element_offset;
 761         unsigned long elements_to_go;
 762         size_t current_offset;
 763 };
 764
 765 static ssize_t
 766 vchiq_ioc_copy_element_data(
 767         void *context,
 768         void *dest,
 769         size_t offset,
 770         size_t maxsize)
 771 {
 772         long res;
 773         size_t bytes_this_round;
 774         struct vchiq_io_copy_callback_context *copy_context =
 775                 (struct vchiq_io_copy_callback_context *)context;
 776
 777         if (offset != copy_context->current_offset)
 778                 return 0;
 779
 780         if (!copy_context->elements_to_go)
 781                 return 0;
 782
 783         /*
 784          * Complex logic here to handle the case of 0 size elements
 785          * in the middle of the array of elements.
 786          *
 787          * Need to skip over these 0 size elements.
 788          */
 789         while (1) {
 790                 bytes_this_round = min(copy_context->current_element->size -
 791                                        copy_context->current_element_offset,
 792                                        maxsize);
 793
 794                 if (bytes_this_round)
 795                         break;
 796
 797                 copy_context->elements_to_go--;
 798                 copy_context->current_element++;
 799                 copy_context->current_element_offset = 0;
 800
 801                 if (!copy_context->elements_to_go)
 802                         return 0;
 803         }
 804
 805         res = copy_from_user(dest,
 806                              copy_context->current_element->data +
 807                              copy_context->current_element_offset,
 808                              bytes_this_round);
 809
 810         if (res != 0)
 811                 return -EFAULT;
 812
 813         copy_context->current_element_offset += bytes_this_round;
 814         copy_context->current_offset += bytes_this_round;
 815
 816         /*
 817          * Check if done with current element, and if so advance to the next.
 818          */
 819         if (copy_context->current_element_offset ==
 820             copy_context->current_element->size) {
 821                 copy_context->elements_to_go--;
 822                 copy_context->current_element++;
 823                 copy_context->current_element_offset = 0;
 824         }
 825
 826         return bytes_this_round;
 827 }
 828
 829 /**************************************************************************
 830  *
 831  *   vchiq_ioc_queue_message
 832  *
 833  **************************************************************************/
 834 static VCHIQ_STATUS_T
 835 vchiq_ioc_queue_message(VCHIQ_SERVICE_HANDLE_T handle,
 836                         struct vchiq_element *elements,
 837                         unsigned long count)
 838 {
 839         struct vchiq_io_copy_callback_context context;
 840         unsigned long i;
 841         size_t total_size = 0;
 842
 843         context.current_element = elements;
 844         context.current_element_offset = 0;
 845         context.elements_to_go = count;
 846         context.current_offset = 0;
 847
 848         for (i = 0; i < count; i++) {
 849                 if (!elements[i].data && elements[i].size != 0)
 850                         return -EFAULT;
 851
 852                 total_size += elements[i].size;
 853         }
 854
 855         return vchiq_queue_message(handle, vchiq_ioc_copy_element_data,
 856                                    &context, total_size);
 857 }
 858
 859 /****************************************************************************
 860 *
 861 *   vchiq_ioctl
 862 *
 863 ***************************************************************************/
 864 static long
 865 vchiq_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 866 {
 867         VCHIQ_INSTANCE_T instance = file->private_data;
 868         VCHIQ_STATUS_T status = VCHIQ_SUCCESS;
 869         VCHIQ_SERVICE_T *service = NULL;
 870         long ret = 0;
 871         int i, rc;
 872
 873         DEBUG_INITIALISE(g_state.local)
 874
 875         vchiq_log_trace(vchiq_arm_log_level,
 876                 "%s - instance %pK, cmd %s, arg %lx",
 877                 __func__, instance,
 878                 ((_IOC_TYPE(cmd) == VCHIQ_IOC_MAGIC) &&
 879                 (_IOC_NR(cmd) <= VCHIQ_IOC_MAX)) ?
 880                 ioctl_names[_IOC_NR(cmd)] : "<invalid>", arg);
 881
 882         switch (cmd) {
 883         case VCHIQ_IOC_SHUTDOWN:
 884                 if (!instance->connected)
 885                         break;
 886
 887                 /* Remove all services */
 888                 i = 0;
 889                 while ((service = next_service_by_instance(instance->state,
 890                         instance, &i)) != NULL) {
 891                         status = vchiq_remove_service(service->handle);
 892                         unlock_service(service);
 893                         if (status != VCHIQ_SUCCESS)
 894                                 break;
 895                 }
 896                 service = NULL;
 897
 898                 if (status == VCHIQ_SUCCESS) {
 899                         /* Wake the completion thread and ask it to exit */
 900                         instance->closing = 1;
 901                         up(&instance->insert_event);
 902                 }
 903
 904                 break;
 905
 906         case VCHIQ_IOC_CONNECT:
 907                 if (instance->connected) {
 908                         ret = -EINVAL;
 909                         break;
 910                 }
 911                 rc = mutex_lock_killable(&instance->state->mutex);
 912                 if (rc != 0) {
 913                         vchiq_log_error(vchiq_arm_log_level,
 914                                 "vchiq: connect: could not lock mutex for "
 915                                 "state %d: %d",
 916                                 instance->state->id, rc);
 917                         ret = -EINTR;
 918                         break;
 919                 }
 920                 status = vchiq_connect_internal(instance->state, instance);
 921                 mutex_unlock(&instance->state->mutex);
 922
 923                 if (status == VCHIQ_SUCCESS)
 924                         instance->connected = 1;
 925                 else
 926                         vchiq_log_error(vchiq_arm_log_level,
 927                                 "vchiq: could not connect: %d", status);
 928                 break;
 929
 930         case VCHIQ_IOC_CREATE_SERVICE: {
 931                 VCHIQ_CREATE_SERVICE_T args;
 932                 USER_SERVICE_T *user_service = NULL;
 933                 void *userdata;
 934                 int srvstate;
 935
 936                 if (copy_from_user
 937                          (&args, (const void __user *)arg,
 938                           sizeof(args)) != 0) {
 939                         ret = -EFAULT;
 940                         break;
 941                 }
 942
 943                 user_service = kmalloc(sizeof(USER_SERVICE_T), GFP_KERNEL);
 944                 if (!user_service) {
 945                         ret = -ENOMEM;
 946                         break;
 947                 }
 948
 949                 if (args.is_open) {
 950                         if (!instance->connected) {
 951                                 ret = -ENOTCONN;
 952                                 kfree(user_service);
 953                                 break;
 954                         }
 955                         srvstate = VCHIQ_SRVSTATE_OPENING;
 956                 } else {
 957                         srvstate =
 958                                  instance->connected ?
 959                                  VCHIQ_SRVSTATE_LISTENING :
 960                                  VCHIQ_SRVSTATE_HIDDEN;
 961                 }
 962
 963                 userdata = args.params.userdata;
 964                 args.params.callback = service_callback;
 965                 args.params.userdata = user_service;
 966                 service = vchiq_add_service_internal(
 967                                 instance->state,
 968                                 &args.params, srvstate,
 969                                 instance, user_service_free);
 970
 971                 if (service != NULL) {
 972                         user_service->service = service;
 973                         user_service->userdata = userdata;
 974                         user_service->instance = instance;
 975                         user_service->is_vchi = (args.is_vchi != 0);
 976                         user_service->dequeue_pending = 0;
 977                         user_service->close_pending = 0;
 978                         user_service->message_available_pos =
 979                                 instance->completion_remove - 1;
 980                         user_service->msg_insert = 0;
 981                         user_service->msg_remove = 0;
 982                         sema_init(&user_service->insert_event, 0);
 983                         sema_init(&user_service->remove_event, 0);
 984                         sema_init(&user_service->close_event, 0);
 985
 986                         if (args.is_open) {
 987                                 status = vchiq_open_service_internal
 988                                         (service, instance->pid);
 989                                 if (status != VCHIQ_SUCCESS) {
 990                                         vchiq_remove_service(service->handle);
 991                                         service = NULL;
 992                                         ret = (status == VCHIQ_RETRY) ?
 993                                                 -EINTR : -EIO;
 994                                         break;
 995                                 }
 996                         }
 997
 998                         if (copy_to_user((void __user *)
 999                                 &(((VCHIQ_CREATE_SERVICE_T __user *)
1000                                         arg)->handle),
1001                                 (const void *)&service->handle,
1002                                 sizeof(service->handle)) != 0) {
1003                                 ret = -EFAULT;
1004                                 vchiq_remove_service(service->handle);
1005                         }
1006
1007                         service = NULL;
1008                 } else {
1009                         ret = -EEXIST;
1010                         kfree(user_service);
1011                 }
1012         } break;
1013
1014         case VCHIQ_IOC_CLOSE_SERVICE: {
1015                 VCHIQ_SERVICE_HANDLE_T handle = (VCHIQ_SERVICE_HANDLE_T)arg;
1016
1017                 service = find_service_for_instance(instance, handle);
1018                 if (service != NULL) {
1019                         USER_SERVICE_T *user_service =
1020                                 (USER_SERVICE_T *)service->base.userdata;
1021                         /* close_pending is false on first entry, and when the
1022                            wait in vchiq_close_service has been interrupted. */
1023                         if (!user_service->close_pending) {
1024                                 status = vchiq_close_service(service->handle);
1025                                 if (status != VCHIQ_SUCCESS)
1026                                         break;
1027                         }
1028
1029                         /* close_pending is true once the underlying service
1030                            has been closed until the client library calls the
1031                            CLOSE_DELIVERED ioctl, signalling close_event. */
1032                         if (user_service->close_pending &&
1033                                 down_interruptible(&user_service->close_event))
1034                                 status = VCHIQ_RETRY;
1035                 } else
1036                         ret = -EINVAL;
1037         } break;
1038
1039         case VCHIQ_IOC_REMOVE_SERVICE: {
1040                 VCHIQ_SERVICE_HANDLE_T handle = (VCHIQ_SERVICE_HANDLE_T)arg;
1041
1042                 service = find_service_for_instance(instance, handle);
1043                 if (service != NULL) {
1044                         USER_SERVICE_T *user_service =
1045                                 (USER_SERVICE_T *)service->base.userdata;
1046                         /* close_pending is false on first entry, and when the
1047                            wait in vchiq_close_service has been interrupted. */
1048                         if (!user_service->close_pending) {
1049                                 status = vchiq_remove_service(service->handle);
1050                                 if (status != VCHIQ_SUCCESS)
1051                                         break;
1052                         }
1053
1054                         /* close_pending is true once the underlying service
1055                            has been closed until the client library calls the
1056                            CLOSE_DELIVERED ioctl, signalling close_event. */
1057                         if (user_service->close_pending &&
1058                                 down_interruptible(&user_service->close_event))
1059                                 status = VCHIQ_RETRY;
1060                 } else
1061                         ret = -EINVAL;
1062         } break;
1063
1064         case VCHIQ_IOC_USE_SERVICE:
1065         case VCHIQ_IOC_RELEASE_SERVICE: {
1066                 VCHIQ_SERVICE_HANDLE_T handle = (VCHIQ_SERVICE_HANDLE_T)arg;
1067
1068                 service = find_service_for_instance(instance, handle);
1069                 if (service != NULL) {
1070                         status = (cmd == VCHIQ_IOC_USE_SERVICE) ?
1071                                 vchiq_use_service_internal(service) :
1072                                 vchiq_release_service_internal(service);
1073                         if (status != VCHIQ_SUCCESS) {
1074                                 vchiq_log_error(vchiq_susp_log_level,
1075                                         "%s: cmd %s returned error %d for "
1076                                         "service %c%c%c%c:%03d",
1077                                         __func__,
1078                                         (cmd == VCHIQ_IOC_USE_SERVICE) ?
1079                                                 "VCHIQ_IOC_USE_SERVICE" :
1080                                                 "VCHIQ_IOC_RELEASE_SERVICE",
1081                                         status,
1082                                         VCHIQ_FOURCC_AS_4CHARS(
1083                                                 service->base.fourcc),
1084                                         service->client_id);
1085                                 ret = -EINVAL;
1086                         }
1087                 } else
1088                         ret = -EINVAL;
1089         } break;
1090
1091         case VCHIQ_IOC_QUEUE_MESSAGE: {
1092                 VCHIQ_QUEUE_MESSAGE_T args;
1093
1094                 if (copy_from_user
1095                          (&args, (const void __user *)arg,
1096                           sizeof(args)) != 0) {
1097                         ret = -EFAULT;
1098                         break;
1099                 }
1100
1101                 service = find_service_for_instance(instance, args.handle);
1102
1103                 if ((service != NULL) && (args.count <= MAX_ELEMENTS)) {
1104                         /* Copy elements into kernel space */
1105                         struct vchiq_element elements[MAX_ELEMENTS];
1106
1107                         if (copy_from_user(elements, args.elements,
1108                                 args.count * sizeof(struct vchiq_element)) == 0)
1109                                 status = vchiq_ioc_queue_message
1110                                         (args.handle,
1111                                         elements, args.count);
1112                         else
1113                                 ret = -EFAULT;
1114                 } else {
1115                         ret = -EINVAL;
1116                 }
1117         } break;
1118
1119         case VCHIQ_IOC_QUEUE_BULK_TRANSMIT:
1120         case VCHIQ_IOC_QUEUE_BULK_RECEIVE: {
1121                 VCHIQ_QUEUE_BULK_TRANSFER_T args;
1122                 struct bulk_waiter_node *waiter = NULL;
1123
1124                 VCHIQ_BULK_DIR_T dir =
1125                         (cmd == VCHIQ_IOC_QUEUE_BULK_TRANSMIT) ?
1126                         VCHIQ_BULK_TRANSMIT : VCHIQ_BULK_RECEIVE;
1127
1128                 if (copy_from_user
1129                         (&args, (const void __user *)arg,
1130                         sizeof(args)) != 0) {
1131                         ret = -EFAULT;
1132                         break;
1133                 }
1134
1135                 service = find_service_for_instance(instance, args.handle);
1136                 if (!service) {
1137                         ret = -EINVAL;
1138                         break;
1139                 }
1140
1141                 if (args.mode == VCHIQ_BULK_MODE_BLOCKING) {
1142                         waiter = kzalloc(sizeof(struct bulk_waiter_node),
1143                                 GFP_KERNEL);
1144                         if (!waiter) {
1145                                 ret = -ENOMEM;
1146                                 break;
1147                         }
1148                         args.userdata = &waiter->bulk_waiter;
1149                 } else if (args.mode == VCHIQ_BULK_MODE_WAITING) {
1150                         struct list_head *pos;
1151
1152                         mutex_lock(&instance->bulk_waiter_list_mutex);
1153                         list_for_each(pos, &instance->bulk_waiter_list) {
1154                                 if (list_entry(pos, struct bulk_waiter_node,
1155                                         list)->pid == current->pid) {
1156                                         waiter = list_entry(pos,
1157                                                 struct bulk_waiter_node,
1158                                                 list);
1159                                         list_del(pos);
1160                                         break;
1161                                 }
1162
1163                         }
1164                         mutex_unlock(&instance->bulk_waiter_list_mutex);
1165                         if (!waiter) {
1166                                 vchiq_log_error(vchiq_arm_log_level,
1167                                         "no bulk_waiter found for pid %d",
1168                                         current->pid);
1169                                 ret = -ESRCH;
1170                                 break;
1171                         }
1172                         vchiq_log_info(vchiq_arm_log_level,
1173                                 "found bulk_waiter %pK for pid %d", waiter,
1174                                 current->pid);
1175                         args.userdata = &waiter->bulk_waiter;
1176                 }
1177                 status = vchiq_bulk_transfer
1178                         (args.handle,
1179                          VCHI_MEM_HANDLE_INVALID,
1180                          args.data, args.size,
1181                          args.userdata, args.mode,
1182                          dir);
1183                 if (!waiter)
1184                         break;
1185                 if ((status != VCHIQ_RETRY) || fatal_signal_pending(current) ||
1186                         !waiter->bulk_waiter.bulk) {
1187                         if (waiter->bulk_waiter.bulk) {
1188                                 /* Cancel the signal when the transfer
1189                                 ** completes. */
1190                                 spin_lock(&bulk_waiter_spinlock);
1191                                 waiter->bulk_waiter.bulk->userdata = NULL;
1192                                 spin_unlock(&bulk_waiter_spinlock);
1193                         }
1194                         kfree(waiter);
1195                 } else {
1196                         const VCHIQ_BULK_MODE_T mode_waiting =
1197                                 VCHIQ_BULK_MODE_WAITING;
1198                         waiter->pid = current->pid;
1199                         mutex_lock(&instance->bulk_waiter_list_mutex);
1200                         list_add(&waiter->list, &instance->bulk_waiter_list);
1201                         mutex_unlock(&instance->bulk_waiter_list_mutex);
1202                         vchiq_log_info(vchiq_arm_log_level,
1203                                 "saved bulk_waiter %pK for pid %d",
1204                                 waiter, current->pid);
1205
1206                         if (copy_to_user((void __user *)
1207                                 &(((VCHIQ_QUEUE_BULK_TRANSFER_T __user *)
1208                                         arg)->mode),
1209                                 (const void *)&mode_waiting,
1210                                 sizeof(mode_waiting)) != 0)
1211                                 ret = -EFAULT;
1212                 }
1213         } break;
1214
1215         case VCHIQ_IOC_AWAIT_COMPLETION: {
1216                 VCHIQ_AWAIT_COMPLETION_T args;
1217
1218                 DEBUG_TRACE(AWAIT_COMPLETION_LINE);
1219                 if (!instance->connected) {
1220                         ret = -ENOTCONN;
1221                         break;
1222                 }
1223
1224                 if (copy_from_user(&args, (const void __user *)arg,
1225                         sizeof(args)) != 0) {
1226                         ret = -EFAULT;
1227                         break;
1228                 }
1229
1230                 mutex_lock(&instance->completion_mutex);
1231
1232                 DEBUG_TRACE(AWAIT_COMPLETION_LINE);
1233                 while ((instance->completion_remove ==
1234                         instance->completion_insert)
1235                         && !instance->closing) {
1236                         int rc;
1237
1238                         DEBUG_TRACE(AWAIT_COMPLETION_LINE);
1239                         mutex_unlock(&instance->completion_mutex);
1240                         rc = down_interruptible(&instance->insert_event);
1241                         mutex_lock(&instance->completion_mutex);
1242                         if (rc != 0) {
1243                                 DEBUG_TRACE(AWAIT_COMPLETION_LINE);
1244                                 vchiq_log_info(vchiq_arm_log_level,
1245                                         "AWAIT_COMPLETION interrupted");
1246                                 ret = -EINTR;
1247                                 break;
1248                         }
1249                 }
1250                 DEBUG_TRACE(AWAIT_COMPLETION_LINE);
1251
1252                 if (ret == 0) {
1253                         int msgbufcount = args.msgbufcount;
1254                         int remove = instance->completion_remove;
1255
1256                         for (ret = 0; ret < args.count; ret++) {
1257                                 VCHIQ_COMPLETION_DATA_T *completion;
1258                                 VCHIQ_SERVICE_T *service;
1259                                 USER_SERVICE_T *user_service;
1260                                 VCHIQ_HEADER_T *header;
1261
1262                                 if (remove == instance->completion_insert)
1263                                         break;
1264
1265                                 completion = &instance->completions[
1266                                         remove & (MAX_COMPLETIONS - 1)];
1267
1268                                 /*
1269                                  * A read memory barrier is needed to stop
1270                                  * prefetch of a stale completion record
1271                                  */
1272                                 rmb();
1273
1274                                 service = completion->service_userdata;
1275                                 user_service = service->base.userdata;
1276                                 completion->service_userdata =
1277                                         user_service->userdata;
1278
1279                                 header = completion->header;
1280                                 if (header) {
1281                                         void __user *msgbuf;
1282                                         int msglen;
1283
1284                                         msglen = header->size +
1285                                                 sizeof(VCHIQ_HEADER_T);
1286                                         /* This must be a VCHIQ-style service */
1287                                         if (args.msgbufsize < msglen) {
1288                                                 vchiq_log_error(
1289                                                         vchiq_arm_log_level,
1290                                                         "header %pK: msgbufsize %x < msglen %x",
1291                                                         header, args.msgbufsize,
1292                                                         msglen);
1293                                                 WARN(1, "invalid message "
1294                                                         "size\n");
1295                                                 if (ret == 0)
1296                                                         ret = -EMSGSIZE;
1297                                                 break;
1298                                         }
1299                                         if (msgbufcount <= 0)
1300                                                 /* Stall here for lack of a
1301                                                 ** buffer for the message. */
1302                                                 break;
1303                                         /* Get the pointer from user space */
1304                                         msgbufcount--;
1305                                         if (copy_from_user(&msgbuf,
1306                                                 (const void __user *)
1307                                                 &args.msgbufs[msgbufcount],
1308                                                 sizeof(msgbuf)) != 0) {
1309                                                 if (ret == 0)
1310                                                         ret = -EFAULT;
1311                                                 break;
1312                                         }
1313
1314                                         /* Copy the message to user space */
1315                                         if (copy_to_user(msgbuf, header,
1316                                                 msglen) != 0) {
1317                                                 if (ret == 0)
1318                                                         ret = -EFAULT;
1319                                                 break;
1320                                         }
1321
1322                                         /* Now it has been copied, the message
1323                                         ** can be released. */
1324                                         vchiq_release_message(service->handle,
1325                                                 header);
1326
1327                                         /* The completion must point to the
1328                                         ** msgbuf. */
1329                                         completion->header = msgbuf;
1330                                 }
1331
1332                                 if ((completion->reason ==
1333                                         VCHIQ_SERVICE_CLOSED) &&
1334                                         !instance->use_close_delivered)
1335                                         unlock_service(service);
1336
1337                                 if (copy_to_user((void __user *)(
1338                                         (size_t)args.buf +
1339                                         ret * sizeof(VCHIQ_COMPLETION_DATA_T)),
1340                                         completion,
1341                                         sizeof(VCHIQ_COMPLETION_DATA_T)) != 0) {
1342                                                 if (ret == 0)
1343                                                         ret = -EFAULT;
1344                                         break;
1345                                 }
1346
1347                                 /*
1348                                  * Ensure that the above copy has completed
1349                                  * before advancing the remove pointer.
1350                                  */
1351                                 mb();
1352                                 remove++;
1353                                 instance->completion_remove = remove;
1354                         }
1355
1356                         if (msgbufcount != args.msgbufcount) {
1357                                 if (copy_to_user((void __user *)
1358                                         &((VCHIQ_AWAIT_COMPLETION_T *)arg)->
1359                                                 msgbufcount,
1360                                         &msgbufcount,
1361                                         sizeof(msgbufcount)) != 0) {
1362                                         ret = -EFAULT;
1363                                 }
1364                         }
1365                 }
1366
1367                 if (ret != 0)
1368                         up(&instance->remove_event);
1369                 mutex_unlock(&instance->completion_mutex);
1370                 DEBUG_TRACE(AWAIT_COMPLETION_LINE);
1371         } break;
1372
1373         case VCHIQ_IOC_DEQUEUE_MESSAGE: {
1374                 VCHIQ_DEQUEUE_MESSAGE_T args;
1375                 USER_SERVICE_T *user_service;
1376                 VCHIQ_HEADER_T *header;
1377
1378                 DEBUG_TRACE(DEQUEUE_MESSAGE_LINE);
1379                 if (copy_from_user
1380                          (&args, (const void __user *)arg,
1381                           sizeof(args)) != 0) {
1382                         ret = -EFAULT;
1383                         break;
1384                 }
1385                 service = find_service_for_instance(instance, args.handle);
1386                 if (!service) {
1387                         ret = -EINVAL;
1388                         break;
1389                 }
1390                 user_service = (USER_SERVICE_T *)service->base.userdata;
1391                 if (user_service->is_vchi == 0) {
1392                         ret = -EINVAL;
1393                         break;
1394                 }
1395
1396                 spin_lock(&msg_queue_spinlock);
1397                 if (user_service->msg_remove == user_service->msg_insert) {
1398                         if (!args.blocking) {
1399                                 spin_unlock(&msg_queue_spinlock);
1400                                 DEBUG_TRACE(DEQUEUE_MESSAGE_LINE);
1401                                 ret = -EWOULDBLOCK;
1402                                 break;
1403                         }
1404                         user_service->dequeue_pending = 1;
1405                         do {
1406                                 spin_unlock(&msg_queue_spinlock);
1407                                 DEBUG_TRACE(DEQUEUE_MESSAGE_LINE);
1408                                 if (down_interruptible(
1409                                         &user_service->insert_event) != 0) {
1410                                         vchiq_log_info(vchiq_arm_log_level,
1411                                                 "DEQUEUE_MESSAGE interrupted");
1412                                         ret = -EINTR;
1413                                         break;
1414                                 }
1415                                 spin_lock(&msg_queue_spinlock);
1416                         } while (user_service->msg_remove ==
1417                                 user_service->msg_insert);
1418
1419                         if (ret)
1420                                 break;
1421                 }
1422
1423                 BUG_ON((int)(user_service->msg_insert -
1424                         user_service->msg_remove) < 0);
1425
1426                 header = user_service->msg_queue[user_service->msg_remove &
1427                         (MSG_QUEUE_SIZE - 1)];
1428                 user_service->msg_remove++;
1429                 spin_unlock(&msg_queue_spinlock);
1430
1431                 up(&user_service->remove_event);
1432                 if (header == NULL)
1433                         ret = -ENOTCONN;
1434                 else if (header->size <= args.bufsize) {
1435                         /* Copy to user space if msgbuf is not NULL */
1436                         if ((args.buf == NULL) ||
1437                                 (copy_to_user((void __user *)args.buf,
1438                                 header->data,
1439                                 header->size) == 0)) {
1440                                 ret = header->size;
1441                                 vchiq_release_message(
1442                                         service->handle,
1443                                         header);
1444                         } else
1445                                 ret = -EFAULT;
1446                 } else {
1447                         vchiq_log_error(vchiq_arm_log_level,
1448                                 "header %pK: bufsize %x < size %x",
1449                                 header, args.bufsize, header->size);
1450                         WARN(1, "invalid size\n");
1451                         ret = -EMSGSIZE;
1452                 }
1453                 DEBUG_TRACE(DEQUEUE_MESSAGE_LINE);
1454         } break;
1455
1456         case VCHIQ_IOC_GET_CLIENT_ID: {
1457                 VCHIQ_SERVICE_HANDLE_T handle = (VCHIQ_SERVICE_HANDLE_T)arg;
1458
1459                 ret = vchiq_get_client_id(handle);
1460         } break;
1461
1462         case VCHIQ_IOC_GET_CONFIG: {
1463                 VCHIQ_GET_CONFIG_T args;
1464                 VCHIQ_CONFIG_T config;
1465
1466                 if (copy_from_user(&args, (const void __user *)arg,
1467                         sizeof(args)) != 0) {
1468                         ret = -EFAULT;
1469                         break;
1470                 }
1471                 if (args.config_size > sizeof(config)) {
1472                         ret = -EINVAL;
1473                         break;
1474                 }
1475                 status = vchiq_get_config(instance, args.config_size, &config);
1476                 if (status == VCHIQ_SUCCESS) {
1477                         if (copy_to_user((void __user *)args.pconfig,
1478                                     &config, args.config_size) != 0) {
1479                                 ret = -EFAULT;
1480                                 break;
1481                         }
1482                 }
1483         } break;
1484
1485         case VCHIQ_IOC_SET_SERVICE_OPTION: {
1486                 VCHIQ_SET_SERVICE_OPTION_T args;
1487
1488                 if (copy_from_user(
1489                         &args, (const void __user *)arg,
1490                         sizeof(args)) != 0) {
1491                         ret = -EFAULT;
1492                         break;
1493                 }
1494
1495                 service = find_service_for_instance(instance, args.handle);
1496                 if (!service) {
1497                         ret = -EINVAL;
1498                         break;
1499                 }
1500
1501                 status = vchiq_set_service_option(
1502                                 args.handle, args.option, args.value);
1503         } break;
1504
1505         case VCHIQ_IOC_LIB_VERSION: {
1506                 unsigned int lib_version = (unsigned int)arg;
1507
1508                 if (lib_version < VCHIQ_VERSION_MIN)
1509                         ret = -EINVAL;
1510                 else if (lib_version >= VCHIQ_VERSION_CLOSE_DELIVERED)
1511                         instance->use_close_delivered = 1;
1512         } break;
1513
1514         case VCHIQ_IOC_CLOSE_DELIVERED: {
1515                 VCHIQ_SERVICE_HANDLE_T handle = (VCHIQ_SERVICE_HANDLE_T)arg;
1516
1517                 service = find_closed_service_for_instance(instance, handle);
1518                 if (service != NULL) {
1519                         USER_SERVICE_T *user_service =
1520                                 (USER_SERVICE_T *)service->base.userdata;
1521                         close_delivered(user_service);
1522                 } else
1523                         ret = -EINVAL;
1524         } break;
1525
1526         default:
1527                 ret = -ENOTTY;
1528                 break;
1529         }
1530
1531         if (service)
1532                 unlock_service(service);
1533
1534         if (ret == 0) {
1535                 if (status == VCHIQ_ERROR)
1536                         ret = -EIO;
1537                 else if (status == VCHIQ_RETRY)
1538                         ret = -EINTR;
1539         }
1540
1541         if ((status == VCHIQ_SUCCESS) && (ret < 0) && (ret != -EINTR) &&
1542                 (ret != -EWOULDBLOCK))
1543                 vchiq_log_info(vchiq_arm_log_level,
1544                         "  ioctl instance %lx, cmd %s -> status %d, %ld",
1545                         (unsigned long)instance,
1546                         (_IOC_NR(cmd) <= VCHIQ_IOC_MAX) ?
1547                                 ioctl_names[_IOC_NR(cmd)] :
1548                                 "<invalid>",
1549                         status, ret);
1550         else
1551                 vchiq_log_trace(vchiq_arm_log_level,
1552                         "  ioctl instance %lx, cmd %s -> status %d, %ld",
1553                         (unsigned long)instance,
1554                         (_IOC_NR(cmd) <= VCHIQ_IOC_MAX) ?
1555                                 ioctl_names[_IOC_NR(cmd)] :
1556                                 "<invalid>",
1557                         status, ret);
1558
1559         return ret;
1560 }
1561
1562 #if defined(CONFIG_COMPAT)
1563
1564 struct vchiq_service_params32 {
1565         int fourcc;
1566         compat_uptr_t callback;
1567         compat_uptr_t userdata;
1568         short version; /* Increment for non-trivial changes */
1569         short version_min; /* Update for incompatible changes */
1570 };
1571
1572 struct vchiq_create_service32 {
1573         struct vchiq_service_params32 params;
1574         int is_open;
1575         int is_vchi;
1576         unsigned int handle; /* OUT */
1577 };
1578
1579 #define VCHIQ_IOC_CREATE_SERVICE32 \
1580         _IOWR(VCHIQ_IOC_MAGIC, 2, struct vchiq_create_service32)
1581
1582 static long
1583 vchiq_compat_ioctl_create_service(
1584         struct file *file,
1585         unsigned int cmd,
1586         unsigned long arg)
1587 {
1588         VCHIQ_CREATE_SERVICE_T __user *args;
1589         struct vchiq_create_service32 __user *ptrargs32 =
1590                 (struct vchiq_create_service32 __user *)arg;
1591         struct vchiq_create_service32 args32;
1592         long ret;
1593
1594         args = compat_alloc_user_space(sizeof(*args));
1595         if (!args)
1596                 return -EFAULT;
1597
1598         if (copy_from_user(&args32,
1599                            (struct vchiq_create_service32 __user *)arg,
1600                            sizeof(args32)))
1601                 return -EFAULT;
1602
1603         if (put_user(args32.params.fourcc, &args->params.fourcc) ||
1604             put_user(compat_ptr(args32.params.callback),
1605                      &args->params.callback) ||
1606             put_user(compat_ptr(args32.params.userdata),
1607                      &args->params.userdata) ||
1608             put_user(args32.params.version, &args->params.version) ||
1609             put_user(args32.params.version_min,
1610                      &args->params.version_min) ||
1611             put_user(args32.is_open, &args->is_open) ||
1612             put_user(args32.is_vchi, &args->is_vchi) ||
1613             put_user(args32.handle, &args->handle))
1614                 return -EFAULT;
1615
1616         ret = vchiq_ioctl(file, VCHIQ_IOC_CREATE_SERVICE, (unsigned long)args);
1617
1618         if (ret < 0)
1619                 return ret;
1620
1621         if (get_user(args32.handle, &args->handle))
1622                 return -EFAULT;
1623
1624         if (copy_to_user(&ptrargs32->handle,
1625                          &args32.handle,
1626                          sizeof(args32.handle)))
1627                 return -EFAULT;
1628
1629         return 0;
1630 }
1631
1632 struct vchiq_element32 {
1633         compat_uptr_t data;
1634         unsigned int size;
1635 };
1636
1637 struct vchiq_queue_message32 {
1638         unsigned int handle;
1639         unsigned int count;
1640         compat_uptr_t elements;
1641 };
1642
1643 #define VCHIQ_IOC_QUEUE_MESSAGE32 \
1644         _IOW(VCHIQ_IOC_MAGIC,  4, struct vchiq_queue_message32)
1645
1646 static long
1647 vchiq_compat_ioctl_queue_message(struct file *file,
1648                                  unsigned int cmd,
1649                                  unsigned long arg)
1650 {
1651         VCHIQ_QUEUE_MESSAGE_T *args;
1652         struct vchiq_element *elements;
1653         struct vchiq_queue_message32 args32;
1654         unsigned int count;
1655
1656         if (copy_from_user(&args32,
1657                            (struct vchiq_queue_message32 __user *)arg,
1658                            sizeof(args32)))
1659                 return -EFAULT;
1660
1661         args = compat_alloc_user_space(sizeof(*args) +
1662                                        (sizeof(*elements) * MAX_ELEMENTS));
1663
1664         if (!args)
1665                 return -EFAULT;
1666
1667         if (put_user(args32.handle, &args->handle) ||
1668             put_user(args32.count, &args->count) ||
1669             put_user(compat_ptr(args32.elements), &args->elements))
1670                 return -EFAULT;
1671
1672         if (args32.count > MAX_ELEMENTS)
1673                 return -EINVAL;
1674
1675         if (args32.elements && args32.count) {
1676                 struct vchiq_element32 tempelement32[MAX_ELEMENTS];
1677
1678                 elements = (struct vchiq_element __user *)(args + 1);
1679
1680                 if (copy_from_user(&tempelement32,
1681                                    compat_ptr(args32.elements),
1682                                    sizeof(tempelement32)))
1683                         return -EFAULT;
1684
1685                 for (count = 0; count < args32.count; count++) {
1686                         if (put_user(compat_ptr(tempelement32[count].data),
1687                                      &elements[count].data) ||
1688                             put_user(tempelement32[count].size,
1689                                      &elements[count].size))
1690                                 return -EFAULT;
1691                 }
1692
1693                 if (put_user(elements, &args->elements))
1694                         return -EFAULT;
1695         }
1696
1697         return vchiq_ioctl(file, VCHIQ_IOC_QUEUE_MESSAGE, (unsigned long)args);
1698 }
1699
1700 struct vchiq_queue_bulk_transfer32 {
1701         unsigned int handle;
1702         compat_uptr_t data;
1703         unsigned int size;
1704         compat_uptr_t userdata;
1705         VCHIQ_BULK_MODE_T mode;
1706 };
1707
1708 #define VCHIQ_IOC_QUEUE_BULK_TRANSMIT32 \
1709         _IOWR(VCHIQ_IOC_MAGIC, 5, struct vchiq_queue_bulk_transfer32)
1710 #define VCHIQ_IOC_QUEUE_BULK_RECEIVE32 \
1711         _IOWR(VCHIQ_IOC_MAGIC, 6, struct vchiq_queue_bulk_transfer32)
1712
1713 static long
1714 vchiq_compat_ioctl_queue_bulk(struct file *file,
1715                               unsigned int cmd,
1716                               unsigned long arg)
1717 {
1718         VCHIQ_QUEUE_BULK_TRANSFER_T *args;
1719         struct vchiq_queue_bulk_transfer32 args32;
1720         struct vchiq_queue_bulk_transfer32 *ptrargs32 =
1721                 (struct vchiq_queue_bulk_transfer32 *)arg;
1722         long ret;
1723
1724         args = compat_alloc_user_space(sizeof(*args));
1725         if (!args)
1726                 return -EFAULT;
1727
1728         if (copy_from_user(&args32,
1729                            (struct vchiq_queue_bulk_transfer32 __user *)arg,
1730                            sizeof(args32)))
1731                 return -EFAULT;
1732
1733         if (put_user(args32.handle, &args->handle) ||
1734             put_user(compat_ptr(args32.data), &args->data) ||
1735             put_user(args32.size, &args->size) ||
1736             put_user(compat_ptr(args32.userdata), &args->userdata) ||
1737             put_user(args32.mode, &args->mode))
1738                 return -EFAULT;
1739
1740         if (cmd == VCHIQ_IOC_QUEUE_BULK_TRANSMIT32)
1741                 cmd = VCHIQ_IOC_QUEUE_BULK_TRANSMIT;
1742         else
1743                 cmd = VCHIQ_IOC_QUEUE_BULK_RECEIVE;
1744
1745         ret = vchiq_ioctl(file, cmd, (unsigned long)args);
1746
1747         if (ret < 0)
1748                 return ret;
1749
1750         if (get_user(args32.mode, &args->mode))
1751                 return -EFAULT;
1752
1753         if (copy_to_user(&ptrargs32->mode,
1754                          &args32.mode,
1755                          sizeof(args32.mode)))
1756                 return -EFAULT;
1757
1758         return 0;
1759 }
1760
1761 struct vchiq_completion_data32 {
1762         VCHIQ_REASON_T reason;
1763         compat_uptr_t header;
1764         compat_uptr_t service_userdata;
1765         compat_uptr_t bulk_userdata;
1766 };
1767
1768 struct vchiq_await_completion32 {
1769         unsigned int count;
1770         compat_uptr_t buf;
1771         unsigned int msgbufsize;
1772         unsigned int msgbufcount; /* IN/OUT */
1773         compat_uptr_t msgbufs;
1774 };
1775
1776 #define VCHIQ_IOC_AWAIT_COMPLETION32 \
1777         _IOWR(VCHIQ_IOC_MAGIC, 7, struct vchiq_await_completion32)
1778
1779 static long
1780 vchiq_compat_ioctl_await_completion(struct file *file,
1781                                     unsigned int cmd,
1782                                     unsigned long arg)
1783 {
1784         VCHIQ_AWAIT_COMPLETION_T *args;
1785         VCHIQ_COMPLETION_DATA_T *completion;
1786         VCHIQ_COMPLETION_DATA_T completiontemp;
1787         struct vchiq_await_completion32 args32;
1788         struct vchiq_completion_data32 completion32;
1789         unsigned int *msgbufcount32;
1790         unsigned int msgbufcount_native;
1791         compat_uptr_t msgbuf32;
1792         void *msgbuf;
1793         void **msgbufptr;
1794         long ret;
1795
1796         args = compat_alloc_user_space(sizeof(*args) +
1797                                        sizeof(*completion) +
1798                                        sizeof(*msgbufptr));
1799         if (!args)
1800                 return -EFAULT;
1801
1802         completion = (VCHIQ_COMPLETION_DATA_T *)(args + 1);
1803         msgbufptr = (void __user **)(completion + 1);
1804
1805         if (copy_from_user(&args32,
1806                            (struct vchiq_completion_data32 *)arg,
1807                            sizeof(args32)))
1808                 return -EFAULT;
1809
1810         if (put_user(args32.count, &args->count) ||
1811             put_user(compat_ptr(args32.buf), &args->buf) ||
1812             put_user(args32.msgbufsize, &args->msgbufsize) ||
1813             put_user(args32.msgbufcount, &args->msgbufcount) ||
1814             put_user(compat_ptr(args32.msgbufs), &args->msgbufs))
1815                 return -EFAULT;
1816
1817         /* These are simple cases, so just fall into the native handler */
1818         if (!args32.count || !args32.buf || !args32.msgbufcount)
1819                 return vchiq_ioctl(file,
1820                                    VCHIQ_IOC_AWAIT_COMPLETION,
1821                                    (unsigned long)args);
1822
1823         /*
1824          * These are the more complex cases.  Typical applications of this
1825          * ioctl will use a very large count, with a very large msgbufcount.
1826          * Since the native ioctl can asynchronously fill in the returned
1827          * buffers and the application can in theory begin processing messages
1828          * even before the ioctl returns, a bit of a trick is used here.
1829          *
1830          * By forcing both count and msgbufcount to be 1, it forces the native
1831          * ioctl to only claim at most 1 message is available.   This tricks
1832          * the calling application into thinking only 1 message was actually
1833          * available in the queue so like all good applications it will retry
1834          * waiting until all the required messages are received.
1835          *
1836          * This trick has been tested and proven to work with vchiq_test,
1837          * Minecraft_PI, the "hello pi" examples, and various other
1838          * applications that are included in Raspbian.
1839          */
1840
1841         if (copy_from_user(&msgbuf32,
1842                            compat_ptr(args32.msgbufs) +
1843                            (sizeof(compat_uptr_t) *
1844                            (args32.msgbufcount - 1)),
1845                            sizeof(msgbuf32)))
1846                 return -EFAULT;
1847
1848         msgbuf = compat_ptr(msgbuf32);
1849
1850         if (copy_to_user(msgbufptr,
1851                          &msgbuf,
1852                          sizeof(msgbuf)))
1853                 return -EFAULT;
1854
1855         if (copy_to_user(&args->msgbufs,
1856                          &msgbufptr,
1857                          sizeof(msgbufptr)))
1858                 return -EFAULT;
1859
1860         if (put_user(1U, &args->count) ||
1861             put_user(completion, &args->buf) ||
1862             put_user(1U, &args->msgbufcount))
1863                 return -EFAULT;
1864
1865         ret = vchiq_ioctl(file,
1866                           VCHIQ_IOC_AWAIT_COMPLETION,
1867                           (unsigned long)args);
1868
1869         /*
1870          * An return value of 0 here means that no messages where available
1871          * in the message queue.  In this case the native ioctl does not
1872          * return any data to the application at all.  Not even to update
1873          * msgbufcount.  This functionality needs to be kept here for
1874          * compatibility.
1875          *
1876          * Of course, < 0 means that an error occurred and no data is being
1877          * returned.
1878          *
1879          * Since count and msgbufcount was forced to 1, that means
1880          * the only other possible return value is 1. Meaning that 1 message
1881          * was available, so that multiple message case does not need to be
1882          * handled here.
1883          */
1884         if (ret <= 0)
1885                 return ret;
1886
1887         if (copy_from_user(&completiontemp, completion, sizeof(*completion)))
1888                 return -EFAULT;
1889
1890         completion32.reason = completiontemp.reason;
1891         completion32.header = ptr_to_compat(completiontemp.header);
1892         completion32.service_userdata =
1893                 ptr_to_compat(completiontemp.service_userdata);
1894         completion32.bulk_userdata =
1895                 ptr_to_compat(completiontemp.bulk_userdata);
1896
1897         if (copy_to_user(compat_ptr(args32.buf),
1898                          &completion32,
1899                          sizeof(completion32)))
1900                 return -EFAULT;
1901
1902         if (get_user(msgbufcount_native, &args->msgbufcount))
1903                 return -EFAULT;
1904
1905         if (!msgbufcount_native)
1906                 args32.msgbufcount--;
1907
1908         msgbufcount32 =
1909                 &((struct vchiq_await_completion32 __user *)arg)->msgbufcount;
1910
1911         if (copy_to_user(msgbufcount32,
1912                          &args32.msgbufcount,
1913                          sizeof(args32.msgbufcount)))
1914                 return -EFAULT;
1915
1916         return 1;
1917 }
1918
1919 struct vchiq_dequeue_message32 {
1920         unsigned int handle;
1921         int blocking;
1922         unsigned int bufsize;
1923         compat_uptr_t buf;
1924 };
1925
1926 #define VCHIQ_IOC_DEQUEUE_MESSAGE32 \
1927         _IOWR(VCHIQ_IOC_MAGIC, 8, struct vchiq_dequeue_message32)
1928
1929 static long
1930 vchiq_compat_ioctl_dequeue_message(struct file *file,
1931                                    unsigned int cmd,
1932                                    unsigned long arg)
1933 {
1934         VCHIQ_DEQUEUE_MESSAGE_T *args;
1935         struct vchiq_dequeue_message32 args32;
1936
1937         args = compat_alloc_user_space(sizeof(*args));
1938         if (!args)
1939                 return -EFAULT;
1940
1941         if (copy_from_user(&args32,
1942                            (struct vchiq_dequeue_message32 *)arg,
1943                            sizeof(args32)))
1944                 return -EFAULT;
1945
1946         if (put_user(args32.handle, &args->handle) ||
1947             put_user(args32.blocking, &args->blocking) ||
1948             put_user(args32.bufsize, &args->bufsize) ||
1949             put_user(compat_ptr(args32.buf), &args->buf))
1950                 return -EFAULT;
1951
1952         return vchiq_ioctl(file, VCHIQ_IOC_DEQUEUE_MESSAGE,
1953                            (unsigned long)args);
1954 }
1955
1956 struct vchiq_get_config32 {
1957         unsigned int config_size;
1958         compat_uptr_t pconfig;
1959 };
1960
1961 #define VCHIQ_IOC_GET_CONFIG32 \
1962         _IOWR(VCHIQ_IOC_MAGIC, 10, struct vchiq_get_config32)
1963
1964 static long
1965 vchiq_compat_ioctl_get_config(struct file *file,
1966                               unsigned int cmd,
1967                               unsigned long arg)
1968 {
1969         VCHIQ_GET_CONFIG_T *args;
1970         struct vchiq_get_config32 args32;
1971
1972         args = compat_alloc_user_space(sizeof(*args));
1973         if (!args)
1974                 return -EFAULT;
1975
1976         if (copy_from_user(&args32,
1977                            (struct vchiq_get_config32 *)arg,
1978                            sizeof(args32)))
1979                 return -EFAULT;
1980
1981         if (put_user(args32.config_size, &args->config_size) ||
1982             put_user(compat_ptr(args32.pconfig), &args->pconfig))
1983                 return -EFAULT;
1984
1985         return vchiq_ioctl(file, VCHIQ_IOC_GET_CONFIG, (unsigned long)args);
1986 }
1987
1988 static long
1989 vchiq_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
1990 {
1991         switch (cmd) {
1992         case VCHIQ_IOC_CREATE_SERVICE32:
1993                 return vchiq_compat_ioctl_create_service(file, cmd, arg);
1994         case VCHIQ_IOC_QUEUE_MESSAGE32:
1995                 return vchiq_compat_ioctl_queue_message(file, cmd, arg);
1996         case VCHIQ_IOC_QUEUE_BULK_TRANSMIT32:
1997         case VCHIQ_IOC_QUEUE_BULK_RECEIVE32:
1998                 return vchiq_compat_ioctl_queue_bulk(file, cmd, arg);
1999         case VCHIQ_IOC_AWAIT_COMPLETION32:
2000                 return vchiq_compat_ioctl_await_completion(file, cmd, arg);
2001         case VCHIQ_IOC_DEQUEUE_MESSAGE32:
2002                 return vchiq_compat_ioctl_dequeue_message(file, cmd, arg);
2003         case VCHIQ_IOC_GET_CONFIG32:
2004                 return vchiq_compat_ioctl_get_config(file, cmd, arg);
2005         default:
2006                 return vchiq_ioctl(file, cmd, arg);
2007         }
2008 }
2009
2010 #endif
2011
2012 /****************************************************************************
2013 *
2014 *   vchiq_open
2015 *
2016 ***************************************************************************/
2017
2018 static int
2019 vchiq_open(struct inode *inode, struct file *file)
2020 {
2021         int dev = iminor(inode) & 0x0f;
2022
2023         vchiq_log_info(vchiq_arm_log_level, "vchiq_open");
2024         switch (dev) {
2025         case VCHIQ_MINOR: {
2026                 VCHIQ_STATE_T *state = vchiq_get_state();
2027                 VCHIQ_INSTANCE_T instance;
2028
2029                 if (!state) {
2030                         vchiq_log_error(vchiq_arm_log_level,
2031                                 "vchiq has no connection to VideoCore");
2032                         return -ENOTCONN;
2033                 }
2034
2035                 instance = kzalloc(sizeof(*instance), GFP_KERNEL);
2036                 if (!instance)
2037                         return -ENOMEM;
2038
2039                 instance->state = state;
2040                 instance->pid = current->tgid;
2041
2042                 vchiq_debugfs_add_instance(instance);
2043
2044                 sema_init(&instance->insert_event, 0);
2045                 sema_init(&instance->remove_event, 0);
2046                 mutex_init(&instance->completion_mutex);
2047                 mutex_init(&instance->bulk_waiter_list_mutex);
2048                 INIT_LIST_HEAD(&instance->bulk_waiter_list);
2049
2050                 file->private_data = instance;
2051         } break;
2052
2053         default:
2054                 vchiq_log_error(vchiq_arm_log_level,
2055                         "Unknown minor device: %d", dev);
2056                 return -ENXIO;
2057         }
2058
2059         return 0;
2060 }
2061
2062 /****************************************************************************
2063 *
2064 *   vchiq_release
2065 *
2066 ***************************************************************************/
2067
2068 static int
2069 vchiq_release(struct inode *inode, struct file *file)
2070 {
2071         int dev = iminor(inode) & 0x0f;
2072         int ret = 0;
2073
2074         switch (dev) {
2075         case VCHIQ_MINOR: {
2076                 VCHIQ_INSTANCE_T instance = file->private_data;
2077                 VCHIQ_STATE_T *state = vchiq_get_state();
2078                 VCHIQ_SERVICE_T *service;
2079                 int i;
2080
2081                 vchiq_log_info(vchiq_arm_log_level,
2082                         "%s: instance=%lx",
2083                         __func__, (unsigned long)instance);
2084
2085                 if (!state) {
2086                         ret = -EPERM;
2087                         goto out;
2088                 }
2089
2090                 /* Ensure videocore is awake to allow termination. */
2091                 vchiq_use_internal(instance->state, NULL,
2092                                 USE_TYPE_VCHIQ);
2093
2094                 mutex_lock(&instance->completion_mutex);
2095
2096                 /* Wake the completion thread and ask it to exit */
2097                 instance->closing = 1;
2098                 up(&instance->insert_event);
2099
2100                 mutex_unlock(&instance->completion_mutex);
2101
2102                 /* Wake the slot handler if the completion queue is full. */
2103                 up(&instance->remove_event);
2104
2105                 /* Mark all services for termination... */
2106                 i = 0;
2107                 while ((service = next_service_by_instance(state, instance,
2108                         &i)) != NULL) {
2109                         USER_SERVICE_T *user_service = service->base.userdata;
2110
2111                         /* Wake the slot handler if the msg queue is full. */
2112                         up(&user_service->remove_event);
2113
2114                         vchiq_terminate_service_internal(service);
2115                         unlock_service(service);
2116                 }
2117
2118                 /* ...and wait for them to die */
2119                 i = 0;
2120                 while ((service = next_service_by_instance(state, instance, &i))
2121                         != NULL) {
2122                         USER_SERVICE_T *user_service = service->base.userdata;
2123
2124                         down(&service->remove_event);
2125
2126                         BUG_ON(service->srvstate != VCHIQ_SRVSTATE_FREE);
2127
2128                         spin_lock(&msg_queue_spinlock);
2129
2130                         while (user_service->msg_remove !=
2131                                 user_service->msg_insert) {
2132                                 VCHIQ_HEADER_T *header;
2133                                 int m = user_service->msg_remove &
2134                                         (MSG_QUEUE_SIZE - 1);
2135
2136                                 header = user_service->msg_queue[m];
2137                                 user_service->msg_remove++;
2138                                 spin_unlock(&msg_queue_spinlock);
2139
2140                                 if (header)
2141                                         vchiq_release_message(
2142                                                 service->handle,
2143                                                 header);
2144                                 spin_lock(&msg_queue_spinlock);
2145                         }
2146
2147                         spin_unlock(&msg_queue_spinlock);
2148
2149                         unlock_service(service);
2150                 }
2151
2152                 /* Release any closed services */
2153                 while (instance->completion_remove !=
2154                         instance->completion_insert) {
2155                         VCHIQ_COMPLETION_DATA_T *completion;
2156                         VCHIQ_SERVICE_T *service;
2157
2158                         completion = &instance->completions[
2159                                 instance->completion_remove &
2160                                 (MAX_COMPLETIONS - 1)];
2161                         service = completion->service_userdata;
2162                         if (completion->reason == VCHIQ_SERVICE_CLOSED) {
2163                                 USER_SERVICE_T *user_service =
2164                                         service->base.userdata;
2165
2166                                 /* Wake any blocked user-thread */
2167                                 if (instance->use_close_delivered)
2168                                         up(&user_service->close_event);
2169                                 unlock_service(service);
2170                         }
2171                         instance->completion_remove++;
2172                 }
2173
2174                 /* Release the PEER service count. */
2175                 vchiq_release_internal(instance->state, NULL);
2176
2177                 {
2178                         struct list_head *pos, *next;
2179
2180                         list_for_each_safe(pos, next,
2181                                 &instance->bulk_waiter_list) {
2182                                 struct bulk_waiter_node *waiter;
2183
2184                                 waiter = list_entry(pos,
2185                                         struct bulk_waiter_node,
2186                                         list);
2187                                 list_del(pos);
2188                                 vchiq_log_info(vchiq_arm_log_level,
2189                                         "bulk_waiter - cleaned up %pK for pid %d",
2190                                         waiter, waiter->pid);
2191                                 kfree(waiter);
2192                         }
2193                 }
2194
2195                 vchiq_debugfs_remove_instance(instance);
2196
2197                 kfree(instance);
2198                 file->private_data = NULL;
2199         } break;
2200
2201         default:
2202                 vchiq_log_error(vchiq_arm_log_level,
2203                         "Unknown minor device: %d", dev);
2204                 ret = -ENXIO;
2205         }
2206
2207 out:
2208         return ret;
2209 }
2210
2211 /****************************************************************************
2212 *
2213 *   vchiq_dump
2214 *
2215 ***************************************************************************/
2216
2217 void
2218 vchiq_dump(void *dump_context, const char *str, int len)
2219 {
2220         DUMP_CONTEXT_T *context = (DUMP_CONTEXT_T *)dump_context;
2221
2222         if (context->actual < context->space) {
2223                 int copy_bytes;
2224
2225                 if (context->offset > 0) {
2226                         int skip_bytes = min(len, (int)context->offset);
2227
2228                         str += skip_bytes;
2229                         len -= skip_bytes;
2230                         context->offset -= skip_bytes;
2231                         if (context->offset > 0)
2232                                 return;
2233                 }
2234                 copy_bytes = min(len, (int)(context->space - context->actual));
2235                 if (copy_bytes == 0)
2236                         return;
2237                 if (copy_to_user(context->buf + context->actual, str,
2238                         copy_bytes))
2239                         context->actual = -EFAULT;
2240                 context->actual += copy_bytes;
2241                 len -= copy_bytes;
2242
2243                 /* If tne terminating NUL is included in the length, then it
2244                 ** marks the end of a line and should be replaced with a
2245                 ** carriage return. */
2246                 if ((len == 0) && (str[copy_bytes - 1] == '\0')) {
2247                         char cr = '\n';
2248
2249                         if (copy_to_user(context->buf + context->actual - 1,
2250                                 &cr, 1))
2251                                 context->actual = -EFAULT;
2252                 }
2253         }
2254 }
2255
2256 /****************************************************************************
2257 *
2258 *   vchiq_dump_platform_instance_state
2259 *
2260 ***************************************************************************/
2261
2262 void
2263 vchiq_dump_platform_instances(void *dump_context)
2264 {
2265         VCHIQ_STATE_T *state = vchiq_get_state();
2266         char buf[80];
2267         int len;
2268         int i;
2269
2270         /* There is no list of instances, so instead scan all services,
2271                 marking those that have been dumped. */
2272
2273         for (i = 0; i < state->unused_service; i++) {
2274                 VCHIQ_SERVICE_T *service = state->services[i];
2275                 VCHIQ_INSTANCE_T instance;
2276
2277                 if (service && (service->base.callback == service_callback)) {
2278                         instance = service->instance;
2279                         if (instance)
2280                                 instance->mark = 0;
2281                 }
2282         }
2283
2284         for (i = 0; i < state->unused_service; i++) {
2285                 VCHIQ_SERVICE_T *service = state->services[i];
2286                 VCHIQ_INSTANCE_T instance;
2287
2288                 if (service && (service->base.callback == service_callback)) {
2289                         instance = service->instance;
2290                         if (instance && !instance->mark) {
2291                                 len = snprintf(buf, sizeof(buf),
2292                                         "Instance %pK: pid %d,%s completions %d/%d",
2293                                         instance, instance->pid,
2294                                         instance->connected ? " connected, " :
2295                                                 "",
2296                                         instance->completion_insert -
2297                                                 instance->completion_remove,
2298                                         MAX_COMPLETIONS);
2299
2300                                 vchiq_dump(dump_context, buf, len + 1);
2301
2302                                 instance->mark = 1;
2303                         }
2304                 }
2305         }
2306 }
2307
2308 /****************************************************************************
2309 *
2310 *   vchiq_dump_platform_service_state
2311 *
2312 ***************************************************************************/
2313
2314 void
2315 vchiq_dump_platform_service_state(void *dump_context, VCHIQ_SERVICE_T *service)
2316 {
2317         USER_SERVICE_T *user_service = (USER_SERVICE_T *)service->base.userdata;
2318         char buf[80];
2319         int len;
2320
2321         len = snprintf(buf, sizeof(buf), "  instance %pK", service->instance);
2322
2323         if ((service->base.callback == service_callback) &&
2324                 user_service->is_vchi) {
2325                 len += snprintf(buf + len, sizeof(buf) - len,
2326                         ", %d/%d messages",
2327                         user_service->msg_insert - user_service->msg_remove,
2328                         MSG_QUEUE_SIZE);
2329
2330                 if (user_service->dequeue_pending)
2331                         len += snprintf(buf + len, sizeof(buf) - len,
2332                                 " (dequeue pending)");
2333         }
2334
2335         vchiq_dump(dump_context, buf, len + 1);
2336 }
2337
2338 /****************************************************************************
2339 *
2340 *   vchiq_read
2341 *
2342 ***************************************************************************/
2343
2344 static ssize_t
2345 vchiq_read(struct file *file, char __user *buf,
2346         size_t count, loff_t *ppos)
2347 {
2348         DUMP_CONTEXT_T context;
2349
2350         context.buf = buf;
2351         context.actual = 0;
2352         context.space = count;
2353         context.offset = *ppos;
2354
2355         vchiq_dump_state(&context, &g_state);
2356
2357         *ppos += context.actual;
2358
2359         return context.actual;
2360 }
2361
2362 VCHIQ_STATE_T *
2363 vchiq_get_state(void)
2364 {
2365
2366         if (g_state.remote == NULL)
2367                 printk(KERN_ERR "%s: g_state.remote == NULL\n", __func__);
2368         else if (g_state.remote->initialised != 1)
2369                 printk(KERN_NOTICE "%s: g_state.remote->initialised != 1 (%d)\n",
2370                         __func__, g_state.remote->initialised);
2371
2372         return ((g_state.remote != NULL) &&
2373                 (g_state.remote->initialised == 1)) ? &g_state : NULL;
2374 }
2375
2376 static const struct file_operations
2377 vchiq_fops = {
2378         .owner = THIS_MODULE,
2379         .unlocked_ioctl = vchiq_ioctl,
2380 #if defined(CONFIG_COMPAT)
2381         .compat_ioctl = vchiq_compat_ioctl,
2382 #endif
2383         .open = vchiq_open,
2384         .release = vchiq_release,
2385         .read = vchiq_read
2386 };
2387
2388 /*
2389  * Autosuspend related functionality
2390  */
2391
2392 int
2393 vchiq_videocore_wanted(VCHIQ_STATE_T *state)
2394 {
2395         VCHIQ_ARM_STATE_T *arm_state = vchiq_platform_get_arm_state(state);
2396
2397         if (!arm_state)
2398                 /* autosuspend not supported - always return wanted */
2399                 return 1;
2400         else if (arm_state->blocked_count)
2401                 return 1;
2402         else if (!arm_state->videocore_use_count)
2403                 /* usage count zero - check for override unless we're forcing */
2404                 if (arm_state->resume_blocked)
2405                         return 0;
2406                 else
2407                         return vchiq_platform_videocore_wanted(state);
2408         else
2409                 /* non-zero usage count - videocore still required */
2410                 return 1;
2411 }
2412
2413 static VCHIQ_STATUS_T
2414 vchiq_keepalive_vchiq_callback(VCHIQ_REASON_T reason,
2415         VCHIQ_HEADER_T *header,
2416         VCHIQ_SERVICE_HANDLE_T service_user,
2417         void *bulk_user)
2418 {
2419         vchiq_log_error(vchiq_susp_log_level,
2420                 "%s callback reason %d", __func__, reason);
2421         return 0;
2422 }
2423
2424 static int
2425 vchiq_keepalive_thread_func(void *v)
2426 {
2427         VCHIQ_STATE_T *state = (VCHIQ_STATE_T *) v;
2428         VCHIQ_ARM_STATE_T *arm_state = vchiq_platform_get_arm_state(state);
2429
2430         VCHIQ_STATUS_T status;
2431         VCHIQ_INSTANCE_T instance;
2432         VCHIQ_SERVICE_HANDLE_T ka_handle;
2433
2434         VCHIQ_SERVICE_PARAMS_T params = {
2435                 .fourcc      = VCHIQ_MAKE_FOURCC('K', 'E', 'E', 'P'),
2436                 .callback    = vchiq_keepalive_vchiq_callback,
2437                 .version     = KEEPALIVE_VER,
2438                 .version_min = KEEPALIVE_VER_MIN
2439         };
2440
2441         status = vchiq_initialise(&instance);
2442         if (status != VCHIQ_SUCCESS) {
2443                 vchiq_log_error(vchiq_susp_log_level,
2444                         "%s vchiq_initialise failed %d", __func__, status);
2445                 goto exit;
2446         }
2447
2448         status = vchiq_connect(instance);
2449         if (status != VCHIQ_SUCCESS) {
2450                 vchiq_log_error(vchiq_susp_log_level,
2451                         "%s vchiq_connect failed %d", __func__, status);
2452                 goto shutdown;
2453         }
2454
2455         status = vchiq_add_service(instance, &params, &ka_handle);
2456         if (status != VCHIQ_SUCCESS) {
2457                 vchiq_log_error(vchiq_susp_log_level,
2458                         "%s vchiq_open_service failed %d", __func__, status);
2459                 goto shutdown;
2460         }
2461
2462         while (1) {
2463                 long rc = 0, uc = 0;
2464
2465                 if (wait_for_completion_interruptible(&arm_state->ka_evt)
2466                                 != 0) {
2467                         vchiq_log_error(vchiq_susp_log_level,
2468                                 "%s interrupted", __func__);
2469                         flush_signals(current);
2470                         continue;
2471                 }
2472
2473                 /* read and clear counters.  Do release_count then use_count to
2474                  * prevent getting more releases than uses */
2475                 rc = atomic_xchg(&arm_state->ka_release_count, 0);
2476                 uc = atomic_xchg(&arm_state->ka_use_count, 0);
2477
2478                 /* Call use/release service the requisite number of times.
2479                  * Process use before release so use counts don't go negative */
2480                 while (uc--) {
2481                         atomic_inc(&arm_state->ka_use_ack_count);
2482                         status = vchiq_use_service(ka_handle);
2483                         if (status != VCHIQ_SUCCESS) {
2484                                 vchiq_log_error(vchiq_susp_log_level,
2485                                         "%s vchiq_use_service error %d",
2486                                         __func__, status);
2487                         }
2488                 }
2489                 while (rc--) {
2490                         status = vchiq_release_service(ka_handle);
2491                         if (status != VCHIQ_SUCCESS) {
2492                                 vchiq_log_error(vchiq_susp_log_level,
2493                                         "%s vchiq_release_service error %d",
2494                                         __func__, status);
2495                         }
2496                 }
2497         }
2498
2499 shutdown:
2500         vchiq_shutdown(instance);
2501 exit:
2502         return 0;
2503 }
2504
2505 VCHIQ_STATUS_T
2506 vchiq_arm_init_state(VCHIQ_STATE_T *state, VCHIQ_ARM_STATE_T *arm_state)
2507 {
2508         if (arm_state) {
2509                 rwlock_init(&arm_state->susp_res_lock);
2510
2511                 init_completion(&arm_state->ka_evt);
2512                 atomic_set(&arm_state->ka_use_count, 0);
2513                 atomic_set(&arm_state->ka_use_ack_count, 0);
2514                 atomic_set(&arm_state->ka_release_count, 0);
2515
2516                 init_completion(&arm_state->vc_suspend_complete);
2517
2518                 init_completion(&arm_state->vc_resume_complete);
2519                 /* Initialise to 'done' state.  We only want to block on resume
2520                  * completion while videocore is suspended. */
2521                 set_resume_state(arm_state, VC_RESUME_RESUMED);
2522
2523                 init_completion(&arm_state->resume_blocker);
2524                 /* Initialise to 'done' state.  We only want to block on this
2525                  * completion while resume is blocked */
2526                 complete_all(&arm_state->resume_blocker);
2527
2528                 init_completion(&arm_state->blocked_blocker);
2529                 /* Initialise to 'done' state.  We only want to block on this
2530                  * completion while things are waiting on the resume blocker */
2531                 complete_all(&arm_state->blocked_blocker);
2532
2533                 arm_state->suspend_timer_timeout = SUSPEND_TIMER_TIMEOUT_MS;
2534                 arm_state->suspend_timer_running = 0;
2535                 arm_state->state = state;
2536                 timer_setup(&arm_state->suspend_timer, suspend_timer_callback,
2537                             0);
2538
2539                 arm_state->first_connect = 0;
2540
2541         }
2542         return VCHIQ_SUCCESS;
2543 }
2544
2545 /*
2546 ** Functions to modify the state variables;
2547 **      set_suspend_state
2548 **      set_resume_state
2549 **
2550 ** There are more state variables than we might like, so ensure they remain in
2551 ** step.  Suspend and resume state are maintained separately, since most of
2552 ** these state machines can operate independently.  However, there are a few
2553 ** states where state transitions in one state machine cause a reset to the
2554 ** other state machine.  In addition, there are some completion events which
2555 ** need to occur on state machine reset and end-state(s), so these are also
2556 ** dealt with in these functions.
2557 **
2558 ** In all states we set the state variable according to the input, but in some
2559 ** cases we perform additional steps outlined below;
2560 **
2561 ** VC_SUSPEND_IDLE - Initialise the suspend completion at the same time.
2562 **                      The suspend completion is completed after any suspend
2563 **                      attempt.  When we reset the state machine we also reset
2564 **                      the completion.  This reset occurs when videocore is
2565 **                      resumed, and also if we initiate suspend after a suspend
2566 **                      failure.
2567 **
2568 ** VC_SUSPEND_IN_PROGRESS - This state is considered the point of no return for
2569 **                      suspend - ie from this point on we must try to suspend
2570 **                      before resuming can occur.  We therefore also reset the
2571 **                      resume state machine to VC_RESUME_IDLE in this state.
2572 **
2573 ** VC_SUSPEND_SUSPENDED - Suspend has completed successfully. Also call
2574 **                      complete_all on the suspend completion to notify
2575 **                      anything waiting for suspend to happen.
2576 **
2577 ** VC_SUSPEND_REJECTED - Videocore rejected suspend. Videocore will also
2578 **                      initiate resume, so no need to alter resume state.
2579 **                      We call complete_all on the suspend completion to notify
2580 **                      of suspend rejection.
2581 **
2582 ** VC_SUSPEND_FAILED - We failed to initiate videocore suspend.  We notify the
2583 **                      suspend completion and reset the resume state machine.
2584 **
2585 ** VC_RESUME_IDLE - Initialise the resume completion at the same time.  The
2586 **                      resume completion is in it's 'done' state whenever
2587 **                      videcore is running.  Therefore, the VC_RESUME_IDLE
2588 **                      state implies that videocore is suspended.
2589 **                      Hence, any thread which needs to wait until videocore is
2590 **                      running can wait on this completion - it will only block
2591 **                      if videocore is suspended.
2592 **
2593 ** VC_RESUME_RESUMED - Resume has completed successfully.  Videocore is running.
2594 **                      Call complete_all on the resume completion to unblock
2595 **                      any threads waiting for resume.  Also reset the suspend
2596 **                      state machine to it's idle state.
2597 **
2598 ** VC_RESUME_FAILED - Currently unused - no mechanism to fail resume exists.
2599 */
2600
2601 void
2602 set_suspend_state(VCHIQ_ARM_STATE_T *arm_state,
2603         enum vc_suspend_status new_state)
2604 {
2605         /* set the state in all cases */
2606         arm_state->vc_suspend_state = new_state;
2607
2608         /* state specific additional actions */
2609         switch (new_state) {
2610         case VC_SUSPEND_FORCE_CANCELED:
2611                 complete_all(&arm_state->vc_suspend_complete);
2612                 break;
2613         case VC_SUSPEND_REJECTED:
2614                 complete_all(&arm_state->vc_suspend_complete);
2615                 break;
2616         case VC_SUSPEND_FAILED:
2617                 complete_all(&arm_state->vc_suspend_complete);
2618                 arm_state->vc_resume_state = VC_RESUME_RESUMED;
2619                 complete_all(&arm_state->vc_resume_complete);
2620                 break;
2621         case VC_SUSPEND_IDLE:
2622                 reinit_completion(&arm_state->vc_suspend_complete);
2623                 break;
2624         case VC_SUSPEND_REQUESTED:
2625                 break;
2626         case VC_SUSPEND_IN_PROGRESS:
2627                 set_resume_state(arm_state, VC_RESUME_IDLE);
2628                 break;
2629         case VC_SUSPEND_SUSPENDED:
2630                 complete_all(&arm_state->vc_suspend_complete);
2631                 break;
2632         default:
2633                 BUG();
2634                 break;
2635         }
2636 }
2637
2638 void
2639 set_resume_state(VCHIQ_ARM_STATE_T *arm_state,
2640         enum vc_resume_status new_state)
2641 {
2642         /* set the state in all cases */
2643         arm_state->vc_resume_state = new_state;
2644
2645         /* state specific additional actions */
2646         switch (new_state) {
2647         case VC_RESUME_FAILED:
2648                 break;
2649         case VC_RESUME_IDLE:
2650                 reinit_completion(&arm_state->vc_resume_complete);
2651                 break;
2652         case VC_RESUME_REQUESTED:
2653                 break;
2654         case VC_RESUME_IN_PROGRESS:
2655                 break;
2656         case VC_RESUME_RESUMED:
2657                 complete_all(&arm_state->vc_resume_complete);
2658                 set_suspend_state(arm_state, VC_SUSPEND_IDLE);
2659                 break;
2660         default:
2661                 BUG();
2662                 break;
2663         }
2664 }
2665
2666 /* should be called with the write lock held */
2667 inline void
2668 start_suspend_timer(VCHIQ_ARM_STATE_T *arm_state)
2669 {
2670         del_timer(&arm_state->suspend_timer);
2671         arm_state->suspend_timer.expires = jiffies +
2672                 msecs_to_jiffies(arm_state->suspend_timer_timeout);
2673         add_timer(&arm_state->suspend_timer);
2674         arm_state->suspend_timer_running = 1;
2675 }
2676
2677 /* should be called with the write lock held */
2678 static inline void
2679 stop_suspend_timer(VCHIQ_ARM_STATE_T *arm_state)
2680 {
2681         if (arm_state->suspend_timer_running) {
2682                 del_timer(&arm_state->suspend_timer);
2683                 arm_state->suspend_timer_running = 0;
2684         }
2685 }
2686
2687 static inline int
2688 need_resume(VCHIQ_STATE_T *state)
2689 {
2690         VCHIQ_ARM_STATE_T *arm_state = vchiq_platform_get_arm_state(state);
2691
2692         return (arm_state->vc_suspend_state > VC_SUSPEND_IDLE) &&
2693                         (arm_state->vc_resume_state < VC_RESUME_REQUESTED) &&
2694                         vchiq_videocore_wanted(state);
2695 }
2696
2697 static int
2698 block_resume(VCHIQ_ARM_STATE_T *arm_state)
2699 {
2700         int status = VCHIQ_SUCCESS;
2701         const unsigned long timeout_val =
2702                                 msecs_to_jiffies(FORCE_SUSPEND_TIMEOUT_MS);
2703         int resume_count = 0;
2704
2705         /* Allow any threads which were blocked by the last force suspend to
2706          * complete if they haven't already.  Only give this one shot; if
2707          * blocked_count is incremented after blocked_blocker is completed
2708          * (which only happens when blocked_count hits 0) then those threads
2709          * will have to wait until next time around */
2710         if (arm_state->blocked_count) {
2711                 reinit_completion(&arm_state->blocked_blocker);
2712                 write_unlock_bh(&arm_state->susp_res_lock);
2713                 vchiq_log_info(vchiq_susp_log_level, "%s wait for previously "
2714                         "blocked clients", __func__);
2715                 if (wait_for_completion_interruptible_timeout(
2716                                 &arm_state->blocked_blocker, timeout_val)
2717                                         <= 0) {
2718                         vchiq_log_error(vchiq_susp_log_level, "%s wait for "
2719                                 "previously blocked clients failed", __func__);
2720                         status = VCHIQ_ERROR;
2721                         write_lock_bh(&arm_state->susp_res_lock);
2722                         goto out;
2723                 }
2724                 vchiq_log_info(vchiq_susp_log_level, "%s previously blocked "
2725                         "clients resumed", __func__);
2726                 write_lock_bh(&arm_state->susp_res_lock);
2727         }
2728
2729         /* We need to wait for resume to complete if it's in process */
2730         while (arm_state->vc_resume_state != VC_RESUME_RESUMED &&
2731                         arm_state->vc_resume_state > VC_RESUME_IDLE) {
2732                 if (resume_count > 1) {
2733                         status = VCHIQ_ERROR;
2734                         vchiq_log_error(vchiq_susp_log_level, "%s waited too "
2735                                 "many times for resume", __func__);
2736                         goto out;
2737                 }
2738                 write_unlock_bh(&arm_state->susp_res_lock);
2739                 vchiq_log_info(vchiq_susp_log_level, "%s wait for resume",
2740                         __func__);
2741                 if (wait_for_completion_interruptible_timeout(
2742                                 &arm_state->vc_resume_complete, timeout_val)
2743                                         <= 0) {
2744                         vchiq_log_error(vchiq_susp_log_level, "%s wait for "
2745                                 "resume failed (%s)", __func__,
2746                                 resume_state_names[arm_state->vc_resume_state +
2747                                                         VC_RESUME_NUM_OFFSET]);
2748                         status = VCHIQ_ERROR;
2749                         write_lock_bh(&arm_state->susp_res_lock);
2750                         goto out;
2751                 }
2752                 vchiq_log_info(vchiq_susp_log_level, "%s resumed", __func__);
2753                 write_lock_bh(&arm_state->susp_res_lock);
2754                 resume_count++;
2755         }
2756         reinit_completion(&arm_state->resume_blocker);
2757         arm_state->resume_blocked = 1;
2758
2759 out:
2760         return status;
2761 }
2762
2763 static inline void
2764 unblock_resume(VCHIQ_ARM_STATE_T *arm_state)
2765 {
2766         complete_all(&arm_state->resume_blocker);
2767         arm_state->resume_blocked = 0;
2768 }
2769
2770 /* Initiate suspend via slot handler. Should be called with the write lock
2771  * held */
2772 VCHIQ_STATUS_T
2773 vchiq_arm_vcsuspend(VCHIQ_STATE_T *state)
2774 {
2775         VCHIQ_STATUS_T status = VCHIQ_ERROR;
2776         VCHIQ_ARM_STATE_T *arm_state = vchiq_platform_get_arm_state(state);
2777
2778         if (!arm_state)
2779                 goto out;
2780
2781         vchiq_log_trace(vchiq_susp_log_level, "%s", __func__);
2782         status = VCHIQ_SUCCESS;
2783
2784         switch (arm_state->vc_suspend_state) {
2785         case VC_SUSPEND_REQUESTED:
2786                 vchiq_log_info(vchiq_susp_log_level, "%s: suspend already "
2787                         "requested", __func__);
2788                 break;
2789         case VC_SUSPEND_IN_PROGRESS:
2790                 vchiq_log_info(vchiq_susp_log_level, "%s: suspend already in "
2791                         "progress", __func__);
2792                 break;
2793
2794         default:
2795                 /* We don't expect to be in other states, so log but continue
2796                  * anyway */
2797                 vchiq_log_error(vchiq_susp_log_level,
2798                         "%s unexpected suspend state %s", __func__,
2799                         suspend_state_names[arm_state->vc_suspend_state +
2800                                                 VC_SUSPEND_NUM_OFFSET]);
2801                 /* fall through */
2802         case VC_SUSPEND_REJECTED:
2803         case VC_SUSPEND_FAILED:
2804                 /* Ensure any idle state actions have been run */
2805                 set_suspend_state(arm_state, VC_SUSPEND_IDLE);
2806                 /* fall through */
2807         case VC_SUSPEND_IDLE:
2808                 vchiq_log_info(vchiq_susp_log_level,
2809                         "%s: suspending", __func__);
2810                 set_suspend_state(arm_state, VC_SUSPEND_REQUESTED);
2811                 /* kick the slot handler thread to initiate suspend */
2812                 request_poll(state, NULL, 0);
2813                 break;
2814         }
2815
2816 out:
2817         vchiq_log_trace(vchiq_susp_log_level, "%s exit %d", __func__, status);
2818         return status;
2819 }
2820
2821 void
2822 vchiq_platform_check_suspend(VCHIQ_STATE_T *state)
2823 {
2824         VCHIQ_ARM_STATE_T *arm_state = vchiq_platform_get_arm_state(state);
2825         int susp = 0;
2826
2827         if (!arm_state)
2828                 goto out;
2829
2830         vchiq_log_trace(vchiq_susp_log_level, "%s", __func__);
2831
2832         write_lock_bh(&arm_state->susp_res_lock);
2833         if (arm_state->vc_suspend_state == VC_SUSPEND_REQUESTED &&
2834                         arm_state->vc_resume_state == VC_RESUME_RESUMED) {
2835                 set_suspend_state(arm_state, VC_SUSPEND_IN_PROGRESS);
2836                 susp = 1;
2837         }
2838         write_unlock_bh(&arm_state->susp_res_lock);
2839
2840         if (susp)
2841                 vchiq_platform_suspend(state);
2842
2843 out:
2844         vchiq_log_trace(vchiq_susp_log_level, "%s exit", __func__);
2845         return;
2846 }
2847
2848 static void
2849 output_timeout_error(VCHIQ_STATE_T *state)
2850 {
2851         VCHIQ_ARM_STATE_T *arm_state = vchiq_platform_get_arm_state(state);
2852         char err[50] = "";
2853         int vc_use_count = arm_state->videocore_use_count;
2854         int active_services = state->unused_service;
2855         int i;
2856
2857         if (!arm_state->videocore_use_count) {
2858                 snprintf(err, sizeof(err), " Videocore usecount is 0");
2859                 goto output_msg;
2860         }
2861         for (i = 0; i < active_services; i++) {
2862                 VCHIQ_SERVICE_T *service_ptr = state->services[i];
2863
2864                 if (service_ptr && service_ptr->service_use_count &&
2865                         (service_ptr->srvstate != VCHIQ_SRVSTATE_FREE)) {
2866                         snprintf(err, sizeof(err), " %c%c%c%c(%d) service has "
2867                                 "use count %d%s", VCHIQ_FOURCC_AS_4CHARS(
2868                                         service_ptr->base.fourcc),
2869                                  service_ptr->client_id,
2870                                  service_ptr->service_use_count,
2871                                  service_ptr->service_use_count ==
2872                                          vc_use_count ? "" : " (+ more)");
2873                         break;
2874                 }
2875         }
2876
2877 output_msg:
2878         vchiq_log_error(vchiq_susp_log_level,
2879                 "timed out waiting for vc suspend (%d).%s",
2880                  arm_state->autosuspend_override, err);
2881
2882 }
2883
2884 /* Try to get videocore into suspended state, regardless of autosuspend state.
2885 ** We don't actually force suspend, since videocore may get into a bad state
2886 ** if we force suspend at a bad time.  Instead, we wait for autosuspend to
2887 ** determine a good point to suspend.  If this doesn't happen within 100ms we
2888 ** report failure.
2889 **
2890 ** Returns VCHIQ_SUCCESS if videocore suspended successfully, VCHIQ_RETRY if
2891 ** videocore failed to suspend in time or VCHIQ_ERROR if interrupted.
2892 */
2893 VCHIQ_STATUS_T
2894 vchiq_arm_force_suspend(VCHIQ_STATE_T *state)
2895 {
2896         VCHIQ_ARM_STATE_T *arm_state = vchiq_platform_get_arm_state(state);
2897         VCHIQ_STATUS_T status = VCHIQ_ERROR;
2898         long rc = 0;
2899         int repeat = -1;
2900
2901         if (!arm_state)
2902                 goto out;
2903
2904         vchiq_log_trace(vchiq_susp_log_level, "%s", __func__);
2905
2906         write_lock_bh(&arm_state->susp_res_lock);
2907
2908         status = block_resume(arm_state);
2909         if (status != VCHIQ_SUCCESS)
2910                 goto unlock;
2911         if (arm_state->vc_suspend_state == VC_SUSPEND_SUSPENDED) {
2912                 /* Already suspended - just block resume and exit */
2913                 vchiq_log_info(vchiq_susp_log_level, "%s already suspended",
2914                         __func__);
2915                 status = VCHIQ_SUCCESS;
2916                 goto unlock;
2917         } else if (arm_state->vc_suspend_state <= VC_SUSPEND_IDLE) {
2918                 /* initiate suspend immediately in the case that we're waiting
2919                  * for the timeout */
2920                 stop_suspend_timer(arm_state);
2921                 if (!vchiq_videocore_wanted(state)) {
2922                         vchiq_log_info(vchiq_susp_log_level, "%s videocore "
2923                                 "idle, initiating suspend", __func__);
2924                         status = vchiq_arm_vcsuspend(state);
2925                 } else if (arm_state->autosuspend_override <
2926                                                 FORCE_SUSPEND_FAIL_MAX) {
2927                         vchiq_log_info(vchiq_susp_log_level, "%s letting "
2928                                 "videocore go idle", __func__);
2929                         status = VCHIQ_SUCCESS;
2930                 } else {
2931                         vchiq_log_warning(vchiq_susp_log_level, "%s failed too "
2932                                 "many times - attempting suspend", __func__);
2933                         status = vchiq_arm_vcsuspend(state);
2934                 }
2935         } else {
2936                 vchiq_log_info(vchiq_susp_log_level, "%s videocore suspend "
2937                         "in progress - wait for completion", __func__);
2938                 status = VCHIQ_SUCCESS;
2939         }
2940
2941         /* Wait for suspend to happen due to system idle (not forced..) */
2942         if (status != VCHIQ_SUCCESS)
2943                 goto unblock_resume;
2944
2945         do {
2946                 write_unlock_bh(&arm_state->susp_res_lock);
2947
2948                 rc = wait_for_completion_interruptible_timeout(
2949                                 &arm_state->vc_suspend_complete,
2950                                 msecs_to_jiffies(FORCE_SUSPEND_TIMEOUT_MS));
2951
2952                 write_lock_bh(&arm_state->susp_res_lock);
2953                 if (rc < 0) {
2954                         vchiq_log_warning(vchiq_susp_log_level, "%s "
2955                                 "interrupted waiting for suspend", __func__);
2956                         status = VCHIQ_ERROR;
2957                         goto unblock_resume;
2958                 } else if (rc == 0) {
2959                         if (arm_state->vc_suspend_state > VC_SUSPEND_IDLE) {
2960                                 /* Repeat timeout once if in progress */
2961                                 if (repeat < 0) {
2962                                         repeat = 1;
2963                                         continue;
2964                                 }
2965                         }
2966                         arm_state->autosuspend_override++;
2967                         output_timeout_error(state);
2968
2969                         status = VCHIQ_RETRY;
2970                         goto unblock_resume;
2971                 }
2972         } while (0 < (repeat--));
2973
2974         /* Check and report state in case we need to abort ARM suspend */
2975         if (arm_state->vc_suspend_state != VC_SUSPEND_SUSPENDED) {
2976                 status = VCHIQ_RETRY;
2977                 vchiq_log_error(vchiq_susp_log_level,
2978                         "%s videocore suspend failed (state %s)", __func__,
2979                         suspend_state_names[arm_state->vc_suspend_state +
2980                                                 VC_SUSPEND_NUM_OFFSET]);
2981                 /* Reset the state only if it's still in an error state.
2982                  * Something could have already initiated another suspend. */
2983                 if (arm_state->vc_suspend_state < VC_SUSPEND_IDLE)
2984                         set_suspend_state(arm_state, VC_SUSPEND_IDLE);
2985
2986                 goto unblock_resume;
2987         }
2988
2989         /* successfully suspended - unlock and exit */
2990         goto unlock;
2991
2992 unblock_resume:
2993         /* all error states need to unblock resume before exit */
2994         unblock_resume(arm_state);
2995
2996 unlock:
2997         write_unlock_bh(&arm_state->susp_res_lock);
2998
2999 out:
3000         vchiq_log_trace(vchiq_susp_log_level, "%s exit %d", __func__, status);
3001         return status;
3002 }
3003
3004 void
3005 vchiq_check_suspend(VCHIQ_STATE_T *state)
3006 {
3007         VCHIQ_ARM_STATE_T *arm_state = vchiq_platform_get_arm_state(state);
3008
3009         if (!arm_state)
3010                 goto out;
3011
3012         vchiq_log_trace(vchiq_susp_log_level, "%s", __func__);
3013
3014         write_lock_bh(&arm_state->susp_res_lock);
3015         if (arm_state->vc_suspend_state != VC_SUSPEND_SUSPENDED &&
3016                         arm_state->first_connect &&
3017                         !vchiq_videocore_wanted(state)) {
3018                 vchiq_arm_vcsuspend(state);
3019         }
3020         write_unlock_bh(&arm_state->susp_res_lock);
3021
3022 out:
3023         vchiq_log_trace(vchiq_susp_log_level, "%s exit", __func__);
3024 }
3025
3026 int
3027 vchiq_arm_allow_resume(VCHIQ_STATE_T *state)
3028 {
3029         VCHIQ_ARM_STATE_T *arm_state = vchiq_platform_get_arm_state(state);
3030         int resume = 0;
3031         int ret = -1;
3032
3033         if (!arm_state)
3034                 goto out;
3035
3036         vchiq_log_trace(vchiq_susp_log_level, "%s", __func__);
3037
3038         write_lock_bh(&arm_state->susp_res_lock);
3039         unblock_resume(arm_state);
3040         resume = vchiq_check_resume(state);
3041         write_unlock_bh(&arm_state->susp_res_lock);
3042
3043         if (resume) {
3044                 if (wait_for_completion_interruptible(
3045                         &arm_state->vc_resume_complete) < 0) {
3046                         vchiq_log_error(vchiq_susp_log_level,
3047                                 "%s interrupted", __func__);
3048                         /* failed, cannot accurately derive suspend
3049                          * state, so exit early. */
3050                         goto out;
3051                 }
3052         }
3053
3054         read_lock_bh(&arm_state->susp_res_lock);
3055         if (arm_state->vc_suspend_state == VC_SUSPEND_SUSPENDED) {
3056                 vchiq_log_info(vchiq_susp_log_level,
3057                                 "%s: Videocore remains suspended", __func__);
3058         } else {
3059                 vchiq_log_info(vchiq_susp_log_level,
3060                                 "%s: Videocore resumed", __func__);
3061                 ret = 0;
3062         }
3063         read_unlock_bh(&arm_state->susp_res_lock);
3064 out:
3065         vchiq_log_trace(vchiq_susp_log_level, "%s exit %d", __func__, ret);
3066         return ret;
3067 }
3068
3069 /* This function should be called with the write lock held */
3070 int
3071 vchiq_check_resume(VCHIQ_STATE_T *state)
3072 {
3073         VCHIQ_ARM_STATE_T *arm_state = vchiq_platform_get_arm_state(state);
3074         int resume = 0;
3075
3076         if (!arm_state)
3077                 goto out;
3078
3079         vchiq_log_trace(vchiq_susp_log_level, "%s", __func__);
3080
3081         if (need_resume(state)) {
3082                 set_resume_state(arm_state, VC_RESUME_REQUESTED);
3083                 request_poll(state, NULL, 0);
3084                 resume = 1;
3085         }
3086
3087 out:
3088         vchiq_log_trace(vchiq_susp_log_level, "%s exit", __func__);
3089         return resume;
3090 }
3091
3092 VCHIQ_STATUS_T
3093 vchiq_use_internal(VCHIQ_STATE_T *state, VCHIQ_SERVICE_T *service,
3094                 enum USE_TYPE_E use_type)
3095 {
3096         VCHIQ_ARM_STATE_T *arm_state = vchiq_platform_get_arm_state(state);
3097         VCHIQ_STATUS_T ret = VCHIQ_SUCCESS;
3098         char entity[16];
3099         int *entity_uc;
3100         int local_uc, local_entity_uc;
3101
3102         if (!arm_state)
3103                 goto out;
3104
3105         vchiq_log_trace(vchiq_susp_log_level, "%s", __func__);
3106
3107         if (use_type == USE_TYPE_VCHIQ) {
3108                 sprintf(entity, "VCHIQ:   ");
3109                 entity_uc = &arm_state->peer_use_count;
3110         } else if (service) {
3111                 sprintf(entity, "%c%c%c%c:%03d",
3112                         VCHIQ_FOURCC_AS_4CHARS(service->base.fourcc),
3113                         service->client_id);
3114                 entity_uc = &service->service_use_count;
3115         } else {
3116                 vchiq_log_error(vchiq_susp_log_level, "%s null service "
3117                                 "ptr", __func__);
3118                 ret = VCHIQ_ERROR;
3119                 goto out;
3120         }
3121
3122         write_lock_bh(&arm_state->susp_res_lock);
3123         while (arm_state->resume_blocked) {
3124                 /* If we call 'use' while force suspend is waiting for suspend,
3125                  * then we're about to block the thread which the force is
3126                  * waiting to complete, so we're bound to just time out. In this
3127                  * case, set the suspend state such that the wait will be
3128                  * canceled, so we can complete as quickly as possible. */
3129                 if (arm_state->resume_blocked && arm_state->vc_suspend_state ==
3130                                 VC_SUSPEND_IDLE) {
3131                         set_suspend_state(arm_state, VC_SUSPEND_FORCE_CANCELED);
3132                         break;
3133                 }
3134                 /* If suspend is already in progress then we need to block */
3135                 if (!try_wait_for_completion(&arm_state->resume_blocker)) {
3136                         /* Indicate that there are threads waiting on the resume
3137                          * blocker.  These need to be allowed to complete before
3138                          * a _second_ call to force suspend can complete,
3139                          * otherwise low priority threads might never actually
3140                          * continue */
3141                         arm_state->blocked_count++;
3142                         write_unlock_bh(&arm_state->susp_res_lock);
3143                         vchiq_log_info(vchiq_susp_log_level, "%s %s resume "
3144                                 "blocked - waiting...", __func__, entity);
3145                         if (wait_for_completion_killable(
3146                                         &arm_state->resume_blocker) != 0) {
3147                                 vchiq_log_error(vchiq_susp_log_level, "%s %s "
3148                                         "wait for resume blocker interrupted",
3149                                         __func__, entity);
3150                                 ret = VCHIQ_ERROR;
3151                                 write_lock_bh(&arm_state->susp_res_lock);
3152                                 arm_state->blocked_count--;
3153                                 write_unlock_bh(&arm_state->susp_res_lock);
3154                                 goto out;
3155                         }
3156                         vchiq_log_info(vchiq_susp_log_level, "%s %s resume "
3157                                 "unblocked", __func__, entity);
3158                         write_lock_bh(&arm_state->susp_res_lock);
3159                         if (--arm_state->blocked_count == 0)
3160                                 complete_all(&arm_state->blocked_blocker);
3161                 }
3162         }
3163
3164         stop_suspend_timer(arm_state);
3165
3166         local_uc = ++arm_state->videocore_use_count;
3167         local_entity_uc = ++(*entity_uc);
3168
3169         /* If there's a pending request which hasn't yet been serviced then
3170          * just clear it.  If we're past VC_SUSPEND_REQUESTED state then
3171          * vc_resume_complete will block until we either resume or fail to
3172          * suspend */
3173         if (arm_state->vc_suspend_state <= VC_SUSPEND_REQUESTED)
3174                 set_suspend_state(arm_state, VC_SUSPEND_IDLE);
3175
3176         if ((use_type != USE_TYPE_SERVICE_NO_RESUME) && need_resume(state)) {
3177                 set_resume_state(arm_state, VC_RESUME_REQUESTED);
3178                 vchiq_log_info(vchiq_susp_log_level,
3179                         "%s %s count %d, state count %d",
3180                         __func__, entity, local_entity_uc, local_uc);
3181                 request_poll(state, NULL, 0);
3182         } else
3183                 vchiq_log_trace(vchiq_susp_log_level,
3184                         "%s %s count %d, state count %d",
3185                         __func__, entity, *entity_uc, local_uc);
3186
3187         write_unlock_bh(&arm_state->susp_res_lock);
3188
3189         /* Completion is in a done state when we're not suspended, so this won't
3190          * block for the non-suspended case. */
3191         if (!try_wait_for_completion(&arm_state->vc_resume_complete)) {
3192                 vchiq_log_info(vchiq_susp_log_level, "%s %s wait for resume",
3193                         __func__, entity);
3194                 if (wait_for_completion_killable(
3195                                 &arm_state->vc_resume_complete) != 0) {
3196                         vchiq_log_error(vchiq_susp_log_level, "%s %s wait for "
3197                                 "resume interrupted", __func__, entity);
3198                         ret = VCHIQ_ERROR;
3199                         goto out;
3200                 }
3201                 vchiq_log_info(vchiq_susp_log_level, "%s %s resumed", __func__,
3202                         entity);
3203         }
3204
3205         if (ret == VCHIQ_SUCCESS) {
3206                 VCHIQ_STATUS_T status = VCHIQ_SUCCESS;
3207                 long ack_cnt = atomic_xchg(&arm_state->ka_use_ack_count, 0);
3208
3209                 while (ack_cnt && (status == VCHIQ_SUCCESS)) {
3210                         /* Send the use notify to videocore */
3211                         status = vchiq_send_remote_use_active(state);
3212                         if (status == VCHIQ_SUCCESS)
3213                                 ack_cnt--;
3214                         else
3215                                 atomic_add(ack_cnt,
3216                                         &arm_state->ka_use_ack_count);
3217                 }
3218         }
3219
3220 out:
3221         vchiq_log_trace(vchiq_susp_log_level, "%s exit %d", __func__, ret);
3222         return ret;
3223 }
3224
3225 VCHIQ_STATUS_T
3226 vchiq_release_internal(VCHIQ_STATE_T *state, VCHIQ_SERVICE_T *service)
3227 {
3228         VCHIQ_ARM_STATE_T *arm_state = vchiq_platform_get_arm_state(state);
3229         VCHIQ_STATUS_T ret = VCHIQ_SUCCESS;
3230         char entity[16];
3231         int *entity_uc;
3232         int local_uc, local_entity_uc;
3233
3234         if (!arm_state)
3235                 goto out;
3236
3237         vchiq_log_trace(vchiq_susp_log_level, "%s", __func__);
3238
3239         if (service) {
3240                 sprintf(entity, "%c%c%c%c:%03d",
3241                         VCHIQ_FOURCC_AS_4CHARS(service->base.fourcc),
3242                         service->client_id);
3243                 entity_uc = &service->service_use_count;
3244         } else {
3245                 sprintf(entity, "PEER:   ");
3246                 entity_uc = &arm_state->peer_use_count;
3247         }
3248
3249         write_lock_bh(&arm_state->susp_res_lock);
3250         if (!arm_state->videocore_use_count || !(*entity_uc)) {
3251                 /* Don't use BUG_ON - don't allow user thread to crash kernel */
3252                 WARN_ON(!arm_state->videocore_use_count);
3253                 WARN_ON(!(*entity_uc));
3254                 ret = VCHIQ_ERROR;
3255                 goto unlock;
3256         }
3257         local_uc = --arm_state->videocore_use_count;
3258         local_entity_uc = --(*entity_uc);
3259
3260         if (!vchiq_videocore_wanted(state)) {
3261                 if (vchiq_platform_use_suspend_timer() &&
3262                                 !arm_state->resume_blocked) {
3263                         /* Only use the timer if we're not trying to force
3264                          * suspend (=> resume_blocked) */
3265                         start_suspend_timer(arm_state);
3266                 } else {
3267                         vchiq_log_info(vchiq_susp_log_level,
3268                                 "%s %s count %d, state count %d - suspending",
3269                                 __func__, entity, *entity_uc,
3270                                 arm_state->videocore_use_count);
3271                         vchiq_arm_vcsuspend(state);
3272                 }
3273         } else
3274                 vchiq_log_trace(vchiq_susp_log_level,
3275                         "%s %s count %d, state count %d",
3276                         __func__, entity, *entity_uc,
3277                         arm_state->videocore_use_count);
3278
3279 unlock:
3280         write_unlock_bh(&arm_state->susp_res_lock);
3281
3282 out:
3283         vchiq_log_trace(vchiq_susp_log_level, "%s exit %d", __func__, ret);
3284         return ret;
3285 }
3286
3287 void
3288 vchiq_on_remote_use(VCHIQ_STATE_T *state)
3289 {
3290         VCHIQ_ARM_STATE_T *arm_state = vchiq_platform_get_arm_state(state);
3291
3292         vchiq_log_trace(vchiq_susp_log_level, "%s", __func__);
3293         atomic_inc(&arm_state->ka_use_count);
3294         complete(&arm_state->ka_evt);
3295 }
3296
3297 void
3298 vchiq_on_remote_release(VCHIQ_STATE_T *state)
3299 {
3300         VCHIQ_ARM_STATE_T *arm_state = vchiq_platform_get_arm_state(state);
3301
3302         vchiq_log_trace(vchiq_susp_log_level, "%s", __func__);
3303         atomic_inc(&arm_state->ka_release_count);
3304         complete(&arm_state->ka_evt);
3305 }
3306
3307 VCHIQ_STATUS_T
3308 vchiq_use_service_internal(VCHIQ_SERVICE_T *service)
3309 {
3310         return vchiq_use_internal(service->state, service, USE_TYPE_SERVICE);
3311 }
3312
3313 VCHIQ_STATUS_T
3314 vchiq_release_service_internal(VCHIQ_SERVICE_T *service)
3315 {
3316         return vchiq_release_internal(service->state, service);
3317 }
3318
3319 VCHIQ_DEBUGFS_NODE_T *
3320 vchiq_instance_get_debugfs_node(VCHIQ_INSTANCE_T instance)
3321 {
3322         return &instance->debugfs_node;
3323 }
3324
3325 int
3326 vchiq_instance_get_use_count(VCHIQ_INSTANCE_T instance)
3327 {
3328         VCHIQ_SERVICE_T *service;
3329         int use_count = 0, i;
3330
3331         i = 0;
3332         while ((service = next_service_by_instance(instance->state,
3333                 instance, &i)) != NULL) {
3334                 use_count += service->service_use_count;
3335                 unlock_service(service);
3336         }
3337         return use_count;
3338 }
3339
3340 int
3341 vchiq_instance_get_pid(VCHIQ_INSTANCE_T instance)
3342 {
3343         return instance->pid;
3344 }
3345
3346 int
3347 vchiq_instance_get_trace(VCHIQ_INSTANCE_T instance)
3348 {
3349         return instance->trace;
3350 }
3351
3352 void
3353 vchiq_instance_set_trace(VCHIQ_INSTANCE_T instance, int trace)
3354 {
3355         VCHIQ_SERVICE_T *service;
3356         int i;
3357
3358         i = 0;
3359         while ((service = next_service_by_instance(instance->state,
3360                 instance, &i)) != NULL) {
3361                 service->trace = trace;
3362                 unlock_service(service);
3363         }
3364         instance->trace = (trace != 0);
3365 }
3366
3367 static void suspend_timer_callback(struct timer_list *t)
3368 {
3369         VCHIQ_ARM_STATE_T *arm_state = from_timer(arm_state, t, suspend_timer);
3370         VCHIQ_STATE_T *state = arm_state->state;
3371
3372         vchiq_log_info(vchiq_susp_log_level,
3373                 "%s - suspend timer expired - check suspend", __func__);
3374         vchiq_check_suspend(state);
3375 }
3376
3377 VCHIQ_STATUS_T
3378 vchiq_use_service_no_resume(VCHIQ_SERVICE_HANDLE_T handle)
3379 {
3380         VCHIQ_STATUS_T ret = VCHIQ_ERROR;
3381         VCHIQ_SERVICE_T *service = find_service_by_handle(handle);
3382
3383         if (service) {
3384                 ret = vchiq_use_internal(service->state, service,
3385                                 USE_TYPE_SERVICE_NO_RESUME);
3386                 unlock_service(service);
3387         }
3388         return ret;
3389 }
3390
3391 VCHIQ_STATUS_T
3392 vchiq_use_service(VCHIQ_SERVICE_HANDLE_T handle)
3393 {
3394         VCHIQ_STATUS_T ret = VCHIQ_ERROR;
3395         VCHIQ_SERVICE_T *service = find_service_by_handle(handle);
3396
3397         if (service) {
3398                 ret = vchiq_use_internal(service->state, service,
3399                                 USE_TYPE_SERVICE);
3400                 unlock_service(service);
3401         }
3402         return ret;
3403 }
3404
3405 VCHIQ_STATUS_T
3406 vchiq_release_service(VCHIQ_SERVICE_HANDLE_T handle)
3407 {
3408         VCHIQ_STATUS_T ret = VCHIQ_ERROR;
3409         VCHIQ_SERVICE_T *service = find_service_by_handle(handle);
3410
3411         if (service) {
3412                 ret = vchiq_release_internal(service->state, service);
3413                 unlock_service(service);
3414         }
3415         return ret;
3416 }
3417
3418 struct service_data_struct {
3419         int fourcc;
3420         int clientid;
3421         int use_count;
3422 };
3423
3424 void
3425 vchiq_dump_service_use_state(VCHIQ_STATE_T *state)
3426 {
3427         VCHIQ_ARM_STATE_T *arm_state = vchiq_platform_get_arm_state(state);
3428         struct service_data_struct *service_data;
3429         int i, found = 0;
3430         /* If there's more than 64 services, only dump ones with
3431          * non-zero counts */
3432         int only_nonzero = 0;
3433         static const char *nz = "<-- preventing suspend";
3434
3435         enum vc_suspend_status vc_suspend_state;
3436         enum vc_resume_status  vc_resume_state;
3437         int peer_count;
3438         int vc_use_count;
3439         int active_services;
3440
3441         if (!arm_state)
3442                 return;
3443
3444         service_data = kmalloc_array(MAX_SERVICES, sizeof(*service_data),
3445                                      GFP_KERNEL);
3446         if (!service_data)
3447                 return;
3448
3449         read_lock_bh(&arm_state->susp_res_lock);
3450         vc_suspend_state = arm_state->vc_suspend_state;
3451         vc_resume_state  = arm_state->vc_resume_state;
3452         peer_count = arm_state->peer_use_count;
3453         vc_use_count = arm_state->videocore_use_count;
3454         active_services = state->unused_service;
3455         if (active_services > MAX_SERVICES)
3456                 only_nonzero = 1;
3457
3458         for (i = 0; i < active_services; i++) {
3459                 VCHIQ_SERVICE_T *service_ptr = state->services[i];
3460
3461                 if (!service_ptr)
3462                         continue;
3463
3464                 if (only_nonzero && !service_ptr->service_use_count)
3465                         continue;
3466
3467                 if (service_ptr->srvstate == VCHIQ_SRVSTATE_FREE)
3468                         continue;
3469
3470                 service_data[found].fourcc = service_ptr->base.fourcc;
3471                 service_data[found].clientid = service_ptr->client_id;
3472                 service_data[found].use_count = service_ptr->service_use_count;
3473                 found++;
3474                 if (found >= MAX_SERVICES)
3475                         break;
3476         }
3477
3478         read_unlock_bh(&arm_state->susp_res_lock);
3479
3480         vchiq_log_warning(vchiq_susp_log_level,
3481                 "-- Videcore suspend state: %s --",
3482                 suspend_state_names[vc_suspend_state + VC_SUSPEND_NUM_OFFSET]);
3483         vchiq_log_warning(vchiq_susp_log_level,
3484                 "-- Videcore resume state: %s --",
3485                 resume_state_names[vc_resume_state + VC_RESUME_NUM_OFFSET]);
3486
3487         if (only_nonzero)
3488                 vchiq_log_warning(vchiq_susp_log_level, "Too many active "
3489                         "services (%d).  Only dumping up to first %d services "
3490                         "with non-zero use-count", active_services, found);
3491
3492         for (i = 0; i < found; i++) {
3493                 vchiq_log_warning(vchiq_susp_log_level,
3494                         "----- %c%c%c%c:%d service count %d %s",
3495                         VCHIQ_FOURCC_AS_4CHARS(service_data[i].fourcc),
3496                         service_data[i].clientid,
3497                         service_data[i].use_count,
3498                         service_data[i].use_count ? nz : "");
3499         }
3500         vchiq_log_warning(vchiq_susp_log_level,
3501                 "----- VCHIQ use count count %d", peer_count);
3502         vchiq_log_warning(vchiq_susp_log_level,
3503                 "--- Overall vchiq instance use count %d", vc_use_count);
3504
3505         kfree(service_data);
3506
3507         vchiq_dump_platform_use_state(state);
3508 }
3509
3510 VCHIQ_STATUS_T
3511 vchiq_check_service(VCHIQ_SERVICE_T *service)
3512 {
3513         VCHIQ_ARM_STATE_T *arm_state;
3514         VCHIQ_STATUS_T ret = VCHIQ_ERROR;
3515
3516         if (!service || !service->state)
3517                 goto out;
3518
3519         vchiq_log_trace(vchiq_susp_log_level, "%s", __func__);
3520
3521         arm_state = vchiq_platform_get_arm_state(service->state);
3522
3523         read_lock_bh(&arm_state->susp_res_lock);
3524         if (service->service_use_count)
3525                 ret = VCHIQ_SUCCESS;
3526         read_unlock_bh(&arm_state->susp_res_lock);
3527
3528         if (ret == VCHIQ_ERROR) {
3529                 vchiq_log_error(vchiq_susp_log_level,
3530                         "%s ERROR - %c%c%c%c:%d service count %d, "
3531                         "state count %d, videocore suspend state %s", __func__,
3532                         VCHIQ_FOURCC_AS_4CHARS(service->base.fourcc),
3533                         service->client_id, service->service_use_count,
3534                         arm_state->videocore_use_count,
3535                         suspend_state_names[arm_state->vc_suspend_state +
3536                                                 VC_SUSPEND_NUM_OFFSET]);
3537                 vchiq_dump_service_use_state(service->state);
3538         }
3539 out:
3540         return ret;
3541 }
3542
3543 /* stub functions */
3544 void vchiq_on_remote_use_active(VCHIQ_STATE_T *state)
3545 {
3546         (void)state;
3547 }
3548
3549 void vchiq_platform_conn_state_changed(VCHIQ_STATE_T *state,
3550         VCHIQ_CONNSTATE_T oldstate, VCHIQ_CONNSTATE_T newstate)
3551 {
3552         VCHIQ_ARM_STATE_T *arm_state = vchiq_platform_get_arm_state(state);
3553
3554         vchiq_log_info(vchiq_susp_log_level, "%d: %s->%s", state->id,
3555                 get_conn_state_name(oldstate), get_conn_state_name(newstate));
3556         if (state->conn_state == VCHIQ_CONNSTATE_CONNECTED) {
3557                 write_lock_bh(&arm_state->susp_res_lock);
3558                 if (!arm_state->first_connect) {
3559                         char threadname[16];
3560
3561                         arm_state->first_connect = 1;
3562                         write_unlock_bh(&arm_state->susp_res_lock);
3563                         snprintf(threadname, sizeof(threadname), "vchiq-keep/%d",
3564                                 state->id);
3565                         arm_state->ka_thread = kthread_create(
3566                                 &vchiq_keepalive_thread_func,
3567                                 (void *)state,
3568                                 threadname);
3569                         if (IS_ERR(arm_state->ka_thread)) {
3570                                 vchiq_log_error(vchiq_susp_log_level,
3571                                         "vchiq: FATAL: couldn't create thread %s",
3572                                         threadname);
3573                         } else {
3574                                 wake_up_process(arm_state->ka_thread);
3575                         }
3576                 } else
3577                         write_unlock_bh(&arm_state->susp_res_lock);
3578         }
3579 }
3580
3581 static int vchiq_probe(struct platform_device *pdev)
3582 {
3583         struct device_node *fw_node;
3584         struct rpi_firmware *fw;
3585         int err;
3586
3587         fw_node = of_find_compatible_node(NULL, NULL,
3588                                           "raspberrypi,bcm2835-firmware");
3589         if (!fw_node) {
3590                 dev_err(&pdev->dev, "Missing firmware node\n");
3591                 return -ENOENT;
3592         }
3593
3594         fw = rpi_firmware_get(fw_node);
3595         of_node_put(fw_node);
3596         if (!fw)
3597                 return -EPROBE_DEFER;
3598
3599         platform_set_drvdata(pdev, fw);
3600
3601         err = vchiq_platform_init(pdev, &g_state);
3602         if (err != 0)
3603                 goto failed_platform_init;
3604
3605         err = alloc_chrdev_region(&vchiq_devid, VCHIQ_MINOR, 1, DEVICE_NAME);
3606         if (err != 0) {
3607                 vchiq_log_error(vchiq_arm_log_level,
3608                         "Unable to allocate device number");
3609                 goto failed_platform_init;
3610         }
3611         cdev_init(&vchiq_cdev, &vchiq_fops);
3612         vchiq_cdev.owner = THIS_MODULE;
3613         err = cdev_add(&vchiq_cdev, vchiq_devid, 1);
3614         if (err != 0) {
3615                 vchiq_log_error(vchiq_arm_log_level,
3616                         "Unable to register device");
3617                 goto failed_cdev_add;
3618         }
3619
3620         /* create sysfs entries */
3621         vchiq_class = class_create(THIS_MODULE, DEVICE_NAME);
3622         err = PTR_ERR(vchiq_class);
3623         if (IS_ERR(vchiq_class))
3624                 goto failed_class_create;
3625
3626         vchiq_dev = device_create(vchiq_class, NULL,
3627                 vchiq_devid, NULL, "vchiq");
3628         err = PTR_ERR(vchiq_dev);
3629         if (IS_ERR(vchiq_dev))
3630                 goto failed_device_create;
3631
3632         /* create debugfs entries */
3633         vchiq_debugfs_init();
3634
3635         vchiq_log_info(vchiq_arm_log_level,
3636                 "vchiq: initialised - version %d (min %d), device %d.%d",
3637                 VCHIQ_VERSION, VCHIQ_VERSION_MIN,
3638                 MAJOR(vchiq_devid), MINOR(vchiq_devid));
3639
3640         bcm2835_camera = platform_device_register_data(&pdev->dev,
3641                                                        "bcm2835-camera", -1,
3642                                                        NULL, 0);
3643
3644         return 0;
3645
3646 failed_device_create:
3647         class_destroy(vchiq_class);
3648 failed_class_create:
3649         cdev_del(&vchiq_cdev);
3650 failed_cdev_add:
3651         unregister_chrdev_region(vchiq_devid, 1);
3652 failed_platform_init:
3653         vchiq_log_warning(vchiq_arm_log_level, "could not load vchiq");
3654         return err;
3655 }
3656
3657 static int vchiq_remove(struct platform_device *pdev)
3658 {
3659         platform_device_unregister(bcm2835_camera);
3660         vchiq_debugfs_deinit();
3661         device_destroy(vchiq_class, vchiq_devid);
3662         class_destroy(vchiq_class);
3663         cdev_del(&vchiq_cdev);
3664         unregister_chrdev_region(vchiq_devid, 1);
3665
3666         return 0;
3667 }
3668
3669 static const struct of_device_id vchiq_of_match[] = {
3670         { .compatible = "brcm,bcm2835-vchiq", },
3671         {},
3672 };
3673 MODULE_DEVICE_TABLE(of, vchiq_of_match);
3674
3675 static struct platform_driver vchiq_driver = {
3676         .driver = {
3677                 .name = "bcm2835_vchiq",
3678                 .of_match_table = vchiq_of_match,
3679         },
3680         .probe = vchiq_probe,
3681         .remove = vchiq_remove,
3682 };
3683 module_platform_driver(vchiq_driver);
3684
3685 MODULE_LICENSE("Dual BSD/GPL");
3686 MODULE_DESCRIPTION("Videocore VCHIQ driver");
3687 MODULE_AUTHOR("Broadcom Corporation");