| // SPDX-License-Identifier: GPL-2.0 |
| /* |
| * drivers/android/staging/vsoc.c |
| * |
| * Android Virtual System on a Chip (VSoC) driver |
| * |
| * Copyright (C) 2017 Google, Inc. |
| * |
| * Author: ghartman@google.com |
| * |
| * Based on drivers/char/kvm_ivshmem.c - driver for KVM Inter-VM shared memory |
| * Copyright 2009 Cam Macdonell <cam@cs.ualberta.ca> |
| * |
| * Based on cirrusfb.c and 8139cp.c: |
| * Copyright 1999-2001 Jeff Garzik |
| * Copyright 2001-2004 Jeff Garzik |
| */ |
| |
| #include <linux/dma-mapping.h> |
| #include <linux/freezer.h> |
| #include <linux/futex.h> |
| #include <linux/init.h> |
| #include <linux/kernel.h> |
| #include <linux/module.h> |
| #include <linux/mutex.h> |
| #include <linux/pci.h> |
| #include <linux/proc_fs.h> |
| #include <linux/sched.h> |
| #include <linux/syscalls.h> |
| #include <linux/uaccess.h> |
| #include <linux/interrupt.h> |
| #include <linux/cdev.h> |
| #include <linux/file.h> |
| #include "uapi/vsoc_shm.h" |
| |
| #define VSOC_DEV_NAME "vsoc" |
| |
| /* |
| * Description of the ivshmem-doorbell PCI device used by QEmu. These |
| * constants follow docs/specs/ivshmem-spec.txt, which can be found in |
| * the QEmu repository. This was last reconciled with the version that |
| * came out with 2.8 |
| */ |
| |
| /* |
| * These constants are determined KVM Inter-VM shared memory device |
| * register offsets |
| */ |
| enum { |
| INTR_MASK = 0x00, /* Interrupt Mask */ |
| INTR_STATUS = 0x04, /* Interrupt Status */ |
| IV_POSITION = 0x08, /* VM ID */ |
| DOORBELL = 0x0c, /* Doorbell */ |
| }; |
| |
| static const int REGISTER_BAR; /* Equal to 0 */ |
| static const int MAX_REGISTER_BAR_LEN = 0x100; |
| /* |
| * The MSI-x BAR is not used directly. |
| * |
| * static const int MSI_X_BAR = 1; |
| */ |
| static const int SHARED_MEMORY_BAR = 2; |
| |
| struct vsoc_region_data { |
| char name[VSOC_DEVICE_NAME_SZ + 1]; |
| wait_queue_head_t interrupt_wait_queue; |
| /* TODO(b/73664181): Use multiple futex wait queues */ |
| wait_queue_head_t futex_wait_queue; |
| /* Flag indicating that an interrupt has been signalled by the host. */ |
| atomic_t *incoming_signalled; |
| /* Flag indicating the guest has signalled the host. */ |
| atomic_t *outgoing_signalled; |
| bool irq_requested; |
| bool device_created; |
| }; |
| |
| struct vsoc_device { |
| /* Kernel virtual address of REGISTER_BAR. */ |
| void __iomem *regs; |
| /* Physical address of SHARED_MEMORY_BAR. */ |
| phys_addr_t shm_phys_start; |
| /* Kernel virtual address of SHARED_MEMORY_BAR. */ |
| void __iomem *kernel_mapped_shm; |
| /* Size of the entire shared memory window in bytes. */ |
| size_t shm_size; |
| /* |
| * Pointer to the virtual address of the shared memory layout structure. |
| * This is probably identical to kernel_mapped_shm, but saving this |
| * here saves a lot of annoying casts. |
| */ |
| struct vsoc_shm_layout_descriptor *layout; |
| /* |
| * Points to a table of region descriptors in the kernel's virtual |
| * address space. Calculated from |
| * vsoc_shm_layout_descriptor.vsoc_region_desc_offset |
| */ |
| struct vsoc_device_region *regions; |
| /* Head of a list of permissions that have been granted. */ |
| struct list_head permissions; |
| struct pci_dev *dev; |
| /* Per-region (and therefore per-interrupt) information. */ |
| struct vsoc_region_data *regions_data; |
| /* |
| * Table of msi-x entries. This has to be separated from struct |
| * vsoc_region_data because the kernel deals with them as an array. |
| */ |
| struct msix_entry *msix_entries; |
| /* Mutex that protectes the permission list */ |
| struct mutex mtx; |
| /* Major number assigned by the kernel */ |
| int major; |
| /* Character device assigned by the kernel */ |
| struct cdev cdev; |
| /* Device class assigned by the kernel */ |
| struct class *class; |
| /* |
| * Flags that indicate what we've initialized. These are used to do an |
| * orderly cleanup of the device. |
| */ |
| bool enabled_device; |
| bool requested_regions; |
| bool cdev_added; |
| bool class_added; |
| bool msix_enabled; |
| }; |
| |
| static struct vsoc_device vsoc_dev; |
| |
| /* |
| * TODO(ghartman): Add a /sys filesystem entry that summarizes the permissions. |
| */ |
| |
| struct fd_scoped_permission_node { |
| struct fd_scoped_permission permission; |
| struct list_head list; |
| }; |
| |
| struct vsoc_private_data { |
| struct fd_scoped_permission_node *fd_scoped_permission_node; |
| }; |
| |
| static long vsoc_ioctl(struct file *, unsigned int, unsigned long); |
| static int vsoc_mmap(struct file *, struct vm_area_struct *); |
| static int vsoc_open(struct inode *, struct file *); |
| static int vsoc_release(struct inode *, struct file *); |
| static ssize_t vsoc_read(struct file *, char __user *, size_t, loff_t *); |
| static ssize_t vsoc_write(struct file *, const char __user *, size_t, loff_t *); |
| static loff_t vsoc_lseek(struct file *filp, loff_t offset, int origin); |
| static int |
| do_create_fd_scoped_permission(struct vsoc_device_region *region_p, |
| struct fd_scoped_permission_node *np, |
| struct fd_scoped_permission_arg __user *arg); |
| static void |
| do_destroy_fd_scoped_permission(struct vsoc_device_region *owner_region_p, |
| struct fd_scoped_permission *perm); |
| static long do_vsoc_describe_region(struct file *, |
| struct vsoc_device_region __user *); |
| static ssize_t vsoc_get_area(struct file *filp, __u32 *perm_off); |
| |
| /** |
| * Validate arguments on entry points to the driver. |
| */ |
| inline int vsoc_validate_inode(struct inode *inode) |
| { |
| if (iminor(inode) >= vsoc_dev.layout->region_count) { |
| dev_err(&vsoc_dev.dev->dev, |
| "describe_region: invalid region %d\n", iminor(inode)); |
| return -ENODEV; |
| } |
| return 0; |
| } |
| |
| inline int vsoc_validate_filep(struct file *filp) |
| { |
| int ret = vsoc_validate_inode(file_inode(filp)); |
| |
| if (ret) |
| return ret; |
| if (!filp->private_data) { |
| dev_err(&vsoc_dev.dev->dev, |
| "No private data on fd, region %d\n", |
| iminor(file_inode(filp))); |
| return -EBADFD; |
| } |
| return 0; |
| } |
| |
| /* Converts from shared memory offset to virtual address */ |
| static inline void *shm_off_to_virtual_addr(__u32 offset) |
| { |
| return (void __force *)vsoc_dev.kernel_mapped_shm + offset; |
| } |
| |
| /* Converts from shared memory offset to physical address */ |
| static inline phys_addr_t shm_off_to_phys_addr(__u32 offset) |
| { |
| return vsoc_dev.shm_phys_start + offset; |
| } |
| |
| /** |
| * Convenience functions to obtain the region from the inode or file. |
| * Dangerous to call before validating the inode/file. |
| */ |
| static |
| inline struct vsoc_device_region *vsoc_region_from_inode(struct inode *inode) |
| { |
| return &vsoc_dev.regions[iminor(inode)]; |
| } |
| |
| static |
| inline struct vsoc_device_region *vsoc_region_from_filep(struct file *inode) |
| { |
| return vsoc_region_from_inode(file_inode(inode)); |
| } |
| |
| static inline uint32_t vsoc_device_region_size(struct vsoc_device_region *r) |
| { |
| return r->region_end_offset - r->region_begin_offset; |
| } |
| |
| static const struct file_operations vsoc_ops = { |
| .owner = THIS_MODULE, |
| .open = vsoc_open, |
| .mmap = vsoc_mmap, |
| .read = vsoc_read, |
| .unlocked_ioctl = vsoc_ioctl, |
| .compat_ioctl = vsoc_ioctl, |
| .write = vsoc_write, |
| .llseek = vsoc_lseek, |
| .release = vsoc_release, |
| }; |
| |
| static struct pci_device_id vsoc_id_table[] = { |
| {0x1af4, 0x1110, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0}, |
| {0}, |
| }; |
| |
| MODULE_DEVICE_TABLE(pci, vsoc_id_table); |
| |
| static void vsoc_remove_device(struct pci_dev *pdev); |
| static int vsoc_probe_device(struct pci_dev *pdev, |
| const struct pci_device_id *ent); |
| |
| static struct pci_driver vsoc_pci_driver = { |
| .name = "vsoc", |
| .id_table = vsoc_id_table, |
| .probe = vsoc_probe_device, |
| .remove = vsoc_remove_device, |
| }; |
| |
| static int |
| do_create_fd_scoped_permission(struct vsoc_device_region *region_p, |
| struct fd_scoped_permission_node *np, |
| struct fd_scoped_permission_arg __user *arg) |
| { |
| struct file *managed_filp; |
| s32 managed_fd; |
| atomic_t *owner_ptr = NULL; |
| struct vsoc_device_region *managed_region_p; |
| |
| if (copy_from_user(&np->permission, &arg->perm, sizeof(*np)) || |
| copy_from_user(&managed_fd, |
| &arg->managed_region_fd, sizeof(managed_fd))) { |
| return -EFAULT; |
| } |
| managed_filp = fdget(managed_fd).file; |
| /* Check that it's a valid fd, */ |
| if (!managed_filp || vsoc_validate_filep(managed_filp)) |
| return -EPERM; |
| /* EEXIST if the given fd already has a permission. */ |
| if (((struct vsoc_private_data *)managed_filp->private_data)-> |
| fd_scoped_permission_node) |
| return -EEXIST; |
| managed_region_p = vsoc_region_from_filep(managed_filp); |
| /* Check that the provided region is managed by this one */ |
| if (&vsoc_dev.regions[managed_region_p->managed_by] != region_p) |
| return -EPERM; |
| /* The area must be well formed and have non-zero size */ |
| if (np->permission.begin_offset >= np->permission.end_offset) |
| return -EINVAL; |
| /* The area must fit in the memory window */ |
| if (np->permission.end_offset > |
| vsoc_device_region_size(managed_region_p)) |
| return -ERANGE; |
| /* The area must be in the region data section */ |
| if (np->permission.begin_offset < |
| managed_region_p->offset_of_region_data) |
| return -ERANGE; |
| /* The area must be page aligned */ |
| if (!PAGE_ALIGNED(np->permission.begin_offset) || |
| !PAGE_ALIGNED(np->permission.end_offset)) |
| return -EINVAL; |
| /* Owner offset must be naturally aligned in the window */ |
| if (np->permission.owner_offset & |
| (sizeof(np->permission.owner_offset) - 1)) |
| return -EINVAL; |
| /* The owner flag must reside in the owner memory */ |
| if (np->permission.owner_offset + sizeof(np->permission.owner_offset) > |
| vsoc_device_region_size(region_p)) |
| return -ERANGE; |
| /* The owner flag must reside in the data section */ |
| if (np->permission.owner_offset < region_p->offset_of_region_data) |
| return -EINVAL; |
| /* The owner value must change to claim the memory */ |
| if (np->permission.owned_value == VSOC_REGION_FREE) |
| return -EINVAL; |
| owner_ptr = |
| (atomic_t *)shm_off_to_virtual_addr(region_p->region_begin_offset + |
| np->permission.owner_offset); |
| /* We've already verified that this is in the shared memory window, so |
| * it should be safe to write to this address. |
| */ |
| if (atomic_cmpxchg(owner_ptr, |
| VSOC_REGION_FREE, |
| np->permission.owned_value) != VSOC_REGION_FREE) { |
| return -EBUSY; |
| } |
| ((struct vsoc_private_data *)managed_filp->private_data)-> |
| fd_scoped_permission_node = np; |
| /* The file offset needs to be adjusted if the calling |
| * process did any read/write operations on the fd |
| * before creating the permission. |
| */ |
| if (managed_filp->f_pos) { |
| if (managed_filp->f_pos > np->permission.end_offset) { |
| /* If the offset is beyond the permission end, set it |
| * to the end. |
| */ |
| managed_filp->f_pos = np->permission.end_offset; |
| } else { |
| /* If the offset is within the permission interval |
| * keep it there otherwise reset it to zero. |
| */ |
| if (managed_filp->f_pos < np->permission.begin_offset) { |
| managed_filp->f_pos = 0; |
| } else { |
| managed_filp->f_pos -= |
| np->permission.begin_offset; |
| } |
| } |
| } |
| return 0; |
| } |
| |
| static void |
| do_destroy_fd_scoped_permission_node(struct vsoc_device_region *owner_region_p, |
| struct fd_scoped_permission_node *node) |
| { |
| if (node) { |
| do_destroy_fd_scoped_permission(owner_region_p, |
| &node->permission); |
| mutex_lock(&vsoc_dev.mtx); |
| list_del(&node->list); |
| mutex_unlock(&vsoc_dev.mtx); |
| kfree(node); |
| } |
| } |
| |
| static void |
| do_destroy_fd_scoped_permission(struct vsoc_device_region *owner_region_p, |
| struct fd_scoped_permission *perm) |
| { |
| atomic_t *owner_ptr = NULL; |
| int prev = 0; |
| |
| if (!perm) |
| return; |
| owner_ptr = (atomic_t *)shm_off_to_virtual_addr |
| (owner_region_p->region_begin_offset + perm->owner_offset); |
| prev = atomic_xchg(owner_ptr, VSOC_REGION_FREE); |
| if (prev != perm->owned_value) |
| dev_err(&vsoc_dev.dev->dev, |
| "%x-%x: owner (%s) %x: expected to be %x was %x", |
| perm->begin_offset, perm->end_offset, |
| owner_region_p->device_name, perm->owner_offset, |
| perm->owned_value, prev); |
| } |
| |
| static long do_vsoc_describe_region(struct file *filp, |
| struct vsoc_device_region __user *dest) |
| { |
| struct vsoc_device_region *region_p; |
| int retval = vsoc_validate_filep(filp); |
| |
| if (retval) |
| return retval; |
| region_p = vsoc_region_from_filep(filp); |
| if (copy_to_user(dest, region_p, sizeof(*region_p))) |
| return -EFAULT; |
| return 0; |
| } |
| |
| /** |
| * Implements the inner logic of cond_wait. Copies to and from userspace are |
| * done in the helper function below. |
| */ |
| static int handle_vsoc_cond_wait(struct file *filp, struct vsoc_cond_wait *arg) |
| { |
| DEFINE_WAIT(wait); |
| u32 region_number = iminor(file_inode(filp)); |
| struct vsoc_region_data *data = vsoc_dev.regions_data + region_number; |
| struct hrtimer_sleeper timeout, *to = NULL; |
| int ret = 0; |
| struct vsoc_device_region *region_p = vsoc_region_from_filep(filp); |
| atomic_t *address = NULL; |
| ktime_t wake_time; |
| |
| /* Ensure that the offset is aligned */ |
| if (arg->offset & (sizeof(uint32_t) - 1)) |
| return -EADDRNOTAVAIL; |
| /* Ensure that the offset is within shared memory */ |
| if (((uint64_t)arg->offset) + region_p->region_begin_offset + |
| sizeof(uint32_t) > region_p->region_end_offset) |
| return -E2BIG; |
| address = shm_off_to_virtual_addr(region_p->region_begin_offset + |
| arg->offset); |
| |
| /* Ensure that the type of wait is valid */ |
| switch (arg->wait_type) { |
| case VSOC_WAIT_IF_EQUAL: |
| break; |
| case VSOC_WAIT_IF_EQUAL_TIMEOUT: |
| to = &timeout; |
| break; |
| default: |
| return -EINVAL; |
| } |
| |
| if (to) { |
| /* Copy the user-supplied timesec into the kernel structure. |
| * We do things this way to flatten differences between 32 bit |
| * and 64 bit timespecs. |
| */ |
| if (arg->wake_time_nsec >= NSEC_PER_SEC) |
| return -EINVAL; |
| wake_time = ktime_set(arg->wake_time_sec, arg->wake_time_nsec); |
| |
| hrtimer_init_on_stack(&to->timer, CLOCK_MONOTONIC, |
| HRTIMER_MODE_ABS); |
| hrtimer_set_expires_range_ns(&to->timer, wake_time, |
| current->timer_slack_ns); |
| |
| hrtimer_init_sleeper(to, current); |
| } |
| |
| while (1) { |
| prepare_to_wait(&data->futex_wait_queue, &wait, |
| TASK_INTERRUPTIBLE); |
| /* |
| * Check the sentinel value after prepare_to_wait. If the value |
| * changes after this check the writer will call signal, |
| * changing the task state from INTERRUPTIBLE to RUNNING. That |
| * will ensure that schedule() will eventually schedule this |
| * task. |
| */ |
| if (atomic_read(address) != arg->value) { |
| ret = 0; |
| break; |
| } |
| if (to) { |
| hrtimer_start_expires(&to->timer, HRTIMER_MODE_ABS); |
| if (likely(to->task)) |
| freezable_schedule(); |
| hrtimer_cancel(&to->timer); |
| if (!to->task) { |
| ret = -ETIMEDOUT; |
| break; |
| } |
| } else { |
| freezable_schedule(); |
| } |
| /* Count the number of times that we woke up. This is useful |
| * for unit testing. |
| */ |
| ++arg->wakes; |
| if (signal_pending(current)) { |
| ret = -EINTR; |
| break; |
| } |
| } |
| finish_wait(&data->futex_wait_queue, &wait); |
| if (to) |
| destroy_hrtimer_on_stack(&to->timer); |
| return ret; |
| } |
| |
| /** |
| * Handles the details of copying from/to userspace to ensure that the copies |
| * happen on all of the return paths of cond_wait. |
| */ |
| static int do_vsoc_cond_wait(struct file *filp, |
| struct vsoc_cond_wait __user *untrusted_in) |
| { |
| struct vsoc_cond_wait arg; |
| int rval = 0; |
| |
| if (copy_from_user(&arg, untrusted_in, sizeof(arg))) |
| return -EFAULT; |
| /* wakes is an out parameter. Initialize it to something sensible. */ |
| arg.wakes = 0; |
| rval = handle_vsoc_cond_wait(filp, &arg); |
| if (copy_to_user(untrusted_in, &arg, sizeof(arg))) |
| return -EFAULT; |
| return rval; |
| } |
| |
| static int do_vsoc_cond_wake(struct file *filp, uint32_t offset) |
| { |
| struct vsoc_device_region *region_p = vsoc_region_from_filep(filp); |
| u32 region_number = iminor(file_inode(filp)); |
| struct vsoc_region_data *data = vsoc_dev.regions_data + region_number; |
| /* Ensure that the offset is aligned */ |
| if (offset & (sizeof(uint32_t) - 1)) |
| return -EADDRNOTAVAIL; |
| /* Ensure that the offset is within shared memory */ |
| if (((uint64_t)offset) + region_p->region_begin_offset + |
| sizeof(uint32_t) > region_p->region_end_offset) |
| return -E2BIG; |
| /* |
| * TODO(b/73664181): Use multiple futex wait queues. |
| * We need to wake every sleeper when the condition changes. Typically |
| * only a single thread will be waiting on the condition, but there |
| * are exceptions. The worst case is about 10 threads. |
| */ |
| wake_up_interruptible_all(&data->futex_wait_queue); |
| return 0; |
| } |
| |
| static long vsoc_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) |
| { |
| int rv = 0; |
| struct vsoc_device_region *region_p; |
| u32 reg_num; |
| struct vsoc_region_data *reg_data; |
| int retval = vsoc_validate_filep(filp); |
| |
| if (retval) |
| return retval; |
| region_p = vsoc_region_from_filep(filp); |
| reg_num = iminor(file_inode(filp)); |
| reg_data = vsoc_dev.regions_data + reg_num; |
| switch (cmd) { |
| case VSOC_CREATE_FD_SCOPED_PERMISSION: |
| { |
| struct fd_scoped_permission_node *node = NULL; |
| |
| node = kzalloc(sizeof(*node), GFP_KERNEL); |
| /* We can't allocate memory for the permission */ |
| if (!node) |
| return -ENOMEM; |
| INIT_LIST_HEAD(&node->list); |
| rv = do_create_fd_scoped_permission |
| (region_p, |
| node, |
| (struct fd_scoped_permission_arg __user *)arg); |
| if (!rv) { |
| mutex_lock(&vsoc_dev.mtx); |
| list_add(&node->list, &vsoc_dev.permissions); |
| mutex_unlock(&vsoc_dev.mtx); |
| } else { |
| kfree(node); |
| return rv; |
| } |
| } |
| break; |
| |
| case VSOC_GET_FD_SCOPED_PERMISSION: |
| { |
| struct fd_scoped_permission_node *node = |
| ((struct vsoc_private_data *)filp->private_data)-> |
| fd_scoped_permission_node; |
| if (!node) |
| return -ENOENT; |
| if (copy_to_user |
| ((struct fd_scoped_permission __user *)arg, |
| &node->permission, sizeof(node->permission))) |
| return -EFAULT; |
| } |
| break; |
| |
| case VSOC_MAYBE_SEND_INTERRUPT_TO_HOST: |
| if (!atomic_xchg(reg_data->outgoing_signalled, 1)) { |
| writel(reg_num, vsoc_dev.regs + DOORBELL); |
| return 0; |
| } else { |
| return -EBUSY; |
| } |
| break; |
| |
| case VSOC_SEND_INTERRUPT_TO_HOST: |
| writel(reg_num, vsoc_dev.regs + DOORBELL); |
| return 0; |
| case VSOC_WAIT_FOR_INCOMING_INTERRUPT: |
| wait_event_interruptible |
| (reg_data->interrupt_wait_queue, |
| (atomic_read(reg_data->incoming_signalled) != 0)); |
| break; |
| |
| case VSOC_DESCRIBE_REGION: |
| return do_vsoc_describe_region |
| (filp, |
| (struct vsoc_device_region __user *)arg); |
| |
| case VSOC_SELF_INTERRUPT: |
| atomic_set(reg_data->incoming_signalled, 1); |
| wake_up_interruptible(®_data->interrupt_wait_queue); |
| break; |
| |
| case VSOC_COND_WAIT: |
| return do_vsoc_cond_wait(filp, |
| (struct vsoc_cond_wait __user *)arg); |
| case VSOC_COND_WAKE: |
| return do_vsoc_cond_wake(filp, arg); |
| |
| default: |
| return -EINVAL; |
| } |
| return 0; |
| } |
| |
| static ssize_t vsoc_read(struct file *filp, char __user *buffer, size_t len, |
| loff_t *poffset) |
| { |
| __u32 area_off; |
| const void *area_p; |
| ssize_t area_len; |
| int retval = vsoc_validate_filep(filp); |
| |
| if (retval) |
| return retval; |
| area_len = vsoc_get_area(filp, &area_off); |
| area_p = shm_off_to_virtual_addr(area_off); |
| area_p += *poffset; |
| area_len -= *poffset; |
| if (area_len <= 0) |
| return 0; |
| if (area_len < len) |
| len = area_len; |
| if (copy_to_user(buffer, area_p, len)) |
| return -EFAULT; |
| *poffset += len; |
| return len; |
| } |
| |
| static loff_t vsoc_lseek(struct file *filp, loff_t offset, int origin) |
| { |
| ssize_t area_len = 0; |
| int retval = vsoc_validate_filep(filp); |
| |
| if (retval) |
| return retval; |
| area_len = vsoc_get_area(filp, NULL); |
| switch (origin) { |
| case SEEK_SET: |
| break; |
| |
| case SEEK_CUR: |
| if (offset > 0 && offset + filp->f_pos < 0) |
| return -EOVERFLOW; |
| offset += filp->f_pos; |
| break; |
| |
| case SEEK_END: |
| if (offset > 0 && offset + area_len < 0) |
| return -EOVERFLOW; |
| offset += area_len; |
| break; |
| |
| case SEEK_DATA: |
| if (offset >= area_len) |
| return -EINVAL; |
| if (offset < 0) |
| offset = 0; |
| break; |
| |
| case SEEK_HOLE: |
| /* Next hole is always the end of the region, unless offset is |
| * beyond that |
| */ |
| if (offset < area_len) |
| offset = area_len; |
| break; |
| |
| default: |
| return -EINVAL; |
| } |
| |
| if (offset < 0 || offset > area_len) |
| return -EINVAL; |
| filp->f_pos = offset; |
| |
| return offset; |
| } |
| |
| static ssize_t vsoc_write(struct file *filp, const char __user *buffer, |
| size_t len, loff_t *poffset) |
| { |
| __u32 area_off; |
| void *area_p; |
| ssize_t area_len; |
| int retval = vsoc_validate_filep(filp); |
| |
| if (retval) |
| return retval; |
| area_len = vsoc_get_area(filp, &area_off); |
| area_p = shm_off_to_virtual_addr(area_off); |
| area_p += *poffset; |
| area_len -= *poffset; |
| if (area_len <= 0) |
| return 0; |
| if (area_len < len) |
| len = area_len; |
| if (copy_from_user(area_p, buffer, len)) |
| return -EFAULT; |
| *poffset += len; |
| return len; |
| } |
| |
| static irqreturn_t vsoc_interrupt(int irq, void *region_data_v) |
| { |
| struct vsoc_region_data *region_data = |
| (struct vsoc_region_data *)region_data_v; |
| int reg_num = region_data - vsoc_dev.regions_data; |
| |
| if (unlikely(!region_data)) |
| return IRQ_NONE; |
| |
| if (unlikely(reg_num < 0 || |
| reg_num >= vsoc_dev.layout->region_count)) { |
| dev_err(&vsoc_dev.dev->dev, |
| "invalid irq @%p reg_num=0x%04x\n", |
| region_data, reg_num); |
| return IRQ_NONE; |
| } |
| if (unlikely(vsoc_dev.regions_data + reg_num != region_data)) { |
| dev_err(&vsoc_dev.dev->dev, |
| "irq not aligned @%p reg_num=0x%04x\n", |
| region_data, reg_num); |
| return IRQ_NONE; |
| } |
| wake_up_interruptible(®ion_data->interrupt_wait_queue); |
| return IRQ_HANDLED; |
| } |
| |
| static int vsoc_probe_device(struct pci_dev *pdev, |
| const struct pci_device_id *ent) |
| { |
| int result; |
| int i; |
| resource_size_t reg_size; |
| dev_t devt; |
| |
| vsoc_dev.dev = pdev; |
| result = pci_enable_device(pdev); |
| if (result) { |
| dev_err(&pdev->dev, |
| "pci_enable_device failed %s: error %d\n", |
| pci_name(pdev), result); |
| return result; |
| } |
| vsoc_dev.enabled_device = true; |
| result = pci_request_regions(pdev, "vsoc"); |
| if (result < 0) { |
| dev_err(&pdev->dev, "pci_request_regions failed\n"); |
| vsoc_remove_device(pdev); |
| return -EBUSY; |
| } |
| vsoc_dev.requested_regions = true; |
| /* Set up the control registers in BAR 0 */ |
| reg_size = pci_resource_len(pdev, REGISTER_BAR); |
| if (reg_size > MAX_REGISTER_BAR_LEN) |
| vsoc_dev.regs = |
| pci_iomap(pdev, REGISTER_BAR, MAX_REGISTER_BAR_LEN); |
| else |
| vsoc_dev.regs = pci_iomap(pdev, REGISTER_BAR, reg_size); |
| |
| if (!vsoc_dev.regs) { |
| dev_err(&pdev->dev, |
| "cannot map registers of size %zu\n", |
| (size_t)reg_size); |
| vsoc_remove_device(pdev); |
| return -EBUSY; |
| } |
| |
| /* Map the shared memory in BAR 2 */ |
| vsoc_dev.shm_phys_start = pci_resource_start(pdev, SHARED_MEMORY_BAR); |
| vsoc_dev.shm_size = pci_resource_len(pdev, SHARED_MEMORY_BAR); |
| |
| dev_info(&pdev->dev, "shared memory @ DMA %pa size=0x%zx\n", |
| &vsoc_dev.shm_phys_start, vsoc_dev.shm_size); |
| vsoc_dev.kernel_mapped_shm = pci_iomap_wc(pdev, SHARED_MEMORY_BAR, 0); |
| if (!vsoc_dev.kernel_mapped_shm) { |
| dev_err(&vsoc_dev.dev->dev, "cannot iomap region\n"); |
| vsoc_remove_device(pdev); |
| return -EBUSY; |
| } |
| |
| vsoc_dev.layout = (struct vsoc_shm_layout_descriptor __force *) |
| vsoc_dev.kernel_mapped_shm; |
| dev_info(&pdev->dev, "major_version: %d\n", |
| vsoc_dev.layout->major_version); |
| dev_info(&pdev->dev, "minor_version: %d\n", |
| vsoc_dev.layout->minor_version); |
| dev_info(&pdev->dev, "size: 0x%x\n", vsoc_dev.layout->size); |
| dev_info(&pdev->dev, "regions: %d\n", vsoc_dev.layout->region_count); |
| if (vsoc_dev.layout->major_version != |
| CURRENT_VSOC_LAYOUT_MAJOR_VERSION) { |
| dev_err(&vsoc_dev.dev->dev, |
| "driver supports only major_version %d\n", |
| CURRENT_VSOC_LAYOUT_MAJOR_VERSION); |
| vsoc_remove_device(pdev); |
| return -EBUSY; |
| } |
| result = alloc_chrdev_region(&devt, 0, vsoc_dev.layout->region_count, |
| VSOC_DEV_NAME); |
| if (result) { |
| dev_err(&vsoc_dev.dev->dev, "alloc_chrdev_region failed\n"); |
| vsoc_remove_device(pdev); |
| return -EBUSY; |
| } |
| vsoc_dev.major = MAJOR(devt); |
| cdev_init(&vsoc_dev.cdev, &vsoc_ops); |
| vsoc_dev.cdev.owner = THIS_MODULE; |
| result = cdev_add(&vsoc_dev.cdev, devt, vsoc_dev.layout->region_count); |
| if (result) { |
| dev_err(&vsoc_dev.dev->dev, "cdev_add error\n"); |
| vsoc_remove_device(pdev); |
| return -EBUSY; |
| } |
| vsoc_dev.cdev_added = true; |
| vsoc_dev.class = class_create(THIS_MODULE, VSOC_DEV_NAME); |
| if (IS_ERR(vsoc_dev.class)) { |
| dev_err(&vsoc_dev.dev->dev, "class_create failed\n"); |
| vsoc_remove_device(pdev); |
| return PTR_ERR(vsoc_dev.class); |
| } |
| vsoc_dev.class_added = true; |
| vsoc_dev.regions = (struct vsoc_device_region __force *) |
| ((void *)vsoc_dev.layout + |
| vsoc_dev.layout->vsoc_region_desc_offset); |
| vsoc_dev.msix_entries = |
| kcalloc(vsoc_dev.layout->region_count, |
| sizeof(vsoc_dev.msix_entries[0]), GFP_KERNEL); |
| if (!vsoc_dev.msix_entries) { |
| dev_err(&vsoc_dev.dev->dev, |
| "unable to allocate msix_entries\n"); |
| vsoc_remove_device(pdev); |
| return -ENOSPC; |
| } |
| vsoc_dev.regions_data = |
| kcalloc(vsoc_dev.layout->region_count, |
| sizeof(vsoc_dev.regions_data[0]), GFP_KERNEL); |
| if (!vsoc_dev.regions_data) { |
| dev_err(&vsoc_dev.dev->dev, |
| "unable to allocate regions' data\n"); |
| vsoc_remove_device(pdev); |
| return -ENOSPC; |
| } |
| for (i = 0; i < vsoc_dev.layout->region_count; ++i) |
| vsoc_dev.msix_entries[i].entry = i; |
| |
| result = pci_enable_msix_exact(vsoc_dev.dev, vsoc_dev.msix_entries, |
| vsoc_dev.layout->region_count); |
| if (result) { |
| dev_info(&pdev->dev, "pci_enable_msix failed: %d\n", result); |
| vsoc_remove_device(pdev); |
| return -ENOSPC; |
| } |
| /* Check that all regions are well formed */ |
| for (i = 0; i < vsoc_dev.layout->region_count; ++i) { |
| const struct vsoc_device_region *region = vsoc_dev.regions + i; |
| |
| if (!PAGE_ALIGNED(region->region_begin_offset) || |
| !PAGE_ALIGNED(region->region_end_offset)) { |
| dev_err(&vsoc_dev.dev->dev, |
| "region %d not aligned (%x:%x)", i, |
| region->region_begin_offset, |
| region->region_end_offset); |
| vsoc_remove_device(pdev); |
| return -EFAULT; |
| } |
| if (region->region_begin_offset >= region->region_end_offset || |
| region->region_end_offset > vsoc_dev.shm_size) { |
| dev_err(&vsoc_dev.dev->dev, |
| "region %d offsets are wrong: %x %x %zx", |
| i, region->region_begin_offset, |
| region->region_end_offset, vsoc_dev.shm_size); |
| vsoc_remove_device(pdev); |
| return -EFAULT; |
| } |
| if (region->managed_by >= vsoc_dev.layout->region_count) { |
| dev_err(&vsoc_dev.dev->dev, |
| "region %d has invalid owner: %u", |
| i, region->managed_by); |
| vsoc_remove_device(pdev); |
| return -EFAULT; |
| } |
| } |
| vsoc_dev.msix_enabled = true; |
| for (i = 0; i < vsoc_dev.layout->region_count; ++i) { |
| const struct vsoc_device_region *region = vsoc_dev.regions + i; |
| size_t name_sz = sizeof(vsoc_dev.regions_data[i].name) - 1; |
| const struct vsoc_signal_table_layout *h_to_g_signal_table = |
| ®ion->host_to_guest_signal_table; |
| const struct vsoc_signal_table_layout *g_to_h_signal_table = |
| ®ion->guest_to_host_signal_table; |
| |
| vsoc_dev.regions_data[i].name[name_sz] = '\0'; |
| memcpy(vsoc_dev.regions_data[i].name, region->device_name, |
| name_sz); |
| dev_info(&pdev->dev, "region %d name=%s\n", |
| i, vsoc_dev.regions_data[i].name); |
| init_waitqueue_head |
| (&vsoc_dev.regions_data[i].interrupt_wait_queue); |
| init_waitqueue_head(&vsoc_dev.regions_data[i].futex_wait_queue); |
| vsoc_dev.regions_data[i].incoming_signalled = |
| shm_off_to_virtual_addr(region->region_begin_offset) + |
| h_to_g_signal_table->interrupt_signalled_offset; |
| vsoc_dev.regions_data[i].outgoing_signalled = |
| shm_off_to_virtual_addr(region->region_begin_offset) + |
| g_to_h_signal_table->interrupt_signalled_offset; |
| result = request_irq(vsoc_dev.msix_entries[i].vector, |
| vsoc_interrupt, 0, |
| vsoc_dev.regions_data[i].name, |
| vsoc_dev.regions_data + i); |
| if (result) { |
| dev_info(&pdev->dev, |
| "request_irq failed irq=%d vector=%d\n", |
| i, vsoc_dev.msix_entries[i].vector); |
| vsoc_remove_device(pdev); |
| return -ENOSPC; |
| } |
| vsoc_dev.regions_data[i].irq_requested = true; |
| if (!device_create(vsoc_dev.class, NULL, |
| MKDEV(vsoc_dev.major, i), |
| NULL, vsoc_dev.regions_data[i].name)) { |
| dev_err(&vsoc_dev.dev->dev, "device_create failed\n"); |
| vsoc_remove_device(pdev); |
| return -EBUSY; |
| } |
| vsoc_dev.regions_data[i].device_created = true; |
| } |
| return 0; |
| } |
| |
| /* |
| * This should undo all of the allocations in the probe function in reverse |
| * order. |
| * |
| * Notes: |
| * |
| * The device may have been partially initialized, so double check |
| * that the allocations happened. |
| * |
| * This function may be called multiple times, so mark resources as freed |
| * as they are deallocated. |
| */ |
| static void vsoc_remove_device(struct pci_dev *pdev) |
| { |
| int i; |
| /* |
| * pdev is the first thing to be set on probe and the last thing |
| * to be cleared here. If it's NULL then there is no cleanup. |
| */ |
| if (!pdev || !vsoc_dev.dev) |
| return; |
| dev_info(&pdev->dev, "remove_device\n"); |
| if (vsoc_dev.regions_data) { |
| for (i = 0; i < vsoc_dev.layout->region_count; ++i) { |
| if (vsoc_dev.regions_data[i].device_created) { |
| device_destroy(vsoc_dev.class, |
| MKDEV(vsoc_dev.major, i)); |
| vsoc_dev.regions_data[i].device_created = false; |
| } |
| if (vsoc_dev.regions_data[i].irq_requested) |
| free_irq(vsoc_dev.msix_entries[i].vector, NULL); |
| vsoc_dev.regions_data[i].irq_requested = false; |
| } |
| kfree(vsoc_dev.regions_data); |
| vsoc_dev.regions_data = NULL; |
| } |
| if (vsoc_dev.msix_enabled) { |
| pci_disable_msix(pdev); |
| vsoc_dev.msix_enabled = false; |
| } |
| kfree(vsoc_dev.msix_entries); |
| vsoc_dev.msix_entries = NULL; |
| vsoc_dev.regions = NULL; |
| if (vsoc_dev.class_added) { |
| class_destroy(vsoc_dev.class); |
| vsoc_dev.class_added = false; |
| } |
| if (vsoc_dev.cdev_added) { |
| cdev_del(&vsoc_dev.cdev); |
| vsoc_dev.cdev_added = false; |
| } |
| if (vsoc_dev.major && vsoc_dev.layout) { |
| unregister_chrdev_region(MKDEV(vsoc_dev.major, 0), |
| vsoc_dev.layout->region_count); |
| vsoc_dev.major = 0; |
| } |
| vsoc_dev.layout = NULL; |
| if (vsoc_dev.kernel_mapped_shm) { |
| pci_iounmap(pdev, vsoc_dev.kernel_mapped_shm); |
| vsoc_dev.kernel_mapped_shm = NULL; |
| } |
| if (vsoc_dev.regs) { |
| pci_iounmap(pdev, vsoc_dev.regs); |
| vsoc_dev.regs = NULL; |
| } |
| if (vsoc_dev.requested_regions) { |
| pci_release_regions(pdev); |
| vsoc_dev.requested_regions = false; |
| } |
| if (vsoc_dev.enabled_device) { |
| pci_disable_device(pdev); |
| vsoc_dev.enabled_device = false; |
| } |
| /* Do this last: it indicates that the device is not initialized. */ |
| vsoc_dev.dev = NULL; |
| } |
| |
| static void __exit vsoc_cleanup_module(void) |
| { |
| vsoc_remove_device(vsoc_dev.dev); |
| pci_unregister_driver(&vsoc_pci_driver); |
| } |
| |
| static int __init vsoc_init_module(void) |
| { |
| int err = -ENOMEM; |
| |
| INIT_LIST_HEAD(&vsoc_dev.permissions); |
| mutex_init(&vsoc_dev.mtx); |
| |
| err = pci_register_driver(&vsoc_pci_driver); |
| if (err < 0) |
| return err; |
| return 0; |
| } |
| |
| static int vsoc_open(struct inode *inode, struct file *filp) |
| { |
| /* Can't use vsoc_validate_filep because filp is still incomplete */ |
| int ret = vsoc_validate_inode(inode); |
| |
| if (ret) |
| return ret; |
| filp->private_data = |
| kzalloc(sizeof(struct vsoc_private_data), GFP_KERNEL); |
| if (!filp->private_data) |
| return -ENOMEM; |
| return 0; |
| } |
| |
| static int vsoc_release(struct inode *inode, struct file *filp) |
| { |
| struct vsoc_private_data *private_data = NULL; |
| struct fd_scoped_permission_node *node = NULL; |
| struct vsoc_device_region *owner_region_p = NULL; |
| int retval = vsoc_validate_filep(filp); |
| |
| if (retval) |
| return retval; |
| private_data = (struct vsoc_private_data *)filp->private_data; |
| if (!private_data) |
| return 0; |
| |
| node = private_data->fd_scoped_permission_node; |
| if (node) { |
| owner_region_p = vsoc_region_from_inode(inode); |
| if (owner_region_p->managed_by != VSOC_REGION_WHOLE) { |
| owner_region_p = |
| &vsoc_dev.regions[owner_region_p->managed_by]; |
| } |
| do_destroy_fd_scoped_permission_node(owner_region_p, node); |
| private_data->fd_scoped_permission_node = NULL; |
| } |
| kfree(private_data); |
| filp->private_data = NULL; |
| |
| return 0; |
| } |
| |
| /* |
| * Returns the device relative offset and length of the area specified by the |
| * fd scoped permission. If there is no fd scoped permission set, a default |
| * permission covering the entire region is assumed, unless the region is owned |
| * by another one, in which case the default is a permission with zero size. |
| */ |
| static ssize_t vsoc_get_area(struct file *filp, __u32 *area_offset) |
| { |
| __u32 off = 0; |
| ssize_t length = 0; |
| struct vsoc_device_region *region_p; |
| struct fd_scoped_permission *perm; |
| |
| region_p = vsoc_region_from_filep(filp); |
| off = region_p->region_begin_offset; |
| perm = &((struct vsoc_private_data *)filp->private_data)-> |
| fd_scoped_permission_node->permission; |
| if (perm) { |
| off += perm->begin_offset; |
| length = perm->end_offset - perm->begin_offset; |
| } else if (region_p->managed_by == VSOC_REGION_WHOLE) { |
| /* No permission set and the regions is not owned by another, |
| * default to full region access. |
| */ |
| length = vsoc_device_region_size(region_p); |
| } else { |
| /* return zero length, access is denied. */ |
| length = 0; |
| } |
| if (area_offset) |
| *area_offset = off; |
| return length; |
| } |
| |
| static int vsoc_mmap(struct file *filp, struct vm_area_struct *vma) |
| { |
| unsigned long len = vma->vm_end - vma->vm_start; |
| __u32 area_off; |
| phys_addr_t mem_off; |
| ssize_t area_len; |
| int retval = vsoc_validate_filep(filp); |
| |
| if (retval) |
| return retval; |
| area_len = vsoc_get_area(filp, &area_off); |
| /* Add the requested offset */ |
| area_off += (vma->vm_pgoff << PAGE_SHIFT); |
| area_len -= (vma->vm_pgoff << PAGE_SHIFT); |
| if (area_len < len) |
| return -EINVAL; |
| vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); |
| mem_off = shm_off_to_phys_addr(area_off); |
| if (io_remap_pfn_range(vma, vma->vm_start, mem_off >> PAGE_SHIFT, |
| len, vma->vm_page_prot)) |
| return -EAGAIN; |
| return 0; |
| } |
| |
| module_init(vsoc_init_module); |
| module_exit(vsoc_cleanup_module); |
| |
| MODULE_LICENSE("GPL"); |
| MODULE_AUTHOR("Greg Hartman <ghartman@google.com>"); |
| MODULE_DESCRIPTION("VSoC interpretation of QEmu's ivshmem device"); |
| MODULE_VERSION("1.0"); |