// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. * Copyright (c) 2021, The Linux Foundation. All rights reserved. */ #define pr_fmt(fmt) "%s: " fmt, __func__ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define CLASS_NAME "sync" #define DRV_NAME "spec_sync" #define DRV_VERSION 1 #define NAME_LEN 32 #define FENCE_MIN 1 #define FENCE_MAX 32 #if IS_ENABLED(CONFIG_DEBUG_FS) #define MAX_DEVICE_SUPPORTED 2 #else #define MAX_DEVICE_SUPPORTED 1 #endif #define DUMMY_CONTEXT 0xfafadadafafadada #define DUMMY_SEQNO 0xefa9ce00efa9ce00 struct dummy_spec_fence { struct dma_fence fence; spinlock_t lock; }; struct sync_device { /* device info */ struct class *dev_class; dev_t dev_num; struct device *dev; struct cdev *cdev; struct mutex lock; struct dummy_spec_fence *dummy_fence; /* device drv data */ atomic_t device_available; char name[NAME_LEN]; uint32_t version; struct mutex l_lock; struct list_head fence_array_list; wait_queue_head_t wait_queue; }; struct fence_array_node { struct dma_fence_array *fence_array; struct list_head list; }; /* Speculative Sync Device Driver State */ static struct sync_device sync_dev; static const char *spec_fence_get_name_dummy(struct dma_fence *fence) { return "dummy_fence"; } static const struct dma_fence_ops dummy_spec_fence_ops = { .get_driver_name = spec_fence_get_name_dummy, .get_timeline_name = spec_fence_get_name_dummy, }; static bool sanitize_fence_array(struct dma_fence_array *fence) { struct fence_array_node *node; int ret = false; mutex_lock(&sync_dev.l_lock); list_for_each_entry(node, &sync_dev.fence_array_list, list) { if (node->fence_array == fence) { ret = true; break; } } mutex_unlock(&sync_dev.l_lock); return ret; } static void clear_fence_array_tracker(bool force_clear) { struct fence_array_node *node, *temp; struct dma_fence_array *array; struct dma_fence *fence; bool is_signaled; mutex_lock(&sync_dev.l_lock); list_for_each_entry_safe(node, temp, &sync_dev.fence_array_list, list) { array = node->fence_array; fence = &array->base; is_signaled = dma_fence_is_signaled(fence); if (force_clear && !array->fences) array->num_fences = 0; pr_debug("force_clear:%d is_signaled:%d pending:%d\n", force_clear, is_signaled, atomic_read(&array->num_pending)); if (force_clear && !is_signaled && atomic_dec_and_test(&array->num_pending)) dma_fence_signal(fence); if (force_clear || is_signaled) { dma_fence_put(fence); list_del(&node->list); kfree(node); } } mutex_unlock(&sync_dev.l_lock); } static struct sync_device *spec_fence_init_locked(struct sync_device *obj, const char *name) { if (atomic_read(&obj->device_available) >= MAX_DEVICE_SUPPORTED) { pr_err("number of device fds are limited to %d, device opened:%d\n", MAX_DEVICE_SUPPORTED, atomic_read(&obj->device_available)); return NULL; } else if (!atomic_read(&obj->device_available)) { memset(obj->name, 0, NAME_LEN); strscpy(obj->name, name, sizeof(obj->name)); } atomic_inc(&obj->device_available); return obj; } static int spec_sync_open(struct inode *inode, struct file *file) { char task_comm[TASK_COMM_LEN]; struct sync_device *obj = &sync_dev; int ret = 0; if (!inode || !inode->i_cdev || !file) { pr_err("NULL pointer passed\n"); return -EINVAL; } mutex_lock(&sync_dev.lock); get_task_comm(task_comm, current); obj = spec_fence_init_locked(obj, task_comm); if (!obj) { pr_err("Spec device exists owner:%s caller:%s\n", sync_dev.name, task_comm); ret = -EEXIST; goto end; } file->private_data = obj; end: mutex_unlock(&sync_dev.lock); return ret; } static int spec_sync_release(struct inode *inode, struct file *file) { int ret = 0; struct sync_device *obj = file->private_data; mutex_lock(&sync_dev.lock); if (!atomic_read(&obj->device_available)) { pr_err("no device to release!!\n"); ret = -ENODEV; goto end; } atomic_dec(&obj->device_available); if (!atomic_read(&obj->device_available)) clear_fence_array_tracker(true); end: mutex_unlock(&sync_dev.lock); return ret; } static int spec_sync_ioctl_get_ver(struct sync_device *obj, unsigned long __user arg) { uint32_t version = obj->version; if (copy_to_user((void __user *)arg, &version, sizeof(uint32_t))) return -EFAULT; return 0; } static int spec_sync_create_array(struct fence_create_data *f) { int fd = get_unused_fd_flags(O_CLOEXEC); struct sync_file *sync_file; struct dma_fence_array *fence_array; struct fence_array_node *node; struct dma_fence **fences; struct dummy_spec_fence *dummy_fence_p = sync_dev.dummy_fence; bool signal_any; int i, ret = 0; if (fd < 0) { pr_err("failed to get_unused_fd_flags\n"); return fd; } if (f->num_fences < FENCE_MIN || f->num_fences > FENCE_MAX) { pr_err("invalid arguments num_fences:%d\n", f->num_fences); ret = -ERANGE; goto error_args; } fences = kmalloc_array(f->num_fences, sizeof(void *), GFP_KERNEL|__GFP_ZERO); if (!fences) { ret = -ENOMEM; goto error_args; } for (i = 0; i < f->num_fences; i++) { fences[i] = &dummy_fence_p->fence; /* * Increase dummy-fences refcount here, we must do this since any call to * fence-array release while dummy-fences are the children of the fence-array * will decrement the dummy_fence refcount. Therefore, to prevent the release * of the dummy_fence fences, we must keep an extra refcount for every time that * the fence-array->release can decrement its children's refcount. the extra * refcount will be decreased impilictly when dma_fence_put(&fence_array->base) * called. */ dma_fence_get(&dummy_fence_p->fence); } signal_any = f->flags & SPEC_FENCE_SIGNAL_ALL ? false : true; fence_array = dma_fence_array_create(f->num_fences, fences, dma_fence_context_alloc(1), 0, signal_any); if (!fence_array) { /* fence-array create failed, remove extra refcounts */ for (i = 0; i < f->num_fences; i++) dma_fence_put(&dummy_fence_p->fence); kfree(fences); ret = -EINVAL; goto error_args; } /* Set the enable signal such that signalling is not done during wait*/ set_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &fence_array->base.flags); set_bit(SPEC_FENCE_FLAG_FENCE_ARRAY, &fence_array->base.flags); sync_file = sync_file_create(&fence_array->base); if (!sync_file) { pr_err("sync_file_create fail\n"); ret = -EINVAL; goto err; } node = kzalloc((sizeof(struct fence_array_node)), GFP_KERNEL); if (!node) { fput(sync_file->file); ret = -ENOMEM; goto err; } fd_install(fd, sync_file->file); node->fence_array = fence_array; mutex_lock(&sync_dev.l_lock); list_add_tail(&node->list, &sync_dev.fence_array_list); mutex_unlock(&sync_dev.l_lock); pr_debug("spec fd:%d num_fences:%u\n", fd, f->num_fences); return fd; err: dma_fence_put(&fence_array->base); error_args: put_unused_fd(fd); return ret; } static int spec_sync_ioctl_create_fence(struct sync_device *obj, unsigned long __user arg) { struct fence_create_data f; int fd; if (copy_from_user(&f, (void __user *)arg, sizeof(f))) return -EFAULT; fd = spec_sync_create_array(&f); if (fd < 0) return fd; f.out_bind_fd = fd; if (copy_to_user((void __user *)arg, &f, sizeof(f))) return -EFAULT; return 0; } int spec_sync_wait_bind_array(struct dma_fence_array *fence_array, u32 timeout_ms) { int ret; /* Check if fence-array is a speculative fence */ if (!fence_array || !test_bit(SPEC_FENCE_FLAG_FENCE_ARRAY, &fence_array->base.flags)) { pr_err("invalid fence!\n"); return -EINVAL; } else if (test_bit(SPEC_FENCE_FLAG_FENCE_ARRAY_BOUND, &fence_array->base.flags)) { /* This fence-array is already bound, just return success */ return 0; } /* Wait for the fence-array bind */ ret = wait_event_timeout(sync_dev.wait_queue, test_bit(SPEC_FENCE_FLAG_FENCE_ARRAY_BOUND, &fence_array->base.flags), msecs_to_jiffies(timeout_ms)); if (!ret) { pr_err("timed out waiting for bind fence-array %d\n", timeout_ms); ret = -ETIMEDOUT; } else { ret = 0; } return ret; } EXPORT_SYMBOL(spec_sync_wait_bind_array); static int spec_sync_bind_array(struct fence_bind_data *sync_bind_info) { struct dma_fence_array *fence_array; struct dma_fence *fence = NULL; struct dma_fence *user_fence = NULL; int *user_fds, ret = 0, i; u32 num_fences; fence = sync_file_get_fence(sync_bind_info->out_bind_fd); if (!fence) { pr_err("dma fence failure out_fd:%d\n", sync_bind_info->out_bind_fd); return -EINVAL; } if (dma_fence_is_signaled(fence)) { pr_err("spec fence is already signaled, out_fd:%d\n", sync_bind_info->out_bind_fd); ret = -EINVAL; goto end; } fence_array = container_of(fence, struct dma_fence_array, base); if (!sanitize_fence_array(fence_array)) { pr_err("spec fence not found in the registered list out_fd:%d\n", sync_bind_info->out_bind_fd); ret = -EINVAL; goto end; } num_fences = fence_array->num_fences; for (i = 0; i < num_fences; i++) { if (!(fence_array->fences[i]->context == DUMMY_CONTEXT && fence_array->fences[i]->seqno == DUMMY_SEQNO)) { pr_err("fence array already populated, spec fd:%d status:%d flags:0x%x\n", sync_bind_info->out_bind_fd, dma_fence_get_status(fence), fence->flags); ret = -EINVAL; goto end; } } user_fds = kzalloc(num_fences * (sizeof(int)), GFP_KERNEL); if (!user_fds) { ret = -ENOMEM; goto end; } if (copy_from_user(user_fds, (void __user *)sync_bind_info->fds, num_fences * sizeof(int))) { ret = -EFAULT; goto out; } spin_lock(fence->lock); for (i = 0; i < num_fences; i++) { user_fence = sync_file_get_fence(user_fds[i]); if (!user_fence) { pr_warn("bind fences are invalid !! user_fd:%d out_bind_fd:%d\n", user_fds[i], sync_bind_info->out_bind_fd); ret = -EINVAL; goto bind_invalid; } fence_array->fences[i] = user_fence; /* * At this point the fence-array fully contains valid fences and no more the * dummy-fence, therefore, we must release the extra refcount that the * creation of the speculative fence added to the dummy-fence. */ dma_fence_put(&sync_dev.dummy_fence->fence); pr_debug("spec fd:%d i:%d bind fd:%d error:%d\n", sync_bind_info->out_bind_fd, i, user_fds[i], fence_array->fences[i]->error); } clear_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &fence->flags); spin_unlock(fence->lock); dma_fence_enable_sw_signaling(&fence_array->base); clear_fence_array_tracker(false); bind_invalid: set_bit(SPEC_FENCE_FLAG_FENCE_ARRAY_BOUND, &fence_array->base.flags); wake_up_all(&sync_dev.wait_queue); if (ret) { dma_fence_set_error(fence, -EINVAL); spin_unlock(fence->lock); dma_fence_signal(fence); clear_fence_array_tracker(false); } out: kfree(user_fds); end: dma_fence_put(fence); return ret; } static int spec_sync_ioctl_bind(struct sync_device *obj, unsigned long __user arg) { struct fence_bind_data sync_bind_info; if (copy_from_user(&sync_bind_info, (void __user *)arg, sizeof(struct fence_bind_data))) return -EFAULT; if (sync_bind_info.out_bind_fd < 0) { pr_err("Invalid out_fd:%d\n", sync_bind_info.out_bind_fd); return -EINVAL; } return spec_sync_bind_array(&sync_bind_info); } static long spec_sync_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { struct sync_device *obj = file->private_data; int ret = 0; switch (cmd) { case SPEC_SYNC_IOC_CREATE_FENCE: ret = spec_sync_ioctl_create_fence(obj, arg); break; case SPEC_SYNC_IOC_BIND: ret = spec_sync_ioctl_bind(obj, arg); break; case SPEC_SYNC_IOC_GET_VER: ret = spec_sync_ioctl_get_ver(obj, arg); break; default: ret = -ENOTTY; } return ret; } const struct file_operations spec_sync_fops = { .owner = THIS_MODULE, .open = spec_sync_open, .release = spec_sync_release, .unlocked_ioctl = spec_sync_ioctl, }; static int spec_sync_register_device(void) { struct dummy_spec_fence *dummy_fence_p = NULL; int ret; sync_dev.dev_class = class_create(THIS_MODULE, CLASS_NAME); if (sync_dev.dev_class == NULL) { pr_err("%s: class_create fail.\n", __func__); goto res_err; } ret = alloc_chrdev_region(&sync_dev.dev_num, 0, 1, DRV_NAME); if (ret) { pr_err("%s: alloc_chrdev_region fail.\n", __func__); goto alloc_chrdev_region_err; } sync_dev.dev = device_create(sync_dev.dev_class, NULL, sync_dev.dev_num, &sync_dev, DRV_NAME); if (IS_ERR(sync_dev.dev)) { pr_err("%s: device_create fail.\n", __func__); goto device_create_err; } sync_dev.cdev = cdev_alloc(); if (sync_dev.cdev == NULL) { pr_err("%s: cdev_alloc fail.\n", __func__); goto cdev_alloc_err; } cdev_init(sync_dev.cdev, &spec_sync_fops); sync_dev.cdev->owner = THIS_MODULE; ret = cdev_add(sync_dev.cdev, sync_dev.dev_num, 1); if (ret) { pr_err("%s: cdev_add fail.\n", __func__); goto cdev_add_err; } sync_dev.version = DRV_VERSION; mutex_init(&sync_dev.lock); mutex_init(&sync_dev.l_lock); INIT_LIST_HEAD(&sync_dev.fence_array_list); init_waitqueue_head(&sync_dev.wait_queue); dummy_fence_p = kzalloc(sizeof(struct dummy_spec_fence), GFP_KERNEL); if (!dummy_fence_p) { ret = -ENOMEM; goto cdev_add_err; } spin_lock_init(&dummy_fence_p->lock); dma_fence_init(&dummy_fence_p->fence, &dummy_spec_fence_ops, &dummy_fence_p->lock, DUMMY_CONTEXT, DUMMY_SEQNO); sync_dev.dummy_fence = dummy_fence_p; return 0; cdev_add_err: cdev_del(sync_dev.cdev); cdev_alloc_err: device_destroy(sync_dev.dev_class, sync_dev.dev_num); device_create_err: unregister_chrdev_region(sync_dev.dev_num, 1); alloc_chrdev_region_err: class_destroy(sync_dev.dev_class); res_err: return -ENODEV; } static int __init spec_sync_init(void) { int ret = 0; ret = spec_sync_register_device(); if (ret) { pr_err("%s: speculative sync driver register fail.\n", __func__); return ret; } return ret; } static void __exit spec_sync_deinit(void) { cdev_del(sync_dev.cdev); device_destroy(sync_dev.dev_class, sync_dev.dev_num); unregister_chrdev_region(sync_dev.dev_num, 1); class_destroy(sync_dev.dev_class); dma_fence_put(&sync_dev.dummy_fence->fence); } module_init(spec_sync_init); module_exit(spec_sync_deinit); MODULE_DESCRIPTION("QCOM Speculative Sync Driver"); MODULE_LICENSE("GPL v2");