--- zzzz-none-000/linux-3.10.107/drivers/iommu/iommu.c 2017-06-27 09:49:32.000000000 +0000 +++ scorpion-7490-727/linux-3.10.107/drivers/iommu/iommu.c 2021-02-04 17:41:59.000000000 +0000 @@ -1,6 +1,6 @@ /* * Copyright (C) 2007-2008 Advanced Micro Devices, Inc. - * Author: Joerg Roedel + * Author: Joerg Roedel * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 as published @@ -16,7 +16,7 @@ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ -#define pr_fmt(fmt) "%s: " fmt, __func__ +#define pr_fmt(fmt) "iommu: " fmt #include #include @@ -29,11 +29,19 @@ #include #include #include +#include +#include +#include +#include static struct kset *iommu_group_kset; static struct ida iommu_group_ida; static struct mutex iommu_group_mutex; +struct iommu_callback_data { + const struct iommu_ops *ops; +}; + struct iommu_group { struct kobject kobj; struct kobject *devices_kobj; @@ -44,6 +52,8 @@ void (*iommu_data_release)(void *iommu_data); char *name; int id; + struct iommu_domain *default_domain; + struct iommu_domain *domain; }; struct iommu_device { @@ -68,6 +78,15 @@ #define to_iommu_group(_kobj) \ container_of(_kobj, struct iommu_group, kobj) +static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus, + unsigned type); +static int __iommu_attach_device(struct iommu_domain *domain, + struct device *dev); +static int __iommu_attach_group(struct iommu_domain *domain, + struct iommu_group *group); +static void __iommu_detach_group(struct iommu_domain *domain, + struct iommu_group *group); + static ssize_t iommu_group_attr_show(struct kobject *kobj, struct attribute *__attr, char *buf) { @@ -121,6 +140,8 @@ { struct iommu_group *group = to_iommu_group(kobj); + pr_debug("Releasing group %d\n", group->id); + if (group->iommu_data_release) group->iommu_data_release(group->iommu_data); @@ -128,6 +149,9 @@ ida_remove(&iommu_group_ida, group->id); mutex_unlock(&iommu_group_mutex); + if (group->default_domain) + iommu_domain_free(group->default_domain); + kfree(group->name); kfree(group); } @@ -200,6 +224,8 @@ */ kobject_put(&group->kobj); + pr_debug("Allocated group %d\n", group->id); + return group; } EXPORT_SYMBOL_GPL(iommu_group_alloc); @@ -300,6 +326,52 @@ } EXPORT_SYMBOL_GPL(iommu_group_set_name); +static int iommu_group_create_direct_mappings(struct iommu_group *group, + struct device *dev) +{ + struct iommu_domain *domain = group->default_domain; + struct iommu_dm_region *entry; + struct list_head mappings; + unsigned long pg_size; + int ret = 0; + + if (!domain || domain->type != IOMMU_DOMAIN_DMA) + return 0; + + BUG_ON(!domain->pgsize_bitmap); + + pg_size = 1UL << __ffs(domain->pgsize_bitmap); + INIT_LIST_HEAD(&mappings); + + iommu_get_dm_regions(dev, &mappings); + + /* We need to consider overlapping regions for different devices */ + list_for_each_entry(entry, &mappings, list) { + dma_addr_t start, end, addr; + + start = ALIGN(entry->start, pg_size); + end = ALIGN(entry->start + entry->length, pg_size); + + for (addr = start; addr < end; addr += pg_size) { + phys_addr_t phys_addr; + + phys_addr = iommu_iova_to_phys(domain, addr); + if (phys_addr) + continue; + + ret = iommu_map(domain, addr, addr, pg_size, entry->prot); + if (ret) + goto out; + } + + } + +out: + iommu_put_dm_regions(dev, &mappings); + + return ret; +} + /** * iommu_group_add_device - add a device to an iommu group * @group: the group into which to add the device (reference should be held) @@ -356,13 +428,23 @@ dev->iommu_group = group; + iommu_group_create_direct_mappings(group, dev); + mutex_lock(&group->mutex); list_add_tail(&device->list, &group->devices); + if (group->domain) + __iommu_attach_device(group->domain, dev); mutex_unlock(&group->mutex); + /* Notify any listeners about change to group. */ blocking_notifier_call_chain(&group->notifier, IOMMU_GROUP_NOTIFY_ADD_DEVICE, dev); + + trace_add_device_to_group(group->id, dev); + + pr_info("Adding device %s to group %d\n", dev_name(dev), group->id); + return 0; } EXPORT_SYMBOL_GPL(iommu_group_add_device); @@ -379,6 +461,8 @@ struct iommu_group *group = dev->iommu_group; struct iommu_device *tmp_device, *device = NULL; + pr_info("Removing device %s from group %d\n", dev_name(dev), group->id); + /* Pre-notify listeners that a device is being removed. */ blocking_notifier_call_chain(&group->notifier, IOMMU_GROUP_NOTIFY_DEL_DEVICE, dev); @@ -399,6 +483,8 @@ sysfs_remove_link(group->devices_kobj, device->name); sysfs_remove_link(&dev->kobj, "iommu_group"); + trace_remove_device_from_group(group->id, dev); + kfree(device->name); kfree(device); dev->iommu_group = NULL; @@ -406,6 +492,17 @@ } EXPORT_SYMBOL_GPL(iommu_group_remove_device); +static int iommu_group_device_count(struct iommu_group *group) +{ + struct iommu_device *entry; + int ret = 0; + + list_for_each_entry(entry, &group->devices, list) + ret++; + + return ret; +} + /** * iommu_group_for_each_dev - iterate over each device in the group * @group: the group @@ -417,19 +514,30 @@ * The group->mutex is held across callbacks, which will block calls to * iommu_group_add/remove_device. */ -int iommu_group_for_each_dev(struct iommu_group *group, void *data, - int (*fn)(struct device *, void *)) +static int __iommu_group_for_each_dev(struct iommu_group *group, void *data, + int (*fn)(struct device *, void *)) { struct iommu_device *device; int ret = 0; - mutex_lock(&group->mutex); list_for_each_entry(device, &group->devices, list) { ret = fn(device->dev, data); if (ret) break; } + return ret; +} + + +int iommu_group_for_each_dev(struct iommu_group *group, void *data, + int (*fn)(struct device *, void *)) +{ + int ret; + + mutex_lock(&group->mutex); + ret = __iommu_group_for_each_dev(group, data, fn); mutex_unlock(&group->mutex); + return ret; } EXPORT_SYMBOL_GPL(iommu_group_for_each_dev); @@ -454,6 +562,19 @@ EXPORT_SYMBOL_GPL(iommu_group_get); /** + * iommu_group_ref_get - Increment reference on a group + * @group: the group to use, must not be NULL + * + * This function is called by iommu drivers to take additional references on an + * existing group. Returns the given group for convenience. + */ +struct iommu_group *iommu_group_ref_get(struct iommu_group *group) +{ + kobject_get(group->devices_kobj); + return group; +} + +/** * iommu_group_put - Decrement group reference * @group: the group to use * @@ -509,16 +630,288 @@ } EXPORT_SYMBOL_GPL(iommu_group_id); +static struct iommu_group *get_pci_alias_group(struct pci_dev *pdev, + unsigned long *devfns); + +/* + * To consider a PCI device isolated, we require ACS to support Source + * Validation, Request Redirection, Completer Redirection, and Upstream + * Forwarding. This effectively means that devices cannot spoof their + * requester ID, requests and completions cannot be redirected, and all + * transactions are forwarded upstream, even as it passes through a + * bridge where the target device is downstream. + */ +#define REQ_ACS_FLAGS (PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF) + +/* + * For multifunction devices which are not isolated from each other, find + * all the other non-isolated functions and look for existing groups. For + * each function, we also need to look for aliases to or from other devices + * that may already have a group. + */ +static struct iommu_group *get_pci_function_alias_group(struct pci_dev *pdev, + unsigned long *devfns) +{ + struct pci_dev *tmp = NULL; + struct iommu_group *group; + + if (!pdev->multifunction || pci_acs_enabled(pdev, REQ_ACS_FLAGS)) + return NULL; + + for_each_pci_dev(tmp) { + if (tmp == pdev || tmp->bus != pdev->bus || + PCI_SLOT(tmp->devfn) != PCI_SLOT(pdev->devfn) || + pci_acs_enabled(tmp, REQ_ACS_FLAGS)) + continue; + + group = get_pci_alias_group(tmp, devfns); + if (group) { + pci_dev_put(tmp); + return group; + } + } + + return NULL; +} + +/* + * Look for aliases to or from the given device for exisiting groups. The + * dma_alias_devfn only supports aliases on the same bus, therefore the search + * space is quite small (especially since we're really only looking at pcie + * device, and therefore only expect multiple slots on the root complex or + * downstream switch ports). It's conceivable though that a pair of + * multifunction devices could have aliases between them that would cause a + * loop. To prevent this, we use a bitmap to track where we've been. + */ +static struct iommu_group *get_pci_alias_group(struct pci_dev *pdev, + unsigned long *devfns) +{ + struct pci_dev *tmp = NULL; + struct iommu_group *group; + + if (test_and_set_bit(pdev->devfn & 0xff, devfns)) + return NULL; + + group = iommu_group_get(&pdev->dev); + if (group) + return group; + + for_each_pci_dev(tmp) { + if (tmp == pdev || tmp->bus != pdev->bus) + continue; + + /* We alias them or they alias us */ + if (((pdev->dev_flags & PCI_DEV_FLAGS_DMA_ALIAS_DEVFN) && + pdev->dma_alias_devfn == tmp->devfn) || + ((tmp->dev_flags & PCI_DEV_FLAGS_DMA_ALIAS_DEVFN) && + tmp->dma_alias_devfn == pdev->devfn)) { + + group = get_pci_alias_group(tmp, devfns); + if (group) { + pci_dev_put(tmp); + return group; + } + + group = get_pci_function_alias_group(tmp, devfns); + if (group) { + pci_dev_put(tmp); + return group; + } + } + } + + return NULL; +} + +struct group_for_pci_data { + struct pci_dev *pdev; + struct iommu_group *group; +}; + +/* + * DMA alias iterator callback, return the last seen device. Stop and return + * the IOMMU group if we find one along the way. + */ +static int get_pci_alias_or_group(struct pci_dev *pdev, u16 alias, void *opaque) +{ + struct group_for_pci_data *data = opaque; + + data->pdev = pdev; + data->group = iommu_group_get(&pdev->dev); + + return data->group != NULL; +} + +/* + * Generic device_group call-back function. It just allocates one + * iommu-group per device. + */ +struct iommu_group *generic_device_group(struct device *dev) +{ + struct iommu_group *group; + + group = iommu_group_alloc(); + if (IS_ERR(group)) + return NULL; + + return group; +} + +/* + * Use standard PCI bus topology, isolation features, and DMA alias quirks + * to find or create an IOMMU group for a device. + */ +struct iommu_group *pci_device_group(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct group_for_pci_data data; + struct pci_bus *bus; + struct iommu_group *group = NULL; + u64 devfns[4] = { 0 }; + + if (WARN_ON(!dev_is_pci(dev))) + return ERR_PTR(-EINVAL); + + /* + * Find the upstream DMA alias for the device. A device must not + * be aliased due to topology in order to have its own IOMMU group. + * If we find an alias along the way that already belongs to a + * group, use it. + */ + if (pci_for_each_dma_alias(pdev, get_pci_alias_or_group, &data)) + return data.group; + + pdev = data.pdev; + + /* + * Continue upstream from the point of minimum IOMMU granularity + * due to aliases to the point where devices are protected from + * peer-to-peer DMA by PCI ACS. Again, if we find an existing + * group, use it. + */ + for (bus = pdev->bus; !pci_is_root_bus(bus); bus = bus->parent) { + if (!bus->self) + continue; + + if (pci_acs_path_enabled(bus->self, NULL, REQ_ACS_FLAGS)) + break; + + pdev = bus->self; + + group = iommu_group_get(&pdev->dev); + if (group) + return group; + } + + /* + * Look for existing groups on device aliases. If we alias another + * device or another device aliases us, use the same group. + */ + group = get_pci_alias_group(pdev, (unsigned long *)devfns); + if (group) + return group; + + /* + * Look for existing groups on non-isolated functions on the same + * slot and aliases of those funcions, if any. No need to clear + * the search bitmap, the tested devfns are still valid. + */ + group = get_pci_function_alias_group(pdev, (unsigned long *)devfns); + if (group) + return group; + + /* No shared group found, allocate new */ + group = iommu_group_alloc(); + if (IS_ERR(group)) + return NULL; + + return group; +} + +/** + * iommu_group_get_for_dev - Find or create the IOMMU group for a device + * @dev: target device + * + * This function is intended to be called by IOMMU drivers and extended to + * support common, bus-defined algorithms when determining or creating the + * IOMMU group for a device. On success, the caller will hold a reference + * to the returned IOMMU group, which will already include the provided + * device. The reference should be released with iommu_group_put(). + */ +struct iommu_group *iommu_group_get_for_dev(struct device *dev) +{ + const struct iommu_ops *ops = dev->bus->iommu_ops; + struct iommu_group *group; + int ret; + + group = iommu_group_get(dev); + if (group) + return group; + + group = ERR_PTR(-EINVAL); + + if (ops && ops->device_group) + group = ops->device_group(dev); + + if (IS_ERR(group)) + return group; + + /* + * Try to allocate a default domain - needs support from the + * IOMMU driver. + */ + if (!group->default_domain) { + group->default_domain = __iommu_domain_alloc(dev->bus, + IOMMU_DOMAIN_DMA); + if (!group->domain) + group->domain = group->default_domain; + } + + ret = iommu_group_add_device(group, dev); + if (ret) { + iommu_group_put(group); + return ERR_PTR(ret); + } + + return group; +} + +struct iommu_domain *iommu_group_default_domain(struct iommu_group *group) +{ + return group->default_domain; +} + static int add_iommu_group(struct device *dev, void *data) { - struct iommu_ops *ops = data; + struct iommu_callback_data *cb = data; + const struct iommu_ops *ops = cb->ops; + int ret; if (!ops->add_device) - return -ENODEV; + return 0; WARN_ON(dev->iommu_group); - ops->add_device(dev); + ret = ops->add_device(dev); + + /* + * We ignore -ENODEV errors for now, as they just mean that the + * device is not translated by an IOMMU. We still care about + * other errors and fail to initialize when they happen. + */ + if (ret == -ENODEV) + ret = 0; + + return ret; +} + +static int remove_iommu_group(struct device *dev, void *data) +{ + struct iommu_callback_data *cb = data; + const struct iommu_ops *ops = cb->ops; + + if (ops->remove_device && dev->iommu_group) + ops->remove_device(dev); return 0; } @@ -527,7 +920,7 @@ unsigned long action, void *data) { struct device *dev = data; - struct iommu_ops *ops = dev->bus->iommu_ops; + const struct iommu_ops *ops = dev->bus->iommu_ops; struct iommu_group *group; unsigned long group_action = 0; @@ -536,9 +929,10 @@ * result in ADD/DEL notifiers to group->notifier */ if (action == BUS_NOTIFY_ADD_DEVICE) { - if (ops->add_device) - return ops->add_device(dev); - } else if (action == BUS_NOTIFY_DEL_DEVICE) { + /*if (ops->add_device) + * return ops->add_device(dev); + */ + } else if (action == BUS_NOTIFY_REMOVED_DEVICE) { if (ops->remove_device && dev->iommu_group) { ops->remove_device(dev); return 0; @@ -576,14 +970,40 @@ return 0; } -static struct notifier_block iommu_bus_nb = { - .notifier_call = iommu_bus_notifier, -}; - -static void iommu_bus_init(struct bus_type *bus, struct iommu_ops *ops) +static int iommu_bus_init(struct bus_type *bus, const struct iommu_ops *ops) { - bus_register_notifier(bus, &iommu_bus_nb); - bus_for_each_dev(bus, NULL, ops, add_iommu_group); + int err; + struct notifier_block *nb; + struct iommu_callback_data cb = { + .ops = ops, + }; + + nb = kzalloc(sizeof(struct notifier_block), GFP_KERNEL); + if (!nb) + return -ENOMEM; + + nb->notifier_call = iommu_bus_notifier; + + err = bus_register_notifier(bus, nb); + if (err) + goto out_free; + + err = bus_for_each_dev(bus, NULL, &cb, add_iommu_group); + if (err) + goto out_err; + + + return 0; + +out_err: + /* Clean up */ + bus_for_each_dev(bus, NULL, &cb, remove_iommu_group); + bus_unregister_notifier(bus, nb); + +out_free: + kfree(nb); + + return err; } /** @@ -599,17 +1019,21 @@ * is set up. With this function the iommu-driver can set the iommu-ops * afterwards. */ -int bus_set_iommu(struct bus_type *bus, struct iommu_ops *ops) +int bus_set_iommu(struct bus_type *bus, const struct iommu_ops *ops) { + int err; + if (bus->iommu_ops != NULL) return -EBUSY; bus->iommu_ops = ops; /* Do IOMMU specific setup for this bus-type */ - iommu_bus_init(bus, ops); + err = iommu_bus_init(bus, ops); + if (err) + bus->iommu_ops = NULL; - return 0; + return err; } EXPORT_SYMBOL_GPL(bus_set_iommu); @@ -619,6 +1043,15 @@ } EXPORT_SYMBOL_GPL(iommu_present); +bool iommu_capable(struct bus_type *bus, enum iommu_cap cap) +{ + if (!bus->iommu_ops || !bus->iommu_ops->capable) + return false; + + return bus->iommu_ops->capable(cap); +} +EXPORT_SYMBOL_GPL(iommu_capable); + /** * iommu_set_fault_handler() - set a fault handler for an iommu domain * @domain: iommu domain @@ -642,60 +1075,138 @@ } EXPORT_SYMBOL_GPL(iommu_set_fault_handler); -struct iommu_domain *iommu_domain_alloc(struct bus_type *bus) +static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus, + unsigned type) { struct iommu_domain *domain; - int ret; if (bus == NULL || bus->iommu_ops == NULL) return NULL; - domain = kzalloc(sizeof(*domain), GFP_KERNEL); + domain = bus->iommu_ops->domain_alloc(type); if (!domain) return NULL; - domain->ops = bus->iommu_ops; - - ret = domain->ops->domain_init(domain); - if (ret) - goto out_free; + domain->ops = bus->iommu_ops; + domain->type = type; + /* Assume all sizes by default; the driver may override this later */ + domain->pgsize_bitmap = bus->iommu_ops->pgsize_bitmap; + memset(domain->name, 0, IOMMU_DOMAIN_NAME_LEN); return domain; +} -out_free: - kfree(domain); - - return NULL; +struct iommu_domain *iommu_domain_alloc(struct bus_type *bus) +{ + return __iommu_domain_alloc(bus, IOMMU_DOMAIN_UNMANAGED); } EXPORT_SYMBOL_GPL(iommu_domain_alloc); void iommu_domain_free(struct iommu_domain *domain) { - if (likely(domain->ops->domain_destroy != NULL)) - domain->ops->domain_destroy(domain); - - kfree(domain); + domain->ops->domain_free(domain); } EXPORT_SYMBOL_GPL(iommu_domain_free); -int iommu_attach_device(struct iommu_domain *domain, struct device *dev) +static int __iommu_attach_device(struct iommu_domain *domain, + struct device *dev) { + int ret; if (unlikely(domain->ops->attach_dev == NULL)) return -ENODEV; - return domain->ops->attach_dev(domain, dev); + ret = domain->ops->attach_dev(domain, dev); + if (!ret) { + trace_attach_device_to_domain(dev); + + if (!strnlen(domain->name, IOMMU_DOMAIN_NAME_LEN)) { + strlcpy(domain->name, dev_name(dev), + IOMMU_DOMAIN_NAME_LEN); + } + } + return ret; +} + +int iommu_attach_device(struct iommu_domain *domain, struct device *dev) +{ + struct iommu_group *group; + int ret; + + group = iommu_group_get(dev); + /* FIXME: Remove this when groups a mandatory for iommu drivers */ + if (group == NULL) + return __iommu_attach_device(domain, dev); + + /* + * We have a group - lock it to make sure the device-count doesn't + * change while we are attaching + */ + mutex_lock(&group->mutex); + ret = -EINVAL; + if (iommu_group_device_count(group) != 1) + goto out_unlock; + + ret = __iommu_attach_group(domain, group); + +out_unlock: + mutex_unlock(&group->mutex); + iommu_group_put(group); + + return ret; } EXPORT_SYMBOL_GPL(iommu_attach_device); -void iommu_detach_device(struct iommu_domain *domain, struct device *dev) +static void __iommu_detach_device(struct iommu_domain *domain, + struct device *dev) { if (unlikely(domain->ops->detach_dev == NULL)) return; domain->ops->detach_dev(domain, dev); + trace_detach_device_from_domain(dev); +} + +void iommu_detach_device(struct iommu_domain *domain, struct device *dev) +{ + struct iommu_group *group; + + group = iommu_group_get(dev); + /* FIXME: Remove this when groups a mandatory for iommu drivers */ + if (group == NULL) + return __iommu_detach_device(domain, dev); + + mutex_lock(&group->mutex); + if (iommu_group_device_count(group) != 1) { + WARN_ON(1); + goto out_unlock; + } + + __iommu_detach_group(domain, group); + +out_unlock: + mutex_unlock(&group->mutex); + iommu_group_put(group); } EXPORT_SYMBOL_GPL(iommu_detach_device); +struct iommu_domain *iommu_get_domain_for_dev(struct device *dev) +{ + struct iommu_domain *domain; + struct iommu_group *group; + + group = iommu_group_get(dev); + /* FIXME: Remove this when groups a mandatory for iommu drivers */ + if (group == NULL) + return NULL; + + domain = group->domain; + + iommu_group_put(group); + + return domain; +} +EXPORT_SYMBOL_GPL(iommu_get_domain_for_dev); + /* * IOMMU groups are really the natrual working unit of the IOMMU, but * the IOMMU API works on domains and devices. Bridge that gap by @@ -710,13 +1221,34 @@ { struct iommu_domain *domain = data; - return iommu_attach_device(domain, dev); + return __iommu_attach_device(domain, dev); +} + +static int __iommu_attach_group(struct iommu_domain *domain, + struct iommu_group *group) +{ + int ret; + + if (group->default_domain && group->domain != group->default_domain) + return -EBUSY; + + ret = __iommu_group_for_each_dev(group, domain, + iommu_group_do_attach_device); + if (ret == 0) + group->domain = domain; + + return ret; } int iommu_attach_group(struct iommu_domain *domain, struct iommu_group *group) { - return iommu_group_for_each_dev(group, domain, - iommu_group_do_attach_device); + int ret; + + mutex_lock(&group->mutex); + ret = __iommu_attach_group(domain, group); + mutex_unlock(&group->mutex); + + return ret; } EXPORT_SYMBOL_GPL(iommu_attach_group); @@ -724,14 +1256,40 @@ { struct iommu_domain *domain = data; - iommu_detach_device(domain, dev); + __iommu_detach_device(domain, dev); return 0; } +static void __iommu_detach_group(struct iommu_domain *domain, + struct iommu_group *group) +{ + int ret; + + if (!group->default_domain) { + __iommu_group_for_each_dev(group, domain, + iommu_group_do_detach_device); + group->domain = NULL; + return; + } + + if (group->domain == group->default_domain) + return; + + /* Detach by re-attaching to the default domain */ + ret = __iommu_group_for_each_dev(group, group->default_domain, + iommu_group_do_attach_device); + if (ret != 0) + WARN_ON(1); + else + group->domain = group->default_domain; +} + void iommu_detach_group(struct iommu_domain *domain, struct iommu_group *group) { - iommu_group_for_each_dev(group, domain, iommu_group_do_detach_device); + mutex_lock(&group->mutex); + __iommu_detach_group(domain, group); + mutex_unlock(&group->mutex); } EXPORT_SYMBOL_GPL(iommu_detach_group); @@ -744,15 +1302,37 @@ } EXPORT_SYMBOL_GPL(iommu_iova_to_phys); -int iommu_domain_has_cap(struct iommu_domain *domain, - unsigned long cap) +static size_t iommu_pgsize(struct iommu_domain *domain, + unsigned long addr_merge, size_t size) { - if (unlikely(domain->ops->domain_has_cap == NULL)) - return 0; + unsigned int pgsize_idx; + size_t pgsize; + + /* Max page size that still fits into 'size' */ + pgsize_idx = __fls(size); + + /* need to consider alignment requirements ? */ + if (likely(addr_merge)) { + /* Max page size allowed by address */ + unsigned int align_pgsize_idx = __ffs(addr_merge); + pgsize_idx = min(pgsize_idx, align_pgsize_idx); + } + + /* build a mask of acceptable page sizes */ + pgsize = (1UL << (pgsize_idx + 1)) - 1; + + /* throw away page sizes not supported by the hardware */ + pgsize &= domain->pgsize_bitmap; + + /* make sure we're still sane */ + BUG_ON(!pgsize); - return domain->ops->domain_has_cap(domain, cap); + /* pick the biggest page */ + pgsize_idx = __fls(pgsize); + pgsize = 1UL << pgsize_idx; + + return pgsize; } -EXPORT_SYMBOL_GPL(iommu_domain_has_cap); int iommu_map(struct iommu_domain *domain, unsigned long iova, phys_addr_t paddr, size_t size, int prot) @@ -760,14 +1340,18 @@ unsigned long orig_iova = iova; unsigned int min_pagesz; size_t orig_size = size; + phys_addr_t orig_paddr = paddr; int ret = 0; - if (unlikely(domain->ops->unmap == NULL || - domain->ops->pgsize_bitmap == 0UL)) + if (unlikely(domain->ops->map == NULL || + domain->pgsize_bitmap == 0UL)) return -ENODEV; + if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING))) + return -EINVAL; + /* find out the minimum page size supported */ - min_pagesz = 1 << __ffs(domain->ops->pgsize_bitmap); + min_pagesz = 1 << __ffs(domain->pgsize_bitmap); /* * both the virtual address and the physical one, as well as @@ -775,45 +1359,18 @@ * size of the smallest page supported by the hardware */ if (!IS_ALIGNED(iova | paddr | size, min_pagesz)) { - pr_err("unaligned: iova 0x%lx pa 0x%lx size 0x%lx min_pagesz " - "0x%x\n", iova, (unsigned long)paddr, - (unsigned long)size, min_pagesz); + pr_err("unaligned: iova 0x%lx pa %pa size 0x%zx min_pagesz 0x%x\n", + iova, &paddr, size, min_pagesz); return -EINVAL; } - pr_debug("map: iova 0x%lx pa 0x%lx size 0x%lx\n", iova, - (unsigned long)paddr, (unsigned long)size); + pr_debug("map: iova 0x%lx pa %pa size 0x%zx\n", iova, &paddr, size); while (size) { - unsigned long pgsize, addr_merge = iova | paddr; - unsigned int pgsize_idx; + size_t pgsize = iommu_pgsize(domain, iova | paddr, size); - /* Max page size that still fits into 'size' */ - pgsize_idx = __fls(size); - - /* need to consider alignment requirements ? */ - if (likely(addr_merge)) { - /* Max page size allowed by both iova and paddr */ - unsigned int align_pgsize_idx = __ffs(addr_merge); - - pgsize_idx = min(pgsize_idx, align_pgsize_idx); - } - - /* build a mask of acceptable page sizes */ - pgsize = (1UL << (pgsize_idx + 1)) - 1; - - /* throw away page sizes not supported by the hardware */ - pgsize &= domain->ops->pgsize_bitmap; - - /* make sure we're still sane */ - BUG_ON(!pgsize); - - /* pick the biggest page */ - pgsize_idx = __fls(pgsize); - pgsize = 1UL << pgsize_idx; - - pr_debug("mapping: iova 0x%lx pa 0x%lx pgsize %lu\n", iova, - (unsigned long)paddr, pgsize); + pr_debug("mapping: iova 0x%lx pa %pa pgsize 0x%zx\n", + iova, &paddr, pgsize); ret = domain->ops->map(domain, iova, paddr, pgsize, prot); if (ret) @@ -827,6 +1384,8 @@ /* unroll mapping in case something went wrong */ if (ret) iommu_unmap(domain, orig_iova, orig_size - size); + else + trace_map(domain, orig_iova, orig_paddr, orig_size, prot); return ret; } @@ -836,13 +1395,17 @@ { size_t unmapped_page, unmapped = 0; unsigned int min_pagesz; + unsigned long orig_iova = iova; if (unlikely(domain->ops->unmap == NULL || - domain->ops->pgsize_bitmap == 0UL)) + domain->pgsize_bitmap == 0UL)) return -ENODEV; + if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING))) + return -EINVAL; + /* find out the minimum page size supported */ - min_pagesz = 1 << __ffs(domain->ops->pgsize_bitmap); + min_pagesz = 1 << __ffs(domain->pgsize_bitmap); /* * The virtual address, as well as the size of the mapping, must be @@ -850,36 +1413,90 @@ * by the hardware */ if (!IS_ALIGNED(iova | size, min_pagesz)) { - pr_err("unaligned: iova 0x%lx size 0x%lx min_pagesz 0x%x\n", - iova, (unsigned long)size, min_pagesz); + pr_err("unaligned: iova 0x%lx size 0x%zx min_pagesz 0x%x\n", + iova, size, min_pagesz); return -EINVAL; } - pr_debug("unmap this: iova 0x%lx size 0x%lx\n", iova, - (unsigned long)size); + pr_debug("unmap this: iova 0x%lx size 0x%zx\n", iova, size); /* * Keep iterating until we either unmap 'size' bytes (or more) * or we hit an area that isn't mapped. */ while (unmapped < size) { - size_t left = size - unmapped; + size_t pgsize = iommu_pgsize(domain, iova, size - unmapped); - unmapped_page = domain->ops->unmap(domain, iova, left); + unmapped_page = domain->ops->unmap(domain, iova, pgsize); if (!unmapped_page) break; - pr_debug("unmapped: iova 0x%lx size %lx\n", iova, - (unsigned long)unmapped_page); + pr_debug("unmapped: iova 0x%lx size 0x%zx\n", + iova, unmapped_page); iova += unmapped_page; unmapped += unmapped_page; } + trace_unmap(domain, orig_iova, size, unmapped); return unmapped; } EXPORT_SYMBOL_GPL(iommu_unmap); +size_t iommu_map_sg(struct iommu_domain *domain, + unsigned long iova, struct scatterlist *sg, + unsigned int nents, int prot) +{ + size_t mapped; + + mapped = domain->ops->map_sg(domain, iova, sg, nents, prot); + trace_map_sg(domain, iova, mapped, prot); + return mapped; +} +EXPORT_SYMBOL(iommu_map_sg); + +size_t default_iommu_map_sg(struct iommu_domain *domain, unsigned long iova, + struct scatterlist *sg, unsigned int nents, int prot) +{ + struct scatterlist *s; + size_t mapped = 0; + unsigned int i, min_pagesz; + int ret; + + if (unlikely(domain->pgsize_bitmap == 0UL)) + return 0; + + min_pagesz = 1 << __ffs(domain->pgsize_bitmap); + + for_each_sg(sg, s, nents, i) { + phys_addr_t phys = page_to_phys(sg_page(s)) + s->offset; + + /* + * We are mapping on IOMMU page boundaries, so offset within + * the page must be 0. However, the IOMMU may support pages + * smaller than PAGE_SIZE, so s->offset may still represent + * an offset of that boundary within the CPU page. + */ + if (!IS_ALIGNED(s->offset, min_pagesz)) + goto out_err; + + ret = iommu_map(domain, iova + mapped, phys, s->length, prot); + if (ret) + goto out_err; + + mapped += s->length; + } + + return mapped; + +out_err: + /* undo mappings already done */ + iommu_unmap(domain, iova, mapped); + + return 0; + +} +EXPORT_SYMBOL_GPL(default_iommu_map_sg); int iommu_domain_window_enable(struct iommu_domain *domain, u32 wnd_nr, phys_addr_t paddr, u64 size, int prot) @@ -901,6 +1518,47 @@ } EXPORT_SYMBOL_GPL(iommu_domain_window_disable); +/** + * report_iommu_fault() - report about an IOMMU fault to the IOMMU framework + * @domain: the iommu domain where the fault has happened + * @dev: the device where the fault has happened + * @iova: the faulting address + * @flags: mmu fault flags (e.g. IOMMU_FAULT_READ/IOMMU_FAULT_WRITE/...) + * + * This function should be called by the low-level IOMMU implementations + * whenever IOMMU faults happen, to allow high-level users, that are + * interested in such events, to know about them. + * + * This event may be useful for several possible use cases: + * - mere logging of the event + * - dynamic TLB/PTE loading + * - if restarting of the faulting device is required + * + * Returns 0 on success and an appropriate error code otherwise (if dynamic + * PTE/TLB loading will one day be supported, implementations will be able + * to tell whether it succeeded or not according to this return value). + * + * Specifically, -ENOSYS is returned if a fault handler isn't installed + * (though fault handlers can also return -ENOSYS, in case they want to + * elicit the default behavior of the IOMMU drivers). + */ +int report_iommu_fault(struct iommu_domain *domain, + struct device *dev, unsigned long iova, int flags) +{ + int ret = -ENOSYS; + + /* + * if upper layers showed interest and installed a fault handler, + * invoke it. + */ + if (domain->handler) + ret = domain->handler(domain, dev, iova, flags, + domain->handler_token); + + trace_io_page_fault(dev, iova, flags); + return ret; +} + static int __init iommu_init(void) { iommu_group_kset = kset_create_and_add("iommu_groups", @@ -912,7 +1570,7 @@ return 0; } -arch_initcall(iommu_init); +core_initcall(iommu_init); int iommu_domain_get_attr(struct iommu_domain *domain, enum iommu_attr attr, void *data) @@ -930,7 +1588,7 @@ break; case DOMAIN_ATTR_PAGING: paging = data; - *paging = (domain->ops->pgsize_bitmap != 0UL); + *paging = (domain->pgsize_bitmap != 0UL); break; case DOMAIN_ATTR_WINDOWS: count = data; @@ -978,3 +1636,129 @@ return ret; } EXPORT_SYMBOL_GPL(iommu_domain_set_attr); + +void iommu_get_dm_regions(struct device *dev, struct list_head *list) +{ + const struct iommu_ops *ops = dev->bus->iommu_ops; + + if (ops && ops->get_dm_regions) + ops->get_dm_regions(dev, list); +} + +void iommu_put_dm_regions(struct device *dev, struct list_head *list) +{ + const struct iommu_ops *ops = dev->bus->iommu_ops; + + if (ops && ops->put_dm_regions) + ops->put_dm_regions(dev, list); +} + +/* Request that a device is direct mapped by the IOMMU */ +int iommu_request_dm_for_dev(struct device *dev) +{ + struct iommu_domain *dm_domain; + struct iommu_group *group; + int ret; + + /* Device must already be in a group before calling this function */ + group = iommu_group_get_for_dev(dev); + if (IS_ERR(group)) + return PTR_ERR(group); + + mutex_lock(&group->mutex); + + /* Check if the default domain is already direct mapped */ + ret = 0; + if (group->default_domain && + group->default_domain->type == IOMMU_DOMAIN_IDENTITY) + goto out; + + /* Don't change mappings of existing devices */ + ret = -EBUSY; + if (iommu_group_device_count(group) != 1) + goto out; + + /* Allocate a direct mapped domain */ + ret = -ENOMEM; + dm_domain = __iommu_domain_alloc(dev->bus, IOMMU_DOMAIN_IDENTITY); + if (!dm_domain) + goto out; + + /* Attach the device to the domain */ + ret = __iommu_attach_group(dm_domain, group); + if (ret) { + iommu_domain_free(dm_domain); + goto out; + } + + /* Make the direct mapped domain the default for this group */ + if (group->default_domain) + iommu_domain_free(group->default_domain); + group->default_domain = dm_domain; + + pr_info("Using direct mapping for device %s\n", dev_name(dev)); + + ret = 0; +out: + mutex_unlock(&group->mutex); + iommu_group_put(group); + + return ret; +} + +int iommu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode, + const struct iommu_ops *ops) +{ + struct iommu_fwspec *fwspec = dev->iommu_fwspec; + + if (fwspec) + return ops == fwspec->ops ? 0 : -EINVAL; + + fwspec = kzalloc(sizeof(*fwspec), GFP_KERNEL); + if (!fwspec) + return -ENOMEM; + + of_node_get(to_of_node(iommu_fwnode)); + fwspec->iommu_fwnode = iommu_fwnode; + fwspec->ops = ops; + dev->iommu_fwspec = fwspec; + return 0; +} +EXPORT_SYMBOL_GPL(iommu_fwspec_init); + +void iommu_fwspec_free(struct device *dev) +{ + struct iommu_fwspec *fwspec = dev->iommu_fwspec; + + if (fwspec) { + fwnode_handle_put(fwspec->iommu_fwnode); + kfree(fwspec); + dev->iommu_fwspec = NULL; + } +} +EXPORT_SYMBOL_GPL(iommu_fwspec_free); + +int iommu_fwspec_add_ids(struct device *dev, u32 *ids, int num_ids) +{ + struct iommu_fwspec *fwspec = dev->iommu_fwspec; + size_t size; + int i; + + if (!fwspec) + return -EINVAL; + + size = offsetof(struct iommu_fwspec, ids[fwspec->num_ids + num_ids]); + if (size > sizeof(*fwspec)) { + fwspec = krealloc(dev->iommu_fwspec, size, GFP_KERNEL); + if (!fwspec) + return -ENOMEM; + } + + for (i = 0; i < num_ids; i++) + fwspec->ids[fwspec->num_ids + i] = ids[i]; + + fwspec->num_ids += num_ids; + dev->iommu_fwspec = fwspec; + return 0; +} +EXPORT_SYMBOL_GPL(iommu_fwspec_add_ids);