vmd.c 28 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Volume Management Device driver
  4. * Copyright (c) 2015, Intel Corporation.
  5. */
  6. #include <linux/device.h>
  7. #include <linux/interrupt.h>
  8. #include <linux/irq.h>
  9. #include <linux/kernel.h>
  10. #include <linux/module.h>
  11. #include <linux/msi.h>
  12. #include <linux/pci.h>
  13. #include <linux/pci-acpi.h>
  14. #include <linux/pci-ecam.h>
  15. #include <linux/srcu.h>
  16. #include <linux/rculist.h>
  17. #include <linux/rcupdate.h>
  18. #include <asm/irqdomain.h>
  19. #define VMD_CFGBAR 0
  20. #define VMD_MEMBAR1 2
  21. #define VMD_MEMBAR2 4
  22. #define PCI_REG_VMCAP 0x40
  23. #define BUS_RESTRICT_CAP(vmcap) (vmcap & 0x1)
  24. #define PCI_REG_VMCONFIG 0x44
  25. #define BUS_RESTRICT_CFG(vmcfg) ((vmcfg >> 8) & 0x3)
  26. #define VMCONFIG_MSI_REMAP 0x2
  27. #define PCI_REG_VMLOCK 0x70
  28. #define MB2_SHADOW_EN(vmlock) (vmlock & 0x2)
  29. #define MB2_SHADOW_OFFSET 0x2000
  30. #define MB2_SHADOW_SIZE 16
  31. enum vmd_features {
  32. /*
  33. * Device may contain registers which hint the physical location of the
  34. * membars, in order to allow proper address translation during
  35. * resource assignment to enable guest virtualization
  36. */
  37. VMD_FEAT_HAS_MEMBAR_SHADOW = (1 << 0),
  38. /*
  39. * Device may provide root port configuration information which limits
  40. * bus numbering
  41. */
  42. VMD_FEAT_HAS_BUS_RESTRICTIONS = (1 << 1),
  43. /*
  44. * Device contains physical location shadow registers in
  45. * vendor-specific capability space
  46. */
  47. VMD_FEAT_HAS_MEMBAR_SHADOW_VSCAP = (1 << 2),
  48. /*
  49. * Device may use MSI-X vector 0 for software triggering and will not
  50. * be used for MSI remapping
  51. */
  52. VMD_FEAT_OFFSET_FIRST_VECTOR = (1 << 3),
  53. /*
  54. * Device can bypass remapping MSI-X transactions into its MSI-X table,
  55. * avoiding the requirement of a VMD MSI domain for child device
  56. * interrupt handling.
  57. */
  58. VMD_FEAT_CAN_BYPASS_MSI_REMAP = (1 << 4),
  59. };
  60. static DEFINE_IDA(vmd_instance_ida);
  61. /*
  62. * Lock for manipulating VMD IRQ lists.
  63. */
  64. static DEFINE_RAW_SPINLOCK(list_lock);
  65. /**
  66. * struct vmd_irq - private data to map driver IRQ to the VMD shared vector
  67. * @node: list item for parent traversal.
  68. * @irq: back pointer to parent.
  69. * @enabled: true if driver enabled IRQ
  70. * @virq: the virtual IRQ value provided to the requesting driver.
  71. *
  72. * Every MSI/MSI-X IRQ requested for a device in a VMD domain will be mapped to
  73. * a VMD IRQ using this structure.
  74. */
  75. struct vmd_irq {
  76. struct list_head node;
  77. struct vmd_irq_list *irq;
  78. bool enabled;
  79. unsigned int virq;
  80. };
  81. /**
  82. * struct vmd_irq_list - list of driver requested IRQs mapping to a VMD vector
  83. * @irq_list: the list of irq's the VMD one demuxes to.
  84. * @srcu: SRCU struct for local synchronization.
  85. * @count: number of child IRQs assigned to this vector; used to track
  86. * sharing.
  87. * @virq: The underlying VMD Linux interrupt number
  88. */
  89. struct vmd_irq_list {
  90. struct list_head irq_list;
  91. struct srcu_struct srcu;
  92. unsigned int count;
  93. unsigned int virq;
  94. };
  95. struct vmd_dev {
  96. struct pci_dev *dev;
  97. spinlock_t cfg_lock;
  98. void __iomem *cfgbar;
  99. int msix_count;
  100. struct vmd_irq_list *irqs;
  101. struct pci_sysdata sysdata;
  102. struct resource resources[3];
  103. struct irq_domain *irq_domain;
  104. struct pci_bus *bus;
  105. u8 busn_start;
  106. u8 first_vec;
  107. char *name;
  108. int instance;
  109. };
  110. static inline struct vmd_dev *vmd_from_bus(struct pci_bus *bus)
  111. {
  112. return container_of(bus->sysdata, struct vmd_dev, sysdata);
  113. }
  114. static inline unsigned int index_from_irqs(struct vmd_dev *vmd,
  115. struct vmd_irq_list *irqs)
  116. {
  117. return irqs - vmd->irqs;
  118. }
  119. /*
  120. * Drivers managing a device in a VMD domain allocate their own IRQs as before,
  121. * but the MSI entry for the hardware it's driving will be programmed with a
  122. * destination ID for the VMD MSI-X table. The VMD muxes interrupts in its
  123. * domain into one of its own, and the VMD driver de-muxes these for the
  124. * handlers sharing that VMD IRQ. The vmd irq_domain provides the operations
  125. * and irq_chip to set this up.
  126. */
  127. static void vmd_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
  128. {
  129. struct vmd_irq *vmdirq = data->chip_data;
  130. struct vmd_irq_list *irq = vmdirq->irq;
  131. struct vmd_dev *vmd = irq_data_get_irq_handler_data(data);
  132. memset(msg, 0, sizeof(*msg));
  133. msg->address_hi = X86_MSI_BASE_ADDRESS_HIGH;
  134. msg->arch_addr_lo.base_address = X86_MSI_BASE_ADDRESS_LOW;
  135. msg->arch_addr_lo.destid_0_7 = index_from_irqs(vmd, irq);
  136. }
  137. /*
  138. * We rely on MSI_FLAG_USE_DEF_CHIP_OPS to set the IRQ mask/unmask ops.
  139. */
  140. static void vmd_irq_enable(struct irq_data *data)
  141. {
  142. struct vmd_irq *vmdirq = data->chip_data;
  143. unsigned long flags;
  144. raw_spin_lock_irqsave(&list_lock, flags);
  145. WARN_ON(vmdirq->enabled);
  146. list_add_tail_rcu(&vmdirq->node, &vmdirq->irq->irq_list);
  147. vmdirq->enabled = true;
  148. raw_spin_unlock_irqrestore(&list_lock, flags);
  149. data->chip->irq_unmask(data);
  150. }
  151. static void vmd_irq_disable(struct irq_data *data)
  152. {
  153. struct vmd_irq *vmdirq = data->chip_data;
  154. unsigned long flags;
  155. data->chip->irq_mask(data);
  156. raw_spin_lock_irqsave(&list_lock, flags);
  157. if (vmdirq->enabled) {
  158. list_del_rcu(&vmdirq->node);
  159. vmdirq->enabled = false;
  160. }
  161. raw_spin_unlock_irqrestore(&list_lock, flags);
  162. }
  163. /*
  164. * XXX: Stubbed until we develop acceptable way to not create conflicts with
  165. * other devices sharing the same vector.
  166. */
  167. static int vmd_irq_set_affinity(struct irq_data *data,
  168. const struct cpumask *dest, bool force)
  169. {
  170. return -EINVAL;
  171. }
  172. static struct irq_chip vmd_msi_controller = {
  173. .name = "VMD-MSI",
  174. .irq_enable = vmd_irq_enable,
  175. .irq_disable = vmd_irq_disable,
  176. .irq_compose_msi_msg = vmd_compose_msi_msg,
  177. .irq_set_affinity = vmd_irq_set_affinity,
  178. };
  179. static irq_hw_number_t vmd_get_hwirq(struct msi_domain_info *info,
  180. msi_alloc_info_t *arg)
  181. {
  182. return 0;
  183. }
  184. /*
  185. * XXX: We can be even smarter selecting the best IRQ once we solve the
  186. * affinity problem.
  187. */
  188. static struct vmd_irq_list *vmd_next_irq(struct vmd_dev *vmd, struct msi_desc *desc)
  189. {
  190. unsigned long flags;
  191. int i, best;
  192. if (vmd->msix_count == 1 + vmd->first_vec)
  193. return &vmd->irqs[vmd->first_vec];
  194. /*
  195. * White list for fast-interrupt handlers. All others will share the
  196. * "slow" interrupt vector.
  197. */
  198. switch (msi_desc_to_pci_dev(desc)->class) {
  199. case PCI_CLASS_STORAGE_EXPRESS:
  200. break;
  201. default:
  202. return &vmd->irqs[vmd->first_vec];
  203. }
  204. raw_spin_lock_irqsave(&list_lock, flags);
  205. best = vmd->first_vec + 1;
  206. for (i = best; i < vmd->msix_count; i++)
  207. if (vmd->irqs[i].count < vmd->irqs[best].count)
  208. best = i;
  209. vmd->irqs[best].count++;
  210. raw_spin_unlock_irqrestore(&list_lock, flags);
  211. return &vmd->irqs[best];
  212. }
  213. static int vmd_msi_init(struct irq_domain *domain, struct msi_domain_info *info,
  214. unsigned int virq, irq_hw_number_t hwirq,
  215. msi_alloc_info_t *arg)
  216. {
  217. struct msi_desc *desc = arg->desc;
  218. struct vmd_dev *vmd = vmd_from_bus(msi_desc_to_pci_dev(desc)->bus);
  219. struct vmd_irq *vmdirq = kzalloc(sizeof(*vmdirq), GFP_KERNEL);
  220. if (!vmdirq)
  221. return -ENOMEM;
  222. INIT_LIST_HEAD(&vmdirq->node);
  223. vmdirq->irq = vmd_next_irq(vmd, desc);
  224. vmdirq->virq = virq;
  225. irq_domain_set_info(domain, virq, vmdirq->irq->virq, info->chip, vmdirq,
  226. handle_untracked_irq, vmd, NULL);
  227. return 0;
  228. }
  229. static void vmd_msi_free(struct irq_domain *domain,
  230. struct msi_domain_info *info, unsigned int virq)
  231. {
  232. struct vmd_irq *vmdirq = irq_get_chip_data(virq);
  233. unsigned long flags;
  234. synchronize_srcu(&vmdirq->irq->srcu);
  235. /* XXX: Potential optimization to rebalance */
  236. raw_spin_lock_irqsave(&list_lock, flags);
  237. vmdirq->irq->count--;
  238. raw_spin_unlock_irqrestore(&list_lock, flags);
  239. kfree(vmdirq);
  240. }
  241. static int vmd_msi_prepare(struct irq_domain *domain, struct device *dev,
  242. int nvec, msi_alloc_info_t *arg)
  243. {
  244. struct pci_dev *pdev = to_pci_dev(dev);
  245. struct vmd_dev *vmd = vmd_from_bus(pdev->bus);
  246. if (nvec > vmd->msix_count)
  247. return vmd->msix_count;
  248. memset(arg, 0, sizeof(*arg));
  249. return 0;
  250. }
  251. static void vmd_set_desc(msi_alloc_info_t *arg, struct msi_desc *desc)
  252. {
  253. arg->desc = desc;
  254. }
  255. static struct msi_domain_ops vmd_msi_domain_ops = {
  256. .get_hwirq = vmd_get_hwirq,
  257. .msi_init = vmd_msi_init,
  258. .msi_free = vmd_msi_free,
  259. .msi_prepare = vmd_msi_prepare,
  260. .set_desc = vmd_set_desc,
  261. };
  262. static struct msi_domain_info vmd_msi_domain_info = {
  263. .flags = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
  264. MSI_FLAG_PCI_MSIX,
  265. .ops = &vmd_msi_domain_ops,
  266. .chip = &vmd_msi_controller,
  267. };
  268. static void vmd_set_msi_remapping(struct vmd_dev *vmd, bool enable)
  269. {
  270. u16 reg;
  271. pci_read_config_word(vmd->dev, PCI_REG_VMCONFIG, &reg);
  272. reg = enable ? (reg & ~VMCONFIG_MSI_REMAP) :
  273. (reg | VMCONFIG_MSI_REMAP);
  274. pci_write_config_word(vmd->dev, PCI_REG_VMCONFIG, reg);
  275. }
  276. static int vmd_create_irq_domain(struct vmd_dev *vmd)
  277. {
  278. struct fwnode_handle *fn;
  279. fn = irq_domain_alloc_named_id_fwnode("VMD-MSI", vmd->sysdata.domain);
  280. if (!fn)
  281. return -ENODEV;
  282. vmd->irq_domain = pci_msi_create_irq_domain(fn, &vmd_msi_domain_info, NULL);
  283. if (!vmd->irq_domain) {
  284. irq_domain_free_fwnode(fn);
  285. return -ENODEV;
  286. }
  287. return 0;
  288. }
  289. static void vmd_remove_irq_domain(struct vmd_dev *vmd)
  290. {
  291. /*
  292. * Some production BIOS won't enable remapping between soft reboots.
  293. * Ensure remapping is restored before unloading the driver.
  294. */
  295. if (!vmd->msix_count)
  296. vmd_set_msi_remapping(vmd, true);
  297. if (vmd->irq_domain) {
  298. struct fwnode_handle *fn = vmd->irq_domain->fwnode;
  299. irq_domain_remove(vmd->irq_domain);
  300. irq_domain_free_fwnode(fn);
  301. }
  302. }
  303. static void __iomem *vmd_cfg_addr(struct vmd_dev *vmd, struct pci_bus *bus,
  304. unsigned int devfn, int reg, int len)
  305. {
  306. unsigned int busnr_ecam = bus->number - vmd->busn_start;
  307. u32 offset = PCIE_ECAM_OFFSET(busnr_ecam, devfn, reg);
  308. if (offset + len >= resource_size(&vmd->dev->resource[VMD_CFGBAR]))
  309. return NULL;
  310. return vmd->cfgbar + offset;
  311. }
  312. /*
  313. * CPU may deadlock if config space is not serialized on some versions of this
  314. * hardware, so all config space access is done under a spinlock.
  315. */
  316. static int vmd_pci_read(struct pci_bus *bus, unsigned int devfn, int reg,
  317. int len, u32 *value)
  318. {
  319. struct vmd_dev *vmd = vmd_from_bus(bus);
  320. void __iomem *addr = vmd_cfg_addr(vmd, bus, devfn, reg, len);
  321. unsigned long flags;
  322. int ret = 0;
  323. if (!addr)
  324. return -EFAULT;
  325. spin_lock_irqsave(&vmd->cfg_lock, flags);
  326. switch (len) {
  327. case 1:
  328. *value = readb(addr);
  329. break;
  330. case 2:
  331. *value = readw(addr);
  332. break;
  333. case 4:
  334. *value = readl(addr);
  335. break;
  336. default:
  337. ret = -EINVAL;
  338. break;
  339. }
  340. spin_unlock_irqrestore(&vmd->cfg_lock, flags);
  341. return ret;
  342. }
  343. /*
  344. * VMD h/w converts non-posted config writes to posted memory writes. The
  345. * read-back in this function forces the completion so it returns only after
  346. * the config space was written, as expected.
  347. */
  348. static int vmd_pci_write(struct pci_bus *bus, unsigned int devfn, int reg,
  349. int len, u32 value)
  350. {
  351. struct vmd_dev *vmd = vmd_from_bus(bus);
  352. void __iomem *addr = vmd_cfg_addr(vmd, bus, devfn, reg, len);
  353. unsigned long flags;
  354. int ret = 0;
  355. if (!addr)
  356. return -EFAULT;
  357. spin_lock_irqsave(&vmd->cfg_lock, flags);
  358. switch (len) {
  359. case 1:
  360. writeb(value, addr);
  361. readb(addr);
  362. break;
  363. case 2:
  364. writew(value, addr);
  365. readw(addr);
  366. break;
  367. case 4:
  368. writel(value, addr);
  369. readl(addr);
  370. break;
  371. default:
  372. ret = -EINVAL;
  373. break;
  374. }
  375. spin_unlock_irqrestore(&vmd->cfg_lock, flags);
  376. return ret;
  377. }
  378. static struct pci_ops vmd_ops = {
  379. .read = vmd_pci_read,
  380. .write = vmd_pci_write,
  381. };
  382. #ifdef CONFIG_ACPI
  383. static struct acpi_device *vmd_acpi_find_companion(struct pci_dev *pci_dev)
  384. {
  385. struct pci_host_bridge *bridge;
  386. u32 busnr, addr;
  387. if (pci_dev->bus->ops != &vmd_ops)
  388. return NULL;
  389. bridge = pci_find_host_bridge(pci_dev->bus);
  390. busnr = pci_dev->bus->number - bridge->bus->number;
  391. /*
  392. * The address computation below is only applicable to relative bus
  393. * numbers below 32.
  394. */
  395. if (busnr > 31)
  396. return NULL;
  397. addr = (busnr << 24) | ((u32)pci_dev->devfn << 16) | 0x8000FFFFU;
  398. dev_dbg(&pci_dev->dev, "Looking for ACPI companion (address 0x%x)\n",
  399. addr);
  400. return acpi_find_child_device(ACPI_COMPANION(bridge->dev.parent), addr,
  401. false);
  402. }
  403. static bool hook_installed;
  404. static void vmd_acpi_begin(void)
  405. {
  406. if (pci_acpi_set_companion_lookup_hook(vmd_acpi_find_companion))
  407. return;
  408. hook_installed = true;
  409. }
  410. static void vmd_acpi_end(void)
  411. {
  412. if (!hook_installed)
  413. return;
  414. pci_acpi_clear_companion_lookup_hook();
  415. hook_installed = false;
  416. }
  417. #else
  418. static inline void vmd_acpi_begin(void) { }
  419. static inline void vmd_acpi_end(void) { }
  420. #endif /* CONFIG_ACPI */
  421. static void vmd_domain_reset(struct vmd_dev *vmd)
  422. {
  423. u16 bus, max_buses = resource_size(&vmd->resources[0]);
  424. u8 dev, functions, fn, hdr_type;
  425. char __iomem *base;
  426. for (bus = 0; bus < max_buses; bus++) {
  427. for (dev = 0; dev < 32; dev++) {
  428. base = vmd->cfgbar + PCIE_ECAM_OFFSET(bus,
  429. PCI_DEVFN(dev, 0), 0);
  430. hdr_type = readb(base + PCI_HEADER_TYPE);
  431. functions = (hdr_type & 0x80) ? 8 : 1;
  432. for (fn = 0; fn < functions; fn++) {
  433. base = vmd->cfgbar + PCIE_ECAM_OFFSET(bus,
  434. PCI_DEVFN(dev, fn), 0);
  435. hdr_type = readb(base + PCI_HEADER_TYPE) &
  436. PCI_HEADER_TYPE_MASK;
  437. if (hdr_type != PCI_HEADER_TYPE_BRIDGE ||
  438. (readw(base + PCI_CLASS_DEVICE) !=
  439. PCI_CLASS_BRIDGE_PCI))
  440. continue;
  441. /*
  442. * Temporarily disable the I/O range before updating
  443. * PCI_IO_BASE.
  444. */
  445. writel(0x0000ffff, base + PCI_IO_BASE_UPPER16);
  446. /* Update lower 16 bits of I/O base/limit */
  447. writew(0x00f0, base + PCI_IO_BASE);
  448. /* Update upper 16 bits of I/O base/limit */
  449. writel(0, base + PCI_IO_BASE_UPPER16);
  450. /* MMIO Base/Limit */
  451. writel(0x0000fff0, base + PCI_MEMORY_BASE);
  452. /* Prefetchable MMIO Base/Limit */
  453. writel(0, base + PCI_PREF_LIMIT_UPPER32);
  454. writel(0x0000fff0, base + PCI_PREF_MEMORY_BASE);
  455. writel(0xffffffff, base + PCI_PREF_BASE_UPPER32);
  456. }
  457. }
  458. }
  459. }
  460. static void vmd_attach_resources(struct vmd_dev *vmd)
  461. {
  462. vmd->dev->resource[VMD_MEMBAR1].child = &vmd->resources[1];
  463. vmd->dev->resource[VMD_MEMBAR2].child = &vmd->resources[2];
  464. }
  465. static void vmd_detach_resources(struct vmd_dev *vmd)
  466. {
  467. vmd->dev->resource[VMD_MEMBAR1].child = NULL;
  468. vmd->dev->resource[VMD_MEMBAR2].child = NULL;
  469. }
  470. /*
  471. * VMD domains start at 0x10000 to not clash with ACPI _SEG domains.
  472. * Per ACPI r6.0, sec 6.5.6, _SEG returns an integer, of which the lower
  473. * 16 bits are the PCI Segment Group (domain) number. Other bits are
  474. * currently reserved.
  475. */
  476. static int vmd_find_free_domain(void)
  477. {
  478. int domain = 0xffff;
  479. struct pci_bus *bus = NULL;
  480. while ((bus = pci_find_next_bus(bus)) != NULL)
  481. domain = max_t(int, domain, pci_domain_nr(bus));
  482. return domain + 1;
  483. }
  484. static int vmd_get_phys_offsets(struct vmd_dev *vmd, bool native_hint,
  485. resource_size_t *offset1,
  486. resource_size_t *offset2)
  487. {
  488. struct pci_dev *dev = vmd->dev;
  489. u64 phys1, phys2;
  490. if (native_hint) {
  491. u32 vmlock;
  492. int ret;
  493. ret = pci_read_config_dword(dev, PCI_REG_VMLOCK, &vmlock);
  494. if (ret || PCI_POSSIBLE_ERROR(vmlock))
  495. return -ENODEV;
  496. if (MB2_SHADOW_EN(vmlock)) {
  497. void __iomem *membar2;
  498. membar2 = pci_iomap(dev, VMD_MEMBAR2, 0);
  499. if (!membar2)
  500. return -ENOMEM;
  501. phys1 = readq(membar2 + MB2_SHADOW_OFFSET);
  502. phys2 = readq(membar2 + MB2_SHADOW_OFFSET + 8);
  503. pci_iounmap(dev, membar2);
  504. } else
  505. return 0;
  506. } else {
  507. /* Hypervisor-Emulated Vendor-Specific Capability */
  508. int pos = pci_find_capability(dev, PCI_CAP_ID_VNDR);
  509. u32 reg, regu;
  510. pci_read_config_dword(dev, pos + 4, &reg);
  511. /* "SHDW" */
  512. if (pos && reg == 0x53484457) {
  513. pci_read_config_dword(dev, pos + 8, &reg);
  514. pci_read_config_dword(dev, pos + 12, &regu);
  515. phys1 = (u64) regu << 32 | reg;
  516. pci_read_config_dword(dev, pos + 16, &reg);
  517. pci_read_config_dword(dev, pos + 20, &regu);
  518. phys2 = (u64) regu << 32 | reg;
  519. } else
  520. return 0;
  521. }
  522. *offset1 = dev->resource[VMD_MEMBAR1].start -
  523. (phys1 & PCI_BASE_ADDRESS_MEM_MASK);
  524. *offset2 = dev->resource[VMD_MEMBAR2].start -
  525. (phys2 & PCI_BASE_ADDRESS_MEM_MASK);
  526. return 0;
  527. }
  528. static int vmd_get_bus_number_start(struct vmd_dev *vmd)
  529. {
  530. struct pci_dev *dev = vmd->dev;
  531. u16 reg;
  532. pci_read_config_word(dev, PCI_REG_VMCAP, &reg);
  533. if (BUS_RESTRICT_CAP(reg)) {
  534. pci_read_config_word(dev, PCI_REG_VMCONFIG, &reg);
  535. switch (BUS_RESTRICT_CFG(reg)) {
  536. case 0:
  537. vmd->busn_start = 0;
  538. break;
  539. case 1:
  540. vmd->busn_start = 128;
  541. break;
  542. case 2:
  543. vmd->busn_start = 224;
  544. break;
  545. default:
  546. pci_err(dev, "Unknown Bus Offset Setting (%d)\n",
  547. BUS_RESTRICT_CFG(reg));
  548. return -ENODEV;
  549. }
  550. }
  551. return 0;
  552. }
  553. static irqreturn_t vmd_irq(int irq, void *data)
  554. {
  555. struct vmd_irq_list *irqs = data;
  556. struct vmd_irq *vmdirq;
  557. int idx;
  558. idx = srcu_read_lock(&irqs->srcu);
  559. list_for_each_entry_rcu(vmdirq, &irqs->irq_list, node)
  560. generic_handle_irq(vmdirq->virq);
  561. srcu_read_unlock(&irqs->srcu, idx);
  562. return IRQ_HANDLED;
  563. }
  564. static int vmd_alloc_irqs(struct vmd_dev *vmd)
  565. {
  566. struct pci_dev *dev = vmd->dev;
  567. int i, err;
  568. vmd->msix_count = pci_msix_vec_count(dev);
  569. if (vmd->msix_count < 0)
  570. return -ENODEV;
  571. vmd->msix_count = pci_alloc_irq_vectors(dev, vmd->first_vec + 1,
  572. vmd->msix_count, PCI_IRQ_MSIX);
  573. if (vmd->msix_count < 0)
  574. return vmd->msix_count;
  575. vmd->irqs = devm_kcalloc(&dev->dev, vmd->msix_count, sizeof(*vmd->irqs),
  576. GFP_KERNEL);
  577. if (!vmd->irqs)
  578. return -ENOMEM;
  579. for (i = 0; i < vmd->msix_count; i++) {
  580. err = init_srcu_struct(&vmd->irqs[i].srcu);
  581. if (err)
  582. return err;
  583. INIT_LIST_HEAD(&vmd->irqs[i].irq_list);
  584. vmd->irqs[i].virq = pci_irq_vector(dev, i);
  585. err = devm_request_irq(&dev->dev, vmd->irqs[i].virq,
  586. vmd_irq, IRQF_NO_THREAD,
  587. vmd->name, &vmd->irqs[i]);
  588. if (err)
  589. return err;
  590. }
  591. return 0;
  592. }
  593. /*
  594. * Since VMD is an aperture to regular PCIe root ports, only allow it to
  595. * control features that the OS is allowed to control on the physical PCI bus.
  596. */
  597. static void vmd_copy_host_bridge_flags(struct pci_host_bridge *root_bridge,
  598. struct pci_host_bridge *vmd_bridge)
  599. {
  600. vmd_bridge->native_pcie_hotplug = root_bridge->native_pcie_hotplug;
  601. vmd_bridge->native_shpc_hotplug = root_bridge->native_shpc_hotplug;
  602. vmd_bridge->native_aer = root_bridge->native_aer;
  603. vmd_bridge->native_pme = root_bridge->native_pme;
  604. vmd_bridge->native_ltr = root_bridge->native_ltr;
  605. vmd_bridge->native_dpc = root_bridge->native_dpc;
  606. }
  607. static int vmd_enable_domain(struct vmd_dev *vmd, unsigned long features)
  608. {
  609. struct pci_sysdata *sd = &vmd->sysdata;
  610. struct resource *res;
  611. u32 upper_bits;
  612. unsigned long flags;
  613. LIST_HEAD(resources);
  614. resource_size_t offset[2] = {0};
  615. resource_size_t membar2_offset = 0x2000;
  616. struct pci_bus *child;
  617. struct pci_dev *dev;
  618. int ret;
  619. /*
  620. * Shadow registers may exist in certain VMD device ids which allow
  621. * guests to correctly assign host physical addresses to the root ports
  622. * and child devices. These registers will either return the host value
  623. * or 0, depending on an enable bit in the VMD device.
  624. */
  625. if (features & VMD_FEAT_HAS_MEMBAR_SHADOW) {
  626. membar2_offset = MB2_SHADOW_OFFSET + MB2_SHADOW_SIZE;
  627. ret = vmd_get_phys_offsets(vmd, true, &offset[0], &offset[1]);
  628. if (ret)
  629. return ret;
  630. } else if (features & VMD_FEAT_HAS_MEMBAR_SHADOW_VSCAP) {
  631. ret = vmd_get_phys_offsets(vmd, false, &offset[0], &offset[1]);
  632. if (ret)
  633. return ret;
  634. }
  635. /*
  636. * Certain VMD devices may have a root port configuration option which
  637. * limits the bus range to between 0-127, 128-255, or 224-255
  638. */
  639. if (features & VMD_FEAT_HAS_BUS_RESTRICTIONS) {
  640. ret = vmd_get_bus_number_start(vmd);
  641. if (ret)
  642. return ret;
  643. }
  644. res = &vmd->dev->resource[VMD_CFGBAR];
  645. vmd->resources[0] = (struct resource) {
  646. .name = "VMD CFGBAR",
  647. .start = vmd->busn_start,
  648. .end = vmd->busn_start + (resource_size(res) >> 20) - 1,
  649. .flags = IORESOURCE_BUS | IORESOURCE_PCI_FIXED,
  650. };
  651. /*
  652. * If the window is below 4GB, clear IORESOURCE_MEM_64 so we can
  653. * put 32-bit resources in the window.
  654. *
  655. * There's no hardware reason why a 64-bit window *couldn't*
  656. * contain a 32-bit resource, but pbus_size_mem() computes the
  657. * bridge window size assuming a 64-bit window will contain no
  658. * 32-bit resources. __pci_assign_resource() enforces that
  659. * artificial restriction to make sure everything will fit.
  660. *
  661. * The only way we could use a 64-bit non-prefetchable MEMBAR is
  662. * if its address is <4GB so that we can convert it to a 32-bit
  663. * resource. To be visible to the host OS, all VMD endpoints must
  664. * be initially configured by platform BIOS, which includes setting
  665. * up these resources. We can assume the device is configured
  666. * according to the platform needs.
  667. */
  668. res = &vmd->dev->resource[VMD_MEMBAR1];
  669. upper_bits = upper_32_bits(res->end);
  670. flags = res->flags & ~IORESOURCE_SIZEALIGN;
  671. if (!upper_bits)
  672. flags &= ~IORESOURCE_MEM_64;
  673. vmd->resources[1] = (struct resource) {
  674. .name = "VMD MEMBAR1",
  675. .start = res->start,
  676. .end = res->end,
  677. .flags = flags,
  678. .parent = res,
  679. };
  680. res = &vmd->dev->resource[VMD_MEMBAR2];
  681. upper_bits = upper_32_bits(res->end);
  682. flags = res->flags & ~IORESOURCE_SIZEALIGN;
  683. if (!upper_bits)
  684. flags &= ~IORESOURCE_MEM_64;
  685. vmd->resources[2] = (struct resource) {
  686. .name = "VMD MEMBAR2",
  687. .start = res->start + membar2_offset,
  688. .end = res->end,
  689. .flags = flags,
  690. .parent = res,
  691. };
  692. sd->vmd_dev = vmd->dev;
  693. sd->domain = vmd_find_free_domain();
  694. if (sd->domain < 0)
  695. return sd->domain;
  696. sd->node = pcibus_to_node(vmd->dev->bus);
  697. /*
  698. * Currently MSI remapping must be enabled in guest passthrough mode
  699. * due to some missing interrupt remapping plumbing. This is probably
  700. * acceptable because the guest is usually CPU-limited and MSI
  701. * remapping doesn't become a performance bottleneck.
  702. */
  703. if (!(features & VMD_FEAT_CAN_BYPASS_MSI_REMAP) ||
  704. offset[0] || offset[1]) {
  705. ret = vmd_alloc_irqs(vmd);
  706. if (ret)
  707. return ret;
  708. vmd_set_msi_remapping(vmd, true);
  709. ret = vmd_create_irq_domain(vmd);
  710. if (ret)
  711. return ret;
  712. /*
  713. * Override the IRQ domain bus token so the domain can be
  714. * distinguished from a regular PCI/MSI domain.
  715. */
  716. irq_domain_update_bus_token(vmd->irq_domain, DOMAIN_BUS_VMD_MSI);
  717. } else {
  718. vmd_set_msi_remapping(vmd, false);
  719. }
  720. pci_add_resource(&resources, &vmd->resources[0]);
  721. pci_add_resource_offset(&resources, &vmd->resources[1], offset[0]);
  722. pci_add_resource_offset(&resources, &vmd->resources[2], offset[1]);
  723. vmd->bus = pci_create_root_bus(&vmd->dev->dev, vmd->busn_start,
  724. &vmd_ops, sd, &resources);
  725. if (!vmd->bus) {
  726. pci_free_resource_list(&resources);
  727. vmd_remove_irq_domain(vmd);
  728. return -ENODEV;
  729. }
  730. vmd_copy_host_bridge_flags(pci_find_host_bridge(vmd->dev->bus),
  731. to_pci_host_bridge(vmd->bus->bridge));
  732. vmd_attach_resources(vmd);
  733. if (vmd->irq_domain)
  734. dev_set_msi_domain(&vmd->bus->dev, vmd->irq_domain);
  735. else
  736. dev_set_msi_domain(&vmd->bus->dev,
  737. dev_get_msi_domain(&vmd->dev->dev));
  738. vmd_acpi_begin();
  739. pci_scan_child_bus(vmd->bus);
  740. vmd_domain_reset(vmd);
  741. /* When Intel VMD is enabled, the OS does not discover the Root Ports
  742. * owned by Intel VMD within the MMCFG space. pci_reset_bus() applies
  743. * a reset to the parent of the PCI device supplied as argument. This
  744. * is why we pass a child device, so the reset can be triggered at
  745. * the Intel bridge level and propagated to all the children in the
  746. * hierarchy.
  747. */
  748. list_for_each_entry(child, &vmd->bus->children, node) {
  749. if (!list_empty(&child->devices)) {
  750. dev = list_first_entry(&child->devices,
  751. struct pci_dev, bus_list);
  752. ret = pci_reset_bus(dev);
  753. if (ret)
  754. pci_warn(dev, "can't reset device: %d\n", ret);
  755. break;
  756. }
  757. }
  758. pci_assign_unassigned_bus_resources(vmd->bus);
  759. /*
  760. * VMD root buses are virtual and don't return true on pci_is_pcie()
  761. * and will fail pcie_bus_configure_settings() early. It can instead be
  762. * run on each of the real root ports.
  763. */
  764. list_for_each_entry(child, &vmd->bus->children, node)
  765. pcie_bus_configure_settings(child);
  766. pci_bus_add_devices(vmd->bus);
  767. vmd_acpi_end();
  768. WARN(sysfs_create_link(&vmd->dev->dev.kobj, &vmd->bus->dev.kobj,
  769. "domain"), "Can't create symlink to domain\n");
  770. return 0;
  771. }
  772. static int vmd_probe(struct pci_dev *dev, const struct pci_device_id *id)
  773. {
  774. unsigned long features = (unsigned long) id->driver_data;
  775. struct vmd_dev *vmd;
  776. int err;
  777. if (resource_size(&dev->resource[VMD_CFGBAR]) < (1 << 20))
  778. return -ENOMEM;
  779. vmd = devm_kzalloc(&dev->dev, sizeof(*vmd), GFP_KERNEL);
  780. if (!vmd)
  781. return -ENOMEM;
  782. vmd->dev = dev;
  783. vmd->instance = ida_simple_get(&vmd_instance_ida, 0, 0, GFP_KERNEL);
  784. if (vmd->instance < 0)
  785. return vmd->instance;
  786. vmd->name = devm_kasprintf(&dev->dev, GFP_KERNEL, "vmd%d",
  787. vmd->instance);
  788. if (!vmd->name) {
  789. err = -ENOMEM;
  790. goto out_release_instance;
  791. }
  792. err = pcim_enable_device(dev);
  793. if (err < 0)
  794. goto out_release_instance;
  795. vmd->cfgbar = pcim_iomap(dev, VMD_CFGBAR, 0);
  796. if (!vmd->cfgbar) {
  797. err = -ENOMEM;
  798. goto out_release_instance;
  799. }
  800. pci_set_master(dev);
  801. if (dma_set_mask_and_coherent(&dev->dev, DMA_BIT_MASK(64)) &&
  802. dma_set_mask_and_coherent(&dev->dev, DMA_BIT_MASK(32))) {
  803. err = -ENODEV;
  804. goto out_release_instance;
  805. }
  806. if (features & VMD_FEAT_OFFSET_FIRST_VECTOR)
  807. vmd->first_vec = 1;
  808. spin_lock_init(&vmd->cfg_lock);
  809. pci_set_drvdata(dev, vmd);
  810. err = vmd_enable_domain(vmd, features);
  811. if (err)
  812. goto out_release_instance;
  813. dev_info(&vmd->dev->dev, "Bound to PCI domain %04x\n",
  814. vmd->sysdata.domain);
  815. return 0;
  816. out_release_instance:
  817. ida_simple_remove(&vmd_instance_ida, vmd->instance);
  818. return err;
  819. }
  820. static void vmd_cleanup_srcu(struct vmd_dev *vmd)
  821. {
  822. int i;
  823. for (i = 0; i < vmd->msix_count; i++)
  824. cleanup_srcu_struct(&vmd->irqs[i].srcu);
  825. }
  826. static void vmd_remove(struct pci_dev *dev)
  827. {
  828. struct vmd_dev *vmd = pci_get_drvdata(dev);
  829. sysfs_remove_link(&vmd->dev->dev.kobj, "domain");
  830. pci_stop_root_bus(vmd->bus);
  831. pci_remove_root_bus(vmd->bus);
  832. vmd_cleanup_srcu(vmd);
  833. vmd_detach_resources(vmd);
  834. vmd_remove_irq_domain(vmd);
  835. ida_simple_remove(&vmd_instance_ida, vmd->instance);
  836. }
  837. static void vmd_shutdown(struct pci_dev *dev)
  838. {
  839. struct vmd_dev *vmd = pci_get_drvdata(dev);
  840. vmd_remove_irq_domain(vmd);
  841. }
  842. #ifdef CONFIG_PM_SLEEP
  843. static int vmd_suspend(struct device *dev)
  844. {
  845. struct pci_dev *pdev = to_pci_dev(dev);
  846. struct vmd_dev *vmd = pci_get_drvdata(pdev);
  847. int i;
  848. for (i = 0; i < vmd->msix_count; i++)
  849. devm_free_irq(dev, vmd->irqs[i].virq, &vmd->irqs[i]);
  850. return 0;
  851. }
  852. static int vmd_resume(struct device *dev)
  853. {
  854. struct pci_dev *pdev = to_pci_dev(dev);
  855. struct vmd_dev *vmd = pci_get_drvdata(pdev);
  856. int err, i;
  857. if (vmd->irq_domain)
  858. vmd_set_msi_remapping(vmd, true);
  859. else
  860. vmd_set_msi_remapping(vmd, false);
  861. for (i = 0; i < vmd->msix_count; i++) {
  862. err = devm_request_irq(dev, vmd->irqs[i].virq,
  863. vmd_irq, IRQF_NO_THREAD,
  864. vmd->name, &vmd->irqs[i]);
  865. if (err)
  866. return err;
  867. }
  868. return 0;
  869. }
  870. #endif
  871. static SIMPLE_DEV_PM_OPS(vmd_dev_pm_ops, vmd_suspend, vmd_resume);
  872. static const struct pci_device_id vmd_ids[] = {
  873. {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_VMD_201D),
  874. .driver_data = VMD_FEAT_HAS_MEMBAR_SHADOW_VSCAP,},
  875. {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_VMD_28C0),
  876. .driver_data = VMD_FEAT_HAS_MEMBAR_SHADOW |
  877. VMD_FEAT_HAS_BUS_RESTRICTIONS |
  878. VMD_FEAT_CAN_BYPASS_MSI_REMAP,},
  879. {PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x467f),
  880. .driver_data = VMD_FEAT_HAS_MEMBAR_SHADOW_VSCAP |
  881. VMD_FEAT_HAS_BUS_RESTRICTIONS |
  882. VMD_FEAT_OFFSET_FIRST_VECTOR,},
  883. {PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x4c3d),
  884. .driver_data = VMD_FEAT_HAS_MEMBAR_SHADOW_VSCAP |
  885. VMD_FEAT_HAS_BUS_RESTRICTIONS |
  886. VMD_FEAT_OFFSET_FIRST_VECTOR,},
  887. {PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xa77f),
  888. .driver_data = VMD_FEAT_HAS_MEMBAR_SHADOW_VSCAP |
  889. VMD_FEAT_HAS_BUS_RESTRICTIONS |
  890. VMD_FEAT_OFFSET_FIRST_VECTOR,},
  891. {PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7d0b),
  892. .driver_data = VMD_FEAT_HAS_MEMBAR_SHADOW_VSCAP |
  893. VMD_FEAT_HAS_BUS_RESTRICTIONS |
  894. VMD_FEAT_OFFSET_FIRST_VECTOR,},
  895. {PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xad0b),
  896. .driver_data = VMD_FEAT_HAS_MEMBAR_SHADOW_VSCAP |
  897. VMD_FEAT_HAS_BUS_RESTRICTIONS |
  898. VMD_FEAT_OFFSET_FIRST_VECTOR,},
  899. {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_VMD_9A0B),
  900. .driver_data = VMD_FEAT_HAS_MEMBAR_SHADOW_VSCAP |
  901. VMD_FEAT_HAS_BUS_RESTRICTIONS |
  902. VMD_FEAT_OFFSET_FIRST_VECTOR,},
  903. {0,}
  904. };
  905. MODULE_DEVICE_TABLE(pci, vmd_ids);
  906. static struct pci_driver vmd_drv = {
  907. .name = "vmd",
  908. .id_table = vmd_ids,
  909. .probe = vmd_probe,
  910. .remove = vmd_remove,
  911. .shutdown = vmd_shutdown,
  912. .driver = {
  913. .pm = &vmd_dev_pm_ops,
  914. },
  915. };
  916. module_pci_driver(vmd_drv);
  917. MODULE_AUTHOR("Intel Corporation");
  918. MODULE_LICENSE("GPL v2");
  919. MODULE_VERSION("0.6");