1. 前后端数据共享 #

/* The Guest publishes the used index for which it expects an interrupt
 * at the end of the avail ring. Host should ignore the avail->flags field. */
/* The Host publishes the avail index for which it expects a kick
 * at the end of the used ring. Guest should ignore the used->flags field. */
 
struct virtq_used { 
#define VIRTQ_USED_F_NO_NOTIFY  1 
        le16 flags; 
        le16 idx; 
        struct virtq_used_elem ring[ /* Queue Size */]; 
        le16 avail_event; /* Only if VIRTIO_F_EVENT_IDX */ 
}; 
 
/* le32 is used here for ids for padding reasons. */ 
struct virtq_used_elem { 
        /* Index of start of used descriptor chain. */ 
        le32 id; 
        /* Total length of the descriptor chain which was used (written to) */ 
        le32 len; 
};

前端驱动通知后端：
内核流程mark一下，PCI设备驱动流程这个后面可以学习一下，先扫描PCI bus发现是virtio设备再扫描virtio-bus。
worker_thread --> process_one_work --> pciehp_power_thread --> pciehp_enable_slot --> 
pciehp_configure_device --> pci_bus_add_devices --> pci_bus_add_device --> device_attach --> 
__device_attach --> bus_for_each_drv --> __device_attach_driver --> driver_probe_device --> 
pci_device_probe --> local_pci_probe --> virtio_pci_probe --> register_virtio_device --> 
device_register --> device_add --> bus_probe_device --> device_initial_probe 
--> __device_attach --> bus_for_each_drv --> __device_attach_driver -->
driver_probe_device --> virtio_dev_probe --> virtnet_probe (网卡设备驱动加载的入口)

static int virtnet_probe(struct virtio_device *vdev) 
{
    ......
    virtio_device_ready(vdev);
}

/**
 * virtio_device_ready - enable vq use in probe function
 * @vdev: the device
 *
 * Driver must call this to use vqs in the probe function.
 *
 * Note: vqs are enabled automatically after probe returns.
 */
static inline
void virtio_device_ready(struct virtio_device *dev)
{
        unsigned status = dev->config->get_status(dev);

        BUG_ON(status & VIRTIO_CONFIG_S_DRIVER_OK);
        dev->config->set_status(dev, status | VIRTIO_CONFIG_S_DRIVER_OK);
}

# QEMU/KVM后端的处理流程如下：
# 前端驱动写Status位，val & VIRTIO_CONFIG_S_DRIVER_OK，这时候前端驱动已经ready
virtio_pci_config_write  --> virtio_ioport_write --> virtio_pci_start_ioeventfd
--> virtio_bus_set_host_notifier --> virtio_bus_start_ioeventfd --> virtio_device_start_ioeventfd_impl
--> virtio_bus_set_host_notifier
    --> virtio_pci_ioeventfd_assign
        --> memory_region_add_eventfd
            --> memory_region_transaction_commit
              --> address_space_update_ioeventfds
                --> address_space_add_del_ioeventfds
                  --> kvm_io_ioeventfd_add/vhost_eventfd_add
                    --> kvm_set_ioeventfd_pio
                      --> kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &kick)

#kvm内核代码virt/kvm/eventfd.c中
kvm_vm_ioctl(KVM_IOEVENTFD)
  --> kvm_ioeventfd
    --> kvm_assign_ioeventfd
      --> kvm_assign_ioeventfd_idx

# MMIO处理流程中（handle_ept_misconfig）最后会调用到ioeventfd_write通知QEMU。
/* MMIO/PIO writes trigger an event if the addr/val match */
static int
ioeventfd_write(struct kvm_vcpu *vcpu, struct kvm_io_device *this, gpa_t addr,
                int len, const void *val)
{
        struct _ioeventfd *p = to_ioeventfd(this);

        if (!ioeventfd_in_range(p, addr, len, val))
                return -EOPNOTSUPP;

        eventfd_signal(p->eventfd, 1);
        return 0;
}

virtnet_probe
  --> init_vqs
    --> virtnet_find_vqs
      --> vi->vdev->config->find_vqs [vp_modern_find_vqs]
        --> vp_find_vqs
          --> vp_find_vqs_msix // 为每virtqueue申请一个MSIx中断，通常收发各一个队列
            --> vp_request_msix_vectors // 主要的MSIx中断申请逻辑都在这个函数里面
              --> pci_alloc_irq_vectors_affinity // 申请MSIx中断描述符(__pci_enable_msix_range)
                --> request_irq  // 注册中断处理函数
               
	       // virtio-net网卡至少申请了3个MSIx中断：
                // 一个是configuration change中断（配置空间发生变化后，QEMU通知前端）
                // 发送队列1个MSIx中断，接收队列1MSIx中断

virtio_pci_config_write
  --> virtio_ioport_write
    --> virtio_set_status
      --> virtio_net_vhost_status
        --> vhost_net_start
          --> virtio_pci_set_guest_notifiers
            --> kvm_virtio_pci_vector_use 
              |--> kvm_irqchip_add_msi_route //更新中断路由表
              |--> kvm_virtio_pci_irqfd_use  //使能MSI中断
                 --> kvm_irqchip_add_irqfd_notifier_gsi
                   --> kvm_irqchip_assign_irqfd
                  
# 申请MSIx中断的时候，会为MSIx分配一个gsi，并为这个gsi绑定一个irqfd，然后调用ioctl KVM_IRQFD注册到内核中。               
static int kvm_irqchip_assign_irqfd(KVMState *s, int fd, int rfd, int virq,
                                    bool assign)
{
    struct kvm_irqfd irqfd = {
        .fd = fd,
        .gsi = virq,
        .flags = assign ? 0 : KVM_IRQFD_FLAG_DEASSIGN,
    };

    if (rfd != -1) {
        irqfd.flags |= KVM_IRQFD_FLAG_RESAMPLE;
        irqfd.resamplefd = rfd;
    }

    if (!kvm_irqfds_enabled()) {
        return -ENOSYS;
    }

    return kvm_vm_ioctl(s, KVM_IRQFD, &irqfd);
}

# KVM内核代码virt/kvm/eventfd.c
kvm_vm_ioctl(s, KVM_IRQFD, &irqfd)
  --> kvm_irqfd_assign
    --> vfs_poll(f.file, &irqfd->pt) // 在内核中poll这个irqfd

#0  0x00005604798d569b in virtio_notify_irqfd (vdev=0x56047d12d670, vq=0x7fab10006110) at  hw/virtio/virtio.c:1684
#1  0x00005604798adea4 in virtio_scsi_complete_req (req=0x56047d09fa70) at  hw/scsi/virtio-scsi.c:76
#2  0x00005604798aecfb in virtio_scsi_complete_cmd_req (req=0x56047d09fa70) at  hw/scsi/virtio-scsi.c:468
#3  0x00005604798aee9d in virtio_scsi_command_complete (r=0x56047ccb0be0, status=0, resid=0) at  hw/scsi/virtio-scsi.c:495
#4  0x0000560479b397cf in scsi_req_complete (req=0x56047ccb0be0, status=0) at hw/scsi/scsi-bus.c:1404
#5  0x0000560479b2b503 in scsi_dma_complete_noio (r=0x56047ccb0be0, ret=0) at hw/scsi/scsi-disk.c:279
#6  0x0000560479b2b610 in scsi_dma_complete (opaque=0x56047ccb0be0, ret=0) at hw/scsi/scsi-disk.c:300
#7  0x00005604799b89e3 in dma_complete (dbs=0x56047c6e9ab0, ret=0) at dma-helpers.c:118
#8  0x00005604799b8a90 in dma_blk_cb (opaque=0x56047c6e9ab0, ret=0) at dma-helpers.c:136
#9  0x0000560479cf5220 in blk_aio_complete (acb=0x56047cd77d40) at block/block-backend.c:1327
#10 0x0000560479cf5470 in blk_aio_read_entry (opaque=0x56047cd77d40) at block/block-backend.c:1387
#11 0x0000560479df49c4 in coroutine_trampoline (i0=2095821104, i1=22020) at util/coroutine-ucontext.c:115
#12 0x00007fab214d82c0 in __start_context () at /usr/lib64/libc.so.6

void virtio_notify_irqfd(VirtIODevice *vdev, VirtQueue *vq)
{
    ...
     /*
     * virtio spec 1.0 says ISR bit 0 should be ignored with MSI, but
     * windows drivers included in virtio-win 1.8.0 (circa 2015) are
     * incorrectly polling this bit during crashdump and hibernation
     * in MSI mode, causing a hang if this bit is never updated.
     * Recent releases of Windows do not really shut down, but rather
     * log out and hibernate to make the next startup faster.  Hence,
     * this manifested as a more serious hang during shutdown with
     *
     * Next driver release from 2016 fixed this problem, so working around it
     * is not a must, but it's easy to do so let's do it here.
     *
     * Note: it's safe to update ISR from any thread as it was switched
     * to an atomic operation.
     */
    virtio_set_isr(vq->vdev, 0x1);
    event_notifier_set(&vq->guest_notifier);   //写vq->guest_notifier，即irqfd
}

static int
irqfd_wakeup(wait_queue_entry_t *wait, unsigned mode, int sync, void *key)
{
        if (flags & EPOLLIN) {
                idx = srcu_read_lock(&kvm->irq_srcu);
                do {
                        seq = read_seqcount_begin(&irqfd->irq_entry_sc);
                        irq = irqfd->irq_entry;
                } while (read_seqcount_retry(&irqfd->irq_entry_sc, seq));
                /* An event has been signaled, inject an interrupt */
                if (kvm_arch_set_irq_inatomic(&irq, kvm,
                                             KVM_USERSPACE_IRQ_SOURCE_ID, 1,
                                             false) == -EWOULDBLOCK)
                        schedule_work(&irqfd->inject);
                srcu_read_unlock(&kvm->irq_srcu, idx);
        }

# 先在PCI总线上调用probe设备，调用了virtio_pci_probe，然后再virtio-bus上调用virtio_dev_probe
# virtio_dev_probe最后调用到virtnet_probe
pci_device_probe --> local_pci_probe --> virtio_pci_probe --> register_virtio_device --> 
device_register --> device_add --> bus_probe_device --> device_initial_probe 
--> __device_attach --> bus_for_each_drv --> __device_attach_driver --> driver_probe_device --> 
virtio_dev_probe --> virtnet_probe

# 在virtio_pci_probe里先尝试以virtio modern方式读取设备配置数据结构，如果失败则尝试virio legacy方式。
# 对于virtio legacy，我们前面提到了virtio legacy协议规定设备的配置数据结构放在PCI BAR0里面。
/* the PCI probing function */
int virtio_pci_legacy_probe(struct virtio_pci_device *vp_dev)
{
        rc = pci_request_region(pci_dev, 0, "virtio-pci-legacy");  //将设备的BAR0映射到物理地址空间
        vp_dev->ioaddr = pci_iomap(pci_dev, 0, 0);   //获得BAR0的内核地址
}

#对于virtio modern，通过capability方式报告配置数据结构的位置，配置数据结构有5种类型。
int virtio_pci_modern_probe(struct virtio_pci_device *vp_dev)
{
        /* check for a common config: if not, use legacy mode (bar 0). */
        common = virtio_pci_find_capability(pci_dev, VIRTIO_PCI_CAP_COMMON_CFG,
                                            IORESOURCE_IO | IORESOURCE_MEM,
                                            &vp_dev->modern_bars);

        /* If common is there, these should be too... */
        isr = virtio_pci_find_capability(pci_dev, VIRTIO_PCI_CAP_ISR_CFG,
                                         IORESOURCE_IO | IORESOURCE_MEM,
                                         &vp_dev->modern_bars);
        notify = virtio_pci_find_capability(pci_dev, VIRTIO_PCI_CAP_NOTIFY_CFG,
                                            IORESOURCE_IO | IORESOURCE_MEM,
                                            &vp_dev->modern_bars);
                                            
        /* Device capability is only mandatory for devices that have
        * device-specific configuration.
        */
        device = virtio_pci_find_capability(pci_dev, VIRTIO_PCI_CAP_DEVICE_CFG,
                                            IORESOURCE_IO | IORESOURCE_MEM,
                                            &vp_dev->modern_bars);

        err = pci_request_selected_regions(pci_dev, vp_dev->modern_bars,
                                            "virtio-pci-modern");
                                        sizeof(struct virtio_pci_common_cfg), 4,
                                        0, sizeof(struct virtio_pci_common_cfg),
                                        NULL);
        // 将配virtio置结构所在的BAR空间MAP到内核地址空间里                                
        vp_dev->common = map_capability(pci_dev, common,
                                        sizeof(struct virtio_pci_common_cfg), 4,
                                        0, sizeof(struct virtio_pci_common_cfg),
                                        NULL);
        ......                              
}

# 接着来到virtio_dev_probe里面看下：
static int virtio_dev_probe(struct device *_d)
{
        /* We have a driver! */
        virtio_add_status(dev, VIRTIO_CONFIG_S_DRIVER);     // 更新status bit，这里要写配置数据结构

        /* Figure out what features the device supports. */
        device_features = dev->config->get_features(dev);   // 查询后端支持哪些feature bits
        
        // feature set协商，取交集
        err = virtio_finalize_features(dev); 
        
        // 调用特定virtio设备的驱动程序probe，例如: virtnet_probe, virtblk_probe
        err = drv->probe(dev); 
}

static int virtnet_probe(struct virtio_device *vdev)
{
       // check后端是否支持多队列，并按情况创建队列
       /* Allocate ourselves a network device with room for our info */
        dev = alloc_etherdev_mq(sizeof(struct virtnet_info), max_queue_pairs);
        
        // 定义一个网络设备并配置一些属性，例如MAC地址
        dev->ethtool_ops = &virtnet_ethtool_ops;
	       SET_NETDEV_DEV(dev, &vdev->dev);
 
        // 初始化virtqueue
        err = init_vqs(vi);
        
        // 注册一个网络设备
        err = register_netdev(dev);
        
        // 写状态位DRIVER_OK，告诉后端，前端已经ready
        virtio_device_ready(vdev);
        
        // 将网卡up起来
        netif_carrier_on(dev);
}

init_vqs --> virtnet_find_vqs --> vi->vdev->config->find_vqs --> vp_modern_find_vqs
--> vp_find_vqs --> vp_find_vqs_msix

static int vp_find_vqs_msix(struct virtio_device *vdev, unsigned nvqs,
		struct virtqueue *vqs[], vq_callback_t *callbacks[],
		const char * const names[], bool per_vq_vectors,
		const bool *ctx,
		struct irq_affinity *desc)
{
        /* 为configuration change申请MSIx中断 */
	err = vp_request_msix_vectors(vdev, nvectors, per_vq_vectors,
			      per_vq_vectors ? desc : NULL);
        for (i = 0; i < nvqs; ++i) {
		 // 创建队列 --> vring_create_virtqueue --> vring_create_virtqueue_split --> vring_alloc_queue
	         vqs[i] = vp_setup_vq(vdev, queue_idx++, callbacks[i], names[i],
                                ctx ? ctx[i] : false,
                                msix_vec);
		// 每个队列申请一个MSIx中断
                err = request_irq(pci_irq_vector(vp_dev->pci_dev, msix_vec),
                                  vring_interrupt, 0,
                                  vp_dev->msix_names[msix_vec],
                                  vqs[i]);
	}

irqreturn_t vring_interrupt(int irq, void *_vq)
{
        struct vring_virtqueue *vq = to_vvq(_vq);

        if (!more_used(vq)) {
                pr_debug("virtqueue interrupt with no work for %p\n", vq);
                return IRQ_NONE;
        }

        if (unlikely(vq->broken))
                return IRQ_HANDLED;

        pr_debug("virtqueue callback for %p (%p)\n", vq, vq->vq.callback);
        if (vq->vq.callback)
                vq->vq.callback(&vq->vq);

        return IRQ_HANDLED;
}
EXPORT_SYMBOL_GPL(vring_interrupt);

static const struct net_device_ops netdev_ops = {
        .ndo_open               = rio_open,
        .ndo_start_xmit = start_xmit,
        .ndo_stop               = rio_close,
        .ndo_get_stats          = get_stats,
        .ndo_validate_addr      = eth_validate_addr,
        .ndo_set_mac_address    = eth_mac_addr,
        .ndo_set_rx_mode        = set_multicast,
        .ndo_do_ioctl           = rio_ioctl,
        .ndo_tx_timeout         = rio_tx_timeout,
};

内核驱动 virtio_net.c
start_xmit
	// 将skb放到virtqueue队列中
 	-> xmit_skb -> sg_init_table,virtqueue_add_outbuf -> virtqueue_add
	// kick通知qemu后端去取
	virtqueue_kick_prepare && virtqueue_notify 
	// kick次数加1
	sq->stats.kicks++

Qemu代码virtio-net相关代码:
virtio_queue_host_notifier_read -> virtio_queue_notify_vq
    -> vq->handle_output -> virtio_net_handle_tx_bh 队列注册的时候，回注册回调函数
        -> qemu_bh_schedule -> virtio_net_tx_bh
            -> virtio_net_flush_tx
	        -> virtqueue_pop
		-> qemu_sendv_packet_async // 报文放到发送队列上，写tap设备的fd去发包
		    -> tap_receive_iov -> tap_write_packet
		    
// 最后调用 tap_write_packet 把数据包发给tap设备投递出去

tap_send -> qemu_send_packet_async -> qemu_send_packet_async_with_flags
    -> qemu_net_queue_send
        -> qemu_net_queue_deliver
	    -> qemu_deliver_packet_iov
	        -> nc_sendv_compat
		    -> virtio_net_receive
		        -> virtio_net_receive_rcu