// SPDX-License-Identifier: GPL-2.0

/*
 * Copyright (c) 2022, Microsoft Corporation.
 *
 * Author:
 *   Iouri Tarassov <iourit@linux.microsoft.com>
 *
 * Dxgkrnl Graphics Driver
 * VM bus interface implementation
 *
 */

#include <linux/kernel.h>
#include <linux/completion.h>
#include <linux/slab.h>
#include <linux/eventfd.h>
#include <linux/hyperv.h>
#include <linux/mman.h>
#include <linux/delay.h>
#include <linux/pagemap.h>
#include <linux/vmalloc.h>
#include "dxgkrnl.h"
#include "dxgvmbus.h"

#pragma GCC diagnostic ignored "-Warray-bounds"

#undef dev_fmt
#define dev_fmt(fmt)	"dxgk: " fmt

#define RING_BUFSIZE (256 * 1024)

/*
 * The structure is used to track VM bus packets, waiting for completion.
 */
struct dxgvmbuspacket {
	struct list_head packet_list_entry;
	u64 request_id;
	struct completion wait;
	void *buffer;
	u32 buffer_length;
	int status;
	bool completed;
};

struct dxgvmb_ext_header {
	/* Offset from the start of the message to DXGKVMB_COMMAND_BASE */
	u32		command_offset;
	u32		reserved;
	struct winluid	vgpu_luid;
};

#define VMBUSMESSAGEONSTACK	64

struct dxgvmbusmsg {
/* Points to the allocated buffer */
	struct dxgvmb_ext_header	*hdr;
/* Points to dxgkvmb_command_vm_to_host or dxgkvmb_command_vgpu_to_host */
	void				*msg;
/* The vm bus channel, used to pass the message to the host */
	struct dxgvmbuschannel		*channel;
/* Message size in bytes including the header and the payload */
	u32				size;
/* Buffer used for small messages */
	char				msg_on_stack[VMBUSMESSAGEONSTACK];
};

struct dxgvmbusmsgres {
/* Points to the allocated buffer */
	struct dxgvmb_ext_header	*hdr;
/* Points to dxgkvmb_command_vm_to_host or dxgkvmb_command_vgpu_to_host */
	void				*msg;
/* The vm bus channel, used to pass the message to the host */
	struct dxgvmbuschannel		*channel;
/* Message size in bytes including the header, the payload and the result */
	u32				size;
/* Result buffer size in bytes */
	u32				res_size;
/* Points to the result within the allocated buffer */
	void				*res;
};

static int init_message(struct dxgvmbusmsg *msg, struct dxgadapter *adapter,
			struct dxgprocess *process, u32 size)
{
	struct dxgglobal *dxgglobal = dxggbl();

	bool use_ext_header = dxgglobal->vmbus_ver >=
			      DXGK_VMBUS_INTERFACE_VERSION;

	if (use_ext_header)
		size += sizeof(struct dxgvmb_ext_header);
	msg->size = size;
	if (size <= VMBUSMESSAGEONSTACK) {
		msg->hdr = (void *)msg->msg_on_stack;
		memset(msg->hdr, 0, size);
	} else {
		msg->hdr = vzalloc(size);
		if (msg->hdr == NULL)
			return -ENOMEM;
	}
	if (use_ext_header) {
		msg->msg = (char *)&msg->hdr[1];
		msg->hdr->command_offset = sizeof(msg->hdr[0]);
		if (adapter)
			msg->hdr->vgpu_luid = adapter->host_vgpu_luid;
	} else {
		msg->msg = (char *)msg->hdr;
	}
	if (adapter && !dxgglobal->async_msg_enabled)
		msg->channel = &adapter->channel;
	else
		msg->channel = &dxgglobal->channel;
	return 0;
}

static int init_message_res(struct dxgvmbusmsgres *msg,
			    struct dxgadapter *adapter,
			    u32 size,
			    u32 result_size)
{
	struct dxgglobal *dxgglobal = dxggbl();
	bool use_ext_header = dxgglobal->vmbus_ver >=
			      DXGK_VMBUS_INTERFACE_VERSION;

	if (use_ext_header)
		size += sizeof(struct dxgvmb_ext_header);
	msg->size = size;
	msg->res_size += (result_size + 7) & ~7;
	size += msg->res_size;
	msg->hdr = vzalloc(size);
	if (msg->hdr == NULL) {
		DXG_ERR("Failed to allocate VM bus message: %d", size);
		return -ENOMEM;
	}
	if (use_ext_header) {
		msg->msg = (char *)&msg->hdr[1];
		msg->hdr->command_offset = sizeof(msg->hdr[0]);
		if (adapter)
			msg->hdr->vgpu_luid = adapter->host_vgpu_luid;
	} else {
		msg->msg = (char *)msg->hdr;
	}
	msg->res = (char *)msg->hdr + msg->size;
	if (adapter && !dxgglobal->async_msg_enabled)
		msg->channel = &adapter->channel;
	else
		msg->channel = &dxgglobal->channel;
	return 0;
}

static void free_message(struct dxgvmbusmsg *msg)
{
	if (msg->hdr && (char *)msg->hdr != msg->msg_on_stack)
		vfree(msg->hdr);
}

/*
 * Helper functions
 */

static void command_vm_to_host_init2(struct dxgkvmb_command_vm_to_host *command,
				     enum dxgkvmb_commandtype_global t,
				     struct d3dkmthandle process)
{
	command->command_type	= t;
	command->process	= process;
	command->command_id	= 0;
	command->channel_type	= DXGKVMB_VM_TO_HOST;
}

static void command_vgpu_to_host_init1(struct dxgkvmb_command_vgpu_to_host
					*command,
					enum dxgkvmb_commandtype type)
{
	command->command_type	= type;
	command->process.v	= 0;
	command->command_id	= 0;
	command->channel_type	= DXGKVMB_VGPU_TO_HOST;
}

static void command_vgpu_to_host_init2(struct dxgkvmb_command_vgpu_to_host
					*command,
					enum dxgkvmb_commandtype type,
					struct d3dkmthandle process)
{
	command->command_type	= type;
	command->process	= process;
	command->command_id	= 0;
	command->channel_type	= DXGKVMB_VGPU_TO_HOST;
}

int ntstatus2int(struct ntstatus status)
{
	if (NT_SUCCESS(status))
		return (int)status.v;
	switch (status.v) {
	case STATUS_OBJECT_NAME_COLLISION:
		return -EEXIST;
	case STATUS_NO_MEMORY:
		return -ENOMEM;
	case STATUS_INVALID_PARAMETER:
		return -EINVAL;
	case STATUS_OBJECT_NAME_INVALID:
	case STATUS_OBJECT_NAME_NOT_FOUND:
		return -ENOENT;
	case STATUS_TIMEOUT:
		return -EAGAIN;
	case STATUS_BUFFER_TOO_SMALL:
		return -EOVERFLOW;
	case STATUS_DEVICE_REMOVED:
		return -ENODEV;
	case STATUS_ACCESS_DENIED:
		return -EACCES;
	case STATUS_NOT_SUPPORTED:
		return -EPERM;
	case STATUS_ILLEGAL_INSTRUCTION:
		return -EOPNOTSUPP;
	case STATUS_INVALID_HANDLE:
		return -EBADF;
	case STATUS_GRAPHICS_ALLOCATION_BUSY:
		return -EINPROGRESS;
	case STATUS_OBJECT_TYPE_MISMATCH:
		return -EPROTOTYPE;
	case STATUS_NOT_IMPLEMENTED:
		return -EPERM;
	default:
		return -EINVAL;
	}
}

int dxgvmbuschannel_init(struct dxgvmbuschannel *ch, struct hv_device *hdev)
{
	int ret;

	ch->hdev = hdev;
	spin_lock_init(&ch->packet_list_mutex);
	INIT_LIST_HEAD(&ch->packet_list_head);
	atomic64_set(&ch->packet_request_id, 0);

	ch->packet_cache = kmem_cache_create("DXGK packet cache",
					     sizeof(struct dxgvmbuspacket), 0,
					     0, NULL);
	if (ch->packet_cache == NULL) {
		DXG_ERR("packet_cache alloc failed");
		ret = -ENOMEM;
		goto cleanup;
	}

#if KERNEL_VERSION(5, 15, 0) <= LINUX_VERSION_CODE
	hdev->channel->max_pkt_size = DXG_MAX_VM_BUS_PACKET_SIZE;
#endif
	ret = vmbus_open(hdev->channel, RING_BUFSIZE, RING_BUFSIZE,
			 NULL, 0, dxgvmbuschannel_receive, ch);
	if (ret) {
		DXG_ERR("vmbus_open failed: %d", ret);
		goto cleanup;
	}

	ch->channel = hdev->channel;

cleanup:

	return ret;
}

void dxgvmbuschannel_destroy(struct dxgvmbuschannel *ch)
{
	kmem_cache_destroy(ch->packet_cache);
	ch->packet_cache = NULL;

	if (ch->channel) {
		vmbus_close(ch->channel);
		ch->channel = NULL;
	}
}

static void command_vm_to_host_init1(struct dxgkvmb_command_vm_to_host *command,
				     enum dxgkvmb_commandtype_global type)
{
	command->command_type = type;
	command->process.v = 0;
	command->command_id = 0;
	command->channel_type = DXGKVMB_VM_TO_HOST;
}

static void set_guest_data(struct dxgkvmb_command_host_to_vm *packet,
			   u32 packet_length)
{
	struct dxgkvmb_command_setguestdata *command = (void *)packet;
	struct dxgglobal *dxgglobal = dxggbl();

	DXG_TRACE("Setting guest data: %d %d %p %p",
		command->data_type,
		command->data32,
		command->guest_pointer,
		&dxgglobal->device_state_counter);
	if (command->data_type == SETGUESTDATA_DATATYPE_DWORD &&
	    command->guest_pointer == &dxgglobal->device_state_counter &&
	    command->data32 != 0) {
		atomic_inc(&dxgglobal->device_state_counter);
	}
}

static void signal_guest_event(struct dxgkvmb_command_host_to_vm *packet,
			       u32 packet_length)
{
	struct dxgkvmb_command_signalguestevent *command = (void *)packet;

	if (packet_length < sizeof(struct dxgkvmb_command_signalguestevent)) {
		DXG_ERR("invalid signal guest event packet size");
		return;
	}
	if (command->event == 0) {
		DXG_ERR("invalid event pointer");
		return;
	}
	dxgglobal_signal_host_event(command->event);
}

static void process_inband_packet(struct dxgvmbuschannel *channel,
				  struct vmpacket_descriptor *desc)
{
	u32 packet_length = hv_pkt_datalen(desc);
	struct dxgkvmb_command_host_to_vm *packet;

	if (channel->adapter == NULL) {
		if (packet_length < sizeof(struct dxgkvmb_command_host_to_vm)) {
			DXG_ERR("Invalid global packet");
		} else {
			packet = hv_pkt_data(desc);
			DXG_TRACE("global packet %d",
				packet->command_type);
			switch (packet->command_type) {
			case DXGK_VMBCOMMAND_SETGUESTDATA:
				set_guest_data(packet, packet_length);
				break;
			case DXGK_VMBCOMMAND_SIGNALGUESTEVENT:
			case DXGK_VMBCOMMAND_SIGNALGUESTEVENTPASSIVE:
				signal_guest_event(packet, packet_length);
				break;
			case DXGK_VMBCOMMAND_SENDWNFNOTIFICATION:
				break;
			default:
				DXG_ERR("unexpected host message %d",
					packet->command_type);
			}
		}
	} else {
		DXG_ERR("Unexpected packet for adapter channel");
	}
}

static void process_completion_packet(struct dxgvmbuschannel *channel,
				      struct vmpacket_descriptor *desc)
{
	struct dxgvmbuspacket *packet = NULL;
	struct dxgvmbuspacket *entry;
	u32 packet_length = hv_pkt_datalen(desc);
	unsigned long flags;

	spin_lock_irqsave(&channel->packet_list_mutex, flags);
	list_for_each_entry(entry, &channel->packet_list_head,
			    packet_list_entry) {
		if (desc->trans_id == entry->request_id) {
			packet = entry;
			list_del(&packet->packet_list_entry);
			packet->completed = true;
			break;
		}
	}
	spin_unlock_irqrestore(&channel->packet_list_mutex, flags);
	if (packet) {
		if (packet->buffer_length) {
			if (packet_length < packet->buffer_length) {
				DXG_TRACE("invalid size %d Expected:%d",
					packet_length,
					packet->buffer_length);
				packet->status = -EOVERFLOW;
			} else {
				memcpy(packet->buffer, hv_pkt_data(desc),
				       packet->buffer_length);
			}
		}
		complete(&packet->wait);
	} else {
		DXG_ERR("did not find packet to complete");
	}
}

/* Receive callback for messages from the host */
void dxgvmbuschannel_receive(void *ctx)
{
	struct dxgvmbuschannel *channel = ctx;
	struct vmpacket_descriptor *desc;
	u32 packet_length = 0;

	DXG_TRACE("New adapter message: %p", channel->adapter);
	foreach_vmbus_pkt(desc, channel->channel) {
		packet_length = hv_pkt_datalen(desc);
		DXG_TRACE("next packet (id, size, type): %llu %d %d",
			desc->trans_id, packet_length, desc->type);
		if (desc->type == VM_PKT_COMP) {
			process_completion_packet(channel, desc);
		} else {
			if (desc->type != VM_PKT_DATA_INBAND)
				DXG_ERR("unexpected packet type");
			else
				process_inband_packet(channel, desc);
		}
	}
}

int dxgvmb_send_sync_msg(struct dxgvmbuschannel *channel,
			 void *command,
			 u32 cmd_size,
			 void *result,
			 u32 result_size)
{
	int ret;
	struct dxgvmbuspacket *packet = NULL;
	struct dxgkvmb_command_vm_to_host *cmd1;
	struct dxgkvmb_command_vgpu_to_host *cmd2;
	int try_count = 0;

	if (cmd_size > DXG_MAX_VM_BUS_PACKET_SIZE ||
	    result_size > DXG_MAX_VM_BUS_PACKET_SIZE) {
		DXG_ERR("%s invalid data size", __func__);
		return -EINVAL;
	}

	packet = kmem_cache_alloc(channel->packet_cache, 0);
	if (packet == NULL) {
		DXG_ERR("kmem_cache_alloc failed");
		return -ENOMEM;
	}

	if (channel->adapter == NULL) {
		cmd1 = command;
		DXG_TRACE("send_sync_msg global: %d %p %d %d",
			cmd1->command_type, command, cmd_size, result_size);
	} else {
		cmd2 = command;
		DXG_TRACE("send_sync_msg adapter: %d %p %d %d",
			cmd2->command_type, command, cmd_size, result_size);
	}

	packet->request_id = atomic64_inc_return(&channel->packet_request_id);
	init_completion(&packet->wait);
	packet->buffer = result;
	packet->buffer_length = result_size;
	packet->status = 0;
	packet->completed = false;
	spin_lock_irq(&channel->packet_list_mutex);
	list_add_tail(&packet->packet_list_entry, &channel->packet_list_head);
	spin_unlock_irq(&channel->packet_list_mutex);

	do {
		ret = vmbus_sendpacket(channel->channel, command, cmd_size,
				packet->request_id, VM_PKT_DATA_INBAND,
				VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
		/*
		 * -EAGAIN is returned when the VM bus ring buffer if full.
		 * Wait 2ms to allow the host to process messages and try again.
		 */
		if (ret == -EAGAIN) {
			usleep_range(1000, 2000);
			try_count++;
		}
	} while (ret == -EAGAIN && try_count < 50);
	if (ret) {
		DXG_ERR("vmbus_sendpacket failed: %x", ret);
		spin_lock_irq(&channel->packet_list_mutex);
		list_del(&packet->packet_list_entry);
		spin_unlock_irq(&channel->packet_list_mutex);
		goto cleanup;
	}

	DXG_TRACE("waiting completion: %llu", packet->request_id);
	ret = wait_for_completion_killable(&packet->wait);
	if (ret) {
		DXG_ERR("wait_for_completion failed: %x", ret);
		spin_lock_irq(&channel->packet_list_mutex);
		if (!packet->completed)
			list_del(&packet->packet_list_entry);
		spin_unlock_irq(&channel->packet_list_mutex);
		goto cleanup;
	}
	DXG_TRACE("completion done: %llu %x",
		packet->request_id, packet->status);
	ret = packet->status;

cleanup:

	kmem_cache_free(channel->packet_cache, packet);
	if (ret < 0)
		DXG_TRACE("Error: %x", ret);
	return ret;
}

int dxgvmb_send_async_msg(struct dxgvmbuschannel *channel,
			  void *command,
			  u32 cmd_size)
{
	int ret;
	int try_count = 0;

	if (cmd_size > DXG_MAX_VM_BUS_PACKET_SIZE) {
		DXG_ERR("%s invalid data size", __func__);
		return -EINVAL;
	}

	if (channel->adapter) {
		DXG_ERR("Async message sent to the adapter channel");
		return -EINVAL;
	}

	do {
		ret = vmbus_sendpacket(channel->channel, command, cmd_size,
				0, VM_PKT_DATA_INBAND, 0);
		/*
		 * -EAGAIN is returned when the VM bus ring buffer if full.
		 * Wait 2ms to allow the host to process messages and try again.
		 */
		if (ret == -EAGAIN) {
			usleep_range(1000, 2000);
			try_count++;
		}
	} while (ret == -EAGAIN && try_count < 5000);
	if (ret < 0)
		DXG_ERR("vmbus_sendpacket failed: %x", ret);

	return ret;
}

static int
dxgvmb_send_sync_msg_ntstatus(struct dxgvmbuschannel *channel,
			      void *command, u32 cmd_size)
{
	struct ntstatus status;
	int ret;

	ret = dxgvmb_send_sync_msg(channel, command, cmd_size,
				   &status, sizeof(status));
	if (ret >= 0)
		ret = ntstatus2int(status);
	return ret;
}

static int check_iospace_address(unsigned long address, u32 size)
{
	struct dxgglobal *dxgglobal = dxggbl();

	if (address < dxgglobal->mmiospace_base ||
	    size > dxgglobal->mmiospace_size ||
	    address >= (dxgglobal->mmiospace_base +
			dxgglobal->mmiospace_size - size)) {
		DXG_ERR("invalid iospace address %lx", address);
		return -EINVAL;
	}
	return 0;
}

int dxg_unmap_iospace(void *va, u32 size)
{
	int ret = 0;

	DXG_TRACE("Unmapping io space: %p %x", va, size);

	/*
	 * When an app calls exit(), dxgkrnl is called to close the device
	 * with current->mm equal to NULL.
	 */
	if (current->mm) {
		ret = vm_munmap((unsigned long)va, size);
		if (ret) {
			DXG_ERR("vm_munmap failed %d", ret);
			return -ENOTRECOVERABLE;
		}
	}
	return 0;
}

static u8 *dxg_map_iospace(u64 iospace_address, u32 size,
			   unsigned long protection, bool cached)
{
	struct vm_area_struct *vma;
	unsigned long va;
	int ret = 0;

	DXG_TRACE("Mapping io space: %llx %x %lx",
		iospace_address, size, protection);
	if (check_iospace_address(iospace_address, size) < 0) {
		DXG_ERR("invalid address to map");
		return NULL;
	}

	va = vm_mmap(NULL, 0, size, protection, MAP_SHARED | MAP_ANONYMOUS, 0);
	if ((long)va <= 0) {
		DXG_ERR("vm_mmap failed %lx %d", va, size);
		return NULL;
	}

	mmap_write_lock(current->mm);
	vma = find_vma(current->mm, (unsigned long)va);
	if (vma) {
		pgprot_t prot = vma->vm_page_prot;

		if (!cached)
			prot = pgprot_writecombine(prot);
		DXG_TRACE("vma: %lx %lx %lx",
			vma->vm_start, vma->vm_end, va);
		vma->vm_pgoff = iospace_address >> PAGE_SHIFT;
		ret = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
					 size, prot);
		if (ret)
			DXG_ERR("io_remap_pfn_range failed: %d", ret);
	} else {
		DXG_ERR("failed to find vma: %p %lx", vma, va);
		ret = -ENOMEM;
	}
	mmap_write_unlock(current->mm);

	if (ret) {
		dxg_unmap_iospace((void *)va, size);
		return NULL;
	}
	DXG_TRACE("Mapped VA: %lx", va);
	return (u8 *) va;
}

/*
 * Global messages to the host
 */

int dxgvmb_send_set_iospace_region(u64 start, u64 len)
{
	int ret;
	struct dxgkvmb_command_setiospaceregion *command;
	struct dxgvmbusmsg msg;
	struct dxgglobal *dxgglobal = dxggbl();

	ret = init_message(&msg, NULL, NULL, sizeof(*command));
	if (ret)
		return ret;
	command = (void *)msg.msg;

	ret = dxgglobal_acquire_channel_lock();
	if (ret < 0)
		goto cleanup;

	command_vm_to_host_init1(&command->hdr,
				 DXGK_VMBCOMMAND_SETIOSPACEREGION);
	command->start = start;
	command->length = len;
	ret = dxgvmb_send_sync_msg_ntstatus(&dxgglobal->channel, msg.hdr,
					    msg.size);
	if (ret < 0)
		DXG_ERR("send_set_iospace_region failed %x", ret);

	dxgglobal_release_channel_lock();
cleanup:
	free_message(&msg);
	if (ret)
		DXG_TRACE("Error: %d", ret);
	return ret;
}

int dxgvmb_send_create_process(struct dxgprocess *process)
{
	int ret;
	struct dxgkvmb_command_createprocess *command;
	struct dxgkvmb_command_createprocess_return result = { 0 };
	struct dxgvmbusmsg msg;
	char s[WIN_MAX_PATH];
	int i;
	struct dxgglobal *dxgglobal = dxggbl();

	ret = init_message(&msg, NULL, process, sizeof(*command));
	if (ret)
		return ret;
	command = (void *)msg.msg;

	ret = dxgglobal_acquire_channel_lock();
	if (ret < 0)
		goto cleanup;

	command_vm_to_host_init1(&command->hdr, DXGK_VMBCOMMAND_CREATEPROCESS);
	command->process = process;
	command->process_id = process->pid;
	command->linux_process = 1;
	s[0] = 0;
	get_task_comm(s, current);
	for (i = 0; i < WIN_MAX_PATH; i++) {
		command->process_name[i] = s[i];
		if (s[i] == 0)
			break;
	}

	ret = dxgvmb_send_sync_msg(&dxgglobal->channel, msg.hdr, msg.size,
				   &result, sizeof(result));
	if (ret < 0) {
		DXG_ERR("create_process failed %d", ret);
	} else if (result.hprocess.v == 0) {
		DXG_ERR("create_process returned 0 handle");
		ret = -ENOTRECOVERABLE;
	} else {
		process->host_handle = result.hprocess;
		DXG_TRACE("create_process returned %x",
			process->host_handle.v);
	}

	dxgglobal_release_channel_lock();

cleanup:
	free_message(&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

int dxgvmb_send_destroy_process(struct d3dkmthandle process)
{
	int ret;
	struct dxgkvmb_command_destroyprocess *command;
	struct dxgvmbusmsg msg;
	struct dxgglobal *dxgglobal = dxggbl();

	ret = init_message(&msg, NULL, NULL, sizeof(*command));
	if (ret)
		return ret;
	command = (void *)msg.msg;

	ret = dxgglobal_acquire_channel_lock();
	if (ret < 0)
		goto cleanup;
	command_vm_to_host_init2(&command->hdr, DXGK_VMBCOMMAND_DESTROYPROCESS,
				 process);
	ret = dxgvmb_send_sync_msg_ntstatus(&dxgglobal->channel,
					    msg.hdr, msg.size);
	dxgglobal_release_channel_lock();

cleanup:
	free_message(&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

int dxgvmb_send_open_sync_object_nt(struct dxgprocess *process,
				    struct dxgvmbuschannel *channel,
				    struct d3dkmt_opensyncobjectfromnthandle2
				    *args,
				    struct dxgsyncobject *syncobj)
{
	struct dxgkvmb_command_opensyncobject *command;
	struct dxgkvmb_command_opensyncobject_return result = { };
	int ret;
	struct dxgvmbusmsg msg;

	ret = init_message(&msg, NULL, process, sizeof(*command));
	if (ret)
		return ret;
	command = (void *)msg.msg;

	command_vm_to_host_init2(&command->hdr, DXGK_VMBCOMMAND_OPENSYNCOBJECT,
				 process->host_handle);
	command->device = args->device;
	command->global_sync_object = syncobj->shared_owner->host_shared_handle;
	command->flags = args->flags;
	if (syncobj->monitored_fence)
		command->engine_affinity =
			args->monitored_fence.engine_affinity;

	ret = dxgglobal_acquire_channel_lock();
	if (ret < 0)
		goto cleanup;

	ret = dxgvmb_send_sync_msg(channel, msg.hdr, msg.size,
				   &result, sizeof(result));

	dxgglobal_release_channel_lock();

	if (ret < 0)
		goto cleanup;

	ret = ntstatus2int(result.status);
	if (ret < 0)
		goto cleanup;

	args->sync_object = result.sync_object;
	if (syncobj->monitored_fence) {
		void *va = dxg_map_iospace(result.guest_cpu_physical_address,
					   PAGE_SIZE, PROT_READ | PROT_WRITE,
					   true);
		if (va == NULL) {
			ret = -ENOMEM;
			goto cleanup;
		}
		args->monitored_fence.fence_value_cpu_va = va;
		args->monitored_fence.fence_value_gpu_va =
		    result.gpu_virtual_address;
		syncobj->mapped_address = va;
	}

cleanup:
	free_message(&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

int dxgvmb_send_open_sync_object(struct dxgprocess *process,
				struct d3dkmthandle device,
				struct d3dkmthandle host_shared_syncobj,
				struct d3dkmthandle *syncobj)
{
	struct dxgkvmb_command_opensyncobject *command;
	struct dxgkvmb_command_opensyncobject_return result = { };
	int ret;
	struct dxgvmbusmsg msg;
	struct dxgglobal *dxgglobal = dxggbl();

	ret = init_message(&msg, NULL, process, sizeof(*command));
	if (ret)
		return ret;
	command = (void *)msg.msg;

	command_vm_to_host_init2(&command->hdr, DXGK_VMBCOMMAND_OPENSYNCOBJECT,
				 process->host_handle);
	command->device = device;
	command->global_sync_object = host_shared_syncobj;
	command->flags.shared = 1;
	command->flags.nt_security_sharing = 1;
	command->flags.no_signal = 1;

	ret = dxgglobal_acquire_channel_lock();
	if (ret < 0)
		goto cleanup;

	ret = dxgvmb_send_sync_msg(&dxgglobal->channel, msg.hdr, msg.size,
				   &result, sizeof(result));

	dxgglobal_release_channel_lock();

	if (ret < 0)
		goto cleanup;

	ret = ntstatus2int(result.status);
	if (ret < 0)
		goto cleanup;

	*syncobj = result.sync_object;

cleanup:
	free_message(&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

int dxgvmb_send_create_nt_shared_object(struct dxgprocess *process,
					struct d3dkmthandle object,
					struct d3dkmthandle *shared_handle)
{
	struct dxgkvmb_command_createntsharedobject *command;
	int ret;
	struct dxgvmbusmsg msg;

	ret = init_message(&msg, NULL, process, sizeof(*command));
	if (ret)
		return ret;
	command = (void *)msg.msg;

	command_vm_to_host_init2(&command->hdr,
				 DXGK_VMBCOMMAND_CREATENTSHAREDOBJECT,
				 process->host_handle);
	command->object = object;

	ret = dxgglobal_acquire_channel_lock();
	if (ret < 0)
		goto cleanup;

	ret = dxgvmb_send_sync_msg(dxgglobal_get_dxgvmbuschannel(),
				   msg.hdr, msg.size, shared_handle,
				   sizeof(*shared_handle));

	dxgglobal_release_channel_lock();

	if (ret < 0)
		goto cleanup;
	if (shared_handle->v == 0) {
		DXG_ERR("failed to create NT shared object");
		ret = -ENOTRECOVERABLE;
	}

cleanup:
	free_message(&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

int dxgvmb_send_destroy_nt_shared_object(struct d3dkmthandle shared_handle)
{
	struct dxgkvmb_command_destroyntsharedobject *command;
	int ret;
	struct dxgvmbusmsg msg;

	ret = init_message(&msg, NULL, NULL, sizeof(*command));
	if (ret)
		return ret;
	command = (void *)msg.msg;

	command_vm_to_host_init1(&command->hdr,
				 DXGK_VMBCOMMAND_DESTROYNTSHAREDOBJECT);
	command->shared_handle = shared_handle;

	ret = dxgglobal_acquire_channel_lock();
	if (ret < 0)
		goto cleanup;

	ret = dxgvmb_send_sync_msg_ntstatus(dxgglobal_get_dxgvmbuschannel(),
					    msg.hdr, msg.size);

	dxgglobal_release_channel_lock();

cleanup:
	free_message(&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

int dxgvmb_send_destroy_sync_object(struct dxgprocess *process,
				    struct d3dkmthandle sync_object)
{
	struct dxgkvmb_command_destroysyncobject *command;
	int ret;
	struct dxgvmbusmsg msg = {.hdr = NULL};

	ret = init_message(&msg, NULL, process, sizeof(*command));
	if (ret)
		return ret;
	command = (void *)msg.msg;

	ret = dxgglobal_acquire_channel_lock();
	if (ret < 0)
		goto cleanup;

	command_vm_to_host_init2(&command->hdr,
				 DXGK_VMBCOMMAND_DESTROYSYNCOBJECT,
				 process->host_handle);
	command->sync_object = sync_object;

	ret = dxgvmb_send_sync_msg_ntstatus(dxgglobal_get_dxgvmbuschannel(),
					    msg.hdr, msg.size);

	dxgglobal_release_channel_lock();

cleanup:
	free_message(&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

int dxgvmb_send_share_object_with_host(struct dxgprocess *process,
				struct d3dkmt_shareobjectwithhost *args)
{
	struct dxgkvmb_command_shareobjectwithhost *command;
	struct dxgkvmb_command_shareobjectwithhost_return result = {};
	int ret;
	struct dxgvmbusmsg msg = {.hdr = NULL};

	ret = init_message(&msg, NULL, process, sizeof(*command));
	if (ret)
		return ret;
	command = (void *)msg.msg;

	ret = dxgglobal_acquire_channel_lock();
	if (ret < 0)
		goto cleanup;

	command_vm_to_host_init2(&command->hdr,
				 DXGK_VMBCOMMAND_SHAREOBJECTWITHHOST,
				 process->host_handle);
	command->device_handle = args->device_handle;
	command->object_handle = args->object_handle;

	ret = dxgvmb_send_sync_msg(dxgglobal_get_dxgvmbuschannel(),
				   msg.hdr, msg.size, &result, sizeof(result));

	dxgglobal_release_channel_lock();

	if (ret || !NT_SUCCESS(result.status)) {
		if (ret == 0)
			ret = ntstatus2int(result.status);
		DXG_ERR("Host failed to share object with host: %d %x",
			ret, result.status.v);
		goto cleanup;
	}
	args->object_vail_nt_handle = result.vail_nt_handle;

cleanup:
	free_message(&msg);
	if (ret)
		DXG_ERR("err: %d", ret);
	return ret;
}

/*
 * Virtual GPU messages to the host
 */

int dxgvmb_send_open_adapter(struct dxgadapter *adapter)
{
	int ret;
	struct dxgkvmb_command_openadapter *command;
	struct dxgkvmb_command_openadapter_return result = { };
	struct dxgvmbusmsg msg;
	struct dxgglobal *dxgglobal = dxggbl();

	ret = init_message(&msg, adapter, NULL, sizeof(*command));
	if (ret)
		return ret;
	command = (void *)msg.msg;

	command_vgpu_to_host_init1(&command->hdr, DXGK_VMBCOMMAND_OPENADAPTER);
	command->vmbus_interface_version = dxgglobal->vmbus_ver;
	command->vmbus_last_compatible_interface_version =
	    DXGK_VMBUS_LAST_COMPATIBLE_INTERFACE_VERSION;

	ret = dxgvmb_send_sync_msg(msg.channel, msg.hdr, msg.size,
				   &result, sizeof(result));
	if (ret < 0)
		goto cleanup;

	ret = ntstatus2int(result.status);
	adapter->host_handle = result.host_adapter_handle;

cleanup:
	free_message(&msg);
	if (ret)
		DXG_ERR("Failed to open adapter: %d", ret);
	return ret;
}

int dxgvmb_send_close_adapter(struct dxgadapter *adapter)
{
	int ret;
	struct dxgkvmb_command_closeadapter *command;
	struct dxgvmbusmsg msg;

	ret = init_message(&msg, adapter, NULL, sizeof(*command));
	if (ret)
		return ret;
	command = (void *)msg.msg;

	command_vgpu_to_host_init1(&command->hdr, DXGK_VMBCOMMAND_CLOSEADAPTER);
	command->host_handle = adapter->host_handle;

	ret = dxgvmb_send_sync_msg(msg.channel, msg.hdr, msg.size,
				   NULL, 0);
	free_message(&msg);
	if (ret)
		DXG_ERR("Failed to close adapter: %d", ret);
	return ret;
}

int dxgvmb_send_get_internal_adapter_info(struct dxgadapter *adapter)
{
	int ret;
	struct dxgkvmb_command_getinternaladapterinfo *command;
	struct dxgkvmb_command_getinternaladapterinfo_return result = { };
	struct dxgvmbusmsg msg;
	u32 result_size = sizeof(result);
	struct dxgglobal *dxgglobal = dxggbl();

	ret = init_message(&msg, adapter, NULL, sizeof(*command));
	if (ret)
		return ret;
	command = (void *)msg.msg;

	command_vgpu_to_host_init1(&command->hdr,
				   DXGK_VMBCOMMAND_GETINTERNALADAPTERINFO);
	if (dxgglobal->vmbus_ver < DXGK_VMBUS_INTERFACE_VERSION)
		result_size -= sizeof(struct winluid);

	ret = dxgvmb_send_sync_msg(msg.channel, msg.hdr, msg.size,
				   &result, result_size);
	if (ret >= 0) {
		adapter->host_adapter_luid = result.host_adapter_luid;
		adapter->host_vgpu_luid = result.host_vgpu_luid;
		wcsncpy(adapter->device_description, result.device_description,
			sizeof(adapter->device_description) / sizeof(u16));
		wcsncpy(adapter->device_instance_id, result.device_instance_id,
			sizeof(adapter->device_instance_id) / sizeof(u16));
		dxgglobal->async_msg_enabled = result.async_msg_enabled != 0;
	}
	free_message(&msg);
	if (ret)
		DXG_ERR("Failed to get adapter info: %d", ret);
	return ret;
}

struct d3dkmthandle dxgvmb_send_create_device(struct dxgadapter *adapter,
					struct dxgprocess *process,
					struct d3dkmt_createdevice *args)
{
	int ret;
	struct dxgkvmb_command_createdevice *command;
	struct dxgkvmb_command_createdevice_return result = { };
	struct dxgvmbusmsg msg;
	struct dxgglobal *dxgglobal = dxggbl();

	ret = init_message(&msg, adapter, process, sizeof(*command));
	if (ret)
		goto cleanup;
	command = (void *)msg.msg;

	command_vgpu_to_host_init2(&command->hdr, DXGK_VMBCOMMAND_CREATEDEVICE,
				   process->host_handle);
	command->flags = args->flags;
	command->error_code = &dxgglobal->device_state_counter;

	ret = dxgvmb_send_sync_msg(msg.channel, msg.hdr, msg.size,
				   &result, sizeof(result));
	if (ret < 0)
		result.device.v = 0;
	free_message(&msg);
cleanup:
	if (ret)
		DXG_TRACE("err: %d", ret);
	return result.device;
}

int dxgvmb_send_destroy_device(struct dxgadapter *adapter,
			       struct dxgprocess *process,
			       struct d3dkmthandle h)
{
	int ret;
	struct dxgkvmb_command_destroydevice *command;
	struct dxgvmbusmsg msg = {.hdr = NULL};

	ret = init_message(&msg, adapter, process, sizeof(*command));
	if (ret)
		goto cleanup;
	command = (void *)msg.msg;

	command_vgpu_to_host_init2(&command->hdr, DXGK_VMBCOMMAND_DESTROYDEVICE,
				   process->host_handle);
	command->device = h;

	ret = dxgvmb_send_sync_msg_ntstatus(msg.channel, msg.hdr, msg.size);
cleanup:
	free_message(&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

int dxgvmb_send_flush_device(struct dxgdevice *device,
			     enum dxgdevice_flushschedulerreason reason)
{
	int ret;
	struct dxgkvmb_command_flushdevice *command = NULL;
	struct dxgvmbusmsg msg = {.hdr = NULL};
	struct dxgprocess *process = device->process;

	ret = init_message(&msg, device->adapter, process, sizeof(*command));
	if (ret)
		goto cleanup;
	command = (void *)msg.msg;

	command_vgpu_to_host_init2(&command->hdr, DXGK_VMBCOMMAND_FLUSHDEVICE,
				   process->host_handle);
	command->device = device->handle;
	command->reason = reason;

	ret = dxgvmb_send_sync_msg_ntstatus(msg.channel, msg.hdr, msg.size);

cleanup:
	free_message(&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

struct d3dkmthandle
dxgvmb_send_create_context(struct dxgadapter *adapter,
			   struct dxgprocess *process,
			   struct d3dkmt_createcontextvirtual *args)
{
	struct dxgkvmb_command_createcontextvirtual *command = NULL;
	u32 cmd_size;
	int ret;
	struct d3dkmthandle context = {};
	struct dxgvmbusmsg msg = {.hdr = NULL};

	if (args->priv_drv_data_size > DXG_MAX_VM_BUS_PACKET_SIZE) {
		DXG_ERR("PrivateDriverDataSize is invalid");
		ret = -EINVAL;
		goto cleanup;
	}
	cmd_size = sizeof(struct dxgkvmb_command_createcontextvirtual) +
	    args->priv_drv_data_size - 1;

	ret = init_message(&msg, adapter, process, cmd_size);
	if (ret)
		goto cleanup;
	command = (void *)msg.msg;

	command_vgpu_to_host_init2(&command->hdr,
				   DXGK_VMBCOMMAND_CREATECONTEXTVIRTUAL,
				   process->host_handle);
	command->device = args->device;
	command->node_ordinal = args->node_ordinal;
	command->engine_affinity = args->engine_affinity;
	command->flags = args->flags;
	command->client_hint = args->client_hint;
	command->priv_drv_data_size = args->priv_drv_data_size;
	if (args->priv_drv_data_size) {
		ret = copy_from_user(command->priv_drv_data,
				     args->priv_drv_data,
				     args->priv_drv_data_size);
		if (ret) {
			DXG_ERR("Faled to copy private data");
			ret = -EFAULT;
			goto cleanup;
		}
	}
	/* Input command is returned back as output */
	ret = dxgvmb_send_sync_msg(msg.channel, msg.hdr, msg.size,
				   command, cmd_size);
	if (ret < 0) {
		goto cleanup;
	} else {
		context = command->context;
		if (args->priv_drv_data_size) {
			ret = copy_to_user(args->priv_drv_data,
					   command->priv_drv_data,
					   args->priv_drv_data_size);
			if (ret) {
				DXG_ERR(
					"Faled to copy private data to user");
				ret = -EFAULT;
				dxgvmb_send_destroy_context(adapter, process,
							    context);
				context.v = 0;
			}
		}
	}

cleanup:
	free_message(&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return context;
}

int dxgvmb_send_destroy_context(struct dxgadapter *adapter,
				struct dxgprocess *process,
				struct d3dkmthandle h)
{
	int ret;
	struct dxgkvmb_command_destroycontext *command;
	struct dxgvmbusmsg msg = {.hdr = NULL};

	ret = init_message(&msg, adapter, process, sizeof(*command));
	if (ret)
		goto cleanup;
	command = (void *)msg.msg;

	command_vgpu_to_host_init2(&command->hdr,
				   DXGK_VMBCOMMAND_DESTROYCONTEXT,
				   process->host_handle);
	command->context = h;

	ret = dxgvmb_send_sync_msg_ntstatus(msg.channel, msg.hdr, msg.size);
cleanup:
	free_message(&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

int dxgvmb_send_create_paging_queue(struct dxgprocess *process,
				    struct dxgdevice *device,
				    struct d3dkmt_createpagingqueue *args,
				    struct dxgpagingqueue *pqueue)
{
	struct dxgkvmb_command_createpagingqueue_return result;
	struct dxgkvmb_command_createpagingqueue *command;
	int ret;
	struct dxgvmbusmsg msg = {.hdr = NULL};

	ret = init_message(&msg, device->adapter, process, sizeof(*command));
	if (ret)
		goto cleanup;
	command = (void *)msg.msg;

	command_vgpu_to_host_init2(&command->hdr,
				   DXGK_VMBCOMMAND_CREATEPAGINGQUEUE,
				   process->host_handle);
	command->args = *args;
	args->paging_queue.v = 0;

	ret = dxgvmb_send_sync_msg(msg.channel, msg.hdr, msg.size, &result,
				   sizeof(result));
	if (ret < 0) {
		DXG_ERR("send_create_paging_queue failed %x", ret);
		goto cleanup;
	}

	args->paging_queue = result.paging_queue;
	args->sync_object = result.sync_object;
	args->fence_cpu_virtual_address =
	    dxg_map_iospace(result.fence_storage_physical_address, PAGE_SIZE,
			    PROT_READ | PROT_WRITE, true);
	if (args->fence_cpu_virtual_address == NULL) {
		ret = -ENOMEM;
		goto cleanup;
	}
	pqueue->mapped_address = args->fence_cpu_virtual_address;
	pqueue->handle = args->paging_queue;

cleanup:
	free_message(&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

int dxgvmb_send_destroy_paging_queue(struct dxgprocess *process,
				     struct dxgadapter *adapter,
				     struct d3dkmthandle h)
{
	int ret;
	struct dxgkvmb_command_destroypagingqueue *command;
	struct dxgvmbusmsg msg = {.hdr = NULL};

	ret = init_message(&msg, adapter, process, sizeof(*command));
	if (ret)
		goto cleanup;
	command = (void *)msg.msg;

	command_vgpu_to_host_init2(&command->hdr,
				   DXGK_VMBCOMMAND_DESTROYPAGINGQUEUE,
				   process->host_handle);
	command->paging_queue = h;

	ret = dxgvmb_send_sync_msg(msg.channel, msg.hdr, msg.size, NULL, 0);

cleanup:
	free_message(&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

static int
copy_private_data(struct d3dkmt_createallocation *args,
		  struct dxgkvmb_command_createallocation *command,
		  struct d3dddi_allocationinfo2 *input_alloc_info,
		  struct d3dkmt_createstandardallocation *standard_alloc)
{
	struct dxgkvmb_command_createallocation_allocinfo *alloc_info;
	struct d3dddi_allocationinfo2 *input_alloc;
	int ret = 0;
	int i;
	u8 *private_data_dest = (u8 *) &command[1] +
	    (args->alloc_count *
	     sizeof(struct dxgkvmb_command_createallocation_allocinfo));

	if (args->private_runtime_data_size) {
		ret = copy_from_user(private_data_dest,
				     args->private_runtime_data,
				     args->private_runtime_data_size);
		if (ret) {
			DXG_ERR("failed to copy runtime data");
			ret = -EFAULT;
			goto cleanup;
		}
		private_data_dest += args->private_runtime_data_size;
	}

	if (args->flags.standard_allocation) {
		DXG_TRACE("private data offset %d",
			(u32) (private_data_dest - (u8 *) command));

		args->priv_drv_data_size = sizeof(*args->standard_allocation);
		memcpy(private_data_dest, standard_alloc,
		       sizeof(*standard_alloc));
		private_data_dest += args->priv_drv_data_size;
	} else if (args->priv_drv_data_size) {
		ret = copy_from_user(private_data_dest,
				     args->priv_drv_data,
				     args->priv_drv_data_size);
		if (ret) {
			DXG_ERR("failed to copy private data");
			ret = -EFAULT;
			goto cleanup;
		}
		private_data_dest += args->priv_drv_data_size;
	}

	alloc_info = (void *)&command[1];
	input_alloc = input_alloc_info;
	if (input_alloc_info[0].sysmem)
		command->flags.existing_sysmem = 1;
	for (i = 0; i < args->alloc_count; i++) {
		alloc_info->flags = input_alloc->flags.value;
		alloc_info->vidpn_source_id = input_alloc->vidpn_source_id;
		alloc_info->priv_drv_data_size =
		    input_alloc->priv_drv_data_size;
		if (input_alloc->priv_drv_data_size) {
			ret = copy_from_user(private_data_dest,
					     input_alloc->priv_drv_data,
					     input_alloc->priv_drv_data_size);
			if (ret) {
				DXG_ERR("failed to copy alloc data");
				ret = -EFAULT;
				goto cleanup;
			}
			private_data_dest += input_alloc->priv_drv_data_size;
		}
		alloc_info++;
		input_alloc++;
	}

cleanup:
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

static
int create_existing_sysmem(struct dxgdevice *device,
			   struct dxgkvmb_command_allocinfo_return *host_alloc,
			   struct dxgallocation *dxgalloc,
			   bool read_only,
			   const void *sysmem)
{
	int ret1 = 0;
	void *kmem = NULL;
	int ret = 0;
	struct dxgkvmb_command_setexistingsysmemstore *set_store_command;
	struct dxgkvmb_command_setexistingsysmempages *set_pages_command;
	u64 alloc_size = host_alloc->allocation_size;
	u32 npages = alloc_size >> PAGE_SHIFT;
	struct dxgvmbusmsg msg = {.hdr = NULL};
	const u32 max_pfns_in_message =
		(DXG_MAX_VM_BUS_PACKET_SIZE - sizeof(*set_pages_command) -
		PAGE_SIZE) / sizeof(__u64);
	u32 alloc_offset_in_pages = 0;
	struct page **page_in;
	u64 *pfn;
	u32 pages_to_send;
	u32 i;
	u32 gup_flags = FOLL_LONGTERM;
	struct dxgglobal *dxgglobal = dxggbl();

	/*
	 * Create a guest physical address list and set it as the allocation
	 * backing store in the host. This is done after creating the host
	 * allocation, because only now the allocation size is known.
	 */

	DXG_TRACE("Alloc size: %lld", alloc_size);

	dxgalloc->cpu_address = (void *)sysmem;

	dxgalloc->pages = vzalloc(npages * sizeof(void *));
	if (dxgalloc->pages == NULL) {
		DXG_ERR("failed to allocate pages");
		ret = -ENOMEM;
		goto cleanup;
	}
	if (!read_only)
		gup_flags |= FOLL_WRITE;
	ret1 = pin_user_pages_fast((unsigned long)sysmem, npages, gup_flags,
				   dxgalloc->pages);
	if (ret1 != npages) {
		DXG_ERR("get_user_pages_fast failed: %d", ret1);
		if (ret1 > 0 && ret1 < npages) {
			unpin_user_pages(dxgalloc->pages, ret1);
		}
		vfree(dxgalloc->pages);
		dxgalloc->pages = NULL;
		ret = -ENOMEM;
		goto cleanup;
	}
	if (!dxgglobal->map_guest_pages_enabled) {
		ret = init_message(&msg, device->adapter, device->process,
				sizeof(*set_store_command));
		if (ret)
			goto cleanup;
		set_store_command = (void *)msg.msg;

		kmem = vmap(dxgalloc->pages, npages, VM_MAP, PAGE_KERNEL);
		if (kmem == NULL) {
			DXG_ERR("vmap failed");
			ret = -ENOMEM;
			goto cleanup;
		}
		ret1 = vmbus_establish_gpadl(dxgglobal_get_vmbus(), kmem,
					alloc_size, &dxgalloc->gpadl);
		if (ret1) {
			DXG_ERR("establish_gpadl failed: %d", ret1);
			ret = -ENOMEM;
			goto cleanup;
		}
		DXG_TRACE("New gpadl %d", dxgalloc->gpadl.gpadl_handle);

		command_vgpu_to_host_init2(&set_store_command->hdr,
					DXGK_VMBCOMMAND_SETEXISTINGSYSMEMSTORE,
					device->process->host_handle);
		set_store_command->device = device->handle;
		set_store_command->allocation = host_alloc->allocation;
		set_store_command->gpadl = dxgalloc->gpadl.gpadl_handle;
		ret = dxgvmb_send_sync_msg_ntstatus(msg.channel, msg.hdr,
						    msg.size);
		if (ret < 0)
			DXG_ERR("failed set existing store: %x", ret);
	} else {
		/*
		 * Send the list of the allocation PFNs to the host. The host
		 * will map the pages for GPU access.
		 */

		ret = init_message(&msg, device->adapter, device->process,
				sizeof(*set_pages_command) +
				max_pfns_in_message * sizeof(u64));
		if (ret)
			goto cleanup;
		set_pages_command = (void *)msg.msg;
		command_vgpu_to_host_init2(&set_pages_command->hdr,
					DXGK_VMBCOMMAND_SETEXISTINGSYSMEMPAGES,
					device->process->host_handle);
		set_pages_command->device = device->handle;
		set_pages_command->allocation = host_alloc->allocation;

		page_in = dxgalloc->pages;
		while (alloc_offset_in_pages < npages) {
			pfn = (u64 *)((char *)msg.msg +
				sizeof(*set_pages_command));
			pages_to_send = min(npages - alloc_offset_in_pages,
					    max_pfns_in_message);
			set_pages_command->num_pages = pages_to_send;
			set_pages_command->alloc_offset_in_pages =
				alloc_offset_in_pages;

			for (i = 0; i < pages_to_send; i++)
				*pfn++ = page_to_pfn(*page_in++);

			ret = dxgvmb_send_sync_msg_ntstatus(msg.channel,
							    msg.hdr,
							    msg.size);
			if (ret < 0) {
				DXG_ERR("failed set existing pages: %x", ret);
				break;
			}
			alloc_offset_in_pages += pages_to_send;
		}
	}

cleanup:
	if (kmem)
		vunmap(kmem);
	free_message(&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

static int
process_allocation_handles(struct dxgprocess *process,
			   struct dxgdevice *device,
			   struct d3dkmt_createallocation *args,
			   struct dxgkvmb_command_createallocation_return *res,
			   struct dxgallocation **dxgalloc,
			   struct dxgresource *resource)
{
	int ret = 0;
	int i = 0;
	int k;
	struct dxgkvmb_command_allocinfo_return *host_alloc;

	/*
	 * Assign handle to the internal objects, so VM bus messages will be
	 * sent to the host to free them during object destruction.
	 */
	if (args->flags.create_resource)
		resource->handle = res->resource;
	for (i = 0; i < args->alloc_count; i++) {
		host_alloc = &res->allocation_info[i];
		dxgalloc[i]->alloc_handle = host_alloc->allocation;
	}

	/*
	 * Assign handle to the handle table.
	 * In case of a failure all handles should be freed.
	 * When the function returns, the objects could be destroyed  by
	 * handle immediately.
	 */
	hmgrtable_lock(&process->handle_table, DXGLOCK_EXCL);
	if (args->flags.create_resource) {
		ret = hmgrtable_assign_handle(&process->handle_table, resource,
					      HMGRENTRY_TYPE_DXGRESOURCE,
					      res->resource);
		if (ret < 0) {
			DXG_ERR("failed to assign resource handle %x",
				res->resource.v);
			goto cleanup;
		} else {
			resource->handle_valid = 1;
		}
	}
	for (i = 0; i < args->alloc_count; i++) {
		host_alloc = &res->allocation_info[i];
		ret = hmgrtable_assign_handle(&process->handle_table,
					      dxgalloc[i],
					      HMGRENTRY_TYPE_DXGALLOCATION,
					      host_alloc->allocation);
		if (ret < 0) {
			DXG_ERR("failed assign alloc handle %x %d %d",
				host_alloc->allocation.v,
				args->alloc_count, i);
			break;
		}
		dxgalloc[i]->handle_valid = 1;
	}
	if (ret < 0) {
		if (args->flags.create_resource) {
			hmgrtable_free_handle(&process->handle_table,
					      HMGRENTRY_TYPE_DXGRESOURCE,
					      res->resource);
			resource->handle_valid = 0;
		}
		for (k = 0; k < i; k++) {
			host_alloc = &res->allocation_info[i];
			hmgrtable_free_handle(&process->handle_table,
					      HMGRENTRY_TYPE_DXGALLOCATION,
					      host_alloc->allocation);
			dxgalloc[i]->handle_valid = 0;
		}
	}

cleanup:

	hmgrtable_unlock(&process->handle_table, DXGLOCK_EXCL);

	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

static int
create_local_allocations(struct dxgprocess *process,
			 struct dxgdevice *device,
			 struct d3dkmt_createallocation *args,
			 struct d3dkmt_createallocation *__user input_args,
			 struct d3dddi_allocationinfo2 *alloc_info,
			 struct dxgkvmb_command_createallocation_return *result,
			 struct dxgresource *resource,
			 struct dxgallocation **dxgalloc,
			 u32 destroy_buffer_size)
{
	int i;
	int alloc_count = args->alloc_count;
	u8 *alloc_private_data = NULL;
	int ret = 0;
	int ret1;
	struct dxgkvmb_command_destroyallocation *destroy_buf;
	struct dxgvmbusmsg msg = {.hdr = NULL};

	ret = init_message(&msg, device->adapter, process,
			    destroy_buffer_size);
	if (ret)
		goto cleanup;
	destroy_buf = (void *)msg.msg;

	/* Prepare the command to destroy allocation in case of failure */
	command_vgpu_to_host_init2(&destroy_buf->hdr,
				   DXGK_VMBCOMMAND_DESTROYALLOCATION,
				   process->host_handle);
	destroy_buf->device = args->device;
	destroy_buf->resource = args->resource;
	destroy_buf->alloc_count = alloc_count;
	destroy_buf->flags.assume_not_in_use = 1;
	for (i = 0; i < alloc_count; i++) {
		DXG_TRACE("host allocation: %d %x",
			i, result->allocation_info[i].allocation.v);
		destroy_buf->allocations[i] =
		    result->allocation_info[i].allocation;
	}

	if (args->flags.create_resource) {
		DXG_TRACE("new resource: %x", result->resource.v);
		ret = copy_to_user(&input_args->resource, &result->resource,
				   sizeof(struct d3dkmthandle));
		if (ret) {
			DXG_ERR("failed to copy resource handle");
			ret = -EFAULT;
			goto cleanup;
		}
	}

	alloc_private_data = (u8 *) result +
	    sizeof(struct dxgkvmb_command_createallocation_return) +
	    sizeof(struct dxgkvmb_command_allocinfo_return) * (alloc_count - 1);

	for (i = 0; i < alloc_count; i++) {
		struct dxgkvmb_command_allocinfo_return *host_alloc;
		struct d3dddi_allocationinfo2 *user_alloc;

		host_alloc = &result->allocation_info[i];
		user_alloc = &alloc_info[i];
		dxgalloc[i]->num_pages =
		    host_alloc->allocation_size >> PAGE_SHIFT;
		if (user_alloc->sysmem) {
			ret = create_existing_sysmem(device, host_alloc,
						     dxgalloc[i],
						     args->flags.read_only != 0,
						     user_alloc->sysmem);
			if (ret < 0)
				goto cleanup;
		}
		dxgalloc[i]->cached = host_alloc->allocation_flags.cached;
		if (host_alloc->priv_drv_data_size) {
			ret = copy_to_user(user_alloc->priv_drv_data,
					   alloc_private_data,
					   host_alloc->priv_drv_data_size);
			if (ret) {
				DXG_ERR("failed to copy private data");
				ret = -EFAULT;
				goto cleanup;
			}
			alloc_private_data += host_alloc->priv_drv_data_size;
		}
		ret = copy_to_user(&args->allocation_info[i].allocation,
				   &host_alloc->allocation,
				   sizeof(struct d3dkmthandle));
		if (ret) {
			DXG_ERR("failed to copy alloc handle");
			ret = -EFAULT;
			goto cleanup;
		}
	}

	ret = copy_to_user(&input_args->global_share, &args->global_share,
			   sizeof(struct d3dkmthandle));
	if (ret) {
		DXG_ERR("failed to copy global share");
		ret = -EFAULT;
		goto cleanup;
	}

	ret = process_allocation_handles(process, device, args, result,
					 dxgalloc, resource);

cleanup:

	if (ret < 0) {
		/* Free local handles before freeing the handles in the host */
		dxgdevice_acquire_alloc_list_lock(device);
		if (dxgalloc)
			for (i = 0; i < alloc_count; i++)
				if (dxgalloc[i])
					dxgallocation_free_handle(dxgalloc[i]);
		if (resource && args->flags.create_resource)
			dxgresource_free_handle(resource);
		dxgdevice_release_alloc_list_lock(device);

		/* Destroy allocations in the host to unmap gpadls */
		ret1 = dxgvmb_send_sync_msg_ntstatus(msg.channel, msg.hdr,
						     msg.size);
		if (ret1 < 0)
			DXG_ERR("failed to destroy allocations: %x",
				ret1);

		dxgdevice_acquire_alloc_list_lock(device);
		if (dxgalloc) {
			for (i = 0; i < alloc_count; i++) {
				if (dxgalloc[i]) {
					dxgalloc[i]->alloc_handle.v = 0;
					dxgallocation_destroy(dxgalloc[i]);
					dxgalloc[i] = NULL;
				}
			}
		}
		if (resource && args->flags.create_resource) {
			/*
			 * Prevent the resource memory from freeing.
			 * It will be freed in the top level function.
			 */
			kref_get(&resource->resource_kref);
			dxgresource_destroy(resource);
		}
		dxgdevice_release_alloc_list_lock(device);
	}

	free_message(&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

int dxgvmb_send_create_allocation(struct dxgprocess *process,
				  struct dxgdevice *device,
				  struct d3dkmt_createallocation *args,
				  struct d3dkmt_createallocation *__user
				  input_args,
				  struct dxgresource *resource,
				  struct dxgallocation **dxgalloc,
				  struct d3dddi_allocationinfo2 *alloc_info,
				  struct d3dkmt_createstandardallocation
				  *standard_alloc)
{
	struct dxgkvmb_command_createallocation *command = NULL;
	struct dxgkvmb_command_createallocation_return *result = NULL;
	int ret = -EINVAL;
	int i;
	u32 result_size = 0;
	u32 cmd_size = 0;
	u32 destroy_buffer_size = 0;
	u32 priv_drv_data_size;
	struct dxgvmbusmsg msg = {.hdr = NULL};

	if (args->private_runtime_data_size >= DXG_MAX_VM_BUS_PACKET_SIZE ||
	    args->priv_drv_data_size >= DXG_MAX_VM_BUS_PACKET_SIZE) {
		ret = -EOVERFLOW;
		goto cleanup;
	}

	/*
	 * Preallocate the buffer, which will be used for destruction in case
	 * of a failure
	 */
	destroy_buffer_size = sizeof(struct dxgkvmb_command_destroyallocation) +
	    args->alloc_count * sizeof(struct d3dkmthandle);

	/* Compute the total private driver size */

	priv_drv_data_size = 0;

	for (i = 0; i < args->alloc_count; i++) {
		if (alloc_info[i].priv_drv_data_size >=
		    DXG_MAX_VM_BUS_PACKET_SIZE) {
			ret = -EOVERFLOW;
			goto cleanup;
		} else {
			priv_drv_data_size += alloc_info[i].priv_drv_data_size;
		}
		if (priv_drv_data_size >= DXG_MAX_VM_BUS_PACKET_SIZE) {
			ret = -EOVERFLOW;
			goto cleanup;
		}
	}

	/*
	 * Private driver data for the result includes only per allocation
	 * private data
	 */
	result_size = sizeof(struct dxgkvmb_command_createallocation_return) +
	    (args->alloc_count - 1) *
	    sizeof(struct dxgkvmb_command_allocinfo_return) +
	    priv_drv_data_size;
	result = vzalloc(result_size);
	if (result == NULL) {
		ret = -ENOMEM;
		goto cleanup;
	}

	/* Private drv data for the command includes the global private data */
	priv_drv_data_size += args->priv_drv_data_size;

	cmd_size = sizeof(struct dxgkvmb_command_createallocation) +
	    args->alloc_count *
	    sizeof(struct dxgkvmb_command_createallocation_allocinfo) +
	    args->private_runtime_data_size + priv_drv_data_size;
	if (cmd_size > DXG_MAX_VM_BUS_PACKET_SIZE) {
		ret = -EOVERFLOW;
		goto cleanup;
	}

	DXG_TRACE("command size, driver_data_size %d %d %ld %ld",
		cmd_size, priv_drv_data_size,
		sizeof(struct dxgkvmb_command_createallocation),
		sizeof(struct dxgkvmb_command_createallocation_allocinfo));

	ret = init_message(&msg, device->adapter, process,
			   cmd_size);
	if (ret)
		goto cleanup;
	command = (void *)msg.msg;

	command_vgpu_to_host_init2(&command->hdr,
				   DXGK_VMBCOMMAND_CREATEALLOCATION,
				   process->host_handle);
	command->device = args->device;
	command->flags = args->flags;
	command->resource = args->resource;
	command->private_runtime_resource_handle =
	    args->private_runtime_resource_handle;
	command->alloc_count = args->alloc_count;
	command->private_runtime_data_size = args->private_runtime_data_size;
	command->priv_drv_data_size = args->priv_drv_data_size;

	ret = copy_private_data(args, command, alloc_info, standard_alloc);
	if (ret < 0)
		goto cleanup;

	ret = dxgvmb_send_sync_msg(msg.channel, msg.hdr, msg.size,
				   result, result_size);
	if (ret < 0) {
		DXG_ERR("send_create_allocation failed %x", ret);
		goto cleanup;
	}

	ret = create_local_allocations(process, device, args, input_args,
				       alloc_info, result, resource, dxgalloc,
				       destroy_buffer_size);
cleanup:

	if (result)
		vfree(result);
	free_message(&msg);

	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

int dxgvmb_send_destroy_allocation(struct dxgprocess *process,
				   struct dxgdevice *device,
				   struct d3dkmt_destroyallocation2 *args,
				   struct d3dkmthandle *alloc_handles)
{
	struct dxgkvmb_command_destroyallocation *destroy_buffer;
	u32 destroy_buffer_size;
	int ret;
	int allocations_size = args->alloc_count * sizeof(struct d3dkmthandle);
	struct dxgvmbusmsg msg = {.hdr = NULL};

	destroy_buffer_size = sizeof(struct dxgkvmb_command_destroyallocation) +
	    allocations_size;

	ret = init_message(&msg, device->adapter, process,
			    destroy_buffer_size);
	if (ret)
		goto cleanup;
	destroy_buffer = (void *)msg.msg;

	command_vgpu_to_host_init2(&destroy_buffer->hdr,
				   DXGK_VMBCOMMAND_DESTROYALLOCATION,
				   process->host_handle);
	destroy_buffer->device = args->device;
	destroy_buffer->resource = args->resource;
	destroy_buffer->alloc_count = args->alloc_count;
	destroy_buffer->flags = args->flags;
	if (allocations_size)
		memcpy(destroy_buffer->allocations, alloc_handles,
		       allocations_size);

	ret = dxgvmb_send_sync_msg_ntstatus(msg.channel, msg.hdr, msg.size);

cleanup:

	free_message(&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

int dxgvmb_send_query_clock_calibration(struct dxgprocess *process,
					struct dxgadapter *adapter,
					struct d3dkmt_queryclockcalibration
					*args,
					struct d3dkmt_queryclockcalibration
					*__user inargs)
{
	struct dxgkvmb_command_queryclockcalibration *command;
	struct dxgkvmb_command_queryclockcalibration_return *result;
	int ret;
	struct dxgvmbusmsgres msg = {.hdr = NULL};

	ret = init_message_res(&msg, adapter, sizeof(*command),
				sizeof(*result));
	if (ret)
		goto cleanup;
	command = (void *)msg.msg;
	result = msg.res;

	command_vgpu_to_host_init2(&command->hdr,
				   DXGK_VMBCOMMAND_QUERYCLOCKCALIBRATION,
				   process->host_handle);
	command->args = *args;

	ret = dxgvmb_send_sync_msg(msg.channel, msg.hdr, msg.size,
				   result, sizeof(*result));
	if (ret < 0)
		goto cleanup;
	ret = copy_to_user(&inargs->clock_data, &result->clock_data,
			   sizeof(result->clock_data));
	if (ret) {
		DXG_ERR("failed to copy clock data");
		ret = -EFAULT;
		goto cleanup;
	}
	ret = ntstatus2int(result->status);

cleanup:
	free_message((struct dxgvmbusmsg *)&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

int dxgvmb_send_flush_heap_transitions(struct dxgprocess *process,
				       struct dxgadapter *adapter,
				       struct d3dkmt_flushheaptransitions *args)
{
	struct dxgkvmb_command_flushheaptransitions *command;
	int ret;
	struct dxgvmbusmsg msg = {.hdr = NULL};

	ret = init_message(&msg, adapter, process, sizeof(*command));
	if (ret)
		goto cleanup;
	command = (void *)msg.msg;
	command_vgpu_to_host_init2(&command->hdr,
				   DXGK_VMBCOMMAND_FLUSHHEAPTRANSITIONS,
				   process->host_handle);
	ret = dxgvmb_send_sync_msg_ntstatus(msg.channel, msg.hdr, msg.size);
cleanup:
	free_message(&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

int dxgvmb_send_invalidate_cache(struct dxgprocess *process,
				struct dxgadapter *adapter,
				struct d3dkmt_invalidatecache *args)
{
	struct dxgkvmb_command_invalidatecache *command;
	int ret;
	struct dxgvmbusmsg msg = {.hdr = NULL};

	ret = init_message(&msg, adapter, process, sizeof(*command));
	if (ret)
		goto cleanup;
	command = (void *)msg.msg;
	command_vgpu_to_host_init2(&command->hdr,
				   DXGK_VMBCOMMAND_INVALIDATECACHE,
				   process->host_handle);
	command->device = args->device;
	command->allocation = args->allocation;
	command->offset = args->offset;
	command->length = args->length;
	ret = dxgvmb_send_sync_msg_ntstatus(msg.channel, msg.hdr, msg.size);
cleanup:
	free_message(&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

int dxgvmb_send_is_feature_enabled(struct dxgadapter *adapter,
				   struct d3dkmt_isfeatureenabled *args)
{
	int ret;
	struct dxgkvmb_command_isfeatureenabled_return *result;
	struct dxgvmbusmsgres msg = {.hdr = NULL};
	int res_size = sizeof(*result);

	if (adapter) {
		struct dxgkvmb_command_isfeatureenabled *command;

		ret = init_message_res(&msg, adapter, sizeof(*command),
					res_size);
		if (ret)
			goto cleanup;
		command = (void *)msg.msg;
		command->feature_id = args->feature_id;
		result = msg.res;
		command_vgpu_to_host_init1(&command->hdr,
					   DXGK_VMBCOMMAND_ISFEATUREENABLED);
	} else {
		struct dxgkvmb_command_isfeatureenabled_gbl *command;

		ret = init_message_res(&msg, adapter, sizeof(*command),
					res_size);
		if (ret)
			goto cleanup;
		command = (void *)msg.msg;
		command->feature_id = args->feature_id;
		result = msg.res;
		command_vm_to_host_init1(&command->hdr,
				DXGK_VMBCOMMAND_ISFEATUREENABLED_GLOBAL);
	}
	ret = dxgvmb_send_sync_msg(msg.channel, msg.hdr, msg.size,
				   result, res_size);
	if (ret == 0) {
		ret = ntstatus2int(result->status);
		if (ret == 0)
			args->result = result->result;
		goto cleanup;
	}

cleanup:
	free_message((struct dxgvmbusmsg *)&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

int dxgvmb_send_query_alloc_residency(struct dxgprocess *process,
				      struct dxgadapter *adapter,
				      struct d3dkmt_queryallocationresidency
				      *args)
{
	int ret = -EINVAL;
	struct dxgkvmb_command_queryallocationresidency *command = NULL;
	u32 cmd_size = sizeof(*command);
	u32 alloc_size = 0;
	u32 result_allocation_size = 0;
	struct dxgkvmb_command_queryallocationresidency_return *result = NULL;
	u32 result_size = sizeof(*result);
	struct dxgvmbusmsgres msg = {.hdr = NULL};

	if (args->allocation_count > DXG_MAX_VM_BUS_PACKET_SIZE) {
		ret = -EINVAL;
		goto cleanup;
	}

	if (args->allocation_count) {
		alloc_size = args->allocation_count *
			     sizeof(struct d3dkmthandle);
		cmd_size += alloc_size;
		result_allocation_size = args->allocation_count *
		    sizeof(args->residency_status[0]);
	} else {
		result_allocation_size = sizeof(args->residency_status[0]);
	}
	result_size += result_allocation_size;

	ret = init_message_res(&msg, adapter, cmd_size, result_size);
	if (ret)
		goto cleanup;
	command = (void *)msg.msg;
	result = msg.res;

	command_vgpu_to_host_init2(&command->hdr,
				   DXGK_VMBCOMMAND_QUERYALLOCATIONRESIDENCY,
				   process->host_handle);
	command->args = *args;
	if (alloc_size) {
		ret = copy_from_user(&command[1], args->allocations,
				     alloc_size);
		if (ret) {
			DXG_ERR("failed to copy alloc handles");
			ret = -EFAULT;
			goto cleanup;
		}
	}

	ret = dxgvmb_send_sync_msg(msg.channel, msg.hdr, msg.size,
				   result, msg.res_size);
	if (ret < 0)
		goto cleanup;

	ret = ntstatus2int(result->status);
	if (ret < 0)
		goto cleanup;

	ret = copy_to_user(args->residency_status, &result[1],
			   result_allocation_size);
	if (ret) {
		DXG_ERR("failed to copy residency status");
		ret = -EFAULT;
	}

cleanup:
	free_message((struct dxgvmbusmsg *)&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

int dxgvmb_send_escape(struct dxgprocess *process,
		       struct dxgadapter *adapter,
		       struct d3dkmt_escape *args,
		       bool user_mode)
{
	int ret;
	struct dxgkvmb_command_escape *command = NULL;
	u32 cmd_size = sizeof(*command);
	struct dxgvmbusmsg msg = {.hdr = NULL};

	if (args->priv_drv_data_size > DXG_MAX_VM_BUS_PACKET_SIZE) {
		ret = -EINVAL;
		goto cleanup;
	}

	cmd_size = cmd_size - sizeof(args->priv_drv_data[0]) +
	    args->priv_drv_data_size;

	ret = init_message(&msg, adapter, process, cmd_size);
	if (ret)
		goto cleanup;
	command = (void *)msg.msg;
	command_vgpu_to_host_init2(&command->hdr,
				   DXGK_VMBCOMMAND_ESCAPE,
				   process->host_handle);
	command->adapter = args->adapter;
	command->device = args->device;
	command->type = args->type;
	command->flags = args->flags;
	command->priv_drv_data_size = args->priv_drv_data_size;
	command->context = args->context;
	if (args->priv_drv_data_size) {
		if (user_mode) {
			ret = copy_from_user(command->priv_drv_data,
					args->priv_drv_data,
					args->priv_drv_data_size);
			if (ret) {
				DXG_ERR("failed to copy priv data");
				ret = -EFAULT;
				goto cleanup;
			}
		} else {
			memcpy(command->priv_drv_data, args->priv_drv_data,
				args->priv_drv_data_size);
		}
	}

	ret = dxgvmb_send_sync_msg(msg.channel, msg.hdr, msg.size,
				   command->priv_drv_data,
				   args->priv_drv_data_size);
	if (ret < 0)
		goto cleanup;

	if (args->priv_drv_data_size) {
		if (user_mode) {
			ret = copy_to_user(args->priv_drv_data,
					command->priv_drv_data,
					args->priv_drv_data_size);
			if (ret) {
				DXG_ERR("failed to copy priv data");
				ret = -EINVAL;
			}
		} else {
			memcpy(args->priv_drv_data,
				command->priv_drv_data,
				args->priv_drv_data_size);
		}
	}

cleanup:
	free_message(&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

int dxgvmb_send_query_vidmem_info(struct dxgprocess *process,
				  struct dxgadapter *adapter,
				  struct d3dkmt_queryvideomemoryinfo *args,
				  struct d3dkmt_queryvideomemoryinfo *__user
				  output)
{
	int ret;
	struct dxgkvmb_command_queryvideomemoryinfo *command;
	struct dxgkvmb_command_queryvideomemoryinfo_return result = { };
	struct dxgvmbusmsg msg = {.hdr = NULL};

	ret = init_message(&msg, adapter, process, sizeof(*command));
	if (ret)
		goto cleanup;
	command = (void *)msg.msg;
	command_vgpu_to_host_init2(&command->hdr,
				   dxgk_vmbcommand_queryvideomemoryinfo,
				   process->host_handle);
	command->adapter = args->adapter;
	command->memory_segment_group = args->memory_segment_group;
	command->physical_adapter_index = args->physical_adapter_index;

	ret = dxgvmb_send_sync_msg(msg.channel, msg.hdr, msg.size,
				   &result, sizeof(result));
	if (ret < 0)
		goto cleanup;

	ret = copy_to_user(&output->budget, &result.budget,
			   sizeof(output->budget));
	if (ret) {
		DXG_ERR("failed to copy budget");
		ret = -EFAULT;
		goto cleanup;
	}
	ret = copy_to_user(&output->current_usage, &result.current_usage,
			   sizeof(output->current_usage));
	if (ret) {
		DXG_ERR("failed to copy current usage");
		ret = -EFAULT;
		goto cleanup;
	}
	ret = copy_to_user(&output->current_reservation,
			   &result.current_reservation,
			   sizeof(output->current_reservation));
	if (ret) {
		DXG_ERR("failed to copy reservation");
		ret = -EFAULT;
		goto cleanup;
	}
	ret = copy_to_user(&output->available_for_reservation,
			   &result.available_for_reservation,
			   sizeof(output->available_for_reservation));
	if (ret) {
		DXG_ERR("failed to copy avail reservation");
		ret = -EFAULT;
	}

cleanup:
	free_message(&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

int dxgvmb_send_get_device_state(struct dxgprocess *process,
				 struct dxgadapter *adapter,
				 struct d3dkmt_getdevicestate *args,
				 struct d3dkmt_getdevicestate *__user output)
{
	int ret;
	struct dxgkvmb_command_getdevicestate *command;
	struct dxgkvmb_command_getdevicestate_return result = { };
	struct dxgvmbusmsg msg = {.hdr = NULL};

	ret = init_message(&msg, adapter, process, sizeof(*command));
	if (ret)
		goto cleanup;
	command = (void *)msg.msg;

	command_vgpu_to_host_init2(&command->hdr,
				   DXGK_VMBCOMMAND_GETDEVICESTATE,
				   process->host_handle);
	command->args = *args;

	ret = dxgvmb_send_sync_msg(msg.channel, msg.hdr, msg.size,
				   &result, sizeof(result));
	if (ret < 0)
		goto cleanup;

	ret = ntstatus2int(result.status);
	if (ret < 0)
		goto cleanup;

	ret = copy_to_user(output, &result.args, sizeof(result.args));
	if (ret) {
		DXG_ERR("failed to copy output args");
		ret = -EFAULT;
	}

	if (args->state_type == _D3DKMT_DEVICESTATE_EXECUTION)
		args->execution_state = result.args.execution_state;

cleanup:
	free_message(&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

int dxgvmb_send_open_resource(struct dxgprocess *process,
			      struct dxgadapter *adapter,
			      struct d3dkmthandle device,
			      struct d3dkmthandle global_share,
			      u32 allocation_count,
			      u32 total_priv_drv_data_size,
			      struct d3dkmthandle *resource_handle,
			      struct d3dkmthandle *alloc_handles)
{
	struct dxgkvmb_command_openresource *command;
	struct dxgkvmb_command_openresource_return *result;
	struct d3dkmthandle *handles;
	int ret;
	int i;
	u32 result_size = allocation_count * sizeof(struct d3dkmthandle) +
			   sizeof(*result);
	struct dxgvmbusmsgres msg = {.hdr = NULL};

	ret = init_message_res(&msg, adapter,  sizeof(*command), result_size);
	if (ret)
		goto cleanup;
	command = msg.msg;
	result = msg.res;

	command_vgpu_to_host_init2(&command->hdr, DXGK_VMBCOMMAND_OPENRESOURCE,
				   process->host_handle);
	command->device = device;
	command->nt_security_sharing = 1;
	command->global_share = global_share;
	command->allocation_count = allocation_count;
	command->total_priv_drv_data_size = total_priv_drv_data_size;

	ret = dxgvmb_send_sync_msg(msg.channel, msg.hdr, msg.size,
				   result, msg.res_size);
	if (ret < 0)
		goto cleanup;

	ret = ntstatus2int(result->status);
	if (ret < 0)
		goto cleanup;

	*resource_handle = result->resource;
	handles = (struct d3dkmthandle *) &result[1];
	for (i = 0; i < allocation_count; i++)
		alloc_handles[i] = handles[i];

cleanup:
	free_message((struct dxgvmbusmsg *)&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

int dxgvmb_send_get_stdalloc_data(struct dxgdevice *device,
				  enum d3dkmdt_standardallocationtype alloctype,
				  struct d3dkmdt_gdisurfacedata *alloc_data,
				  u32 physical_adapter_index,
				  u32 *alloc_priv_driver_size,
				  void *priv_alloc_data,
				  u32 *res_priv_data_size,
				  void *priv_res_data)
{
	struct dxgkvmb_command_getstandardallocprivdata *command;
	struct dxgkvmb_command_getstandardallocprivdata_return *result = NULL;
	u32 result_size = sizeof(*result);
	int ret;
	struct dxgvmbusmsgres msg = {.hdr = NULL};

	if (priv_alloc_data)
		result_size += *alloc_priv_driver_size;
	if (priv_res_data)
		result_size += *res_priv_data_size;
	ret = init_message_res(&msg, device->adapter,
			       sizeof(*command), result_size);
	if (ret)
		goto cleanup;
	command = msg.msg;
	result = msg.res;

	command_vgpu_to_host_init2(&command->hdr,
			DXGK_VMBCOMMAND_DDIGETSTANDARDALLOCATIONDRIVERDATA,
			device->process->host_handle);

	command->alloc_type = alloctype;
	command->priv_driver_data_size = *alloc_priv_driver_size;
	command->physical_adapter_index = physical_adapter_index;
	command->priv_driver_resource_size = *res_priv_data_size;
	switch (alloctype) {
	case _D3DKMDT_STANDARDALLOCATION_GDISURFACE:
		command->gdi_surface = *alloc_data;
		break;
	case _D3DKMDT_STANDARDALLOCATION_SHAREDPRIMARYSURFACE:
	case _D3DKMDT_STANDARDALLOCATION_SHADOWSURFACE:
	case _D3DKMDT_STANDARDALLOCATION_STAGINGSURFACE:
	default:
		DXG_ERR("Invalid standard alloc type");
		goto cleanup;
	}

	ret = dxgvmb_send_sync_msg(msg.channel, msg.hdr, msg.size,
				   result, msg.res_size);
	if (ret < 0)
		goto cleanup;

	ret = ntstatus2int(result->status);
	if (ret < 0)
		goto cleanup;

	if (*alloc_priv_driver_size &&
	    result->priv_driver_data_size != *alloc_priv_driver_size) {
		DXG_ERR("Priv data size mismatch");
		goto cleanup;
	}
	if (*res_priv_data_size &&
	    result->priv_driver_resource_size != *res_priv_data_size) {
		DXG_ERR("Resource priv data size mismatch");
		goto cleanup;
	}
	*alloc_priv_driver_size = result->priv_driver_data_size;
	*res_priv_data_size = result->priv_driver_resource_size;
	if (priv_alloc_data) {
		memcpy(priv_alloc_data, &result[1],
		       result->priv_driver_data_size);
	}
	if (priv_res_data) {
		memcpy(priv_res_data,
		       (char *)(&result[1]) + result->priv_driver_data_size,
		       result->priv_driver_resource_size);
	}

cleanup:

	free_message((struct dxgvmbusmsg *)&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

int dxgvmb_send_make_resident(struct dxgprocess *process,
			      struct dxgadapter *adapter,
			      struct d3dddi_makeresident *args)
{
	int ret;
	u32 cmd_size;
	struct dxgkvmb_command_makeresident_return result = { };
	struct dxgkvmb_command_makeresident *command = NULL;
	struct dxgvmbusmsg msg = {.hdr = NULL};

	cmd_size = (args->alloc_count - 1) * sizeof(struct d3dkmthandle) +
		   sizeof(struct dxgkvmb_command_makeresident);

	ret = init_message(&msg, adapter, process, cmd_size);
	if (ret)
		goto cleanup;
	command = (void *)msg.msg;

	ret = copy_from_user(command->allocations, args->allocation_list,
			     args->alloc_count *
			     sizeof(struct d3dkmthandle));
	if (ret) {
		DXG_ERR("failed to copy alloc handles");
		ret = -EFAULT;
		goto cleanup;
	}
	command_vgpu_to_host_init2(&command->hdr,
				   DXGK_VMBCOMMAND_MAKERESIDENT,
				   process->host_handle);
	command->alloc_count = args->alloc_count;
	command->paging_queue = args->paging_queue;
	command->flags = args->flags;

	ret = dxgvmb_send_sync_msg(msg.channel, msg.hdr, msg.size,
				   &result, sizeof(result));
	if (ret < 0) {
		DXG_ERR("send_make_resident failed %x", ret);
		goto cleanup;
	}

	args->paging_fence_value = result.paging_fence_value;
	args->num_bytes_to_trim = result.num_bytes_to_trim;
	ret = ntstatus2int(result.status);

cleanup:

	free_message(&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

int dxgvmb_send_evict(struct dxgprocess *process,
		      struct dxgadapter *adapter,
		      struct d3dkmt_evict *args)
{
	int ret;
	u32 cmd_size;
	struct dxgkvmb_command_evict_return result = { };
	struct dxgkvmb_command_evict *command = NULL;
	struct dxgvmbusmsg msg = {.hdr = NULL};

	cmd_size = (args->alloc_count - 1) * sizeof(struct d3dkmthandle) +
	    sizeof(struct dxgkvmb_command_evict);
	ret = init_message(&msg, adapter, process, cmd_size);
	if (ret)
		goto cleanup;
	command = (void *)msg.msg;
	ret = copy_from_user(command->allocations, args->allocations,
			     args->alloc_count *
			     sizeof(struct d3dkmthandle));
	if (ret) {
		DXG_ERR("failed to copy alloc handles");
		ret = -EFAULT;
		goto cleanup;
	}
	command_vgpu_to_host_init2(&command->hdr,
				   DXGK_VMBCOMMAND_EVICT, process->host_handle);
	command->alloc_count = args->alloc_count;
	command->device = args->device;
	command->flags = args->flags;

	ret = dxgvmb_send_sync_msg(msg.channel, msg.hdr, msg.size,
				   &result, sizeof(result));
	if (ret < 0) {
		DXG_ERR("send_evict failed %x", ret);
		goto cleanup;
	}
	args->num_bytes_to_trim = result.num_bytes_to_trim;

cleanup:

	free_message(&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

int dxgvmb_send_submit_command(struct dxgprocess *process,
			       struct dxgadapter *adapter,
			       struct d3dkmt_submitcommand *args)
{
	int ret;
	u32 cmd_size;
	struct dxgkvmb_command_submitcommand *command;
	u32 hbufsize = args->num_history_buffers * sizeof(struct d3dkmthandle);
	struct dxgvmbusmsg msg = {.hdr = NULL};
	struct dxgglobal *dxgglobal = dxggbl();

	cmd_size = sizeof(struct dxgkvmb_command_submitcommand) +
	    hbufsize + args->priv_drv_data_size;

	ret = init_message(&msg, adapter, process, cmd_size);
	if (ret)
		goto cleanup;
	command = (void *)msg.msg;

	ret = copy_from_user(&command[1], args->history_buffer_array,
			     hbufsize);
	if (ret) {
		DXG_ERR(" failed to copy history buffer");
		ret = -EFAULT;
		goto cleanup;
	}
	ret = copy_from_user((u8 *) &command[1] + hbufsize,
			     args->priv_drv_data, args->priv_drv_data_size);
	if (ret) {
		DXG_ERR("failed to copy history priv data");
		ret = -EFAULT;
		goto cleanup;
	}

	command_vgpu_to_host_init2(&command->hdr,
				   DXGK_VMBCOMMAND_SUBMITCOMMAND,
				   process->host_handle);
	command->args = *args;

	if (dxgglobal->async_msg_enabled) {
		command->hdr.async_msg = 1;
		ret = dxgvmb_send_async_msg(msg.channel, msg.hdr, msg.size);
	} else {
		ret = dxgvmb_send_sync_msg_ntstatus(msg.channel, msg.hdr,
						    msg.size);
	}

cleanup:

	free_message(&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

int dxgvmb_send_map_gpu_va(struct dxgprocess *process,
			   struct d3dkmthandle device,
			   struct dxgadapter *adapter,
			   struct d3dddi_mapgpuvirtualaddress *args)
{
	struct dxgkvmb_command_mapgpuvirtualaddress *command;
	struct dxgkvmb_command_mapgpuvirtualaddress_return result;
	int ret;
	struct dxgvmbusmsg msg = {.hdr = NULL};

	ret = init_message(&msg, adapter, process, sizeof(*command));
	if (ret)
		goto cleanup;
	command = (void *)msg.msg;

	command_vgpu_to_host_init2(&command->hdr,
				   DXGK_VMBCOMMAND_MAPGPUVIRTUALADDRESS,
				   process->host_handle);
	command->args = *args;
	command->device = device;

	ret = dxgvmb_send_sync_msg(msg.channel, msg.hdr, msg.size, &result,
				   sizeof(result));
	if (ret < 0)
		goto cleanup;
	args->virtual_address = result.virtual_address;
	args->paging_fence_value = result.paging_fence_value;
	ret = ntstatus2int(result.status);

cleanup:

	free_message(&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

int dxgvmb_send_reserve_gpu_va(struct dxgprocess *process,
			       struct dxgadapter *adapter,
			       struct d3dddi_reservegpuvirtualaddress *args)
{
	struct dxgkvmb_command_reservegpuvirtualaddress *command;
	struct dxgkvmb_command_reservegpuvirtualaddress_return result;
	int ret;
	struct dxgvmbusmsg msg = {.hdr = NULL};

	ret = init_message(&msg, adapter, process, sizeof(*command));
	if (ret)
		goto cleanup;
	command = (void *)msg.msg;

	command_vgpu_to_host_init2(&command->hdr,
				   DXGK_VMBCOMMAND_RESERVEGPUVIRTUALADDRESS,
				   process->host_handle);
	command->args = *args;

	ret = dxgvmb_send_sync_msg(msg.channel, msg.hdr, msg.size, &result,
				   sizeof(result));
	args->virtual_address = result.virtual_address;

cleanup:
	free_message(&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

int dxgvmb_send_free_gpu_va(struct dxgprocess *process,
			    struct dxgadapter *adapter,
			    struct d3dkmt_freegpuvirtualaddress *args)
{
	struct dxgkvmb_command_freegpuvirtualaddress *command;
	int ret;
	struct dxgvmbusmsg msg = {.hdr = NULL};

	ret = init_message(&msg, adapter, process, sizeof(*command));
	if (ret)
		goto cleanup;
	command = (void *)msg.msg;

	command_vgpu_to_host_init2(&command->hdr,
				   DXGK_VMBCOMMAND_FREEGPUVIRTUALADDRESS,
				   process->host_handle);
	command->args = *args;

	ret = dxgvmb_send_sync_msg_ntstatus(msg.channel, msg.hdr, msg.size);

cleanup:
	free_message(&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

int dxgvmb_send_update_gpu_va(struct dxgprocess *process,
			      struct dxgadapter *adapter,
			      struct d3dkmt_updategpuvirtualaddress *args)
{
	struct dxgkvmb_command_updategpuvirtualaddress *command;
	u32 cmd_size;
	u32 op_size;
	int ret;
	struct dxgvmbusmsg msg = {.hdr = NULL};

	if (args->num_operations == 0 ||
	    (DXG_MAX_VM_BUS_PACKET_SIZE /
	     sizeof(struct d3dddi_updategpuvirtualaddress_operation)) <
	    args->num_operations) {
		ret = -EINVAL;
		DXG_ERR("Invalid number of operations: %d",
			args->num_operations);
		goto cleanup;
	}

	op_size = args->num_operations *
	    sizeof(struct d3dddi_updategpuvirtualaddress_operation);
	cmd_size = sizeof(struct dxgkvmb_command_updategpuvirtualaddress) +
	    op_size - sizeof(args->operations[0]);

	ret = init_message(&msg, adapter, process, cmd_size);
	if (ret)
		goto cleanup;
	command = (void *)msg.msg;

	command_vgpu_to_host_init2(&command->hdr,
				   DXGK_VMBCOMMAND_UPDATEGPUVIRTUALADDRESS,
				   process->host_handle);
	command->fence_value = args->fence_value;
	command->device = args->device;
	command->context = args->context;
	command->fence_object = args->fence_object;
	command->num_operations = args->num_operations;
	command->flags = args->flags.value;
	ret = copy_from_user(command->operations, args->operations,
			     op_size);
	if (ret) {
		DXG_ERR("failed to copy operations");
		ret = -EFAULT;
		goto cleanup;
	}

	ret = dxgvmb_send_sync_msg_ntstatus(msg.channel, msg.hdr, msg.size);

cleanup:
	free_message(&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

static void set_result(struct d3dkmt_createsynchronizationobject2 *args,
		       u64 fence_gpu_va, u8 *va)
{
	args->info.periodic_monitored_fence.fence_gpu_virtual_address =
	    fence_gpu_va;
	args->info.periodic_monitored_fence.fence_cpu_virtual_address = va;
}

int
dxgvmb_send_create_sync_object(struct dxgprocess *process,
			       struct dxgadapter *adapter,
			       struct d3dkmt_createsynchronizationobject2 *args,
			       struct dxgsyncobject *syncobj)
{
	struct dxgkvmb_command_createsyncobject_return result = { };
	struct dxgkvmb_command_createsyncobject *command;
	int ret;
	u8 *va = 0;
	struct dxgvmbusmsg msg = {.hdr = NULL};

	ret = init_message(&msg, adapter, process, sizeof(*command));
	if (ret)
		goto cleanup;
	command = (void *)msg.msg;

	command_vgpu_to_host_init2(&command->hdr,
				   DXGK_VMBCOMMAND_CREATESYNCOBJECT,
				   process->host_handle);
	command->args = *args;
	command->client_hint = 1;	/* CLIENTHINT_UMD */

	ret = dxgvmb_send_sync_msg(msg.channel, msg.hdr, msg.size, &result,
				   sizeof(result));
	if (ret < 0) {
		DXG_ERR("failed %d", ret);
		goto cleanup;
	}
	args->sync_object = result.sync_object;
	if (syncobj->shared) {
		if (result.global_sync_object.v == 0) {
			DXG_ERR("shared handle is 0");
			ret = -EINVAL;
			goto cleanup;
		}
		args->info.shared_handle = result.global_sync_object;
	}

	if (syncobj->monitored_fence) {
		va = dxg_map_iospace(result.fence_storage_address, PAGE_SIZE,
				     PROT_READ | PROT_WRITE, true);
		if (va == NULL) {
			ret = -ENOMEM;
			goto cleanup;
		}
		if (args->info.type == _D3DDDI_MONITORED_FENCE) {
			args->info.monitored_fence.fence_gpu_virtual_address =
			    result.fence_gpu_va;
			args->info.monitored_fence.fence_cpu_virtual_address =
			    va;
			{
				unsigned long value;

				DXG_TRACE("fence cpu va: %p", va);
				ret = copy_from_user(&value, va,
						     sizeof(u64));
				if (ret) {
					DXG_ERR("failed to read fence");
					ret = -EFAULT;
				} else {
					DXG_TRACE("fence value:%lx",
						value);
				}
			}
		} else {
			set_result(args, result.fence_gpu_va, va);
		}
		syncobj->mapped_address = va;
	}

cleanup:
	free_message(&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

int dxgvmb_send_signal_sync_object(struct dxgprocess *process,
				   struct dxgadapter *adapter,
				   struct d3dddicb_signalflags flags,
				   u64 legacy_fence_value,
				   struct d3dkmthandle context,
				   u32 object_count,
				   struct d3dkmthandle __user *objects,
				   u32 context_count,
				   struct d3dkmthandle __user *contexts,
				   u32 fence_count,
				   u64 __user *fences,
				   struct eventfd_ctx *cpu_event_handle,
				   struct d3dkmthandle device)
{
	int ret;
	struct dxgkvmb_command_signalsyncobject *command;
	u32 object_size = object_count * sizeof(struct d3dkmthandle);
	u32 context_size = context_count * sizeof(struct d3dkmthandle);
	u32 fence_size = fences ? fence_count * sizeof(u64) : 0;
	u8 *current_pos;
	u32 cmd_size = sizeof(struct dxgkvmb_command_signalsyncobject) +
	    object_size + context_size + fence_size;
	struct dxgvmbusmsg msg = {.hdr = NULL};
	struct dxgglobal *dxgglobal = dxggbl();

	if (context.v)
		cmd_size += sizeof(struct d3dkmthandle);

	ret = init_message(&msg, adapter, process, cmd_size);
	if (ret)
		goto cleanup;
	command = (void *)msg.msg;

	command_vgpu_to_host_init2(&command->hdr,
				   DXGK_VMBCOMMAND_SIGNALSYNCOBJECT,
				   process->host_handle);

	if (flags.enqueue_cpu_event)
		command->cpu_event_handle = (u64) cpu_event_handle;
	else
		command->device = device;
	command->flags = flags;
	command->fence_value = legacy_fence_value;
	command->object_count = object_count;
	command->context_count = context_count;
	current_pos = (u8 *) &command[1];
	ret = copy_from_user(current_pos, objects, object_size);
	if (ret) {
		DXG_ERR("Failed to read objects %p %d",
			objects, object_size);
		ret = -EFAULT;
		goto cleanup;
	}
	current_pos += object_size;
	if (context.v) {
		command->context_count++;
		*(struct d3dkmthandle *) current_pos = context;
		current_pos += sizeof(struct d3dkmthandle);
	}
	if (context_size) {
		ret = copy_from_user(current_pos, contexts, context_size);
		if (ret) {
			DXG_ERR("Failed to read contexts %p %d",
				contexts, context_size);
			ret = -EFAULT;
			goto cleanup;
		}
		current_pos += context_size;
	}
	if (fence_size) {
		ret = copy_from_user(current_pos, fences, fence_size);
		if (ret) {
			DXG_ERR("Failed to read fences %p %d",
				fences, fence_size);
			ret = -EFAULT;
			goto cleanup;
		}
	}

	if (dxgglobal->async_msg_enabled) {
		command->hdr.async_msg = 1;
		ret = dxgvmb_send_async_msg(msg.channel, msg.hdr, msg.size);
	} else {
		ret = dxgvmb_send_sync_msg_ntstatus(msg.channel, msg.hdr,
						    msg.size);
	}

cleanup:
	free_message(&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

int dxgvmb_send_wait_sync_object_cpu(struct dxgprocess *process,
				     struct dxgadapter *adapter,
				     struct
				     d3dkmt_waitforsynchronizationobjectfromcpu
				     *args,
				     bool user_address,
				     u64 cpu_event)
{
	int ret = -EINVAL;
	struct dxgkvmb_command_waitforsyncobjectfromcpu *command;
	u32 object_size = args->object_count * sizeof(struct d3dkmthandle);
	u32 fence_size = args->object_count * sizeof(u64);
	u8 *current_pos;
	u32 cmd_size = sizeof(*command) + object_size + fence_size;
	struct dxgvmbusmsg msg = {.hdr = NULL};

	ret = init_message(&msg, adapter, process, cmd_size);
	if (ret)
		goto cleanup;
	command = (void *)msg.msg;

	command_vgpu_to_host_init2(&command->hdr,
				   DXGK_VMBCOMMAND_WAITFORSYNCOBJECTFROMCPU,
				   process->host_handle);
	command->device = args->device;
	command->flags = args->flags;
	command->object_count = args->object_count;
	command->guest_event_pointer = (u64) cpu_event;
	current_pos = (u8 *) &command[1];

	if (user_address) {
		ret = copy_from_user(current_pos, args->objects, object_size);
		if (ret) {
			DXG_ERR("failed to copy objects");
			ret = -EFAULT;
			goto cleanup;
		}
		current_pos += object_size;
		ret = copy_from_user(current_pos, args->fence_values,
					fence_size);
		if (ret) {
			DXG_ERR("failed to copy fences");
			ret = -EFAULT;
			goto cleanup;
		}
	} else {
		memcpy(current_pos, args->objects, object_size);
		current_pos += object_size;
		memcpy(current_pos, args->fence_values, fence_size);
	}

	ret = dxgvmb_send_sync_msg_ntstatus(msg.channel, msg.hdr, msg.size);

cleanup:
	free_message(&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

int dxgvmb_send_wait_sync_object_gpu(struct dxgprocess *process,
				     struct dxgadapter *adapter,
				     struct d3dkmthandle context,
				     u32 object_count,
				     struct d3dkmthandle *objects,
				     u64 *fences,
				     bool legacy_fence)
{
	int ret;
	struct dxgkvmb_command_waitforsyncobjectfromgpu *command;
	u32 fence_size = object_count * sizeof(u64);
	u32 object_size = object_count * sizeof(struct d3dkmthandle);
	u8 *current_pos;
	u32 cmd_size = object_size + fence_size - sizeof(u64) +
	    sizeof(struct dxgkvmb_command_waitforsyncobjectfromgpu);
	struct dxgvmbusmsg msg = {.hdr = NULL};
	struct dxgglobal *dxgglobal = dxggbl();

	if (object_count == 0 || object_count > D3DDDI_MAX_OBJECT_WAITED_ON) {
		ret = -EINVAL;
		goto cleanup;
	}
	ret = init_message(&msg, adapter, process, cmd_size);
	if (ret)
		goto cleanup;
	command = (void *)msg.msg;

	command_vgpu_to_host_init2(&command->hdr,
				   DXGK_VMBCOMMAND_WAITFORSYNCOBJECTFROMGPU,
				   process->host_handle);
	command->context = context;
	command->object_count = object_count;
	command->legacy_fence_object = legacy_fence;
	current_pos = (u8 *) command->fence_values;
	memcpy(current_pos, fences, fence_size);
	current_pos += fence_size;
	memcpy(current_pos, objects, object_size);

	if (dxgglobal->async_msg_enabled) {
		command->hdr.async_msg = 1;
		ret = dxgvmb_send_async_msg(msg.channel, msg.hdr, msg.size);
	} else {
		ret = dxgvmb_send_sync_msg_ntstatus(msg.channel, msg.hdr,
						    msg.size);
	}

cleanup:
	free_message(&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

int dxgvmb_send_lock2(struct dxgprocess *process,
		      struct dxgadapter *adapter,
		      struct d3dkmt_lock2 *args,
		      struct d3dkmt_lock2 *__user outargs)
{
	int ret;
	struct dxgkvmb_command_lock2 *command;
	struct dxgkvmb_command_lock2_return result = { };
	struct dxgallocation *alloc = NULL;
	struct dxgvmbusmsg msg = {.hdr = NULL};

	ret = init_message(&msg, adapter, process, sizeof(*command));
	if (ret)
		goto cleanup;
	command = (void *)msg.msg;

	command_vgpu_to_host_init2(&command->hdr,
				   DXGK_VMBCOMMAND_LOCK2, process->host_handle);
	command->args = *args;

	ret = dxgvmb_send_sync_msg(msg.channel, msg.hdr, msg.size,
				   &result, sizeof(result));
	if (ret < 0)
		goto cleanup;

	ret = ntstatus2int(result.status);
	if (ret < 0)
		goto cleanup;

	hmgrtable_lock(&process->handle_table, DXGLOCK_EXCL);
	alloc = hmgrtable_get_object_by_type(&process->handle_table,
					     HMGRENTRY_TYPE_DXGALLOCATION,
					     args->allocation);
	if (alloc == NULL) {
		DXG_ERR("invalid alloc");
		ret = -EINVAL;
	} else {
		if (alloc->cpu_address) {
			args->data = alloc->cpu_address;
			if (alloc->cpu_address_mapped)
				alloc->cpu_address_refcount++;
		} else {
			u64 offset = (u64)result.cpu_visible_buffer_offset;

			args->data = dxg_map_iospace(offset,
					alloc->num_pages << PAGE_SHIFT,
					PROT_READ | PROT_WRITE, alloc->cached);
			if (args->data) {
				alloc->cpu_address_refcount = 1;
				alloc->cpu_address_mapped = true;
				alloc->cpu_address = args->data;
			}
		}
		if (args->data == NULL) {
			ret = -ENOMEM;
		} else {
			ret = copy_to_user(&outargs->data, &args->data,
					   sizeof(args->data));
			if (ret) {
				DXG_ERR("failed to copy data");
				ret = -EFAULT;
				alloc->cpu_address_refcount--;
				if (alloc->cpu_address_refcount == 0) {
					dxg_unmap_iospace(alloc->cpu_address,
					   alloc->num_pages << PAGE_SHIFT);
					alloc->cpu_address_mapped = false;
					alloc->cpu_address = NULL;
				}
			}
		}
	}
	hmgrtable_unlock(&process->handle_table, DXGLOCK_EXCL);

cleanup:
	free_message(&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

int dxgvmb_send_unlock2(struct dxgprocess *process,
			struct dxgadapter *adapter,
			struct d3dkmt_unlock2 *args)
{
	int ret;
	struct dxgkvmb_command_unlock2 *command;
	struct dxgvmbusmsg msg = {.hdr = NULL};

	ret = init_message(&msg, adapter, process, sizeof(*command));
	if (ret)
		goto cleanup;
	command = (void *)msg.msg;

	command_vgpu_to_host_init2(&command->hdr,
				   DXGK_VMBCOMMAND_UNLOCK2,
				   process->host_handle);
	command->args = *args;

	ret = dxgvmb_send_sync_msg_ntstatus(msg.channel, msg.hdr, msg.size);

cleanup:
	free_message(&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

int dxgvmb_send_update_alloc_property(struct dxgprocess *process,
				      struct dxgadapter *adapter,
				      struct d3dddi_updateallocproperty *args,
				      struct d3dddi_updateallocproperty *__user
				      inargs)
{
	int ret;
	int ret1;
	struct dxgkvmb_command_updateallocationproperty *command;
	struct dxgkvmb_command_updateallocationproperty_return result = { };
	struct dxgvmbusmsg msg = {.hdr = NULL};

	ret = init_message(&msg, adapter, process, sizeof(*command));
	if (ret)
		goto cleanup;
	command = (void *)msg.msg;

	command_vgpu_to_host_init2(&command->hdr,
				   DXGK_VMBCOMMAND_UPDATEALLOCATIONPROPERTY,
				   process->host_handle);
	command->args = *args;

	ret = dxgvmb_send_sync_msg(msg.channel, msg.hdr, msg.size,
				   &result, sizeof(result));

	if (ret < 0)
		goto cleanup;
	ret = ntstatus2int(result.status);
	/* STATUS_PENING is a success code > 0 */
	if (ret == STATUS_PENDING) {
		ret1 = copy_to_user(&inargs->paging_fence_value,
				    &result.paging_fence_value,
				    sizeof(u64));
		if (ret1) {
			DXG_ERR("failed to copy paging fence");
			ret = -EFAULT;
		}
	}
cleanup:
	free_message(&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

int dxgvmb_send_mark_device_as_error(struct dxgprocess *process,
				     struct dxgadapter *adapter,
				     struct d3dkmt_markdeviceaserror *args)
{
	struct dxgkvmb_command_markdeviceaserror *command;
	int ret;
	struct dxgvmbusmsg msg = {.hdr = NULL};

	ret = init_message(&msg, adapter, process, sizeof(*command));
	if (ret)
		goto cleanup;
	command = (void *)msg.msg;

	command_vgpu_to_host_init2(&command->hdr,
				   DXGK_VMBCOMMAND_MARKDEVICEASERROR,
				   process->host_handle);
	command->args = *args;
	ret = dxgvmb_send_sync_msg_ntstatus(msg.channel, msg.hdr, msg.size);
cleanup:
	free_message(&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

int dxgvmb_send_set_allocation_priority(struct dxgprocess *process,
				struct dxgadapter *adapter,
				struct d3dkmt_setallocationpriority *args)
{
	u32 cmd_size = sizeof(struct dxgkvmb_command_setallocationpriority);
	u32 alloc_size = 0;
	u32 priority_size = 0;
	struct dxgkvmb_command_setallocationpriority *command;
	int ret;
	struct d3dkmthandle *allocations;
	struct dxgvmbusmsg msg = {.hdr = NULL};

	if (args->allocation_count > DXG_MAX_VM_BUS_PACKET_SIZE) {
		ret = -EINVAL;
		goto cleanup;
	}
	if (args->resource.v) {
		priority_size = sizeof(u32);
		if (args->allocation_count != 0) {
			ret = -EINVAL;
			goto cleanup;
		}
	} else {
		if (args->allocation_count == 0) {
			ret = -EINVAL;
			goto cleanup;
		}
		alloc_size = args->allocation_count *
			     sizeof(struct d3dkmthandle);
		cmd_size += alloc_size;
		priority_size = sizeof(u32) * args->allocation_count;
	}
	cmd_size += priority_size;

	ret = init_message(&msg, adapter, process, cmd_size);
	if (ret)
		goto cleanup;
	command = (void *)msg.msg;

	command_vgpu_to_host_init2(&command->hdr,
				   DXGK_VMBCOMMAND_SETALLOCATIONPRIORITY,
				   process->host_handle);
	command->device = args->device;
	command->allocation_count = args->allocation_count;
	command->resource = args->resource;
	allocations = (struct d3dkmthandle *) &command[1];
	ret = copy_from_user(allocations, args->allocation_list,
			     alloc_size);
	if (ret) {
		DXG_ERR("failed to copy alloc handle");
		ret = -EFAULT;
		goto cleanup;
	}
	ret = copy_from_user((u8 *) allocations + alloc_size,
				args->priorities, priority_size);
	if (ret) {
		DXG_ERR("failed to copy alloc priority");
		ret = -EFAULT;
		goto cleanup;
	}

	ret = dxgvmb_send_sync_msg_ntstatus(msg.channel, msg.hdr, msg.size);

cleanup:
	free_message(&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

int dxgvmb_send_get_allocation_priority(struct dxgprocess *process,
				struct dxgadapter *adapter,
				struct d3dkmt_getallocationpriority *args)
{
	u32 cmd_size = sizeof(struct dxgkvmb_command_getallocationpriority);
	u32 result_size;
	u32 alloc_size = 0;
	u32 priority_size = 0;
	struct dxgkvmb_command_getallocationpriority *command;
	struct dxgkvmb_command_getallocationpriority_return *result;
	int ret;
	struct d3dkmthandle *allocations;
	struct dxgvmbusmsgres msg = {.hdr = NULL};

	if (args->allocation_count > DXG_MAX_VM_BUS_PACKET_SIZE) {
		ret = -EINVAL;
		goto cleanup;
	}
	if (args->resource.v) {
		priority_size = sizeof(u32);
		if (args->allocation_count != 0) {
			ret = -EINVAL;
			goto cleanup;
		}
	} else {
		if (args->allocation_count == 0) {
			ret = -EINVAL;
			goto cleanup;
		}
		alloc_size = args->allocation_count *
			sizeof(struct d3dkmthandle);
		cmd_size += alloc_size;
		priority_size = sizeof(u32) * args->allocation_count;
	}
	result_size = sizeof(*result) + priority_size;

	ret = init_message_res(&msg, adapter, cmd_size, result_size);
	if (ret)
		goto cleanup;
	command = (void *)msg.msg;
	result = msg.res;

	command_vgpu_to_host_init2(&command->hdr,
				   DXGK_VMBCOMMAND_GETALLOCATIONPRIORITY,
				   process->host_handle);
	command->device = args->device;
	command->allocation_count = args->allocation_count;
	command->resource = args->resource;
	allocations = (struct d3dkmthandle *) &command[1];
	ret = copy_from_user(allocations, args->allocation_list,
			     alloc_size);
	if (ret) {
		DXG_ERR("failed to copy alloc handles");
		ret = -EFAULT;
		goto cleanup;
	}

	ret = dxgvmb_send_sync_msg(msg.channel, msg.hdr,
				   msg.size + msg.res_size,
				   result, msg.res_size);
	if (ret < 0)
		goto cleanup;

	ret = ntstatus2int(result->status);
	if (ret < 0)
		goto cleanup;

	ret = copy_to_user(args->priorities,
			   (u8 *) result + sizeof(*result),
			   priority_size);
	if (ret) {
		DXG_ERR("failed to copy priorities");
		ret = -EFAULT;
	}

cleanup:
	free_message((struct dxgvmbusmsg *)&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

int dxgvmb_send_set_context_sch_priority(struct dxgprocess *process,
					 struct dxgadapter *adapter,
					 struct d3dkmthandle context,
					 int priority,
					 bool in_process)
{
	struct dxgkvmb_command_setcontextschedulingpriority2 *command;
	int ret;
	struct dxgvmbusmsg msg = {.hdr = NULL};

	ret = init_message(&msg, adapter, process, sizeof(*command));
	if (ret)
		goto cleanup;
	command = (void *)msg.msg;

	command_vgpu_to_host_init2(&command->hdr,
				   DXGK_VMBCOMMAND_SETCONTEXTSCHEDULINGPRIORITY,
				   process->host_handle);
	command->context = context;
	command->priority = priority;
	command->in_process = in_process;
	ret = dxgvmb_send_sync_msg_ntstatus(msg.channel, msg.hdr, msg.size);
cleanup:
	free_message(&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

int dxgvmb_send_get_context_sch_priority(struct dxgprocess *process,
					 struct dxgadapter *adapter,
					 struct d3dkmthandle context,
					 int *priority,
					 bool in_process)
{
	struct dxgkvmb_command_getcontextschedulingpriority *command;
	struct dxgkvmb_command_getcontextschedulingpriority_return result = { };
	int ret;
	struct dxgvmbusmsg msg = {.hdr = NULL};

	ret = init_message(&msg, adapter, process, sizeof(*command));
	if (ret)
		goto cleanup;
	command = (void *)msg.msg;

	command_vgpu_to_host_init2(&command->hdr,
				   DXGK_VMBCOMMAND_GETCONTEXTSCHEDULINGPRIORITY,
				   process->host_handle);
	command->context = context;
	command->in_process = in_process;
	ret = dxgvmb_send_sync_msg(msg.channel, msg.hdr, msg.size,
				   &result, sizeof(result));
	if (ret >= 0) {
		ret = ntstatus2int(result.status);
		*priority = result.priority;
	}
cleanup:
	free_message(&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

int dxgvmb_send_offer_allocations(struct dxgprocess *process,
				  struct dxgadapter *adapter,
				  struct d3dkmt_offerallocations *args)
{
	struct dxgkvmb_command_offerallocations *command;
	int ret = -EINVAL;
	u32 alloc_size = sizeof(struct d3dkmthandle) * args->allocation_count;
	u32 cmd_size = sizeof(struct dxgkvmb_command_offerallocations) +
			alloc_size - sizeof(struct d3dkmthandle);
	struct dxgvmbusmsg msg = {.hdr = NULL};

	ret = init_message(&msg, adapter, process, cmd_size);
	if (ret)
		goto cleanup;
	command = (void *)msg.msg;

	command_vgpu_to_host_init2(&command->hdr,
				   DXGK_VMBCOMMAND_OFFERALLOCATIONS,
				   process->host_handle);
	command->flags = args->flags;
	command->priority = args->priority;
	command->device = args->device;
	command->allocation_count = args->allocation_count;
	if (args->resources) {
		command->resources = true;
		ret = copy_from_user(command->allocations, args->resources,
				     alloc_size);
	} else {
		ret = copy_from_user(command->allocations,
				     args->allocations, alloc_size);
	}
	if (ret) {
		DXG_ERR("failed to copy input handles");
		ret = -EFAULT;
		goto cleanup;
	}

	ret = dxgvmb_send_sync_msg_ntstatus(msg.channel, msg.hdr, msg.size);

cleanup:
	free_message(&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

int dxgvmb_send_reclaim_allocations(struct dxgprocess *process,
				    struct dxgadapter *adapter,
				    struct d3dkmthandle device,
				    struct d3dkmt_reclaimallocations2 *args,
				    u64  __user *paging_fence_value)
{
	struct dxgkvmb_command_reclaimallocations *command;
	struct dxgkvmb_command_reclaimallocations_return *result;
	int ret;
	u32 alloc_size = sizeof(struct d3dkmthandle) * args->allocation_count;
	u32 cmd_size = sizeof(struct dxgkvmb_command_reclaimallocations) +
	    alloc_size - sizeof(struct d3dkmthandle);
	u32 result_size = sizeof(*result);
	struct dxgvmbusmsgres msg = {.hdr = NULL};

	if (args->results)
		result_size += (args->allocation_count - 1) *
				sizeof(enum d3dddi_reclaim_result);

	ret = init_message_res(&msg, adapter, cmd_size, result_size);
	if (ret)
		goto cleanup;
	command = (void *)msg.msg;
	result = msg.res;

	command_vgpu_to_host_init2(&command->hdr,
				   DXGK_VMBCOMMAND_RECLAIMALLOCATIONS,
				   process->host_handle);
	command->device = device;
	command->paging_queue = args->paging_queue;
	command->allocation_count = args->allocation_count;
	command->write_results = args->results != NULL;
	if (args->resources) {
		command->resources = true;
		ret = copy_from_user(command->allocations, args->resources,
					 alloc_size);
	} else {
		ret = copy_from_user(command->allocations,
					 args->allocations, alloc_size);
	}
	if (ret) {
		DXG_ERR("failed to copy input handles");
		ret = -EFAULT;
		goto cleanup;
	}

	ret = dxgvmb_send_sync_msg(msg.channel, msg.hdr, msg.size,
				   result, msg.res_size);
	if (ret < 0)
		goto cleanup;
	ret = copy_to_user(paging_fence_value,
			   &result->paging_fence_value, sizeof(u64));
	if (ret) {
		DXG_ERR("failed to copy paging fence");
		ret = -EFAULT;
		goto cleanup;
	}

	ret = ntstatus2int(result->status);
	if (NT_SUCCESS(result->status) && args->results) {
		ret = copy_to_user(args->results, result->discarded,
				   sizeof(result->discarded[0]) *
				   args->allocation_count);
		if (ret) {
			DXG_ERR("failed to copy results");
			ret = -EFAULT;
		}
	}

cleanup:
	free_message((struct dxgvmbusmsg *)&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

int dxgvmb_send_change_vidmem_reservation(struct dxgprocess *process,
					  struct dxgadapter *adapter,
					  struct d3dkmthandle other_process,
					  struct
					  d3dkmt_changevideomemoryreservation
					  *args)
{
	struct dxgkvmb_command_changevideomemoryreservation *command;
	int ret;
	struct dxgvmbusmsg msg = {.hdr = NULL};

	ret = init_message(&msg, adapter, process, sizeof(*command));
	if (ret)
		goto cleanup;
	command = (void *)msg.msg;

	command_vgpu_to_host_init2(&command->hdr,
				   DXGK_VMBCOMMAND_CHANGEVIDEOMEMORYRESERVATION,
				   process->host_handle);
	command->args = *args;
	command->args.process = other_process.v;

	ret = dxgvmb_send_sync_msg_ntstatus(msg.channel, msg.hdr, msg.size);
cleanup:
	free_message(&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

int dxgvmb_send_create_hwqueue(struct dxgprocess *process,
			       struct dxgadapter *adapter,
			       struct d3dkmt_createhwqueue *args,
			       struct d3dkmt_createhwqueue *__user inargs,
			       struct dxghwqueue *hwqueue)
{
	struct dxgkvmb_command_createhwqueue *command = NULL;
	u32 cmd_size = sizeof(struct dxgkvmb_command_createhwqueue);
	int ret;
	struct dxgvmbusmsg msg = {.hdr = NULL};

	if (args->priv_drv_data_size > DXG_MAX_VM_BUS_PACKET_SIZE) {
		DXG_ERR("invalid private driver data size: %d",
			args->priv_drv_data_size);
		ret = -EINVAL;
		goto cleanup;
	}

	if (args->priv_drv_data_size)
		cmd_size += args->priv_drv_data_size - 1;

	ret = init_message(&msg, adapter, process, cmd_size);
	if (ret)
		goto cleanup;
	command = (void *)msg.msg;

	command_vgpu_to_host_init2(&command->hdr,
				   DXGK_VMBCOMMAND_CREATEHWQUEUE,
				   process->host_handle);
	command->context = args->context;
	command->flags = args->flags;
	command->priv_drv_data_size = args->priv_drv_data_size;
	if (args->priv_drv_data_size) {
		ret = copy_from_user(command->priv_drv_data,
				     args->priv_drv_data,
				     args->priv_drv_data_size);
		if (ret) {
			DXG_ERR("failed to copy private data");
			ret = -EFAULT;
			goto cleanup;
		}
	}

	ret = dxgvmb_send_sync_msg(msg.channel, msg.hdr, msg.size,
				   command, cmd_size);
	if (ret < 0)
		goto cleanup;

	ret = ntstatus2int(command->status);
	if (ret < 0) {
		DXG_ERR("dxgvmb_send_sync_msg failed: %x",
			command->status.v);
		goto cleanup;
	}

	hwqueue->progress_fence_mapped_address =
		dxg_map_iospace((u64)command->hwqueue_progress_fence_cpuva,
				PAGE_SIZE, PROT_READ | PROT_WRITE, true);
	if (hwqueue->progress_fence_mapped_address == NULL) {
		ret = -ENOMEM;
		goto cleanup;
	}

	ret = copy_to_user(&inargs->queue, &command->hwqueue,
			   sizeof(struct d3dkmthandle));
	if (ret) {
		DXG_ERR("failed to copy hwqueue handle");
		ret = -EFAULT;
		goto cleanup;
	}
	ret = copy_to_user(&inargs->queue_progress_fence,
			   &command->hwqueue_progress_fence,
			   sizeof(struct d3dkmthandle));
	if (ret) {
		DXG_ERR("failed to progress fence");
		ret = -EFAULT;
		goto cleanup;
	}
	ret = copy_to_user(&inargs->queue_progress_fence_cpu_va,
			   &hwqueue->progress_fence_mapped_address,
			   sizeof(inargs->queue_progress_fence_cpu_va));
	if (ret) {
		DXG_ERR("failed to copy fence cpu va");
		ret = -EFAULT;
		goto cleanup;
	}
	ret = copy_to_user(&inargs->queue_progress_fence_gpu_va,
			   &command->hwqueue_progress_fence_gpuva,
			   sizeof(u64));
	if (ret) {
		DXG_ERR("failed to copy fence gpu va");
		ret = -EFAULT;
		goto cleanup;
	}
	if (args->priv_drv_data_size) {
		ret = copy_to_user(args->priv_drv_data,
				   command->priv_drv_data,
				   args->priv_drv_data_size);
		if (ret) {
			DXG_ERR("failed to copy private data");
			ret = -EFAULT;
		}
	}

	ret = hmgrtable_assign_handle_safe(&process->handle_table,
				NULL,
				HMGRENTRY_TYPE_MONITOREDFENCE,
				command->hwqueue_progress_fence);
	if (ret < 0)
		goto cleanup;

	hwqueue->progress_fence_sync_object = command->hwqueue_progress_fence;
	hwqueue->handle = command->hwqueue;

	ret = hmgrtable_assign_handle_safe(&process->handle_table, hwqueue,
					   HMGRENTRY_TYPE_DXGHWQUEUE,
					   command->hwqueue);
	if (ret < 0)
		hwqueue->handle.v = 0;

cleanup:
	if (ret < 0) {
		DXG_ERR("failed %x", ret);
		if (hwqueue->handle.v) {
			hmgrtable_free_handle_safe(&process->handle_table,
						   HMGRENTRY_TYPE_DXGHWQUEUE,
						   hwqueue->handle);
			hwqueue->handle.v = 0;
		}
		if (command && command->hwqueue.v)
			dxgvmb_send_destroy_hwqueue(process, adapter,
						    command->hwqueue);
	}
	free_message(&msg);
	return ret;
}

int dxgvmb_send_destroy_hwqueue(struct dxgprocess *process,
				struct dxgadapter *adapter,
				struct d3dkmthandle handle)
{
	int ret;
	struct dxgkvmb_command_destroyhwqueue *command;
	struct dxgvmbusmsg msg = {.hdr = NULL};

	ret = init_message(&msg, adapter, process, sizeof(*command));
	if (ret)
		goto cleanup;
	command = (void *)msg.msg;

	command_vgpu_to_host_init2(&command->hdr,
				   DXGK_VMBCOMMAND_DESTROYHWQUEUE,
				   process->host_handle);
	command->hwqueue = handle;

	ret = dxgvmb_send_sync_msg_ntstatus(msg.channel, msg.hdr, msg.size);

cleanup:
	free_message(&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

int dxgvmb_send_query_adapter_info(struct dxgprocess *process,
				   struct dxgadapter *adapter,
				   struct d3dkmt_queryadapterinfo *args)
{
	struct dxgkvmb_command_queryadapterinfo *command;
	u32 cmd_size = sizeof(*command) + args->private_data_size - 1;
	int ret;
	u32 private_data_size;
	void *private_data;
	struct dxgvmbusmsg msg = {.hdr = NULL};
	struct dxgglobal *dxgglobal = dxggbl();

	ret = init_message(&msg, adapter, process, cmd_size);
	if (ret)
		goto cleanup;
	command = (void *)msg.msg;

	ret = copy_from_user(command->private_data,
			     args->private_data, args->private_data_size);
	if (ret) {
		DXG_ERR("Faled to copy private data");
		ret = -EFAULT;
		goto cleanup;
	}

	command_vgpu_to_host_init2(&command->hdr,
				   DXGK_VMBCOMMAND_QUERYADAPTERINFO,
				   process->host_handle);
	command->private_data_size = args->private_data_size;
	command->query_type = args->type;

	if (dxgglobal->vmbus_ver >= DXGK_VMBUS_INTERFACE_VERSION) {
		private_data = msg.msg;
		private_data_size = command->private_data_size +
				    sizeof(struct ntstatus);
	} else {
		private_data = command->private_data;
		private_data_size = command->private_data_size;
	}

	ret = dxgvmb_send_sync_msg(msg.channel, msg.hdr, msg.size,
				   private_data, private_data_size);
	if (ret < 0)
		goto cleanup;

	if (dxgglobal->vmbus_ver >= DXGK_VMBUS_INTERFACE_VERSION) {
		ret = ntstatus2int(*(struct ntstatus *)private_data);
		if (ret < 0)
			goto cleanup;
		private_data = (char *)private_data + sizeof(struct ntstatus);
	}

	switch (args->type) {
	case _KMTQAITYPE_ADAPTERTYPE:
	case _KMTQAITYPE_ADAPTERTYPE_RENDER:
		{
			struct d3dkmt_adaptertype *adapter_type =
			    (void *)private_data;
			adapter_type->paravirtualized = 1;
			adapter_type->display_supported = 0;
			adapter_type->post_device = 0;
			adapter_type->indirect_display_device = 0;
			adapter_type->acg_supported = 0;
			adapter_type->support_set_timings_from_vidpn = 0;
			adapter_type->compute_only = !!adapter->compute_only;
			break;
		}
	default:
		break;
	}
	ret = copy_to_user(args->private_data, private_data,
			   args->private_data_size);
	if (ret) {
		DXG_ERR("Faled to copy private data to user");
		ret = -EFAULT;
	}

cleanup:
	free_message(&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

int dxgvmb_send_submit_command_hwqueue(struct dxgprocess *process,
				       struct dxgadapter *adapter,
				       struct d3dkmt_submitcommandtohwqueue
				       *args)
{
	int ret = -EINVAL;
	u32 cmd_size;
	struct dxgkvmb_command_submitcommandtohwqueue *command;
	u32 primaries_size = args->num_primaries * sizeof(struct d3dkmthandle);
	struct dxgvmbusmsg msg = {.hdr = NULL};
	struct dxgglobal *dxgglobal = dxggbl();

	cmd_size = sizeof(*command) + args->priv_drv_data_size + primaries_size;
	ret = init_message(&msg, adapter, process, cmd_size);
	if (ret)
		goto cleanup;
	command = (void *)msg.msg;

	if (primaries_size) {
		ret = copy_from_user(&command[1], args->written_primaries,
					 primaries_size);
		if (ret) {
			DXG_ERR("failed to copy primaries handles");
			ret = -EFAULT;
			goto cleanup;
		}
	}
	if (args->priv_drv_data_size) {
		ret = copy_from_user((char *)&command[1] + primaries_size,
				      args->priv_drv_data,
				      args->priv_drv_data_size);
		if (ret) {
			DXG_ERR("failed to copy primaries data");
			ret = -EFAULT;
			goto cleanup;
		}
	}

	command_vgpu_to_host_init2(&command->hdr,
				   DXGK_VMBCOMMAND_SUBMITCOMMANDTOHWQUEUE,
				   process->host_handle);
	command->args = *args;

	if (dxgglobal->async_msg_enabled) {
		command->hdr.async_msg = 1;
		ret = dxgvmb_send_async_msg(msg.channel, msg.hdr, msg.size);
	} else {
		ret = dxgvmb_send_sync_msg_ntstatus(msg.channel, msg.hdr,
						    msg.size);
	}

cleanup:
	free_message(&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}

int dxgvmb_send_query_statistics(struct d3dkmthandle host_process_handle,
				 struct dxgadapter *adapter,
				 struct d3dkmt_querystatistics *args)
{
	struct dxgkvmb_command_querystatistics *command;
	struct dxgkvmb_command_querystatistics_return *result;
	int ret;
	struct dxgvmbusmsgres msg = {.hdr = NULL};

	ret = init_message_res(&msg, adapter, sizeof(*command),
			       sizeof(*result));
	if (ret)
		goto cleanup;
	command = msg.msg;
	result = msg.res;

	command_vgpu_to_host_init2(&command->hdr,
				   DXGK_VMBCOMMAND_QUERYSTATISTICS,
				   host_process_handle);
	command->args = *args;

	ret = dxgvmb_send_sync_msg(msg.channel, msg.hdr, msg.size,
				   result, msg.res_size);
	if (ret < 0)
		goto cleanup;

	args->result = result->result;
	ret = ntstatus2int(result->status);

cleanup:
	free_message((struct dxgvmbusmsg *)&msg);
	if (ret)
		DXG_TRACE("err: %d", ret);
	return ret;
}
