# HG changeset patch # User David Scott # Date 1263553029 0 # Node ID 6c0dc1298ff409515511ca313480ad1c7cb3a32f # Parent c2268131edc640465fed22cef492b2e10b8a4579 CA-36384: [experimental PCI passthrough]: unplug/plug devices around suspend/resume. Also refactor the Device.PCI.* functions to make it clear that a response ("pci-inserted", "pci-removed") is always expected from a hotplug/unplug. Signed-off-by: David Scott diff -r c2268131edc6 -r 6c0dc1298ff4 ocaml/xapi/vmops.ml --- a/ocaml/xapi/vmops.ml Tue Jan 12 23:37:40 2010 +0000 +++ b/ocaml/xapi/vmops.ml Fri Jan 15 10:57:09 2010 +0000 @@ -494,7 +494,7 @@ devs (* Hotplug the PCI devices into the domain (as opposed to 'attach_pcis') *) -let plug_pcidevs ~__context ~vm domid pcidevs = +let plug_pcidevs_noexn ~__context ~vm domid pcidevs = Helpers.log_exn_continue "plug_pcidevs" (fun () -> if List.length pcidevs > 0 then begin @@ -515,6 +515,26 @@ ) end; ) () + +(* Hot unplug the PCI devices from the domain. Note this is done serially due to a limitation of the + xenstore protocol. *) +let unplug_pcidevs_noexn ~__context ~vm domid pcidevs = + Helpers.log_exn_continue "unplug_pcidevs" + (fun () -> + Vmopshelpers.with_xc_and_xs + (fun xc xs -> + if (Xc.domain_getinfo xc domid).Xc.hvm_guest then begin + List.iter + (fun (devid, devices) -> + List.iter + (fun device -> + debug "requesting hotunplug of PCI device %s" (Device.PCI.to_string device); + Device.PCI.unplug ~xc ~xs device domid; + ) devices + ) (sort_pcidevs pcidevs) + end + ) + ) () let has_platform_flag platform feature = try bool_of_string (List.assoc feature platform) with _ -> false @@ -1018,7 +1038,7 @@ progress_cb 0.80; debug "creating device emulator"; let vncport = create_device_emulator ~__context ~xc ~xs ~self:vm domid vifs snapshot in - if hvm then plug_pcidevs ~__context ~vm domid pcidevs; + if hvm then plug_pcidevs_noexn ~__context ~vm domid pcidevs; create_console ~__context ~vM:vm ~vncport (); debug "writing memory policy"; write_memory_policy ~xs snapshot domid; diff -r c2268131edc6 -r 6c0dc1298ff4 ocaml/xapi/xapi_vm.ml --- a/ocaml/xapi/xapi_vm.ml Tue Jan 12 23:37:40 2010 +0000 +++ b/ocaml/xapi/xapi_vm.ml Fri Jan 15 10:57:09 2010 +0000 @@ -737,11 +737,15 @@ try (* Balloon down the guest as far as we can to force it to clear unnecessary caches etc. *) - debug "suspend phase 0/3: asking guest to balloon down"; + debug "suspend phase 0/4: asking guest to balloon down"; Domain.set_memory_dynamic_range ~xs ~min ~max:min domid; Memory_control.balance_memory ~__context ~xc ~xs; - debug "suspend phase 1/3: calling Vmops.suspend"; + debug "suspend phase 1/4: hot-unplugging any PCI devices"; + let hvm = (Xc.domain_getinfo xc domid).Xc.hvm_guest in + if hvm then Vmops.unplug_pcidevs_noexn ~__context ~vm domid (Device.PCI.list xc xs domid); + + debug "suspend phase 2/4: calling Vmops.suspend"; (* Call the memory image creating 90%, *) (* the device un-hotplug the final 10% *) Vmops.suspend ~__context ~xc ~xs ~vm ~live:false @@ -749,7 +753,7 @@ TaskHelper.set_progress ~__context (x *. 0.9) ); - debug "suspend phase 2/3: recording memory usage"; + debug "suspend phase 3/4: recording memory usage"; (* Record the final memory usage of the VM, so *) (* that we know how much memory to free before *) (* attempting to resume this VM in future. *) @@ -758,7 +762,7 @@ debug "total_memory_pages=%Ld; storing target=%Ld" (Int64.of_nativeint di.Xc.total_memory_pages) final_memory_bytes; (* CA-31759: avoid using the LBR to simplify upgrade *) Db.VM.set_memory_target ~__context ~self:vm ~value:final_memory_bytes; - debug "suspend phase 3/3: destroying the domain"; + debug "suspend phase 4/4: destroying the domain"; Vmops.destroy ~clear_currently_attached:false ~__context ~xc ~xs ~self:vm domid `Suspended; with e -> @@ -811,13 +815,15 @@ Vmops.restore ~__context ~xc ~xs ~self:vm domid; Db.VM.set_domid ~__context ~self:vm ~value:(Int64.of_int domid); - Vmops.plug_pcidevs ~__context ~vm domid (Vmops.pcidevs_of_vm ~__context ~vm); debug "resume phase 3/3: %s unpausing domain" (if start_paused then "not" else ""); if not start_paused then begin Domain.unpause ~xc domid; end; + + Vmops.plug_pcidevs_noexn ~__context ~vm domid (Vmops.pcidevs_of_vm ~__context ~vm); + (* VM is now resident here *) let localhost = Helpers.get_localhost ~__context in Helpers.call_api_functions ~__context diff -r c2268131edc6 -r 6c0dc1298ff4 ocaml/xapi/xapi_vm_migrate.ml --- a/ocaml/xapi/xapi_vm_migrate.ml Tue Jan 12 23:37:40 2010 +0000 +++ b/ocaml/xapi/xapi_vm_migrate.ml Fri Jan 15 10:57:09 2010 +0000 @@ -215,7 +215,7 @@ let (id, device) = List.hd devices in let (domain, bus, dev, func) = device in debug "requesting unplug of %.4x:%.2x:%.2x.%.1x" domain bus dev func; - Device.PCI.unplug ~xc ~xs device domid (-1); + Device.PCI.unplug ~xc ~xs device domid; pci_devices_to_unplug := [ device ] end end) () in @@ -499,7 +499,7 @@ debug "Receiver 7b. unpausing domain"; Domain.unpause ~xc domid; - Vmops.plug_pcidevs ~__context ~vm domid (Vmops.pcidevs_of_vm ~__context ~vm); + Vmops.plug_pcidevs_noexn ~__context ~vm domid (Vmops.pcidevs_of_vm ~__context ~vm); Db.VM.set_domid ~__context ~self:vm ~value:(Int64.of_int domid); Helpers.call_api_functions ~__context diff -r c2268131edc6 -r 6c0dc1298ff4 ocaml/xenops/device.ml --- a/ocaml/xenops/device.ml Tue Jan 12 23:37:40 2010 +0000 +++ b/ocaml/xenops/device.ml Fri Jan 15 10:57:09 2010 +0000 @@ -1159,19 +1159,25 @@ let be_path = xs.Xs.getdomainpath be_domid in Printf.sprintf "%s/backend/pci/%d/0" be_path fe_domid + let signal_device_model ~xc ~xs domid cmd parameter = debug "Device.Pci.signal_device_model domid=%d cmd=%s param=%s" domid cmd parameter; let be_domid = 0 in (* XXX: assume device model is in domain 0 *) let be_path = xs.Xs.getdomainpath be_domid in + (* Currently responses go in this global place. Blank it to prevent request/response/request confusion *) + xs.Xs.rm (device_model_state_path xs be_domid domid); + Xs.transaction xs (fun t -> t.Xst.writev be_path [ Printf.sprintf "device-model/%d/command" domid, cmd; Printf.sprintf "device-model/%d/parameter" domid, parameter ]; - (* Currently responses go in this global place. Blank it to prevent request/response/request confusion *) - t.Xst.rm (device_model_state_path xs be_domid domid) - ); - (* XXX: no response protocol *) - () + ) +let wait_device_model ~xc ~xs domid = + let be_domid = 0 in + let answer = Watch.wait_for ~xs (Watch.value_to_appear (device_model_state_path xs be_domid domid)) in + xs.Xs.rm (device_model_state_path xs be_domid domid); + answer + (* Return a list of PCI devices *) let list ~xc ~xs domid = let path = device_model_pci_device_path xs 0 domid in @@ -1191,21 +1197,25 @@ let pci = to_string (domain, bus, dev, func) in signal_device_model ~xc ~xs domid "pci-ins" pci; - xs.Xs.write (device_model_pci_device_path xs 0 domid ^ "/dev-" ^ (string_of_int next_idx)) pci -let unplug ~xc ~xs (domain, bus, dev, func) domid devid = + match wait_device_model ~xc ~xs domid with + | "pci-inserted" -> + (* success *) + xs.Xs.write (device_model_pci_device_path xs 0 domid ^ "/dev-" ^ (string_of_int next_idx)) pci; + | x -> failwith (Printf.sprintf "Waiting for state=pci-inserted; got state=%s" x) + +let unplug ~xc ~xs (domain, bus, dev, func) domid = let current = list ~xc ~xs domid in - let pci = (domain, bus, dev, func) in - let idx = fst (List.find (fun x -> snd x = pci) current) in - signal_device_model ~xc ~xs domid "pci-rem" (to_string (domain, bus, dev, func)); - xs.Xs.rm (device_model_pci_device_path xs 0 domid ^ "/dev-" ^ (string_of_int idx)) -(* XXX: this really should be tied to the device rather than being global. Also we should make it clear that 'unplug' - is asynchronous. *) -let unplug_wait ~xc ~xs domid = - match Watch.wait_for ~xs (Watch.value_to_appear (device_model_state_path xs 0 domid)) with (* XXX: assume dom0 *) - | "pci-removed" -> () (* success *) - | x -> failwith (Printf.sprintf "Waiting for state=pci-removed; got state=%s" x) + let pci = to_string (domain, bus, dev, func) in + let idx = fst (List.find (fun x -> snd x = (domain, bus, dev, func)) current) in + signal_device_model ~xc ~xs domid "pci-rem" pci; + + match wait_device_model ~xc ~xs domid with + | "pci-removed" -> + (* success *) + xs.Xs.rm (device_model_pci_device_path xs 0 domid ^ "/dev-" ^ (string_of_int idx)) + | x -> failwith (Printf.sprintf "Waiting for state=pci-removed; got state=%s" x) end diff -r c2268131edc6 -r 6c0dc1298ff4 ocaml/xenops/device.mli --- a/ocaml/xenops/device.mli Tue Jan 12 23:37:40 2010 +0000 +++ b/ocaml/xenops/device.mli Fri Jan 15 10:57:09 2010 +0000 @@ -128,6 +128,9 @@ resources: (int64 * int64 * int64) list; driver: string; } + type dev = int * int * int * int + val to_string: dev -> string + val of_string: string -> dev exception Cannot_use_pci_with_no_pciback of t list @@ -140,8 +143,7 @@ val plug : xc:Xc.handle -> xs:Xs.xsh -> (int * int * int * int) -> Xc.domid -> int -> unit val unplug : xc:Xc.handle -> xs:Xs.xsh - -> (int * int * int * int) -> Xc.domid -> int -> unit - val unplug_wait : xc:Xc.handle -> xs:Xs.xsh -> Xc.domid -> unit + -> (int * int * int * int) -> Xc.domid -> unit val list : xc:Xc.handle -> xs:Xs.xsh -> Xc.domid -> (int * (int * int * int * int)) list end diff -r c2268131edc6 -r 6c0dc1298ff4 ocaml/xenops/xenops.ml --- a/ocaml/xenops/xenops.ml Tue Jan 12 23:37:40 2010 +0000 +++ b/ocaml/xenops/xenops.ml Fri Jan 15 10:57:09 2010 +0000 @@ -264,7 +264,7 @@ let unplug_pci ~xc ~xs ~domid ~devid ~pci = let pcidev = pci_of_string pci in - Device.PCI.unplug ~xc ~xs pcidev domid devid + Device.PCI.unplug ~xc ~xs pcidev domid let del_pci ~xc ~xs ~hvm ~domid ~devid ~pci = let pcidevs = List.map (fun d ->