# HG changeset patch # User David Scott # Date 1259589188 0 # Node ID c90daafe028b57f2b597c4a8961355e23770ff93 # Parent 91eb1c3b26e7c10b249c0a2a2197587abb0c9a92 CA-35397: add a simple cache of per-domain data in the ballooning daemon. The cache stores: * values of xenstore keys written only by the ballooning daemon * values of maxmem (set only by the ballooning daemon) but not * values of dynamic-min, dynamic-max, written by the toolstack. Signed-off-by: David Scott diff -r 91eb1c3b26e7 -r c90daafe028b ocaml/xenops/squeeze_xen.ml --- a/ocaml/xenops/squeeze_xen.ml Wed Nov 25 16:01:11 2009 -0800 +++ b/ocaml/xenops/squeeze_xen.ml Mon Nov 30 13:53:08 2009 +0000 @@ -25,13 +25,14 @@ let debug = Squeeze.debug let error = Squeeze.error -let initial_reservation_path dom_path = dom_path ^ "/memory/initial-reservation" -let target_path dom_path = dom_path ^ "/memory/target" -let dynamic_min_path dom_path = dom_path ^ "/memory/dynamic-min" -let dynamic_max_path dom_path = dom_path ^ "/memory/dynamic-max" -let feature_balloon_path dom_path = dom_path ^ "/control/feature-balloon" -let memory_offset_path dom_path = dom_path ^ "/memory/memory-offset" -let uncooperative_path dom_path = dom_path ^ "/memory/uncooperative" +let _initial_reservation = "/memory/initial-reservation" (* immutable *) +let _target = "/memory/target" (* immutable *) +let _feature_balloon = "/control/feature-balloon" (* immutable *) +let _memory_offset = "/memory/memory-offset" (* immutable *) +let _uncooperative = "/memory/uncooperative" (* mutable: written by us *) + +let _dynamic_min = "/memory/dynamic-min" (* mutable: written by domain manager *) +let _dynamic_max = "/memory/dynamic-max" (* mutable: written by domain manager *) let ( ** ) = Int64.mul let ( +* ) = Int64.add @@ -42,70 +43,156 @@ let low_mem_emergency_pool = 1L ** mib (** Same as xen commandline *) -let exists xs path = try ignore(xs.Xs.read path); true with Xb.Noent -> false - -let xs_read xs path = try xs.Xs.read path with Xb.Noent as e -> begin debug "xenstore-read %s returned ENOENT" path; raise e end - (** Return the extra amount we always add onto maxmem *) -let xen_max_offset_kib di = - let maxmem_mib = if di.Xc.hvm_guest then Memory.HVM.xen_max_offset_mib else Memory.Linux.xen_max_offset_mib in +let xen_max_offset_kib hvm = + let maxmem_mib = if hvm then Memory.HVM.xen_max_offset_mib else Memory.Linux.xen_max_offset_mib in Memory.kib_of_mib maxmem_mib -let domain_setmaxmem_noexn xc domid target_kib = - let di = Xc.domain_getinfo xc domid in - let maxmem_kib = xen_max_offset_kib di +* target_kib in - try - let existing_maxmem_kib = Xc.pages_to_kib (Int64.of_nativeint di.Xc.max_memory_pages) in - if existing_maxmem_kib <> maxmem_kib then begin - debug "Xc.domain_setmaxmem domid=%d target=%Ld max=%Ld" domid target_kib maxmem_kib; - Xc.domain_setmaxmem xc domid maxmem_kib - end - with e -> - (* someone probably just destroyed the domain *) - error "Xc.domain_setmaxmem domid=%d max=%Ld: %s" domid maxmem_kib (Printexc.to_string e) - -let set_target_noexn xs dom_path target_kib = - let path = target_path dom_path in - try - Xs.transaction xs - (fun t -> - (* make sure no-one deletes the tree *) - let existing_target_kib = Int64.of_string (t.Xst.read path) in - if existing_target_kib <> target_kib then begin - debug "xenstore-write %s = %Ld" path target_kib; - t.Xst.write path (Int64.to_string target_kib) - end - ) - with e -> - (* someone probably just destroyed the domain *) - error "xenstore-write %s = %Ld: %s" path target_kib (Printexc.to_string e) +(** Cache of per-domain info, to avoid repeated per-domain queries *) +module Domain = struct + type per_domain = { path: string; + hvm: bool; + mutable maxmem: int64; + keys: (string, string option) Hashtbl.t } + type t = (int, per_domain) Hashtbl.t + + let cache = Hashtbl.create 10 + let get_per_domain (xc, xs) domid = + if Hashtbl.mem cache domid + then Hashtbl.find cache domid + else + let di = Xc.domain_getinfo xc domid in + let maxmem = Xc.pages_to_kib (Int64.of_nativeint di.Xc.max_memory_pages) in + let d = { path = xs.Xs.getdomainpath domid; + hvm = di.Xc.hvm_guest; + maxmem = maxmem; + keys = Hashtbl.create 10 } in + Hashtbl.replace cache domid d; + d + let get_hvm cnx domid = (get_per_domain cnx domid).hvm + let get_maxmem cnx domid = (get_per_domain cnx domid).maxmem + let set_maxmem_noexn (xc, xs) domid m = + let per_domain = get_per_domain (xc, xs) domid in + if per_domain.maxmem <> m then begin + debug "Xc.domain_setmaxmem domid=%d max=%Ld" domid m; + try + Xc.domain_setmaxmem xc domid m; + per_domain.maxmem <- m + with e -> + error "Xc.domain_setmaxmem domid=%d max=%Ld: %s" domid m (Printexc.to_string e) + end -let set_uncooperative_noexn xs dom_path x = - try - if x - then - Xs.transaction xs - (fun t -> - (* make sure no-one deletes the tree *) - ignore (t.Xst.read dom_path); - t.Xst.write (uncooperative_path dom_path) "" - ) - else xs.Xs.rm (uncooperative_path dom_path) - with e -> - error "set_uncooperative %s %b: %s" dom_path x (Printexc.to_string e) + (** Read a particular domain's key, using the cache *) + let read (xc, xs) domid key = + let per_domain = get_per_domain (xc, xs) domid in + let x = + if Hashtbl.mem per_domain.keys key + then Hashtbl.find per_domain.keys key + else begin + let x = (try Some (xs.Xs.read (per_domain.path ^ key)) + with Xb.Noent -> None) in + Hashtbl.replace per_domain.keys key x; + x + end in + match x with + | Some y -> y + | None -> + debug "xenstore-read %s%s returned ENOENT" per_domain.path key; + raise Xb.Noent -let get_uncooperative_noexn xs dom_path = exists xs (uncooperative_path dom_path) + (** Read a particular domain's key from xenstore, for when we believe the cache is out of date *) + let read_nocache (xc, xs) domid key = + let per_domain = get_per_domain (xc, xs) domid in + if Hashtbl.mem per_domain.keys key then Hashtbl.remove per_domain.keys key; + read (xc, xs) domid key + (** Write a new (key, value) pair into a domain's directory in xenstore. Don't write anything + if the domain's directory doesn't exist. Don't throw exceptions. *) -let set_memory_offset_noexn xs dom_path offset_kib = - try - Xs.transaction xs - (fun t -> - (* Make sure the domain still exists *) - ignore (xs.Xs.read dom_path); - t.Xst.write (memory_offset_path dom_path) (Int64.to_string offset_kib) - ) - with e -> - error "set_memory_offset_noexn %s %Ld: %s" dom_path offset_kib (Printexc.to_string e) + let write_noexn (xc, xs) domid key value = + let per_domain = get_per_domain (xc, xs) domid in + if not(Hashtbl.mem per_domain.keys key) || Hashtbl.find per_domain.keys key <> (Some value) then begin + try + Xs.transaction xs + (fun t -> + (* Fail if the directory has been deleted *) + ignore (xs.Xs.read per_domain.path); + t.Xst.write (per_domain.path ^ key) value + ); + Hashtbl.replace per_domain.keys key (Some value); + with e -> + (* Log but don't throw an exception *) + error "xenstore-write %d %s = %s failed: %s" domid key value (Printexc.to_string e) + end + (** Returns true if the key exists, false otherwise *) + let exists (xc, xs) domid key = try ignore(read (xc, xs) domid key); true with Xb.Noent -> false + (** Delete the key. Don't throw exceptions. *) + let rm_noexn (xc, xs) domid key = + let per_domain = get_per_domain (xc, xs) domid in + Hashtbl.replace per_domain.keys key None; + try + xs.Xs.rm (per_domain.path ^ key) + with e -> + error "xenstore-rm %d %s: %s" domid key (Printexc.to_string e) + + (** {High-level functions} *) + + (** Set a domain's memory target. Don't throw an exception if the domain has been destroyed. *) + let set_target_noexn cnx domid target_kib = + write_noexn cnx domid _target (Int64.to_string target_kib) + + (** Get a domain's memory target. Throws Xb.Noent if the domain has been destroyed *) + let get_target cnx domid = + Int64.of_string (read cnx domid _target) + + (** Mark a domain as uncooperative. Don't throw an exception if the domain has been destroyed. *) + let set_uncooperative_noexn cnx domid x = + if x + then write_noexn cnx domid _uncooperative "" + else rm_noexn cnx domid _uncooperative + + (** Query a domain's uncooperative status. Throws Xb.Noent if the domain has been destroyed *) + let get_uncooperative_noexn cnx domid = exists cnx domid _uncooperative + + (** Set a domain's memory-offset. Don't throw an exception if the domain has been destroyed *) + let set_memory_offset_noexn cnx domid offset_kib = + write_noexn cnx domid _memory_offset (Int64.to_string offset_kib) + + (** Query a domain's memory-offset. Throws Xb.Noent if the domain has been destroyed *) + let get_memory_offset cnx domid = + Int64.of_string (read cnx domid _memory_offset) + + (** Set a domain's maxmem. Don't throw an exception if the domain has been destroyed *) + let set_maxmem_noexn cnx domid target_kib = + let maxmem_kib = xen_max_offset_kib (get_hvm cnx domid) +* target_kib in + set_maxmem_noexn cnx domid maxmem_kib + + (** Return true if feature_balloon has been advertised *) + let get_feature_balloon (xc, xs) domid = + (* Cache the presence of this key when it appears but always read through when it hasn't yet *) + let per_domain = get_per_domain (xc, xs) domid in + if Hashtbl.mem per_domain.keys _feature_balloon && Hashtbl.find per_domain.keys _feature_balloon <> None + then true + else (try ignore (read_nocache (xc, xs) domid _feature_balloon); true with Xb.Noent -> false) + + (** Query a domain's initial reservation. Throws Xb.Noent if the domain has been destroyed *) + let get_initial_reservation cnx domid = + Int64.of_string (read cnx domid _initial_reservation) + + (** Query a domain's dynamic_min. Throws Xb.Noent if the domain has been destroyed *) + let get_dynamic_min cnx domid = + Int64.of_string (read_nocache cnx domid _dynamic_min) + + (** Query a domain's dynamic_max. Throws Xb.Noent if the domain has been destroyed *) + let get_dynamic_max cnx domid = + Int64.of_string (read_nocache cnx domid _dynamic_max) + + (** Expire old domain information from the cache *) + let gc live_domids = + let cached_domids = Hashtbl.fold (fun d _ ds -> d::ds) cache [] in + let to_remove = set_difference cached_domids live_domids in + List.iter (Hashtbl.remove cache) to_remove +end + (** Record when the domain was last co-operative *) @@ -127,13 +214,12 @@ ) host.Squeeze.domains (** Update all the flags in xenstore *) -let update_cooperative_flags xs = +let update_cooperative_flags cnx = let now = Unix.gettimeofday () in Hashtbl.iter (fun domid last_time -> - let dom_path = xs.Xs.getdomainpath domid in - let old_value = get_uncooperative_noexn xs dom_path in + let old_value = Domain.get_uncooperative_noexn cnx domid in let new_value = now -. last_time > 20. in - if old_value <> new_value then set_uncooperative_noexn xs dom_path new_value + if old_value <> new_value then Domain.set_uncooperative_noexn cnx domid new_value ) when_domain_was_last_cooperative @@ -171,30 +257,30 @@ and total_pages_kib = Xc.pages_to_kib (Int64.of_nativeint physinfo.Xc.total_pages) in let free_mem_kib = free_pages_kib +* scrub_pages_kib in + let cnx = xc, xs in let domains = List.concat (List.map (fun di -> try - let path = xs.Xs.getdomainpath di.Xc.domid in let memory_actual_kib = Xc.pages_to_kib (Int64.of_nativeint di.Xc.total_memory_pages) in (* dom0 is special for some reason *) let memory_max_kib = if di.Xc.domid = 0 then 0L else Xc.pages_to_kib (Int64.of_nativeint di.Xc.max_memory_pages) in (* Misc other stuff appears in max_memory_pages *) - let memory_max_kib = max 0L (memory_max_kib -* (xen_max_offset_kib di)) in - let can_balloon = exists xs (feature_balloon_path path) in + let memory_max_kib = max 0L (memory_max_kib -* (xen_max_offset_kib di.Xc.hvm_guest)) in + let can_balloon = Domain.get_feature_balloon cnx di.Xc.domid in (* Once the domain tells us it can balloon, we assume it's not currently ballooning and record the offset between memory_actual and target. We assume this is constant over the lifetime of the domain. *) - let offset_kib = + let offset_kib : int64 = if not can_balloon then 0L else begin try - Int64.of_string (xs_read xs (memory_offset_path path)) + Domain.get_memory_offset cnx di.Xc.domid with Xb.Noent -> - let target_kib = Int64.of_string (xs_read xs (target_path path)) in + let target_kib = Domain.get_target cnx di.Xc.domid in let offset_kib = memory_actual_kib -* target_kib in debug "domid %d just exposed feature-balloon; calibrating total_pages offset = %Ld KiB" di.Xc.domid offset_kib; - set_memory_offset_noexn xs path offset_kib; + Domain.set_memory_offset_noexn cnx di.Xc.domid offset_kib; offset_kib end in let memory_actual_kib = memory_actual_kib -* offset_kib in @@ -218,7 +304,7 @@ (* If the domain has yet to expose it's feature-balloon flag then we assume it is using at least its "initial-reservation". *) if not can_balloon then begin - let initial_reservation_kib = Int64.of_string (xs_read xs (initial_reservation_path path)) in + let initial_reservation_kib = Domain.get_initial_reservation cnx di.Xc.domid in (* memory_actual_kib is memory which xen has accounted to this domain. We bump this up to the "initial-reservation" and compute how much memory to subtract from the host's free memory *) @@ -233,13 +319,13 @@ } ] end else begin - let target_kib = Int64.of_string (xs_read xs (target_path path)) in + let target_kib = Domain.get_target cnx di.Xc.domid in (* min and max are written separately; if we notice they *) (* are missing set them both to the target for now. *) let min_kib, max_kib = try - Int64.of_string (xs_read xs (dynamic_min_path path)), - Int64.of_string (xs_read xs (dynamic_max_path path)) + Domain.get_dynamic_min cnx di.Xc.domid, + Domain.get_dynamic_max cnx di.Xc.domid with _ -> target_kib, target_kib in @@ -252,7 +338,7 @@ end with | Xb.Noent -> - (* useful debug message is printed by the xs_read function *) + (* useful debug message is printed by the Domain.read* functions *) [] | e -> debug "Skipping domid %d: %s" @@ -270,7 +356,9 @@ let host = Squeeze.make_host ~domains ~free_mem_kib:(Int64.sub free_mem_kib !reserved_kib) in update_cooperative_table host; - update_cooperative_flags xs; + update_cooperative_flags cnx; + + Domain.gc (List.map (fun di -> di.Xc.domid) domain_infolist); Printf.sprintf "F%Ld S%Ld R%Ld T%Ld" free_pages_kib scrub_pages_kib !reserved_kib total_pages_kib, host @@ -278,18 +366,17 @@ let execute_action ~xc ~xs action = try let domid = action.Squeeze.action_domid in - let path = xs.Xs.getdomainpath domid in let target_kib = action.Squeeze.new_target_kib in if target_kib < 0L then failwith "Proposed target is negative (domid %d): %Ld" domid target_kib; - let di = Xc.domain_getinfo xc domid in - let memory_max_kib = Xc.pages_to_kib (Int64.of_nativeint di.Xc.max_memory_pages) in + let cnx = (xc, xs) in + let memory_max_kib = Domain.get_maxmem cnx domid in if target_kib > memory_max_kib then begin - domain_setmaxmem_noexn xc domid target_kib; - set_target_noexn xs path target_kib; + Domain.set_maxmem_noexn cnx domid target_kib; + Domain.set_target_noexn cnx domid target_kib; end else begin - set_target_noexn xs path target_kib; - domain_setmaxmem_noexn xc domid target_kib; + Domain.set_target_noexn cnx domid target_kib; + Domain.set_maxmem_noexn cnx domid target_kib; end with e -> debug "Failed to reset balloon target (domid: %d) (target: %Ld): %s"