WARNING - OLD ARCHIVES

This is an archived copy of the Xen.org mailing list, which we have preserved to ensure that existing links to archives are not broken. The live archive, which contains the latest emails, can be found at http://lists.xen.org/
   
 
 
Xen 
 
Home Products Support Community News
 
   
 

xen-api

[Xen-API] [PATCH] Work around firewalls killing connections in export co

To: xen-api@xxxxxxxxxxxxxxxxxxx
Subject: [Xen-API] [PATCH] Work around firewalls killing connections in export code
From: David Scott <dave.scott@xxxxxxxxxxxxx>
Date: Fri, 9 Apr 2010 19:58:38 +0100
Delivery-date: Fri, 09 Apr 2010 11:56:01 -0700
Envelope-to: www-data@xxxxxxxxxxxxxxxxxxx
List-help: <mailto:xen-api-request@lists.xensource.com?subject=help>
List-id: Discussion of API issues surrounding Xen <xen-api.lists.xensource.com>
List-post: <mailto:xen-api@lists.xensource.com>
List-subscribe: <http://lists.xensource.com/mailman/listinfo/xen-api>, <mailto:xen-api-request@lists.xensource.com?subject=subscribe>
List-unsubscribe: <http://lists.xensource.com/mailman/listinfo/xen-api>, <mailto:xen-api-request@lists.xensource.com?subject=unsubscribe>
Sender: xen-api-bounces@xxxxxxxxxxxxxxxxxxx
# HG changeset patch
# User David Scott <dave.scott@xxxxxxxxxxxxx>
# Date 1270839463 -3600
# Node ID 9efb5684022b96d8e0da86b869038dfe2cbd362b
# Parent  6182bc7d23af84bb7e5138c698dc2efda20c35dc
CA-39291: Work around firewalls which kill idle TCP connections by inserting a 
small empty block into an export every 5s or so.

The failure happens whenever a disk has a lot of zeroes in it: the TCP 
connection goes idle while the server is scanning for the next non-zero block. 
Even setting SO_KEEPALIVE on the stunnel sockets and reducing the window probe 
interval down to 30s didn't fix it. We wish to keep the ability to have a basic 
client do an export via HTTP GET so we can't add application-level keepalives 
to the protocol... we must add them to the export itself.

Note this change is backwards compatible. The receiver code expects:
* a common prefix
* a monotonically increasing chunk number
* the first and last blocks to be the same size and included verbatim (even if 
all zeroes)
* blocks of zeroes the same size as the first block represented as gaps in the 
increasing chunk number sequence

Therefore including extra files of length 0 in the stream will be ignored 
provided they
* share the common prefix and chunk numbering scheme
* are not the first or last blocks

Signed-off-by: David Scott <dave.scott@xxxxxxxxxxxxx>

diff -r 6182bc7d23af -r 9efb5684022b ocaml/xapi/stream_vdi.ml
--- a/ocaml/xapi/stream_vdi.ml  Fri Apr 09 10:56:14 2010 +0100
+++ b/ocaml/xapi/stream_vdi.ml  Fri Apr 09 19:57:43 2010 +0100
@@ -86,6 +86,27 @@
   end
 
 
+(** Write a block of checksummed data of length [len] with name [filename] to 
[ofd] *)
+let write_block ~__context filename buffer ofd len = 
+  let hdr = Tar.Header.make filename (Int64.of_int len) in
+
+  try
+       let csum = Sha1sum.sha1sum
+         (fun checksumfd ->
+                  Tar.write_block hdr (fun ofd -> 
Tar.Archive.multicast_n_string buffer 
+                                                                       [ ofd; 
checksumfd ] len) ofd
+         ) in
+       (* Write the checksum as a separate file *)
+       let hdr' = Tar.Header.make (filename ^ checksum_extension) 
(Int64.of_int (String.length csum)) in
+       Tar.write_block hdr' (fun ofd -> ignore(Unix.write ofd csum 0 
(String.length csum))) ofd
+  with
+       Unix.Unix_error (a,b,c) as e ->
+               if TaskHelper.is_cancelling ~__context
+               then raise (Api_errors.Server_error (Api_errors.task_cancelled, 
[]))
+               else 
+                 (if b="write" 
+                  then raise (Api_errors.Server_error 
(Api_errors.client_error, [ExnHelper.string_of_exn e]))
+                  else raise e)
 
 
 (** Stream a set of VDIs split into chunks in a tar format in a defined order. 
Return an
@@ -95,56 +116,53 @@
 
   let progress = new_progress_record __context prefix_vdis in
 
+  (* Remember when we last wrote something so that we can work around 
firewalls which close 'idle' connections *)
+  let last_transmission_time = ref 0. in
+
   let send_one ofd (__context:Context.t) (prefix, vdi_ref, size) = 
     let size = Db.VDI.get_virtual_size ~__context ~self:vdi_ref in
-    let buffer = String.make (Int64.to_int chunk_size) '\000' in
 
     with_open_vdi __context rpc session_id vdi_ref `RO [Unix.O_RDONLY] 0o644
       (fun ifd ->
-        
+
+
         (* NB. It used to be that chunks could be larger than a native int *)
         (* could handle, but this is no longer the case! Ensure all chunks *)
         (* are strictly less than 2^30 bytes *)
         let rec stream_from (chunk_no: int) (offset: int64) = 
           refresh_session ();
-
           let remaining = Int64.sub size offset in
           if remaining > 0L
           then 
             begin
               let this_chunk = (min remaining chunk_size) in
-              let last_chunk = this_chunk=remaining in
+                  let last_chunk = this_chunk = remaining in
               let this_chunk = Int64.to_int this_chunk in
               let filename = Printf.sprintf "%s/%08d" prefix chunk_no in
               let hdr = Tar.Header.make filename (Int64.of_int this_chunk) in
-              Unixext.really_read ifd buffer 0 this_chunk;
-              
-              (* Only write the chunk if it's not all zeros, or if it's the 
first *)
-              if not (Zerocheck.is_all_zeros buffer this_chunk) || chunk_no=0 
|| last_chunk then
-                begin
-                  let csum = Sha1sum.sha1sum
-                    (fun checksumfd ->
-                      try
-                        Tar.write_block hdr (fun ofd -> 
Tar.Archive.multicast_n_string buffer 
-                          [ ofd; checksumfd ] this_chunk) ofd
-                      with
-                          Unix.Unix_error (a,b,c) as e ->
-                            if TaskHelper.is_cancelling ~__context
-                            then raise (Api_errors.Server_error 
(Api_errors.task_cancelled, []))
-                            else 
-                              (if b="write" 
-                              then raise (Api_errors.Server_error 
(Api_errors.client_error, [ExnHelper.string_of_exn e]))
-                                else raise e)
-                    ) in       
+
+                  let now = Unix.gettimeofday () in
+                  let time_since_transmission = now -. !last_transmission_time 
in
                   
-                  (* Write the checksum as a separate file *)
-                  let hdr' = Tar.Header.make (filename ^ checksum_extension) 
(Int64.of_int (String.length csum)) in
-                  Tar.write_block hdr' (fun ofd -> ignore(Unix.write ofd csum 
0 (String.length csum))) ofd
-                end;
+                  (* We always include the first and last blocks *)
+                  let first_or_last = chunk_no = 0 || last_chunk in
 
-              made_progress __context progress (Int64.of_int this_chunk);
-              stream_from (chunk_no + 1) (Int64.add offset chunk_size)
-            end;
+                  if time_since_transmission > 5. && not first_or_last then 
begin
+                        last_transmission_time := now;
+                        write_block ~__context filename "" ofd 0;
+                        (* no progress has been made *)
+                        stream_from (chunk_no + 1) offset
+                  end else begin
+                        let buffer = String.make this_chunk '\000' in
+                        Unixext.really_read ifd buffer 0 this_chunk;
+                        if not (Zerocheck.is_all_zeros buffer this_chunk) || 
first_or_last then begin
+                          last_transmission_time := now;
+                          write_block ~__context filename buffer ofd 
this_chunk;
+                        end;
+                        made_progress __context progress (Int64.of_int 
this_chunk);
+                        stream_from (chunk_no + 1) (Int64.add offset 
chunk_size);
+              end
+                end
         in
         stream_from 0 0L);
     debug "Finished streaming VDI" in
1 file changed, 48 insertions(+), 30 deletions(-)
ocaml/xapi/stream_vdi.ml |   78 ++++++++++++++++++++++++++++------------------


Attachment: xen-api.hg.patch
Description: Text Data

_______________________________________________
xen-api mailing list
xen-api@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/mailman/listinfo/xen-api
<Prev in Thread] Current Thread [Next in Thread>
  • [Xen-API] [PATCH] Work around firewalls killing connections in export code, David Scott <=