WARNING - OLD ARCHIVES

This is an archived copy of the Xen.org mailing list, which we have preserved to ensure that existing links to archives are not broken. The live archive, which contains the latest emails, can be found at http://lists.xen.org/
   
 
 
Xen 
 
Home Products Support Community News
 
   
 

xen-api

[Xen-API] [PATCH 1 of 3] CP-1883: Allow raw VDI import to receive an exp

To: xen-api@xxxxxxxxxxxxxxxxxxx
Subject: [Xen-API] [PATCH 1 of 3] CP-1883: Allow raw VDI import to receive an export-like chunked encoding
From: David Scott <dave.scott@xxxxxxxxxxxxx>
Date: Mon, 23 Aug 2010 13:06:40 +0100
Delivery-date: Mon, 23 Aug 2010 05:28:44 -0700
Envelope-to: www-data@xxxxxxxxxxxxxxxxxxx
In-reply-to: <patchbomb.1282565199@ely>
List-help: <mailto:xen-api-request@lists.xensource.com?subject=help>
List-id: Discussion of API issues surrounding Xen <xen-api.lists.xensource.com>
List-post: <mailto:xen-api@lists.xensource.com>
List-subscribe: <http://lists.xensource.com/mailman/listinfo/xen-api>, <mailto:xen-api-request@lists.xensource.com?subject=subscribe>
List-unsubscribe: <http://lists.xensource.com/mailman/listinfo/xen-api>, <mailto:xen-api-request@lists.xensource.com?subject=unsubscribe>
References: <patchbomb.1282565199@ely>
Sender: xen-api-bounces@xxxxxxxxxxxxxxxxxxx
User-agent: Mercurial-patchbomb/1.4.3
# HG changeset patch
# User David Scott <dave.scott@xxxxxxxxxxxxx>
# Date 1282565147 -3600
# Node ID 23fa063db91d4eae0179fb29179002a85a75cf31
# Parent  acfa0e8405cb12be60262655c8f21d97284e1a3b
CP-1883: Allow raw VDI import to receive an export-like chunked encoding.

The raw VDI import HTTP handler currently assumes the whole disk is being 
uploaded at once. Instead we add a 'chunked' mode which allows arbitrary-sized 
disk blocks to be selectively uploaded.

Signed-off-by: David Scott <dave.scott@xxxxxxxxxxxxx>

diff -r acfa0e8405cb -r 23fa063db91d ocaml/xapi/OMakefile
--- a/ocaml/xapi/OMakefile      Mon Aug 23 13:03:21 2010 +0100
+++ b/ocaml/xapi/OMakefile      Mon Aug 23 13:05:47 2010 +0100
@@ -73,6 +73,7 @@
        console \
        xen_helpers \
        importexport \
+       sparse_encoding \
        create_storage \
        create_networks \
        xapi_fist \
diff -r acfa0e8405cb -r 23fa063db91d ocaml/xapi/import_raw_vdi.ml
--- a/ocaml/xapi/import_raw_vdi.ml      Mon Aug 23 13:03:21 2010 +0100
+++ b/ocaml/xapi/import_raw_vdi.ml      Mon Aug 23 13:05:47 2010 +0100
@@ -20,9 +20,13 @@
 
 open Http
 open Importexport
+open Sparse_encoding
 open Unixext
 open Pervasiveext
 
+let receive_chunks (s: Unix.file_descr) (fd: Unix.file_descr) = 
+       Chunk.fold (fun () -> Chunk.write fd) () s
+
 let vdi_of_req ~__context (req: request) = 
        let vdi = 
                if List.mem_assoc "vdi" req.Http.query
@@ -37,6 +41,7 @@
   Xapi_http.with_context "Importing raw VDI" req s
     (fun __context ->
       let vdi = vdi_of_req ~__context req in
+      let chunked = List.mem_assoc "chunked" req.Http.query in
       try
        match req.transfer_encoding, req.content_length with
        | Some "chunked", _ ->
@@ -56,7 +61,9 @@
                      finally 
                        (fun () -> 
                           try
-                            Unixext.copy_file ~limit:len s fd;
+                            if chunked
+                            then receive_chunks s fd
+                            else ignore(Unixext.copy_file ~limit:len s fd);
                             Unixext.fsync fd
                           with Unix.Unix_error(Unix.EIO, _, _) ->
                             raise (Api_errors.Server_error 
(Api_errors.vdi_io_error, ["Device I/O errors"]))
diff -r acfa0e8405cb -r 23fa063db91d ocaml/xapi/importexport.ml
--- a/ocaml/xapi/importexport.ml        Mon Aug 23 13:03:21 2010 +0100
+++ b/ocaml/xapi/importexport.ml        Mon Aug 23 13:05:47 2010 +0100
@@ -212,3 +212,4 @@
                         Helpers.log_exn_continue "executing cleanup action" 
(action __context rpc) session_id) x
         )
     )
+
diff -r acfa0e8405cb -r 23fa063db91d ocaml/xapi/sparse_encoding.ml
--- /dev/null   Thu Jan 01 00:00:00 1970 +0000
+++ b/ocaml/xapi/sparse_encoding.ml     Mon Aug 23 13:05:47 2010 +0100
@@ -0,0 +1,122 @@
+(*
+ * Copyright (C) 2010 Citrix Systems Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation; version 2.1 only. with the special
+ * exception on linking described in file LICENSE.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *)
+(** Utility functions for reading and writing disk blocks to/from a network 
stream.
+ * @group Import and Export
+ *)
+
+module Unmarshal = struct
+       let int64 (s, offset) = 
+               let (<<) a b = Int64.shift_left a b
+               and (||) a b = Int64.logor a b in
+               let a = Int64.of_int (int_of_char (s.[offset + 0])) 
+               and b = Int64.of_int (int_of_char (s.[offset + 1])) 
+               and c = Int64.of_int (int_of_char (s.[offset + 2])) 
+               and d = Int64.of_int (int_of_char (s.[offset + 3]))
+               and e = Int64.of_int (int_of_char (s.[offset + 4]))
+               and f = Int64.of_int (int_of_char (s.[offset + 5]))
+               and g = Int64.of_int (int_of_char (s.[offset + 6]))
+               and h = Int64.of_int (int_of_char (s.[offset + 7])) in
+               (a << 0) || (b << 8) || (c << 16) || (d << 24) || (e << 32) || 
(f << 40) || (g << 48) || (h << 56), 
+               (s, offset + 8)
+       let int32 (s, offset) = 
+               let (<<) a b = Int32.shift_left a b
+               and (||) a b = Int32.logor a b in
+               let a = Int32.of_int (int_of_char (s.[offset + 0])) 
+               and b = Int32.of_int (int_of_char (s.[offset + 1])) 
+               and c = Int32.of_int (int_of_char (s.[offset + 2])) 
+               and d = Int32.of_int (int_of_char (s.[offset + 3])) in
+               (a << 0) || (b << 8) || (c << 16) || (d << 24), (s, offset + 4)
+end
+
+module Marshal = struct
+       let int64 x = 
+               let (>>) a b = Int64.shift_right_logical a b
+               and (&&) a b = Int64.logand a b in
+               let a = (x >> 0) && 0xffL 
+               and b = (x >> 8) && 0xffL
+               and c = (x >> 16) && 0xffL
+               and d = (x >> 24) && 0xffL
+               and e = (x >> 32) && 0xffL
+               and f = (x >> 40) && 0xffL
+               and g = (x >> 48) && 0xffL
+               and h = (x >> 56) && 0xffL in
+               let result = String.make 8 '\000' in
+               result.[0] <- char_of_int (Int64.to_int a);
+               result.[1] <- char_of_int (Int64.to_int b);
+               result.[2] <- char_of_int (Int64.to_int c);
+               result.[3] <- char_of_int (Int64.to_int d);
+               result.[4] <- char_of_int (Int64.to_int e);
+               result.[5] <- char_of_int (Int64.to_int f);
+               result.[6] <- char_of_int (Int64.to_int g);
+               result.[7] <- char_of_int (Int64.to_int h);
+               result
+       let int32 x = 
+               let (>>) a b = Int32.shift_right_logical a b
+               and (&&) a b = Int32.logand a b in
+               let a = (x >> 0) && 0xffl 
+               and b = (x >> 8) && 0xffl
+               and c = (x >> 16) && 0xffl
+               and d = (x >> 24) && 0xffl in
+               let result = String.make 4 '\000' in
+               result.[0] <- char_of_int (Int32.to_int a);
+               result.[1] <- char_of_int (Int32.to_int b);
+               result.[2] <- char_of_int (Int32.to_int c);
+               result.[3] <- char_of_int (Int32.to_int d);
+               result
+
+end
+
+module Chunk = struct
+       (** Represents an single block of data to write *)
+       type t = {
+               start: int64;
+               data: string;
+       }
+
+       let really_write fd offset buf off len = 
+               ignore(Unix.LargeFile.lseek fd offset Unix.SEEK_SET);
+               let n = Unix.write fd buf off len in
+               if n < len 
+               then failwith "Short write: attempted to write %d bytes at %Ld, 
only wrote %d" len offset n
+
+       (** Writes a single block of data to the output device *)
+       let write fd x = really_write fd x.start x.data 0 (String.length x.data)
+
+       (** Reads a type t from a file descriptor *)
+       let unmarshal fd = 
+               let buf = String.make 12 '\000' in
+               Unixext.really_read fd buf 0 (String.length buf);
+               let stream = (buf, 0) in
+               let start, stream = Unmarshal.int64 stream in
+               let len, stream = Unmarshal.int32 stream in
+               let payload = String.make (Int32.to_int len) '\000' in
+               Unixext.really_read fd payload 0 (String.length payload);
+               { start = start; data = payload }
+
+       (** Writes a type t from a file descriptor *)
+       let marshal fd x = 
+               let start' = Marshal.int64 x.start in
+               let len' = Marshal.int32 (Int32.of_int (String.length x.data)) 
in
+               really_write fd 0L start' 0 (String.length start');
+               really_write fd 8L len' 0 (String.length len');
+               really_write fd 12L x.data 0 (String.length x.data)
+
+       (** Fold [f] across all ts unmarshalled from [fd] *)
+       let rec fold f init fd = 
+               let x = unmarshal fd in
+               if x.data = "" 
+               then init
+               else fold f (f init x) fd
+end
+
 ocaml/xapi/OMakefile          |    1 +
 ocaml/xapi/import_raw_vdi.ml  |    9 ++-
 ocaml/xapi/importexport.ml    |    1 +
 ocaml/xapi/sparse_encoding.ml |  122 ++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 132 insertions(+), 1 deletions(-)


Attachment: xen-api.hg-1.patch
Description: Text Data

_______________________________________________
xen-api mailing list
xen-api@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/mailman/listinfo/xen-api