diff --git a/CHANGES.md b/CHANGES.md index 632e2c4..b07a088 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,20 +4,20 @@ (#119, #120, @MisterDA) - Support GNU LongLink and LongName. Prior, `Tar.HeaderWriter` and `Tar.HeaderReader` supported both, but `Tar.Header.Link` only had `LongLink` - and (de)serialized to (from) GNU LongName. + and (de)serialized to (from) GNU LongName (#127) - Compatibility level when reading / parsing is removed. Only GNU LongLink/LongName extensions were affected by the compatibility level when - reading. + reading (#127) - Add module types `Tar.HEADERREADER` and `Tar.HEADERWRITER` describing the - output of `Tar.HeaderReader` and `Tar.HeaderWriter` respectively. -- Types `Tar.READER.t` and `Tar.WRITER.t` are renamed to `io`. -- Add `write_global` function for writing a global PAX extended header. -- Rework IO-specific modules (tar-unix etc.) harmonizing them. + output of `Tar.HeaderReader` and `Tar.HeaderWriter` respectively (#127) +- Types `Tar.READER.t` and `Tar.WRITER.t` are renamed to `io` (#127) +- Add `write_global` function for writing a global PAX extended header (#127) +- Rework IO-specific modules (tar-unix etc.) harmonizing them (#127) - Avoid exceptions in tar and use result instead. The exceptions - `End_of_stream` and `Checksum_mismatch` are removed. -- Remove the `Tar_cstruct` module as it was unused. -- Remove debug printers. -- Finally remove the unused camlp-streams dependency. + `End_of_stream` and `Checksum_mismatch` are removed (#127) +- Remove the `Tar_cstruct` module as it was unused (#127) +- Remove debug printers (#127) +- Finally remove the unused camlp-streams dependency (#127) ## v2.6.0 (2023-09-07) diff --git a/bin/otar.ml b/bin/otar.ml index 4c40592..33de886 100644 --- a/bin/otar.ml +++ b/bin/otar.ml @@ -22,22 +22,13 @@ module Tar_gz = Tar_gz.Make let return x = x end) (struct type out_channel = Stdlib.out_channel type 'a io = 'a - let really_write oc cs = - let str = Cstruct.to_string cs in + let really_write oc str = output_string oc str end) (struct type in_channel = Stdlib.in_channel type 'a io = 'a - let really_read ic cs = - let len = Cstruct.length cs in - let buf = Bytes.create len in - really_input ic buf 0 len ; - Cstruct.blit_from_bytes buf 0 cs 0 len - let skip ic len = really_read ic (Cstruct.create len) - let read ic cs = - let max = Cstruct.length cs in - let buf = Bytes.create max in - let len = input ic buf 0 max in - Cstruct.blit_from_bytes buf 0 cs 0 len ; len end) + let read ic buf = + input ic buf 0 (Bytes.length buf) + end) let ( / ) = Filename.concat @@ -107,7 +98,7 @@ let bytes_to_size ?(decimals = 2) ppf = function let list filename = let ic = open_in filename in - let ic = Tar_gz.of_in_channel ~internal:(Cstruct.create 0x1000) ic in + let ic = Tar_gz.of_in_channel ~internal:(De.bigstring_create 0x1000) ic in let rec go global () = match Tar_gz.HeaderReader.read ~global ic with | Ok (hdr, global) -> Format.printf "%s (%s, %a)\n%!" diff --git a/dune-project b/dune-project index 401fc94..480654e 100644 --- a/dune-project +++ b/dune-project @@ -32,7 +32,6 @@ (tags ("org:xapi-project" "org:mirage")) (depends (ocaml (>= 4.08.0)) - (cstruct (>= 6.0.0)) (decompress (>= 1.5.1)) ) ) @@ -47,8 +46,6 @@ (tags ("org:xapi-project" "org:mirage")) (depends (ocaml (>= 4.08.0)) - (cstruct (>= 6.0.0)) - cstruct-lwt lwt (tar (= :version)) ) @@ -66,7 +63,7 @@ (conflicts (result (< 1.5))) (depends (ocaml (>= 4.08.0)) - (cstruct (>= 1.9.0)) + (cstruct (>= 6.0.0)) (lwt (>= 5.6.0)) (mirage-block (>= 2.0.0)) (mirage-clock (>= 4.0.0)) diff --git a/eio/tar_eio.ml b/eio/tar_eio.ml index 27adba2..553b126 100644 --- a/eio/tar_eio.ml +++ b/eio/tar_eio.ml @@ -27,7 +27,11 @@ end module Io = struct type in_channel = Flow.source type 'a io = 'a - let really_read f b = Flow.read_exact f b + let really_read f b = + let len = Bytes.length b in + let cs = Cstruct.create len in + Flow.read_exact f cs; + Cstruct.blit_to_bytes cs 0 b 0 len let skip f (n: int) = let buffer_size = 32768 in let buffer = Cstruct.create buffer_size in @@ -36,15 +40,18 @@ module Io = struct else let amount = min n buffer_size in let block = Cstruct.sub buffer 0 amount in - really_read f block; + Flow.read_exact f block; loop (n - amount) in loop n type out_channel = Flow.sink - let really_write f b = Flow.write f [ b ] + let really_write f str = Flow.write f [ Cstruct.of_string str ] end -include Io +let really_read = Flow.read_exact +let skip = Io.skip +let really_write f b = Flow.write f [ b ] + module HeaderReader = Tar.HeaderReader(Monad)(Io) module HeaderWriter = Tar.HeaderWriter(Monad)(Io) diff --git a/lib/dune b/lib/dune index 1718831..60befad 100644 --- a/lib/dune +++ b/lib/dune @@ -2,8 +2,7 @@ (name tar) (modules tar) (public_name tar) - (wrapped false) - (libraries cstruct)) + (wrapped false)) (library (name tar_gz) diff --git a/lib/tar.ml b/lib/tar.ml index 08e9b61..ce5f646 100644 --- a/lib/tar.ml +++ b/lib/tar.ml @@ -34,28 +34,32 @@ module Header = struct String.(trim (map (function '\000' -> ' ' | x -> x) s)) (** Unmarshal an integer field (stored as 0-padded octal) *) - let unmarshal_int x = - let tmp = "0o0" ^ (trim_numerical x) in + let unmarshal_int ~off ~len x = + let tmp = "0o0" ^ (trim_numerical (String.sub x off len)) in try Ok (int_of_string tmp) with Failure msg -> Error (`Unmarshal (Printf.sprintf "%s: failed to parse integer %S" msg tmp)) (** Unmarshal an int64 field (stored as 0-padded octal) *) - let unmarshal_int64 x = - let tmp = "0o0" ^ (trim_numerical x) in + let unmarshal_int64 ~off ~len x = + let tmp = "0o0" ^ (trim_numerical (String.sub x off len)) in try Ok (Int64.of_string tmp) with Failure msg -> Error (`Unmarshal (Printf.sprintf "%s: failed to parse int64 %S" msg tmp)) (** Unmarshal a string *) - let unmarshal_string x = + let unmarshal_string ?(off = 0) ?len x = + let len = Option.value ~default:(String.length x - off) len in try - let first_0 = String.index x '\000' in - Ok (String.sub x 0 first_0) + let first_0 = String.index_from x off '\000' in + if first_0 - off < len then + Ok (String.sub x off (first_0 - off)) + else + raise Not_found with Not_found -> - Ok x + Ok (String.sub x off len) (** Marshal an integer field of size 'n' *) let marshal_int x n = @@ -136,113 +140,98 @@ module Header = struct let sizeof_hdr_prefix = 155 let get_hdr_file_name buf = - unmarshal_string - (Cstruct.to_string ~off:hdr_file_name_off ~len:sizeof_hdr_file_name buf) + unmarshal_string ~off:hdr_file_name_off ~len:sizeof_hdr_file_name buf let set_hdr_file_name buf v = let v = marshal_string v sizeof_hdr_file_name in - Cstruct.blit_from_string v 0 buf hdr_file_name_off sizeof_hdr_file_name + Bytes.blit_string v 0 buf hdr_file_name_off sizeof_hdr_file_name let get_hdr_file_mode buf = - unmarshal_int - (Cstruct.to_string ~off:hdr_file_mode_off ~len:sizeof_hdr_file_mode buf) + unmarshal_int ~off:hdr_file_mode_off ~len:sizeof_hdr_file_mode buf let set_hdr_file_mode buf v = let v = marshal_int v sizeof_hdr_file_mode in - Cstruct.blit_from_string v 0 buf hdr_file_mode_off sizeof_hdr_file_mode + Bytes.blit_string v 0 buf hdr_file_mode_off sizeof_hdr_file_mode let get_hdr_user_id buf = - unmarshal_int - (Cstruct.to_string ~off:hdr_user_id_off ~len:sizeof_hdr_user_id buf) + unmarshal_int ~off:hdr_user_id_off ~len:sizeof_hdr_user_id buf let set_hdr_user_id buf v = let v = marshal_int v sizeof_hdr_user_id in - Cstruct.blit_from_string v 0 buf hdr_user_id_off sizeof_hdr_user_id + Bytes.blit_string v 0 buf hdr_user_id_off sizeof_hdr_user_id let get_hdr_group_id buf = - unmarshal_int - (Cstruct.to_string ~off:hdr_group_id_off ~len:sizeof_hdr_group_id buf) + unmarshal_int ~off:hdr_group_id_off ~len:sizeof_hdr_group_id buf let set_hdr_group_id buf v = let v = marshal_int v sizeof_hdr_group_id in - Cstruct.blit_from_string v 0 buf hdr_group_id_off sizeof_hdr_group_id + Bytes.blit_string v 0 buf hdr_group_id_off sizeof_hdr_group_id let get_hdr_file_size buf = - unmarshal_int64 - (Cstruct.to_string ~off:hdr_file_size_off ~len:sizeof_hdr_file_size buf) + unmarshal_int64 ~off:hdr_file_size_off ~len:sizeof_hdr_file_size buf let set_hdr_file_size buf v = let v = marshal_int64 v sizeof_hdr_file_size in - Cstruct.blit_from_string v 0 buf hdr_file_size_off sizeof_hdr_file_size + Bytes.blit_string v 0 buf hdr_file_size_off sizeof_hdr_file_size let get_hdr_mod_time buf = - unmarshal_int64 - (Cstruct.to_string ~off:hdr_mod_time_off ~len:sizeof_hdr_mod_time buf) + unmarshal_int64 ~off:hdr_mod_time_off ~len:sizeof_hdr_mod_time buf let set_hdr_mod_time buf v = let v = marshal_int64 v sizeof_hdr_mod_time in - Cstruct.blit_from_string v 0 buf hdr_mod_time_off sizeof_hdr_mod_time + Bytes.blit_string v 0 buf hdr_mod_time_off sizeof_hdr_mod_time let get_hdr_chksum buf = - unmarshal_int64 - (Cstruct.to_string ~off:hdr_chksum_off ~len:sizeof_hdr_chksum buf) + unmarshal_int64 ~off:hdr_chksum_off ~len:sizeof_hdr_chksum buf let set_hdr_chksum buf v = let v = marshal_int64 v sizeof_hdr_chksum in - Cstruct.blit_from_string v 0 buf hdr_chksum_off sizeof_hdr_chksum + Bytes.blit_string v 0 buf hdr_chksum_off sizeof_hdr_chksum - let get_hdr_link_indicator buf = Cstruct.get_char buf hdr_link_indicator_off + let get_hdr_link_indicator buf = String.get buf hdr_link_indicator_off let set_hdr_link_indicator buf v = - Cstruct.set_char buf hdr_link_indicator_off v + Bytes.set buf hdr_link_indicator_off v let get_hdr_link_name buf = - unmarshal_string - (Cstruct.to_string ~off:hdr_link_name_off ~len:sizeof_hdr_link_name buf) + unmarshal_string ~off:hdr_link_name_off ~len:sizeof_hdr_link_name buf let set_hdr_link_name buf v = let v = marshal_string v sizeof_hdr_link_name in - Cstruct.blit_from_string v 0 buf hdr_link_name_off sizeof_hdr_link_name + Bytes.blit_string v 0 buf hdr_link_name_off sizeof_hdr_link_name let get_hdr_magic buf = - unmarshal_string - (Cstruct.to_string ~off:hdr_magic_off ~len:sizeof_hdr_magic buf) + unmarshal_string ~off:hdr_magic_off ~len:sizeof_hdr_magic buf let set_hdr_magic buf v = let v = marshal_string v sizeof_hdr_magic in - Cstruct.blit_from_string v 0 buf hdr_magic_off sizeof_hdr_magic + Bytes.blit_string v 0 buf hdr_magic_off sizeof_hdr_magic let _get_hdr_version buf = - unmarshal_string - (Cstruct.to_string ~off:hdr_version_off ~len:sizeof_hdr_version buf) + unmarshal_string ~off:hdr_version_off ~len:sizeof_hdr_version buf let set_hdr_version buf v = let v = marshal_string v sizeof_hdr_version in - Cstruct.blit_from_string v 0 buf hdr_version_off sizeof_hdr_version + Bytes.blit_string v 0 buf hdr_version_off sizeof_hdr_version let get_hdr_uname buf = - unmarshal_string - (Cstruct.to_string ~off:hdr_uname_off ~len:sizeof_hdr_uname buf) + unmarshal_string ~off:hdr_uname_off ~len:sizeof_hdr_uname buf let set_hdr_uname buf v = let v = marshal_string v sizeof_hdr_uname in - Cstruct.blit_from_string v 0 buf hdr_uname_off sizeof_hdr_uname + Bytes.blit_string v 0 buf hdr_uname_off sizeof_hdr_uname let get_hdr_gname buf = - unmarshal_string - (Cstruct.to_string ~off:hdr_gname_off ~len:sizeof_hdr_gname buf) + unmarshal_string ~off:hdr_gname_off ~len:sizeof_hdr_gname buf let set_hdr_gname buf v = let v = marshal_string v sizeof_hdr_gname in - Cstruct.blit_from_string v 0 buf hdr_gname_off sizeof_hdr_gname + Bytes.blit_string v 0 buf hdr_gname_off sizeof_hdr_gname let get_hdr_devmajor buf = - unmarshal_int - (Cstruct.to_string ~off:hdr_devmajor_off ~len:sizeof_hdr_devmajor buf) + unmarshal_int ~off:hdr_devmajor_off ~len:sizeof_hdr_devmajor buf let set_hdr_devmajor buf v = let v = marshal_int v sizeof_hdr_devmajor in - Cstruct.blit_from_string v 0 buf hdr_devmajor_off sizeof_hdr_devmajor + Bytes.blit_string v 0 buf hdr_devmajor_off sizeof_hdr_devmajor let get_hdr_devminor buf = - unmarshal_int - (Cstruct.to_string ~off:hdr_devminor_off ~len:sizeof_hdr_devminor buf) + unmarshal_int ~off:hdr_devminor_off ~len:sizeof_hdr_devminor buf let set_hdr_devminor buf v = let v = marshal_int v sizeof_hdr_devminor in - Cstruct.blit_from_string v 0 buf hdr_devminor_off sizeof_hdr_devminor + Bytes.blit_string v 0 buf hdr_devminor_off sizeof_hdr_devminor let get_hdr_prefix buf = - unmarshal_string - (Cstruct.to_string ~off:hdr_prefix_off ~len:sizeof_hdr_prefix buf) + unmarshal_string ~off:hdr_prefix_off ~len:sizeof_hdr_prefix buf let set_hdr_prefix buf v = let v = marshal_string v sizeof_hdr_prefix in - Cstruct.blit_from_string v 0 buf hdr_prefix_off sizeof_hdr_prefix + Bytes.blit_string v 0 buf hdr_prefix_off sizeof_hdr_prefix type compatibility = | OldGNU @@ -367,11 +356,10 @@ module Header = struct @ (match t.file_size with None -> [] | Some x -> [ "size", Int64.to_string x ]) @ (match t.user_id with None -> [] | Some x -> [ "uid", string_of_int x ]) @ (match t.uname with None -> [] | Some x -> [ "uname", x ]) in - let txt = String.concat "" (List.map (fun (k, v) -> - let length = 8 + 1 + (String.length k) + 1 + (String.length v) + 1 in - Printf.sprintf "%08d %s=%s\n" length k v - ) pairs) in - Cstruct.of_string txt + String.concat "" (List.map (fun (k, v) -> + let length = 8 + 1 + (String.length k) + 1 + (String.length v) + 1 in + Printf.sprintf "%08d %s=%s\n" length k v + ) pairs) let merge global extended = match global with @@ -412,41 +400,30 @@ module Header = struct - the cannot contain an equals sign - the is the number of octets of the record, including \n *) - let find buffer char = - let rec loop i = - if i = Cstruct.length buffer - then None - else if Cstruct.get_char buffer i = char - then Some i - else loop (i + 1) - in - loop 0 + let find start char = + try Ok (String.index_from c start char) + with Not_found -> Error (`Unmarshal "Failed to decode pax extended header record") in - let rec loop remaining = - if Cstruct.length remaining = 0 - then Ok [] + let slen = String.length c in + let rec loop acc idx = + if idx >= slen + then Ok (List.rev acc) else begin (* Find the space, then decode the length *) - match find remaining ' ' with - | None -> Error (`Unmarshal "Failed to decode pax extended header record") - | Some i -> - let length = int_of_string @@ Cstruct.to_string @@ Cstruct.sub remaining 0 i in - let record = Cstruct.sub remaining 0 length in - let remaining = Cstruct.shift remaining length in - begin match find record '=' with - | None -> Error (`Unmarshal "Failed to decode pax extended header record") - | Some j -> - let keyword = Cstruct.to_string @@ Cstruct.sub record (i + 1) (j - i - 1) in - let v = Cstruct.to_string @@ Cstruct.sub record (j + 1) (Cstruct.length record - j - 2) in - let* rem = loop remaining in - Ok ((keyword, v) :: rem) - end + let* i = find idx ' ' in + let* length = + try Ok (int_of_string (String.sub c idx (i - idx))) with + Failure _ -> Error (`Unmarshal "Failed to decode pax extended header record") + in + let* j = find i '=' in + let keyword = String.sub c (i + 1) (j - i - 1) in + let v = String.sub c (j + 1) (length - (j - idx) - 2) in + loop ((keyword, v) :: acc) (idx + length) end in - let* pairs = loop c in + let* pairs = loop [] 0 in let option name f = - if List.mem_assoc name pairs - then + if List.mem_assoc name pairs then let* v = f (List.assoc name pairs) in Ok (Some v) else @@ -519,7 +496,7 @@ module Header = struct let length = 512 (** A blank header block (two of these in series mark the end of the tar) *) - let zero_block = Cstruct.create length + let zero_block = String.make length '\000' (** Pretty-print the header record *) let to_detailed_string (x: t) = @@ -534,29 +511,29 @@ module Header = struct "{\n\t" ^ (String.concat "\n\t" (List.map (fun (k, v) -> k ^ ": " ^ v) table)) ^ "}" (** From an already-marshalled block, compute what the checksum should be *) - let checksum (x: Cstruct.t) : int64 = + let checksum x : int64 = (* Sum of all the byte values of the header with the checksum field taken as 8 ' ' (spaces) *) let result = ref 0 in let in_checksum_range i = i >= hdr_chksum_off && i < hdr_chksum_off + sizeof_hdr_chksum in - for i = 0 to Cstruct.length x - 1 do + for i = 0 to String.length x - 1 do let v = if in_checksum_range i then int_of_char ' ' else - Cstruct.get_uint8 x i + int_of_char (String.get x i) in result := !result + v done; Int64.of_int !result (** Unmarshal a header block, returning None if it's all zeroes *) - let unmarshal ?(extended = Extended.make ()) (c: Cstruct.t) + let unmarshal ?(extended = Extended.make ()) c : (t, [>`Zero_block | `Checksum_mismatch]) result = - if Cstruct.length c <> length then Error (`Unmarshal "buffer is not of block size") - else if Cstruct.equal zero_block c then Error `Zero_block + if String.length c <> length then Error (`Unmarshal "buffer is not of block size") + else if String.equal zero_block c then Error `Zero_block else let* chksum = get_hdr_chksum c in if checksum c <> chksum then Error `Checksum_mismatch @@ -668,7 +645,7 @@ module Header = struct in set_hdr_link_name c x.link_name; (* Finally, compute the checksum *) - let chksum = checksum c in + let chksum = checksum (Bytes.unsafe_to_string c) in set_hdr_chksum c chksum; Ok () @@ -682,7 +659,7 @@ module Header = struct (** Return the required zero-padding as a string *) let zero_padding (x: t) = let zero_padding_len = compute_zero_padding_length x in - Cstruct.sub zero_block 0 zero_padding_len + String.sub zero_block 0 zero_padding_len let to_sectors (x: t) = Int64.(div (add (pred (of_int length)) x.file_size) (of_int length)) @@ -697,14 +674,14 @@ end module type READER = sig type in_channel type 'a io - val really_read: in_channel -> Cstruct.t -> unit io + val really_read: in_channel -> bytes -> unit io val skip: in_channel -> int -> unit io end module type WRITER = sig type out_channel type 'a io - val really_write: out_channel -> Cstruct.t -> unit io + val really_write: out_channel -> string -> unit io end module type HEADERREADER = sig @@ -752,18 +729,18 @@ module HeaderReader(Async: ASYNC)(Reader: READER with type 'a io = 'a Async.t) = let read ~global (ifd: Reader.in_channel) : (Header.t * Header.Extended.t option, [ `Eof | `Fatal of [ `Checksum_mismatch | `Corrupt_pax_header | `Unmarshal of string ] ]) result t = (* We might need to read 2 headers at once if we encounter a Pax header *) - let buffer = Cstruct.create Header.length in - let real_header_buf = Cstruct.create Header.length in + let buffer = Bytes.make Header.length '\000' in + let real_header_buf = Bytes.make Header.length '\000' in let next_block global () = really_read ifd buffer >>= fun () -> - return (Header.unmarshal ?extended:global buffer) + return (Header.unmarshal ?extended:global (Bytes.unsafe_to_string buffer)) in let rec get_hdr ~next_longname ~next_longlink global () : (Header.t * Header.Extended.t option, [> `Eof | `Fatal of [ `Checksum_mismatch | `Corrupt_pax_header | `Unmarshal of string ] ]) result t = next_block global () >>= function | Ok x when x.Header.link_indicator = Header.Link.GlobalExtendedHeader -> - let extra_header_buf = Cstruct.create (Int64.to_int x.Header.file_size) in + let extra_header_buf = Bytes.make (Int64.to_int x.Header.file_size) '\000' in really_read ifd extra_header_buf >>= fun () -> skip ifd (Header.compute_zero_padding_length x) >>= fun () -> (* unmarshal merges the previous global (if any) with the @@ -771,31 +748,31 @@ module HeaderReader(Async: ASYNC)(Reader: READER with type 'a io = 'a Async.t) = let^* global = Result.map_error (fun e -> `Fatal e) - (Header.Extended.unmarshal ~global extra_header_buf) + (Header.Extended.unmarshal ~global (Bytes.unsafe_to_string extra_header_buf)) in get_hdr ~next_longname ~next_longlink (Some global) () | Ok x when x.Header.link_indicator = Header.Link.PerFileExtendedHeader -> - let extra_header_buf = Cstruct.create (Int64.to_int x.Header.file_size) in + let extra_header_buf = Bytes.make (Int64.to_int x.Header.file_size) '\000' in really_read ifd extra_header_buf >>= fun () -> skip ifd (Header.compute_zero_padding_length x) >>= fun () -> let^* extended = Result.map_error (fun e -> `Fatal e) - (Header.Extended.unmarshal ~global extra_header_buf) + (Header.Extended.unmarshal ~global (Bytes.unsafe_to_string extra_header_buf)) in really_read ifd real_header_buf >>= fun () -> let^* x = Result.map_error (fun _ -> `Fatal `Corrupt_pax_header) - (Header.unmarshal ~extended real_header_buf) + (Header.unmarshal ~extended (Bytes.unsafe_to_string real_header_buf)) in let x = fix_link_indicator x in return (Ok (x, global)) | Ok ({ Header.link_indicator = Header.Link.LongLink | Header.Link.LongName; _ } as x) when x.Header.file_name = longlink -> - let extra_header_buf = Cstruct.create (Int64.to_int x.Header.file_size) in + let extra_header_buf = Bytes.create (Int64.to_int x.Header.file_size) in really_read ifd extra_header_buf >>= fun () -> skip ifd (Header.compute_zero_padding_length x) >>= fun () -> - let name = Cstruct.to_string ~len:(Cstruct.length extra_header_buf - 1) extra_header_buf in + let name = String.sub (Bytes.unsafe_to_string extra_header_buf) 0 (Bytes.length extra_header_buf - 1) in let next_longlink = if x.Header.link_indicator = Header.Link.LongLink then Some name else next_longlink in let next_longname = if x.Header.link_indicator = Header.Link.LongName then Some name else next_longname in get_hdr ~next_longname ~next_longlink global () @@ -834,20 +811,20 @@ module HeaderWriter(Async: ASYNC)(Writer: WRITER with type 'a io = 'a Async.t) = let write_unextended ?level header fd = let level = Header.get_level level in - let buffer = Cstruct.create Header.length in let blank = {Header.file_name = longlink; file_mode = 0; user_id = 0; group_id = 0; mod_time = 0L; file_size = 0L; link_indicator = Header.Link.LongLink; link_name = ""; uname = "root"; gname = "root"; devmajor = 0; devminor = 0; extended = None} in (if level = Header.GNU then begin begin if String.length header.Header.link_name > Header.sizeof_hdr_link_name then begin let file_size = String.length header.Header.link_name + 1 in let blank = {blank with Header.file_size = Int64.of_int file_size} in + let buffer = Bytes.make Header.length '\000' in match Header.marshal ~level buffer { blank with link_indicator = Header.Link.LongLink } with | Error _ as e -> return e | Ok () -> - really_write fd buffer >>= fun () -> - let payload = Cstruct.of_string (header.Header.link_name ^ "\000") in + really_write fd (Bytes.unsafe_to_string buffer) >>= fun () -> + let payload = header.Header.link_name ^ "\000" in really_write fd payload >>= fun () -> really_write fd (Header.zero_padding blank) >>= fun () -> return (Ok ()) @@ -860,13 +837,14 @@ module HeaderWriter(Async: ASYNC)(Writer: WRITER with type 'a io = 'a Async.t) = if String.length header.Header.file_name > Header.sizeof_hdr_file_name then begin let file_size = String.length header.Header.file_name + 1 in let blank = {blank with Header.file_size = Int64.of_int file_size} in + let buffer = Bytes.make Header.length '\000' in match Header.marshal ~level buffer { blank with link_indicator = Header.Link.LongName } with | Error _ as e -> return e | Ok () -> - really_write fd buffer >>= fun () -> - let payload = Cstruct.of_string (header.Header.file_name ^ "\000") in + really_write fd (Bytes.unsafe_to_string buffer) >>= fun () -> + let payload = header.Header.file_name ^ "\000" in really_write fd payload >>= fun () -> really_write fd (Header.zero_padding blank) >>= fun () -> return (Ok ()) @@ -874,17 +852,16 @@ module HeaderWriter(Async: ASYNC)(Writer: WRITER with type 'a io = 'a Async.t) = return (Ok ()) end >>= function | Error _ as e -> return e - | Ok () -> - Cstruct.memset buffer 0; - return (Ok ()) + | Ok () -> return (Ok ()) end else return (Ok ())) >>= function | Error _ as e -> return e | Ok () -> + let buffer = Bytes.make Header.length '\000' in match Header.marshal ~level buffer header with | Error _ as e -> return e | Ok () -> - really_write fd buffer >>= fun () -> + really_write fd (Bytes.unsafe_to_string buffer) >>= fun () -> return (Ok ()) let write_extended ?level ~link_indicator hdr fd = @@ -895,7 +872,7 @@ module HeaderWriter(Async: ASYNC)(Writer: WRITER with type 'a io = 'a Async.t) = in let pax_payload = Header.Extended.marshal hdr in let pax = Header.make ~link_indicator link_indicator_name - (Int64.of_int @@ Cstruct.length pax_payload) in + (Int64.of_int @@ String.length pax_payload) in write_unextended ?level pax fd >>= function | Error _ as e -> return e | Ok () -> diff --git a/lib/tar.mli b/lib/tar.mli index d05eddd..aa11188 100644 --- a/lib/tar.mli +++ b/lib/tar.mli @@ -82,7 +82,7 @@ module Header : sig (** Unmarshal a pax Extended Header block. This header block may be preceded by [global] blocks which will override some fields. *) - val unmarshal : global:t option -> Cstruct.t -> (t, [> error ]) result + val unmarshal : global:t option -> string -> (t, [> error ]) result end (** Represents a standard archive (note checksum not stored). *) @@ -115,7 +115,7 @@ module Header : sig val length : int (** A blank header block (two of these in series mark the end of the tar). *) - val zero_block : Cstruct.t + val zero_block : string (** Pretty-print the header record. *) val to_detailed_string : t -> string @@ -123,17 +123,17 @@ module Header : sig (** Unmarshal a header block, returning [None] if it's all zeroes. This header block may be preceded by an [?extended] block which will override some fields. *) - val unmarshal : ?extended:Extended.t -> Cstruct.t -> (t, [`Zero_block | `Checksum_mismatch | `Unmarshal of string]) result + val unmarshal : ?extended:Extended.t -> string -> (t, [`Zero_block | `Checksum_mismatch | `Unmarshal of string]) result (** Marshal a header block, computing and inserting the checksum. *) - val marshal : ?level:compatibility -> Cstruct.t -> t -> (unit, [> `Msg of string ]) result + val marshal : ?level:compatibility -> bytes -> t -> (unit, [> `Msg of string ]) result (** Compute the amount of zero-padding required to round up the file size to a whole number of blocks. *) val compute_zero_padding_length : t -> int (** Return the required zero-padding as a string. *) - val zero_padding : t -> Cstruct.t + val zero_padding : t -> string (** [to_sectors t] is the number of sectors occupied by the data. *) val to_sectors: t -> int64 @@ -148,14 +148,14 @@ end module type READER = sig type in_channel type 'a io - val really_read: in_channel -> Cstruct.t -> unit io + val really_read: in_channel -> bytes -> unit io val skip: in_channel -> int -> unit io end module type WRITER = sig type out_channel type 'a io - val really_write: out_channel -> Cstruct.t -> unit io + val really_write: out_channel -> string -> unit io end module type HEADERREADER = sig diff --git a/lib/tar_gz.ml b/lib/tar_gz.ml index 1f4239c..e197482 100644 --- a/lib/tar_gz.ml +++ b/lib/tar_gz.ml @@ -15,11 +15,62 @@ *) module type READER = sig - include Tar.READER - - val read : in_channel -> Cstruct.t -> int io + type in_channel + type 'a io + val read : in_channel -> bytes -> int io end +external ba_get_int32_ne : De.bigstring -> int -> int32 = "%caml_bigstring_get32" +external ba_set_int32_ne : De.bigstring -> int -> int32 -> unit = "%caml_bigstring_set32" + +let bigstring_to_string ?(off= 0) ?len ba = + let len = match len with + | Some len -> len + | None -> De.bigstring_length ba - off in + let res = Bytes.create len in + let len0 = len land 3 in + let len1 = len asr 2 in + for i = 0 to len1 - 1 do + let i = i * 4 in + let v = ba_get_int32_ne ba i in + Bytes.set_int32_ne res i v + done; + for i = 0 to len0 - 1 do + let i = (len1 * 4) + i in + let v = Bigarray.Array1.get ba i in + Bytes.set res i v + done; + Bytes.unsafe_to_string res + +let bigstring_blit_string src ~src_off dst ~dst_off ~len = + let len0 = len land 3 in + let len1 = len asr 2 in + for i = 0 to len1 - 1 do + let i = i * 4 in + (* TODO: use String.get_int32_ne when ocaml-tar requires OCaml >= 4.13 *) + let v = Bytes.get_int32_ne (Bytes.unsafe_of_string src) (src_off + i) in + ba_set_int32_ne dst (dst_off + i) v + done; + for i = 0 to len0 - 1 do + let i = (len1 * 4) + i in + let v = String.get src (src_off + i) in + Bigarray.Array1.set dst (dst_off + i) v + done + +let bigstring_blit_bytes src ~src_off dst ~dst_off ~len = + let len0 = len land 3 in + let len1 = len asr 2 in + for i = 0 to len1 - 1 do + let i = i * 4 in + let v = ba_get_int32_ne src (src_off + i) in + Bytes.set_int32_ne dst (dst_off + i) v + done; + for i = 0 to len0 - 1 do + let i = (len1 * 4) + i in + let v = Bigarray.Array1.get src (src_off + i) in + Bytes.set dst (dst_off + i) v + done + module Make (Async : Tar.ASYNC) (Writer : Tar.WRITER with type 'a io = 'a Async.t) @@ -30,83 +81,84 @@ module Make module Gz_writer = struct type out_channel = { mutable gz : Gz.Def.encoder - ; ic_buffer : Cstruct.t - ; oc_buffer : Cstruct.t + ; ic_buffer : De.bigstring + ; oc_buffer : De.bigstring ; out_channel : Writer.out_channel } type 'a io = 'a Async.t - let really_write ({ gz; oc_buffer; out_channel; _ } as state) cs = + let really_write ({ gz; ic_buffer; oc_buffer; out_channel; _ } as state) str = let rec until_await gz = match Gz.Def.encode gz with - | `Await gz -> state.gz <- gz ; Async.return () + | `Await gz -> Async.return gz | `Flush gz -> - let max = Cstruct.length oc_buffer - Gz.Def.dst_rem gz in - Writer.really_write out_channel (Cstruct.sub oc_buffer 0 max) >>= fun () -> - let { Cstruct.buffer; off= cs_off; len= cs_len; } = oc_buffer in - until_await (Gz.Def.dst gz buffer cs_off cs_len) - | `End _gz -> assert false in - if Cstruct.length cs = 0 - then Async.return () - else ( let { Cstruct.buffer; off; len; } = cs in - let gz = Gz.Def.src gz buffer off len in - until_await gz ) + let len = De.bigstring_length oc_buffer - Gz.Def.dst_rem gz in + let str = bigstring_to_string oc_buffer ~off:0 ~len in + Writer.really_write out_channel str >>= fun () -> + until_await (Gz.Def.dst gz oc_buffer 0 (De.bigstring_length oc_buffer)) + | `End _gz -> assert false + and go gz (str, str_off, str_len) = + if str_len = 0 + then ( state.gz <- gz ; Async.return () ) + else ( let len = min str_len (De.bigstring_length ic_buffer) in + bigstring_blit_string str ~src_off:0 ic_buffer ~dst_off:0 ~len; + let gz = Gz.Def.src gz ic_buffer 0 len in + until_await gz >>= fun gz -> + go gz (str, str_off + len, str_len - len) ) in + go gz (str, 0, String.length str) end module Gz_reader = struct type in_channel = { mutable gz : Gz.Inf.decoder - ; ic_buffer : Cstruct.t - ; oc_buffer : Cstruct.t + ; ic_buffer : De.bigstring + ; oc_buffer : De.bigstring + ; tp_buffer : bytes ; in_channel : Reader.in_channel ; mutable pos : int } type 'a io = 'a Async.t let really_read - : in_channel -> Cstruct.t -> unit io - = fun ({ ic_buffer; oc_buffer; in_channel; _ } as state) res -> - let rec until_full_or_end gz res = + : in_channel -> bytes -> unit io + = fun ({ ic_buffer; oc_buffer; in_channel; tp_buffer; _ } as state) res -> + let rec until_full_or_end gz (res, res_off, res_len) = match Gz.Inf.decode gz with | `Flush gz -> - let max = Cstruct.length oc_buffer - Gz.Inf.dst_rem gz in - let len = min (Cstruct.length res) max in - Cstruct.blit oc_buffer 0 res 0 len ; + let max = De.bigstring_length oc_buffer - Gz.Inf.dst_rem gz in + let len = min res_len max in + bigstring_blit_bytes oc_buffer ~src_off:0 res ~dst_off:res_off ~len; if len < max then ( state.pos <- len ; state.gz <- gz ; Async.return () ) - else until_full_or_end (Gz.Inf.flush gz) (Cstruct.shift res len) + else until_full_or_end (Gz.Inf.flush gz) (res, res_off + len, res_len - len) | `End gz -> - let max = Cstruct.length oc_buffer - Gz.Inf.dst_rem gz in - let len = min (Cstruct.length res) max in - Cstruct.blit oc_buffer 0 res 0 len ; - if Cstruct.length res > len + let max = De.bigstring_length oc_buffer - Gz.Inf.dst_rem gz in + let len = min res_len max in + bigstring_blit_bytes oc_buffer ~src_off:0 res ~dst_off:res_off ~len; + if res_len > len then raise End_of_file else ( state.pos <- len ; state.gz <- gz ; Async.return () ) | `Await gz -> - Reader.read in_channel ic_buffer >>= fun len -> - let { Cstruct.buffer; off; len= _; } = ic_buffer in - let gz = Gz.Inf.src gz buffer off len in - until_full_or_end gz res + Reader.read in_channel tp_buffer >>= fun len -> + bigstring_blit_string (Bytes.unsafe_to_string tp_buffer) ~src_off:0 ic_buffer ~dst_off:0 ~len; + let gz = Gz.Inf.src gz ic_buffer 0 len in + until_full_or_end gz (res, res_off, res_len) | `Malformed err -> failwith ("gzip: " ^ err) in - let max = (Cstruct.length oc_buffer - Gz.Inf.dst_rem state.gz) - state.pos in - let len = min (Cstruct.length res) max in - Cstruct.blit oc_buffer state.pos res 0 len ; - + let max = (De.bigstring_length oc_buffer - Gz.Inf.dst_rem state.gz) - state.pos in + let len = min (Bytes.length res) max in + bigstring_blit_bytes oc_buffer ~src_off:state.pos res ~dst_off:0 ~len; if len < max then ( state.pos <- state.pos + len ; Async.return () ) - else ( let res = Cstruct.shift res len in - until_full_or_end (Gz.Inf.flush state.gz) res ) - - let skip - : in_channel -> int -> unit io - = fun state len -> - let oc_buffer = Cstruct.create len in - really_read state oc_buffer + else until_full_or_end (Gz.Inf.flush state.gz) (res, len, Bytes.length res - len) + + let skip : in_channel -> int -> unit io = fun state len -> + let res = Bytes.create len in + really_read state res end module HeaderWriter = Tar.HeaderWriter (Async) (Gz_writer) @@ -115,11 +167,10 @@ module Make type in_channel = Gz_reader.in_channel let of_in_channel ~internal:oc_buffer in_channel = - let { Cstruct.buffer; off; len; } = oc_buffer in - let o = Bigarray.Array1.sub buffer off len in - { Gz_reader.gz= Gz.Inf.decoder `Manual ~o + { Gz_reader.gz= Gz.Inf.decoder `Manual ~o:oc_buffer ; oc_buffer - ; ic_buffer= Cstruct.create 0x1000 + ; ic_buffer= De.bigstring_create 0x1000 + ; tp_buffer= Bytes.create 0x1000 ; in_channel ; pos= 0 } @@ -129,14 +180,13 @@ module Make type out_channel = Gz_writer.out_channel let of_out_channel ?bits:(w_bits= 15) ?q:(q_len= 0x1000) ~level ~mtime os out_channel = - let ic_buffer = Cstruct.create (4 * 4 * 1024) in - let oc_buffer = Cstruct.create 4096 in + let ic_buffer = De.bigstring_create (4 * 4 * 1024) in + let oc_buffer = De.bigstring_create 4096 in let gz = let w = De.Lz77.make_window ~bits:w_bits in let q = De.Queue.create q_len in Gz.Def.encoder `Manual `Manual ~mtime os ~q ~w ~level in - let { Cstruct.buffer; off; len; } = oc_buffer in - let gz = Gz.Def.dst gz buffer off len in + let gz = Gz.Def.dst gz oc_buffer 0 (De.bigstring_length oc_buffer) in { Gz_writer.gz; ic_buffer; oc_buffer; out_channel; } let write_block ?level hdr ({ Gz_writer.ic_buffer= buf; oc_buffer; out_channel; _ } as state) block = @@ -145,23 +195,22 @@ module Make | Ok () -> (* XXX(dinosaure): we can refactor this codec with [Gz_writer.really_write] but this loop saves and uses [ic_buffer]/[buf] to avoid extra - allocations on the case between [string] and [Cstruct.t]. *) + allocations on the case between [string] and [bigstring]. *) let rec deflate (str, off, len) gz = match Gz.Def.encode gz with | `Await gz -> if len = 0 then block () >>= function | None -> state.gz <- gz ; Async.return () | Some str -> deflate (str, 0, String.length str) gz - else ( let len' = min len (Cstruct.length buf) in - Cstruct.blit_from_string str off buf 0 len' ; - let { Cstruct.buffer; off= cs_off; len= _; } = buf in + else ( let len' = min len (De.bigstring_length buf) in + bigstring_blit_string str ~src_off:off buf ~dst_off:0 ~len:len'; deflate (str, off + len', len - len') - (Gz.Def.src gz buffer cs_off len') ) + (Gz.Def.src gz buf 0 len') ) | `Flush gz -> - let max = Cstruct.length oc_buffer - Gz.Def.dst_rem gz in - Writer.really_write out_channel (Cstruct.sub oc_buffer 0 max) >>= fun () -> - let { Cstruct.buffer; off= cs_off; len= cs_len; } = oc_buffer in - deflate (str, off, len) (Gz.Def.dst gz buffer cs_off cs_len) + let len = De.bigstring_length oc_buffer - Gz.Def.dst_rem gz in + let out = bigstring_to_string oc_buffer ~len in + Writer.really_write out_channel out >>= fun () -> + deflate (str, off, len) (Gz.Def.dst gz oc_buffer 0 (De.bigstring_length oc_buffer)) | `End _gz -> assert false in deflate ("", 0, 0) state.gz >>= fun () -> Gz_writer.really_write state (Tar.Header.zero_padding hdr) >>= fun () -> @@ -173,12 +222,11 @@ module Make let rec until_end gz = match Gz.Def.encode gz with | `Await _gz -> assert false | `Flush gz | `End gz as flush_or_end -> - let max = Cstruct.length oc_buffer - Gz.Def.dst_rem gz in - Writer.really_write out_channel (Cstruct.sub oc_buffer 0 max) >>= fun () -> + let max = De.bigstring_length oc_buffer - Gz.Def.dst_rem gz in + Writer.really_write out_channel (bigstring_to_string oc_buffer ~len:max) >>= fun () -> match flush_or_end with | `Flush gz -> - let { Cstruct.buffer; off= cs_off; len= cs_len; } = oc_buffer in - until_end (Gz.Def.dst gz buffer cs_off cs_len) + until_end (Gz.Def.dst gz oc_buffer 0 (De.bigstring_length oc_buffer)) | `End _gz -> Async.return () in until_end (Gz.Def.src state.gz De.bigstring_empty 0 0) end diff --git a/lib/tar_gz.mli b/lib/tar_gz.mli index b4198b0..de18b76 100644 --- a/lib/tar_gz.mli +++ b/lib/tar_gz.mli @@ -15,9 +15,9 @@ *) module type READER = sig - include Tar.READER - - val read : in_channel -> Cstruct.t -> int io + type in_channel + type 'a io + val read : in_channel -> bytes -> int io end module Make @@ -27,9 +27,9 @@ module Make : sig type in_channel - val of_in_channel : internal:Cstruct.t -> Reader.in_channel -> in_channel + val of_in_channel : internal:De.bigstring -> Reader.in_channel -> in_channel - val really_read : in_channel -> Cstruct.t -> unit Async.t + val really_read : in_channel -> bytes -> unit Async.t (** [really_read fd buf] fills [buf] with data from [fd] or raises {!Stdlib.End_of_file}. *) diff --git a/lib_test/global_extended_headers_test.ml b/lib_test/global_extended_headers_test.ml index 2affcde..a5ae6de 100644 --- a/lib_test/global_extended_headers_test.ml +++ b/lib_test/global_extended_headers_test.ml @@ -3,8 +3,7 @@ let level = Tar.Header.Ustar module Writer = struct type out_channel = Stdlib.out_channel type 'a io = 'a - let really_write oc cs = - let str = Cstruct.to_string cs in + let really_write oc str = output_string oc str end @@ -17,17 +16,14 @@ module HW = Tar.HeaderWriter module Reader = struct type in_channel = Stdlib.in_channel type 'a io = 'a - let really_read ic cs = - let len = Cstruct.length cs in - let buf = Bytes.create len in - really_input ic buf 0 len ; - Cstruct.blit_from_bytes buf 0 cs 0 len - let skip ic len = really_read ic (Cstruct.create len) - let read ic cs = - let max = Cstruct.length cs in - let buf = Bytes.create max in - let len = input ic buf 0 max in - Cstruct.blit_from_bytes buf 0 cs 0 len ; len + let really_read ic buf = + really_input ic buf 0 (Bytes.length buf) + let skip ic len = + let cur = pos_in ic in + seek_in ic (cur + len) + let read ic buf = + let max = Bytes.length buf in + input ic buf 0 max end module HR = Tar.HeaderReader @@ -47,7 +43,6 @@ let make_file = let hdr = Tar.Header.make name 0L in hdr, fun cout -> Tar.Header.zero_padding hdr - |> Cstruct.to_string |> output_string cout (* Tests that global and per-file extended headers correctly override @@ -102,7 +97,8 @@ let use_global_extended_headers _test_ctxt = Alcotest.(check int) "expected user" 1000 hdr.Tar.Header.user_id; let to_skip = Tar.Header.(Int64.to_int (to_sectors hdr) * length) in Reader.skip cin to_skip; - | Error _ -> failwith "Couldn't read header" ); + | Error `Eof -> failwith "Couldn't read header, end of file" + | Error (`Fatal err) -> Fmt.failwith "Couldn't read header: %a" Tar.pp_error err ); ( match HR.read ~global:!global cin with | Ok (hdr, global') -> Alcotest.check header "expected global header" (Some g0) global'; diff --git a/lib_test/parse_test.ml b/lib_test/parse_test.ml index c0ae596..05ee8e2 100644 --- a/lib_test/parse_test.ml +++ b/lib_test/parse_test.ml @@ -47,7 +47,6 @@ let list fd = List.iter (fun h -> print_endline h.Tar.Header.file_name) r; r -let cstruct = Alcotest.testable Cstruct.hexdump_pp Cstruct.equal let pp_header f x = Fmt.pf f "%s" (Tar.Header.to_detailed_string x) let header = Alcotest.testable pp_header ( = ) @@ -59,14 +58,14 @@ let header () = (* check header marshalling and unmarshalling *) let h = Tar.Header.make ~file_mode:5 ~user_id:1001 ~group_id:1002 ~mod_time:55L ~link_name:"" "hello" 1234L in let txt = "hello\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\0000000005\0000001751\0000001752\00000000002322\00000000000067\0000005534\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000" in - let c = Cstruct.create (String.length txt) in - Cstruct.blit_from_string txt 0 c 0 (String.length txt); - let c' = Cstruct.create Tar.Header.length in - for i = 0 to Tar.Header.length - 1 do Cstruct.set_uint8 c' i 0 done; + let c = Bytes.create (String.length txt) in + Bytes.blit_string txt 0 c 0 (String.length txt); + let c' = Bytes.create Tar.Header.length in + for i = 0 to Tar.Header.length - 1 do Bytes.set c' i '\000' done; match Tar.Header.marshal c' h with | Ok () -> - Alcotest.(check cstruct) "marshalled headers" c c'; - Alcotest.(check (result header error)) "unmarshalled headers" (Ok h) (Tar.Header.unmarshal c'); + Alcotest.(check bytes) "marshalled headers" c c'; + Alcotest.(check (result header error)) "unmarshalled headers" (Ok h) (Tar.Header.unmarshal (Bytes.unsafe_to_string c')); Alcotest.(check int) "zero padding length" 302 (Tar.Header.compute_zero_padding_length h) | Error `Msg msg -> Alcotest.failf "error marshalling: %s" msg diff --git a/mirage/tar_mirage.ml b/mirage/tar_mirage.ml index 9d42035..35f5b55 100644 --- a/mirage/tar_mirage.ml +++ b/mirage/tar_mirage.ml @@ -84,7 +84,7 @@ module Make_KV_RO (BLOCK : Mirage_block.S) = struct } type 'a io = 'a Lwt.t let really_read in_channel buffer = - let len = Cstruct.length buffer in + let len = Bytes.length buffer in assert(len <= 512); (* Tar assumes 512 byte sectors, but BLOCK might have 4096 byte sectors for example *) let sector_size = in_channel.info.Mirage_block.sector_size in @@ -102,7 +102,7 @@ module Make_KV_RO (BLOCK : Mirage_block.S) = struct (* If the BLOCK sector size is big, then we need to select the 512 bytes we want *) let offset = Int64.(to_int (sub in_channel.offset (mul sector' (of_int sector_size)))) in in_channel.offset <- Int64.(add in_channel.offset (of_int len)); - Cstruct.blit tmp offset buffer 0 len; + Cstruct.blit_to_bytes tmp offset buffer 0 len; Lwt.return_unit let skip in_channel n = in_channel.offset <- Int64.(add in_channel.offset (of_int n)); @@ -156,8 +156,9 @@ module Make_KV_RO (BLOCK : Mirage_block.S) = struct let buf = Cstruct.create (to_int (mul n_sectors sector_size)) in (* XXX: this is to work around limitations in some block implementations *) let tmps = + let sec_size_int = to_int sector_size in List.init (to_int n_sectors) - (fun sec -> Cstruct.sub buf (sec * to_int sector_size) (to_int sector_size)) + (fun sec -> Cstruct.sub buf (sec * sec_size_int) sec_size_int) in read t start_sector tmps >|= function | Error _ as e -> e @@ -373,9 +374,13 @@ module Make_KV_RW (CLOCK : Mirage_clock.PCLOCK) (BLOCK : Mirage_block.S) = struc type 'a io = 'a Lwt.t exception Read of BLOCK.error exception Write of BLOCK.write_error - let really_write out_channel data = - assert (Cstruct.length data <= 512); - let data = Cstruct.(append data (create (512 - length data))) in + let really_write out_channel str = + assert (String.length str <= Tar.Header.length); + let data = + let cs = Cstruct.create Tar.Header.length in + Cstruct.blit_from_string str 0 cs 0 (String.length str); + cs + in let sector_size = out_channel.info.sector_size in let sector = Int64.(div out_channel.offset (of_int sector_size)) in let block = Cstruct.create sector_size in diff --git a/tar-mirage.opam b/tar-mirage.opam index b3f6d8f..0cf420e 100644 --- a/tar-mirage.opam +++ b/tar-mirage.opam @@ -23,7 +23,7 @@ bug-reports: "https://github.com/mirage/ocaml-tar/issues" depends: [ "dune" {>= "2.9"} "ocaml" {>= "4.08.0"} - "cstruct" {>= "1.9.0"} + "cstruct" {>= "6.0.0"} "lwt" {>= "5.6.0"} "mirage-block" {>= "2.0.0"} "mirage-clock" {>= "4.0.0"} diff --git a/tar-unix.opam b/tar-unix.opam index 79710b0..ad160b7 100644 --- a/tar-unix.opam +++ b/tar-unix.opam @@ -22,8 +22,6 @@ bug-reports: "https://github.com/mirage/ocaml-tar/issues" depends: [ "dune" {>= "2.9"} "ocaml" {>= "4.08.0"} - "cstruct" {>= "6.0.0"} - "cstruct-lwt" "lwt" "tar" {= version} "odoc" {with-doc} diff --git a/tar.opam b/tar.opam index d5ac9b0..df09506 100644 --- a/tar.opam +++ b/tar.opam @@ -24,7 +24,6 @@ bug-reports: "https://github.com/mirage/ocaml-tar/issues" depends: [ "dune" {>= "2.9"} "ocaml" {>= "4.08.0"} - "cstruct" {>= "6.0.0"} "decompress" {>= "1.5.1"} "odoc" {with-doc} ] diff --git a/unix/dune b/unix/dune index eb838ac..bf45322 100644 --- a/unix/dune +++ b/unix/dune @@ -1,5 +1,5 @@ (library (name tar_unix) (public_name tar-unix) - (libraries tar lwt cstruct-lwt) + (libraries tar lwt lwt.unix) (wrapped false)) diff --git a/unix/tar_lwt_unix.ml b/unix/tar_lwt_unix.ml index d455c2b..8db418a 100644 --- a/unix/tar_lwt_unix.ml +++ b/unix/tar_lwt_unix.ml @@ -20,23 +20,30 @@ open Lwt.Infix module Io = struct type in_channel = Lwt_unix.file_descr type 'a io = 'a Lwt.t - let really_read fd = Lwt_cstruct.(complete (read fd)) - let skip (ifd: Lwt_unix.file_descr) (n: int) = - (* Here it would make sense to use [Lwt_unix.lseek] if we can detect if - [ifd] is seekable *) - let buffer_size = 32768 in - let buffer = Cstruct.create buffer_size in - let rec loop (n: int) = - if n <= 0 then Lwt.return_unit + let really_read fd buf = + let len = Bytes.length buf in + let rec loop idx = + if idx = len then + Lwt.return_unit else - let amount = min n buffer_size in - let block = Cstruct.sub buffer 0 amount in - really_read ifd block >>= fun () -> - loop (n - amount) in - loop n + Lwt_unix.read fd buf idx (len - idx) >>= fun n -> + loop (n + idx) + in + loop 0 + let skip (ifd: Lwt_unix.file_descr) (n: int) = + Lwt_unix.(lseek ifd n SEEK_CUR) >|= ignore type out_channel = Lwt_unix.file_descr - let really_write fd = Lwt_cstruct.(complete (write fd)) + let really_write fd buf = + let len = String.length buf in + let rec loop idx = + if idx = len then + Lwt.return_unit + else + Lwt_unix.write_string fd buf idx (len - idx) >>= fun n -> + loop (idx + n) + in + loop 0 end include Io diff --git a/unix/tar_lwt_unix.mli b/unix/tar_lwt_unix.mli index f153391..9b97e4d 100644 --- a/unix/tar_lwt_unix.mli +++ b/unix/tar_lwt_unix.mli @@ -16,11 +16,11 @@ (** Lwt_unix I/O for tar-formatted data *) -val really_read: Lwt_unix.file_descr -> Cstruct.t -> unit Lwt.t +val really_read: Lwt_unix.file_descr -> bytes -> unit Lwt.t (** [really_read fd buf] fills [buf] with data from [fd] or fails with {!Stdlib.End_of_file}. *) -val really_write: Lwt_unix.file_descr -> Cstruct.t -> unit Lwt.t +val really_write: Lwt_unix.file_descr -> string -> unit Lwt.t (** [really_write fd buf] writes the full contents of [buf] to [fd] or fails with {!Stdlib.End_of_file}. *) diff --git a/unix/tar_unix.ml b/unix/tar_unix.ml index bc364b1..e0bf5ae 100644 --- a/unix/tar_unix.ml +++ b/unix/tar_unix.ml @@ -32,36 +32,22 @@ module Driver = struct with_restart op fd buf off len let really_read fd buf = - (* Change code once this is merged: - https://github.com/mirage/ocaml-cstruct/pull/302 *) - let b = Bytes.create (min 4096 (Cstruct.length buf)) in + let len = Bytes.length buf in let rec loop offset = - let len = min (Bytes.length b) (Cstruct.length buf - offset) in - if len > 0 then - let n = with_restart Unix.read fd b 0 len in + if offset < len then + let n = with_restart Unix.read fd buf offset (len - offset) in if n = 0 then raise End_of_file; - Cstruct.blit_from_bytes b 0 buf offset n; loop (offset + n) in loop 0 let skip fd n = - (* Here it would make sense to use [Lwt_unix.lseek] if we can detect if - [ifd] is seekable *) - let b = Bytes.create (min 4096 n) in - let rem = ref n in - while !rem > 0 do - let len = min (Bytes.length b) !rem in - rem := !rem - with_restart Unix.read fd b 0 len - done + ignore (Unix.lseek fd n Unix.SEEK_CUR) let really_write fd buf = - (* FIXME: This is not very good :( - also: https://github.com/mirage/ocaml-cstruct/pull/302 *) - let b = Cstruct.to_bytes buf in let offset = ref 0 in - while !offset < Bytes.length b do - offset := !offset + with_restart Unix.write fd b 0 (Bytes.length b) + while !offset < String.length buf do + offset := !offset + with_restart Unix.write_substring fd buf !offset (String.length buf - !offset) done end diff --git a/unix/tar_unix.mli b/unix/tar_unix.mli index 32ddd81..b21ad57 100644 --- a/unix/tar_unix.mli +++ b/unix/tar_unix.mli @@ -16,11 +16,11 @@ (** Unix I/O for tar-formatted data. *) -val really_read: Unix.file_descr -> Cstruct.t -> unit +val really_read: Unix.file_descr -> bytes -> unit (** [really_read fd buf] fills [buf] with data from [fd] or raises {!Stdlib.End_of_file}. *) -val really_write: Unix.file_descr -> Cstruct.t -> unit +val really_write: Unix.file_descr -> string -> unit (** [really_write fd buf] writes the full contents of [buf] to [fd] or {!Stdlib.End_of_file}. *)