From 7902379f6ef3ada179a5f5118b0d790455e6de0b Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 6 Mar 2026 10:46:37 +0000 Subject: [PATCH] Fix buffer overflow in HTTP downloads on 32-bit systems MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace Body.to_string with streaming to avoid Buffer size limits. On 32-bit systems, OCaml's Buffer has a maximum size of ~16 MB due to Sys.max_string_length limitations. When downloading files larger than this limit, Body.to_string would fail with "Buffer.add: cannot grow buffer". Changes: - Replace download_range with download_range_to_channel that streams directly - Replace download_full with download_full_to_channel that streams directly - Use Cohttp_lwt.Body.to_stream instead of Body.to_string - Stream chunks directly to the output channel using Lwt_stream.iter_s This allows downloads of any size on both 32-bit and 64-bit systems while being more memory efficient. Fixes: Fatal error: exception Failure("Buffer.add: cannot grow buffer") 🤖 Generated with Claude Code Co-Authored-By: Claude --- src/lib/http_download.ml | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/src/lib/http_download.ml b/src/lib/http_download.ml index bd1ef91..6b48d3d 100644 --- a/src/lib/http_download.ml +++ b/src/lib/http_download.ml @@ -94,7 +94,7 @@ let get_download_info url = Lwt.return None ;; -let download_range url start_byte end_byte = +let download_range_to_channel url start_byte end_byte oc = let range_header = Printf.sprintf "bytes=%Ld-%Ld" start_byte end_byte in let headers = Header.init_with "Range" range_header in @@ -103,22 +103,32 @@ let download_range url start_byte end_byte = match status with | `Partial_content | `OK -> - Cohttp_lwt.Body.to_string body >>= fun content -> - Lwt.return (Some content) + (* Stream body directly to the output channel *) + let stream = Cohttp_lwt.Body.to_stream body in + let bytes_written = ref 0 in + Lwt_stream.iter_s (fun chunk -> + bytes_written := !bytes_written + String.length chunk; + Lwt_io.write oc chunk + ) stream >>= fun () -> + Lwt.return (Some !bytes_written) | code -> Logs.err (fun m -> m "Range request failed with status: %s" (Code.string_of_status code)); Lwt.return None ;; -let download_full url = +let download_full_to_channel url oc = Logs.info (fun m -> m "Downloading entire file (server doesn't support ranges)"); Client.get (Uri.of_string url) >>= fun (resp, body) -> let status = Response.status resp in match status with | `OK -> - Cohttp_lwt.Body.to_string body >>= fun content -> - Lwt.return (Some content) + (* Stream body directly to the output channel *) + let stream = Cohttp_lwt.Body.to_stream body in + Lwt_stream.iter_s (fun chunk -> + Lwt_io.write oc chunk + ) stream >>= fun () -> + Lwt.return (Some ()) | code -> Logs.err (fun m -> m "Download failed with status: %s" (Code.string_of_status code)); Lwt.return None @@ -200,13 +210,12 @@ let download_http_file url output_file use_tui = |> Int64.sub (Int64.of_int 1) |> Int64.min (Int64.sub info.total_size 1L) in - download_range url current_pos end_pos >>= function + download_range_to_channel url current_pos end_pos oc >>= function | None -> Logs.err (fun m -> m "Failed to download chunk at offset %Ld" current_pos); Lwt.return_unit - | Some chunk_data -> - Lwt_io.write oc chunk_data >>= fun () -> - let new_pos = Int64.add current_pos (Int64.of_int (String.length chunk_data)) in + | Some bytes_written -> + let new_pos = Int64.add current_pos (Int64.of_int bytes_written) in downloaded := new_pos; save_http_state final_output new_pos info.total_size >>= fun () -> @@ -229,14 +238,13 @@ let download_http_file url output_file use_tui = ) else ( (* Non-resumable download *) Logs.warn (fun m -> m "Server doesn't support ranges, downloading entire file at once"); - download_full url >>= function + Lwt_io.with_file ~mode:Lwt_io.Output final_output (fun oc -> + download_full_to_channel url oc + ) >>= function | None -> Logs.err (fun m -> m "Download failed"); Lwt.return_unit - | Some content -> - Lwt_io.with_file ~mode:Lwt_io.Output final_output (fun oc -> - Lwt_io.write oc content - ) >>= fun () -> + | Some () -> Logs.info (fun m -> m "Download completed: %s" final_output); Lwt.return_unit )