Moved malloc and free upcalls into rust runtime.

Author: Elliott Slaughter

src/libcore/rt.rs

@@ -1,13 +1,27 @@
 //! Runtime calls emitted by the compiler.
 
 import libc::c_char;
+import libc::c_void;
 import libc::size_t;
+import libc::uintptr_t;
 
-type rust_task = libc::c_void;
+type rust_task = c_void;
 
 extern mod rustrt {
     #[rust_stack]
     fn rust_upcall_fail(expr: *c_char, file: *c_char, line: size_t);
+
+    #[rust_stack]
+    fn rust_upcall_exchange_malloc(td: *c_char, size: uintptr_t) -> *c_char;
+
+    #[rust_stack]
+    fn rust_upcall_exchange_free(ptr: *c_char);
+
+    #[rust_stack]
+    fn rust_upcall_malloc(td: *c_char, size: uintptr_t) -> *c_char;
+
+    #[rust_stack]
+    fn rust_upcall_free(ptr: *c_char);
 }
 
 // FIXME (#2861): This needs both the attribute, and the name prefixed with
@@ -18,6 +32,26 @@ fn rt_fail(expr: *c_char, file: *c_char, line: size_t) {
     rustrt::rust_upcall_fail(expr, file, line);
 }
 
+#[rt(exchange_malloc)]
+fn rt_exchange_malloc(td: *c_char, size: uintptr_t) -> *c_char {
+    ret rustrt::rust_upcall_exchange_malloc(td, size);
+}
+
+#[rt(exchange_free)]
+fn rt_exchange_free(ptr: *c_char) {
+    rustrt::rust_upcall_exchange_free(ptr);
+}
+
+#[rt(malloc)]
+fn rt_malloc(td: *c_char, size: uintptr_t) -> *c_char {
+    ret rustrt::rust_upcall_malloc(td, size);
+}
+
+#[rt(free)]
+fn rt_free(ptr: *c_char) {
+    rustrt::rust_upcall_free(ptr);
+}
+
 // Local Variables:
 // mode: rust;
 // fill-column: 78;

src/rt/rust_upcall.cpp

@@ -170,6 +170,14 @@ upcall_exchange_malloc(type_desc *td, uintptr_t size) {
     return args.retval;
 }
 
+// FIXME (#2861): Alias used by libcore/rt.rs to avoid naming conflicts with
+// autogenerated wrappers for upcall_exchange_malloc. Remove this when we
+// fully move away away from the C upcall path.
+extern "C" CDECL uintptr_t
+rust_upcall_exchange_malloc(type_desc *td, uintptr_t size) {
+    return upcall_exchange_malloc(td, size);
+}
+
 struct s_exchange_free_args {
     rust_task *task;
     void *ptr;
@@ -189,6 +197,14 @@ upcall_exchange_free(void *ptr) {
     UPCALL_SWITCH_STACK(task, &args, upcall_s_exchange_free);
 }
 
+// FIXME (#2861): Alias used by libcore/rt.rs to avoid naming conflicts with
+// autogenerated wrappers for upcall_exchange_free. Remove this when we fully
+// move away away from the C upcall path.
+extern "C" CDECL void
+rust_upcall_exchange_free(void *ptr) {
+    return upcall_exchange_free(ptr);
+}
+
 /**********************************************************************
  * Allocate an object in the task-local heap.
  */
@@ -230,6 +246,14 @@ upcall_malloc(type_desc *td, uintptr_t size) {
     return args.retval;
 }
 
+// FIXME (#2861): Alias used by libcore/rt.rs to avoid naming conflicts with
+// autogenerated wrappers for upcall_malloc. Remove this when we fully move
+// away away from the C upcall path.
+extern "C" CDECL uintptr_t
+rust_upcall_malloc(type_desc *td, uintptr_t size) {
+    return upcall_malloc(td, size);
+}
+
 /**********************************************************************
  * Called whenever an object in the task-local heap is freed.
  */
@@ -262,6 +286,14 @@ upcall_free(void* ptr) {
     UPCALL_SWITCH_STACK(task, &args, upcall_s_free);
 }
 
+// FIXME (#2861): Alias used by libcore/rt.rs to avoid naming conflicts with
+// autogenerated wrappers for upcall_free. Remove this when we fully move away
+// away from the C upcall path.
+extern "C" CDECL void
+rust_upcall_free(void* ptr) {
+    upcall_free(ptr);
+}
+
 /**********************************************************************
  * Sanity checks on boxes, insert when debugging possible
  * use-after-free bugs.  See maybe_validate_box() in trans.rs.

src/rustc/middle/trans/base.rs

@@ -244,15 +244,13 @@ fn trans_foreign_call(cx: block, externs: hashmap<~str, ValueRef>,
 
 fn trans_free(cx: block, v: ValueRef) -> block {
     let _icx = cx.insn_ctxt(~"trans_free");
-    Call(cx, cx.ccx().upcalls.free, ~[PointerCast(cx, v, T_ptr(T_i8()))]);
-    cx
+    trans_rtcall(cx, ~"free", ~[PointerCast(cx, v, T_ptr(T_i8()))], ignore)
 }
 
 fn trans_unique_free(cx: block, v: ValueRef) -> block {
     let _icx = cx.insn_ctxt(~"trans_unique_free");
-    Call(cx, cx.ccx().upcalls.exchange_free,
-         ~[PointerCast(cx, v, T_ptr(T_i8()))]);
-    ret cx;
+    trans_rtcall(cx, ~"exchange_free", ~[PointerCast(cx, v, T_ptr(T_i8()))],
+                 ignore)
 }
 
 fn umax(cx: block, a: ValueRef, b: ValueRef) -> ValueRef {
@@ -356,15 +354,13 @@ fn opaque_box_body(bcx: block,
 // malloc_raw_dyn: allocates a box to contain a given type, but with a
 // potentially dynamic size.
 fn malloc_raw_dyn(bcx: block, t: ty::t, heap: heap,
-                  size: ValueRef) -> ValueRef {
+                  size: ValueRef) -> result {
     let _icx = bcx.insn_ctxt(~"malloc_raw");
     let ccx = bcx.ccx();
 
-    let (mk_fn, upcall) = alt heap {
-      heap_shared { (ty::mk_imm_box, ccx.upcalls.malloc) }
-      heap_exchange {
-        (ty::mk_imm_uniq, ccx.upcalls.exchange_malloc )
-      }
+    let (mk_fn, rtcall) = alt heap {
+      heap_shared { (ty::mk_imm_box, ~"malloc") }
+      heap_exchange { (ty::mk_imm_uniq, ~"exchange_malloc") }
     };
 
     // Grab the TypeRef type of box_ptr_ty.
@@ -376,37 +372,42 @@ fn malloc_raw_dyn(bcx: block, t: ty::t, heap: heap,
     lazily_emit_all_tydesc_glue(ccx, static_ti);
 
     // Allocate space:
-    let rval = Call(bcx, upcall, ~[static_ti.tydesc, size]);
-    ret PointerCast(bcx, rval, llty);
+    let tydesc = PointerCast(bcx, static_ti.tydesc, T_ptr(T_i8()));
+    let rval = alloca_zeroed(bcx, T_ptr(T_i8()));
+    let bcx = trans_rtcall(bcx, rtcall, ~[tydesc, size], save_in(rval));
+    let retval = {bcx: bcx, val: PointerCast(bcx, Load(bcx, rval), llty)};
+    ret retval;
 }
 
 // malloc_raw: expects an unboxed type and returns a pointer to
 // enough space for a box of that type.  This includes a rust_opaque_box
 // header.
-fn malloc_raw(bcx: block, t: ty::t, heap: heap) -> ValueRef {
+fn malloc_raw(bcx: block, t: ty::t, heap: heap) -> result {
     malloc_raw_dyn(bcx, t, heap, llsize_of(bcx.ccx(), type_of(bcx.ccx(), t)))
 }
 
 // malloc_general_dyn: usefully wraps malloc_raw_dyn; allocates a box,
 // and pulls out the body
-fn malloc_general_dyn(bcx: block, t: ty::t, heap: heap, size: ValueRef) ->
-    {box: ValueRef, body: ValueRef} {
+fn malloc_general_dyn(bcx: block, t: ty::t, heap: heap, size: ValueRef)
+    -> {bcx: block, box: ValueRef, body: ValueRef} {
     let _icx = bcx.insn_ctxt(~"malloc_general");
-    let llbox = malloc_raw_dyn(bcx, t, heap, size);
+    let {bcx: bcx, val: llbox} = malloc_raw_dyn(bcx, t, heap, size);
     let non_gc_box = non_gc_box_cast(bcx, llbox);
     let body = GEPi(bcx, non_gc_box, ~[0u, abi::box_field_body]);
-    ret {box: llbox, body: body};
+    ret {bcx: bcx, box: llbox, body: body};
 }
 
-fn malloc_general(bcx: block, t: ty::t, heap: heap) ->
-    {box: ValueRef, body: ValueRef} {
+fn malloc_general(bcx: block, t: ty::t, heap: heap)
+    -> {bcx: block, box: ValueRef, body: ValueRef} {
     malloc_general_dyn(bcx, t, heap,
                        llsize_of(bcx.ccx(), type_of(bcx.ccx(), t)))
 }
-fn malloc_boxed(bcx: block, t: ty::t) -> {box: ValueRef, body: ValueRef} {
+fn malloc_boxed(bcx: block, t: ty::t)
+    -> {bcx: block, box: ValueRef, body: ValueRef} {
     malloc_general(bcx, t, heap_shared)
 }
-fn malloc_unique(bcx: block, t: ty::t) -> {box: ValueRef, body: ValueRef} {
+fn malloc_unique(bcx: block, t: ty::t)
+    -> {bcx: block, box: ValueRef, body: ValueRef} {
     malloc_general(bcx, t, heap_exchange)
 }
 
@@ -1464,7 +1465,7 @@ fn trans_boxed_expr(bcx: block, contents: @ast::expr,
                     t: ty::t, heap: heap,
                     dest: dest) -> block {
     let _icx = bcx.insn_ctxt(~"trans_boxed_expr");
-    let {box, body} = malloc_general(bcx, t, heap);
+    let {bcx, box, body} = malloc_general(bcx, t, heap);
     add_clean_free(bcx, box, heap);
     let bcx = trans_expr_save_in(bcx, contents, body);
     revoke_clean(bcx, box);
@@ -3942,12 +3943,13 @@ fn trans_fail_value(bcx: block, sp_opt: option<span>,
     let V_str = PointerCast(bcx, V_fail_str, T_ptr(T_i8()));
     let V_filename = PointerCast(bcx, V_filename, T_ptr(T_i8()));
     let args = ~[V_str, V_filename, C_int(ccx, V_line)];
-    let bcx = trans_rtcall(bcx, ~"fail", args);
+    let bcx = trans_rtcall(bcx, ~"fail", args, ignore);
     Unreachable(bcx);
     ret bcx;
 }
 
-fn trans_rtcall(bcx: block, name: ~str, args: ~[ValueRef]) -> block {
+fn trans_rtcall(bcx: block, name: ~str, args: ~[ValueRef], dest: dest)
+    -> block {
     let did = bcx.ccx().rtcalls[name];
     let fty = if did.crate == ast::local_crate {
         ty::node_id_to_type(bcx.ccx().tcx, did.node)
@@ -3958,7 +3960,7 @@ fn trans_rtcall(bcx: block, name: ~str, args: ~[ValueRef]) -> block {
     ret trans_call_inner(
         bcx, none, fty, rty,
         |bcx| lval_static_fn_inner(bcx, did, 0, ~[], none),
-        arg_vals(args), ignore);
+        arg_vals(args), dest);
 }
 
 fn trans_break_cont(bcx: block, to_end: bool)
@@ -5396,8 +5398,12 @@ fn gather_rtcalls(ccx: @crate_ctxt, crate: @ast::crate) {
     // supported. Also probably want to check type signature so we don't crash
     // in some obscure place in LLVM if the user provides the wrong signature
     // for an rtcall.
-    if !ccx.rtcalls.contains_key(~"fail") {
-        fail ~"no definition for runtime call fail";
+    let expected_rtcalls =
+        ~[~"exchange_free", ~"exchange_malloc", ~"fail", ~"free", ~"malloc"];
+    for vec::each(expected_rtcalls) |name| {
+        if !ccx.rtcalls.contains_key(name) {
+            fail #fmt("no definition for runtime call %s", name);
+        }
     }
 }
 

src/rustc/middle/trans/closure.rs

@@ -138,7 +138,7 @@ fn mk_closure_tys(tcx: ty::ctxt,
 fn allocate_cbox(bcx: block,
                  ck: ty::closure_kind,
                  cdata_ty: ty::t)
-    -> ValueRef {
+    -> result {
     let _icx = bcx.insn_ctxt(~"closure::allocate_cbox");
     let ccx = bcx.ccx(), tcx = ccx.tcx;
 
@@ -153,7 +153,7 @@ fn allocate_cbox(bcx: block,
     }
 
     // Allocate and initialize the box:
-    let llbox = alt ck {
+    let {bcx, val} = alt ck {
       ty::ck_box {
         malloc_raw(bcx, cdata_ty, heap_shared)
       }
@@ -164,11 +164,11 @@ fn allocate_cbox(bcx: block,
         let cbox_ty = tuplify_box_ty(tcx, cdata_ty);
         let llbox = base::alloc_ty(bcx, cbox_ty);
         nuke_ref_count(bcx, llbox);
-        llbox
+        {bcx: bcx, val: llbox}
       }
     };
 
-    ret llbox;
+    ret {bcx: bcx, val: val};
 }
 
 type closure_result = {
@@ -191,7 +191,7 @@ fn store_environment(bcx: block,
     let cdata_ty = mk_closure_tys(tcx, bound_values);
 
     // allocate closure in the heap
-    let llbox = allocate_cbox(bcx, ck, cdata_ty);
+    let {bcx: bcx, val: llbox} = allocate_cbox(bcx, ck, cdata_ty);
     let mut temp_cleanups = ~[];
 
     // cbox_ty has the form of a tuple: (a, b, c) we want a ptr to a
@@ -362,14 +362,14 @@ fn trans_expr_fn(bcx: block,
                  dest: dest) -> block {
     let _icx = bcx.insn_ctxt(~"closure::trans_expr_fn");
     if dest == ignore { ret bcx; }
-    let ccx = bcx.ccx(), bcx = bcx;
+    let ccx = bcx.ccx();
     let fty = node_id_type(bcx, id);
     let llfnty = type_of_fn_from_ty(ccx, fty);
     let sub_path = vec::append_one(bcx.fcx.path, path_name(@~"anon"));
     let s = mangle_internal_name_by_path(ccx, sub_path);
     let llfn = decl_internal_cdecl_fn(ccx.llmod, s, llfnty);
 
-    let trans_closure_env = fn@(ck: ty::closure_kind) -> ValueRef {
+    let trans_closure_env = fn@(ck: ty::closure_kind) -> result {
         let cap_vars = capture::compute_capture_vars(
             ccx.tcx, id, proto, cap_clause);
         let ret_handle = alt is_loop_body { some(x) { x } none { none } };
@@ -384,20 +384,21 @@ fn trans_expr_fn(bcx: block,
                 Store(bcx, C_bool(true), bcx.fcx.llretptr);
             }
         });
-        llbox
+        {bcx: bcx, val: llbox}
     };
 
-    let closure = alt proto {
+    let {bcx: bcx, val: closure} = alt proto {
       ast::proto_any | ast::proto_block { trans_closure_env(ty::ck_block) }
       ast::proto_box { trans_closure_env(ty::ck_box) }
       ast::proto_uniq { trans_closure_env(ty::ck_uniq) }
       ast::proto_bare {
         trans_closure(ccx, sub_path, decl, body, llfn, no_self, none,
                       id, |_fcx| { }, |_bcx| { });
-        C_null(T_opaque_box_ptr(ccx))
+        {bcx: bcx, val: C_null(T_opaque_box_ptr(ccx))}
       }
     };
     fill_fn_pair(bcx, get_dest_addr(dest), llfn, closure);
+
     ret bcx;
 }
 
@@ -459,9 +460,12 @@ fn make_opaque_cbox_take_glue(
         let sz = Add(bcx, sz, shape::llsize_of(ccx, T_box_header(ccx)));
 
         // Allocate memory, update original ptr, and copy existing data
-        let malloc = ccx.upcalls.exchange_malloc;
-        let cbox_out = Call(bcx, malloc, ~[tydesc, sz]);
-        let cbox_out = PointerCast(bcx, cbox_out, llopaquecboxty);
+        let malloc = ~"exchange_malloc";
+        let opaque_tydesc = PointerCast(bcx, tydesc, T_ptr(T_i8()));
+        let rval = alloca_zeroed(bcx, T_ptr(T_i8()));
+        let bcx = trans_rtcall(bcx, malloc, ~[opaque_tydesc, sz],
+                               save_in(rval));
+        let cbox_out = PointerCast(bcx, Load(bcx, rval), llopaquecboxty);
         call_memmove(bcx, cbox_out, cbox_in, sz);
         Store(bcx, cbox_out, cboxptr);
 

src/rustc/middle/trans/impl.rs

@@ -288,7 +288,7 @@ fn trans_cast(bcx: block, val: @ast::expr, id: ast::node_id, dest: dest)
     if dest == ignore { ret trans_expr(bcx, val, ignore); }
     let ccx = bcx.ccx();
     let v_ty = expr_ty(bcx, val);
-    let {box: llbox, body: body} = malloc_boxed(bcx, v_ty);
+    let {bcx: bcx, box: llbox, body: body} = malloc_boxed(bcx, v_ty);
     add_clean_free(bcx, llbox, heap_shared);
     let bcx = trans_expr_save_in(bcx, val, body);
     revoke_clean(bcx, llbox);

src/rustc/middle/trans/tvec.rs

@@ -68,7 +68,8 @@ fn alloc_raw(bcx: block, unit_ty: ty::t,
     let vecbodyty = ty::mk_mut_unboxed_vec(bcx.tcx(), unit_ty);
     let vecsize = Add(bcx, alloc, llsize_of(ccx, ccx.opaque_vec_type));
 
-    let {box, body} = base::malloc_general_dyn(bcx, vecbodyty, heap, vecsize);
+    let {bcx, box, body} =
+        base::malloc_general_dyn(bcx, vecbodyty, heap, vecsize);
     Store(bcx, fill, GEPi(bcx, body, ~[0u, abi::vec_elt_fill]));
     Store(bcx, alloc, GEPi(bcx, body, ~[0u, abi::vec_elt_alloc]));
     ret {bcx: bcx, val: box};

src/rustc/middle/trans/uniq.rs

@@ -34,7 +34,8 @@ fn autoderef(bcx: block, v: ValueRef, t: ty::t) -> {v: ValueRef, t: ty::t} {
 fn duplicate(bcx: block, v: ValueRef, t: ty::t) -> result {
     let _icx = bcx.insn_ctxt(~"uniq::duplicate");
     let content_ty = content_ty(t);
-    let {box: dst_box, body: dst_body} = malloc_unique(bcx, content_ty);
+    let {bcx: bcx, box: dst_box, body: dst_body} =
+        malloc_unique(bcx, content_ty);
 
     let src_box = v;
     let src_body = opaque_box_body(bcx, content_ty, src_box);