@@ -757,17 +757,34 @@ bool _mi_arena_contains(const void* p) {
757
757
// sets the thread_id.
758
758
bool _mi_arena_segment_clear_abandoned (mi_segment_t * segment )
759
759
{
760
- if (segment -> memid .memkind != MI_MEM_ARENA ) {
761
- // not in an arena, consider it un-abandoned now.
762
- // but we need to still claim it atomically -- we use the thread_id for that.
760
+ if mi_unlikely (segment -> memid .memkind != MI_MEM_ARENA ) {
761
+ // not in an arena
762
+ // if abandoned visiting is allowed, we need to take a lock on the abandoned os list
763
+ bool has_lock = false;
764
+ if (mi_option_is_enabled (mi_option_visit_abandoned )) {
765
+ has_lock = mi_lock_try_acquire (& segment -> subproc -> abandoned_os_lock );
766
+ if (!has_lock ) {
767
+ return false; // failed to acquire the lock, we just give up
768
+ }
769
+ }
770
+ // abandon it, but we need to still claim it atomically -- we use the thread_id for that.
771
+ bool reclaimed = false;
763
772
size_t expected = 0 ;
764
773
if (mi_atomic_cas_strong_acq_rel (& segment -> thread_id , & expected , _mi_thread_id ())) {
774
+ // reclaim
765
775
mi_atomic_decrement_relaxed (& segment -> subproc -> abandoned_count );
766
- return true;
767
- }
768
- else {
769
- return false;
776
+ reclaimed = true;
777
+ // and remove from the abandoned os list (if needed)
778
+ mi_segment_t * const next = segment -> abandoned_os_next ;
779
+ mi_segment_t * const prev = segment -> abandoned_os_prev ;
780
+ if (prev != NULL ) { prev -> abandoned_os_next = next ; }
781
+ else { segment -> subproc -> abandoned_os_list = next ; }
782
+ if (next != NULL ) { next -> abandoned_os_prev = prev ; }
783
+ segment -> abandoned_os_next = NULL ;
784
+ segment -> abandoned_os_prev = NULL ;
770
785
}
786
+ if (has_lock ) { mi_lock_release (& segment -> subproc -> abandoned_os_lock ); }
787
+ return reclaimed ;
771
788
}
772
789
// arena segment: use the blocks_abandoned bitmap.
773
790
size_t arena_idx ;
@@ -794,12 +811,30 @@ void _mi_arena_segment_mark_abandoned(mi_segment_t* segment)
794
811
{
795
812
mi_atomic_store_release (& segment -> thread_id , 0 );
796
813
mi_assert_internal (segment -> used == segment -> abandoned );
797
- if (segment -> memid .memkind != MI_MEM_ARENA ) {
798
- // not in an arena; count it as abandoned and return
814
+ if mi_unlikely (segment -> memid .memkind != MI_MEM_ARENA ) {
815
+ // not in an arena; count it as abandoned and return (these can be reclaimed on a `free`)
799
816
mi_atomic_increment_relaxed (& segment -> subproc -> abandoned_count );
817
+ // if abandoned visiting is allowed, we need to take a lock on the abandoned os list to insert it
818
+ if (mi_option_is_enabled (mi_option_visit_abandoned )) {
819
+ if (!mi_lock_acquire (& segment -> subproc -> abandoned_os_lock )) {
820
+ _mi_error_message (EFAULT , "internal error: failed to acquire the abandoned (os) segment lock to mark abandonment" );
821
+ }
822
+ else {
823
+ // push on the front of the list
824
+ mi_segment_t * next = segment -> subproc -> abandoned_os_list ;
825
+ mi_assert_internal (next == NULL || next -> abandoned_os_prev == NULL );
826
+ mi_assert_internal (segment -> abandoned_os_prev == NULL );
827
+ mi_assert_internal (segment -> abandoned_os_next == NULL );
828
+ if (next != NULL ) { next -> abandoned_os_prev = segment ; }
829
+ segment -> abandoned_os_prev = NULL ;
830
+ segment -> abandoned_os_next = next ;
831
+ segment -> subproc -> abandoned_os_list = segment ;
832
+ mi_lock_release (& segment -> subproc -> abandoned_os_lock );
833
+ }
834
+ }
800
835
return ;
801
836
}
802
- // segment is in an arena
837
+ // segment is in an arena, mark it in the arena `blocks_abandoned` bitmap
803
838
size_t arena_idx ;
804
839
size_t bitmap_idx ;
805
840
mi_arena_memid_indices (segment -> memid , & arena_idx , & bitmap_idx );
@@ -822,6 +857,29 @@ void _mi_arena_field_cursor_init(mi_heap_t* heap, mi_subproc_t* subproc, mi_aren
822
857
current -> subproc = subproc ;
823
858
}
824
859
860
+ static mi_segment_t * mi_arena_segment_clear_abandoned_at (mi_arena_t * arena , mi_subproc_t * subproc , mi_bitmap_index_t bitmap_idx ) {
861
+ // try to reclaim an abandoned segment in the arena atomically
862
+ if (!_mi_bitmap_unclaim (arena -> blocks_abandoned , arena -> field_count , 1 , bitmap_idx )) return NULL ;
863
+ mi_assert_internal (_mi_bitmap_is_claimed (arena -> blocks_inuse , arena -> field_count , 1 , bitmap_idx ));
864
+ mi_segment_t * segment = (mi_segment_t * )mi_arena_block_start (arena , bitmap_idx );
865
+ mi_assert_internal (mi_atomic_load_relaxed (& segment -> thread_id ) == 0 );
866
+ // check that the segment belongs to our sub-process
867
+ // note: this is the reason we need a lock in the case abandoned visiting is enabled.
868
+ // without the lock an abandoned visit may otherwise fail to visit all segments.
869
+ // for regular reclaim it is fine to miss one sometimes so without abandoned visiting we don't need the arena lock.
870
+ if (segment -> subproc != subproc ) {
871
+ // it is from another subprocess, re-mark it and continue searching
872
+ const bool was_zero = _mi_bitmap_claim (arena -> blocks_abandoned , arena -> field_count , 1 , bitmap_idx , NULL );
873
+ mi_assert_internal (was_zero ); MI_UNUSED (was_zero );
874
+ return NULL ;
875
+ }
876
+ else {
877
+ // success, we unabandoned a segment in our sub-process
878
+ mi_atomic_decrement_relaxed (& subproc -> abandoned_count );
879
+ return segment ;
880
+ }
881
+ }
882
+
825
883
// reclaim abandoned segments
826
884
// this does not set the thread id (so it appears as still abandoned)
827
885
mi_segment_t * _mi_arena_segment_clear_abandoned_next (mi_arena_field_cursor_t * previous , bool visit_all )
@@ -848,7 +906,7 @@ mi_segment_t* _mi_arena_segment_clear_abandoned_next(mi_arena_field_cursor_t* pr
848
906
has_lock = (visit_all ? mi_lock_acquire (& arena -> abandoned_visit_lock ) : mi_lock_try_acquire (& arena -> abandoned_visit_lock ));
849
907
if (!has_lock ) {
850
908
if (visit_all ) {
851
- _mi_error_message (EINVAL , "failed to visit all abandoned segments due to failure to acquire the visitor lock" );
909
+ _mi_error_message (EFAULT , "internal error: failed to visit all abandoned segments due to failure to acquire the visitor lock" );
852
910
}
853
911
// skip to next arena
854
912
break ;
@@ -860,31 +918,14 @@ mi_segment_t* _mi_arena_segment_clear_abandoned_next(mi_arena_field_cursor_t* pr
860
918
// pre-check if the bit is set
861
919
size_t mask = ((size_t )1 << bit_idx );
862
920
if mi_unlikely ((field & mask ) == mask ) {
863
- mi_bitmap_index_t bitmap_idx = mi_bitmap_index_create (field_idx , bit_idx );
864
- // try to reclaim it atomically
865
- if (_mi_bitmap_unclaim (arena -> blocks_abandoned , arena -> field_count , 1 , bitmap_idx )) {
866
- mi_assert_internal (_mi_bitmap_is_claimed (arena -> blocks_inuse , arena -> field_count , 1 , bitmap_idx ));
867
- mi_segment_t * segment = (mi_segment_t * )mi_arena_block_start (arena , bitmap_idx );
868
- mi_assert_internal (mi_atomic_load_relaxed (& segment -> thread_id ) == 0 );
869
- // check that the segment belongs to our sub-process
870
- // note: this is the reason we need a lock in the case abandoned visiting is enabled.
871
- // without the lock an abandoned visit may otherwise fail to visit all segments.
872
- // for regular reclaim it is fine to miss one sometimes so without abandoned visiting we don't need the arena lock.
873
- if (segment -> subproc != previous -> subproc ) {
874
- // it is from another subprocess, re-mark it and continue searching
875
- const bool was_zero = _mi_bitmap_claim (arena -> blocks_abandoned , arena -> field_count , 1 , bitmap_idx , NULL );
876
- mi_assert_internal (was_zero );
877
- }
878
- else {
879
- // success, we unabandoned a segment in our sub-process
880
- mi_atomic_decrement_relaxed (& previous -> subproc -> abandoned_count );
881
- previous -> bitmap_idx = bitmap_idx ;
882
- previous -> count = count ;
883
-
884
- //mi_assert_internal(arena->blocks_committed == NULL || _mi_bitmap_is_claimed(arena->blocks_committed, arena->field_count, 1, bitmap_idx));
885
- if (has_lock ) { mi_lock_release (& arena -> abandoned_visit_lock ); }
886
- return segment ;
887
- }
921
+ const mi_bitmap_index_t bitmap_idx = mi_bitmap_index_create (field_idx , bit_idx );
922
+ mi_segment_t * const segment = mi_arena_segment_clear_abandoned_at (arena , previous -> subproc , bitmap_idx );
923
+ if (segment != NULL ) {
924
+ previous -> bitmap_idx = bitmap_idx ;
925
+ previous -> count = count ;
926
+ //mi_assert_internal(arena->blocks_committed == NULL || _mi_bitmap_is_claimed(arena->blocks_committed, arena->field_count, 1, bitmap_idx));
927
+ if (has_lock ) { mi_lock_release (& arena -> abandoned_visit_lock ); }
928
+ return segment ;
888
929
}
889
930
}
890
931
}
@@ -910,16 +951,35 @@ static bool mi_arena_visit_abandoned_blocks(mi_subproc_t* subproc, int heap_tag,
910
951
return true;
911
952
}
912
953
954
+ static bool mi_subproc_visit_abandoned_os_blocks (mi_subproc_t * subproc , int heap_tag , bool visit_blocks , mi_block_visit_fun * visitor , void * arg ) {
955
+ if (!mi_lock_acquire (& subproc -> abandoned_os_lock )) {
956
+ _mi_error_message (EFAULT , "internal error: failed to acquire abandoned (OS) segment lock" );
957
+ return false;
958
+ }
959
+ bool all_visited = true;
960
+ for (mi_segment_t * segment = subproc -> abandoned_os_list ; segment != NULL ; segment = segment -> abandoned_os_next ) {
961
+ if (!_mi_segment_visit_blocks (segment , heap_tag , visit_blocks , visitor , arg )) {
962
+ all_visited = false;
963
+ break ;
964
+ }
965
+ }
966
+ mi_lock_release (& subproc -> abandoned_os_lock );
967
+ return all_visited ;
968
+ }
969
+
913
970
bool mi_abandoned_visit_blocks (mi_subproc_id_t subproc_id , int heap_tag , bool visit_blocks , mi_block_visit_fun * visitor , void * arg ) {
914
971
// (unfortunately) the visit_abandoned option must be enabled from the start.
915
972
// This is to avoid taking locks if abandoned list visiting is not required (as for most programs)
916
973
if (!mi_option_is_enabled (mi_option_visit_abandoned )) {
917
- mi_assert (false);
918
- _mi_error_message (EINVAL , "internal error: can only visit abandoned blocks when MIMALLOC_VISIT_ABANDONED=ON" );
974
+ _mi_error_message (EFAULT , "internal error: can only visit abandoned blocks when MIMALLOC_VISIT_ABANDONED=ON" );
919
975
return false;
920
976
}
977
+ mi_subproc_t * const subproc = _mi_subproc_from_id (subproc_id );
921
978
// visit abandoned segments in the arena's
922
- return mi_arena_visit_abandoned_blocks (_mi_subproc_from_id (subproc_id ), heap_tag , visit_blocks , visitor , arg );
979
+ if (!mi_arena_visit_abandoned_blocks (subproc , heap_tag , visit_blocks , visitor , arg )) return false;
980
+ // and visit abandoned segments outside arena's (in OS allocated memory)
981
+ if (!mi_subproc_visit_abandoned_os_blocks (subproc , heap_tag , visit_blocks , visitor , arg )) return false;
982
+ return true;
923
983
}
924
984
925
985
0 commit comments