Skip to content
New issue

Have a question about this project? # for a free GitHub account to open an issue and contact its maintainers and the community.

#

By clicking “#”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? # to your account

Use Intersect to Narrow Iterate Range and Reduce Memory Allocation #9271

Closed
wants to merge 8 commits into from
Closed

Use Intersect to Narrow Iterate Range and Reduce Memory Allocation #9271

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
3b23c92
Use Intersect to Narrow Iterate Range and Reduce Memory Allocation
Jan 9, 2025
25a8814
Use Intersect to Narrow Iterate Range and Reduce Memory Allocation
Jan 10, 2025
a5576a1
Use Intersect to Narrow Iterate Range and Reduce Memory Allocation
Jan 10, 2025
17d0349
Use Intersect to Narrow Iterate Range and Reduce Memory Allocation
Jan 10, 2025
2121ec2
Use Intersect to Narrow Iterate Range and Reduce Memory Allocation
Jan 13, 2025
5516de8
Use Intersect to Narrow Iterate Range and Reduce Memory Allocation
Jan 13, 2025
f25dfee
handle negative opt.First
Jan 15, 2025
0d023b0
Use native min function to replace custom implementation
Jan 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 54 additions & 10 deletions posting/list.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1672,10 +1672,8 @@ func (l *List) Uids(opt ListOptions) (*pb.List, error) {
if opt.First == 0 {
opt.First = math.MaxInt32
}
// Pre-assign length to make it faster.
l.RLock()
// Use approximate length for initial capacity.
res := make([]uint64, 0, l.mutationMap.len()+codec.ApproxLen(l.plist.Pack))

out := &pb.List{}
if l.mutationMap.len() == 0 && opt.Intersect != nil && len(l.plist.Splits) == 0 {
if opt.ReadTs < l.minTs {
Expand All @@ -1687,16 +1685,62 @@ func (l *List) Uids(opt ListOptions) (*pb.List, error) {
return out, nil
}

absFirst := opt.First
if opt.First < 0 {
absFirst = -opt.First
}
preAllowcateLength := x.MinInt(absFirst, l.mutationMap.len()+codec.ApproxLen(l.plist.Pack))
if opt.Intersect != nil {
preAllowcateLength = x.MinInt(preAllowcateLength, len(opt.Intersect.Uids))
}
// Pre-assign length to make it faster.
res := make([]uint64, 0, preAllowcateLength)

checkLimit := func() bool {
// We need the last N.
// TODO: This could be optimized by only considering some of the last UidBlocks.
if opt.First < 0 {
if len(res) > -opt.First {
res = res[1:]
}
} else if len(res) > opt.First {
return true
}
return false
}

if opt.Intersect != nil && len(opt.Intersect.Uids) < l.mutationMap.len()+codec.ApproxLen(l.plist.Pack) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

same: use exact length.
lets create another function inside list that intersects with the list of uids.

start := 0
if opt.AfterUid > 0 {
start = sort.Search(len(opt.Intersect.Uids), func(i int) bool {
return opt.Intersect.Uids[i] > opt.AfterUid
})
}

for i := start; i < len(opt.Intersect.Uids); i++ {
uid := opt.Intersect.Uids[i]

found, _, err := l.findPosting(opt.ReadTs, uid)
if err != nil {
l.RUnlock()
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Instead of manually calling RUnlock() each time a return happens, you could do defer l.RUnlock() at the start of the function for the same effect.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Instead of manually calling RUnlock() each time a return happens, you could do defer l.RUnlock() at the start of the function for the same effect.

When implementing this code, I also considered making this change, but I referred to the previous implementation and believe it was designed to minimize the time spent holding the read lock. Therefore, I chose not to change this implementation.

return out, errors.Wrapf(err, "While find posting for UIDs")
}
if found {
res = append(res, uid)
if checkLimit() {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

instead of checking for negative first here like this, why not just check the ids in reverse.

break
}
}
}
out.Uids = res
l.RUnlock()
return out, nil
}

err := l.iterate(opt.ReadTs, opt.AfterUid, func(p *pb.Posting) error {
if p.PostingType == pb.Posting_REF {
res = append(res, p.Uid)
if opt.First < 0 {
// We need the last N.
// TODO: This could be optimized by only considering some of the last UidBlocks.
if len(res) > -opt.First {
res = res[1:]
}
} else if len(res) > opt.First {
if checkLimit() {
return ErrStopIteration
}
}
Expand Down
32 changes: 32 additions & 0 deletions x/x.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -619,6 +619,38 @@ func Max(a, b uint64) uint64 {
return b
}

// MinInt returns the smallest integer among the given numbers.
// The first two arguments are mandatory, additional numbers are optional.
func MinInt(a, b int, nums ...int) int {
gooohgb marked this conversation as resolved.
Show resolved Hide resolved
min := a
if b < min {
min = b
}

for _, num := range nums {
if num < min {
min = num
}
}
return min
}

// MaxInt returns the largest integer among the given numbers.
// The first two arguments are mandatory, additional numbers are optional.
func MaxInt(a, b int, nums ...int) int {
max := a
if b > max {
max = b
}

for _, num := range nums {
if num > max {
max = num
}
}
return max
}

// ExponentialRetry runs the given function until it succeeds or can no longer be retried.
func ExponentialRetry(maxRetries int, waitAfterFailure time.Duration,
f func() error) error {
Expand Down
Loading