Optimize allocations (#21)

* Initial allocation optimisations
* Updated init_grams to support UTF8 texts

Author: PyDataBlog

src/dictdb.jl

@@ -65,8 +65,8 @@ function DictDB(x::WordNGrams)
         x,
         String[],
         DefaultDict{Int, Set{String}}( () -> Set{String}() ),
-        DefaultDict{ Int, DefaultOrderedDict{Tuple{NTuple{x.n, String}, Int}, Set{String}}  }( () -> DefaultOrderedDict{Tuple{NTuple{x.n, String}, Int}, Set{String} }(Set{String})),
-        DefaultDict{ Int, DefaultDict{Tuple{NTuple{x.n, String}, Int}, Set{String}} }( () -> DefaultDict{Tuple{NTuple{x.n, String}, Int}, Set{String}}(Set{String}))
+        DefaultDict{ Int, DefaultOrderedDict{Tuple{SubArray{SubString{String}}, Int}, Set{String}}  }( () -> DefaultOrderedDict{Tuple{SubArray{SubString{String}}, Int}, Set{String} }(Set{String})),
+        DefaultDict{ Int, DefaultDict{Tuple{SubArray{SubString{String}}, Int}, Set{String}} }( () -> DefaultDict{Tuple{SubArray{SubString{String}}, Int}, Set{String}}(Set{String}))
     )
 end
 

src/features.jl

@@ -10,7 +10,7 @@ end
 Internal function to pad AbstractVector types with specified padder
 """
 function pad_string(x::AbstractVector, padder::AbstractString)
-    # Insert a padder as the first and last element of x
+    # TODO: Insert a padder as the first and last element of x with undef
     insert!(x, 1, padder)
     push!(x, padder)
     return x
@@ -21,18 +21,29 @@ end
 Internal function to generate intial uncounted ngrams on a character level
 """
 function init_ngrams(extractor::CharacterNGrams, x, n)
-    map(0:length(x)-n) do i
-        x[i+1: i+n]
+    y = Vector{SubString{String}}(undef, length(x)-n+1)
+
+    i = 0
+    i1 = nextind(x, 0, 1)
+    i2 = nextind(x, i1, n-1)
+
+    while i2 <= lastindex(x)
+        i += 1
+        y[i] = SubString(x, i1:i2)
+        i1 = nextind(x, i1)
+        i2 = nextind(x, i2)
     end
+    return y
 end
 
 
+
 """
 Internal function to generate intial uncounted ngrams on a word level
 """
 function init_ngrams(extractor::WordNGrams, x, n)
     map(0:length(x)-n) do i
-        tuple(String.(x[i+1: i+n])...)
+        @view x[i+1: i+n]
     end
 end
 
@@ -85,6 +96,7 @@ end
 Internal function to count and pad generated character-level ngrams (including duplicates)
 """
 function cummulative_ngram_count(x)
+    # TODO: Use length of x initiate non allocated ngrams
     counter = Dict{eltype(x), Int}()
 
     return map(x) do val

test/test01_dictdb.jl

@@ -59,7 +59,7 @@ end
     @test collect(values(db.string_feature_map[9]))[5] == Set(["You are a really really really cool dude.", "Sometimes you are not really really cool tho"])
     @test collect(values(db.string_feature_map[9]))[7] == Set(["You are a really really really cool dude.", "Sometimes you are not really really cool tho"])
 
-    @test eltype(collect(keys(db.string_feature_map[9]))) == Tuple{Tuple{String,String},Int64}
+    @test eltype(collect(keys(db.string_feature_map[9]))) == Tuple{SubArray{SubString{String}},Int64}
 end
 
 
@@ -104,7 +104,7 @@ end
     @test collect(values(db.string_feature_map[9]))[5] == Set(["You are a really really really cool dude.", "Sometimes you are not really really cool tho"])
     @test collect(values(db.string_feature_map[9]))[7] == Set(["You are a really really really cool dude.", "Sometimes you are not really really cool tho"])
 
-    @test eltype(collect(keys(db.string_feature_map[9]))) == Tuple{Tuple{String,String},Int64}
+    @test eltype(collect(keys(db.string_feature_map[9]))) == Tuple{SubArray{SubString{String}},Int64}
 
 end
 

test/test02_features.jl

@@ -7,8 +7,12 @@ using Test
     char_ngram_res = SimString.extract_features(CharacterNGrams(3, " "), "prepress")
     @test char_ngram_res[5] == ("pre", 2)
 
-    word_ngram_res = SimString.extract_features(WordNGrams(2, " ", " "), "You are a really really really cool dude.")
-    @test word_ngram_res[5] == (("really", "really"), 2)
+    # Unicode text test
+    @test SimString.extract_features(CharacterNGrams(2, " "), "∀∃😄🍕")[3] == ("😄🍕", 1)
+
+    word_ngram_res = SimString.extract_features(WordNGrams(2, " ", " "), "You are a really really really cool dude 😄🍕")
+    @test word_ngram_res[5] == (["really", "really"], 2)
+    @test word_ngram_res[8] == (["dude", "😄🍕"], 1)
 end