Add built-in fzy fuzzy filter (#92)

* .

* .

* .

* Work for vim now

* Refactoring

* Update README.md

* .

* Try fixing CI

* Replace the gif

* .

* Skip linting filter.vim

* .

* Fix marks preview

* Update default fuzzy matches hl group

.github/workflows/ci.yml

@@ -31,4 +31,4 @@ jobs:
     - name: Setup dependencies
       run: pip install vim-vint
     - name: Run Vimscript Linter
-      run: vint .
+      run: test/run_vint.sh

.gitignore

@@ -9,3 +9,4 @@ __pycache__
 !.travis.yml
 !.gitignore
 !.github
+!.vintrc.yaml

.vintrc.yaml

@@ -0,0 +1,5 @@
+cmdargs:
+  severity: style_problem
+  color: true
+  env:
+    neovim: false

README.md

@@ -8,7 +8,7 @@
 
 Vim-clap is a modern generic interactive finder and dispatcher, based on the newly feature: `floating_win` of neovim or `popup` of vim. The goal of vim-clap is to work everywhere out of the box, with fast response.
 
-![vim-clap-1024-98](https://user-images.githubusercontent.com/8850248/65813749-d6562c00-e20b-11e9-8161-c42801b1056c.gif)
+![fzy-filter-c](https://user-images.githubusercontent.com/8850248/67620599-3b1c9a80-f83b-11e9-8d8c-72bfae9d9177.gif)
 
 ## Table of Contents
 
@@ -44,7 +44,7 @@ Vim-clap is a modern generic interactive finder and dispatcher, based on the new
 - [x] Avoid touching the current window layout, less eye movement.
 - [x] Support multi-selection, use vim's regexp as filter by default.
 - [x] Support the preview functionality when navigating the result list.
-- [x] Support builtin match and external fuzzy filter tools.
+- [x] Support built-in fuzzy match and external fuzzy filter tools.
 - [ ] Support searching by multiple providers simultaneously.
 - [ ] Add the preview support for more providers.
 - [ ] Add the multi-selection support for more providers.
@@ -63,6 +63,16 @@ Vim-clap is a modern generic interactive finder and dispatcher, based on the new
 - Vim: `:echo has('patch-8.1.2114')`.
 - NeoVim: `:echo has('nvim-0.4')`.
 
+The `python` support is actually not neccessary. However, if you want to use the advanced built-in fuzzy match filter which uses the [fzy algorithm](https://github.com/jhawthorn/fzy/blob/master/ALGORITHM.md) implemented in python, then the `python` support is required:
+
+- Vim: `:pyx print("Hello")` should be `Hello`.
+- NeoVim:
+
+  ```bash
+  # ensure you have installed pynvim
+  $ python3 -m pip install pynvim
+  ```
+
 ## Installation
 
 ```vim
@@ -77,25 +87,25 @@ Vim-clap is utterly easy to use, just type, press Ctrl-J/K to locate the wanted
 
 The paradigm is `Clap [provider_id_or_alias] {provider_args}`, where the `provider_id_or_alias` is obviously either the name or alias of provider. Technically the `provider_id` can be anything that can be used a key of a Dict, but I recommend you using an _identifier_ like name as the provider id, and use the alias rule if you prefer a special name.
 
-Command                                | List                               | Requirement
-:----                                  | :----                              | :----
-`Clap bcommits`**<sup>!</sup>**        | Git commits for the current buffer | **[git][git]**
-`Clap blines`                          | Lines in the current buffer        | _none_
-`Clap buffers`                         | Open buffers                       | _none_
-`Clap colors`                          | Colorschemes                       | _none_
-`Clap hist:` or `Clap command_history` | Command history                    | _none_
-`Clap commits` **<sup>!</sup>**        | Git commits                        | **[git][git]**
-`Clap files`                           | Files                              | **[fd][fd]**/**[git][git]**/**[rg][rg]**/find
-`Clap filetypes`                       | File types                         | _none_
-`Clap gfiles` or `Clap git_files`      | Files managed by git               | **[git][git]**
-`Clap grep`**<sup>+</sup>**            | Grep on the fly                    | **[rg][rg]**
-`Clap history`                         | Open buffers and `v:oldfiles`      | _none_
-`Clap jumps`                           | Jumps                              | _none_
-`Clap lines`                           | Lines in the loaded buffers        | _none_
-`Clap marks`                           | Marks                              | _none_
-`Clap tags`                            | Tags in the current buffer         | **[vista.vim][vista.vim]**
-`Clap yanks`                           | Yank stack of the current vim session    | _none_
-`Clap windows` **<sup>!</sup>**        | Windows                            | _none_
+Command                                | List                                  | Requirement
+:----                                  | :----                                 | :----
+`Clap bcommits`**<sup>!</sup>**        | Git commits for the current buffer    | **[git][git]**
+`Clap blines`                          | Lines in the current buffer           | _none_
+`Clap buffers`                         | Open buffers                          | _none_
+`Clap colors`                          | Colorschemes                          | _none_
+`Clap hist:` or `Clap command_history` | Command history                       | _none_
+`Clap commits` **<sup>!</sup>**        | Git commits                           | **[git][git]**
+`Clap files`                           | Files                                 | **[fd][fd]**/**[git][git]**/**[rg][rg]**/find
+`Clap filetypes`                       | File types                            | _none_
+`Clap gfiles` or `Clap git_files`      | Files managed by git                  | **[git][git]**
+`Clap grep`**<sup>+</sup>**            | Grep on the fly                       | **[rg][rg]**
+`Clap history`                         | Open buffers and `v:oldfiles`         | _none_
+`Clap jumps`                           | Jumps                                 | _none_
+`Clap lines`                           | Lines in the loaded buffers           | _none_
+`Clap marks`                           | Marks                                 | _none_
+`Clap tags`                            | Tags in the current buffer            | **[vista.vim][vista.vim]**
+`Clap yanks`                           | Yank stack of the current vim session | _none_
+`Clap windows` **<sup>!</sup>**        | Windows                               | _none_
 
 [fd]: https://github.com/sharkdp/fd
 [rg]: https://github.com/BurntSushi/ripgrep
@@ -137,6 +147,7 @@ The option naming convention for provider is `g:clap_provider_{provider_id}_{opt
 - `g:clap_provider_grep_delay`: 300ms by default, delay for actually spawning the grep job in the background.
 
 - `g:clap_provider_grep_blink`: [2, 100] by default, blink 2 times with 100ms timeout when jumping the result. Set it to [0, 0] to disable the blink.
+
 - `g:clap_provider_grep_opts`: An empty string by default, allows you to enable flags such as `'--hidden -g "!.git/"'`.
 
 See `:help clap-options` for more information.
@@ -328,6 +339,7 @@ If you'd liked to discuss the project more directly, check out [![][G1]][G2].
 
 - Vim-clap is initially enlightened by [snails](https://github.com/manateelazycat/snails).
 - Some providers' idea and code are borrowed from [fzf.vim](https://github.com/junegunn/fzf.vim).
+- The built-in fzy python implementation is based on [sweep.py](https://github.com/aslpavel/sweep.py).
 
 ## [License](LICENSE)
 

autoload/clap.vim

@@ -22,6 +22,8 @@ let s:builtin_providers = map(
       \ 'fnamemodify(v:val, '':t:r'')'
       \ )
 
+let g:__clap_dir = s:cur_dir
+
 let g:clap#builtin_providers = s:builtin_providers
 
 let g:__t_func = 0
@@ -140,6 +142,10 @@ function! clap#_exit() abort
     call remove(g:clap.provider, 'args')
   endif
 
+  if exists('g:__clap_fuzzy_matched_indices')
+    unlet g:__clap_fuzzy_matched_indices
+  endif
+
   call clap#sign#reset()
 
   call map(g:clap.tmps, 'delete(v:val)')

autoload/clap/debugging.vim

@@ -28,7 +28,7 @@ function! s:get_global_variables() abort
 endfunction
 
 function! s:get_third_party_providers() abort
-  let all_providers = split(globpath(&rtp, 'autoload/clap/provider/*.vim'), "\n")
+  let all_providers = split(globpath(&runtimepath, 'autoload/clap/provider/*.vim'), "\n")
   let third_party_providers = filter(all_providers, 'index(g:clap#builtin_providers, v:val) != -1')
   return third_party_providers
 endfunction

autoload/clap/filter.vim

@@ -8,6 +8,12 @@ let s:pattern_builder = {}
 
 let s:default_ext_filter = v:null
 
+if has('python3') || has('python')
+  let s:py_exe = has('python3') ? 'python3' : 'python'
+else
+  let s:py_exe = v:null
+endif
+
 let s:ext_cmd = {}
 let s:ext_cmd.fzy = 'fzy --show-matches="%s"'
 let s:ext_cmd.fzf = 'fzf --filter="%s"'
@@ -83,11 +89,52 @@ function! s:filter(line, pattern) abort
   return a:line =~ a:pattern
 endfunction
 
-function! clap#filter#(lines, input) abort
-  let s:pattern_builder.input = a:input
+function! s:fallback_filter(query, candidates) abort
+  let s:pattern_builder.input = a:query
   let l:filter_pattern = s:pattern_builder.build()
-  return filter(a:lines, 's:filter(v:val, l:filter_pattern)')
+  return filter(a:candidates, 's:filter(v:val, l:filter_pattern)')
 endfunction
 
+if s:py_exe isnot v:null
+
+  if has('nvim')
+
+execute s:py_exe "<< EOF"
+from clap.fzy import clap_fzy
+EOF
+
+  else
+
+execute s:py_exe "<< EOF"
+import sys
+from os.path import normpath, join
+import vim
+plugin_root_dir = vim.eval('g:__clap_dir')
+python_root_dir = normpath(join(plugin_root_dir, '..', 'pythonx'))
+sys.path.insert(0, python_root_dir)
+import clap
+
+from clap.fzy import clap_fzy
+EOF
+
+  endif
+
+  function! clap#filter#(query, candidates) abort
+    try
+      let [g:__clap_fuzzy_matched_indices, filtered] = pyxeval("clap_fzy()")
+      return filtered
+    catch
+      return s:fallback_filter(a:query, a:candidates)
+    endtry
+  endfunction
+
+else
+
+  function! clap#filter#(query, candidates) abort
+    return s:fallback_filter(a:query, a:candidates)
+  endfunction
+
+endif
+
 let &cpoptions = s:save_cpo
 unlet s:save_cpo

autoload/clap/handler.vim

@@ -207,5 +207,5 @@ function! clap#handler#try_open(action) abort
   endif
 endfunction
 
-let &cpo = s:save_cpo
+let &cpoptions = s:save_cpo
 unlet s:save_cpo

autoload/clap/impl.vim

@@ -5,24 +5,21 @@ let s:save_cpo = &cpoptions
 set cpoptions&vim
 
 let s:is_nvim = has('nvim')
-let s:async_threshold = 5000
-
-function! s:on_typed_sync_impl() abort
-  call g:clap.display.clear_highlight()
-
-  let l:cur_input = g:clap.input.get()
-
-  if empty(l:cur_input)
-    call g:clap.display.set_lines_lazy(g:clap.provider.get_source())
-    let l:matches_cnt = g:clap.display.line_count() + len(g:clap.display.cache)
-    call clap#indicator#set_matches('['.l:matches_cnt.']')
-    call clap#sign#toggle_cursorline()
-    call g:clap#display_win.compact_if_undersize()
-    return
-  endif
-
-  call clap#spinner#set_busy()
+let s:async_threshold = 10000
+
+" =======================================
+" sync implementation
+" =======================================
+function! s:reset_on_empty_input() abort
+  call g:clap.display.set_lines_lazy(g:clap.provider.get_source())
+  let l:matches_cnt = g:clap.display.line_count() + len(g:clap.display.cache)
+  call clap#indicator#set_matches('['.l:matches_cnt.']')
+  call clap#sign#toggle_cursorline()
+  call g:clap#display_win.compact_if_undersize()
+endfunction
 
+" FIXME: some sources could be cached.
+function! s:get_source() abort
   if get(g:, '__clap_should_refilter', v:false)
         \ || get(g:, '__clap_do_not_use_cache', v:false)
     let l:lines = g:clap.provider.get_source()
@@ -37,10 +34,25 @@ function! s:on_typed_sync_impl() abort
       let l:lines = g:clap.provider.get_source()
     endif
   endif
+  return l:lines
+endfunction
+
+function! s:on_typed_sync_impl() abort
+  call g:clap.display.clear_highlight()
+
+  let l:cur_input = g:clap.input.get()
+
+  if empty(l:cur_input)
+    call s:reset_on_empty_input()
+    return
+  endif
+
+  call clap#spinner#set_busy()
 
   let l:has_no_matches = v:false
 
-  let l:lines = call(g:clap.provider.filter(), [l:lines, l:cur_input])
+  let l:raw_lines = s:get_source()
+  let l:lines = call(g:clap.provider.filter(), [l:cur_input, l:raw_lines])
 
   if empty(l:lines)
     let l:lines = [g:clap_no_matches_msg]
@@ -60,7 +72,7 @@ function! s:on_typed_sync_impl() abort
     call clap#indicator#set_matches('['.l:count.']')
     call clap#sign#disable_cursorline()
   else
-    let l:matches_cnt = string(len(lines))
+    let l:matches_cnt = string(len(l:lines))
     if get(g:clap.display, 'initial_size', -1) > 0
       let l:matches_cnt .= '/'.g:clap.display.initial_size
     endif
@@ -72,10 +84,41 @@ function! s:on_typed_sync_impl() abort
   call clap#spinner#set_idle()
 
   if !l:has_no_matches
-    call g:clap.display.add_highlight()
+    if exists('g:__clap_fuzzy_matched_indices')
+      call s:add_highlight_for_fuzzy_matched()
+    else
+      call g:clap.display.add_highlight(l:cur_input)
+    endif
   endif
 endfunction
 
+function! s:add_highlight_for_fuzzy_matched() abort
+  " Due the cache strategy, g:__clap_fuzzy_matched_indices may be oversize
+  " than the actual display buffer, the rest highlight indices of g:__clap_fuzzy_matched_indices
+  " belong to the cached lines.
+  "
+  " TODO: also add highlights for the cached lines?
+  let hl_lines = g:__clap_fuzzy_matched_indices[:g:clap.display.line_count()-1]
+
+  let lnum = 0
+
+  for indices in hl_lines
+    let group_idx = 1
+    for idx in indices
+      if group_idx < g:__clap_fuzzy_matches_hl_group_cnt + 1
+        call clap#util#add_highlight_at(lnum, idx, 'ClapFuzzyMatches'.group_idx)
+        let group_idx += 1
+      else
+        call clap#util#add_highlight_at(lnum, idx, 'ClapMatches')
+      endif
+    endfor
+    let lnum += 1
+  endfor
+endfunction
+
+" =======================================
+" async implementation
+" =======================================
 function! s:apply_source_async() abort
   let cmd = g:clap.provider.source_async_or_default()
   call clap#dispatcher#job_start(cmd)
@@ -121,7 +164,7 @@ function! s:should_switch_to_async() abort
 endfunction
 
 "                          filter
-"                       /  (sync)
+"                       /  (sync/async)
 "             on_typed -
 "           /           \
 "          /              dispatcher

autoload/clap/init.vim

@@ -88,18 +88,18 @@ endfunction
 
 function! s:init_fuzzy_matches_hl_group() abort
   let clap_fuzzy_matches = [
-        \ [46, '#00ff00'],
-        \ [47, '#00ff5f'],
-        \ [48, '#00ff87'],
-        \ [49, '#00ffaf'],
-        \ [50, '#00ffd7'],
-        \ [51, '#00ffff'],
-        \ [40, '#00d700'],
-        \ [41, '#00d75f'],
-        \ [42, '#00d787'],
-        \ [43, '#00d7af'],
-        \ [44, '#00d7d7'],
-        \ [45, '#00d7ff'],
+        \ [118 , '#87ff00'] ,
+        \ [82  , '#5fff00'] ,
+        \ [46  , '#00ff00'] ,
+        \ [47  , '#00ff5f'] ,
+        \ [48  , '#00ff87'] ,
+        \ [49  , '#00ffaf'] ,
+        \ [50  , '#00ffd7'] ,
+        \ [51  , '#00ffff'] ,
+        \ [87  , '#5fffff'] ,
+        \ [123 , '#87ffff'] ,
+        \ [159 , '#afffff'] ,
+        \ [195 , '#d7ffff'] ,
         \ ]
 
   let idx = 1