Support Fennel. (#530)

This adds support for the Fennel programming language: https://fennel-lang.org

I couldn't find much explanation for what the different lexer rules
meant, so I based it off of Clojure's lexer since the two languages
share a very similar syntax.

I also included a program to generate a list of keywords from Fennel's
own listing, which will make it easier to update in the future.

lexers/f/fennel.go

@@ -0,0 +1,66 @@
+package f
+
+import (
+	. "github.com/alecthomas/chroma" // nolint
+	"github.com/alecthomas/chroma/lexers/internal"
+)
+
+// Fennel lexer.
+var Fennel = internal.Register(MustNewLazyLexer(
+	&Config{
+		Name:      "Fennel",
+		Aliases:   []string{"fennel", "fnl"},
+		Filenames: []string{"*.fennel"},
+		MimeTypes: []string{"text/x-fennel", "application/x-fennel"},
+	},
+	fennelRules,
+))
+
+// Here's some Fennel code used to generate the lists of keywords:
+// (local fennel (require :fennel))
+//
+// (fn member? [t x] (each [_ y (ipairs t)] (when (= y x) (lua "return true"))))
+//
+// (local declarations [:fn :lambda :λ :local :var :global :macro :macros])
+// (local keywords [])
+// (local globals [])
+//
+// (each [name data (pairs (fennel.syntax))]
+//   (if (member? declarations name) nil ; already populated
+//       data.special? (table.insert keywords name)
+//       data.macro? (table.insert keywords name)
+//       data.global? (table.insert globals name)))
+//
+// (fn quoted [tbl]
+//   (table.sort tbl)
+//   (table.concat (icollect [_ k (ipairs tbl)]
+//                   (string.format "`%s`" k)) ", "))
+//
+// (print :Keyword (quoted keywords))
+// (print :KeywordDeclaration (quoted declarations))
+// (print :NameBuiltin (quoted globals))
+
+func fennelRules() Rules {
+	return Rules{
+		"root": {
+			{`;.*$`, CommentSingle, nil},
+			{`\s+`, Whitespace, nil},
+			{`-?\d+\.\d+`, LiteralNumberFloat, nil},
+			{`-?\d+`, LiteralNumberInteger, nil},
+			{`0x-?[abcdef\d]+`, LiteralNumberHex, nil},
+			{`"(\\\\|\\"|[^"])*"`, LiteralString, nil},
+			{`'(?!#)[\w!$%*+<=>?/.#-]+`, LiteralStringSymbol, nil},
+			{`\\(.|[a-z]+)`, LiteralStringChar, nil},
+			{`::?#?(?!#)[\w!$%*+<=>?/.#-]+`, LiteralStringSymbol, nil},
+			{"~@|[`\\'#^~&@]", Operator, nil},
+			{Words(``, ` `, `#`, `%`, `*`, `+`, `-`, `->`, `->>`, `-?>`, `-?>>`, `.`, `..`, `/`, `//`, `:`, `<`, `<=`, `=`, `>`, `>=`, `?.`, `^`, `accumulate`, `and`, `band`, `bnot`, `bor`, `bxor`, `collect`, `comment`, `do`, `doc`, `doto`, `each`, `eval-compiler`, `for`, `hashfn`, `icollect`, `if`, `import-macros`, `include`, `length`, `let`, `lshift`, `lua`, `macrodebug`, `match`, `not`, `not=`, `or`, `partial`, `pick-args`, `pick-values`, `quote`, `require-macros`, `rshift`, `set`, `set-forcibly!`, `tset`, `values`, `when`, `while`, `with-open`, `~=`), Keyword, nil},
+			{Words(``, ` `, `fn`, `global`, `lambda`, `local`, `macro`, `macros`, `var`, `λ`), KeywordDeclaration, nil},
+			{Words(``, ` `, `_G`, `arg`, `assert`, `bit32`, `bit32.arshift`, `bit32.band`, `bit32.bnot`, `bit32.bor`, `bit32.btest`, `bit32.bxor`, `bit32.extract`, `bit32.lrotate`, `bit32.lshift`, `bit32.replace`, `bit32.rrotate`, `bit32.rshift`, `collectgarbage`, `coroutine`, `coroutine.create`, `coroutine.resume`, `coroutine.running`, `coroutine.status`, `coroutine.wrap`, `coroutine.yield`, `debug`, `debug.debug`, `debug.gethook`, `debug.getinfo`, `debug.getlocal`, `debug.getmetatable`, `debug.getregistry`, `debug.getupvalue`, `debug.getuservalue`, `debug.sethook`, `debug.setlocal`, `debug.setmetatable`, `debug.setupvalue`, `debug.setuservalue`, `debug.traceback`, `debug.upvalueid`, `debug.upvaluejoin`, `dofile`, `error`, `getmetatable`, `io`, `io.close`, `io.flush`, `io.input`, `io.lines`, `io.open`, `io.output`, `io.popen`, `io.read`, `io.tmpfile`, `io.type`, `io.write`, `ipairs`, `load`, `loadfile`, `loadstring`, `math`, `math.abs`, `math.acos`, `math.asin`, `math.atan`, `math.atan2`, `math.ceil`, `math.cos`, `math.cosh`, `math.deg`, `math.exp`, `math.floor`, `math.fmod`, `math.frexp`, `math.ldexp`, `math.log`, `math.log10`, `math.max`, `math.min`, `math.modf`, `math.pow`, `math.rad`, `math.random`, `math.randomseed`, `math.sin`, `math.sinh`, `math.sqrt`, `math.tan`, `math.tanh`, `module`, `next`, `os`, `os.clock`, `os.date`, `os.difftime`, `os.execute`, `os.exit`, `os.getenv`, `os.remove`, `os.rename`, `os.setlocale`, `os.time`, `os.tmpname`, `package`, `package.loadlib`, `package.searchpath`, `package.seeall`, `pairs`, `pcall`, `print`, `rawequal`, `rawget`, `rawlen`, `rawset`, `require`, `select`, `setmetatable`, `string`, `string.byte`, `string.char`, `string.dump`, `string.find`, `string.format`, `string.gmatch`, `string.gsub`, `string.len`, `string.lower`, `string.match`, `string.rep`, `string.reverse`, `string.sub`, `string.upper`, `table`, `table.concat`, `table.insert`, `table.maxn`, `table.pack`, `table.remove`, `table.sort`, `table.unpack`, `tonumber`, `tostring`, `type`, `unpack`, `xpcall`), NameBuiltin, nil},
+			{`(?<=\()(?!#)[\w!$%*+<=>?/.#-]+`, NameFunction, nil},
+			{`(?!#)[\w!$%*+<=>?/.#-]+`, NameVariable, nil},
+			{`(\[|\])`, Punctuation, nil},
+			{`(\{|\})`, Punctuation, nil},
+			{`(\(|\))`, Punctuation, nil},
+		},
+	}
+}

lexers/testdata/fennel.actual

@@ -0,0 +1,75 @@
+;; An example of some possible linters using Fennel's --plugin option.
+
+;; The first two linters here can only function on static module
+;; use. For instance, this code can be checked because they use static
+;; field access on a local directly bound to a require call:
+
+;; (local m (require :mymodule))
+;; (print m.field) ; fails if mymodule lacks a :field field
+;; (print (m.function 1 2 3)) ; fails unless mymodule.function takes 3 args
+
+;; However, these cannot:
+
+;; (local m (do (require :mymodule)) ; m is not directly bound
+;; (print (. m field)) ; not a static field reference
+;; (let [f m.function]
+;;   (print (f 1 2 3)) ; intermediate local, not a static field call on m
+
+;; Still, pretty neat, huh?
+
+;; This file is provided as an example and is not part of Fennel's public API.
+
+(fn save-require-meta [from to scope]
+  "When destructuring, save module name if local is bound to a `require' call.
+Doesn't do any linting on its own; just saves the data for other linters."
+  (when (and (sym? to) (not (multi-sym? to)) (list? from)
+             (sym? (. from 1)) (= :require (tostring (. from 1)))
+             (= :string (type (. from 2))))
+    (let [meta (. scope.symmeta (tostring to))]
+      (set meta.required (tostring (. from 2))))))
+
+(fn check-module-fields [symbol scope]
+  "When referring to a field in a local that's a module, make sure it exists."
+  (let [[module-local field] (or (multi-sym? symbol) [])
+        module-name (-?> scope.symmeta (. (tostring module-local)) (. :required))
+        module (and module-name (require module-name))]
+    (assert-compile (or (= module nil) (not= (. module field) nil))
+                    (string.format "Missing field %s in module %s"
+                                   (or field :?) (or module-name :?)) symbol)))
+
+(fn arity-check? [module] (-?> module getmetatable (. :arity-check?)))
+
+(fn arity-check-call [[f & args] scope]
+  "Perform static arity checks on static function calls in a module."
+  (let [arity (# args)
+        last-arg (. args arity)
+        [f-local field] (or (multi-sym? f) [])
+        module-name (-?> scope.symmeta (. (tostring f-local)) (. :required))
+        module (and module-name (require module-name))]
+    (when (and (arity-check? module) _G.debug _G.debug.getinfo
+               (not (varg? last-arg)) (not (list? last-arg)))
+      (assert-compile (= (type (. module field)) :function)
+                      (string.format "Missing function %s in module %s"
+                                     (or field :?) module-name) f)
+      (match (_G.debug.getinfo (. module field))
+        {: nparams :what "Lua" :isvararg true}
+        (assert-compile (<= nparams (# args))
+                        (: "Called %s.%s with %s arguments, expected %s+"
+                           :format f-local field arity nparams) f)
+        {: nparams :what "Lua" :isvararg false}
+        (assert-compile (= nparams (# args))
+                        (: "Called %s.%s with %s arguments, expected %s"
+                           :format f-local field arity nparams) f)))))
+
+(fn check-unused [ast scope]
+  (each [symname (pairs scope.symmeta)]
+    (assert-compile (or (. scope.symmeta symname :used) (symname:find "^_"))
+                    (string.format "unused local %s" (or symname :?)) ast)))
+
+{:destructure save-require-meta
+ :symbol-to-expression check-module-fields
+ :call arity-check-call
+ ;; Note that this will only check unused args inside functions and let blocks,
+ ;; not top-level locals of a chunk.
+ :fn check-unused
+ :do check-unused}