From ad71187a9fa3f86d67f86a193aa37fbacef9e429 Mon Sep 17 00:00:00 2001 From: David Anthoff Date: Tue, 25 Jul 2023 14:43:07 +0200 Subject: [PATCH 1/3] Migrate to test item framework --- Project.toml | 3 +- test/check_base.jl | 159 --- test/iterate.jl | 838 ------------ test/iterate/test_iterators.jl | 905 +++++++++++++ test/iterate/test_selftest.jl | 12 + test/parser.jl | 1575 ---------------------- test/parser/test_function_calls.jl | 58 + test/parser/test_keyword_blocks.jl | 191 +++ test/parser/test_modules.jl | 22 + test/parser/test_operators.jl | 224 +++ test/parser/test_parser.jl | 1088 +++++++++++++++ test/parser/test_square.jl | 138 ++ test/parser/test_types.jl | 64 + test/runtests.jl | 15 +- test/shared.jl | 110 ++ test/test_check_base.jl | 165 +++ test/{display.jl => test_display.jl} | 2 +- test/{errparse.jl => test_errparse.jl} | 20 +- test/{interface.jl => test_interface.jl} | 12 +- test/{spec.jl => test_spec.jl} | 224 +-- 20 files changed, 3126 insertions(+), 2699 deletions(-) delete mode 100644 test/check_base.jl delete mode 100644 test/iterate.jl create mode 100644 test/iterate/test_iterators.jl create mode 100644 test/iterate/test_selftest.jl delete mode 100644 test/parser.jl create mode 100644 test/parser/test_function_calls.jl create mode 100644 test/parser/test_keyword_blocks.jl create mode 100644 test/parser/test_modules.jl create mode 100644 test/parser/test_operators.jl create mode 100644 test/parser/test_parser.jl create mode 100644 test/parser/test_square.jl create mode 100644 test/parser/test_types.jl create mode 100644 test/shared.jl create mode 100644 test/test_check_base.jl rename test/{display.jl => test_display.jl} (91%) rename test/{errparse.jl => test_errparse.jl} (90%) rename test/{interface.jl => test_interface.jl} (96%) rename test/{spec.jl => test_spec.jl} (68%) diff --git a/Project.toml b/Project.toml index 47624344..a5cf7a97 100644 --- a/Project.toml +++ b/Project.toml @@ -8,10 +8,11 @@ Tokenize = "0796e94c-ce3b-5d07-9a54-7f471281c624" [extras] Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a" [compat] julia = "1" Tokenize = "0.5.24" [targets] -test = ["Test"] +test = ["Test", "TestItemRunner"] diff --git a/test/check_base.jl b/test/check_base.jl deleted file mode 100644 index 96cdd3d0..00000000 --- a/test/check_base.jl +++ /dev/null @@ -1,159 +0,0 @@ -function norm_ast(a::Any) - if isa(a, Expr) - for (i, arg) in enumerate(a.args) - a.args[i] = norm_ast(arg) - end - if a.head === :line - return Expr(:line, a.args[1], :none) - end - if a.head === :macrocall - fa = a.args[1] - long_enough = length(a.args) >= 3 - if fa === Symbol("@int128_str") && long_enough - return Base.parse(Int128, a.args[3]) - elseif fa === Symbol("@uint128_str") && long_enough - return Base.parse(UInt128, a.args[3]) - elseif fa === Symbol("@bigint_str") && long_enough - return Base.parse(BigInt, a.args[3]) - elseif fa == Symbol("@big_str") && long_enough - s = a.args[3] - n = tryparse(BigInt, s) - if !(n === nothing) - return (n) - end - n = tryparse(BigFloat, s) - if !(n === nothing) - return isnan((n)) ? :NaN : (n) - end - return s - end - elseif length(a.args) >= 2 && Meta.isexpr(a, :call) && a.args[1] == :- && isa(a.args[2], Number) - return -a.args[2] - end - return a - elseif isa(a, QuoteNode) - return Expr(:quote, norm_ast(a.value)) - elseif isa(a, AbstractFloat) && isnan(a) - return :NaN - end - return a -end - -function meta_parse_has_error(x::Expr) - if x.head == :incomplete - return true - else - for a in x.args - if meta_parse_has_error(a) - return true - end - end - end - return false -end -meta_parse_has_error(_) = false - -function meta_parse_file(str) - pos = 1 - x1 = Expr(:file) - try - while pos <= sizeof(str) - x, pos = Meta.parse(str, pos) - push!(x1.args, x) - end - catch er - isa(er, InterruptException) && rethrow(er) - - return x1, true - end - if length(x1.args) > 0 && x1.args[end] === nothing - pop!(x1.args) - end - x1 = norm_ast(x1) - CSTParser.remlineinfo!(x1) - return x1, meta_parse_has_error(x1) -end - -find_error(x, offset = 0) = if CSTParser.headof(x) === :errortoken - @show offset - else - for a in x - find_error(a, offset) - offset += a.fullspan - end -end - -function cst_parse_file(str) - x, ps = CSTParser.parse(CSTParser.ParseState(str), true) - sp = CSTParser.check_span(x) - # remove leading/trailing nothings - if length(x.args) > 0 && CSTParser.is_nothing(x.args[1]) - popfirst!(x.args) - end - - if !isempty(sp) - @error "CST spans inconsistent!" - end - - x0 = norm_ast(to_codeobject(x)) - x0, CSTParser.has_error(ps), isempty(sp) -end - -_compare(x, y) = x == y - -function _compare(x::Expr, y::Expr) - if x == y - return true - else - if x.head != y.head || length(x.args) != length(y.args) - printstyled(x, bold = true, color = :light_red) - println() - printstyled(y, bold=true, color=:light_green) - println() - end - for i = 1:min(length(x.args), length(y.args)) - if !_compare(x.args[i], y.args[i]) - printstyled(x.args[i], bold = true, color = :light_red) - println() - printstyled(y.args[i], bold=true, color=:light_green) - println() - end - end - return false - end -end - -@testset "Parsing files in Base" begin - dir = joinpath(Sys.BINDIR, Base.DATAROOTDIR) - for (root, _, files) in walkdir(dir; follow_symlinks=true) - for fpath in files - file = joinpath(root, fpath) - endswith(file, ".jl") || continue - - str = read(file, String) - - cst = CSTParser.parse(str, true) - cst_expr, cst_err, span_err = cst_parse_file(str) - meta_expr, meta_err = meta_parse_file(str) - @test cst_err == meta_err - @test span_err - @test cst.fullspan == sizeof(str) - - if cst_err || meta_err - if cst_err && !meta_err - @error "CSTParser.parse errored, but Meta.parse didn't." file=file - elseif !cst_err && meta_err - @error "Meta.parse errored, but CSTParser.parse didn't." file=file - end - else - if cst_expr == meta_expr - @test true - else - @error "parsing difference" file=file - _compare(cst_expr, meta_expr) - @test false - end - end - end - end -end diff --git a/test/iterate.jl b/test/iterate.jl deleted file mode 100644 index c894c2f2..00000000 --- a/test/iterate.jl +++ /dev/null @@ -1,838 +0,0 @@ -using CSTParser: @cst_str, headof, valof - -function test_iter_spans(x) - n = 0 - for i = 1:length(x) - a = x[i] - if !(a isa EXPR) - @info i, headof(x), to_codeobject(x) - end - @test a isa EXPR - test_iter_spans(a) - n += a.fullspan - end - length(x) > 0 && @test n == x.fullspan -end - -@testset "iterators" begin - @testset "const local global return" begin - @testset "local" begin - x = cst"local a = 1" - @test length(x) == 2 - @test x[1] === x.trivia[1] - @test x[2] === x.args[1] - end - - @testset "global" begin - x = cst"global a = 1" - @test length(x) == 2 - @test x[1] === x.trivia[1] - @test x[2] === x.args[1] - end - - @testset "global tuple" begin - x = cst"global (a = 1,b = 2)" - @test length(x) == 6 - @test x[1] === x.trivia[1] - @test x[2] === x.trivia[2] - @test x[3] === x.args[1] - @test x[4] === x.trivia[3] - @test x[5] === x.args[2] - @test x[6] === x.trivia[4] - end - - @testset "const" begin - x = cst"const a = 1" - @test length(x) == 2 - @test x[1] === x.trivia[1] - @test x[2] === x.args[1] - end - @testset "simple" begin - x = cst"global const a = 1" - @test length(x) == 2 - @test x[1] === x.trivia[1] - @test x[2] === x.args[1] - @test length(x[2]) == 2 - @test x[2][1] === x.args[1].trivia[1] - @test x[2][2] === x.args[1].args[1] - end - - @testset "return" begin - x = cst"return a" - @test length(x) == 2 - @test x[1] === x.trivia[1] - @test x[2] === x.args[1] - end - end - - @testset "datatype declarations" begin - @testset "abstract" begin - x = cst"abstract type T end" - @test length(x) == 4 - @test x[1] === x.trivia[1] - @test x[2] === x.trivia[2] - @test x[3] === x.args[1] - @test x[4] === x.trivia[3] - end - @testset "primitive" begin - x = cst"primitive type T N end" - @test length(x) == 5 - @test x[1] === x.trivia[1] - @test x[2] === x.trivia[2] - @test x[3] === x.args[1] - @test x[4] === x.args[2] - @test x[5] === x.trivia[3] - end - @testset "struct" begin - x = cst"struct T body end" - @test length(x) == 5 - @test x[1] === x.trivia[1] - @test x[2] === x.args[1] - @test x[3] === x.args[2] - @test x[4] === x.args[3] - @test x[5] === x.trivia[2] - end - @testset "mutable" begin - x = cst"mutable struct T body end" - @test length(x) == 6 - @test x[1] === x.trivia[1] - @test x[2] === x.trivia[2] - @test x[3] === x.args[1] - @test x[4] === x.args[2] - @test x[5] === x.args[3] - @test x[6] === x.trivia[3] - end - end - - @testset ":quote" begin - @testset "block" begin - x = cst"""quote - ex1 - ex2 - end""" - @test length(x) == 3 - @test x[1] === x.trivia[1] - @test x[2] === x.args[1] - @test x[3] === x.trivia[2] - end - @testset "op" begin - x = cst""":(body + 1)""" - @test length(x) == 2 - @test x[1] === x.trivia[1] - @test x[2] === x.args[1] - end - end - - @testset ":block" begin - @testset "begin" begin - x = cst"""begin - ex1 - ex2 - end""" - @test length(x) == 4 - @test x[1] === x.trivia[1] - @test x[2] === x.args[1] - @test x[3] === x.args[2] - @test x[4] === x.trivia[2] - end - end - @testset ":for" begin - x = cst"""for itr in itr - ex1 - ex2 - end""" - @test length(x) == 4 - @test x[1] === x.trivia[1] - @test x[2] === x.args[1] - @test x[3] === x.args[2] - @test x[4] === x.trivia[2] - end - - @testset ":outer" begin - x = cst"""for outer itr in itr - ex1 - ex2 - end""".args[1].args[1] - @test length(x) == 2 - @test x[1] === x.trivia[1] - @test x[2] === x.args[1] - end - - - @testset ":function" begin - @testset "name only" begin - x = cst"""function name end""" - @test length(x) == 3 - @test x[1] === x.trivia[1] - @test x[2] === x.args[1] - @test x[3] === x.trivia[2] - end - @testset "full" begin - x = cst"""function sig() - ex1 - ex2 - end""" - @test length(x) == 4 - @test x[1] === x.trivia[1] - @test x[2] === x.args[1] - @test x[3] === x.args[2] - @test x[4] === x.trivia[2] - end - end - @testset ":braces" begin - @testset "simple" begin - x = cst"{a,b,c}" - @test length(x) == 7 - @test x[1] === x.trivia[1] - @test x[2] === x.args[1] - @test x[3] === x.trivia[2] - @test x[4] === x.args[2] - @test x[5] === x.trivia[3] - @test x[6] === x.args[3] - @test x[7] === x.trivia[4] - end - - @testset "params" begin - x = cst"{a,b;c}" - @test length(x) == 6 - @test x[1] === x.trivia[1] - @test x[2] === x.args[2] - @test x[3] === x.trivia[2] - @test x[4] === x.args[3] - @test x[5] === x.args[1] - @test x[6] === x.trivia[3] - end - end - - @testset ":curly" begin - @testset "simple" begin - x = cst"x{a,b}" - @test length(x) == 6 - @test x[1] === x.args[1] - @test x[2] === x.trivia[1] - @test x[3] === x.args[2] - @test x[4] === x.trivia[2] - @test x[5] === x.args[3] - @test x[6] === x.trivia[3] - end - @testset "params" begin - x = cst"x{a,b;c}" - @test length(x) == 7 - @test x[1] === x.args[1] - @test x[2] === x.trivia[1] - @test x[3] === x.args[3] - @test x[4] === x.trivia[2] - @test x[5] === x.args[4] - @test x[6] === x.args[2] - @test x[7] === x.trivia[3] - end - end - - @testset ":comparison" begin - x = cst"a < b < c" - @test length(x) == 5 - @test x[1] === x.args[1] - @test x[2] === x.args[2] - @test x[3] === x.args[3] - @test x[4] === x.args[4] - @test x[5] === x.args[5] - end - - @testset ":using" begin - @testset ":using" begin - x = cst"using a" - @test length(x) == 2 - @test x[1] === x.trivia[1] - @test x[2] === x.args[1] - - x = cst"using a, b, c" - @test length(x) == 6 - @test x[1] === x.trivia[1] - @test x[2] === x.args[1] - @test x[3] === x.trivia[2] - @test x[4] === x.args[2] - @test x[5] === x.trivia[3] - @test x[6] === x.args[3] - - x = cst"using .a" - @test length(x) == 2 - @test x[1] === x.trivia[1] - @test x[2] === x.args[1] - - x = cst"using a: b, c" - @test length(x) == 2 - @test x[1] === x.trivia[1] - @test x[2] === x.args[1] - end - - @testset ":" begin - x = cst"using a: b, c".args[1] - @test length(x) == 5 - @test x[1] === x.args[1] - @test x[2] === x.head - @test x[3] === x.args[2] - @test x[3] === x.args[2] - end - - @testset "." begin - x = cst"using a".args[1] - @test length(x) == 1 - @test x[1] === x.args[1] - - x = cst"using a.b".args[1] - @test length(x) == 3 - @test x[1] === x.args[1] - @test x[2] === x.trivia[1] - @test x[3] === x.args[2] - - x = cst"using ..a.b".args[1] - @test length(x) == 5 - @test x[1] === x.args[1] - @test x[2] === x.args[2] - @test x[3] === x.args[3] - @test x[4] === x.trivia[1] - @test x[5] === x.args[4] - - x = cst"using .a.b".args[1] - @test length(x) == 4 - @test x[1] === x.args[1] - @test x[2] === x.args[2] - @test x[3] === x.trivia[1] - @test x[4] === x.args[3] - - x = cst"using ...a.b".args[1] - @test length(x) == 6 - @test x[1] === x.args[1] - @test x[2] === x.args[2] - @test x[3] === x.args[3] - @test x[4] === x.args[4] - @test x[5] === x.trivia[1] - @test x[6] === x.args[5] - end - end - - @testset ":kw" begin - x = cst"f(a=1)".args[2] - @test length(x) == 3 - @test x[1] === x.args[1] - @test x[2] === x.trivia[1] - @test x[3] === x.args[2] - end - - @testset ":tuple" begin - x = cst"a,b" - @test length(x) == 3 - @test x[1] === x.args[1] - @test x[2] === x.trivia[1] - @test x[3] === x.args[2] - - x = cst"(a,b)" - @test length(x) == 5 - @test x[1] === x.trivia[1] - @test x[2] === x.args[1] - @test x[3] === x.trivia[2] - @test x[4] === x.args[2] - @test x[5] === x.trivia[3] - - x = cst"(a,b,)" - @test length(x) == 6 - @test x[1] === x.trivia[1] - @test x[2] === x.args[1] - @test x[3] === x.trivia[2] - @test x[4] === x.args[2] - @test x[5] === x.trivia[3] - @test x[6] === x.trivia[4] - end - - @testset ":call" begin - x = cst"f()" - @test length(x) == 3 - @test x[1] === x.args[1] - @test x[2] === x.trivia[1] - @test x[3] === x.trivia[2] - - x = cst"f(a)" - @test length(x) == 4 - @test x[1] === x.args[1] - @test x[2] === x.trivia[1] - @test x[3] === x.args[2] - @test x[4] === x.trivia[2] - - x = cst"f(a,)" - @test length(x) == 5 - @test x[1] === x.args[1] - @test x[2] === x.trivia[1] - @test x[3] === x.args[2] - @test x[4] === x.trivia[2] - @test x[5] === x.trivia[3] - - x = cst"f(;)" - @test length(x) == 4 - @test x[1] === x.args[1] - @test x[2] === x.trivia[1] - @test x[3] === x.args[2] - @test x[4] === x.trivia[2] - - x = cst"f(a, b;c = 1)" - @test length(x) == 7 - @test x[1] === x.args[1] - @test x[2] === x.trivia[1] - @test x[3] === x.args[3] - @test x[4] === x.trivia[2] - @test x[5] === x.args[4] - @test x[6] === x.args[2] - @test x[7] === x.trivia[3] - - x = cst"f(a, b,;c = 1)" - @test length(x) == 8 - @test x[1] === x.args[1] - @test x[2] === x.trivia[1] - @test x[3] === x.args[3] - @test x[4] === x.trivia[2] - @test x[5] === x.args[4] - @test x[6] === x.trivia[3] - @test x[7] === x.args[2] - @test x[8] === x.trivia[4] - end - - @testset ":where" begin - x = cst"a where {b,c;d}" - @test length(x) == 8 - @test x[1] === x.args[1] - @test x[2] === x.trivia[1] - @test x[3] === x.trivia[2] - @test x[4] === x.args[3] - @test x[5] === x.trivia[3] - @test x[6] === x.args[4] - @test x[7] === x.args[2] - @test x[8] === x.trivia[4] - end - - @testset ":quotenode" begin - x = cst"a.b".args[2] - @test length(x) == 1 - @test x[1] === x.args[1] - - x = cst"a.:b".args[2] - @test length(x) == 2 - @test x[1] === x.trivia[1] - @test x[2] === x.args[1] - end - - @testset ":if" begin - x = cst"if cond end" - @test length(x) == 4 - @test headof(x[1]) === :IF - @test x[2] === x.args[1] - @test x[3] === x.args[2] - @test headof(x[4]) === :END - - x = cst"if cond else end" - @test length(x) == 6 - @test headof(x[1]) === :IF - @test x[2] === x.args[1] - @test x[3] === x.args[2] - @test headof(x[4]) === :ELSE - @test x[5] === x.args[3] - @test headof(x[6]) === :END - - x = cst"if cond args elseif a end" - @test length(x) == 5 - @test headof(x[1]) === :IF - @test x[2] === x.args[1] - @test x[3] === x.args[2] - @test headof(x[4]) === :elseif - @test headof(x[5]) === :END - - x = cst"a ? b : c" - @test length(x) == 5 - @test valof(x[1]) === "a" - @test CSTParser.isoperator(x[2]) - @test valof(x[3]) === "b" - @test CSTParser.isoperator(x[4]) - @test valof(x[5]) === "c" - end - - @testset ":elseif" begin - x = cst"if cond elseif c args end".args[3] - @test length(x) == 3 - @test headof(x[1]) === :ELSEIF - @test x[2] === x.args[1] - @test x[3] === x.args[2] - - x = cst"if cond elseif c args else args end".args[3] - @test length(x) == 5 - @test headof(x[1]) === :ELSEIF - @test x[2] === x.args[1] - @test x[3] === x.args[2] - @test headof(x[4]) === :ELSE - @test x[5] === x.args[3] - end - - @testset ":string" begin - x = cst"\"txt$interp txt\"" - @test length(x) == 4 - @test x[1] === x.args[1] - @test x[2] === x.trivia[1] - @test x[3] === x.args[2] - @test x[4] === x.args[3] - - x = cst"\"txt1 $interp1 txt2 $interp2 txt3\"" - @test length(x) == 7 - @test x[1] === x.args[1] - @test x[2] === x.trivia[1] - @test x[3] === x.args[2] - @test x[4] === x.args[3] - @test x[5] === x.trivia[2] - @test x[6] === x.args[4] - @test x[7] === x.args[5] - - x = cst"\"$interp\"" - @test length(x) == 4 - @test x[1] === x.trivia[1] - @test x[2] === x.trivia[2] - @test x[3] === x.args[1] - @test x[4] === x.trivia[3] - - x = cst"\"$interp txt\"" - @test length(x) == 4 - @test x[1] === x.trivia[1] - @test x[2] === x.trivia[2] - @test x[3] === x.args[1] - @test x[4] === x.args[2] - - x = cst"\"$(interp)\"" - @test length(x) == 6 - @test x[1] === x.trivia[1] - @test x[2] === x.trivia[2] - @test x[3] === x.trivia[3] - @test x[4] === x.args[1] - @test x[5] === x.trivia[4] - @test x[6] === x.trivia[5] - - x = cst"\"a$b$c \"" - @test length(x) == 6 - @test x[1] === x.args[1] - @test x[2] === x.trivia[1] - @test x[3] === x.args[2] - @test x[4] === x.trivia[2] - @test x[5] === x.args[3] - - x = cst"\"$(a)$(b)$(c)d\"" - @test length(x) == 14 - @test x[1] === x.trivia[1] - @test x[2] === x.trivia[2] - @test x[3] === x.trivia[3] - @test x[4] === x.args[1] - @test x[5] === x.trivia[4] - @test x[6] === x.trivia[5] - @test x[7] === x.trivia[6] - @test x[8] === x.args[2] - @test x[9] === x.trivia[7] - @test x[10] === x.trivia[8] - @test x[11] === x.trivia[9] - @test x[12] === x.args[3] - @test x[13] === x.trivia[10] - @test x[14] === x.args[4] - - x = cst""" - "$(()$)" - """ - @test x[6] === x.trivia[5] - - x = cst"\"$(\"\")\"" - @test length(x) == 6 - @test x[1] === x.trivia[1] - @test x[2] === x.trivia[2] - @test x[3] === x.trivia[3] - @test x[4] === x.args[1] - @test x[5] === x.trivia[4] - @test x[6] === x.trivia[5] - - x = EXPR(:string, EXPR[cst"\" \"", EXPR(:errortoken, 0, 0), EXPR(:errortoken, 0, 0)], EXPR[cst"$"]) - @test x[4] == x.args[3] - end - - @testset ":macrocall" begin - x = cst"@mac a" - @test length(x) == 3 - @test x[1] === x.args[1] - @test x[2] === x.args[2] - @test x[3] === x.args[3] - - x = cst"@mac(a)" - @test length(x) == 5 - @test x[1] === x.args[1] - @test x[2] === x.args[2] - @test x[3] === x.trivia[1] - @test x[4] === x.args[3] - @test x[5] === x.trivia[2] - - x = cst"@mac(a, b)" - @test length(x) == 7 - @test x[1] === x.args[1] - @test x[2] === x.args[2] - @test x[3] === x.trivia[1] - @test x[4] === x.args[3] - @test x[5] === x.trivia[2] - @test x[6] === x.args[4] - @test x[7] === x.trivia[3] - - x = cst"@mac(a; b = 1)" - @test length(x) == 6 - @test x[1] === x.args[1] - @test x[2] === x.args[2] - @test x[3] === x.trivia[1] - @test x[4] === x.args[4] - @test x[5] === x.args[3] - @test x[6] === x.trivia[2] - - x = cst"@mac(a, b; x)" - @test length(x) == 8 - @test x[1] === x.args[1] - @test x[2] === x.args[2] - @test x[3] === x.trivia[1] - @test x[4] === x.args[4] - @test x[5] === x.trivia[2] - @test x[6] === x.args[5] - @test x[7] === x.args[3] - @test x[8] === x.trivia[3] - end - - @testset ":brackets" begin - x = cst"(x)" - @test length(x) == 3 - @test x[1] === x.trivia[1] - @test x[2] === x.args[1] - @test x[3] === x.trivia[2] - end - - @testset ":ref" begin - x = cst"x[i]" - @test length(x) == 4 - @test x[1] === x.args[1] - @test x[2] === x.trivia[1] - @test x[3] === x.args[2] - @test x[4] === x.trivia[2] - - x = cst"x[i, j]" - @test length(x) == 6 - @test x[1] === x.args[1] - @test x[2] === x.trivia[1] - @test x[3] === x.args[2] - @test x[4] === x.trivia[2] - @test x[5] === x.args[3] - @test x[6] === x.trivia[3] - - x = cst"x[i, j; k = 1]" - @test length(x) == 7 - @test x[1] === x.args[1] - @test x[2] === x.trivia[1] - @test x[3] === x.args[3] - @test x[4] === x.trivia[2] - @test x[5] === x.args[4] - @test x[6] === x.args[2] - @test x[7] === x.trivia[3] - end - @testset ":typed_vcat" begin - x = cst"x[i;j]" - @test length(x) == 5 - @test x[1] === x.args[1] - @test x[2] === x.trivia[1] - @test x[3] === x.args[2] - @test x[4] === x.args[3] - @test x[5] === x.trivia[2] - end - - @testset ":row" begin - x = cst"[a b; c d ]".args[1] - @test length(x) == 2 - @test x[1] === x.args[1] - @test x[2] === x.args[2] - end - - @testset ":module" begin - x = cst"module a end" - @test length(x) == 5 - @test x[1] === x.trivia[1] - @test x[2] === x.args[1] - @test x[3] === x.args[2] - @test x[4] === x.args[3] - @test x[5] === x.trivia[2] - end - - @testset ":export" begin - x = cst"export a, b, c" - @test length(x) == 6 - @test x[1] === x.trivia[1] - @test x[2] === x.args[1] - @test x[3] === x.trivia[2] - @test x[4] === x.args[2] - @test x[5] === x.trivia[3] - @test x[6] === x.args[3] - end - - @testset ":parameters" begin - x = cst"f(a; b=1, c=1, d=1)"[4] - @test length(x) == 5 - @test x[1] === x.args[1] - @test x[2] === x.trivia[1] - @test x[3] === x.args[2] - @test x[4] === x.trivia[2] - @test x[5] === x.args[3] - end - - @testset "lowered iterator" begin - x = cst"for a in b end".args[1] - @test length(x) == 3 - @test x[1] === x.args[1] - @test x[2] === x.trivia[1] - @test x[3] === x.args[2] - end - - @testset ":do" begin - x = cst"f(x) do arg something end" - @test length(x) == 4 - @test x[1] === x.args[1] - @test x[2] === x.trivia[1] - @test x[3] === x.args[2] - @test x[4] === x.trivia[2] - end - - @testset ":generator" begin - x = cst"(a for a in A)".args[1] - @test length(x) == 3 - @test x[1] === x.args[1] - @test x[2] === x.trivia[1] - @test x[3] === x.args[2] - - x = cst"(a for a in A, b in B)".args[1] - @test length(x) == 5 - @test x[1] === x.args[1] - @test x[2] === x.trivia[1] - @test x[3] === x.args[2] - @test x[4] === x.trivia[2] - @test x[5] === x.args[3] - end - - @testset ":flatten" begin - function flatten(x) - if length(x) == 0 - [x] - else - vcat([flatten(a) for a in x]...) - end - end - function testflattenorder(s) - x = CSTParser.parse(s)[2] - issorted([Base.parse(Int, a.val) for a in flatten(x) if a.head === :INTEGER]) - end - - @test testflattenorder("(1 for 2 in 3)") - @test testflattenorder("(1 for 2 in 3 for 4 in 5)") - @test testflattenorder("(1 for 2 in 3, 4 in 5 for 6 in 7)") - @test testflattenorder("(1 for 2 in 3 for 4 in 5, 6 in 7)") - @test testflattenorder("(1 for 2 in 3 for 4 in 5, 6 in 7 if 8)") - end - - @testset ":filter" begin - x = cst"(a for a in A if a)".args[1].args[2] - @test length(x) == 3 - @test valof(headof(x[1])) == "=" - @test headof(x[2]) === :IF - @test valof(x[3]) == "a" - end - - @testset ":try" begin - x = cst"try expr catch e end" - @test length(x) == 6 - @test headof(x[1]) === :TRY - @test headof(x[2]) === :block - @test headof(x[3]) === :CATCH - @test valof(x[4]) == "e" - @test headof(x[5]) === :block - @test headof(x[6]) === :END - - x = cst"try expr finally expr2 end" - @test length(x) == 8 - @test headof(x[1]) === :TRY - @test headof(x[2]) === :block - @test headof(x[3]) === :CATCH - @test x[3].fullspan == 0 - @test headof(x[4]) === :FALSE - @test headof(x[5]) === :FALSE - @test headof(x[6]) === :FINALLY - @test headof(x[7]) === :block - @test headof(x[8]) === :END - - x = cst"try expr catch err expr2 finally expr3 end" - @test length(x) == 8 - @test headof(x[1]) === :TRY - @test headof(x[2]) === :block - @test headof(x[3]) === :CATCH - @test valof(x[4]) == "err" - @test headof(x[5]) === :block - @test headof(x[6]) === :FINALLY - @test headof(x[7]) === :block - @test headof(x[8]) === :END - end - - @testset ":comprehension" begin - x = cst"[a for a in A]" - @test length(x) == 3 - @test headof(x[1]) === :LSQUARE - @test headof(x[2]) === :generator - @test headof(x[3]) === :RSQUARE - end - - @testset ":typed_comprehension" begin - x = cst"T[a for a in A]" - @test length(x) == 4 - @test headof(x[1]) === :IDENTIFIER - @test headof(x[2]) === :LSQUARE - @test headof(x[3]) === :generator - @test headof(x[4]) === :RSQUARE - end - - @testset "unary syntax" begin - x = cst"<:a" - @test length(x) == 2 - @test headof(x[1]) === :OPERATOR - @test headof(x[2]) === :IDENTIFIER - - x = cst">:a" - @test length(x) == 2 - @test headof(x[1]) === :OPERATOR - @test headof(x[2]) === :IDENTIFIER - - x = cst"::a" - @test length(x) == 2 - @test headof(x[1]) === :OPERATOR - @test headof(x[2]) === :IDENTIFIER - - x = cst"&a" - @test length(x) == 2 - @test headof(x[1]) === :OPERATOR - @test headof(x[2]) === :IDENTIFIER - - x = cst"a..." - @test length(x) == 2 - @test headof(x[1]) === :IDENTIFIER - @test headof(x[2]) === :OPERATOR - - x = cst"$a" - @test length(x) == 2 - @test headof(x[1]) === :OPERATOR - @test headof(x[2]) === :IDENTIFIER - end -end - -@testset "self test" begin - test_iter_spans(CSTParser.parse(String(read("parser.jl")), true)) - - for f in joinpath.(abspath("../src"), readdir("../src")) - if endswith(f, ".jl") - test_iter_spans(CSTParser.parse(String(read(f)), true)) - end - end -end diff --git a/test/iterate/test_iterators.jl b/test/iterate/test_iterators.jl new file mode 100644 index 00000000..fb8d1ecb --- /dev/null +++ b/test/iterate/test_iterators.jl @@ -0,0 +1,905 @@ +using CSTParser: @cst_str, headof, valof + +@testitem "const local global return.local" begin + using CSTParser: @cst_str, headof, valof + + x = cst"local a = 1" + @test length(x) == 2 + @test x[1] === x.trivia[1] + @test x[2] === x.args[1] +end + +@testitem "const local global return.global" begin + using CSTParser: @cst_str, headof, valof + + x = cst"global a = 1" + @test length(x) == 2 + @test x[1] === x.trivia[1] + @test x[2] === x.args[1] +end + +@testitem "const local global return.global tuple" begin + using CSTParser: @cst_str, headof, valof + + x = cst"global (a = 1,b = 2)" + @test length(x) == 6 + @test x[1] === x.trivia[1] + @test x[2] === x.trivia[2] + @test x[3] === x.args[1] + @test x[4] === x.trivia[3] + @test x[5] === x.args[2] + @test x[6] === x.trivia[4] +end + +@testitem "const local global return.const" begin + using CSTParser: @cst_str, headof, valof + + x = cst"const a = 1" + @test length(x) == 2 + @test x[1] === x.trivia[1] + @test x[2] === x.args[1] +end +@testitem "const local global return.simple" begin + using CSTParser: @cst_str, headof, valof + + x = cst"global const a = 1" + @test length(x) == 2 + @test x[1] === x.trivia[1] + @test x[2] === x.args[1] + @test length(x[2]) == 2 + @test x[2][1] === x.args[1].trivia[1] + @test x[2][2] === x.args[1].args[1] +end + +@testitem "const local global return.return" begin + using CSTParser: @cst_str, headof, valof + + x = cst"return a" + @test length(x) == 2 + @test x[1] === x.trivia[1] + @test x[2] === x.args[1] +end + +@testitem "datatype declarations.abstract" begin + using CSTParser: @cst_str, headof, valof + + x = cst"abstract type T end" + @test length(x) == 4 + @test x[1] === x.trivia[1] + @test x[2] === x.trivia[2] + @test x[3] === x.args[1] + @test x[4] === x.trivia[3] +end + +@testitem "datatype declarations.primitive" begin + using CSTParser: @cst_str, headof, valof + + x = cst"primitive type T N end" + @test length(x) == 5 + @test x[1] === x.trivia[1] + @test x[2] === x.trivia[2] + @test x[3] === x.args[1] + @test x[4] === x.args[2] + @test x[5] === x.trivia[3] +end + +@testitem "datatype declarations.struct" begin + using CSTParser: @cst_str, headof, valof + + x = cst"struct T body end" + @test length(x) == 5 + @test x[1] === x.trivia[1] + @test x[2] === x.args[1] + @test x[3] === x.args[2] + @test x[4] === x.args[3] + @test x[5] === x.trivia[2] +end + +@testitem "datatype declarations.mutable" begin + using CSTParser: @cst_str, headof, valof + + x = cst"mutable struct T body end" + @test length(x) == 6 + @test x[1] === x.trivia[1] + @test x[2] === x.trivia[2] + @test x[3] === x.args[1] + @test x[4] === x.args[2] + @test x[5] === x.args[3] + @test x[6] === x.trivia[3] +end + +@testitem "quote.block" begin + using CSTParser: @cst_str, headof, valof + + x = cst"""quote + ex1 + ex2 + end""" + @test length(x) == 3 + @test x[1] === x.trivia[1] + @test x[2] === x.args[1] + @test x[3] === x.trivia[2] +end + +@testitem "quote.op" begin + using CSTParser: @cst_str, headof, valof + + x = cst""":(body + 1)""" + @test length(x) == 2 + @test x[1] === x.trivia[1] + @test x[2] === x.args[1] +end + +@testitem "block.begin" begin + using CSTParser: @cst_str, headof, valof + + x = cst"""begin + ex1 + ex2 + end""" + @test length(x) == 4 + @test x[1] === x.trivia[1] + @test x[2] === x.args[1] + @test x[3] === x.args[2] + @test x[4] === x.trivia[2] +end + +@testitem ":for" begin + using CSTParser: @cst_str, headof, valof + + x = cst"""for itr in itr + ex1 + ex2 + end""" + @test length(x) == 4 + @test x[1] === x.trivia[1] + @test x[2] === x.args[1] + @test x[3] === x.args[2] + @test x[4] === x.trivia[2] +end + +@testitem ":outer" begin + using CSTParser: @cst_str, headof, valof + + x = cst"""for outer itr in itr + ex1 + ex2 + end""".args[1].args[1] + @test length(x) == 2 + @test x[1] === x.trivia[1] + @test x[2] === x.args[1] +end + + +@testitem "function.name only" begin + using CSTParser: @cst_str, headof, valof + + x = cst"""function name end""" + @test length(x) == 3 + @test x[1] === x.trivia[1] + @test x[2] === x.args[1] + @test x[3] === x.trivia[2] +end + +@testitem "function.full" begin + using CSTParser: @cst_str, headof, valof + + x = cst"""function sig() + ex1 + ex2 + end""" + @test length(x) == 4 + @test x[1] === x.trivia[1] + @test x[2] === x.args[1] + @test x[3] === x.args[2] + @test x[4] === x.trivia[2] +end + +@testitem "braces.simple" begin + using CSTParser: @cst_str, headof, valof + + x = cst"{a,b,c}" + @test length(x) == 7 + @test x[1] === x.trivia[1] + @test x[2] === x.args[1] + @test x[3] === x.trivia[2] + @test x[4] === x.args[2] + @test x[5] === x.trivia[3] + @test x[6] === x.args[3] + @test x[7] === x.trivia[4] +end + +@testitem "braces.params" begin + using CSTParser: @cst_str, headof, valof + + x = cst"{a,b;c}" + @test length(x) == 6 + @test x[1] === x.trivia[1] + @test x[2] === x.args[2] + @test x[3] === x.trivia[2] + @test x[4] === x.args[3] + @test x[5] === x.args[1] + @test x[6] === x.trivia[3] +end + +@testitem "curly.simple" begin + using CSTParser: @cst_str, headof, valof + + x = cst"x{a,b}" + @test length(x) == 6 + @test x[1] === x.args[1] + @test x[2] === x.trivia[1] + @test x[3] === x.args[2] + @test x[4] === x.trivia[2] + @test x[5] === x.args[3] + @test x[6] === x.trivia[3] +end + +@testitem "curly.params" begin + using CSTParser: @cst_str, headof, valof + + x = cst"x{a,b;c}" + @test length(x) == 7 + @test x[1] === x.args[1] + @test x[2] === x.trivia[1] + @test x[3] === x.args[3] + @test x[4] === x.trivia[2] + @test x[5] === x.args[4] + @test x[6] === x.args[2] + @test x[7] === x.trivia[3] +end + +@testitem ":comparison" begin + using CSTParser: @cst_str, headof, valof + + x = cst"a < b < c" + @test length(x) == 5 + @test x[1] === x.args[1] + @test x[2] === x.args[2] + @test x[3] === x.args[3] + @test x[4] === x.args[4] + @test x[5] === x.args[5] +end + +@testitem "using.:using" begin + using CSTParser: @cst_str, headof, valof + + x = cst"using a" + @test length(x) == 2 + @test x[1] === x.trivia[1] + @test x[2] === x.args[1] + + x = cst"using a, b, c" + @test length(x) == 6 + @test x[1] === x.trivia[1] + @test x[2] === x.args[1] + @test x[3] === x.trivia[2] + @test x[4] === x.args[2] + @test x[5] === x.trivia[3] + @test x[6] === x.args[3] + + x = cst"using .a" + @test length(x) == 2 + @test x[1] === x.trivia[1] + @test x[2] === x.args[1] + + x = cst"using a: b, c" + @test length(x) == 2 + @test x[1] === x.trivia[1] + @test x[2] === x.args[1] +end + +@testitem "using.:" begin + using CSTParser: @cst_str, headof, valof + + x = cst"using a: b, c".args[1] + @test length(x) == 5 + @test x[1] === x.args[1] + @test x[2] === x.head + @test x[3] === x.args[2] + @test x[3] === x.args[2] +end + +@testitem "using.." begin + using CSTParser: @cst_str, headof, valof + + x = cst"using a".args[1] + @test length(x) == 1 + @test x[1] === x.args[1] + + x = cst"using a.b".args[1] + @test length(x) == 3 + @test x[1] === x.args[1] + @test x[2] === x.trivia[1] + @test x[3] === x.args[2] + + x = cst"using ..a.b".args[1] + @test length(x) == 5 + @test x[1] === x.args[1] + @test x[2] === x.args[2] + @test x[3] === x.args[3] + @test x[4] === x.trivia[1] + @test x[5] === x.args[4] + + x = cst"using .a.b".args[1] + @test length(x) == 4 + @test x[1] === x.args[1] + @test x[2] === x.args[2] + @test x[3] === x.trivia[1] + @test x[4] === x.args[3] + + x = cst"using ...a.b".args[1] + @test length(x) == 6 + @test x[1] === x.args[1] + @test x[2] === x.args[2] + @test x[3] === x.args[3] + @test x[4] === x.args[4] + @test x[5] === x.trivia[1] + @test x[6] === x.args[5] +end + +@testitem ":kw" begin + using CSTParser: @cst_str, headof, valof + + x = cst"f(a=1)".args[2] + @test length(x) == 3 + @test x[1] === x.args[1] + @test x[2] === x.trivia[1] + @test x[3] === x.args[2] +end + +@testitem ":tuple" begin + using CSTParser: @cst_str, headof, valof + + x = cst"a,b" + @test length(x) == 3 + @test x[1] === x.args[1] + @test x[2] === x.trivia[1] + @test x[3] === x.args[2] + + x = cst"(a,b)" + @test length(x) == 5 + @test x[1] === x.trivia[1] + @test x[2] === x.args[1] + @test x[3] === x.trivia[2] + @test x[4] === x.args[2] + @test x[5] === x.trivia[3] + + x = cst"(a,b,)" + @test length(x) == 6 + @test x[1] === x.trivia[1] + @test x[2] === x.args[1] + @test x[3] === x.trivia[2] + @test x[4] === x.args[2] + @test x[5] === x.trivia[3] + @test x[6] === x.trivia[4] +end + +@testitem ":call" begin + using CSTParser: @cst_str, headof, valof + + x = cst"f()" + @test length(x) == 3 + @test x[1] === x.args[1] + @test x[2] === x.trivia[1] + @test x[3] === x.trivia[2] + + x = cst"f(a)" + @test length(x) == 4 + @test x[1] === x.args[1] + @test x[2] === x.trivia[1] + @test x[3] === x.args[2] + @test x[4] === x.trivia[2] + + x = cst"f(a,)" + @test length(x) == 5 + @test x[1] === x.args[1] + @test x[2] === x.trivia[1] + @test x[3] === x.args[2] + @test x[4] === x.trivia[2] + @test x[5] === x.trivia[3] + + x = cst"f(;)" + @test length(x) == 4 + @test x[1] === x.args[1] + @test x[2] === x.trivia[1] + @test x[3] === x.args[2] + @test x[4] === x.trivia[2] + + x = cst"f(a, b;c = 1)" + @test length(x) == 7 + @test x[1] === x.args[1] + @test x[2] === x.trivia[1] + @test x[3] === x.args[3] + @test x[4] === x.trivia[2] + @test x[5] === x.args[4] + @test x[6] === x.args[2] + @test x[7] === x.trivia[3] + + x = cst"f(a, b,;c = 1)" + @test length(x) == 8 + @test x[1] === x.args[1] + @test x[2] === x.trivia[1] + @test x[3] === x.args[3] + @test x[4] === x.trivia[2] + @test x[5] === x.args[4] + @test x[6] === x.trivia[3] + @test x[7] === x.args[2] + @test x[8] === x.trivia[4] +end + +@testitem ":where" begin + using CSTParser: @cst_str, headof, valof + + x = cst"a where {b,c;d}" + @test length(x) == 8 + @test x[1] === x.args[1] + @test x[2] === x.trivia[1] + @test x[3] === x.trivia[2] + @test x[4] === x.args[3] + @test x[5] === x.trivia[3] + @test x[6] === x.args[4] + @test x[7] === x.args[2] + @test x[8] === x.trivia[4] +end + +@testitem ":quotenode" begin + using CSTParser: @cst_str, headof, valof + + x = cst"a.b".args[2] + @test length(x) == 1 + @test x[1] === x.args[1] + + x = cst"a.:b".args[2] + @test length(x) == 2 + @test x[1] === x.trivia[1] + @test x[2] === x.args[1] +end + +@testitem ":if" begin + using CSTParser: @cst_str, headof, valof + + x = cst"if cond end" + @test length(x) == 4 + @test headof(x[1]) === :IF + @test x[2] === x.args[1] + @test x[3] === x.args[2] + @test headof(x[4]) === :END + + x = cst"if cond else end" + @test length(x) == 6 + @test headof(x[1]) === :IF + @test x[2] === x.args[1] + @test x[3] === x.args[2] + @test headof(x[4]) === :ELSE + @test x[5] === x.args[3] + @test headof(x[6]) === :END + + x = cst"if cond args elseif a end" + @test length(x) == 5 + @test headof(x[1]) === :IF + @test x[2] === x.args[1] + @test x[3] === x.args[2] + @test headof(x[4]) === :elseif + @test headof(x[5]) === :END + + x = cst"a ? b : c" + @test length(x) == 5 + @test valof(x[1]) === "a" + @test CSTParser.isoperator(x[2]) + @test valof(x[3]) === "b" + @test CSTParser.isoperator(x[4]) + @test valof(x[5]) === "c" +end + +@testitem ":elseif" begin + using CSTParser: @cst_str, headof, valof + + x = cst"if cond elseif c args end".args[3] + @test length(x) == 3 + @test headof(x[1]) === :ELSEIF + @test x[2] === x.args[1] + @test x[3] === x.args[2] + + x = cst"if cond elseif c args else args end".args[3] + @test length(x) == 5 + @test headof(x[1]) === :ELSEIF + @test x[2] === x.args[1] + @test x[3] === x.args[2] + @test headof(x[4]) === :ELSE + @test x[5] === x.args[3] +end + +@testitem ":string" begin + using CSTParser: @cst_str, headof, valof, EXPR + + x = cst"\"txt$interp txt\"" + @test length(x) == 4 + @test x[1] === x.args[1] + @test x[2] === x.trivia[1] + @test x[3] === x.args[2] + @test x[4] === x.args[3] + + x = cst"\"txt1 $interp1 txt2 $interp2 txt3\"" + @test length(x) == 7 + @test x[1] === x.args[1] + @test x[2] === x.trivia[1] + @test x[3] === x.args[2] + @test x[4] === x.args[3] + @test x[5] === x.trivia[2] + @test x[6] === x.args[4] + @test x[7] === x.args[5] + + x = cst"\"$interp\"" + @test length(x) == 4 + @test x[1] === x.trivia[1] + @test x[2] === x.trivia[2] + @test x[3] === x.args[1] + @test x[4] === x.trivia[3] + + x = cst"\"$interp txt\"" + @test length(x) == 4 + @test x[1] === x.trivia[1] + @test x[2] === x.trivia[2] + @test x[3] === x.args[1] + @test x[4] === x.args[2] + + x = cst"\"$(interp)\"" + @test length(x) == 6 + @test x[1] === x.trivia[1] + @test x[2] === x.trivia[2] + @test x[3] === x.trivia[3] + @test x[4] === x.args[1] + @test x[5] === x.trivia[4] + @test x[6] === x.trivia[5] + + x = cst"\"a$b$c \"" + @test length(x) == 6 + @test x[1] === x.args[1] + @test x[2] === x.trivia[1] + @test x[3] === x.args[2] + @test x[4] === x.trivia[2] + @test x[5] === x.args[3] + + x = cst"\"$(a)$(b)$(c)d\"" + @test length(x) == 14 + @test x[1] === x.trivia[1] + @test x[2] === x.trivia[2] + @test x[3] === x.trivia[3] + @test x[4] === x.args[1] + @test x[5] === x.trivia[4] + @test x[6] === x.trivia[5] + @test x[7] === x.trivia[6] + @test x[8] === x.args[2] + @test x[9] === x.trivia[7] + @test x[10] === x.trivia[8] + @test x[11] === x.trivia[9] + @test x[12] === x.args[3] + @test x[13] === x.trivia[10] + @test x[14] === x.args[4] + + x = cst""" + "$(()$)" + """ + @test x[6] === x.trivia[5] + + x = cst"\"$(\"\")\"" + @test length(x) == 6 + @test x[1] === x.trivia[1] + @test x[2] === x.trivia[2] + @test x[3] === x.trivia[3] + @test x[4] === x.args[1] + @test x[5] === x.trivia[4] + @test x[6] === x.trivia[5] + + x = EXPR(:string, EXPR[cst"\" \"", EXPR(:errortoken, 0, 0), EXPR(:errortoken, 0, 0)], EXPR[cst"$"]) + @test x[4] == x.args[3] +end + +@testitem ":macrocall" begin + using CSTParser: @cst_str, headof, valof + + x = cst"@mac a" + @test length(x) == 3 + @test x[1] === x.args[1] + @test x[2] === x.args[2] + @test x[3] === x.args[3] + + x = cst"@mac(a)" + @test length(x) == 5 + @test x[1] === x.args[1] + @test x[2] === x.args[2] + @test x[3] === x.trivia[1] + @test x[4] === x.args[3] + @test x[5] === x.trivia[2] + + x = cst"@mac(a, b)" + @test length(x) == 7 + @test x[1] === x.args[1] + @test x[2] === x.args[2] + @test x[3] === x.trivia[1] + @test x[4] === x.args[3] + @test x[5] === x.trivia[2] + @test x[6] === x.args[4] + @test x[7] === x.trivia[3] + + x = cst"@mac(a; b = 1)" + @test length(x) == 6 + @test x[1] === x.args[1] + @test x[2] === x.args[2] + @test x[3] === x.trivia[1] + @test x[4] === x.args[4] + @test x[5] === x.args[3] + @test x[6] === x.trivia[2] + + x = cst"@mac(a, b; x)" + @test length(x) == 8 + @test x[1] === x.args[1] + @test x[2] === x.args[2] + @test x[3] === x.trivia[1] + @test x[4] === x.args[4] + @test x[5] === x.trivia[2] + @test x[6] === x.args[5] + @test x[7] === x.args[3] + @test x[8] === x.trivia[3] +end + +@testitem ":brackets" begin + using CSTParser: @cst_str, headof, valof + + x = cst"(x)" + @test length(x) == 3 + @test x[1] === x.trivia[1] + @test x[2] === x.args[1] + @test x[3] === x.trivia[2] +end + +@testitem ":ref" begin + using CSTParser: @cst_str, headof, valof + + x = cst"x[i]" + @test length(x) == 4 + @test x[1] === x.args[1] + @test x[2] === x.trivia[1] + @test x[3] === x.args[2] + @test x[4] === x.trivia[2] + + x = cst"x[i, j]" + @test length(x) == 6 + @test x[1] === x.args[1] + @test x[2] === x.trivia[1] + @test x[3] === x.args[2] + @test x[4] === x.trivia[2] + @test x[5] === x.args[3] + @test x[6] === x.trivia[3] + + x = cst"x[i, j; k = 1]" + @test length(x) == 7 + @test x[1] === x.args[1] + @test x[2] === x.trivia[1] + @test x[3] === x.args[3] + @test x[4] === x.trivia[2] + @test x[5] === x.args[4] + @test x[6] === x.args[2] + @test x[7] === x.trivia[3] +end + +@testitem ":typed_vcat" begin + using CSTParser: @cst_str, headof, valof + + x = cst"x[i;j]" + @test length(x) == 5 + @test x[1] === x.args[1] + @test x[2] === x.trivia[1] + @test x[3] === x.args[2] + @test x[4] === x.args[3] + @test x[5] === x.trivia[2] +end + +@testitem ":row" begin + using CSTParser: @cst_str, headof, valof + + x = cst"[a b; c d ]".args[1] + @test length(x) == 2 + @test x[1] === x.args[1] + @test x[2] === x.args[2] +end + +@testitem ":module" begin + using CSTParser: @cst_str, headof, valof + + x = cst"module a end" + @test length(x) == 5 + @test x[1] === x.trivia[1] + @test x[2] === x.args[1] + @test x[3] === x.args[2] + @test x[4] === x.args[3] + @test x[5] === x.trivia[2] +end + +@testitem ":export" begin + using CSTParser: @cst_str, headof, valof + + x = cst"export a, b, c" + @test length(x) == 6 + @test x[1] === x.trivia[1] + @test x[2] === x.args[1] + @test x[3] === x.trivia[2] + @test x[4] === x.args[2] + @test x[5] === x.trivia[3] + @test x[6] === x.args[3] +end + +@testitem ":parameters" begin + using CSTParser: @cst_str, headof, valof + + x = cst"f(a; b=1, c=1, d=1)"[4] + @test length(x) == 5 + @test x[1] === x.args[1] + @test x[2] === x.trivia[1] + @test x[3] === x.args[2] + @test x[4] === x.trivia[2] + @test x[5] === x.args[3] +end + +@testitem "lowered iterator" begin + using CSTParser: @cst_str, headof, valof + + x = cst"for a in b end".args[1] + @test length(x) == 3 + @test x[1] === x.args[1] + @test x[2] === x.trivia[1] + @test x[3] === x.args[2] +end + +@testitem ":do" begin + using CSTParser: @cst_str, headof, valof + + x = cst"f(x) do arg something end" + @test length(x) == 4 + @test x[1] === x.args[1] + @test x[2] === x.trivia[1] + @test x[3] === x.args[2] + @test x[4] === x.trivia[2] +end + +@testitem ":generator" begin + using CSTParser: @cst_str, headof, valof + + x = cst"(a for a in A)".args[1] + @test length(x) == 3 + @test x[1] === x.args[1] + @test x[2] === x.trivia[1] + @test x[3] === x.args[2] + + x = cst"(a for a in A, b in B)".args[1] + @test length(x) == 5 + @test x[1] === x.args[1] + @test x[2] === x.trivia[1] + @test x[3] === x.args[2] + @test x[4] === x.trivia[2] + @test x[5] === x.args[3] +end + +@testitem ":flatten" begin + using CSTParser: @cst_str, headof, valof + + function flatten(x) + if length(x) == 0 + [x] + else + vcat([flatten(a) for a in x]...) + end + end + function testflattenorder(s) + x = CSTParser.parse(s)[2] + issorted([Base.parse(Int, a.val) for a in flatten(x) if a.head === :INTEGER]) + end + + @test testflattenorder("(1 for 2 in 3)") + @test testflattenorder("(1 for 2 in 3 for 4 in 5)") + @test testflattenorder("(1 for 2 in 3, 4 in 5 for 6 in 7)") + @test testflattenorder("(1 for 2 in 3 for 4 in 5, 6 in 7)") + @test testflattenorder("(1 for 2 in 3 for 4 in 5, 6 in 7 if 8)") +end + +@testitem ":filter" begin + using CSTParser: @cst_str, headof, valof + + x = cst"(a for a in A if a)".args[1].args[2] + @test length(x) == 3 + @test valof(headof(x[1])) == "=" + @test headof(x[2]) === :IF + @test valof(x[3]) == "a" +end + +@testitem ":try" begin + using CSTParser: @cst_str, headof, valof + + x = cst"try expr catch e end" + @test length(x) == 6 + @test headof(x[1]) === :TRY + @test headof(x[2]) === :block + @test headof(x[3]) === :CATCH + @test valof(x[4]) == "e" + @test headof(x[5]) === :block + @test headof(x[6]) === :END + + x = cst"try expr finally expr2 end" + @test length(x) == 8 + @test headof(x[1]) === :TRY + @test headof(x[2]) === :block + @test headof(x[3]) === :CATCH + @test x[3].fullspan == 0 + @test headof(x[4]) === :FALSE + @test headof(x[5]) === :FALSE + @test headof(x[6]) === :FINALLY + @test headof(x[7]) === :block + @test headof(x[8]) === :END + + x = cst"try expr catch err expr2 finally expr3 end" + @test length(x) == 8 + @test headof(x[1]) === :TRY + @test headof(x[2]) === :block + @test headof(x[3]) === :CATCH + @test valof(x[4]) == "err" + @test headof(x[5]) === :block + @test headof(x[6]) === :FINALLY + @test headof(x[7]) === :block + @test headof(x[8]) === :END +end + +@testitem ":comprehension" begin + using CSTParser: @cst_str, headof, valof + + x = cst"[a for a in A]" + @test length(x) == 3 + @test headof(x[1]) === :LSQUARE + @test headof(x[2]) === :generator + @test headof(x[3]) === :RSQUARE +end + +@testitem ":typed_comprehension" begin + using CSTParser: @cst_str, headof, valof + + x = cst"T[a for a in A]" + @test length(x) == 4 + @test headof(x[1]) === :IDENTIFIER + @test headof(x[2]) === :LSQUARE + @test headof(x[3]) === :generator + @test headof(x[4]) === :RSQUARE +end + +@testitem "unary syntax" begin + using CSTParser: @cst_str, headof, valof + + x = cst"<:a" + @test length(x) == 2 + @test headof(x[1]) === :OPERATOR + @test headof(x[2]) === :IDENTIFIER + + x = cst">:a" + @test length(x) == 2 + @test headof(x[1]) === :OPERATOR + @test headof(x[2]) === :IDENTIFIER + + x = cst"::a" + @test length(x) == 2 + @test headof(x[1]) === :OPERATOR + @test headof(x[2]) === :IDENTIFIER + + x = cst"&a" + @test length(x) == 2 + @test headof(x[1]) === :OPERATOR + @test headof(x[2]) === :IDENTIFIER + + x = cst"a..." + @test length(x) == 2 + @test headof(x[1]) === :IDENTIFIER + @test headof(x[2]) === :OPERATOR + + x = cst"$a" + @test length(x) == 2 + @test headof(x[1]) === :OPERATOR + @test headof(x[2]) === :IDENTIFIER +end diff --git a/test/iterate/test_selftest.jl b/test/iterate/test_selftest.jl new file mode 100644 index 00000000..7cead93e --- /dev/null +++ b/test/iterate/test_selftest.jl @@ -0,0 +1,12 @@ +@testitem "self test" begin + include("../shared.jl") + + for (root, dirs, files) in walkdir(joinpath(@__DIR__, "..", "..")) + for file in files + if endswith(file, ".jl") + test_iter_spans(CSTParser.parse(read(joinpath(root, file), String), true)) + end + end + end + +end diff --git a/test/parser.jl b/test/parser.jl deleted file mode 100644 index 2892b67c..00000000 --- a/test/parser.jl +++ /dev/null @@ -1,1575 +0,0 @@ -using Test - -randop() = rand(["-->", "→", - "||", - "&&", - "<", "==", "<:", ">:", - "<|", "|>", - ":", - "+", "-", - ">>", "<<", - "*", "/", - "//", - "^", "↑", - "::", - ".", "->"]) - -test_expr_broken(str) = test_expr(str, false) - -function traverse(x) - try - for a in x - @test traverse(a) - end - return true - catch err - @error "EXPR traversal failed." expr = x exception = err - return false - end -end - -function test_expr(str, show_data=true) - x, ps = CSTParser.parse(ParseState(str)) - - x0 = to_codeobject(x) - x1 = remlineinfo!(Meta.parse(str)) - - @test x.args === nothing || all(x === parentof(a) for a in x.args) - @test x.trivia === nothing || all(x === parentof(a) for a in x.trivia) - @test isempty(check_span(x)) - check_parents(x) - @test traverse(x) - @test x.fullspan == sizeof(str) - - if CSTParser.has_error(ps) || x0 != x1 - if show_data - println("Mismatch between flisp and CSTParser when parsing string $str") - println("ParserState:\n $ps\n") - println("CSTParser Expr:\n $x\n") - println("Converted CSTParser Expr:\n $x0\n") - println("Base EXPR:\n $x1\n") - end - return false - end - return true -end - -@testset "All tests" begin - @test Meta.parse("(1,)") == Expr(:tuple, 1) - @testset "Operators" begin - # @testset "Binary Operators" begin - # for iter = 1:25 - # println(iter) - # str = join([["x$(randop())" for i = 1:19];"x"]) - - # @test test_expr(str) - # end - # end - @testset "Conditional Operator" begin - @test test_expr("a ? b : c") - @test test_expr("a ? b : c : d") - @test test_expr("a ? b : c : d : e") - @test test_expr("a ? b : c : d : e") - end - - - @testset "Dot Operator" begin - @test "a.b" |> test_expr - @test "a.b.c" |> test_expr - @test "(a(b)).c" |> test_expr - @test "(a).(b).(c)" |> test_expr - @test "(a).b.(c)" |> test_expr - @test "(a).b.(c+d)" |> test_expr - end - - @testset "Unary Operator" begin - @test "+" |> test_expr - @test "-" |> test_expr - @test "!" |> test_expr - @test "~" |> test_expr - @test "&" |> test_expr - # @test "::" |> test_expr - @test "<:" |> test_expr - @test ">:" |> test_expr - @test "¬" |> test_expr - @test "√" |> test_expr - @test "∛" |> test_expr - @test "∜" |> test_expr - end - - @testset "Unary Operator" begin - @test "a=b..." |> test_expr - @test "a-->b..." |> test_expr - if VERSION >= v"1.6" - @test "a<--b..." |> test_expr - @test "a<-->b..." |> test_expr - end - @test "a&&b..." |> test_expr - @test "a||b..." |> test_expr - @test "a test_expr - @test "a:b..." |> test_expr - @test "a+b..." |> test_expr - @test "a< test_expr - @test "a*b..." |> test_expr - @test "a//b..." |> test_expr - @test "a^b..." |> test_expr - @test "3a^b" |> test_expr - @test "3//a^b" |> test_expr - @test "3^b//a^b" |> test_expr - @test "3^b//a" |> test_expr - @test "a::b..." |> test_expr - @test "a where b..." |> test_expr - @test "a.b..." |> test_expr - end - - @testset "unary op calls" begin - @test "+(a,b)" |> test_expr - @test "-(a,b)" |> test_expr - @test "!(a,b)" |> test_expr - @test "¬(a,b)" |> test_expr - @test "~(a,b)" |> test_expr - if VERSION > v"1.7-" - @test "~(a)(foo...)" |> test_expr - @test "~(&)(foo...)" |> test_expr - end - @test "<:(a,b)" |> test_expr - @test "√(a,b)" |> test_expr - @test "\$(a,b)" |> test_expr - @test ":(a,b)" |> test_expr - @test "&a" |> test_expr - @test "&(a,b)" |> test_expr - @test "::a" |> test_expr - @test "::(a,b)" |> test_expr - end - - @testset "dotted non-calls" begin - @test "f(.+)" |> test_expr - @test "f(.-)" |> test_expr - @test "f(.!)" |> test_expr - @test "f(.¬)" |> test_expr - if VERSION >= v"1.6" - @test_broken "f(.~)" |> test_expr_broken - end - @test "f(.√)" |> test_expr - @test "f(:(.=))" |> test_expr - @test "f(:(.+))" |> test_expr - @test "f(:(.*))" |> test_expr - end - - if VERSION >= v"1.6" - @testset "comment parsing" begin - @test "[1#==#2#==#3]" |> test_expr - @test """ - begin - arraycopy_common(false, LLVM.Builder(B), orig, origops[1], gutils)#=fwd=# - return nothing - end - """ |> test_expr - @test CSTParser.has_error(CSTParser.parse(""" - begin - arraycopy_common(false, LLVM.Builder(B), orig, origops[1], gutils)#=fwd=#return nothing - end - """)) - end - end - - @testset "weird quote parsing" begin - @test ":(;)" |> test_expr - @test ":(;;)" |> test_expr - @test ":(;;;)" |> test_expr - end - - # this errors during *lowering*, not parsing. - @testset "parse const without assignment in quote" begin - @test ":(global const x)" |> test_expr - @test ":(global const x::Int)" |> test_expr - @test ":(const global x)" |> test_expr - @test ":(const global x::Int)" |> test_expr - end - - @testset "where precedence" begin - @test "a = b where c = d" |> test_expr - @test "a = b where c" |> test_expr - @test "b where c = d" |> test_expr - - @test "a ? b where c : d" |> test_expr - if VERSION >= v"1.6" - @test "a --> b where c --> d" |> test_expr - @test "a --> b where c" |> test_expr - @test "b where c --> d" |> test_expr - @test "b where c <-- d" |> test_expr - @test "b where c <--> d" |> test_expr - end - - @test "a || b where c || d" |> test_expr - @test "a || b where c" |> test_expr - @test "b where c || d" |> test_expr - - @test "a && b where c && d" |> test_expr - @test "a && b where c" |> test_expr - @test "b where c && d" |> test_expr - - @test "a <: b where c <: d" |> test_expr - @test "a <: b where c" |> test_expr - @test "b where c <: d" |> test_expr - - @test "a <| b where c <| d" |> test_expr - @test "a <| b where c" |> test_expr - @test "b where c <| d" |> test_expr - - @test "a : b where c : d" |> test_expr - @test "a : b where c" |> test_expr - @test "b where c : d" |> test_expr - - @test "a + b where c + d" |> test_expr - @test "a + b where c" |> test_expr - @test "b where c + d" |> test_expr - - @test "a << b where c << d" |> test_expr - @test "a << b where c" |> test_expr - @test "b where c << d" |> test_expr - - @test "a * b where c * d" |> test_expr - @test "a * b where c" |> test_expr - @test "b where c * d" |> test_expr - - @test "a // b where c // d" |> test_expr - @test "a // b where c" |> test_expr - @test "b where c // d" |> test_expr - - @test "a ^ b where c ^ d" |> test_expr - @test "a ^ b where c" |> test_expr - @test "b where c ^ d" |> test_expr - - @test "a :: b where c :: d" |> test_expr - @test "a :: b where c" |> test_expr - @test "b where c :: d" |> test_expr - - @test "a.b where c.d" |> test_expr - @test "a.b where c" |> test_expr - @test "b where c.d" |> test_expr - - @test "a where b where c" |> test_expr - end - end - - - @testset "Type Annotations" begin - @testset "Curly" begin - @test "x{T}" |> test_expr - @test "x{T,S}" |> test_expr - @test "a.b{T}" |> test_expr - @test "a(b){T}" |> test_expr - @test "(a(b)){T}" |> test_expr - @test "a{b}{T}" |> test_expr - @test "a{b}(c){T}" |> test_expr - @test "a{b}.c{T}" |> test_expr - @test """x{T, - S}""" |> test_expr - end - end - - @testset "Tuples" begin - @static if VERSION > v"1.1-" - @test headof(CSTParser.parse("1,")) === :errortoken - else - @test "1," |> test_expr - end - @test "1,2" |> test_expr - @test "1,2,3" |> test_expr - @test "()" |> test_expr - @test "(==)" |> test_expr - @test "(1)" |> test_expr - @test "(1,)" |> test_expr - @test "(1,2)" |> test_expr - @test "(a,b,c)" |> test_expr - @test "(a...)" |> test_expr - @test "((a,b)...)" |> test_expr - @test "a,b = c,d" |> test_expr - @test "(a,b) = (c,d)" |> test_expr - end - - @testset "Function Calls" begin - @testset "Simple Calls" begin - @test "f(x)" |> test_expr - @test "f(x,y)" |> test_expr - @test "f(g(x))" |> test_expr - @test "f((x,y))" |> test_expr - @test "f((x,y), z)" |> test_expr - @test "f(z, (x,y), z)" |> test_expr - @test "f{a}(x)" |> test_expr - @test "f{a<:T}(x::T)" |> test_expr - end - - @testset "Keyword Arguments" begin - @test "f(x=1)" |> test_expr - @test "f(x=1,y::Int = 1)" |> test_expr - end - - @testset "Compact Declaration" begin - @test "f(x) = x" |> test_expr - @test "f(x) = g(x)" |> test_expr - @test "f(x) = (x)" |> test_expr - @test "f(x) = (x;y)" |> test_expr - @test "f(g(x)) = x" |> test_expr - @test "f(g(x)) = h(x)" |> test_expr - end - - @testset "Standard Declaration" begin - @test "function f end" |> test_expr - @test "function f(x) x end" |> test_expr - @test "function f(x); x; end" |> test_expr - @test "function f(x) x; end" |> test_expr - @test "function f(x); x end" |> test_expr - @test "function f(x) x;y end" |> test_expr - @test """function f(x) x end""" |> test_expr - @test """function f(x,y =1) x end""" |> test_expr - @test """function f(x,y =1;z =2) x end""" |> test_expr - end - @testset "Anonymous" begin - @test "x->y" |> test_expr - @test "(x,y)->x*y" |> test_expr - @test """function () - return - end""" |> test_expr - end - end - - - - - - @testset "Types" begin - @testset "Abstract" begin - @test "abstract type t end" |> test_expr - @test "abstract type t{T} end" |> test_expr - @test "abstract type t <: S end" |> test_expr - @test "abstract type t{T} <: S end" |> test_expr - end - - @testset "primitive" begin - @test "primitive type Int 64 end" |> test_expr - @test "primitive type Int 4*16 end" |> test_expr - end - - @testset "Structs" begin - @test "struct a end" |> test_expr - @test "struct a; end" |> test_expr - @test "struct a; b;end" |> test_expr - @test """struct a - arg1 - end""" |> test_expr - @test """struct a <: T - arg1::Int - arg2::Int - end""" |> test_expr - @test """struct a - arg1::T - end""" |> test_expr - @test """struct a{T} - arg1::T - a(args) = new(args) - end""" |> test_expr - @test """struct a <: Int - arg1::Vector{Int} - end""" |> test_expr - @test """mutable struct a <: Int - arg1::Vector{Int} - end""" |> test_expr - if VERSION > v"1.8-" - @test """mutable struct A - const arg1::Vector{Int} - arg2 - end""" |> test_expr - @test """mutable struct A - const arg1 - arg2 - end""" |> test_expr - @test """struct A - const arg1 - arg2 - end""" |> test_expr - @test """@eval struct A - const arg1 - arg2 - end""" |> test_expr - end - end - end - - - @testset "Modules" begin - @testset "Imports " begin - @test "import ModA" |> test_expr - @test "import .ModA" |> test_expr - @test "import ..ModA.a" |> test_expr - @test "import ModA.subModA" |> test_expr - @test "import ModA.subModA: a" |> test_expr - @test "import ModA.subModA: a, b" |> test_expr - @test "import ModA.subModA: a, b.c" |> test_expr - @test "import .ModA.subModA: a, b.c" |> test_expr - @test "import ..ModA.subModA: a, b.c" |> test_expr - end - @testset "Export " begin - @test "export ModA" |> test_expr - @test "export a, b, c" |> test_expr - end - end - - @testset "Generators" begin - @test "(y for y in X)" |> test_expr - @test "((x,y) for x in X, y in Y)" |> test_expr - @test "(y.x for y in X)" |> test_expr - @test "((y) for y in X)" |> test_expr - @test "(y,x for y in X)" |> test_expr - @test "((y,x) for y in X)" |> test_expr - @test "[y for y in X]" |> test_expr - @test "[(y) for y in X]" |> test_expr - @test "[(y,x) for y in X]" |> test_expr - @test "Int[y for y in X]" |> test_expr - @test "Int[(y) for y in X]" |> test_expr - @test "Int[(y,x) for y in X]" |> test_expr - @test """ - [a - for a = 1:2]""" |> test_expr - @test "[ V[j][i]::T for i=1:length(V[1]), j=1:length(V) ]" |> test_expr - @test "all(d ≥ 0 for d in B.dims)" |> test_expr - @test "(arg for x in X)" |> test_expr - @test "(arg for x in X for y in Y)" |> test_expr - @test "(arg for x in X for y in Y for z in Z)" |> test_expr - @test "(arg for x in X if A)" |> test_expr - @test "(arg for x in X if A for y in Y)" |> test_expr - @test "(arg for x in X if A for y in Y if B)" |> test_expr - @test "(arg for x in X if A for y in Y for z in Z)" |> test_expr - @test "(arg for x in X if A for y in Y if B for z in Z)" |> test_expr - @test "(arg for x in X if A for y in Y if B for z in Z if C)" |> test_expr - @test "(arg for x in X, y in Y for z in Z)" |> test_expr - @test "(arg for x in X, y in Y if A for z in Z)" |> test_expr - end - - @testset "Macros " begin - @test "macro m end" |> test_expr - @test "macro m() end" |> test_expr - @test "macro m() a end" |> test_expr - @test "@mac" |> test_expr - @test "@mac a b c" |> test_expr - @test "@mac f(5)" |> test_expr - @test "(@mac x)" |> test_expr - @test "Mod.@mac a b c" |> test_expr - @test "[@mac a b]" |> test_expr - @test "@inline get_chunks_id(i::Integer) = _div64(Int(i)-1)+1, _mod64(Int(i) -1)" |> test_expr - @test "@inline f() = (), ()" |> test_expr - @test "@sprintf(\"%08d\", id)" |> test_expr - @test "[@m @n a for a in A]" |> test_expr - @test ":(@foo bar baz bat)" |> test_expr - @test ":(@foo bar for i in j end)" |> test_expr - @test "(@foo bar for i in j end)" |> test_expr - @test "foo(@foo bar for i in j)" |> test_expr - @test "foo.(@foo bar for i in j)" |> test_expr - @test CSTParser.parse("@__DIR__\n\nx", true)[1].span == 8 - - if VERSION >= v"1.8.0-" - @test "M43018.@test43018() do; end" |> test_expr - @test "@M43018.test43018() do; end" |> test_expr - end - end - - @testset "Square " begin - @testset "vect" begin - @test "[x]" |> test_expr - @test "[(1,2)]" |> test_expr - @test "[x...]" |> test_expr - @test "[1,2,3,4,5]" |> test_expr - # this is nonsensical, but iteration should still work - @test traverse(CSTParser.parse(raw"""[[:alpha:]a←-]""")) - # unterminated expressions may cause issues - @test traverse(CSTParser.parse("bind_artifact!(\"../Artifacts.toml\",\"example\",hash,download_info=[(\"file://c:/juliaWork/tarballs/example.tar.gz\"\",tarball_hash)],force=true)\n2+2\n")) - @test traverse(CSTParser.parse("[2+3+4")) - @test traverse(CSTParser.parse("[2+3+4+")) - @test traverse(CSTParser.parse("[\"hi\"\"")) - @test traverse(CSTParser.parse("[\"hi\"\"\n")) - @test traverse(CSTParser.parse("[(1,2,3])")) - end - - @testset "ref" begin - @test "t[i]" |> test_expr - @test "t[i, j]" |> test_expr - end - - @testset "vcat" begin - @test "[x;]" |> test_expr - @test "[x;y;z]" |> test_expr - @test """[x - y - z]""" |> test_expr - @test """[x - y;z]""" |> test_expr - @test """[x;y - z]""" |> test_expr - @test "[x,y;z]" |> test_expr - @test "[1,2;]" |> test_expr - end - - @testset "typed_vcat" begin - @test "t[x;]" |> test_expr - @test "t[x;y]" |> test_expr - @test """t[x - y]""" |> test_expr - @test "t[x;y]" |> test_expr - @test "t[x y; z]" |> test_expr - @test "t[x, y; z]" |> test_expr - end - if VERSION > v"1.8-" - @testset "ncat" begin - @test "[;]" |> test_expr - @test "[;;]" |> test_expr - @test "[;;\n]" |> test_expr - @test "[\n ;; \n]" |> test_expr - @test "[;;;;;;;]" |> test_expr - @test "[x;;;;;]" |> test_expr - @test "[x;;]" |> test_expr - @test "[x;; y;; z]" |> test_expr - @test "[x;;; y;;;z]" |> test_expr - @test "[x;;; y;;;z]'" |> test_expr - @test "[1 2; 3 4]" |> test_expr - @test "[1;2;;3;4;;5;6;;;;9]" |> test_expr - if VERSION > v"1.7-" - @test "[let; x; end;; y]" |> test_expr - @test "[let; x; end;;;; y]" |> test_expr - end - end - - @testset "typed_ncat" begin - @test "t[;;]" |> test_expr - @test "t[;;;;;;;]" |> test_expr - @test "t[x;;;;;]" |> test_expr - @test "t[x;;]" |> test_expr - @test "t[x;; y;; z]" |> test_expr - @test "t[x;;; y;;;z]" |> test_expr - @test "t[x;;\ny]" |> test_expr - @test "t[x y;;\nz a]" |> test_expr - @test "t[x y;;\nz a]'" |> test_expr - @test "t[let; x; end;; y]" |> test_expr - @test "t[let; x; end;;;; y]" |> test_expr - end - end - - @testset "hcat" begin - @test "[x y]" |> test_expr - @test "[let; x; end y]" |> test_expr - @test "[let; x; end; y]" |> test_expr - end - - @testset "typed_hcat" begin - @test "t[x y]" |> test_expr - @test "t[let; x; end y]" |> test_expr - @test "t[let; x; end; y]" |> test_expr - end - - @testset "Comprehension" begin - @test "[i for i = 1:10]" |> test_expr - @test "Int[i for i = 1:10]" |> test_expr - @test "[let;x;end for x in x]" |> test_expr - @test "[let; x; end for x in x]" |> test_expr - @test "[let x=x; x+x; end for x in x]" |> test_expr - if VERSION > v"1.7-" - @test """[ - [ - let l = min((d-k),k); - binomial(d-l,l); - end; for k in 1:d-1 - ] for d in 2:9 - ] - """ |> test_expr - end - end - end - - - @testset "Keyword Blocks" begin - @testset "If" begin - @test "if cond end" |> test_expr - @test "if cond; a; end" |> test_expr - @test "if cond a; end" |> test_expr - @test "if cond; a end" |> test_expr - @test """if cond - 1 - 1 - end""" |> test_expr - @test """if cond - else - 2 - 2 - end""" |> test_expr - @test """if cond - 1 - 1 - else - 2 - 2 - end""" |> test_expr - @test "if 1<2 end" |> test_expr - @test """if 1<2 - f(1) - f(2) - end""" |> test_expr - @test """if 1<2 - f(1) - elseif 1<2 - f(2) - end""" |> test_expr - @test """if 1<2 - f(1) - elseif 1<2 - f(2) - else - f(3) - end""" |> test_expr - @test "if cond a end" |> test_expr - end - - - @testset "Try" begin - # @test "try f(1) end" |> test_expr - # @test "try; f(1) end" |> test_expr - # @test "try; f(1); end" |> test_expr - @test "try; f(1); catch e; e; end" |> test_expr - @test "try; f(1); catch e; e end" |> test_expr - @test "try; f(1); catch e e; end" |> test_expr - @test """ - try - f(1) - catch - end - """ |> test_expr - @test """try - f(1) - catch - error(err) - end - """ |> test_expr - @test """ - try - f(1) - catch err - error(err) - end - """ |> test_expr - @test """ - try - f(1) - catch - error(err) - finally - stop(f) - end - """ |> test_expr - @test """ - try - f(1) - catch err - error(err) - finally - stop(f) - end - """ |> test_expr - @test """try - f(1) - finally - stop(f) - end - """ |> test_expr - - if VERSION > v"1.8-" - @test """try - f(1) - catch - x - else - stop(f) - end - """ |> test_expr - @test """try - f(1) - catch - else - stop(f) - end - """ |> test_expr - @test """try - f(1) - catch err - x - else - stop(f) - finally - foo - end - """ |> test_expr - # the most useless try catch ever: - @test """try - catch - else - finally - end - """ |> test_expr - end - end - @testset "For" begin - @test """ - for i = 1:10 - f(i) - end""" |> test_expr - @test """ - for i = 1:10, j = 1:20 - f(i) - end - """ |> test_expr - - @testset "for outer parsing" begin - @test "for outer i in 1:3 end" |> test_expr - @test "for outer i = 1:3 end" |> test_expr - if VERSION >= v"1.6" - @test "for outer \$i = 1:3 end" |> test_expr - @test "for outer \$ i = 1:3 end" |> test_expr - end - end - end - - @testset "Let" begin - @test """ - let x = 1 - f(x) - end - """ |> test_expr - @test """ - let x = 1, y = 2 - f(x) - end - """ |> test_expr - @test """ - let - x - end - """ |> test_expr - end - - @testset "Do" begin - @test """f(X) do x - return x - end""" |> test_expr - @test """f(X,Y) do x,y - return x,y - end""" |> test_expr - @test "f() do x body end" |> test_expr - end - end - - @testset "Triple-quoted string" begin - @test valof(CSTParser.parse("\"\"\" \" \"\"\"")) == " \" " - @test valof(CSTParser.parse("\"\"\"a\"\"\"")) == "a" - @test valof(CSTParser.parse("\"\"\"\"\"\"")) == "" - @test valof(CSTParser.parse("\"\"\"\n\t \ta\n\n\t \tb\"\"\"")) == "a\n\nb" - @test to_codeobject(CSTParser.parse("\"\"\"\ta\n\tb \$c\n\td\n\"\"\"")) == Expr(:string, "\ta\n\tb ", :c, "\n\td\n") - @test to_codeobject(CSTParser.parse("\"\"\"\n\ta\n\tb \$c\n\td\n\"\"\"")) == Expr(:string, "\ta\n\tb ", :c, "\n\td\n") - @test to_codeobject(CSTParser.parse("\"\"\"\n\ta\n\tb \$c\n\td\n\t\"\"\"")) == Expr(:string, "a\nb ", :c, "\nd\n") - @test to_codeobject(CSTParser.parse("\"\"\"\n\t \ta\$(1+\n1)\n\t \tb\"\"\"")) == Expr(:string, "a", :(1 + 1), "\nb") - ws = " " - "\"\"\"\n$ws%rv = atomicrmw \$rmw \$lt* %0, \$lt %1 acq_rel\n$(ws)ret \$lt %rv\n$ws\"\"\"" |> test_expr - ws1 = " " - ws2 = " " - "\"\"\"\n$(ws1)a\n$(ws1)b\n$(ws2)c\n$(ws2)d\n$(ws2)\"\"\"" |> test_expr - "\"\"\"\n$(ws1)a\n\n$(ws1)b\n\n$(ws2)c\n\n$(ws2)d\n\n$(ws2)\"\"\"" |> test_expr - @test "\"\"\"\n$(ws1)α\n$(ws1)β\n$(ws2)γ\n$(ws2)δ\n$(ws2)\"\"\"" |> test_expr - @test "\"\"\"Float\$(bit)\"\"\"" |> test_expr - @test headof(CSTParser.parse("\"\"\"abc\$(de)fg\"\"\"").args[3]) == :STRING - @test headof(CSTParser.parse("\"\"\"abc(de)fg\"\"\"")) == :TRIPLESTRING - @test "\"\"\"\n\t\"\"\"" |> test_expr # Change of behaviour from v1.5 -> v1.6 - end - - @testset "raw strings with unicode" begin - @test raw"""re = r"(\\"[^\\"]*\\") (\d+) bytes α (\\"[^\\"]*\\")\\\\" """ |> test_expr - @test raw"""re = r"(\\"[^\\"]*\\") ⋯ (\d+) bytes ⋯ (\\"[^\\"]*\\")" """ |> test_expr - end - - if VERSION > v"1.7-" - @testset "weird string edge cases" begin - @test """x = raw"a \$(asd) sd\\\n bsf\\\\leq\n\\\\leq" """ |> test_expr - @test """x = "a \$(asd) sd\\\n bsf\\\\leq\n\\\\leq" """ |> test_expr - @test """x = raw\"\"\"a \$(asd) sd\\\n bsf\\\\leq\n\\\\leq\"\"\" """ |> test_expr - @test """x = \"\"\"a \$(asd) sd\\\n bsf\\\\leq\n\\\\leq\"\"\" """ |> test_expr - @test """x = @naah \"\"\"a \$(asd) sd\\\n bsf\\\\leq\n\\\\leq\"\"\" """ |> test_expr - @test """x = Foo.@naah \"\"\"a \$(asd) sd\\\n bsf\\\\leq\n\\\\leq\"\"\" """ |> test_expr - @test """x = Foo.@naah_str \"\"\"a \$(asd) sd\\\n bsf\\\\leq\n\\\\leq\"\"\" """ |> test_expr - @test """x = Foo.naah\"\"\"a \$(asd) sd\\\n bsf\\\\leq\n\\\\leq\"\"\" """ |> test_expr - @test """\"\"\"a \$(asd) sd\\\n bsf\\\\leq\n\\\\leq\"\"\"\nfoo""" |> test_expr - @test """throw(ArgumentError("invalid \$(m == 2 ? "hex (\\\\x)" : - "unicode (\\\$u)") escape sequence"))""" |> test_expr - @test "\"a\\\\\\\\\\\nb\"" |> test_expr - for c in 0:20 - @test test_expr(string("\"a", '\\'^c, "\nb\"")) - @test test_expr(string("\"\"\"a", '\\'^c, "\nb\"\"\"")) - end - for c in 0:20 - @test test_expr(string("`a", '\\'^c, "\nb`")) - @test test_expr(string("```a", '\\'^c, "\nb```")) - end - - @test "\"\"\"\n a\\\n b\"\"\"" |> test_expr - @test "\"\"\"\n a\\\n b\"\"\"" |> test_expr - @test "\"\"\"\na\\\n b\"\"\"" |> test_expr - @test "\"\"\"\na\\\nb\"\"\"" |> test_expr - @test "\"\"\"\n a\\\n b\"\"\"" |> test_expr - end - end - - @testset "No longer broken things" begin - @test "[ V[j][i]::T for i=1:length(V[1]), j=1:length(V) ]" |> test_expr - @test "all(d ≥ 0 for d in B.dims)" |> test_expr - @test ":(=)" |> test_expr - @test ":(1)" |> test_expr - @test ":(a)" |> test_expr - @test "(@_inline_meta(); f(x))" |> test_expr - @test "isa(a,b) != c" |> test_expr - @test "isa(a,a) != isa(a,a)" |> test_expr - @test "@mac return x" |> test_expr - @static if VERSION > v"1.1-" - @test headof(CSTParser.parse("a,b,").trivia[2]) === :errortoken - else - @test "a,b," |> test_expr - end - @test "m!=m" |> test_expr - @test "+(x...)" |> test_expr - @test "+(promote(x,y)...)" |> test_expr - @test "\$(x...)" |> test_expr # - @test "ccall(:gethostname, stdcall, Int32, ())" |> test_expr - @test "@inbounds @ncall a b c" |> test_expr - @test "(a+b)``" |> test_expr - @test "(-, ~)" |> test_expr - @test """function +(x::Bool, y::T)::promote_type(Bool,T) where T<:AbstractFloat - return ifelse(x, oneunit(y) + y, y) - end""" |> test_expr - @test """finalizer(x,x::GClosure->begin - ccall((:g_closure_unref,Gtk.GLib.libgobject),Void,(Ptr{GClosure},),x.handle) - end)""" |> test_expr - @test "function \$A end" |> test_expr - @test "&ctx->exe_ctx_ref" |> test_expr - @test ":(\$(docstr).\$(TEMP_SYM)[\$(key)])" |> test_expr - @test "SpecialFunctions.\$(fsym)(n::Dual)" |> test_expr - @test "(Base.@_pure_meta;)" |> test_expr - @test "@M a b->(@N c = @O d e f->g)" |> test_expr - @test "! = f" |> test_expr - @test "[a=>1, b=>2]" |> test_expr - @test "a.\$(b)" |> test_expr - @test "a.\$f()" |> test_expr - @test "4x/y" |> test_expr - @test """ - ccall(:jl_finalize_th, Void, (Ptr{Void}, Any,), - Core.getptls(), o) - """ |> test_expr - @test """ - A[if n == d - i - else - (indices(A,n) for n = 1:nd) - end...] - """ |> test_expr - @test """ - @spawnat(p, - let m = a - isa(m, Exception) ? m : nothing - end) - """ |> test_expr # - @test "[@spawn f(R, first(c), last(c)) for c in splitrange(length(R), nworkers())]" |> test_expr - @test "M.:(a)" |> test_expr - @test """ - begin - for i in I for j in J - if cond - a - end - end end - end""" |> test_expr - @test "-f.(a.b + c)" |> test_expr - @test ":(import Base: @doc)" |> test_expr - @test "[a for a in A for b in B]" |> test_expr - @test "+(a,b,c...)" |> test_expr - @test """@testset a for t in T - t - end""" |> test_expr - @test "import Base.==" |> test_expr - @test "a`text`" |> test_expr - @test "a``" |> test_expr - @test "a`text`b" |> test_expr - @test "[a; a 0]" |> test_expr - @test "[a, b; c]" |> test_expr - @test "t{a; b} " |> test_expr - @test "a ~ b + c -d" |> test_expr - @test "y[j=1:10,k=3:2:9; isodd(j+k) && k <= 8]" |> test_expr - @test "(8=>32.0, 12=>33.1, 6=>18.2)" |> test_expr - @test "(a,b = c,d)" |> test_expr - @test "[ -1 -2;]" |> test_expr - @test "-2y" |> test_expr # precedence - @test "'''" |> test_expr # tokenize - @test """ - if j+k <= deg +1 - end - """ |> test_expr - @test "function f() ::T end" |> test_expr # ws closer - @test "import Base: +, -, .+, .-" |> test_expr - if VERSION > v"1.6-" - @test "import Base.:+" |> test_expr - @test "import Base.:⋅" |> test_expr - @test "import Base.:sin, Base.:-" |> test_expr - end - @test "[a + + l]" |> test_expr # ws closer - @test "@inbounds C[i,j] = - α[i] * αjc" |> test_expr - @test "@inbounds C[i,j] = - n * p[i] * pj" |> test_expr - @test """ - if ! a - b - end - """ |> test_expr # ws closer - @test "[:-\n:+]" |> test_expr - @test "::a::b" |> test_expr - @test "-[1:nc]" |> test_expr - @test "@assert .!(isna(res[2]))" |> test_expr # v0.6 - @test "-((attr.rise / PANGO_SCALE)pt).value" |> test_expr - @test "!(a = b)" |> test_expr - @test "-(1)a" |> test_expr - @test "!(a)::T" |> test_expr - @test "a::b where T<:S" |> test_expr - @test "+(x::Bool, y::T)::promote_type(Bool,T) where T<:AbstractFloat" |> test_expr - @test "T where V<:(T where T)" |> test_expr - @test "function ^(z::Complex{T}, p::Complex{T})::Complex{T} where T<:AbstractFloat end" |> test_expr - @test "function +(a) where T where S end" |> test_expr - @test "function -(x::Rational{T}) where T<:Signed end" |> test_expr - @test "\$(a)(b)" |> test_expr - @test "if !(a) break end" |> test_expr - @test "module a() end" |> test_expr - if VERSION > v"1.3-" - @test """module var"#43932#" end""" |> test_expr - end - @test "M.r\"str\" " |> test_expr - @test "f(a for a in A if cond)" |> test_expr - @test "\"dimension \$d is not 1 ≤ \$d ≤ \$nd\" " |> test_expr - @test "-(-x)^1" |> test_expr - @test """ - "\\\\\$ch" - """ |> test_expr - @test "µs" |> test_expr # normalize unicode - @test """ - (x, o; p = 1) -> begin - return o, p - end - """ |> test_expr # normalize unicode - @test """ - (x, o...; p...) -> begin - return o, p - end - """ |> test_expr # normalize unicode - @test "function func() where {A where T} x + 1 end" |> test_expr # nested where - @test "(;x)" |> test_expr # issue 39 - @test """ - let f = ((; a = 1, b = 2) -> ()), - m = first(methods(f)) - @test DSE.keywords(f, m) == [:a, :b] - end - """ |> test_expr - @test "-1^a" |> test_expr - @test "function(f, args...; kw...) end" |> test_expr - @test "function(f, args...=1; kw...) end" |> test_expr - @test "2a * b" |> test_expr - @test "(g1090(x::T)::T) where {T} = x+1.0" |> test_expr - @test "(:) = Colon()" |> test_expr - @test "a + in[1]" |> test_expr - @test "function f(ex) +a end" |> test_expr - @test "x`\\\\`" |> test_expr - @test "x\"\\\\\"" |> test_expr - @test "x\"\\\\ \"" |> test_expr - @test "a.{1}" |> test_expr - @test "@~" |> test_expr - @test "\$\$(x)" |> test_expr - @test "\$\$(x)" |> test_expr - @test CSTParser.headof(CSTParser.parse("=")) === :errortoken - @test CSTParser.headof(CSTParser.parse("~")) === :OPERATOR - @test "(1:\n2)" |> test_expr - @test "a[: ]" |> test_expr - @test ".~b" |> test_expr - if VERSION > v"1.1-" - @test "a .~ b" |> test_expr - end - @test "A[a~b]" |> test_expr - @test "[a~b]" |> test_expr - if VERSION >= v"1.6" - @test "[a ~b]" |> test_expr - end - @test "[a ~ b]" |> test_expr - @test "[a~ b]" |> test_expr - @test "1 .< 2 .< 3" |> test_expr - @test "(;)" |> test_expr - if VERSION > v"1.5" - @test "@M{a}-b" |> test_expr - @test "@M{a,b}-b" |> test_expr - end - @test "@M[a]-b" |> test_expr - end - - @testset "interpolation error catching" begin - x = CSTParser.parse("\"a \$ b\"") - @test x.fullspan == 7 - @test CSTParser.headof(x[3]) === :errortoken - x = CSTParser.parse("\"a \$# b\"") - @test x.fullspan == 8 - @test CSTParser.headof(x[3]) === :errortoken - end - if VERSION >= v"1.6" - @testset "string interpolation" begin - @test test_expr(raw""""$("asd")" """) - @test test_expr(raw""""$("asd")a" """) - @test test_expr(raw""""a$("asd")" """) - @test test_expr(raw""""a$("asd")a" """) - @test test_expr(raw"""`$("asd")` """) - @test test_expr(raw"""`$("asd")a` """) - @test test_expr(raw"""`a$("asd")` """) - @test test_expr(raw"""`a$("asd")a` """) - end - @testset "string whitespace handling" begin - @test test_expr("""\"\"\"\n\\t\"\"\" """) - @test test_expr("""\"\"\"\n\\t\n\"\"\" """) - @test test_expr("""\"\"\"\n\\t\\n\"\"\" """) - @test test_expr(raw"""\"\"\"\n\\t\"\"\" """) - @test test_expr(raw"""\"\"\"\n\\t\n\"\"\" """) - @test test_expr(raw"""\"\"\"\n\\t\\n\"\"\" """) - end - end - - @testset "cmd interpolation" begin - @test test_expr("`a \$b c`") - @test test_expr(raw"`a \"\$b $b\" c`") - @test test_expr("`a b c`") - x = CSTParser.parse("`a \$b c`") - @test x.args[1].head == :globalrefcmd - @test x.args[3].head == :string - @test valof(x.args[3].args[1]) == "a " - @test valof(x.args[3].args[2]) == "b" - @test valof(x.args[3].args[3]) == " c" - end - - @testset "multiple ; in kwargs" begin - @test test_expr("f(a; b=1; c=2)") - @test test_expr("f(a; b=1; c=2) = 2") - @test test_expr("f( ; b=1; c=2)") - @test test_expr("f(a; b=1; c=2)") - @test test_expr("f(a; b=1, c=2; d=3)") - @test test_expr("f(a; b=1; c=2, d=3)") - @test test_expr("f(a; b=1; c=2; d=3)") - end - - @testset "Broken things" begin - @test_broken "\$(a) * -\$(b)" |> test_expr_broken - end - -# test_fsig_decl(str) = (x->x.id).(CSTParser._get_fsig(CSTParser.parse(str)).defs) -# @testset "func-sig variable declarations" begin -# @test test_fsig_decl("f(x) = x") == [:x] -# @test test_fsig_decl("""function f(x) -# x -# end""") == [:x] - -# @test test_fsig_decl("f{T}(x::T) = x") == [:T, :x] -# @test test_fsig_decl("""function f{T}(x::T) -# x -# end""") == [:T, :x] - -# @test test_fsig_decl("f(x::T) where T = x") == [:T, :x] -# @test test_fsig_decl("""function f(x::T) where T -# x -# end""") == [:T, :x] - - -# @test test_fsig_decl("f(x::T{S}) where T where S = x") == [:T, :S, :x] -# @test test_fsig_decl("""function f(x::T{S}) where T where S -# x -# end""") == [:T, :S, :x] -# end - - @testset "Spans" begin - CSTParser.parse(raw""" - "ABC$(T)" - """).fullspan >= 9 - CSTParser.parse("\"_\"").fullspan == 3 - CSTParser.parse("T.mutable && print(\"Ok\")").fullspan == 24 - CSTParser.parse("(\"\$T\")").fullspan == 6 - CSTParser.parse("\"\"\"\$T is not supported\"\"\"").fullspan == 25 - CSTParser.parse("using Compat: @compat\n").fullspan == 22 - CSTParser.parse("primitive = 1").fullspan == 13 - end - - @testset "Command or string with unicode" begin - @test "```αhelloworldω```" |> test_expr - @test "\"αhelloworldω\"" |> test_expr - end - - @testset "conversion of floats with underscore" begin - @test "30.424_876_125_859_513" |> test_expr - end - - @testset "errors" begin - @test headof(CSTParser.parse("1? b : c ").args[1]) === :errortoken - @test headof(CSTParser.parse("1 ?b : c ").trivia[1]) === :errortoken - @test headof(CSTParser.parse("1 ? b :c ").trivia[2]) === :errortoken - @test headof(CSTParser.parse("1:\n2").args[1]) === :errortoken - @test headof(CSTParser.parse("1.a").args[2]) === :errortoken - @test headof(CSTParser.parse("f ()")) === :errortoken - @test headof(CSTParser.parse("f{t} ()")) === :errortoken - @test headof(CSTParser.parse(": a").trivia[1]) === :errortoken - @test headof(CSTParser.parse("const a").args[1]) === :errortoken - end - - @testset "colons" begin - @test test_expr("(if true I elseif false J else : end for i in 1:5)") - @test test_expr("if true; : end") - @test test_expr("a + :") - end - - @testset "tuple params" begin - @test "1,2,3" |> test_expr - @test "1;2,3" |> test_expr - @test "1,2;3" |> test_expr - @test "(1,2,3)" |> test_expr - @test "(1;2,3)" |> test_expr - @test "(1,2;3)" |> test_expr - @test "f(;)" |> test_expr - end - - @testset "docs" begin - @test "\"doc\"\nT" |> test_expr - @test "@doc \"doc\" T" |> test_expr - @test "@doc \"doc\"\nT" |> test_expr - @test "@doc \"doc\n\n\n\"\nT" |> test_expr - @test "begin\n@doc \"doc\"\n\nT\nend" |> test_expr - @test "begin\n@doc \"doc\"\nT\nend" |> test_expr - @test "begin\n@doc \"doc\" T\nend" |> test_expr - @test "if true\n@doc \"doc\"\n\nT\nend" |> test_expr - @test "if true\n@doc \"doc\"\nT\nend" |> test_expr - @test "if true\n@doc \"doc\" T\nend" |> test_expr - @test "@doc \"I am a module\" ModuleMacroDoc" |> test_expr - @test """ - @doc(foo) - """ |> test_expr - @test """ - @doc foo - """ |> test_expr - end - - @testset "braces" begin - @test "{a}" |> test_expr - @test "{a, b}" |> test_expr - @test "{a, b; c}" |> test_expr - @test "{a, b; c = 1}" |> test_expr - @test "{a b}" |> test_expr - @test "{a b; c}" |> test_expr - @test "{a b; c = 1}" |> test_expr - end - - @testset "import preceding dot whitespace" begin - @test "using . M" |> test_expr - @test "using .. M" |> test_expr - @test "using ... M" |> test_expr - end - - @testset "issue #116" begin - @test """ - function foo() where {A <:B} - body - end""" |> test_expr - - @test """ - function foo() where {A <: B} - body - end""" |> test_expr - end - - @testset "issue #165" begin - x = CSTParser.parse(""" - a ? b - function f end""") - @test length(x) == 5 # make sure we always give out an EXPR of the right length - @test headof(x.args[3]) === :errortoken - end - - @testset "issue #182" begin - x = CSTParser.parse(""" - quote - \"\"\" - txt - \"\"\" - sym - end""") - @test x.args[1].args[1].args[1].head === :globalrefdoc - end - if VERSION > v"1.3.0-" - @testset "issue #198" begin - @test test_expr(":var\"id\"") - end - end - @testset "vscode issue #1632" begin - @test test_expr("\"\$( a)\"") - @test test_expr("\"\$(#=comment=# a)\"") - end - @testset "issue #210" begin - @test test_expr("function f(a; where = false) end") - end - - @testset "suffixed ops" begin - @test test_expr("a +₊ b *₊ c") - @test test_expr("a *₊ b +₊ c") - end - @static if VERSION > v"1.6-" - @testset "import .. as .. syntax" begin - @test test_expr("import a as b") - @test test_expr("import a as b, c") - @test test_expr("import M: a as b") - @test test_expr("import M: a as b, c") - @test CSTParser.parse("using a as b")[2].head === :errortoken - @test test_expr("using M: a as b") - @test test_expr("using M: a as b, c") - end - end - @testset "exor #201" begin - @test test_expr(raw"$return(x)") - end - if VERSION > v"1.3.0-" - @testset "@var #236" begin - s = raw"""@var" " a""" - @test test_expr(s) - @test CSTParser.ismacroname(CSTParser.parse(s).args[1]) - end - - @testset "nonstandard identifier (var\"blah\") parsing" begin - @test """var"asd" """ |> test_expr - @test """var"#asd" """ |> test_expr - @test """var"#asd#" """ |> test_expr - @test """M.var"asd" """ |> test_expr - @test """M.var"#asd" """ |> test_expr - @test """M.var"#asd#" """ |> test_expr - end - end - @testset "bad uint" begin - @test to_codeobject(CSTParser.parse("0x.")) == Expr(:error) - end - - @testset "endswithtrivia" begin - x = CSTParser.parse("\"some long title \$label1 \$label2\" \na") - @test x[3].span < x[3].fullspan - @test CSTParser.lastchildistrivia(x[3]) - end - - @testset "bad interp with following newline" begin - s = "\"\"\"\$()\n\"\"\"" - x = CSTParser.parse(s) - @test sizeof(s) == x.fullspan - end - - @testset "minimal_reparse" begin - s0 = """ - testsettravx=nothing; - ) fx;ifxend) - # parsing works?""" - s1 = """ - testsettravx=nothing; - ) ;ifxend) - # parsing works?""" - x0 = CSTParser.parse(s0, true) - x1 = CSTParser.parse(s1, true) - x2 = CSTParser.minimal_reparse(s0, s1, x0, x1) - @test CSTParser.comp(x1, x2) - end - - @testset "primes" begin - @test test_expr(""" - f() do x - end' - """) - @test CSTParser.has_error(cst"begin end'") - @test !CSTParser.has_error(cst"[]'") - @test !CSTParser.has_error(cst"'a''") - @test test_expr("(a)'") - @test test_expr("a.a'") - if VERSION >= v"1.6" - @test test_expr("a'ᵀ") - @test test_expr(":(a'ᵀ)") - end - @test test_expr("a'") - @test test_expr("a''") - @test test_expr("a'''") - # @test test_expr(":.'") - # @test test_expr(":?'") - # @test test_expr(":a'") - end - - @testset "end as id juxt" begin - @test test_expr("a[1end]") - if VERSION >= v"1.4" - @test test_expr("a[2begin:1end]") - end - end - - @testset "last child is trivia for :string" begin - @test !CSTParser.lastchildistrivia(cst"""("a $(A) a" )"""[2]) - end - - @testset "toplevel strings" begin - @test test_expr(""""a" in b && c""") - end - - @testset "@doc cont" begin - @test test_expr("module a\n@doc doc\"\"\"doc\"\"\"\nx\nend") - end - - @testset "char escape" begin - @test test_expr(raw"'\$'") - @test test_expr(raw"'\a'") - @test test_expr(raw"'\3'") - @test test_expr(raw"'\000'") - @test test_expr(raw"'\033'") - @test test_expr(raw"'\177'") - @test test_expr(raw"'\u222'") - @test test_expr(raw"'\ufff'") - @test test_expr(raw"'\x2'") - @test test_expr(raw"'\x22'") - @test test_expr(raw"'\u22'") - @test test_expr(raw"'\u2222'") - @test test_expr(raw"'\U2222'") - @test test_expr(raw"'\U22222'") - @test test_expr(raw"'\U00000001'") - - @test CSTParser.parse(raw"'\200'").head == :errortoken - @test CSTParser.parse(raw"'\300'").head == :errortoken - @test CSTParser.parse(raw"'\377'").head == :errortoken - @test CSTParser.parse(raw"'\600'").head == :errortoken - @test CSTParser.parse(raw"'\777'").head == :errortoken - @test CSTParser.parse(raw"'\x222'").head == :errortoken - @test CSTParser.parse(raw"'\u22222'").head == :errortoken - @test CSTParser.parse(raw"'\U222222'").head == :errortoken - @test CSTParser.parse(raw"'\asdd'").head == :errortoken - @test CSTParser.parse(raw"'\α'").head == :errortoken - @test CSTParser.parse(raw"'\αsdd'").head == :errortoken - @test CSTParser.parse(raw"'\u222ää'").head == :errortoken - @test CSTParser.parse(raw"'\x222ää'").head == :errortoken - @test CSTParser.parse(raw"'\U222ää'").head == :errortoken - @test CSTParser.parse(raw"'\U10000001'").head == :errortoken - for c in rand(Char, 1000) - c == '\\' && continue - @test test_expr(string("'", c, "'")) - end - end - - @testset "invalid char in string" begin - @test CSTParser.parse(raw"\"\U222222222\"").head == :errortoken - end - - @testset "string macros" begin - @test test_expr(raw"""test"asd"asd""") - if VERSION >= v"1.6" - @test test_expr(raw"""test"asd"0""") - @test test_expr(raw"""test"asd"0o0""") - @test test_expr(raw"""test"asd"0x0""") - @test test_expr(raw"""test"asd"0.0""") - end - @test test_expr(raw"""test"asd"true""") - @test test_expr(raw"""test""true""") - end - - @testset "number parsing" begin - @test test_expr("0b00000000") - @test test_expr("0b000000000") - @test test_expr("0b0000000000000000") - @test test_expr("0b00000000000000000") - @test test_expr("0b00000000000000000000000000000000") - @test test_expr("0b000000000000000000000000000000000") - @test test_expr("0b0000000000000000000000000000000000000000000000000000000000000000") - @test test_expr("0b00000000000000000000000000000000000000000000000000000000000000000") - @test test_expr("0b00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000") - if VERSION >= v"1.6" - @test test_expr("0b000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000") - end - @test test_expr("0b11111111") - @test test_expr("0b111111111") - @test test_expr("0b1111111111111111") - @test test_expr("0b11111111111111111") - @test test_expr("0b11111111111111111111111111111111") - @test test_expr("0b111111111111111111111111111111111") - @test test_expr("0b1111111111111111111111111111111111111111111111111111111111111111") - @test test_expr("0b11111111111111111111111111111111111111111111111111111111111111111") - @test test_expr("0b11111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111") - if VERSION >= v"1.6" - @test test_expr("0b111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111") - end - - @test test_expr("0o0") - @test test_expr("0o00") - @test test_expr("0o0000") - @test test_expr("0o00000") - @test test_expr("0o000000000") - @test test_expr("0o0000000000") - @test test_expr("0o00000000000000000000") - @test test_expr("0o000000000000000000000") - @test test_expr("0o00000000000000000000000000000000000000000") - @test test_expr("0o0000000000000000000000000000000000000000000") - @test test_expr("0o1") - @test test_expr("0o11") - @test test_expr("0o1111") - @test test_expr("0o11111") - @test test_expr("0o111111111") - @test test_expr("0o1111111111") - @test test_expr("0o11111111111111111111") - @test test_expr("0o111111111111111111111") - @test test_expr("0o11111111111111111111111111111111111111111") - @test test_expr("0o111111111111111111111111111111111111111111") - @test test_expr("0o377777777777777777777777777777777777777777") - @test test_expr("0o1111111111111111111111111111111111111111111") - @test test_expr("0o077") - @test test_expr("0o377") - @test test_expr("0o400") - @test test_expr("0o077777") - @test test_expr("0o177777") - @test test_expr("0o200000") - @test test_expr("0o00000000000") - @test test_expr("0o17777777777") - @test test_expr("0o40000000000") - @test test_expr("0o0000000000000000000000") - @test test_expr("0o1000000000000000000000") - @test test_expr("0o2000000000000000000000") - @test test_expr("0o0000000000000000000000000000000000000000000") - @test test_expr("0o1000000000000000000000000000000000000000000") - @test test_expr("0o2000000000000000000000000000000000000000000") - - @test test_expr("0x00") - @test test_expr("0x000") - @test test_expr("0x0000") - @test test_expr("0x00000") - @test test_expr("0x00000000") - @test test_expr("0x000000000") - @test test_expr("0x0000000000000000") - @test test_expr("0x00000000000000000") - @test test_expr("0x00000000000000000000000000000000") - if VERSION >= v"1.6" - @test test_expr("0x000000000000000000000000000000000") - end - - @test test_expr("0x11") - @test test_expr("0x111") - @test test_expr("0x1111") - @test test_expr("0x11111") - @test test_expr("0x11111111") - @test test_expr("0x111111111") - @test test_expr("0x1111111111111111") - @test test_expr("0x11111111111111111") - @test test_expr("0x11111111111111111111111111111111") - if VERSION >= v"1.6" - @test test_expr("0x111111111111111111111111111111111") - end - end - - @testset "#302" begin - str = """ - const _examples = PlotExample[ - PlotExample( # 1 - "Lines", - "A simple line plot of the columns.", - [:( - begin - plot(Plots.fakedata(50, 5), w = 3) - end - )], - ), - ] - """ - @test test_expr(str) - x, ps = CSTParser.parse(CSTParser.ParseState(str), true) - @test ps.errored == false - end - - @testset "#304" begin - str = """ - const _examples = PlotExample[ - PlotExample( # 40 - "Lens", - "A lens lets you easily magnify a region of a plot. x and y coordinates refer to the to be magnified region and the via the `inset` keyword the subplot index and the bounding box (in relative coordinates) of the inset plot with the magnified plot can be specified. Additional attributes count for the inset plot.", - [ - quote - begin - plot( - [(0, 0), (0, 0.9), (1, 0.9), (2, 1), (3, 0.9), (80, 0)], - legend = :outertopright, - ) - plot!([(0, 0), (0, 0.9), (2, 0.9), (3, 1), (4, 0.9), (80, 0)]) - plot!([(0, 0), (0, 0.9), (3, 0.9), (4, 1), (5, 0.9), (80, 0)]) - plot!([(0, 0), (0, 0.9), (4, 0.9), (5, 1), (6, 0.9), (80, 0)]) - lens!( - [1, 6], - [0.9, 1.1], - inset = (1, bbox(0.5, 0.0, 0.4, 0.4)), - ) - end - end, - ], - ), - ] - """ - @test test_expr(str) - x, ps = CSTParser.parse(CSTParser.ParseState(str), true) - @test ps.errored == false - end - - @testset "#310" begin - x, ps = CSTParser.parse(CSTParser.ParseState("""import a.notvar"papa" """), true) - @test ps.errored == true - x, ps = CSTParser.parse(CSTParser.ParseState("""import notvar"papa" """), true) - @test ps.errored == true - end - - @testset "#311" begin - @test test_expr(raw"import a.$b.c") - end - - @testset "kw interpolation" begin - @test test_expr(raw""""foo $bar" """) - @test test_expr(raw""""foo $type" """) - @test test_expr(raw""""foo $function" """) - @test test_expr(raw""""foo $begin" """) - @test test_expr(raw""""foo $quote" """) - end - - if VERSION > v"1.7-" - @testset "broadcasted && and ||" begin - @test test_expr(raw"""a .&& b""") - @test test_expr(raw"""a .< b .&& b .> a""") - @test test_expr(raw"""a .|| b""") - @test test_expr(raw"""a .< b .|| b .> a""") - end - end - - if VERSION > v"1.7-" - @testset "normalized unicode ops" begin - @test "(·) == (·) == (⋅) == 5" |> test_expr - @test "(−) == (-) == 6" |> test_expr - end - end - - @testset "pair tuple" begin - @test test_expr("a => b") - @test test_expr("a => b, c, d") - @test test_expr("a, a => b, c, d") - end - - @testset "global" begin - @test test_expr("global a") - @test test_expr("global a = 1") - @test test_expr("global a = 1, b") - @test test_expr("global a, b") - @test test_expr("global a, b = 2") - end -end diff --git a/test/parser/test_function_calls.jl b/test/parser/test_function_calls.jl new file mode 100644 index 00000000..f906632a --- /dev/null +++ b/test/parser/test_function_calls.jl @@ -0,0 +1,58 @@ +@testitem "Simple Calls" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test "f(x)" |> test_expr + @test "f(x,y)" |> test_expr + @test "f(g(x))" |> test_expr + @test "f((x,y))" |> test_expr + @test "f((x,y), z)" |> test_expr + @test "f(z, (x,y), z)" |> test_expr + @test "f{a}(x)" |> test_expr + @test "f{a<:T}(x::T)" |> test_expr +end + +@testitem "Keyword Arguments" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test "f(x=1)" |> test_expr + @test "f(x=1,y::Int = 1)" |> test_expr +end + +@testitem "Compact Declaration" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test "f(x) = x" |> test_expr + @test "f(x) = g(x)" |> test_expr + @test "f(x) = (x)" |> test_expr + @test "f(x) = (x;y)" |> test_expr + @test "f(g(x)) = x" |> test_expr + @test "f(g(x)) = h(x)" |> test_expr +end + +@testitem "Standard Declaration" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test "function f end" |> test_expr + @test "function f(x) x end" |> test_expr + @test "function f(x); x; end" |> test_expr + @test "function f(x) x; end" |> test_expr + @test "function f(x); x end" |> test_expr + @test "function f(x) x;y end" |> test_expr + @test """function f(x) x end""" |> test_expr + @test """function f(x,y =1) x end""" |> test_expr + @test """function f(x,y =1;z =2) x end""" |> test_expr +end +@testitem "Anonymous" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test "x->y" |> test_expr + @test "(x,y)->x*y" |> test_expr + @test """function () + return +end""" |> test_expr +end \ No newline at end of file diff --git a/test/parser/test_keyword_blocks.jl b/test/parser/test_keyword_blocks.jl new file mode 100644 index 00000000..6ba58f3b --- /dev/null +++ b/test/parser/test_keyword_blocks.jl @@ -0,0 +1,191 @@ +@testitem "If" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test "if cond end" |> test_expr + @test "if cond; a; end" |> test_expr + @test "if cond a; end" |> test_expr + @test "if cond; a end" |> test_expr + @test """if cond + 1 + 1 +end""" |> test_expr + @test """if cond +else + 2 + 2 +end""" |> test_expr + @test """if cond + 1 + 1 +else + 2 + 2 +end""" |> test_expr + @test "if 1<2 end" |> test_expr + @test """if 1<2 + f(1) + f(2) +end""" |> test_expr + @test """if 1<2 + f(1) +elseif 1<2 + f(2) +end""" |> test_expr + @test """if 1<2 + f(1) +elseif 1<2 + f(2) +else + f(3) +end""" |> test_expr + @test "if cond a end" |> test_expr +end + + +@testitem "Try" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + # @test "try f(1) end" |> test_expr + # @test "try; f(1) end" |> test_expr + # @test "try; f(1); end" |> test_expr + @test "try; f(1); catch e; e; end" |> test_expr + @test "try; f(1); catch e; e end" |> test_expr + @test "try; f(1); catch e e; end" |> test_expr + @test """ + try + f(1) + catch + end + """ |> test_expr + @test """try + f(1) + catch + error(err) + end + """ |> test_expr + @test """ + try + f(1) + catch err + error(err) + end + """ |> test_expr + @test """ + try + f(1) + catch + error(err) + finally + stop(f) + end + """ |> test_expr + @test """ + try + f(1) + catch err + error(err) + finally + stop(f) + end + """ |> test_expr + @test """try + f(1) + finally + stop(f) + end + """ |> test_expr + + if VERSION > v"1.8-" + @test """try + f(1) + catch + x + else + stop(f) + end + """ |> test_expr + @test """try + f(1) + catch + else + stop(f) + end + """ |> test_expr + @test """try + f(1) + catch err + x + else + stop(f) + finally + foo + end + """ |> test_expr + # the most useless try catch ever: + @test """try + catch + else + finally + end + """ |> test_expr + end +end +@testitem "For" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test """ + for i = 1:10 + f(i) + end""" |> test_expr + @test """ + for i = 1:10, j = 1:20 + f(i) + end + """ |> test_expr + + @testset "for outer parsing" begin + @test "for outer i in 1:3 end" |> test_expr + @test "for outer i = 1:3 end" |> test_expr + if VERSION >= v"1.6" + @test "for outer \$i = 1:3 end" |> test_expr + @test "for outer \$ i = 1:3 end" |> test_expr + end + end +end + +@testitem "Let" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test """ + let x = 1 + f(x) + end + """ |> test_expr + @test """ + let x = 1, y = 2 + f(x) + end + """ |> test_expr + @test """ + let + x + end + """ |> test_expr +end + +@testitem "Do" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test """f(X) do x + return x +end""" |> test_expr + @test """f(X,Y) do x,y + return x,y +end""" |> test_expr + @test "f() do x body end" |> test_expr +end diff --git a/test/parser/test_modules.jl b/test/parser/test_modules.jl new file mode 100644 index 00000000..6d942bbc --- /dev/null +++ b/test/parser/test_modules.jl @@ -0,0 +1,22 @@ +@testitem "Imports " begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test "import ModA" |> test_expr + @test "import .ModA" |> test_expr + @test "import ..ModA.a" |> test_expr + @test "import ModA.subModA" |> test_expr + @test "import ModA.subModA: a" |> test_expr + @test "import ModA.subModA: a, b" |> test_expr + @test "import ModA.subModA: a, b.c" |> test_expr + @test "import .ModA.subModA: a, b.c" |> test_expr + @test "import ..ModA.subModA: a, b.c" |> test_expr +end + +@testitem "Export " begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test "export ModA" |> test_expr + @test "export a, b, c" |> test_expr +end diff --git a/test/parser/test_operators.jl b/test/parser/test_operators.jl new file mode 100644 index 00000000..4b091361 --- /dev/null +++ b/test/parser/test_operators.jl @@ -0,0 +1,224 @@ +# @testitem "Binary Operators" begin + # using CSTParser: remlineinfo! + # include("../shared.jl") + +# for iter = 1:25 +# println(iter) +# str = join([["x$(randop())" for i = 1:19];"x"]) + +# @test test_expr(str) +# end +# end + +@testitem "Conditional Operator" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test test_expr("a ? b : c") + @test test_expr("a ? b : c : d") + @test test_expr("a ? b : c : d : e") + @test test_expr("a ? b : c : d : e") +end + + +@testitem "Dot Operator" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test "a.b" |> test_expr + @test "a.b.c" |> test_expr + @test "(a(b)).c" |> test_expr + @test "(a).(b).(c)" |> test_expr + @test "(a).b.(c)" |> test_expr + @test "(a).b.(c+d)" |> test_expr +end + +@testitem "Unary Operator" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test "+" |> test_expr + @test "-" |> test_expr + @test "!" |> test_expr + @test "~" |> test_expr + @test "&" |> test_expr + # @test "::" |> test_expr + @test "<:" |> test_expr + @test ">:" |> test_expr + @test "¬" |> test_expr + @test "√" |> test_expr + @test "∛" |> test_expr + @test "∜" |> test_expr + + @test "a=b..." |> test_expr + @test "a-->b..." |> test_expr + if VERSION >= v"1.6" + @test "a<--b..." |> test_expr + @test "a<-->b..." |> test_expr + end + @test "a&&b..." |> test_expr + @test "a||b..." |> test_expr + @test "a test_expr + @test "a:b..." |> test_expr + @test "a+b..." |> test_expr + @test "a< test_expr + @test "a*b..." |> test_expr + @test "a//b..." |> test_expr + @test "a^b..." |> test_expr + @test "3a^b" |> test_expr + @test "3//a^b" |> test_expr + @test "3^b//a^b" |> test_expr + @test "3^b//a" |> test_expr + @test "a::b..." |> test_expr + @test "a where b..." |> test_expr + @test "a.b..." |> test_expr +end + +@testitem "unary op calls" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test "+(a,b)" |> test_expr + @test "-(a,b)" |> test_expr + @test "!(a,b)" |> test_expr + @test "¬(a,b)" |> test_expr + @test "~(a,b)" |> test_expr + if VERSION > v"1.7-" + @test "~(a)(foo...)" |> test_expr + @test "~(&)(foo...)" |> test_expr + end + @test "<:(a,b)" |> test_expr + @test "√(a,b)" |> test_expr + @test "\$(a,b)" |> test_expr + @test ":(a,b)" |> test_expr + @test "&a" |> test_expr + @test "&(a,b)" |> test_expr + @test "::a" |> test_expr + @test "::(a,b)" |> test_expr +end + +@testitem "dotted non-calls" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test "f(.+)" |> test_expr + @test "f(.-)" |> test_expr + @test "f(.!)" |> test_expr + @test "f(.¬)" |> test_expr + if VERSION >= v"1.6" + @test_broken "f(.~)" |> test_expr_broken + end + @test "f(.√)" |> test_expr + @test "f(:(.=))" |> test_expr + @test "f(:(.+))" |> test_expr + @test "f(:(.*))" |> test_expr +end + + +@testitem "comment parsing" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + if VERSION >= v"1.6" + @test "[1#==#2#==#3]" |> test_expr + @test """ + begin + arraycopy_common(false, LLVM.Builder(B), orig, origops[1], gutils)#=fwd=# + return nothing + end + """ |> test_expr + @test CSTParser.has_error(CSTParser.parse(""" + begin + arraycopy_common(false, LLVM.Builder(B), orig, origops[1], gutils)#=fwd=#return nothing + end + """)) + end +end + +@testitem "weird quote parsing" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test ":(;)" |> test_expr + @test ":(;;)" |> test_expr + @test ":(;;;)" |> test_expr +end + +# this errors during *lowering*, not parsing. +@testitem "parse const without assignment in quote" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test ":(global const x)" |> test_expr + @test ":(global const x::Int)" |> test_expr + @test ":(const global x)" |> test_expr + @test ":(const global x::Int)" |> test_expr +end + +@testitem "where precedence" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test "a = b where c = d" |> test_expr + @test "a = b where c" |> test_expr + @test "b where c = d" |> test_expr + + @test "a ? b where c : d" |> test_expr + if VERSION >= v"1.6" + @test "a --> b where c --> d" |> test_expr + @test "a --> b where c" |> test_expr + @test "b where c --> d" |> test_expr + @test "b where c <-- d" |> test_expr + @test "b where c <--> d" |> test_expr + end + + @test "a || b where c || d" |> test_expr + @test "a || b where c" |> test_expr + @test "b where c || d" |> test_expr + + @test "a && b where c && d" |> test_expr + @test "a && b where c" |> test_expr + @test "b where c && d" |> test_expr + + @test "a <: b where c <: d" |> test_expr + @test "a <: b where c" |> test_expr + @test "b where c <: d" |> test_expr + + @test "a <| b where c <| d" |> test_expr + @test "a <| b where c" |> test_expr + @test "b where c <| d" |> test_expr + + @test "a : b where c : d" |> test_expr + @test "a : b where c" |> test_expr + @test "b where c : d" |> test_expr + + @test "a + b where c + d" |> test_expr + @test "a + b where c" |> test_expr + @test "b where c + d" |> test_expr + + @test "a << b where c << d" |> test_expr + @test "a << b where c" |> test_expr + @test "b where c << d" |> test_expr + + @test "a * b where c * d" |> test_expr + @test "a * b where c" |> test_expr + @test "b where c * d" |> test_expr + + @test "a // b where c // d" |> test_expr + @test "a // b where c" |> test_expr + @test "b where c // d" |> test_expr + + @test "a ^ b where c ^ d" |> test_expr + @test "a ^ b where c" |> test_expr + @test "b where c ^ d" |> test_expr + + @test "a :: b where c :: d" |> test_expr + @test "a :: b where c" |> test_expr + @test "b where c :: d" |> test_expr + + @test "a.b where c.d" |> test_expr + @test "a.b where c" |> test_expr + @test "b where c.d" |> test_expr + + @test "a where b where c" |> test_expr +end diff --git a/test/parser/test_parser.jl b/test/parser/test_parser.jl new file mode 100644 index 00000000..716fd61a --- /dev/null +++ b/test/parser/test_parser.jl @@ -0,0 +1,1088 @@ +@testitem "Parser" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test Meta.parse("(1,)") == Expr(:tuple, 1) +end + +@testitem "Type Annotations Curly" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test "x{T}" |> test_expr + @test "x{T,S}" |> test_expr + @test "a.b{T}" |> test_expr + @test "a(b){T}" |> test_expr + @test "(a(b)){T}" |> test_expr + @test "a{b}{T}" |> test_expr + @test "a{b}(c){T}" |> test_expr + @test "a{b}.c{T}" |> test_expr + @test """x{T, +S}""" |> test_expr +end + +@testitem "Tuples" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @static if VERSION > v"1.1-" + @test headof(CSTParser.parse("1,")) === :errortoken + else + @test "1," |> test_expr + end + @test "1,2" |> test_expr + @test "1,2,3" |> test_expr + @test "()" |> test_expr + @test "(==)" |> test_expr + @test "(1)" |> test_expr + @test "(1,)" |> test_expr + @test "(1,2)" |> test_expr + @test "(a,b,c)" |> test_expr + @test "(a...)" |> test_expr + @test "((a,b)...)" |> test_expr + @test "a,b = c,d" |> test_expr + @test "(a,b) = (c,d)" |> test_expr +end + +@testitem "Generators" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test "(y for y in X)" |> test_expr + @test "((x,y) for x in X, y in Y)" |> test_expr + @test "(y.x for y in X)" |> test_expr + @test "((y) for y in X)" |> test_expr + @test "(y,x for y in X)" |> test_expr + @test "((y,x) for y in X)" |> test_expr + @test "[y for y in X]" |> test_expr + @test "[(y) for y in X]" |> test_expr + @test "[(y,x) for y in X]" |> test_expr + @test "Int[y for y in X]" |> test_expr + @test "Int[(y) for y in X]" |> test_expr + @test "Int[(y,x) for y in X]" |> test_expr + @test """ +[a +for a = 1:2]""" |> test_expr + @test "[ V[j][i]::T for i=1:length(V[1]), j=1:length(V) ]" |> test_expr + @test "all(d ≥ 0 for d in B.dims)" |> test_expr + @test "(arg for x in X)" |> test_expr + @test "(arg for x in X for y in Y)" |> test_expr + @test "(arg for x in X for y in Y for z in Z)" |> test_expr + @test "(arg for x in X if A)" |> test_expr + @test "(arg for x in X if A for y in Y)" |> test_expr + @test "(arg for x in X if A for y in Y if B)" |> test_expr + @test "(arg for x in X if A for y in Y for z in Z)" |> test_expr + @test "(arg for x in X if A for y in Y if B for z in Z)" |> test_expr + @test "(arg for x in X if A for y in Y if B for z in Z if C)" |> test_expr + @test "(arg for x in X, y in Y for z in Z)" |> test_expr + @test "(arg for x in X, y in Y if A for z in Z)" |> test_expr +end + +@testitem "Macros " begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test "macro m end" |> test_expr + @test "macro m() end" |> test_expr + @test "macro m() a end" |> test_expr + @test "@mac" |> test_expr + @test "@mac a b c" |> test_expr + @test "@mac f(5)" |> test_expr + @test "(@mac x)" |> test_expr + @test "Mod.@mac a b c" |> test_expr + @test "[@mac a b]" |> test_expr + @test "@inline get_chunks_id(i::Integer) = _div64(Int(i)-1)+1, _mod64(Int(i) -1)" |> test_expr + @test "@inline f() = (), ()" |> test_expr + @test "@sprintf(\"%08d\", id)" |> test_expr + @test "[@m @n a for a in A]" |> test_expr + @test ":(@foo bar baz bat)" |> test_expr + @test ":(@foo bar for i in j end)" |> test_expr + @test "(@foo bar for i in j end)" |> test_expr + @test "foo(@foo bar for i in j)" |> test_expr + @test "foo.(@foo bar for i in j)" |> test_expr + @test CSTParser.parse("@__DIR__\n\nx", true)[1].span == 8 + + if VERSION >= v"1.8.0-" + @test "M43018.@test43018() do; end" |> test_expr + @test "@M43018.test43018() do; end" |> test_expr + end +end + +@testitem "Triple-quoted string" begin + using CSTParser: remlineinfo!, valof + include("../shared.jl") + + @test valof(CSTParser.parse("\"\"\" \" \"\"\"")) == " \" " + @test valof(CSTParser.parse("\"\"\"a\"\"\"")) == "a" + @test valof(CSTParser.parse("\"\"\"\"\"\"")) == "" + @test valof(CSTParser.parse("\"\"\"\n\t \ta\n\n\t \tb\"\"\"")) == "a\n\nb" + @test to_codeobject(CSTParser.parse("\"\"\"\ta\n\tb \$c\n\td\n\"\"\"")) == Expr(:string, "\ta\n\tb ", :c, "\n\td\n") + @test to_codeobject(CSTParser.parse("\"\"\"\n\ta\n\tb \$c\n\td\n\"\"\"")) == Expr(:string, "\ta\n\tb ", :c, "\n\td\n") + @test to_codeobject(CSTParser.parse("\"\"\"\n\ta\n\tb \$c\n\td\n\t\"\"\"")) == Expr(:string, "a\nb ", :c, "\nd\n") + @test to_codeobject(CSTParser.parse("\"\"\"\n\t \ta\$(1+\n1)\n\t \tb\"\"\"")) == Expr(:string, "a", :(1 + 1), "\nb") + ws = " " + "\"\"\"\n$ws%rv = atomicrmw \$rmw \$lt* %0, \$lt %1 acq_rel\n$(ws)ret \$lt %rv\n$ws\"\"\"" |> test_expr + ws1 = " " + ws2 = " " + "\"\"\"\n$(ws1)a\n$(ws1)b\n$(ws2)c\n$(ws2)d\n$(ws2)\"\"\"" |> test_expr + "\"\"\"\n$(ws1)a\n\n$(ws1)b\n\n$(ws2)c\n\n$(ws2)d\n\n$(ws2)\"\"\"" |> test_expr + @test "\"\"\"\n$(ws1)α\n$(ws1)β\n$(ws2)γ\n$(ws2)δ\n$(ws2)\"\"\"" |> test_expr + @test "\"\"\"Float\$(bit)\"\"\"" |> test_expr + @test headof(CSTParser.parse("\"\"\"abc\$(de)fg\"\"\"").args[3]) == :STRING + @test headof(CSTParser.parse("\"\"\"abc(de)fg\"\"\"")) == :TRIPLESTRING + @test "\"\"\"\n\t\"\"\"" |> test_expr # Change of behaviour from v1.5 -> v1.6 +end + +@testitem "raw strings with unicode" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test raw"""re = r"(\\"[^\\"]*\\") (\d+) bytes α (\\"[^\\"]*\\")\\\\" """ |> test_expr + @test raw"""re = r"(\\"[^\\"]*\\") ⋯ (\d+) bytes ⋯ (\\"[^\\"]*\\")" """ |> test_expr +end + + +@testitem "weird string edge cases" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + if VERSION > v"1.7-" + @test """x = raw"a \$(asd) sd\\\n bsf\\\\leq\n\\\\leq" """ |> test_expr + @test """x = "a \$(asd) sd\\\n bsf\\\\leq\n\\\\leq" """ |> test_expr + @test """x = raw\"\"\"a \$(asd) sd\\\n bsf\\\\leq\n\\\\leq\"\"\" """ |> test_expr + @test """x = \"\"\"a \$(asd) sd\\\n bsf\\\\leq\n\\\\leq\"\"\" """ |> test_expr + @test """x = @naah \"\"\"a \$(asd) sd\\\n bsf\\\\leq\n\\\\leq\"\"\" """ |> test_expr + @test """x = Foo.@naah \"\"\"a \$(asd) sd\\\n bsf\\\\leq\n\\\\leq\"\"\" """ |> test_expr + @test """x = Foo.@naah_str \"\"\"a \$(asd) sd\\\n bsf\\\\leq\n\\\\leq\"\"\" """ |> test_expr + @test """x = Foo.naah\"\"\"a \$(asd) sd\\\n bsf\\\\leq\n\\\\leq\"\"\" """ |> test_expr + @test """\"\"\"a \$(asd) sd\\\n bsf\\\\leq\n\\\\leq\"\"\"\nfoo""" |> test_expr + @test """throw(ArgumentError("invalid \$(m == 2 ? "hex (\\\\x)" : + "unicode (\\\$u)") escape sequence"))""" |> test_expr + @test "\"a\\\\\\\\\\\nb\"" |> test_expr + for c in 0:20 + @test test_expr(string("\"a", '\\'^c, "\nb\"")) + @test test_expr(string("\"\"\"a", '\\'^c, "\nb\"\"\"")) + end + for c in 0:20 + @test test_expr(string("`a", '\\'^c, "\nb`")) + @test test_expr(string("```a", '\\'^c, "\nb```")) + end + + @test "\"\"\"\n a\\\n b\"\"\"" |> test_expr + @test "\"\"\"\n a\\\n b\"\"\"" |> test_expr + @test "\"\"\"\na\\\n b\"\"\"" |> test_expr + @test "\"\"\"\na\\\nb\"\"\"" |> test_expr + @test "\"\"\"\n a\\\n b\"\"\"" |> test_expr + end +end + +@testitem "No longer broken things" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test "[ V[j][i]::T for i=1:length(V[1]), j=1:length(V) ]" |> test_expr + @test "all(d ≥ 0 for d in B.dims)" |> test_expr + @test ":(=)" |> test_expr + @test ":(1)" |> test_expr + @test ":(a)" |> test_expr + @test "(@_inline_meta(); f(x))" |> test_expr + @test "isa(a,b) != c" |> test_expr + @test "isa(a,a) != isa(a,a)" |> test_expr + @test "@mac return x" |> test_expr + @static if VERSION > v"1.1-" + @test headof(CSTParser.parse("a,b,").trivia[2]) === :errortoken + else + @test "a,b," |> test_expr + end + @test "m!=m" |> test_expr + @test "+(x...)" |> test_expr + @test "+(promote(x,y)...)" |> test_expr + @test "\$(x...)" |> test_expr # + @test "ccall(:gethostname, stdcall, Int32, ())" |> test_expr + @test "@inbounds @ncall a b c" |> test_expr + @test "(a+b)``" |> test_expr + @test "(-, ~)" |> test_expr + @test """function +(x::Bool, y::T)::promote_type(Bool,T) where T<:AbstractFloat + return ifelse(x, oneunit(y) + y, y) + end""" |> test_expr + @test """finalizer(x,x::GClosure->begin + ccall((:g_closure_unref,Gtk.GLib.libgobject),Void,(Ptr{GClosure},),x.handle) + end)""" |> test_expr + @test "function \$A end" |> test_expr + @test "&ctx->exe_ctx_ref" |> test_expr + @test ":(\$(docstr).\$(TEMP_SYM)[\$(key)])" |> test_expr + @test "SpecialFunctions.\$(fsym)(n::Dual)" |> test_expr + @test "(Base.@_pure_meta;)" |> test_expr + @test "@M a b->(@N c = @O d e f->g)" |> test_expr + @test "! = f" |> test_expr + @test "[a=>1, b=>2]" |> test_expr + @test "a.\$(b)" |> test_expr + @test "a.\$f()" |> test_expr + @test "4x/y" |> test_expr + @test """ + ccall(:jl_finalize_th, Void, (Ptr{Void}, Any,), + Core.getptls(), o) + """ |> test_expr + @test """ + A[if n == d + i + else + (indices(A,n) for n = 1:nd) + end...] + """ |> test_expr + @test """ + @spawnat(p, + let m = a + isa(m, Exception) ? m : nothing + end) + """ |> test_expr # + @test "[@spawn f(R, first(c), last(c)) for c in splitrange(length(R), nworkers())]" |> test_expr + @test "M.:(a)" |> test_expr + @test """ + begin + for i in I for j in J + if cond + a + end + end end + end""" |> test_expr + @test "-f.(a.b + c)" |> test_expr + @test ":(import Base: @doc)" |> test_expr + @test "[a for a in A for b in B]" |> test_expr + @test "+(a,b,c...)" |> test_expr + @test """@testset a for t in T + t + end""" |> test_expr + @test "import Base.==" |> test_expr + @test "a`text`" |> test_expr + @test "a``" |> test_expr + @test "a`text`b" |> test_expr + @test "[a; a 0]" |> test_expr + @test "[a, b; c]" |> test_expr + @test "t{a; b} " |> test_expr + @test "a ~ b + c -d" |> test_expr + @test "y[j=1:10,k=3:2:9; isodd(j+k) && k <= 8]" |> test_expr + @test "(8=>32.0, 12=>33.1, 6=>18.2)" |> test_expr + @test "(a,b = c,d)" |> test_expr + @test "[ -1 -2;]" |> test_expr + @test "-2y" |> test_expr # precedence + @test "'''" |> test_expr # tokenize + @test """ + if j+k <= deg +1 + end + """ |> test_expr + @test "function f() ::T end" |> test_expr # ws closer + @test "import Base: +, -, .+, .-" |> test_expr + if VERSION > v"1.6-" + @test "import Base.:+" |> test_expr + @test "import Base.:⋅" |> test_expr + @test "import Base.:sin, Base.:-" |> test_expr + end + @test "[a + + l]" |> test_expr # ws closer + @test "@inbounds C[i,j] = - α[i] * αjc" |> test_expr + @test "@inbounds C[i,j] = - n * p[i] * pj" |> test_expr + @test """ + if ! a + b + end + """ |> test_expr # ws closer + @test "[:-\n:+]" |> test_expr + @test "::a::b" |> test_expr + @test "-[1:nc]" |> test_expr + @test "@assert .!(isna(res[2]))" |> test_expr # v0.6 + @test "-((attr.rise / PANGO_SCALE)pt).value" |> test_expr + @test "!(a = b)" |> test_expr + @test "-(1)a" |> test_expr + @test "!(a)::T" |> test_expr + @test "a::b where T<:S" |> test_expr + @test "+(x::Bool, y::T)::promote_type(Bool,T) where T<:AbstractFloat" |> test_expr + @test "T where V<:(T where T)" |> test_expr + @test "function ^(z::Complex{T}, p::Complex{T})::Complex{T} where T<:AbstractFloat end" |> test_expr + @test "function +(a) where T where S end" |> test_expr + @test "function -(x::Rational{T}) where T<:Signed end" |> test_expr + @test "\$(a)(b)" |> test_expr + @test "if !(a) break end" |> test_expr + @test "module a() end" |> test_expr + if VERSION > v"1.3-" + @test """module var"#43932#" end""" |> test_expr + end + @test "M.r\"str\" " |> test_expr + @test "f(a for a in A if cond)" |> test_expr + @test "\"dimension \$d is not 1 ≤ \$d ≤ \$nd\" " |> test_expr + @test "-(-x)^1" |> test_expr + @test """ + "\\\\\$ch" + """ |> test_expr + @test "µs" |> test_expr # normalize unicode + @test """ + (x, o; p = 1) -> begin + return o, p + end + """ |> test_expr # normalize unicode + @test """ + (x, o...; p...) -> begin + return o, p + end + """ |> test_expr # normalize unicode + @test "function func() where {A where T} x + 1 end" |> test_expr # nested where + @test "(;x)" |> test_expr # issue 39 + @test """ + let f = ((; a = 1, b = 2) -> ()), + m = first(methods(f)) + @test DSE.keywords(f, m) == [:a, :b] + end + """ |> test_expr + @test "-1^a" |> test_expr + @test "function(f, args...; kw...) end" |> test_expr + @test "function(f, args...=1; kw...) end" |> test_expr + @test "2a * b" |> test_expr + @test "(g1090(x::T)::T) where {T} = x+1.0" |> test_expr + @test "(:) = Colon()" |> test_expr + @test "a + in[1]" |> test_expr + @test "function f(ex) +a end" |> test_expr + @test "x`\\\\`" |> test_expr + @test "x\"\\\\\"" |> test_expr + @test "x\"\\\\ \"" |> test_expr + @test "a.{1}" |> test_expr + @test "@~" |> test_expr + @test "\$\$(x)" |> test_expr + @test "\$\$(x)" |> test_expr + @test CSTParser.headof(CSTParser.parse("=")) === :errortoken + @test CSTParser.headof(CSTParser.parse("~")) === :OPERATOR + @test "(1:\n2)" |> test_expr + @test "a[: ]" |> test_expr + @test ".~b" |> test_expr + if VERSION > v"1.1-" + @test "a .~ b" |> test_expr + end + @test "A[a~b]" |> test_expr + @test "[a~b]" |> test_expr + if VERSION >= v"1.6" + @test "[a ~b]" |> test_expr + end + @test "[a ~ b]" |> test_expr + @test "[a~ b]" |> test_expr + @test "1 .< 2 .< 3" |> test_expr + @test "(;)" |> test_expr + if VERSION > v"1.5" + @test "@M{a}-b" |> test_expr + @test "@M{a,b}-b" |> test_expr + end + @test "@M[a]-b" |> test_expr +end + +@testitem "interpolation error catching" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + x = CSTParser.parse("\"a \$ b\"") + @test x.fullspan == 7 + @test CSTParser.headof(x[3]) === :errortoken + x = CSTParser.parse("\"a \$# b\"") + @test x.fullspan == 8 + @test CSTParser.headof(x[3]) === :errortoken +end + + + +@testitem "string interpolation" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + if VERSION >= v"1.6" + @test test_expr(raw""""$("asd")" """) + @test test_expr(raw""""$("asd")a" """) + @test test_expr(raw""""a$("asd")" """) + @test test_expr(raw""""a$("asd")a" """) + @test test_expr(raw"""`$("asd")` """) + @test test_expr(raw"""`$("asd")a` """) + @test test_expr(raw"""`a$("asd")` """) + @test test_expr(raw"""`a$("asd")a` """) + end +end +@testitem "string whitespace handling" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + if VERSION >= v"1.6" + @test test_expr("""\"\"\"\n\\t\"\"\" """) + @test test_expr("""\"\"\"\n\\t\n\"\"\" """) + @test test_expr("""\"\"\"\n\\t\\n\"\"\" """) + @test test_expr(raw"""\"\"\"\n\\t\"\"\" """) + @test test_expr(raw"""\"\"\"\n\\t\n\"\"\" """) + @test test_expr(raw"""\"\"\"\n\\t\\n\"\"\" """) + end +end + +@testitem "cmd interpolation" begin + using CSTParser: remlineinfo!, valof + include("../shared.jl") + + @test test_expr("`a \$b c`") + @test test_expr(raw"`a \"\$b $b\" c`") + @test test_expr("`a b c`") + x = CSTParser.parse("`a \$b c`") + @test x.args[1].head == :globalrefcmd + @test x.args[3].head == :string + @test valof(x.args[3].args[1]) == "a " + @test valof(x.args[3].args[2]) == "b" + @test valof(x.args[3].args[3]) == " c" +end + +@testitem "multiple ; in kwargs" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test test_expr("f(a; b=1; c=2)") + @test test_expr("f(a; b=1; c=2) = 2") + @test test_expr("f( ; b=1; c=2)") + @test test_expr("f(a; b=1; c=2)") + @test test_expr("f(a; b=1, c=2; d=3)") + @test test_expr("f(a; b=1; c=2, d=3)") + @test test_expr("f(a; b=1; c=2; d=3)") +end + +@testitem "Broken things" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test_broken "\$(a) * -\$(b)" |> test_expr_broken +end + +# test_fsig_decl(str) = (x->x.id).(CSTParser._get_fsig(CSTParser.parse(str)).defs) +# @testitem "func-sig variable declarations" begin +# @test test_fsig_decl("f(x) = x") == [:x] +# @test test_fsig_decl("""function f(x) +# x +# end""") == [:x] + +# @test test_fsig_decl("f{T}(x::T) = x") == [:T, :x] +# @test test_fsig_decl("""function f{T}(x::T) +# x +# end""") == [:T, :x] + +# @test test_fsig_decl("f(x::T) where T = x") == [:T, :x] +# @test test_fsig_decl("""function f(x::T) where T +# x +# end""") == [:T, :x] + + +# @test test_fsig_decl("f(x::T{S}) where T where S = x") == [:T, :S, :x] +# @test test_fsig_decl("""function f(x::T{S}) where T where S +# x +# end""") == [:T, :S, :x] +# end + +@testitem "Spans" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + CSTParser.parse(raw""" + "ABC$(T)" + """).fullspan >= 9 + CSTParser.parse("\"_\"").fullspan == 3 + CSTParser.parse("T.mutable && print(\"Ok\")").fullspan == 24 + CSTParser.parse("(\"\$T\")").fullspan == 6 + CSTParser.parse("\"\"\"\$T is not supported\"\"\"").fullspan == 25 + CSTParser.parse("using Compat: @compat\n").fullspan == 22 + CSTParser.parse("primitive = 1").fullspan == 13 +end + +@testitem "Command or string with unicode" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test "```αhelloworldω```" |> test_expr + @test "\"αhelloworldω\"" |> test_expr +end + +@testitem "conversion of floats with underscore" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test "30.424_876_125_859_513" |> test_expr +end + +@testitem "errors" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test headof(CSTParser.parse("1? b : c ").args[1]) === :errortoken + @test headof(CSTParser.parse("1 ?b : c ").trivia[1]) === :errortoken + @test headof(CSTParser.parse("1 ? b :c ").trivia[2]) === :errortoken + @test headof(CSTParser.parse("1:\n2").args[1]) === :errortoken + @test headof(CSTParser.parse("1.a").args[2]) === :errortoken + @test headof(CSTParser.parse("f ()")) === :errortoken + @test headof(CSTParser.parse("f{t} ()")) === :errortoken + @test headof(CSTParser.parse(": a").trivia[1]) === :errortoken + @test headof(CSTParser.parse("const a").args[1]) === :errortoken +end + +@testitem "colons" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test test_expr("(if true I elseif false J else : end for i in 1:5)") + @test test_expr("if true; : end") + @test test_expr("a + :") +end + +@testitem "tuple params" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test "1,2,3" |> test_expr + @test "1;2,3" |> test_expr + @test "1,2;3" |> test_expr + @test "(1,2,3)" |> test_expr + @test "(1;2,3)" |> test_expr + @test "(1,2;3)" |> test_expr + @test "f(;)" |> test_expr +end + +@testitem "docs" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test "\"doc\"\nT" |> test_expr + @test "@doc \"doc\" T" |> test_expr + @test "@doc \"doc\"\nT" |> test_expr + @test "@doc \"doc\n\n\n\"\nT" |> test_expr + @test "begin\n@doc \"doc\"\n\nT\nend" |> test_expr + @test "begin\n@doc \"doc\"\nT\nend" |> test_expr + @test "begin\n@doc \"doc\" T\nend" |> test_expr + @test "if true\n@doc \"doc\"\n\nT\nend" |> test_expr + @test "if true\n@doc \"doc\"\nT\nend" |> test_expr + @test "if true\n@doc \"doc\" T\nend" |> test_expr + @test "@doc \"I am a module\" ModuleMacroDoc" |> test_expr + @test """ + @doc(foo) + """ |> test_expr + @test """ + @doc foo + """ |> test_expr +end + +@testitem "braces" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test "{a}" |> test_expr + @test "{a, b}" |> test_expr + @test "{a, b; c}" |> test_expr + @test "{a, b; c = 1}" |> test_expr + @test "{a b}" |> test_expr + @test "{a b; c}" |> test_expr + @test "{a b; c = 1}" |> test_expr +end + +@testitem "import preceding dot whitespace" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test "using . M" |> test_expr + @test "using .. M" |> test_expr + @test "using ... M" |> test_expr +end + +@testitem "issue #116" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test """ + function foo() where {A <:B} + body + end""" |> test_expr + + @test """ + function foo() where {A <: B} + body + end""" |> test_expr +end + +@testitem "issue #165" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + x = CSTParser.parse(""" + a ? b + function f end""") + @test length(x) == 5 # make sure we always give out an EXPR of the right length + @test headof(x.args[3]) === :errortoken +end + +@testitem "issue #182" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + x = CSTParser.parse(""" + quote + \"\"\" + txt + \"\"\" + sym + end""") + @test x.args[1].args[1].args[1].head === :globalrefdoc +end + +@testitem "issue #198" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + if VERSION > v"1.3.0-" + @test test_expr(":var\"id\"") + end +end +@testitem "vscode issue #1632" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test test_expr("\"\$( a)\"") + @test test_expr("\"\$(#=comment=# a)\"") +end +@testitem "issue #210" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test test_expr("function f(a; where = false) end") +end + +@testitem "suffixed ops" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test test_expr("a +₊ b *₊ c") + @test test_expr("a *₊ b +₊ c") +end + + +@testitem "import .. as .. syntax" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @static if VERSION > v"1.6-" + @test test_expr("import a as b") + @test test_expr("import a as b, c") + @test test_expr("import M: a as b") + @test test_expr("import M: a as b, c") + @test CSTParser.parse("using a as b")[2].head === :errortoken + @test test_expr("using M: a as b") + @test test_expr("using M: a as b, c") + end +end +@testitem "exor #201" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test test_expr(raw"$return(x)") +end + + + +@testitem "@var #236" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + if VERSION > v"1.3.0-" + s = raw"""@var" " a""" + @test test_expr(s) + @test CSTParser.ismacroname(CSTParser.parse(s).args[1]) + end +end + +@testitem "nonstandard identifier (var\"blah\") parsing" begin +using CSTParser: remlineinfo! + include("../shared.jl") + + if VERSION > v"1.3.0-" + @test """var"asd" """ |> test_expr + @test """var"#asd" """ |> test_expr + @test """var"#asd#" """ |> test_expr + @test """M.var"asd" """ |> test_expr + @test """M.var"#asd" """ |> test_expr + @test """M.var"#asd#" """ |> test_expr + end +end +@testitem "bad uint" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test to_codeobject(CSTParser.parse("0x.")) == Expr(:error) +end + +@testitem "endswithtrivia" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + x = CSTParser.parse("\"some long title \$label1 \$label2\" \na") + @test x[3].span < x[3].fullspan + @test CSTParser.lastchildistrivia(x[3]) +end + +@testitem "bad interp with following newline" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + s = "\"\"\"\$()\n\"\"\"" + x = CSTParser.parse(s) + @test sizeof(s) == x.fullspan +end + +@testitem "minimal_reparse" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + s0 = """ + testsettravx=nothing; + ) fx;ifxend) + # parsing works?""" + s1 = """ + testsettravx=nothing; + ) ;ifxend) + # parsing works?""" + x0 = CSTParser.parse(s0, true) + x1 = CSTParser.parse(s1, true) + x2 = CSTParser.minimal_reparse(s0, s1, x0, x1) + @test CSTParser.comp(x1, x2) +end + +@testitem "primes" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test test_expr(""" + f() do x + end' + """) + @test CSTParser.has_error(cst"begin end'") + @test !CSTParser.has_error(cst"[]'") + @test !CSTParser.has_error(cst"'a''") + @test test_expr("(a)'") + @test test_expr("a.a'") + if VERSION >= v"1.6" + @test test_expr("a'ᵀ") + @test test_expr(":(a'ᵀ)") + end + @test test_expr("a'") + @test test_expr("a''") + @test test_expr("a'''") + # @test test_expr(":.'") + # @test test_expr(":?'") + # @test test_expr(":a'") +end + +@testitem "end as id juxt" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test test_expr("a[1end]") + if VERSION >= v"1.4" + @test test_expr("a[2begin:1end]") + end +end + +@testitem "last child is trivia for :string" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test !CSTParser.lastchildistrivia(cst"""("a $(A) a" )"""[2]) +end + +@testitem "toplevel strings" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test test_expr(""""a" in b && c""") +end + +@testitem "@doc cont" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test test_expr("module a\n@doc doc\"\"\"doc\"\"\"\nx\nend") +end + +@testitem "char escape" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test test_expr(raw"'\$'") + @test test_expr(raw"'\a'") + @test test_expr(raw"'\3'") + @test test_expr(raw"'\000'") + @test test_expr(raw"'\033'") + @test test_expr(raw"'\177'") + @test test_expr(raw"'\u222'") + @test test_expr(raw"'\ufff'") + @test test_expr(raw"'\x2'") + @test test_expr(raw"'\x22'") + @test test_expr(raw"'\u22'") + @test test_expr(raw"'\u2222'") + @test test_expr(raw"'\U2222'") + @test test_expr(raw"'\U22222'") + @test test_expr(raw"'\U00000001'") + + @test CSTParser.parse(raw"'\200'").head == :errortoken + @test CSTParser.parse(raw"'\300'").head == :errortoken + @test CSTParser.parse(raw"'\377'").head == :errortoken + @test CSTParser.parse(raw"'\600'").head == :errortoken + @test CSTParser.parse(raw"'\777'").head == :errortoken + @test CSTParser.parse(raw"'\x222'").head == :errortoken + @test CSTParser.parse(raw"'\u22222'").head == :errortoken + @test CSTParser.parse(raw"'\U222222'").head == :errortoken + @test CSTParser.parse(raw"'\asdd'").head == :errortoken + @test CSTParser.parse(raw"'\α'").head == :errortoken + @test CSTParser.parse(raw"'\αsdd'").head == :errortoken + @test CSTParser.parse(raw"'\u222ää'").head == :errortoken + @test CSTParser.parse(raw"'\x222ää'").head == :errortoken + @test CSTParser.parse(raw"'\U222ää'").head == :errortoken + @test CSTParser.parse(raw"'\U10000001'").head == :errortoken + for c in rand(Char, 1000) + c == '\\' && continue + @test test_expr(string("'", c, "'")) + end +end + +@testitem "invalid char in string" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test CSTParser.parse(raw"\"\U222222222\"").head == :errortoken +end + +@testitem "string macros" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test test_expr(raw"""test"asd"asd""") + if VERSION >= v"1.6" + @test test_expr(raw"""test"asd"0""") + @test test_expr(raw"""test"asd"0o0""") + @test test_expr(raw"""test"asd"0x0""") + @test test_expr(raw"""test"asd"0.0""") + end + @test test_expr(raw"""test"asd"true""") + @test test_expr(raw"""test""true""") +end + +@testitem "number parsing" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test test_expr("0b00000000") + @test test_expr("0b000000000") + @test test_expr("0b0000000000000000") + @test test_expr("0b00000000000000000") + @test test_expr("0b00000000000000000000000000000000") + @test test_expr("0b000000000000000000000000000000000") + @test test_expr("0b0000000000000000000000000000000000000000000000000000000000000000") + @test test_expr("0b00000000000000000000000000000000000000000000000000000000000000000") + @test test_expr("0b00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000") + if VERSION >= v"1.6" + @test test_expr("0b000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000") + end + @test test_expr("0b11111111") + @test test_expr("0b111111111") + @test test_expr("0b1111111111111111") + @test test_expr("0b11111111111111111") + @test test_expr("0b11111111111111111111111111111111") + @test test_expr("0b111111111111111111111111111111111") + @test test_expr("0b1111111111111111111111111111111111111111111111111111111111111111") + @test test_expr("0b11111111111111111111111111111111111111111111111111111111111111111") + @test test_expr("0b11111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111") + if VERSION >= v"1.6" + @test test_expr("0b111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111") + end + + @test test_expr("0o0") + @test test_expr("0o00") + @test test_expr("0o0000") + @test test_expr("0o00000") + @test test_expr("0o000000000") + @test test_expr("0o0000000000") + @test test_expr("0o00000000000000000000") + @test test_expr("0o000000000000000000000") + @test test_expr("0o00000000000000000000000000000000000000000") + @test test_expr("0o0000000000000000000000000000000000000000000") + @test test_expr("0o1") + @test test_expr("0o11") + @test test_expr("0o1111") + @test test_expr("0o11111") + @test test_expr("0o111111111") + @test test_expr("0o1111111111") + @test test_expr("0o11111111111111111111") + @test test_expr("0o111111111111111111111") + @test test_expr("0o11111111111111111111111111111111111111111") + @test test_expr("0o111111111111111111111111111111111111111111") + @test test_expr("0o377777777777777777777777777777777777777777") + @test test_expr("0o1111111111111111111111111111111111111111111") + @test test_expr("0o077") + @test test_expr("0o377") + @test test_expr("0o400") + @test test_expr("0o077777") + @test test_expr("0o177777") + @test test_expr("0o200000") + @test test_expr("0o00000000000") + @test test_expr("0o17777777777") + @test test_expr("0o40000000000") + @test test_expr("0o0000000000000000000000") + @test test_expr("0o1000000000000000000000") + @test test_expr("0o2000000000000000000000") + @test test_expr("0o0000000000000000000000000000000000000000000") + @test test_expr("0o1000000000000000000000000000000000000000000") + @test test_expr("0o2000000000000000000000000000000000000000000") + + @test test_expr("0x00") + @test test_expr("0x000") + @test test_expr("0x0000") + @test test_expr("0x00000") + @test test_expr("0x00000000") + @test test_expr("0x000000000") + @test test_expr("0x0000000000000000") + @test test_expr("0x00000000000000000") + @test test_expr("0x00000000000000000000000000000000") + if VERSION >= v"1.6" + @test test_expr("0x000000000000000000000000000000000") + end + + @test test_expr("0x11") + @test test_expr("0x111") + @test test_expr("0x1111") + @test test_expr("0x11111") + @test test_expr("0x11111111") + @test test_expr("0x111111111") + @test test_expr("0x1111111111111111") + @test test_expr("0x11111111111111111") + @test test_expr("0x11111111111111111111111111111111") + if VERSION >= v"1.6" + @test test_expr("0x111111111111111111111111111111111") + end +end + +@testitem "#302" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + str = """ + const _examples = PlotExample[ + PlotExample( # 1 + "Lines", + "A simple line plot of the columns.", + [:( + begin + plot(Plots.fakedata(50, 5), w = 3) + end + )], + ), + ] + """ + @test test_expr(str) + x, ps = CSTParser.parse(CSTParser.ParseState(str), true) + @test ps.errored == false +end + +@testitem "#304" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + str = """ + const _examples = PlotExample[ + PlotExample( # 40 + "Lens", + "A lens lets you easily magnify a region of a plot. x and y coordinates refer to the to be magnified region and the via the `inset` keyword the subplot index and the bounding box (in relative coordinates) of the inset plot with the magnified plot can be specified. Additional attributes count for the inset plot.", + [ + quote + begin + plot( + [(0, 0), (0, 0.9), (1, 0.9), (2, 1), (3, 0.9), (80, 0)], + legend = :outertopright, + ) + plot!([(0, 0), (0, 0.9), (2, 0.9), (3, 1), (4, 0.9), (80, 0)]) + plot!([(0, 0), (0, 0.9), (3, 0.9), (4, 1), (5, 0.9), (80, 0)]) + plot!([(0, 0), (0, 0.9), (4, 0.9), (5, 1), (6, 0.9), (80, 0)]) + lens!( + [1, 6], + [0.9, 1.1], + inset = (1, bbox(0.5, 0.0, 0.4, 0.4)), + ) + end + end, + ], + ), + ] + """ + @test test_expr(str) + x, ps = CSTParser.parse(CSTParser.ParseState(str), true) + @test ps.errored == false +end + +@testitem "#310" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + x, ps = CSTParser.parse(CSTParser.ParseState("""import a.notvar"papa" """), true) + @test ps.errored == true + x, ps = CSTParser.parse(CSTParser.ParseState("""import notvar"papa" """), true) + @test ps.errored == true +end + +@testitem "#311" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test test_expr(raw"import a.$b.c") +end + +@testitem "kw interpolation" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test test_expr(raw""""foo $bar" """) + @test test_expr(raw""""foo $type" """) + @test test_expr(raw""""foo $function" """) + @test test_expr(raw""""foo $begin" """) + @test test_expr(raw""""foo $quote" """) +end + +@testitem "broadcasted && and ||" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + if VERSION > v"1.7-" + @test test_expr(raw"""a .&& b""") + @test test_expr(raw"""a .< b .&& b .> a""") + @test test_expr(raw"""a .|| b""") + @test test_expr(raw"""a .< b .|| b .> a""") + end +end + + +@testitem "normalized unicode ops" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + if VERSION > v"1.7-" + @test "(·) == (·) == (⋅) == 5" |> test_expr + @test "(−) == (-) == 6" |> test_expr + end +end + +@testitem "pair tuple" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test test_expr("a => b") + @test test_expr("a => b, c, d") + @test test_expr("a, a => b, c, d") +end + +@testitem "global" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test test_expr("global a") + @test test_expr("global a = 1") + @test test_expr("global a = 1, b") + @test test_expr("global a, b") + @test test_expr("global a, b = 2") +end diff --git a/test/parser/test_square.jl b/test/parser/test_square.jl new file mode 100644 index 00000000..07d5bc4b --- /dev/null +++ b/test/parser/test_square.jl @@ -0,0 +1,138 @@ +@testitem "vect" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test "[x]" |> test_expr + @test "[(1,2)]" |> test_expr + @test "[x...]" |> test_expr + @test "[1,2,3,4,5]" |> test_expr + # this is nonsensical, but iteration should still work + @test traverse(CSTParser.parse(raw"""[[:alpha:]a←-]""")) + # unterminated expressions may cause issues + @test traverse(CSTParser.parse("bind_artifact!(\"../Artifacts.toml\",\"example\",hash,download_info=[(\"file://c:/juliaWork/tarballs/example.tar.gz\"\",tarball_hash)],force=true)\n2+2\n")) + @test traverse(CSTParser.parse("[2+3+4")) + @test traverse(CSTParser.parse("[2+3+4+")) + @test traverse(CSTParser.parse("[\"hi\"\"")) + @test traverse(CSTParser.parse("[\"hi\"\"\n")) + @test traverse(CSTParser.parse("[(1,2,3])")) +end + +@testitem "ref" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test "t[i]" |> test_expr + @test "t[i, j]" |> test_expr +end + +@testitem "vcat" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test "[x;]" |> test_expr + @test "[x;y;z]" |> test_expr + @test """[x + y + z]""" |> test_expr + @test """[x + y;z]""" |> test_expr + @test """[x;y + z]""" |> test_expr + @test "[x,y;z]" |> test_expr + @test "[1,2;]" |> test_expr +end + +@testitem "typed_vcat" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test "t[x;]" |> test_expr + @test "t[x;y]" |> test_expr + @test """t[x + y]""" |> test_expr + @test "t[x;y]" |> test_expr + @test "t[x y; z]" |> test_expr + @test "t[x, y; z]" |> test_expr +end + +@testitem "ncat" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + if VERSION > v"1.8-" + @test "[;]" |> test_expr + @test "[;;]" |> test_expr + @test "[;;\n]" |> test_expr + @test "[\n ;; \n]" |> test_expr + @test "[;;;;;;;]" |> test_expr + @test "[x;;;;;]" |> test_expr + @test "[x;;]" |> test_expr + @test "[x;; y;; z]" |> test_expr + @test "[x;;; y;;;z]" |> test_expr + @test "[x;;; y;;;z]'" |> test_expr + @test "[1 2; 3 4]" |> test_expr + @test "[1;2;;3;4;;5;6;;;;9]" |> test_expr + if VERSION > v"1.7-" + @test "[let; x; end;; y]" |> test_expr + @test "[let; x; end;;;; y]" |> test_expr + end + end +end + +@testitem "typed_ncat" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + if VERSION > v"1.8-" + @test "t[;;]" |> test_expr + @test "t[;;;;;;;]" |> test_expr + @test "t[x;;;;;]" |> test_expr + @test "t[x;;]" |> test_expr + @test "t[x;; y;; z]" |> test_expr + @test "t[x;;; y;;;z]" |> test_expr + @test "t[x;;\ny]" |> test_expr + @test "t[x y;;\nz a]" |> test_expr + @test "t[x y;;\nz a]'" |> test_expr + @test "t[let; x; end;; y]" |> test_expr + @test "t[let; x; end;;;; y]" |> test_expr + end +end + +@testitem "hcat" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test "[x y]" |> test_expr + @test "[let; x; end y]" |> test_expr + @test "[let; x; end; y]" |> test_expr +end + +@testitem "typed_hcat" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test "t[x y]" |> test_expr + @test "t[let; x; end y]" |> test_expr + @test "t[let; x; end; y]" |> test_expr +end + +@testitem "Comprehension" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test "[i for i = 1:10]" |> test_expr + @test "Int[i for i = 1:10]" |> test_expr + @test "[let;x;end for x in x]" |> test_expr + @test "[let; x; end for x in x]" |> test_expr + @test "[let x=x; x+x; end for x in x]" |> test_expr + if VERSION > v"1.7-" + @test """[ + [ + let l = min((d-k),k); + binomial(d-l,l); + end; for k in 1:d-1 + ] for d in 2:9 + ] + """ |> test_expr + end +end diff --git a/test/parser/test_types.jl b/test/parser/test_types.jl new file mode 100644 index 00000000..34c653d4 --- /dev/null +++ b/test/parser/test_types.jl @@ -0,0 +1,64 @@ +@testitem "Abstract" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test "abstract type t end" |> test_expr + @test "abstract type t{T} end" |> test_expr + @test "abstract type t <: S end" |> test_expr + @test "abstract type t{T} <: S end" |> test_expr +end + +@testitem "primitive" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test "primitive type Int 64 end" |> test_expr + @test "primitive type Int 4*16 end" |> test_expr +end + +@testitem "Structs" begin + using CSTParser: remlineinfo! + include("../shared.jl") + + @test "struct a end" |> test_expr + @test "struct a; end" |> test_expr + @test "struct a; b;end" |> test_expr + @test """struct a + arg1 + end""" |> test_expr + @test """struct a <: T + arg1::Int + arg2::Int + end""" |> test_expr + @test """struct a + arg1::T + end""" |> test_expr + @test """struct a{T} + arg1::T + a(args) = new(args) + end""" |> test_expr + @test """struct a <: Int + arg1::Vector{Int} + end""" |> test_expr + @test """mutable struct a <: Int + arg1::Vector{Int} + end""" |> test_expr + if VERSION > v"1.8-" + @test """mutable struct A + const arg1::Vector{Int} + arg2 + end""" |> test_expr + @test """mutable struct A + const arg1 + arg2 + end""" |> test_expr + @test """struct A + const arg1 + arg2 + end""" |> test_expr + @test """@eval struct A + const arg1 + arg2 + end""" |> test_expr + end +end \ No newline at end of file diff --git a/test/runtests.jl b/test/runtests.jl index f4fd0f91..b9e874db 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,14 +1,3 @@ -using CSTParser, Test, CSTParser.Tokenize -using CSTParser: parse, remlineinfo!, span, headof, kindof, valof +using TestItemRunner -@testset "CSTParser" begin - include("spec.jl") - include("parser.jl") - include("interface.jl") - include("display.jl") - include("iterate.jl") - if VERSION >= v"1.6" - include("check_base.jl") - include("errparse.jl") - end -end +@run_package_tests diff --git a/test/shared.jl b/test/shared.jl new file mode 100644 index 00000000..2b2c6ab8 --- /dev/null +++ b/test/shared.jl @@ -0,0 +1,110 @@ +using CSTParser: @cst_str, headof, parentof, check_span, EXPR, to_codeobject + +jl_parse(s) = CSTParser.remlineinfo!(Meta.parse(s)) + +function check_parents(x::EXPR) + if x.args isa Vector{EXPR} + for a in x.args + @test a.parent == x + check_parents(a) + end + end + if x.trivia isa Vector{EXPR} + for a in x.trivia + @test a.parent == x + end + end +end + +function test_iter(ex) + total = 0 + for x in ex + @test x isa EXPR + test_iter(x) + total += x.fullspan + end + if length(ex) > 0 + @test total == ex.fullspan + end +end + +function test_expr(s, head, n, endswithtrivia = false) + x = CSTParser.parse(s) + head === nothing || @test headof(x) === head + @test length(x) === n + @test x.args === nothing || all(x === parentof(a) for a in x.args) + @test x.trivia === nothing || all(x === parentof(a) for a in x.trivia) + @test to_codeobject(x) == jl_parse(s) + @test isempty(check_span(x)) + check_parents(x) + test_iter(x) + @test endswithtrivia ? (x.fullspan-x.span) == (last(x.trivia).fullspan - last(x.trivia).span) : (x.fullspan-x.span) == (last(x.args).fullspan - last(x.args).span) +end + +randop() = rand(["-->", "→", + "||", + "&&", + "<", "==", "<:", ">:", + "<|", "|>", + ":", + "+", "-", + ">>", "<<", + "*", "/", + "//", + "^", "↑", + "::", + ".", "->"]) + +test_expr_broken(str) = test_expr(str, false) + +function traverse(x) + try + for a in x + @test traverse(a) + end + return true + catch err + @error "EXPR traversal failed." expr = x exception = err + return false + end +end + +function test_expr(str, show_data=true) + x, ps = CSTParser.parse(ParseState(str)) + + x0 = to_codeobject(x) + x1 = remlineinfo!(Meta.parse(str)) + + @test x.args === nothing || all(x === parentof(a) for a in x.args) + @test x.trivia === nothing || all(x === parentof(a) for a in x.trivia) + @test isempty(check_span(x)) + check_parents(x) + @test traverse(x) + @test x.fullspan == sizeof(str) + + if CSTParser.has_error(ps) || x0 != x1 + if show_data + println("Mismatch between flisp and CSTParser when parsing string $str") + println("ParserState:\n $ps\n") + println("CSTParser Expr:\n $x\n") + println("Converted CSTParser Expr:\n $x0\n") + println("Base EXPR:\n $x1\n") + end + return false + end + return true +end + +function test_iter_spans(x) + n = 0 + for i = 1:length(x) + a = x[i] + if !(a isa EXPR) + @info i, headof(x), to_codeobject(x) + end + @test a isa EXPR + test_iter_spans(a) + n += a.fullspan + end + length(x) > 0 && @test n == x.fullspan +end diff --git a/test/test_check_base.jl b/test/test_check_base.jl new file mode 100644 index 00000000..e29dc201 --- /dev/null +++ b/test/test_check_base.jl @@ -0,0 +1,165 @@ +@testitem "Parsing files in Base" begin + include("shared.jl") + + function norm_ast(a::Any) + if isa(a, Expr) + for (i, arg) in enumerate(a.args) + a.args[i] = norm_ast(arg) + end + if a.head === :line + return Expr(:line, a.args[1], :none) + end + if a.head === :macrocall + fa = a.args[1] + long_enough = length(a.args) >= 3 + if fa === Symbol("@int128_str") && long_enough + return Base.parse(Int128, a.args[3]) + elseif fa === Symbol("@uint128_str") && long_enough + return Base.parse(UInt128, a.args[3]) + elseif fa === Symbol("@bigint_str") && long_enough + return Base.parse(BigInt, a.args[3]) + elseif fa == Symbol("@big_str") && long_enough + s = a.args[3] + n = tryparse(BigInt, s) + if !(n === nothing) + return (n) + end + n = tryparse(BigFloat, s) + if !(n === nothing) + return isnan((n)) ? :NaN : (n) + end + return s + end + elseif length(a.args) >= 2 && Meta.isexpr(a, :call) && a.args[1] == :- && isa(a.args[2], Number) + return -a.args[2] + end + return a + elseif isa(a, QuoteNode) + return Expr(:quote, norm_ast(a.value)) + elseif isa(a, AbstractFloat) && isnan(a) + return :NaN + end + return a + end + + function meta_parse_has_error(x::Expr) + if x.head == :incomplete + return true + else + for a in x.args + if meta_parse_has_error(a) + return true + end + end + end + return false + end + meta_parse_has_error(_) = false + + function meta_parse_file(str) + pos = 1 + x1 = Expr(:file) + try + while pos <= sizeof(str) + x, pos = Meta.parse(str, pos) + push!(x1.args, x) + end + catch er + isa(er, InterruptException) && rethrow(er) + + return x1, true + end + if length(x1.args) > 0 && x1.args[end] === nothing + pop!(x1.args) + end + x1 = norm_ast(x1) + CSTParser.remlineinfo!(x1) + return x1, meta_parse_has_error(x1) + end + + find_error(x, offset=0) = + if CSTParser.headof(x) === :errortoken + @show offset + else + for a in x + find_error(a, offset) + offset += a.fullspan + end + end + + function cst_parse_file(str) + x, ps = CSTParser.parse(CSTParser.ParseState(str), true) + sp = CSTParser.check_span(x) + # remove leading/trailing nothings + if length(x.args) > 0 && CSTParser.is_nothing(x.args[1]) + popfirst!(x.args) + end + + if !isempty(sp) + @error "CST spans inconsistent!" + end + + x0 = norm_ast(to_codeobject(x)) + x0, CSTParser.has_error(ps), isempty(sp) + end + + _compare(x, y) = x == y + + function _compare(x::Expr, y::Expr) + if x == y + return true + else + if x.head != y.head || length(x.args) != length(y.args) + printstyled(x, bold=true, color=:light_red) + println() + printstyled(y, bold=true, color=:light_green) + println() + end + for i = 1:min(length(x.args), length(y.args)) + if !_compare(x.args[i], y.args[i]) + printstyled(x.args[i], bold=true, color=:light_red) + println() + printstyled(y.args[i], bold=true, color=:light_green) + println() + end + end + return false + end + end + + if VERSION >= v"1.6" + + dir = joinpath(Sys.BINDIR, Base.DATAROOTDIR) + for (root, _, files) in walkdir(dir; follow_symlinks=true) + for fpath in files + file = joinpath(root, fpath) + endswith(file, ".jl") || continue + + str = read(file, String) + + cst = CSTParser.parse(str, true) + cst_expr, cst_err, span_err = cst_parse_file(str) + meta_expr, meta_err = meta_parse_file(str) + @test cst_err == meta_err + @test span_err + @test cst.fullspan == sizeof(str) + + if cst_err || meta_err + if cst_err && !meta_err + @error "CSTParser.parse errored, but Meta.parse didn't." file = file + elseif !cst_err && meta_err + @error "Meta.parse errored, but CSTParser.parse didn't." file = file + end + else + if cst_expr == meta_expr + @test true + else + @error "parsing difference" file = file + _compare(cst_expr, meta_expr) + @test false + end + end + end + end + end +end diff --git a/test/display.jl b/test/test_display.jl similarity index 91% rename from test/display.jl rename to test/test_display.jl index fcaec9eb..cdfbf487 100644 --- a/test/display.jl +++ b/test/test_display.jl @@ -1,4 +1,4 @@ -@testset "show" begin +@testitem "show" begin x = CSTParser.parse("a + (b*c) - d") @test sprint(show, x) === " 1:13 call\n 1:2 OP: -\n 3:12 call\n 3:4 OP: +\n 5:6 a\n 7:12 brackets\n 7:9 call\n 7:7 OP: *\n 8:8 b\n 9:9 c\n 13:13 d" diff --git a/test/errparse.jl b/test/test_errparse.jl similarity index 90% rename from test/errparse.jl rename to test/test_errparse.jl index ac3fec50..e1b9b4e2 100644 --- a/test/errparse.jl +++ b/test/test_errparse.jl @@ -1,15 +1,15 @@ # This deletes arbitrary tokens from files and checks that we can still parse them # and that iteration functions are still correctly ordered. -@testset "invalid jl file parsing" begin - function trav(x, f = x->nothing) +@testitem "invalid jl file parsing" begin + function trav(x, f=x -> nothing) f(x) for a in x trav(a, f) end end - function trav1(x, f = x->nothing) + function trav1(x, f=x -> nothing) f(x) if x.args !== nothing for a in x @@ -18,7 +18,7 @@ end end - function check_err_parse(s, n = length(s)÷100) + function check_err_parse(s, n=length(s) ÷ 100) check_str(s) # parsing works? check_itr_order(s) # iteration produces same text? @@ -72,7 +72,7 @@ segs end - function check_itr_order(s, x = CSTParser.parse(s, true)) + function check_itr_order(s, x=CSTParser.parse(s, true)) length(x) == 0 && return segs = get_segs(x) s0 = join(String(codeunits(s)[seg]) for seg in segs) @@ -103,7 +103,7 @@ all(comp(x[i], y[i]) for i = 1:length(x)) end - function check_reparse(s0, n = length(s0)÷100) + function check_reparse(s0, n=length(s0) ÷ 100) for _ in 1:n x0 = CSTParser.parse(s0, true) CSTParser.has_error(x0) && return @@ -150,7 +150,7 @@ (!isfile(f) || !endswith(f, ".jl")) && continue @info "checking $(nameof(check)) against $f" s = String(read(f)) - if isvalid(s) && length(s) >0 + if isvalid(s) && length(s) > 0 check(s) end end @@ -158,6 +158,8 @@ true end - @test check_dir("..", check_err_parse) - @test check_dir("..", check_reparse) + if VERSION >= v"1.6" + @test check_dir("..", check_err_parse) + @test check_dir("..", check_reparse) + end end diff --git a/test/interface.jl b/test/test_interface.jl similarity index 96% rename from test/interface.jl rename to test/test_interface.jl index 85e06458..e5d2e353 100644 --- a/test/interface.jl +++ b/test/test_interface.jl @@ -1,5 +1,5 @@ -@testset "function defs" begin +@testitem "function defs" begin @test CSTParser.defines_function(CSTParser.parse("function f end")) @test CSTParser.defines_function(CSTParser.parse("function f() end")) @test CSTParser.defines_function(CSTParser.parse("function f()::T end")) @@ -17,7 +17,7 @@ @test !CSTParser.defines_function(CSTParser.parse("a.b = x")) end -@testset "datatype defs" begin +@testitem "datatype defs" begin @test CSTParser.defines_struct(CSTParser.parse("struct T end")) @test CSTParser.defines_struct(CSTParser.parse("mutable struct T end")) @test CSTParser.defines_mutable(CSTParser.parse("mutable struct T end")) @@ -26,7 +26,9 @@ end @test CSTParser.defines_primitive(CSTParser.parse("primitive type a b end")) end -@testset "get_name" begin +@testitem "get_name" begin + using CSTParser: valof + @test valof(CSTParser.get_name(CSTParser.parse("struct T end"))) == "T" @test valof(CSTParser.get_name(CSTParser.parse("struct T{T} end"))) == "T" @test valof(CSTParser.get_name(CSTParser.parse("struct T <: T end"))) == "T" @@ -65,7 +67,7 @@ end end -# @testset "get_sig_params" begin +# @testitem "get_sig_params" begin # f = x -> CSTParser.str_value.(CSTParser.get_args(CSTParser.parse(x))) # @test f("function f(a) end") == ["a"] # @test f("function f(a::T) end") == ["a"] @@ -88,7 +90,7 @@ end # end") == ["a", "b"] # end -@testset "has_error" begin +@testitem "has_error" begin # Just an error token @test CSTParser.has_error(CSTParser.parse(",")) # A nested ErrorToken diff --git a/test/spec.jl b/test/test_spec.jl similarity index 68% rename from test/spec.jl rename to test/test_spec.jl index 59d2d3d2..28354249 100644 --- a/test/spec.jl +++ b/test/test_spec.jl @@ -1,51 +1,12 @@ -using CSTParser: @cst_str, headof, parentof, check_span, EXPR, to_codeobject -jl_parse(s) = CSTParser.remlineinfo!(Meta.parse(s)) - -function check_parents(x::EXPR) - if x.args isa Vector{EXPR} - for a in x.args - @test a.parent == x - check_parents(a) - end - end - if x.trivia isa Vector{EXPR} - for a in x.trivia - @test a.parent == x - end - end -end - -function test_iter(ex) - total = 0 - for x in ex - @test x isa EXPR - test_iter(x) - total += x.fullspan - end - if length(ex) > 0 - @test total == ex.fullspan - end -end +@testitem ":local" begin + include("shared.jl") -function test_expr(s, head, n, endswithtrivia = false) - x = CSTParser.parse(s) - head === nothing || @test headof(x) === head - @test length(x) === n - @test x.args === nothing || all(x === parentof(a) for a in x.args) - @test x.trivia === nothing || all(x === parentof(a) for a in x.trivia) - @test to_codeobject(x) == jl_parse(s) - @test isempty(check_span(x)) - check_parents(x) - test_iter(x) - @test endswithtrivia ? (x.fullspan-x.span) == (last(x.trivia).fullspan - last(x.trivia).span) : (x.fullspan-x.span) == (last(x.args).fullspan - last(x.args).span) -end - - -@testset ":local" begin test_expr("local a", :local, 2) end -@testset ":global" begin +@testitem ":global" begin + include("shared.jl") + test_expr("global a", :global, 2) test_expr("global a, b", :global, 4) test_expr("global a, b = 2", :global, 2) @@ -53,34 +14,48 @@ end test_expr("global const a = 1, b", :const, 2) end -@testset ":const" begin +@testitem ":const" begin + include("shared.jl") + test_expr("const a = 1", :const, 2) test_expr("const global a = 1", :const, 2) end -@testset ":return" begin +@testitem ":return" begin + include("shared.jl") + test_expr("return a", :return, 2) end -@testset ":abstract" begin +@testitem ":abstract" begin + include("shared.jl") + test_expr("abstract type sig end", :abstract, 4, true) end -@testset ":primitive" begin +@testitem ":primitive" begin + include("shared.jl") + test_expr("primitive type sig spec end", :primitive, 5, true) end -@testset ":call" begin +@testitem ":call" begin + include("shared.jl") + test_expr("f()", :call, 3, true) test_expr("f(a, b)", :call, 6, true) test_expr("a + b", :call, 3, false) end -@testset ":brackets" begin +@testitem ":brackets" begin + include("shared.jl") + test_expr("(a)", :brackets, 3, true) end -@testset ":begin" begin +@testitem ":begin" begin + include("shared.jl") + test_expr("begin end", :block, 2, true) test_expr("begin a end", :block, 3, true) test_expr("quote end", :quote, 3, true) @@ -93,12 +68,16 @@ end test_expr("mutable struct T end", :struct, 6, true) end -@testset ":export" begin +@testitem ":export" begin + include("shared.jl") + test_expr("export a", :export, 2, false) test_expr("export a, b", :export, 4, false) end -@testset ":import" begin +@testitem ":import" begin + include("shared.jl") + test_expr("import a", :import, 2, false) test_expr("import a, b", :import, 4, false) test_expr("import a.b", :import, 2, false) @@ -108,23 +87,31 @@ end test_expr("import a:b.c, d", :import, 2, false) end -@testset ":kw" begin +@testitem ":kw" begin + include("shared.jl") + test_expr("f(a=1)", :call, 4, false) end -@testset ":tuple" begin +@testitem ":tuple" begin + include("shared.jl") + test_expr("a,b ", :tuple, 3, false) test_expr("(a,b) ", :tuple, 5, true) test_expr("a,b,c ", :tuple, 5, false) test_expr("(a,b),(c) ", :tuple, 3, false) end -@testset ":curly" begin +@testitem ":curly" begin + include("shared.jl") + test_expr("x{a}", :curly, 4, true) test_expr("x{a,b}", :curly, 6, true) end -@testset "operators" begin +@testitem "operators" begin + include("shared.jl") + test_expr("!a", :call, 2, false) test_expr("&a", nothing, 2, false) test_expr(":a", :quotenode, 2, false) @@ -155,48 +142,67 @@ end test_expr("(\na +\nb +\nc +\n d\n)", :brackets, 3, true) end -@testset ":parameters" begin +@testitem ":parameters" begin + include("shared.jl") + test_expr("f(a;b = 1)", nothing, 5, true) end -@testset "lists" begin - @testset ":vect" begin - test_expr("[]", :vect, 2, true) - test_expr("[a]", :vect, 3, true) - test_expr("[a, b]", :vect, 5, true) - test_expr("[a ]", :vect, 3, true) - end +@testitem "lists :vect" begin + include("shared.jl") + + test_expr("[]", :vect, 2, true) + test_expr("[a]", :vect, 3, true) + test_expr("[a, b]", :vect, 5, true) + test_expr("[a ]", :vect, 3, true) +end - @testset ":vcat" begin - test_expr("[a\nb]", :vcat, 4, true) - test_expr("[a;b]", :vcat, 4, true) - test_expr("[a b\nc d]", :vcat, 4, true) - test_expr("[a\nc d]", :vcat, 4, true) - test_expr("[a;c d]", :vcat, 4, true) - end +@testitem "lists :vcat" begin + include("shared.jl") + + test_expr("[a\nb]", :vcat, 4, true) + test_expr("[a;b]", :vcat, 4, true) + test_expr("[a b\nc d]", :vcat, 4, true) + test_expr("[a\nc d]", :vcat, 4, true) + test_expr("[a;c d]", :vcat, 4, true) +end - @testset ":hcat" begin - test_expr("[a b]", :hcat, 4, true) - end - @testset ":ref" begin - test_expr("T[a]", :ref, 4, true) - test_expr("T[a,b]", :ref, 6, true) - end - @testset ":typed_hcat" begin - test_expr("T[a b]", :typed_hcat, 5, true) - end - @testset ":typed_vcat" begin - test_expr("T[a;b]", :typed_vcat, 5, true) - end +@testitem "lists :hcat" begin + include("shared.jl") + + test_expr("[a b]", :hcat, 4, true) +end + +@testitem "lists :ref" begin + include("shared.jl") + + test_expr("T[a]", :ref, 4, true) + test_expr("T[a,b]", :ref, 6, true) +end + +@testitem "lists :typed_hcat" begin + include("shared.jl") + + test_expr("T[a b]", :typed_hcat, 5, true) +end + +@testitem "lists :typed_vcat" begin + include("shared.jl") + + test_expr("T[a;b]", :typed_vcat, 5, true) end -@testset ":let" begin +@testitem ":let" begin + include("shared.jl") + test_expr("let\n end", :let, 4, true) test_expr("let x = 1 end", :let, 4, true) test_expr("let x = 1, y =1 end", :let, 4, true) end -@testset ":try" begin +@testitem ":try" begin + include("shared.jl") + test_expr("try catch end", :try, 6, true) test_expr("try a catch end", :try, 6, true) test_expr("try catch e end", :try, 6, true) @@ -208,13 +214,17 @@ end test_expr("try a catch e b finally c end", :try, 8, true) end -@testset ":macrocall" begin +@testitem ":macrocall" begin + include("shared.jl") + test_expr("@m", :macrocall, 2, false) test_expr("@m a", :macrocall, 3, false) test_expr("@m a b", :macrocall, 4, false) end -@testset ":if" begin +@testitem ":if" begin + include("shared.jl") + test_expr("if c end", :if, 4, true) test_expr("if c a end", :if, 4, true) test_expr("if c a else end", :if, 6, true) @@ -224,35 +234,47 @@ end test_expr("if c a elseif c b else d end", :if, 5, true) end -@testset ":do" begin +@testitem ":do" begin + include("shared.jl") + test_expr("f() do x end", :do, 4, true) test_expr("f() do x,y end", :do, 4, true) end -@testset "strings" begin +@testitem "strings" begin + include("shared.jl") + test_expr("a\"txt\"", :macrocall, 3, false) test_expr("a\"txt\"b", :macrocall, 4, false) end -@testset ":string" begin +@testitem ":string" begin + include("shared.jl") + test_expr("\"\$a\"", :string, 4, false) test_expr("\" \$a\"", :string, 4, false) test_expr("\" \$a \"", :string, 4, false) end -@testset ":for" begin +@testitem ":for" begin + include("shared.jl") + test_expr("for i = I end", :for, 4, true) test_expr("for i in I end", :for, 4, true) test_expr("for i ∈ I end", :for, 4, true) test_expr("for i ∈ I, j in J end", :for, 4, true) end -@testset "docs" begin +@testitem "docs" begin + include("shared.jl") + test_expr("\"doc\"\nT", :macrocall, 4, false) end -@testset ":generator" begin +@testitem ":generator" begin + include("shared.jl") + test_expr("(arg for x in X)", :brackets, 3, true) test_expr("(arg for x in X if x)", :brackets, 3, true) test_expr("(arg for x in X, y in Y)", :brackets, 3, true) @@ -262,7 +284,9 @@ end test_expr("(arg for x in X for y in Y for z in Z)", :brackets, 3, true) end -@testset ":cmd" begin +@testitem ":cmd" begin + include("shared.jl") + let s = "``" x = CSTParser.parse(s) x1 = jl_parse(s) @@ -282,14 +306,18 @@ end test_expr("a`a`", nothing, 3, false) end -@testset ":macrocall" begin +@testitem "macrocall" begin + include("shared.jl") + test_expr("@m", :macrocall, 2, false) test_expr("@m a", :macrocall, 3, false) test_expr("@m(a)", :macrocall, 5, false) test_expr("@horner(r) + r", nothing, 3, false) end -@testset "_str" begin +@testitem "_str" begin + include("shared.jl") + test_expr("a\"txt\"", :macrocall, 3, false) test_expr("a.b\"txt\"", :macrocall, 3, false) end From e85ef5808d7de01b3e146e89980783227f088be0 Mon Sep 17 00:00:00 2001 From: David Anthoff Date: Tue, 25 Jul 2023 15:01:18 +0200 Subject: [PATCH 2/3] Fix bug in test item migration --- test/test_errparse.jl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/test_errparse.jl b/test/test_errparse.jl index e1b9b4e2..fae720c7 100644 --- a/test/test_errparse.jl +++ b/test/test_errparse.jl @@ -2,6 +2,10 @@ # and that iteration functions are still correctly ordered. @testitem "invalid jl file parsing" begin + using CSTParser: to_codeobject + using Tokenize: tokenize + import Tokenize.Tokens: untokenize + function trav(x, f=x -> nothing) f(x) for a in x From b448d4082c6756af6e7036e781f9d022f088c499 Mon Sep 17 00:00:00 2001 From: David Anthoff Date: Tue, 25 Jul 2023 15:48:41 +0200 Subject: [PATCH 3/3] Temporarily remove Julia 1.9 --- .github/workflows/jlpkgbutler-ci-pr-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/jlpkgbutler-ci-pr-workflow.yml b/.github/workflows/jlpkgbutler-ci-pr-workflow.yml index 8793bf55..9114217f 100644 --- a/.github/workflows/jlpkgbutler-ci-pr-workflow.yml +++ b/.github/workflows/jlpkgbutler-ci-pr-workflow.yml @@ -9,7 +9,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - julia-version: ['1.0', '1.1', '1.2', '1.3', '1.4', '1.5', '1.6', '1.7', '1.8', '1.9'] + julia-version: ['1.0', '1.1', '1.2', '1.3', '1.4', '1.5', '1.6', '1.7', '1.8'] julia-arch: [x64, x86] os: [ubuntu-latest, windows-latest, macOS-latest] exclude: