From 0f2fd3f1f32ebd5e5e322b3445029f645e450c82 Mon Sep 17 00:00:00 2001 From: "Jack Liu S. R" Date: Fri, 10 Sep 2021 18:00:38 +0800 Subject: [PATCH] use pygments 2.10 --- .../libs/pygments/__init__.py | 18 +- .../libs/pygments/__main__.py | 17 + .../libs/pygments/cmdline.py | 404 ++- .../libs/pygments/console.py | 16 +- .../libs/pygments/filter.py | 11 +- .../libs/pygments/filters/__init__.py | 631 +++- .../libs/pygments/formatter.py | 9 +- .../libs/pygments/formatters/__init__.py | 20 +- .../libs/pygments/formatters/_mapping.py | 6 +- .../libs/pygments/formatters/bbcode.py | 3 +- .../libs/pygments/formatters/html.py | 406 ++- .../libs/pygments/formatters/img.py | 141 +- .../libs/pygments/formatters/irc.py | 79 +- .../libs/pygments/formatters/latex.py | 95 +- .../libs/pygments/formatters/other.py | 35 +- .../libs/pygments/formatters/pangomarkup.py | 83 + .../libs/pygments/formatters/rtf.py | 59 +- .../libs/pygments/formatters/svg.py | 45 +- .../libs/pygments/formatters/terminal.py | 57 +- .../libs/pygments/formatters/terminal256.py | 59 +- .../libs/pygments/lexer.py | 91 +- .../libs/pygments/lexers/__init__.py | 46 +- .../libs/pygments/lexers/_asy_builtins.py | 11 +- .../libs/pygments/lexers/_cl_builtins.py | 31 +- .../libs/pygments/lexers/_cocoa_builtins.py | 39 +- .../libs/pygments/lexers/_csound_builtins.py | 3097 ++++++++++------- .../libs/pygments/lexers/_julia_builtins.py | 401 +++ .../libs/pygments/lexers/_lasso_builtins.py | 3 +- .../libs/pygments/lexers/_lua_builtins.py | 7 +- .../libs/pygments/lexers/_mapping.py | 212 +- .../libs/pygments/lexers/_mql_builtins.py | 5 +- .../libs/pygments/lexers/_mysql_builtins.py | 1281 +++++++ .../pygments/lexers/_openedge_builtins.py | 825 +++-- .../libs/pygments/lexers/_php_builtins.py | 38 +- .../pygments/lexers/_postgres_builtins.py | 90 +- .../libs/pygments/lexers/_scilab_builtins.py | 5 +- .../pygments/lexers/_sourcemod_builtins.py | 5 +- .../libs/pygments/lexers/_stan_builtins.py | 277 +- .../libs/pygments/lexers/_stata_builtins.py | 154 +- .../libs/pygments/lexers/_tsql_builtins.py | 3 +- .../libs/pygments/lexers/_usd_builtins.py | 112 + .../pygments/lexers/_vbscript_builtins.py | 279 ++ .../libs/pygments/lexers/_vim_builtins.py | 3 +- .../libs/pygments/lexers/actionscript.py | 28 +- .../libs/pygments/lexers/agile.py | 3 +- .../libs/pygments/lexers/algebra.py | 22 +- .../libs/pygments/lexers/ambient.py | 7 +- .../libs/pygments/lexers/amdgpu.py | 53 + .../libs/pygments/lexers/ampl.py | 7 +- .../libs/pygments/lexers/apdlexer.py | 448 +++ .../libs/pygments/lexers/apl.py | 38 +- .../libs/pygments/lexers/archetype.py | 9 +- .../libs/pygments/lexers/arrow.py | 116 + .../libs/pygments/lexers/asc.py | 51 + .../libs/pygments/lexers/asm.py | 547 ++- .../libs/pygments/lexers/automation.py | 15 +- .../libs/pygments/lexers/bare.py | 103 + .../libs/pygments/lexers/basic.py | 175 +- .../libs/pygments/lexers/bibtex.py | 15 +- .../libs/pygments/lexers/boa.py | 101 + .../libs/pygments/lexers/business.py | 24 +- .../libs/pygments/lexers/c_cpp.py | 237 +- .../libs/pygments/lexers/c_like.py | 213 +- .../libs/pygments/lexers/capnproto.py | 15 +- .../libs/pygments/lexers/cddl.py | 190 + .../libs/pygments/lexers/chapel.py | 75 +- .../libs/pygments/lexers/clean.py | 380 +- .../libs/pygments/lexers/compiled.py | 3 +- .../libs/pygments/lexers/configs.py | 366 +- .../libs/pygments/lexers/console.py | 3 +- .../libs/pygments/lexers/crystal.py | 135 +- .../libs/pygments/lexers/csound.py | 476 ++- .../libs/pygments/lexers/css.py | 51 +- .../libs/pygments/lexers/d.py | 20 +- .../libs/pygments/lexers/dalvik.py | 3 +- .../libs/pygments/lexers/data.py | 338 +- .../libs/pygments/lexers/devicetree.py | 108 + .../libs/pygments/lexers/diff.py | 9 +- .../libs/pygments/lexers/dotnet.py | 77 +- .../libs/pygments/lexers/dsls.py | 211 +- .../libs/pygments/lexers/dylan.py | 35 +- .../libs/pygments/lexers/ecl.py | 22 +- .../libs/pygments/lexers/eiffel.py | 5 +- .../libs/pygments/lexers/elm.py | 11 +- .../libs/pygments/lexers/email.py | 150 + .../libs/pygments/lexers/erlang.py | 62 +- .../libs/pygments/lexers/esoteric.py | 66 +- .../libs/pygments/lexers/ezhil.py | 35 +- .../libs/pygments/lexers/factor.py | 7 +- .../libs/pygments/lexers/fantom.py | 3 +- .../libs/pygments/lexers/felix.py | 3 +- .../libs/pygments/lexers/floscript.py | 82 + .../libs/pygments/lexers/forth.py | 82 +- .../libs/pygments/lexers/fortran.py | 23 +- .../libs/pygments/lexers/foxpro.py | 5 +- .../libs/pygments/lexers/freefem.py | 897 +++++ .../libs/pygments/lexers/functional.py | 3 +- .../libs/pygments/lexers/futhark.py | 111 + .../libs/pygments/lexers/gcodelexer.py | 36 + .../libs/pygments/lexers/gdscript.py | 345 ++ .../libs/pygments/lexers/go.py | 7 +- .../libs/pygments/lexers/grammar_notation.py | 68 +- .../libs/pygments/lexers/graph.py | 24 +- .../libs/pygments/lexers/graphics.py | 304 +- .../libs/pygments/lexers/graphviz.py | 58 + .../libs/pygments/lexers/gsql.py | 92 + .../libs/pygments/lexers/haskell.py | 78 +- .../libs/pygments/lexers/haxe.py | 19 +- .../libs/pygments/lexers/hdl.py | 266 +- .../libs/pygments/lexers/hexdump.py | 3 +- .../libs/pygments/lexers/html.py | 63 +- .../libs/pygments/lexers/idl.py | 30 +- .../libs/pygments/lexers/igor.py | 357 +- .../libs/pygments/lexers/inferno.py | 5 +- .../libs/pygments/lexers/installers.py | 7 +- .../libs/pygments/lexers/int_fiction.py | 56 +- .../libs/pygments/lexers/iolang.py | 7 +- .../libs/pygments/lexers/j.py | 9 +- .../libs/pygments/lexers/javascript.py | 413 ++- .../libs/pygments/lexers/jslt.py | 94 + .../libs/pygments/lexers/julia.py | 335 +- .../libs/pygments/lexers/jvm.py | 740 ++-- .../libs/pygments/lexers/kuin.py | 299 ++ .../libs/pygments/lexers/lisp.py | 1702 ++++----- .../libs/pygments/lexers/make.py | 21 +- .../libs/pygments/lexers/markup.py | 261 +- .../libs/pygments/lexers/math.py | 3 +- .../libs/pygments/lexers/matlab.py | 2820 ++++++++++++++- .../libs/pygments/lexers/meson.py | 155 + .../libs/pygments/lexers/mime.py | 210 ++ .../libs/pygments/lexers/ml.py | 218 +- .../libs/pygments/lexers/modeling.py | 45 +- .../libs/pygments/lexers/modula2.py | 36 +- .../libs/pygments/lexers/monte.py | 3 +- .../libs/pygments/lexers/mosel.py | 447 +++ .../libs/pygments/lexers/ncl.py | 3 +- .../libs/pygments/lexers/nimrod.py | 11 +- .../libs/pygments/lexers/nit.py | 3 +- .../libs/pygments/lexers/nix.py | 3 +- .../libs/pygments/lexers/oberon.py | 19 +- .../libs/pygments/lexers/objective.py | 43 +- .../libs/pygments/lexers/ooc.py | 3 +- .../libs/pygments/lexers/other.py | 3 +- .../libs/pygments/lexers/parasail.py | 3 +- .../libs/pygments/lexers/parsers.py | 132 +- .../libs/pygments/lexers/pascal.py | 45 +- .../libs/pygments/lexers/pawn.py | 59 +- .../libs/pygments/lexers/perl.py | 413 ++- .../libs/pygments/lexers/php.py | 79 +- .../libs/pygments/lexers/pointless.py | 70 + .../libs/pygments/lexers/pony.py | 93 + .../libs/pygments/lexers/praat.py | 69 +- .../libs/pygments/lexers/procfile.py | 43 + .../libs/pygments/lexers/prolog.py | 77 +- .../libs/pygments/lexers/promql.py | 182 + .../libs/pygments/lexers/python.py | 701 ++-- .../libs/pygments/lexers/qvt.py | 13 +- .../libs/pygments/lexers/r.py | 289 +- .../libs/pygments/lexers/rdf.py | 272 +- .../libs/pygments/lexers/rebol.py | 17 +- .../libs/pygments/lexers/resource.py | 7 +- .../libs/pygments/lexers/ride.py | 138 + .../libs/pygments/lexers/rnc.py | 5 +- .../libs/pygments/lexers/roboconf.py | 3 +- .../libs/pygments/lexers/robotframework.py | 55 +- .../libs/pygments/lexers/ruby.py | 98 +- .../libs/pygments/lexers/rust.py | 168 +- .../libs/pygments/lexers/sas.py | 3 +- .../libs/pygments/lexers/scdoc.py | 82 + .../libs/pygments/lexers/scripting.py | 105 +- .../libs/pygments/lexers/sgf.py | 60 + .../libs/pygments/lexers/shell.py | 230 +- .../libs/pygments/lexers/sieve.py | 68 + .../libs/pygments/lexers/slash.py | 184 + .../libs/pygments/lexers/smalltalk.py | 5 +- .../libs/pygments/lexers/smithy.py | 79 + .../libs/pygments/lexers/smv.py | 9 +- .../libs/pygments/lexers/snobol.py | 3 +- .../libs/pygments/lexers/solidity.py | 91 + .../libs/pygments/lexers/special.py | 76 +- .../libs/pygments/lexers/sql.py | 509 ++- .../libs/pygments/lexers/stata.py | 152 +- .../libs/pygments/lexers/supercollider.py | 14 +- .../libs/pygments/lexers/tcl.py | 9 +- .../libs/pygments/lexers/teal.py | 88 + .../libs/pygments/lexers/templates.py | 290 +- .../libs/pygments/lexers/teraterm.py | 334 ++ .../libs/pygments/lexers/testing.py | 15 +- .../libs/pygments/lexers/text.py | 4 +- .../libs/pygments/lexers/textedit.py | 13 +- .../libs/pygments/lexers/textfmts.py | 160 +- .../libs/pygments/lexers/theorem.py | 134 +- .../libs/pygments/lexers/thingsdb.py | 118 + .../libs/pygments/lexers/tnt.py | 272 ++ .../libs/pygments/lexers/trafficscript.py | 5 +- .../libs/pygments/lexers/typoscript.py | 16 +- .../libs/pygments/lexers/unicon.py | 411 +++ .../libs/pygments/lexers/urbi.py | 20 +- .../libs/pygments/lexers/usd.py | 89 + .../libs/pygments/lexers/varnish.py | 9 +- .../libs/pygments/lexers/verification.py | 24 +- .../libs/pygments/lexers/web.py | 3 +- .../libs/pygments/lexers/webassembly.py | 120 + .../libs/pygments/lexers/webidl.py | 298 ++ .../libs/pygments/lexers/webmisc.py | 67 +- .../libs/pygments/lexers/whiley.py | 11 +- .../libs/pygments/lexers/x10.py | 5 +- .../libs/pygments/lexers/xorg.py | 36 + .../libs/pygments/lexers/yang.py | 103 + .../libs/pygments/lexers/zig.py | 123 + .../libs/pygments/modeline.py | 3 +- .../libs/pygments/plugin.py | 7 +- .../libs/pygments/regexopt.py | 5 +- .../libs/pygments/scanner.py | 5 +- .../libs/pygments/sphinxext.py | 7 +- .../libs/pygments/style.py | 86 +- .../libs/pygments/styles/__init__.py | 19 +- .../libs/pygments/styles/abap.py | 3 +- .../libs/pygments/styles/algol.py | 3 +- .../libs/pygments/styles/algol_nu.py | 3 +- .../libs/pygments/styles/arduino.py | 5 +- .../libs/pygments/styles/autumn.py | 3 +- .../libs/pygments/styles/borland.py | 3 +- .../libs/pygments/styles/bw.py | 3 +- .../libs/pygments/styles/colorful.py | 3 +- .../libs/pygments/styles/default.py | 3 +- .../libs/pygments/styles/emacs.py | 3 +- .../libs/pygments/styles/friendly.py | 4 +- .../libs/pygments/styles/fruity.py | 3 +- .../libs/pygments/styles/gruvbox.py | 107 + .../libs/pygments/styles/igor.py | 3 +- .../libs/pygments/styles/inkpot.py | 66 + .../libs/pygments/styles/lovelace.py | 3 +- .../libs/pygments/styles/manni.py | 3 +- .../libs/pygments/styles/material.py | 118 + .../libs/pygments/styles/monokai.py | 8 +- .../libs/pygments/styles/murphy.py | 3 +- .../libs/pygments/styles/native.py | 4 +- .../libs/pygments/styles/paraiso_dark.py | 3 +- .../libs/pygments/styles/paraiso_light.py | 3 +- .../libs/pygments/styles/pastie.py | 3 +- .../libs/pygments/styles/perldoc.py | 3 +- .../libs/pygments/styles/rainbow_dash.py | 3 +- .../libs/pygments/styles/rrt.py | 3 +- .../libs/pygments/styles/sas.py | 3 +- .../libs/pygments/styles/solarized.py | 136 + .../libs/pygments/styles/stata_dark.py | 40 + .../styles/{stata.py => stata_light.py} | 30 +- .../libs/pygments/styles/tango.py | 3 +- .../libs/pygments/styles/trac.py | 3 +- .../libs/pygments/styles/vim.py | 3 +- .../libs/pygments/styles/vs.py | 3 +- .../libs/pygments/styles/xcode.py | 3 +- .../libs/pygments/styles/zenburn.py | 80 + .../libs/pygments/token.py | 3 +- .../libs/pygments/unistring.py | 156 +- src/syntax_highlighting/libs/pygments/util.py | 112 +- src/syntax_highlighting/main.py | 4 +- 258 files changed, 25580 insertions(+), 8188 deletions(-) create mode 100644 src/syntax_highlighting/libs/pygments/__main__.py mode change 100644 => 100755 src/syntax_highlighting/libs/pygments/formatters/_mapping.py create mode 100644 src/syntax_highlighting/libs/pygments/formatters/pangomarkup.py create mode 100644 src/syntax_highlighting/libs/pygments/lexers/_julia_builtins.py create mode 100644 src/syntax_highlighting/libs/pygments/lexers/_mysql_builtins.py create mode 100644 src/syntax_highlighting/libs/pygments/lexers/_usd_builtins.py create mode 100644 src/syntax_highlighting/libs/pygments/lexers/_vbscript_builtins.py create mode 100644 src/syntax_highlighting/libs/pygments/lexers/amdgpu.py create mode 100644 src/syntax_highlighting/libs/pygments/lexers/apdlexer.py create mode 100644 src/syntax_highlighting/libs/pygments/lexers/arrow.py create mode 100644 src/syntax_highlighting/libs/pygments/lexers/asc.py create mode 100644 src/syntax_highlighting/libs/pygments/lexers/bare.py create mode 100644 src/syntax_highlighting/libs/pygments/lexers/boa.py create mode 100644 src/syntax_highlighting/libs/pygments/lexers/cddl.py create mode 100644 src/syntax_highlighting/libs/pygments/lexers/devicetree.py create mode 100644 src/syntax_highlighting/libs/pygments/lexers/email.py create mode 100644 src/syntax_highlighting/libs/pygments/lexers/floscript.py create mode 100644 src/syntax_highlighting/libs/pygments/lexers/freefem.py create mode 100644 src/syntax_highlighting/libs/pygments/lexers/futhark.py create mode 100644 src/syntax_highlighting/libs/pygments/lexers/gcodelexer.py create mode 100644 src/syntax_highlighting/libs/pygments/lexers/gdscript.py create mode 100644 src/syntax_highlighting/libs/pygments/lexers/graphviz.py create mode 100755 src/syntax_highlighting/libs/pygments/lexers/gsql.py create mode 100644 src/syntax_highlighting/libs/pygments/lexers/jslt.py create mode 100644 src/syntax_highlighting/libs/pygments/lexers/kuin.py create mode 100644 src/syntax_highlighting/libs/pygments/lexers/meson.py create mode 100644 src/syntax_highlighting/libs/pygments/lexers/mime.py create mode 100644 src/syntax_highlighting/libs/pygments/lexers/mosel.py create mode 100644 src/syntax_highlighting/libs/pygments/lexers/pointless.py create mode 100644 src/syntax_highlighting/libs/pygments/lexers/pony.py create mode 100644 src/syntax_highlighting/libs/pygments/lexers/procfile.py create mode 100644 src/syntax_highlighting/libs/pygments/lexers/promql.py create mode 100644 src/syntax_highlighting/libs/pygments/lexers/ride.py create mode 100644 src/syntax_highlighting/libs/pygments/lexers/scdoc.py create mode 100644 src/syntax_highlighting/libs/pygments/lexers/sgf.py create mode 100644 src/syntax_highlighting/libs/pygments/lexers/sieve.py create mode 100644 src/syntax_highlighting/libs/pygments/lexers/slash.py create mode 100644 src/syntax_highlighting/libs/pygments/lexers/smithy.py create mode 100644 src/syntax_highlighting/libs/pygments/lexers/solidity.py create mode 100644 src/syntax_highlighting/libs/pygments/lexers/teal.py create mode 100644 src/syntax_highlighting/libs/pygments/lexers/teraterm.py create mode 100644 src/syntax_highlighting/libs/pygments/lexers/thingsdb.py create mode 100644 src/syntax_highlighting/libs/pygments/lexers/tnt.py create mode 100644 src/syntax_highlighting/libs/pygments/lexers/unicon.py create mode 100644 src/syntax_highlighting/libs/pygments/lexers/usd.py create mode 100644 src/syntax_highlighting/libs/pygments/lexers/webassembly.py create mode 100644 src/syntax_highlighting/libs/pygments/lexers/webidl.py create mode 100644 src/syntax_highlighting/libs/pygments/lexers/xorg.py create mode 100644 src/syntax_highlighting/libs/pygments/lexers/yang.py create mode 100644 src/syntax_highlighting/libs/pygments/lexers/zig.py create mode 100644 src/syntax_highlighting/libs/pygments/styles/gruvbox.py create mode 100644 src/syntax_highlighting/libs/pygments/styles/inkpot.py create mode 100644 src/syntax_highlighting/libs/pygments/styles/material.py create mode 100644 src/syntax_highlighting/libs/pygments/styles/solarized.py create mode 100644 src/syntax_highlighting/libs/pygments/styles/stata_dark.py rename src/syntax_highlighting/libs/pygments/styles/{stata.py => stata_light.py} (52%) create mode 100644 src/syntax_highlighting/libs/pygments/styles/zenburn.py diff --git a/src/syntax_highlighting/libs/pygments/__init__.py b/src/syntax_highlighting/libs/pygments/__init__.py index 394a85f..4ffb19f 100644 --- a/src/syntax_highlighting/libs/pygments/__init__.py +++ b/src/syntax_highlighting/libs/pygments/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Pygments ~~~~~~~~ @@ -17,19 +16,18 @@ * it is usable as a command-line tool and as a library * ... and it highlights even Brainfuck! - The `Pygments tip`_ is installable with ``easy_install Pygments==dev``. + The `Pygments master branch`_ is installable with ``easy_install Pygments==dev``. - .. _Pygments tip: - http://bitbucket.org/birkenfeld/pygments-main/get/tip.zip#egg=Pygments-dev + .. _Pygments master branch: + https://github.com/pygments/pygments/archive/master.zip#egg=Pygments-dev - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ import sys +from io import StringIO, BytesIO -from pygments.util import StringIO, BytesIO - -__version__ = '2.2.0' +__version__ = '2.10.0' __docformat__ = 'restructuredtext' __all__ = ['lex', 'format', 'highlight'] @@ -84,7 +82,3 @@ def highlight(code, lexer, formatter, outfile=None): """ return format(lex(code, lexer), formatter, outfile) - -if __name__ == '__main__': # pragma: no cover - from pygments.cmdline import main - sys.exit(main(sys.argv)) diff --git a/src/syntax_highlighting/libs/pygments/__main__.py b/src/syntax_highlighting/libs/pygments/__main__.py new file mode 100644 index 0000000..c6e2517 --- /dev/null +++ b/src/syntax_highlighting/libs/pygments/__main__.py @@ -0,0 +1,17 @@ +""" + pygments.__main__ + ~~~~~~~~~~~~~~~~~ + + Main entry point for ``python -m pygments``. + + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import sys +import pygments.cmdline + +try: + sys.exit(pygments.cmdline.main(sys.argv)) +except KeyboardInterrupt: + sys.exit(1) diff --git a/src/syntax_highlighting/libs/pygments/cmdline.py b/src/syntax_highlighting/libs/pygments/cmdline.py index 5e1f39e..db6de0c 100644 --- a/src/syntax_highlighting/libs/pygments/cmdline.py +++ b/src/syntax_highlighting/libs/pygments/cmdline.py @@ -1,23 +1,23 @@ -# -*- coding: utf-8 -*- """ pygments.cmdline ~~~~~~~~~~~~~~~~ Command line interface. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ -from __future__ import print_function - +import os import sys -import getopt +import shutil +import argparse from textwrap import dedent from pygments import __version__, highlight from pygments.util import ClassNotFound, OptionError, docstring_headline, \ - guess_decode, guess_decode_from_terminal, terminal_encoding + guess_decode, guess_decode_from_terminal, terminal_encoding, \ + UnclosingTextIOWrapper from pygments.lexers import get_all_lexers, get_lexer_by_name, guess_lexer, \ load_lexer_from_file, get_lexer_for_filename, find_lexer_class_for_filename from pygments.lexers.special import TextLexer @@ -25,88 +25,11 @@ from pygments.formatters import get_all_formatters, get_formatter_by_name, \ load_formatter_from_file, get_formatter_for_filename, find_formatter_class from pygments.formatters.terminal import TerminalFormatter +from pygments.formatters.terminal256 import Terminal256Formatter from pygments.filters import get_all_filters, find_filter_class from pygments.styles import get_all_styles, get_style_by_name -USAGE = """\ -Usage: %s [-l | -g] [-F [:]] [-f ] - [-O ] [-P ] [-s] [-v] [-x] [-o ] [] - - %s -S $)', _handle_cssblock), + include('keywords'), include('inline'), ], + 'keywords': [ + (words(( + '\\define', '\\end', 'caption', 'created', 'modified', 'tags', + 'title', 'type'), prefix=r'^', suffix=r'\b'), + Keyword), + ], 'inline': [ # escape (r'\\.', Text), + # created or modified date + (r'\d{17}', Number.Integer), # italics - (r'(\s)([*_][^*_]+[*_])(\W|\n)', bygroups(Text, Generic.Emph, Text)), + (r'(\s)(//[^/]+//)((?=\W|\n))', + bygroups(Text, Generic.Emph, Text)), + # superscript + (r'(\s)(\^\^[^\^]+\^\^)', bygroups(Text, Generic.Emph)), + # subscript + (r'(\s)(,,[^,]+,,)', bygroups(Text, Generic.Emph)), + # underscore + (r'(\s)(__[^_]+__)', bygroups(Text, Generic.Strong)), # bold - # warning: the following rule eats internal tags. eg. **foo _bar_ baz** bar is not italics - (r'(\s)((\*\*|__).*\3)((?=\W|\n))', bygroups(Text, Generic.Strong, None, Text)), - # "proper way" (r'(\s)([*_]{2}[^*_]+[*_]{2})((?=\W|\n))', bygroups(Text, Generic.Strong, Text)), + (r"(\s)(''[^']+'')((?=\W|\n))", + bygroups(Text, Generic.Strong, Text)), # strikethrough - (r'(\s)(~~[^~]+~~)((?=\W|\n))', bygroups(Text, Generic.Deleted, Text)), + (r'(\s)(~~[^~]+~~)((?=\W|\n))', + bygroups(Text, Generic.Deleted, Text)), + # TiddlyWiki variables + (r'<<[^>]+>>', Name.Tag), + (r'\$\$[^$]+\$\$', Name.Tag), + (r'\$\([^)]+\)\$', Name.Tag), + # TiddlyWiki style or class + (r'^@@.*$', Name.Tag), + # HTML tags + (r']+>', Name.Tag), # inline code (r'`[^`]+`', String.Backtick), - # mentions and topics (twitter and github stuff) - (r'[@#][\w/:]+', Name.Entity), - # (image?) links eg: ![Image of Yaktocat](https://octodex.github.com/images/yaktocat.png) - (r'(!?\[)([^]]+)(\])(\()([^)]+)(\))', bygroups(Text, Name.Tag, Text, Text, Name.Attribute, Text)), + # HTML escaped symbols + (r'&\S*?;', String.Regex), + # Wiki links + (r'(\[{2})([^]\|]+)(\]{2})', bygroups(Text, Name.Tag, Text)), + # External links + (r'(\[{2})([^]\|]+)(\|)([^]\|]+)(\]{2})', + bygroups(Text, Name.Tag, Text, Name.Attribute, Text)), + # Transclusion + (r'(\{{2})([^}]+)(\}{2})', bygroups(Text, Name.Tag, Text)), + # URLs + (r'(\b.?.?tps?://[^\s"]+)', bygroups(Name.Attribute)), # general text, must come last! - (r'[^\\\s]+', Text), - (r'.', Text), + (r'[\w]+', Text), + (r'.', Text) ], } diff --git a/src/syntax_highlighting/libs/pygments/lexers/math.py b/src/syntax_highlighting/libs/pygments/lexers/math.py index ea0ebee..88f810e 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/math.py +++ b/src/syntax_highlighting/libs/pygments/lexers/math.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.math ~~~~~~~~~~~~~~~~~~~~ Just export lexers that were contained in this module. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/src/syntax_highlighting/libs/pygments/lexers/matlab.py b/src/syntax_highlighting/libs/pygments/lexers/matlab.py index 56a0f6d..e12303d 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/matlab.py +++ b/src/syntax_highlighting/libs/pygments/lexers/matlab.py @@ -1,17 +1,17 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.matlab ~~~~~~~~~~~~~~~~~~~~~~ Lexers for Matlab and related languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ import re -from pygments.lexer import Lexer, RegexLexer, bygroups, words, do_insertions +from pygments.lexer import Lexer, RegexLexer, bygroups, default, words, \ + do_insertions, include from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ Number, Punctuation, Generic, Whitespace @@ -31,73 +31,20 @@ class MatlabLexer(RegexLexer): filenames = ['*.m'] mimetypes = ['text/matlab'] - # - # These lists are generated automatically. - # Run the following in bash shell: - # - # for f in elfun specfun elmat; do - # echo -n "$f = " - # matlab -nojvm -r "help $f;exit;" | perl -ne \ - # 'push(@c,$1) if /^ (\w+)\s+-/; END {print q{["}.join(q{","},@c).qq{"]\n};}' - # done - # - # elfun: Elementary math functions - # specfun: Special Math functions - # elmat: Elementary matrices and matrix manipulation - # - # taken from Matlab version 7.4.0.336 (R2007a) - # - elfun = ("sin", "sind", "sinh", "asin", "asind", "asinh", "cos", "cosd", "cosh", - "acos", "acosd", "acosh", "tan", "tand", "tanh", "atan", "atand", "atan2", - "atanh", "sec", "secd", "sech", "asec", "asecd", "asech", "csc", "cscd", - "csch", "acsc", "acscd", "acsch", "cot", "cotd", "coth", "acot", "acotd", - "acoth", "hypot", "exp", "expm1", "log", "log1p", "log10", "log2", "pow2", - "realpow", "reallog", "realsqrt", "sqrt", "nthroot", "nextpow2", "abs", - "angle", "complex", "conj", "imag", "real", "unwrap", "isreal", "cplxpair", - "fix", "floor", "ceil", "round", "mod", "rem", "sign") - specfun = ("airy", "besselj", "bessely", "besselh", "besseli", "besselk", "beta", - "betainc", "betaln", "ellipj", "ellipke", "erf", "erfc", "erfcx", - "erfinv", "expint", "gamma", "gammainc", "gammaln", "psi", "legendre", - "cross", "dot", "factor", "isprime", "primes", "gcd", "lcm", "rat", - "rats", "perms", "nchoosek", "factorial", "cart2sph", "cart2pol", - "pol2cart", "sph2cart", "hsv2rgb", "rgb2hsv") - elmat = ("zeros", "ones", "eye", "repmat", "rand", "randn", "linspace", "logspace", - "freqspace", "meshgrid", "accumarray", "size", "length", "ndims", "numel", - "disp", "isempty", "isequal", "isequalwithequalnans", "cat", "reshape", - "diag", "blkdiag", "tril", "triu", "fliplr", "flipud", "flipdim", "rot90", - "find", "end", "sub2ind", "ind2sub", "bsxfun", "ndgrid", "permute", - "ipermute", "shiftdim", "circshift", "squeeze", "isscalar", "isvector", - "ans", "eps", "realmax", "realmin", "pi", "i", "inf", "nan", "isnan", - "isinf", "isfinite", "j", "why", "compan", "gallery", "hadamard", "hankel", - "hilb", "invhilb", "magic", "pascal", "rosser", "toeplitz", "vander", - "wilkinson") + _operators = r'-|==|~=|<=|>=|<|>|&&|&|~|\|\|?|\.\*|\*|\+|\.\^|\.\\|\./|/|\\' tokens = { - 'root': [ - # line starting with '!' is sent as a system command. not sure what - # label to use... - (r'^!.*', String.Other), - (r'%\{\s*\n', Comment.Multiline, 'blockcomment'), - (r'%.*$', Comment), - (r'^\s*function', Keyword, 'deffunc'), - - # from 'iskeyword' on version 7.11 (R2010): - (words(( - 'break', 'case', 'catch', 'classdef', 'continue', 'else', 'elseif', - 'end', 'enumerated', 'events', 'for', 'function', 'global', 'if', - 'methods', 'otherwise', 'parfor', 'persistent', 'properties', - 'return', 'spmd', 'switch', 'try', 'while'), suffix=r'\b'), - Keyword), - - ("(" + "|".join(elfun + specfun + elmat) + r')\b', Name.Builtin), - - # line continuation with following comment: - (r'\.\.\..*$', Comment), - + 'expressions': [ # operators: - (r'-|==|~=|<|>|<=|>=|&&|&|~|\|\|?', Operator), - # operators requiring escape for re: - (r'\.\*|\*|\+|\.\^|\.\\|\.\/|\/|\\', Operator), + (_operators, Operator), + + # numbers (must come before punctuation to handle `.5`; cannot use + # `\b` due to e.g. `5. + .5`). The negative lookahead on operators + # avoids including the dot in `1./x` (the dot is part of `./`). + (r'(? and then + # (equal | open-parenthesis | | ). + (r'(?:^|(?<=;))(\s*)(\w+)(\s+)(?!=|\(|%s\s|\s)' % _operators, + bygroups(Whitespace, Name, Whitespace), 'commandargs'), + + include('expressions') ], 'blockcomment': [ (r'^\s*%\}', Comment.Multiline, '#pop'), @@ -124,19 +2676,62 @@ class MatlabLexer(RegexLexer): (r'.', Comment.Multiline), ], 'deffunc': [ - (r'(\s*)(?:(.+)(\s*)(=)(\s*))?(.+)(\()(.*)(\))(\s*)', + (r'(\s*)(?:(\S+)(\s*)(=)(\s*))?(.+)(\()(.*)(\))(\s*)', bygroups(Whitespace, Text, Whitespace, Punctuation, Whitespace, Name.Function, Punctuation, Text, Punctuation, Whitespace), '#pop'), # function with no args - (r'(\s*)([a-zA-Z_]\w*)', bygroups(Text, Name.Function), '#pop'), + (r'(\s*)([a-zA-Z_]\w*)', + bygroups(Whitespace, Name.Function), '#pop'), + ], + 'propattrs': [ + (r'(\w+)(\s*)(=)(\s*)(\d+)', + bygroups(Name.Builtin, Whitespace, Punctuation, Whitespace, + Number)), + (r'(\w+)(\s*)(=)(\s*)([a-zA-Z]\w*)', + bygroups(Name.Builtin, Whitespace, Punctuation, Whitespace, + Keyword)), + (r',', Punctuation), + (r'\)', Punctuation, '#pop'), + (r'\s+', Whitespace), + (r'.', Text), + ], + 'defprops': [ + (r'%\{\s*\n', Comment.Multiline, 'blockcomment'), + (r'%.*$', Comment), + (r'(?`_ language lexer. + The grammar definition use to transcribe the syntax was retrieved from + https://mesonbuild.com/Syntax.html#grammar for version 0.58 + Some of those definitions are improperly transcribed so the Meson++ + implementation was also checked: https://github.com/dcbaker/meson-plus-plus + + .. versionadded:: 2.10 + """ + + # TODO String interpolation @VARNAME@ inner matches + # TODO keyword_arg: value inner matches + + name = 'Meson' + aliases = ['meson', 'meson.build'] + filenames = ['meson.build', 'meson_options.txt'] + mimetypes = ['text/x-meson'] + + flags = re.MULTILINE | re.UNICODE + + tokens = { + 'root': [ + (r'#.*?$', Comment), + (r"'''.*'''", String.Single), + (r'[1-9][0-9]*', Number.Integer), + (r'0o[0-7]+', Number.Oct), + (r'0x[a-fA-F0-9]+', Number.Hex), + include('string'), + include('keywords'), + include('expr'), + (r'[a-zA-Z_][a-zA-Z_0-9]*', Name), + (r'\s+', Whitespace), + ], + 'string': [ + (r"[']{3}([']{0,2}([^\\']|\\(.|\n)))*[']{3}", String), + (r"'.*?(?', '->', '#', + ':', r'\|', '=', '=>', '->', '#', # Modules ':>', - )) + } - nonid_reserved = set(('(', ')', '[', ']', '{', '}', ',', ';', '...', '_')) + nonid_reserved = {'(', ')', '[', ']', '{', '}', ',', ';', '...', '_'} alphanumid_re = r"[a-zA-Z][\w']*" symbolicid_re = r"[!%&$#+\-/:<=>?@\\~`^|*]+" @@ -142,7 +141,7 @@ def id_callback(self, match): (r'#\s+(%s)' % symbolicid_re, Name.Label), # Some reserved words trigger a special, local lexer state change (r'\b(datatype|abstype)\b(?!\')', Keyword.Reserved, 'dname'), - (r'(?=\b(exception)\b(?!\'))', Text, ('ename')), + (r'\b(exception)\b(?!\')', Keyword.Reserved, 'ename'), (r'\b(functor|include|open|signature|structure)\b(?!\')', Keyword.Reserved, 'sname'), (r'\b(type|eqtype)\b(?!\')', Keyword.Reserved, 'tname'), @@ -315,15 +314,14 @@ def id_callback(self, match): 'ename': [ include('whitespace'), - (r'(exception|and)\b(\s+)(%s)' % alphanumid_re, + (r'(and\b)(\s+)(%s)' % alphanumid_re, bygroups(Keyword.Reserved, Text, Name.Class)), - (r'(exception|and)\b(\s*)(%s)' % symbolicid_re, + (r'(and\b)(\s*)(%s)' % symbolicid_re, bygroups(Keyword.Reserved, Text, Name.Class)), (r'\b(of)\b(?!\')', Keyword.Reserved), + (r'(%s)|(%s)' % (alphanumid_re, symbolicid_re), Name.Class), - include('breakout'), - include('core'), - (r'\S+', Error), + default('#pop'), ], 'datcon': [ @@ -767,3 +765,193 @@ class OpaLexer(RegexLexer): (r'[^\-]+|-', Comment), ], } + + +class ReasonLexer(RegexLexer): + """ + For the ReasonML language (https://reasonml.github.io/). + + .. versionadded:: 2.6 + """ + + name = 'ReasonML' + aliases = ['reasonml', 'reason'] + filenames = ['*.re', '*.rei'] + mimetypes = ['text/x-reasonml'] + + keywords = ( + 'as', 'assert', 'begin', 'class', 'constraint', 'do', 'done', 'downto', + 'else', 'end', 'exception', 'external', 'false', 'for', 'fun', 'esfun', + 'function', 'functor', 'if', 'in', 'include', 'inherit', 'initializer', 'lazy', + 'let', 'switch', 'module', 'pub', 'mutable', 'new', 'nonrec', 'object', 'of', + 'open', 'pri', 'rec', 'sig', 'struct', 'then', 'to', 'true', 'try', + 'type', 'val', 'virtual', 'when', 'while', 'with', + ) + keyopts = ( + '!=', '#', '&', '&&', r'\(', r'\)', r'\*', r'\+', ',', '-', + r'-\.', '=>', r'\.', r'\.\.', r'\.\.\.', ':', '::', ':=', ':>', ';', ';;', '<', + '<-', '=', '>', '>]', r'>\}', r'\?', r'\?\?', r'\[', r'\[<', r'\[>', + r'\[\|', ']', '_', '`', r'\{', r'\{<', r'\|', r'\|\|', r'\|]', r'\}', '~' + ) + + operators = r'[!$%&*+\./:<=>?@^|~-]' + word_operators = ('and', 'asr', 'land', 'lor', 'lsl', 'lsr', 'lxor', 'mod', 'or') + prefix_syms = r'[!?~]' + infix_syms = r'[=<>@^|&+\*/$%-]' + primitives = ('unit', 'int', 'float', 'bool', 'string', 'char', 'list', 'array') + + tokens = { + 'escape-sequence': [ + (r'\\[\\"\'ntbr]', String.Escape), + (r'\\[0-9]{3}', String.Escape), + (r'\\x[0-9a-fA-F]{2}', String.Escape), + ], + 'root': [ + (r'\s+', Text), + (r'false|true|\(\)|\[\]', Name.Builtin.Pseudo), + (r'\b([A-Z][\w\']*)(?=\s*\.)', Name.Namespace, 'dotted'), + (r'\b([A-Z][\w\']*)', Name.Class), + (r'//.*?\n', Comment.Single), + (r'\/\*(?!/)', Comment.Multiline, 'comment'), + (r'\b(%s)\b' % '|'.join(keywords), Keyword), + (r'(%s)' % '|'.join(keyopts[::-1]), Operator.Word), + (r'(%s|%s)?%s' % (infix_syms, prefix_syms, operators), Operator), + (r'\b(%s)\b' % '|'.join(word_operators), Operator.Word), + (r'\b(%s)\b' % '|'.join(primitives), Keyword.Type), + + (r"[^\W\d][\w']*", Name), + + (r'-?\d[\d_]*(.[\d_]*)?([eE][+\-]?\d[\d_]*)', Number.Float), + (r'0[xX][\da-fA-F][\da-fA-F_]*', Number.Hex), + (r'0[oO][0-7][0-7_]*', Number.Oct), + (r'0[bB][01][01_]*', Number.Bin), + (r'\d[\d_]*', Number.Integer), + + (r"'(?:(\\[\\\"'ntbr ])|(\\[0-9]{3})|(\\x[0-9a-fA-F]{2}))'", + String.Char), + (r"'.'", String.Char), + (r"'", Keyword), + + (r'"', String.Double, 'string'), + + (r'[~?][a-z][\w\']*:', Name.Variable), + ], + 'comment': [ + (r'[^/*]+', Comment.Multiline), + (r'\/\*', Comment.Multiline, '#push'), + (r'\*\/', Comment.Multiline, '#pop'), + (r'\*', Comment.Multiline), + ], + 'string': [ + (r'[^\\"]+', String.Double), + include('escape-sequence'), + (r'\\\n', String.Double), + (r'"', String.Double, '#pop'), + ], + 'dotted': [ + (r'\s+', Text), + (r'\.', Punctuation), + (r'[A-Z][\w\']*(?=\s*\.)', Name.Namespace), + (r'[A-Z][\w\']*', Name.Class, '#pop'), + (r'[a-z_][\w\']*', Name, '#pop'), + default('#pop'), + ], + } + + +class FStarLexer(RegexLexer): + """ + For the F* language (https://www.fstar-lang.org/). + .. versionadded:: 2.7 + """ + + name = 'FStar' + aliases = ['fstar'] + filenames = ['*.fst', '*.fsti'] + mimetypes = ['text/x-fstar'] + + keywords = ( + 'abstract', 'attributes', 'noeq', 'unopteq', 'and' + 'begin', 'by', 'default', 'effect', 'else', 'end', 'ensures', + 'exception', 'exists', 'false', 'forall', 'fun', 'function', 'if', + 'in', 'include', 'inline', 'inline_for_extraction', 'irreducible', + 'logic', 'match', 'module', 'mutable', 'new', 'new_effect', 'noextract', + 'of', 'open', 'opaque', 'private', 'range_of', 'reifiable', + 'reify', 'reflectable', 'requires', 'set_range_of', 'sub_effect', + 'synth', 'then', 'total', 'true', 'try', 'type', 'unfold', 'unfoldable', + 'val', 'when', 'with', 'not' + ) + decl_keywords = ('let', 'rec') + assume_keywords = ('assume', 'admit', 'assert', 'calc') + keyopts = ( + r'~', r'-', r'/\\', r'\\/', r'<:', r'<@', r'\(\|', r'\|\)', r'#', r'u#', + r'&', r'\(', r'\)', r'\(\)', r',', r'~>', r'->', r'<-', r'<--', r'<==>', + r'==>', r'\.', r'\?', r'\?\.', r'\.\[', r'\.\(', r'\.\(\|', r'\.\[\|', + r'\{:pattern', r':', r'::', r':=', r';', r';;', r'=', r'%\[', r'!\{', + r'\[', r'\[@', r'\[\|', r'\|>', r'\]', r'\|\]', r'\{', r'\|', r'\}', r'\$' + ) + + operators = r'[!$%&*+\./:<=>?@^|~-]' + prefix_syms = r'[!?~]' + infix_syms = r'[=<>@^|&+\*/$%-]' + primitives = ('unit', 'int', 'float', 'bool', 'string', 'char', 'list', 'array') + + tokens = { + 'escape-sequence': [ + (r'\\[\\"\'ntbr]', String.Escape), + (r'\\[0-9]{3}', String.Escape), + (r'\\x[0-9a-fA-F]{2}', String.Escape), + ], + 'root': [ + (r'\s+', Text), + (r'false|true|False|True|\(\)|\[\]', Name.Builtin.Pseudo), + (r'\b([A-Z][\w\']*)(?=\s*\.)', Name.Namespace, 'dotted'), + (r'\b([A-Z][\w\']*)', Name.Class), + (r'\(\*(?![)])', Comment, 'comment'), + (r'^\/\/.+$', Comment), + (r'\b(%s)\b' % '|'.join(keywords), Keyword), + (r'\b(%s)\b' % '|'.join(assume_keywords), Name.Exception), + (r'\b(%s)\b' % '|'.join(decl_keywords), Keyword.Declaration), + (r'(%s)' % '|'.join(keyopts[::-1]), Operator), + (r'(%s|%s)?%s' % (infix_syms, prefix_syms, operators), Operator), + (r'\b(%s)\b' % '|'.join(primitives), Keyword.Type), + + (r"[^\W\d][\w']*", Name), + + (r'-?\d[\d_]*(.[\d_]*)?([eE][+\-]?\d[\d_]*)', Number.Float), + (r'0[xX][\da-fA-F][\da-fA-F_]*', Number.Hex), + (r'0[oO][0-7][0-7_]*', Number.Oct), + (r'0[bB][01][01_]*', Number.Bin), + (r'\d[\d_]*', Number.Integer), + + (r"'(?:(\\[\\\"'ntbr ])|(\\[0-9]{3})|(\\x[0-9a-fA-F]{2}))'", + String.Char), + (r"'.'", String.Char), + (r"'", Keyword), # a stray quote is another syntax element + (r"\`([\w\'.]+)\`", Operator.Word), # for infix applications + (r"\`", Keyword), # for quoting + (r'"', String.Double, 'string'), + + (r'[~?][a-z][\w\']*:', Name.Variable), + ], + 'comment': [ + (r'[^(*)]+', Comment), + (r'\(\*', Comment, '#push'), + (r'\*\)', Comment, '#pop'), + (r'[(*)]', Comment), + ], + 'string': [ + (r'[^\\"]+', String.Double), + include('escape-sequence'), + (r'\\\n', String.Double), + (r'"', String.Double, '#pop'), + ], + 'dotted': [ + (r'\s+', Text), + (r'\.', Punctuation), + (r'[A-Z][\w\']*(?=\s*\.)', Name.Namespace), + (r'[A-Z][\w\']*', Name.Class, '#pop'), + (r'[a-z_][\w\']*', Name, '#pop'), + default('#pop'), + ], + } diff --git a/src/syntax_highlighting/libs/pygments/lexers/modeling.py b/src/syntax_highlighting/libs/pygments/lexers/modeling.py index b354f1c..b00a7f1 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/modeling.py +++ b/src/syntax_highlighting/libs/pygments/lexers/modeling.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.modeling ~~~~~~~~~~~~~~~~~~~~~~~~ Lexers for modeling languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -13,7 +12,7 @@ from pygments.lexer import RegexLexer, include, bygroups, using, default from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ - Number, Punctuation + Number, Punctuation, Whitespace from pygments.lexers.html import HtmlLexer from pygments.lexers import _stan_builtins @@ -38,7 +37,7 @@ class ModelicaLexer(RegexLexer): tokens = { 'whitespace': [ - (u'[\\s\ufeff]+', Text), + (r'[\s\ufeff]+', Text), (r'//[^\n]*\n?', Comment.Single), (r'/\*.*?\*/', Comment.Multiline) ], @@ -62,8 +61,8 @@ class ModelicaLexer(RegexLexer): r'transpose|vector|zeros)\b', Name.Builtin), (r'(algorithm|annotation|break|connect|constant|constrainedby|der|' r'discrete|each|else|elseif|elsewhen|encapsulated|enumeration|' - r'equation|exit|expandable|extends|external|final|flow|for|if|' - r'import|impure|in|initial|inner|input|loop|nondiscrete|outer|' + r'equation|exit|expandable|extends|external|firstTick|final|flow|for|if|' + r'import|impure|in|initial|inner|input|interval|loop|nondiscrete|outer|' r'output|parameter|partial|protected|public|pure|redeclare|' r'replaceable|return|stream|then|when|while)\b', Keyword.Reserved), @@ -284,8 +283,8 @@ class StanLexer(RegexLexer): """Pygments Lexer for Stan models. The Stan modeling language is specified in the *Stan Modeling Language - User's Guide and Reference Manual, v2.8.0*, - `pdf `__. + User's Guide and Reference Manual, v2.17.0*, + `pdf `__. .. versionadded:: 1.6 """ @@ -316,19 +315,26 @@ class StanLexer(RegexLexer): 'parameters', r'transformed\s+parameters', 'model', r'generated\s+quantities')), bygroups(Keyword.Namespace, Text, Punctuation)), + # target keyword + (r'target\s*\+=', Keyword), # Reserved Words (r'(%s)\b' % r'|'.join(_stan_builtins.KEYWORDS), Keyword), # Truncation (r'T(?=\s*\[)', Keyword), # Data types (r'(%s)\b' % r'|'.join(_stan_builtins.TYPES), Keyword.Type), + # < should be punctuation, but elsewhere I can't tell if it is in + # a range constraint + (r'(<)(\s*)(upper|lower)(\s*)(=)', + bygroups(Operator, Whitespace, Keyword, Whitespace, Punctuation)), + (r'(,)(\s*)(upper)(\s*)(=)', + bygroups(Punctuation, Whitespace, Keyword, Whitespace, Punctuation)), # Punctuation - (r"[;:,\[\]()]", Punctuation), + (r"[;,\[\]()]", Punctuation), # Builtin - (r'(%s)(?=\s*\()' - % r'|'.join(_stan_builtins.FUNCTIONS - + _stan_builtins.DISTRIBUTIONS), - Name.Builtin), + (r'(%s)(?=\s*\()' % '|'.join(_stan_builtins.FUNCTIONS), Name.Builtin), + (r'(~)(\s*)(%s)(?=\s*\()' % '|'.join(_stan_builtins.DISTRIBUTIONS), + bygroups(Operator, Whitespace, Name.Builtin)), # Special names ending in __, like lp__ (r'[A-Za-z]\w*__\b', Name.Builtin.Pseudo), (r'(%s)\b' % r'|'.join(_stan_builtins.RESERVED), Keyword.Reserved), @@ -337,17 +343,18 @@ class StanLexer(RegexLexer): # Regular variable names (r'[A-Za-z]\w*\b', Name), # Real Literals - (r'-?[0-9]+(\.[0-9]+)?[eE]-?[0-9]+', Number.Float), - (r'-?[0-9]*\.[0-9]*', Number.Float), + (r'[0-9]+(\.[0-9]*)?([eE][+-]?[0-9]+)?', Number.Float), + (r'\.[0-9]+([eE][+-]?[0-9]+)?', Number.Float), # Integer Literals - (r'-?[0-9]+', Number.Integer), + (r'[0-9]+', Number.Integer), # Assignment operators - # SLexer makes these tokens Operators. - (r'<-|~', Operator), + (r'<-|(?:\+|-|\.?/|\.?\*|=)?=|~', Operator), # Infix, prefix and postfix operators (and = ) - (r"\+|-|\.?\*|\.?/|\\|'|\^|==?|!=?|<=?|>=?|\|\||&&", Operator), + (r"\+|-|\.?\*|\.?/|\\|'|\^|!=?|<=?|>=?|\|\||&&|%|\?|:", Operator), # Block delimiters (r'[{}]', Punctuation), + # Distribution | + (r'\|', Punctuation) ] } diff --git a/src/syntax_highlighting/libs/pygments/lexers/modula2.py b/src/syntax_highlighting/libs/pygments/lexers/modula2.py index c0a69b4..cad2f4f 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/modula2.py +++ b/src/syntax_highlighting/libs/pygments/lexers/modula2.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.modula2 ~~~~~~~~~~~~~~~~~~~~~~~ Multi-Dialect Lexer for Modula-2. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -228,8 +227,8 @@ class Modula2Lexer(RegexLexer): (r'[0-9A-F]+H', Number.Hex), ], 'string_literals': [ - (r"'(\\\\|\\'|[^'])*'", String), # single quoted string - (r'"(\\\\|\\"|[^"])*"', String), # double quoted string + (r'"(\\\\|\\[^\\]|[^"\\])*"', String.Double), + (r"'(\\\\|\\[^\\]|[^'\\])*'", String.Single), ], 'digraph_operators': [ # Dot Product Operator @@ -1547,15 +1546,34 @@ def get_tokens_unprocessed(self, text): # substitute lexemes when in Algol mode if self.algol_publication_mode: if value == '#': - value = u'≠' + value = '≠' elif value == '<=': - value = u'≤' + value = '≤' elif value == '>=': - value = u'≥' + value = '≥' elif value == '==': - value = u'≡' + value = '≡' elif value == '*.': - value = u'•' + value = '•' # return result yield index, token, value + + def analyse_text(text): + """It's Pascal-like, but does not use FUNCTION -- uses PROCEDURE + instead.""" + + # Check if this looks like Pascal, if not, bail out early + if not ('(*' in text and '*)' in text and ':=' in text): + return + + result = 0 + # Procedure is in Modula2 + if re.search(r'\bPROCEDURE\b', text): + result += 0.6 + + # FUNCTION is only valid in Pascal, but not in Modula2 + if re.search(r'\bFUNCTION\b', text): + result = 0.0 + + return result diff --git a/src/syntax_highlighting/libs/pygments/lexers/monte.py b/src/syntax_highlighting/libs/pygments/lexers/monte.py index ed6e20f..4cd8324 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/monte.py +++ b/src/syntax_highlighting/libs/pygments/lexers/monte.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.monte ~~~~~~~~~~~~~~~~~~~~~ Lexer for the Monte programming language. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/src/syntax_highlighting/libs/pygments/lexers/mosel.py b/src/syntax_highlighting/libs/pygments/lexers/mosel.py new file mode 100644 index 0000000..62cb3b4 --- /dev/null +++ b/src/syntax_highlighting/libs/pygments/lexers/mosel.py @@ -0,0 +1,447 @@ +""" + pygments.lexers.mosel + ~~~~~~~~~~~~~~~~~~~~~ + + Lexers for the mosel language. + http://www.fico.com/en/products/fico-xpress-optimization + + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer, words +from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ + Number, Punctuation + +__all__ = ['MoselLexer'] + +FUNCTIONS = ( + # core functions + '_', + 'abs', + 'arctan', + 'asproc', + 'assert', + 'bitflip', + 'bitneg', + 'bitset', + 'bitshift', + 'bittest', + 'bitval', + 'ceil', + 'cos', + 'create', + 'currentdate', + 'currenttime', + 'cutelt', + 'cutfirst', + 'cuthead', + 'cutlast', + 'cuttail', + 'datablock', + 'delcell', + 'exists', + 'exit', + 'exp', + 'exportprob', + 'fclose', + 'fflush', + 'finalize', + 'findfirst', + 'findlast', + 'floor', + 'fopen', + 'fselect', + 'fskipline', + 'fwrite', + 'fwrite_', + 'fwriteln', + 'fwriteln_', + 'getact', + 'getcoeff', + 'getcoeffs', + 'getdual', + 'getelt', + 'getfid', + 'getfirst', + 'getfname', + 'gethead', + 'getlast', + 'getobjval', + 'getparam', + 'getrcost', + 'getreadcnt', + 'getreverse', + 'getsize', + 'getslack', + 'getsol', + 'gettail', + 'gettype', + 'getvars', + 'isdynamic', + 'iseof', + 'isfinite', + 'ishidden', + 'isinf', + 'isnan', + 'isodd', + 'ln', + 'localsetparam', + 'log', + 'makesos1', + 'makesos2', + 'maxlist', + 'memoryuse', + 'minlist', + 'newmuid', + 'publish', + 'random', + 'read', + 'readln', + 'reset', + 'restoreparam', + 'reverse', + 'round', + 'setcoeff', + 'sethidden', + 'setioerr', + 'setmatherr', + 'setname', + 'setparam', + 'setrandseed', + 'setrange', + 'settype', + 'sin', + 'splithead', + 'splittail', + 'sqrt', + 'strfmt', + 'substr', + 'timestamp', + 'unpublish', + 'versionnum', + 'versionstr', + 'write', + 'write_', + 'writeln', + 'writeln_', + + # mosel exam mmxprs | sed -n -e "s/ [pf][a-z]* \([a-zA-Z0-9_]*\).*/'\1',/p" | sort -u + 'addcut', + 'addcuts', + 'addmipsol', + 'basisstability', + 'calcsolinfo', + 'clearmipdir', + 'clearmodcut', + 'command', + 'copysoltoinit', + 'crossoverlpsol', + 'defdelayedrows', + 'defsecurevecs', + 'delcuts', + 'dropcuts', + 'estimatemarginals', + 'fixglobal', + 'flushmsgq', + 'getbstat', + 'getcnlist', + 'getcplist', + 'getdualray', + 'getiis', + 'getiissense', + 'getiistype', + 'getinfcause', + 'getinfeas', + 'getlb', + 'getlct', + 'getleft', + 'getloadedlinctrs', + 'getloadedmpvars', + 'getname', + 'getprimalray', + 'getprobstat', + 'getrange', + 'getright', + 'getsensrng', + 'getsize', + 'getsol', + 'gettype', + 'getub', + 'getvars', + 'gety', + 'hasfeature', + 'implies', + 'indicator', + 'initglobal', + 'ishidden', + 'isiisvalid', + 'isintegral', + 'loadbasis', + 'loadcuts', + 'loadlpsol', + 'loadmipsol', + 'loadprob', + 'maximise', + 'maximize', + 'minimise', + 'minimize', + 'postsolve', + 'readbasis', + 'readdirs', + 'readsol', + 'refinemipsol', + 'rejectintsol', + 'repairinfeas', + 'repairinfeas_deprec', + 'resetbasis', + 'resetiis', + 'resetsol', + 'savebasis', + 'savemipsol', + 'savesol', + 'savestate', + 'selectsol', + 'setarchconsistency', + 'setbstat', + 'setcallback', + 'setcbcutoff', + 'setgndata', + 'sethidden', + 'setlb', + 'setmipdir', + 'setmodcut', + 'setsol', + 'setub', + 'setucbdata', + 'stopoptimise', + 'stopoptimize', + 'storecut', + 'storecuts', + 'unloadprob', + 'uselastbarsol', + 'writebasis', + 'writedirs', + 'writeprob', + 'writesol', + 'xor', + 'xprs_addctr', + 'xprs_addindic', + + # mosel exam mmsystem | sed -n -e "s/ [pf][a-z]* \([a-zA-Z0-9_]*\).*/'\1',/p" | sort -u + 'addmonths', + 'copytext', + 'cuttext', + 'deltext', + 'endswith', + 'erase', + 'expandpath', + 'fcopy', + 'fdelete', + 'findfiles', + 'findtext', + 'fmove', + 'formattext', + 'getasnumber', + 'getchar', + 'getcwd', + 'getdate', + 'getday', + 'getdaynum', + 'getdays', + 'getdirsep', + 'getdsoparam', + 'getendparse', + 'getenv', + 'getfsize', + 'getfstat', + 'getftime', + 'gethour', + 'getminute', + 'getmonth', + 'getmsec', + 'getoserrmsg', + 'getoserror', + 'getpathsep', + 'getqtype', + 'getsecond', + 'getsepchar', + 'getsize', + 'getstart', + 'getsucc', + 'getsysinfo', + 'getsysstat', + 'gettime', + 'gettmpdir', + 'gettrim', + 'getweekday', + 'getyear', + 'inserttext', + 'isvalid', + 'jointext', + 'makedir', + 'makepath', + 'newtar', + 'newzip', + 'nextfield', + 'openpipe', + 'parseextn', + 'parseint', + 'parsereal', + 'parsetext', + 'pastetext', + 'pathmatch', + 'pathsplit', + 'qsort', + 'quote', + 'readtextline', + 'regmatch', + 'regreplace', + 'removedir', + 'removefiles', + 'setchar', + 'setdate', + 'setday', + 'setdsoparam', + 'setendparse', + 'setenv', + 'sethour', + 'setminute', + 'setmonth', + 'setmsec', + 'setoserror', + 'setqtype', + 'setsecond', + 'setsepchar', + 'setstart', + 'setsucc', + 'settime', + 'settrim', + 'setyear', + 'sleep', + 'splittext', + 'startswith', + 'system', + 'tarlist', + 'textfmt', + 'tolower', + 'toupper', + 'trim', + 'untar', + 'unzip', + 'ziplist', + + # mosel exam mmjobs | sed -n -e "s/ [pf][a-z]* \([a-zA-Z0-9_]*\).*/'\1',/p" | sort -u + 'canceltimer', + 'clearaliases', + 'compile', + 'connect', + 'detach', + 'disconnect', + 'dropnextevent', + 'findxsrvs', + 'getaliases', + 'getannidents', + 'getannotations', + 'getbanner', + 'getclass', + 'getdsoprop', + 'getdsopropnum', + 'getexitcode', + 'getfromgid', + 'getfromid', + 'getfromuid', + 'getgid', + 'gethostalias', + 'getid', + 'getmodprop', + 'getmodpropnum', + 'getnextevent', + 'getnode', + 'getrmtid', + 'getstatus', + 'getsysinfo', + 'gettimer', + 'getuid', + 'getvalue', + 'isqueueempty', + 'load', + 'nullevent', + 'peeknextevent', + 'resetmodpar', + 'run', + 'send', + 'setcontrol', + 'setdefstream', + 'setgid', + 'sethostalias', + 'setmodpar', + 'settimer', + 'setuid', + 'setworkdir', + 'stop', + 'unload', + 'wait', + 'waitexpired', + 'waitfor', + 'waitforend', +) + + +class MoselLexer(RegexLexer): + """ + For the Mosel optimization language. + + .. versionadded:: 2.6 + """ + name = 'Mosel' + aliases = ['mosel'] + filenames = ['*.mos'] + + tokens = { + 'root': [ + (r'\n', Text), + (r'\s+', Text.Whitespace), + (r'!.*?\n', Comment.Single), + (r'\(!(.|\n)*?!\)', Comment.Multiline), + (words(( + 'and', 'as', 'break', 'case', 'count', 'declarations', 'do', + 'dynamic', 'elif', 'else', 'end-', 'end', 'evaluation', 'false', + 'forall', 'forward', 'from', 'function', 'hashmap', 'if', + 'imports', 'include', 'initialisations', 'initializations', 'inter', + 'max', 'min', 'model', 'namespace', 'next', 'not', 'nsgroup', + 'nssearch', 'of', 'options', 'or', 'package', 'parameters', + 'procedure', 'public', 'prod', 'record', 'repeat', 'requirements', + 'return', 'sum', 'then', 'to', 'true', 'union', 'until', 'uses', + 'version', 'while', 'with'), prefix=r'\b', suffix=r'\b'), + Keyword.Builtin), + (words(( + 'range', 'array', 'set', 'list', 'mpvar', 'mpproblem', 'linctr', + 'nlctr', 'integer', 'string', 'real', 'boolean', 'text', 'time', + 'date', 'datetime', 'returned', 'Model', 'Mosel', 'counter', + 'xmldoc', 'is_sos1', 'is_sos2', 'is_integer', 'is_binary', + 'is_continuous', 'is_free', 'is_semcont', 'is_semint', + 'is_partint'), prefix=r'\b', suffix=r'\b'), + Keyword.Type), + (r'(\+|\-|\*|/|=|<=|>=|\||\^|<|>|<>|\.\.|\.|:=|::|:|in|mod|div)', + Operator), + (r'[()\[\]{},;]+', Punctuation), + (words(FUNCTIONS, prefix=r'\b', suffix=r'\b'), Name.Function), + (r'(\d+\.(?!\.)\d*|\.(?!.)\d+)([eE][+-]?\d+)?', Number.Float), + (r'\d+([eE][+-]?\d+)?', Number.Integer), + (r'[+-]?Infinity', Number.Integer), + (r'0[xX][0-9a-fA-F]+', Number), + (r'"', String.Double, 'double_quote'), + (r'\'', String.Single, 'single_quote'), + (r'(\w+|(\.(?!\.)))', Text), + ], + 'single_quote': [ + (r'\'', String.Single, '#pop'), + (r'[^\']+', String.Single), + ], + 'double_quote': [ + (r'(\\"|\\[0-7]{1,3}\D|\\[abfnrtv]|\\\\)', String.Escape), + (r'\"', String.Double, '#pop'), + (r'[^"\\]+', String.Double), + ], + } diff --git a/src/syntax_highlighting/libs/pygments/lexers/ncl.py b/src/syntax_highlighting/libs/pygments/lexers/ncl.py index 3ca5135..f9df40b 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/ncl.py +++ b/src/syntax_highlighting/libs/pygments/lexers/ncl.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.ncl ~~~~~~~~~~~~~~~~~~~ Lexers for NCAR Command Language. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/src/syntax_highlighting/libs/pygments/lexers/nimrod.py b/src/syntax_highlighting/libs/pygments/lexers/nimrod.py index d438c1b..ce6ba87 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/nimrod.py +++ b/src/syntax_highlighting/libs/pygments/lexers/nimrod.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.nimrod ~~~~~~~~~~~~~~~~~~~~~~ Lexer for the Nim language (formerly known as Nimrod). - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -26,7 +25,7 @@ class NimrodLexer(RegexLexer): """ name = 'Nimrod' - aliases = ['nim', 'nimrod'] + aliases = ['nimrod', 'nim'] filenames = ['*.nim', '*.nimrod'] mimetypes = ['text/x-nim'] @@ -43,14 +42,14 @@ def underscorize(words): return "|".join(newWords) keywords = [ - 'addr', 'and', 'as', 'asm', 'atomic', 'bind', 'block', 'break', 'case', + 'addr', 'and', 'as', 'asm', 'bind', 'block', 'break', 'case', 'cast', 'concept', 'const', 'continue', 'converter', 'defer', 'discard', 'distinct', 'div', 'do', 'elif', 'else', 'end', 'enum', 'except', 'export', 'finally', 'for', 'func', 'if', 'in', 'yield', 'interface', 'is', 'isnot', 'iterator', 'let', 'macro', 'method', 'mixin', 'mod', 'not', 'notin', 'object', 'of', 'or', 'out', 'proc', 'ptr', 'raise', - 'ref', 'return', 'shared', 'shl', 'shr', 'static', 'template', 'try', - 'tuple', 'type', 'when', 'while', 'with', 'without', 'xor' + 'ref', 'return', 'shl', 'shr', 'static', 'template', 'try', + 'tuple', 'type', 'using', 'when', 'while', 'xor' ] keywordsPseudo = [ diff --git a/src/syntax_highlighting/libs/pygments/lexers/nit.py b/src/syntax_highlighting/libs/pygments/lexers/nit.py index 2111649..7a15d78 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/nit.py +++ b/src/syntax_highlighting/libs/pygments/lexers/nit.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.nit ~~~~~~~~~~~~~~~~~~~ Lexer for the Nit language. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/src/syntax_highlighting/libs/pygments/lexers/nix.py b/src/syntax_highlighting/libs/pygments/lexers/nix.py index e148c91..bd7afe7 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/nix.py +++ b/src/syntax_highlighting/libs/pygments/lexers/nix.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.nix ~~~~~~~~~~~~~~~~~~~ Lexers for the NixOS Nix language. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/src/syntax_highlighting/libs/pygments/lexers/oberon.py b/src/syntax_highlighting/libs/pygments/lexers/oberon.py index 3b5fb3e..7010e91 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/oberon.py +++ b/src/syntax_highlighting/libs/pygments/lexers/oberon.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.oberon ~~~~~~~~~~~~~~~~~~~~~~ Lexers for Oberon family languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -103,3 +102,19 @@ class ComponentPascalLexer(RegexLexer): (r'(TRUE|FALSE|NIL|INF)\b', Keyword.Constant), ] } + + def analyse_text(text): + """The only other lexer using .cp is the C++ one, so we check if for + a few common Pascal keywords here. Those are unfortunately quite + common across various business languages as well.""" + result = 0 + if 'BEGIN' in text: + result += 0.01 + if 'END' in text: + result += 0.01 + if 'PROCEDURE' in text: + result += 0.01 + if 'END' in text: + result += 0.01 + + return result diff --git a/src/syntax_highlighting/libs/pygments/lexers/objective.py b/src/syntax_highlighting/libs/pygments/lexers/objective.py index 7807255..a4cc44b 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/objective.py +++ b/src/syntax_highlighting/libs/pygments/lexers/objective.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.objective ~~~~~~~~~~~~~~~~~~~~~~~~~ Lexers for Objective-C family languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -66,7 +65,7 @@ class GeneratedObjectiveCVariant(baselexer): 'copy', 'retain', 'assign', 'unsafe_unretained', 'atomic', 'nonatomic', 'readonly', 'readwrite', 'setter', 'getter', 'typeof', 'in', 'out', 'inout', 'release', 'class', '@dynamic', '@optional', - '@required', '@autoreleasepool'), suffix=r'\b'), + '@required', '@autoreleasepool', '@import'), suffix=r'\b'), Keyword), (words(('id', 'instancetype', 'Class', 'IMP', 'SEL', 'BOOL', 'IBOutlet', 'IBAction', 'unichar'), suffix=r'\b'), @@ -87,26 +86,26 @@ class GeneratedObjectiveCVariant(baselexer): ], 'oc_classname': [ # interface definition that inherits - ('([a-zA-Z$_][\w$]*)(\s*:\s*)([a-zA-Z$_][\w$]*)?(\s*)(\{)', + (r'([a-zA-Z$_][\w$]*)(\s*:\s*)([a-zA-Z$_][\w$]*)?(\s*)(\{)', bygroups(Name.Class, Text, Name.Class, Text, Punctuation), ('#pop', 'oc_ivars')), - ('([a-zA-Z$_][\w$]*)(\s*:\s*)([a-zA-Z$_][\w$]*)?', + (r'([a-zA-Z$_][\w$]*)(\s*:\s*)([a-zA-Z$_][\w$]*)?', bygroups(Name.Class, Text, Name.Class), '#pop'), # interface definition for a category - ('([a-zA-Z$_][\w$]*)(\s*)(\([a-zA-Z$_][\w$]*\))(\s*)(\{)', + (r'([a-zA-Z$_][\w$]*)(\s*)(\([a-zA-Z$_][\w$]*\))(\s*)(\{)', bygroups(Name.Class, Text, Name.Label, Text, Punctuation), ('#pop', 'oc_ivars')), - ('([a-zA-Z$_][\w$]*)(\s*)(\([a-zA-Z$_][\w$]*\))', + (r'([a-zA-Z$_][\w$]*)(\s*)(\([a-zA-Z$_][\w$]*\))', bygroups(Name.Class, Text, Name.Label), '#pop'), # simple interface / implementation - ('([a-zA-Z$_][\w$]*)(\s*)(\{)', + (r'([a-zA-Z$_][\w$]*)(\s*)(\{)', bygroups(Name.Class, Text, Punctuation), ('#pop', 'oc_ivars')), - ('([a-zA-Z$_][\w$]*)', Name.Class, '#pop') + (r'([a-zA-Z$_][\w$]*)', Name.Class, '#pop') ], 'oc_forward_classname': [ - ('([a-zA-Z$_][\w$]*)(\s*,\s*)', + (r'([a-zA-Z$_][\w$]*)(\s*,\s*)', bygroups(Name.Class, Text), 'oc_forward_classname'), - ('([a-zA-Z$_][\w$]*)(\s*;?)', + (r'([a-zA-Z$_][\w$]*)(\s*;?)', bygroups(Name.Class, Text), '#pop') ], 'oc_ivars': [ @@ -244,28 +243,28 @@ class LogosLexer(ObjectiveCppLexer): inherit, ], 'logos_init_directive': [ - ('\s+', Text), + (r'\s+', Text), (',', Punctuation, ('logos_init_directive', '#pop')), - ('([a-zA-Z$_][\w$]*)(\s*)(=)(\s*)([^);]*)', + (r'([a-zA-Z$_][\w$]*)(\s*)(=)(\s*)([^);]*)', bygroups(Name.Class, Text, Punctuation, Text, Text)), - ('([a-zA-Z$_][\w$]*)', Name.Class), - ('\)', Punctuation, '#pop'), + (r'([a-zA-Z$_][\w$]*)', Name.Class), + (r'\)', Punctuation, '#pop'), ], 'logos_classname': [ - ('([a-zA-Z$_][\w$]*)(\s*:\s*)([a-zA-Z$_][\w$]*)?', + (r'([a-zA-Z$_][\w$]*)(\s*:\s*)([a-zA-Z$_][\w$]*)?', bygroups(Name.Class, Text, Name.Class), '#pop'), - ('([a-zA-Z$_][\w$]*)', Name.Class, '#pop') + (r'([a-zA-Z$_][\w$]*)', Name.Class, '#pop') ], 'root': [ (r'(%subclass)(\s+)', bygroups(Keyword, Text), 'logos_classname'), (r'(%hook|%group)(\s+)([a-zA-Z$_][\w$]+)', bygroups(Keyword, Text, Name.Class)), - (r'(%config)(\s*\(\s*)(\w+)(\s*=\s*)(.*?)(\s*\)\s*)', + (r'(%config)(\s*\(\s*)(\w+)(\s*=)(.*?)(\)\s*)', bygroups(Keyword, Text, Name.Variable, Text, String, Text)), (r'(%ctor)(\s*)(\{)', bygroups(Keyword, Text, Punctuation), 'function'), - (r'(%new)(\s*)(\()(\s*.*?\s*)(\))', + (r'(%new)(\s*)(\()(.*?)(\))', bygroups(Keyword, Text, Keyword, String, Keyword)), (r'(\s*)(%end)(\s*)', bygroups(Text, Keyword, Text)), inherit, @@ -413,7 +412,7 @@ class SwiftLexer(RegexLexer): ], 'keywords': [ (words(( - 'as', 'break', 'case', 'catch', 'continue', 'default', 'defer', + 'as', 'async', 'await', 'break', 'case', 'catch', 'continue', 'default', 'defer', 'do', 'else', 'fallthrough', 'for', 'guard', 'if', 'in', 'is', 'repeat', 'return', '#selector', 'switch', 'throw', 'try', 'where', 'while'), suffix=r'\b'), @@ -441,8 +440,8 @@ class SwiftLexer(RegexLexer): (r'(var|let)(\s+)([a-zA-Z_]\w*)', bygroups(Keyword.Declaration, Text, Name.Variable)), (words(( - 'class', 'deinit', 'enum', 'extension', 'func', 'import', 'init', - 'internal', 'let', 'operator', 'private', 'protocol', 'public', + 'actor', 'associatedtype', 'class', 'deinit', 'enum', 'extension', 'func', 'import', + 'init', 'internal', 'let', 'operator', 'private', 'protocol', 'public', 'static', 'struct', 'subscript', 'typealias', 'var'), suffix=r'\b'), Keyword.Declaration) ], diff --git a/src/syntax_highlighting/libs/pygments/lexers/ooc.py b/src/syntax_highlighting/libs/pygments/lexers/ooc.py index 957b72f..0c74cde 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/ooc.py +++ b/src/syntax_highlighting/libs/pygments/lexers/ooc.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.ooc ~~~~~~~~~~~~~~~~~~~ Lexers for the Ooc language. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/src/syntax_highlighting/libs/pygments/lexers/other.py b/src/syntax_highlighting/libs/pygments/lexers/other.py index bfce4c3..b093008 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/other.py +++ b/src/syntax_highlighting/libs/pygments/lexers/other.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.other ~~~~~~~~~~~~~~~~~~~~~ Just export lexer classes previously contained in this module. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/src/syntax_highlighting/libs/pygments/lexers/parasail.py b/src/syntax_highlighting/libs/pygments/lexers/parasail.py index 5308802..49d8d67 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/parasail.py +++ b/src/syntax_highlighting/libs/pygments/lexers/parasail.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.parasail ~~~~~~~~~~~~~~~~~~~~~~~~ Lexer for ParaSail. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/src/syntax_highlighting/libs/pygments/lexers/parsers.py b/src/syntax_highlighting/libs/pygments/lexers/parsers.py index 1f3c9b4..0009082 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/parsers.py +++ b/src/syntax_highlighting/libs/pygments/lexers/parsers.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.parsers ~~~~~~~~~~~~~~~~~~~~~~~ Lexers for parser generators. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -28,7 +27,6 @@ 'RagelCppLexer', 'RagelObjectiveCLexer', 'RagelRubyLexer', 'RagelJavaLexer', 'AntlrLexer', 'AntlrPythonLexer', 'AntlrPerlLexer', 'AntlrRubyLexer', 'AntlrCppLexer', - # 'AntlrCLexer', 'AntlrCSharpLexer', 'AntlrObjectiveCLexer', 'AntlrJavaLexer', 'AntlrActionScriptLexer', 'TreetopLexer', 'EbnfLexer'] @@ -65,10 +63,10 @@ class RagelLexer(RegexLexer): (r'[+-]?[0-9]+', Number.Integer), ], 'literals': [ - (r'"(\\\\|\\"|[^"])*"', String), # double quote string - (r"'(\\\\|\\'|[^'])*'", String), # single quote string - (r'\[(\\\\|\\\]|[^\]])*\]', String), # square bracket literals - (r'/(?!\*)(\\\\|\\/|[^/])*/', String.Regex), # regular expressions + (r'"(\\\\|\\[^\\]|[^"\\])*"', String.Double), + (r"'(\\\\|\\[^\\]|[^'\\])*'", String.Single), + (r'\[(\\\\|\\[^\\]|[^\\\]])*\]', String), # square bracket literals + (r'/(?!\*)(\\\\|\\[^\\]|[^/\\])*/', String.Regex), # regular expressions ], 'identifiers': [ (r'[a-zA-Z_]\w*', Name.Variable), @@ -107,15 +105,15 @@ class RagelLexer(RegexLexer): r'[^\\]\\[{}]', # allow escaped { or } # strings and comments may safely contain unsafe characters - r'"(\\\\|\\"|[^"])*"', # double quote string - r"'(\\\\|\\'|[^'])*'", # single quote string + r'"(\\\\|\\[^\\]|[^"\\])*"', + r"'(\\\\|\\[^\\]|[^'\\])*'", r'//.*$\n?', # single line comment r'/\*(.|\n)*?\*/', # multi-line javadoc-style comment r'\#.*$\n?', # ruby comment # regular expression: There's no reason for it to start # with a * and this stops confusion with comments. - r'/(?!\*)(\\\\|\\/|[^/])*/', + r'/(?!\*)(\\\\|\\[^\\]|[^/\\])*/', # / is safe now that we've handled regex and javadoc comments r'/', @@ -148,12 +146,12 @@ class RagelEmbeddedLexer(RegexLexer): r'%(?=[^%]|$)', # a single % sign is okay, just not 2 of them # strings and comments may safely contain unsafe characters - r'"(\\\\|\\"|[^"])*"', # double quote string - r"'(\\\\|\\'|[^'])*'", # single quote string + r'"(\\\\|\\[^\\]|[^"\\])*"', + r"'(\\\\|\\[^\\]|[^'\\])*'", r'/\*(.|\n)*?\*/', # multi-line javadoc-style comment r'//.*$\n?', # single line comment r'\#.*$\n?', # ruby/ragel comment - r'/(?!\*)(\\\\|\\/|[^/])*/', # regular expression + r'/(?!\*)(\\\\|\\[^\\]|[^/\\])*/', # regular expression # / is safe now that we've handled regex and javadoc comments r'/', @@ -183,7 +181,7 @@ class RagelEmbeddedLexer(RegexLexer): # specifically allow regex followed immediately by * # so it doesn't get mistaken for a comment - r'/(?!\*)(\\\\|\\/|[^/])*/\*', + r'/(?!\*)(\\\\|\\[^\\]|[^/\\])*/\*', # allow / as long as it's not followed by another / or by a * r'/(?=[^/*]|$)', @@ -194,9 +192,9 @@ class RagelEmbeddedLexer(RegexLexer): )) + r')+', # strings and comments may safely contain unsafe characters - r'"(\\\\|\\"|[^"])*"', # double quote string - r"'(\\\\|\\'|[^'])*'", # single quote string - r"\[(\\\\|\\\]|[^\]])*\]", # square bracket literal + r'"(\\\\|\\[^\\]|[^"\\])*"', + r"'(\\\\|\\[^\\]|[^'\\])*'", + r"\[(\\\\|\\[^\\]|[^\]\\])*\]", # square bracket literal r'/\*(.|\n)*?\*/', # multi-line javadoc-style comment r'//.*$\n?', # single line comment r'\#.*$\n?', # ruby/ragel comment @@ -222,8 +220,7 @@ class RagelRubyLexer(DelegatingLexer): filenames = ['*.rl'] def __init__(self, **options): - super(RagelRubyLexer, self).__init__(RubyLexer, RagelEmbeddedLexer, - **options) + super().__init__(RubyLexer, RagelEmbeddedLexer, **options) def analyse_text(text): return '@LANG: ruby' in text @@ -241,8 +238,7 @@ class RagelCLexer(DelegatingLexer): filenames = ['*.rl'] def __init__(self, **options): - super(RagelCLexer, self).__init__(CLexer, RagelEmbeddedLexer, - **options) + super().__init__(CLexer, RagelEmbeddedLexer, **options) def analyse_text(text): return '@LANG: c' in text @@ -260,7 +256,7 @@ class RagelDLexer(DelegatingLexer): filenames = ['*.rl'] def __init__(self, **options): - super(RagelDLexer, self).__init__(DLexer, RagelEmbeddedLexer, **options) + super().__init__(DLexer, RagelEmbeddedLexer, **options) def analyse_text(text): return '@LANG: d' in text @@ -278,7 +274,7 @@ class RagelCppLexer(DelegatingLexer): filenames = ['*.rl'] def __init__(self, **options): - super(RagelCppLexer, self).__init__(CppLexer, RagelEmbeddedLexer, **options) + super().__init__(CppLexer, RagelEmbeddedLexer, **options) def analyse_text(text): return '@LANG: c++' in text @@ -296,9 +292,7 @@ class RagelObjectiveCLexer(DelegatingLexer): filenames = ['*.rl'] def __init__(self, **options): - super(RagelObjectiveCLexer, self).__init__(ObjectiveCLexer, - RagelEmbeddedLexer, - **options) + super().__init__(ObjectiveCLexer, RagelEmbeddedLexer, **options) def analyse_text(text): return '@LANG: objc' in text @@ -316,8 +310,7 @@ class RagelJavaLexer(DelegatingLexer): filenames = ['*.rl'] def __init__(self, **options): - super(RagelJavaLexer, self).__init__(JavaLexer, RagelEmbeddedLexer, - **options) + super().__init__(JavaLexer, RagelEmbeddedLexer, **options) def analyse_text(text): return '@LANG: java' in text @@ -364,13 +357,13 @@ class AntlrLexer(RegexLexer): # tokensSpec (r'tokens\b', Keyword, 'tokens'), # attrScope - (r'(scope)(\s*)(' + _id + ')(\s*)(\{)', + (r'(scope)(\s*)(' + _id + r')(\s*)(\{)', bygroups(Keyword, Whitespace, Name.Variable, Whitespace, Punctuation), 'action'), # exception (r'(catch|finally)\b', Keyword, 'exception'), # action - (r'(@' + _id + ')(\s*)(::)?(\s*)(' + _id + ')(\s*)(\{)', + (r'(@' + _id + r')(\s*)(::)?(\s*)(' + _id + r')(\s*)(\{)', bygroups(Name.Label, Whitespace, Punctuation, Whitespace, Name.Label, Whitespace, Punctuation), 'action'), # rule @@ -405,10 +398,10 @@ class AntlrLexer(RegexLexer): # L173 ANTLRv3.g from ANTLR book (r'(scope)(\s+)(\{)', bygroups(Keyword, Whitespace, Punctuation), 'action'), - (r'(scope)(\s+)(' + _id + ')(\s*)(;)', + (r'(scope)(\s+)(' + _id + r')(\s*)(;)', bygroups(Keyword, Whitespace, Name.Label, Whitespace, Punctuation)), # ruleAction - (r'(@' + _id + ')(\s*)(\{)', + (r'(@' + _id + r')(\s*)(\{)', bygroups(Name.Label, Whitespace, Punctuation), 'action'), # finished prelims, go to rule alts! (r':', Punctuation, '#pop') @@ -422,8 +415,8 @@ class AntlrLexer(RegexLexer): (r':', Punctuation), # literals - (r"'(\\\\|\\'|[^'])*'", String), - (r'"(\\\\|\\"|[^"])*"', String), + (r'"(\\\\|\\[^\\]|[^"\\])*"', String.Double), + (r"'(\\\\|\\[^\\]|[^'\\])*'", String.Single), (r'<<([^>]|>[^>])>>', String), # identifiers # Tokens start with capital letter. @@ -442,7 +435,7 @@ class AntlrLexer(RegexLexer): include('comments'), (r'\{', Punctuation), (r'(' + _TOKEN_REF + r')(\s*)(=)?(\s*)(' + _STRING_LITERAL - + ')?(\s*)(;)', + + r')?(\s*)(;)', bygroups(Name.Label, Whitespace, Punctuation, Whitespace, String, Whitespace, Punctuation)), (r'\}', Punctuation, '#pop'), @@ -452,7 +445,7 @@ class AntlrLexer(RegexLexer): include('comments'), (r'\{', Punctuation), (r'(' + _id + r')(\s*)(=)(\s*)(' + - '|'.join((_id, _STRING_LITERAL, _INT, '\*')) + ')(\s*)(;)', + '|'.join((_id, _STRING_LITERAL, _INT, r'\*')) + r')(\s*)(;)', bygroups(Name.Variable, Whitespace, Punctuation, Whitespace, Text, Whitespace, Punctuation)), (r'\}', Punctuation, '#pop'), @@ -462,14 +455,14 @@ class AntlrLexer(RegexLexer): r'[^${}\'"/\\]+', # exclude unsafe characters # strings and comments may safely contain unsafe characters - r'"(\\\\|\\"|[^"])*"', # double quote string - r"'(\\\\|\\'|[^'])*'", # single quote string + r'"(\\\\|\\[^\\]|[^"\\])*"', + r"'(\\\\|\\[^\\]|[^'\\])*'", r'//.*$\n?', # single line comment r'/\*(.|\n)*?\*/', # multi-line javadoc-style comment # regular expression: There's no reason for it to start # with a * and this stops confusion with comments. - r'/(?!\*)(\\\\|\\/|[^/])*/', + r'/(?!\*)(\\\\|\\[^\\]|[^/\\])*/', # backslashes are okay, as long as we are not backslashing a % r'\\(?!%)', @@ -489,14 +482,14 @@ class AntlrLexer(RegexLexer): r'[^$\[\]\'"/]+', # exclude unsafe characters # strings and comments may safely contain unsafe characters - r'"(\\\\|\\"|[^"])*"', # double quote string - r"'(\\\\|\\'|[^'])*'", # single quote string + r'"(\\\\|\\[^\\]|[^"\\])*"', + r"'(\\\\|\\[^\\]|[^'\\])*'", r'//.*$\n?', # single line comment r'/\*(.|\n)*?\*/', # multi-line javadoc-style comment # regular expression: There's no reason for it to start # with a * and this stops confusion with comments. - r'/(?!\*)(\\\\|\\/|[^/])*/', + r'/(?!\*)(\\\\|\\[^\\]|[^/\\])*/', # Now that we've handled regex and javadoc comments # it's safe to let / through. @@ -515,30 +508,8 @@ class AntlrLexer(RegexLexer): def analyse_text(text): return re.search(r'^\s*grammar\s+[a-zA-Z0-9]+\s*;', text, re.M) -# http://www.antlr.org/wiki/display/ANTLR3/Code+Generation+Targets - -# TH: I'm not aware of any language features of C++ that will cause -# incorrect lexing of C files. Antlr doesn't appear to make a distinction, -# so just assume they're C++. No idea how to make Objective C work in the -# future. - -# class AntlrCLexer(DelegatingLexer): -# """ -# ANTLR with C Target -# -# .. versionadded:: 1.1 -# """ -# -# name = 'ANTLR With C Target' -# aliases = ['antlr-c'] -# filenames = ['*.G', '*.g'] -# -# def __init__(self, **options): -# super(AntlrCLexer, self).__init__(CLexer, AntlrLexer, **options) -# -# def analyse_text(text): -# return re.match(r'^\s*language\s*=\s*C\s*;', text) +# http://www.antlr.org/wiki/display/ANTLR3/Code+Generation+Targets class AntlrCppLexer(DelegatingLexer): """ @@ -552,7 +523,7 @@ class AntlrCppLexer(DelegatingLexer): filenames = ['*.G', '*.g'] def __init__(self, **options): - super(AntlrCppLexer, self).__init__(CppLexer, AntlrLexer, **options) + super().__init__(CppLexer, AntlrLexer, **options) def analyse_text(text): return AntlrLexer.analyse_text(text) and \ @@ -571,8 +542,7 @@ class AntlrObjectiveCLexer(DelegatingLexer): filenames = ['*.G', '*.g'] def __init__(self, **options): - super(AntlrObjectiveCLexer, self).__init__(ObjectiveCLexer, - AntlrLexer, **options) + super().__init__(ObjectiveCLexer, AntlrLexer, **options) def analyse_text(text): return AntlrLexer.analyse_text(text) and \ @@ -591,8 +561,7 @@ class AntlrCSharpLexer(DelegatingLexer): filenames = ['*.G', '*.g'] def __init__(self, **options): - super(AntlrCSharpLexer, self).__init__(CSharpLexer, AntlrLexer, - **options) + super().__init__(CSharpLexer, AntlrLexer, **options) def analyse_text(text): return AntlrLexer.analyse_text(text) and \ @@ -611,8 +580,7 @@ class AntlrPythonLexer(DelegatingLexer): filenames = ['*.G', '*.g'] def __init__(self, **options): - super(AntlrPythonLexer, self).__init__(PythonLexer, AntlrLexer, - **options) + super().__init__(PythonLexer, AntlrLexer, **options) def analyse_text(text): return AntlrLexer.analyse_text(text) and \ @@ -631,8 +599,7 @@ class AntlrJavaLexer(DelegatingLexer): filenames = ['*.G', '*.g'] def __init__(self, **options): - super(AntlrJavaLexer, self).__init__(JavaLexer, AntlrLexer, - **options) + super().__init__(JavaLexer, AntlrLexer, **options) def analyse_text(text): # Antlr language is Java by default @@ -651,8 +618,7 @@ class AntlrRubyLexer(DelegatingLexer): filenames = ['*.G', '*.g'] def __init__(self, **options): - super(AntlrRubyLexer, self).__init__(RubyLexer, AntlrLexer, - **options) + super().__init__(RubyLexer, AntlrLexer, **options) def analyse_text(text): return AntlrLexer.analyse_text(text) and \ @@ -671,8 +637,7 @@ class AntlrPerlLexer(DelegatingLexer): filenames = ['*.G', '*.g'] def __init__(self, **options): - super(AntlrPerlLexer, self).__init__(PerlLexer, AntlrLexer, - **options) + super().__init__(PerlLexer, AntlrLexer, **options) def analyse_text(text): return AntlrLexer.analyse_text(text) and \ @@ -687,13 +652,12 @@ class AntlrActionScriptLexer(DelegatingLexer): """ name = 'ANTLR With ActionScript Target' - aliases = ['antlr-as', 'antlr-actionscript'] + aliases = ['antlr-actionscript', 'antlr-as'] filenames = ['*.G', '*.g'] def __init__(self, **options): from pygments.lexers.actionscript import ActionScriptLexer - super(AntlrActionScriptLexer, self).__init__(ActionScriptLexer, - AntlrLexer, **options) + super().__init__(ActionScriptLexer, AntlrLexer, **options) def analyse_text(text): return AntlrLexer.analyse_text(text) and \ @@ -736,8 +700,8 @@ class TreetopBaseLexer(RegexLexer): 'rule': [ include('space'), include('end'), - (r'"(\\\\|\\"|[^"])*"', String.Double), - (r"'(\\\\|\\'|[^'])*'", String.Single), + (r'"(\\\\|\\[^\\]|[^"\\])*"', String.Double), + (r"'(\\\\|\\[^\\]|[^'\\])*'", String.Single), (r'([A-Za-z_]\w*)(:)', bygroups(Name.Label, Punctuation)), (r'[A-Za-z_]\w*', Name), (r'[()]', Punctuation), @@ -781,7 +745,7 @@ class TreetopLexer(DelegatingLexer): filenames = ['*.treetop', '*.tt'] def __init__(self, **options): - super(TreetopLexer, self).__init__(RubyLexer, TreetopBaseLexer, **options) + super().__init__(RubyLexer, TreetopBaseLexer, **options) class EbnfLexer(RegexLexer): diff --git a/src/syntax_highlighting/libs/pygments/lexers/pascal.py b/src/syntax_highlighting/libs/pygments/lexers/pascal.py index 9aa1ac8..0d1ac3f 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/pascal.py +++ b/src/syntax_highlighting/libs/pygments/lexers/pascal.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.pascal ~~~~~~~~~~~~~~~~~~~~~~ Lexers for Pascal family languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -68,29 +67,29 @@ class DelphiLexer(Lexer): 'dispose', 'exit', 'false', 'new', 'true' ) - BLOCK_KEYWORDS = set(( + BLOCK_KEYWORDS = { 'begin', 'class', 'const', 'constructor', 'destructor', 'end', 'finalization', 'function', 'implementation', 'initialization', 'label', 'library', 'operator', 'procedure', 'program', 'property', 'record', 'threadvar', 'type', 'unit', 'uses', 'var' - )) + } - FUNCTION_MODIFIERS = set(( + FUNCTION_MODIFIERS = { 'alias', 'cdecl', 'export', 'inline', 'interrupt', 'nostackframe', 'pascal', 'register', 'safecall', 'softfloat', 'stdcall', 'varargs', 'name', 'dynamic', 'near', 'virtual', 'external', 'override', 'assembler' - )) + } # XXX: those aren't global. but currently we know no way for defining # them just for the type context. - DIRECTIVES = set(( + DIRECTIVES = { 'absolute', 'abstract', 'assembler', 'cppdecl', 'default', 'far', 'far16', 'forward', 'index', 'oldfpccall', 'private', 'protected', 'published', 'public' - )) + } - BUILTIN_TYPES = set(( + BUILTIN_TYPES = { 'ansichar', 'ansistring', 'bool', 'boolean', 'byte', 'bytebool', 'cardinal', 'char', 'comp', 'currency', 'double', 'dword', 'extended', 'int64', 'integer', 'iunknown', 'longbool', 'longint', @@ -104,7 +103,7 @@ class DelphiLexer(Lexer): 'shortstring', 'single', 'smallint', 'string', 'tclass', 'tdate', 'tdatetime', 'textfile', 'thandle', 'tobject', 'ttime', 'variant', 'widechar', 'widestring', 'word', 'wordbool' - )) + } BUILTIN_UNITS = { 'System': ( @@ -246,7 +245,7 @@ class DelphiLexer(Lexer): ) } - ASM_REGISTERS = set(( + ASM_REGISTERS = { 'ah', 'al', 'ax', 'bh', 'bl', 'bp', 'bx', 'ch', 'cl', 'cr0', 'cr1', 'cr2', 'cr3', 'cr4', 'cs', 'cx', 'dh', 'di', 'dl', 'dr0', 'dr1', 'dr2', 'dr3', 'dr4', 'dr5', 'dr6', 'dr7', 'ds', 'dx', @@ -255,9 +254,9 @@ class DelphiLexer(Lexer): 'mm7', 'si', 'sp', 'ss', 'st0', 'st1', 'st2', 'st3', 'st4', 'st5', 'st6', 'st7', 'xmm0', 'xmm1', 'xmm2', 'xmm3', 'xmm4', 'xmm5', 'xmm6', 'xmm7' - )) + } - ASM_INSTRUCTIONS = set(( + ASM_INSTRUCTIONS = { 'aaa', 'aad', 'aam', 'aas', 'adc', 'add', 'and', 'arpl', 'bound', 'bsf', 'bsr', 'bswap', 'bt', 'btc', 'btr', 'bts', 'call', 'cbw', 'cdq', 'clc', 'cld', 'cli', 'clts', 'cmc', 'cmova', 'cmovae', @@ -296,7 +295,7 @@ class DelphiLexer(Lexer): 'sysret', 'test', 'ud1', 'ud2', 'umov', 'verr', 'verw', 'wait', 'wbinvd', 'wrmsr', 'wrshr', 'xadd', 'xbts', 'xchg', 'xlat', 'xlatb', 'xor' - )) + } def __init__(self, **options): Lexer.__init__(self, **options) @@ -563,9 +562,9 @@ class AdaLexer(RegexLexer): 'exception', 'exit', 'interface', 'for', 'goto', 'if', 'is', 'limited', 'loop', 'new', 'null', 'of', 'or', 'others', 'out', 'overriding', 'pragma', 'protected', 'raise', 'range', 'record', 'renames', 'requeue', - 'return', 'reverse', 'select', 'separate', 'subtype', 'synchronized', - 'task', 'tagged', 'terminate', 'then', 'type', 'until', 'when', - 'while', 'xor'), prefix=r'\b', suffix=r'\b'), + 'return', 'reverse', 'select', 'separate', 'some', 'subtype', + 'synchronized', 'task', 'tagged', 'terminate', 'then', 'type', 'until', + 'when', 'while', 'xor'), prefix=r'\b', suffix=r'\b'), Keyword.Reserved), (r'"[^"]*"', String), include('attribute'), @@ -577,7 +576,7 @@ class AdaLexer(RegexLexer): (r'\n+', Text), ], 'numbers': [ - (r'[0-9_]+#[0-9a-f]+#', Number.Hex), + (r'[0-9_]+#[0-9a-f_\.]+#', Number.Hex), (r'[0-9_]+\.[0-9_]*', Number.Float), (r'[0-9_]+', Number.Integer), ], @@ -593,8 +592,8 @@ class AdaLexer(RegexLexer): ], 'end': [ ('(if|case|record|loop|select)', Keyword.Reserved), - ('"[^"]+"|[\w.]+', Name.Function), - ('\s+', Text), + (r'"[^"]+"|[\w.]+', Name.Function), + (r'\s+', Text), (';', Punctuation, '#pop'), ], 'type_def': [ @@ -628,11 +627,11 @@ class AdaLexer(RegexLexer): ], 'package': [ ('body', Keyword.Declaration), - ('is\s+new|renames', Keyword.Reserved), + (r'is\s+new|renames', Keyword.Reserved), ('is', Keyword.Reserved, '#pop'), (';', Punctuation, '#pop'), - ('\(', Punctuation, 'package_instantiation'), - ('([\w.]+)', Name.Class), + (r'\(', Punctuation, 'package_instantiation'), + (r'([\w.]+)', Name.Class), include('root'), ], 'package_instantiation': [ diff --git a/src/syntax_highlighting/libs/pygments/lexers/pawn.py b/src/syntax_highlighting/libs/pygments/lexers/pawn.py index f462a88..5a303e4 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/pawn.py +++ b/src/syntax_highlighting/libs/pygments/lexers/pawn.py @@ -1,17 +1,16 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.pawn ~~~~~~~~~~~~~~~~~~~~ Lexers for the Pawn languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ from pygments.lexer import RegexLexer from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ - Number, Punctuation, Error + Number, Punctuation from pygments.util import get_bool_opt __all__ = ['SourcePawnLexer', 'PawnLexer'] @@ -36,7 +35,7 @@ class SourcePawnLexer(RegexLexer): tokens = { 'root': [ # preprocessor directives: without whitespace - ('^#if\s+0', Comment.Preproc, 'if0'), + (r'^#if\s+0', Comment.Preproc, 'if0'), ('^#', Comment.Preproc, 'macro'), # or with whitespace ('^' + _ws1 + r'#if\s+0', Comment.Preproc, 'if0'), @@ -54,7 +53,6 @@ class SourcePawnLexer(RegexLexer): (r'0x[0-9a-fA-F]+[LlUu]*', Number.Hex), (r'0[0-7]+[LlUu]*', Number.Oct), (r'\d+[LlUu]*', Number.Integer), - (r'\*/', Error), (r'[~!%^&*+=|?:<>/-]', Operator), (r'[()\[\],.;]', Punctuation), (r'(case|const|continue|native|' @@ -62,7 +60,7 @@ class SourcePawnLexer(RegexLexer): r'public|return|sizeof|static|decl|struct|switch)\b', Keyword), (r'(bool|Float)\b', Keyword.Type), (r'(true|false)\b', Keyword.Constant), - ('[a-zA-Z_]\w*', Name), + (r'[a-zA-Z_]\w*', Name), ], 'string': [ (r'"', String, '#pop'), @@ -86,25 +84,25 @@ class SourcePawnLexer(RegexLexer): ] } - SM_TYPES = set(('Action', 'bool', 'Float', 'Plugin', 'String', 'any', - 'AdminFlag', 'OverrideType', 'OverrideRule', 'ImmunityType', - 'GroupId', 'AdminId', 'AdmAccessMode', 'AdminCachePart', - 'CookieAccess', 'CookieMenu', 'CookieMenuAction', 'NetFlow', - 'ConVarBounds', 'QueryCookie', 'ReplySource', - 'ConVarQueryResult', 'ConVarQueryFinished', 'Function', - 'Action', 'Identity', 'PluginStatus', 'PluginInfo', 'DBResult', - 'DBBindType', 'DBPriority', 'PropType', 'PropFieldType', - 'MoveType', 'RenderMode', 'RenderFx', 'EventHookMode', - 'EventHook', 'FileType', 'FileTimeMode', 'PathType', - 'ParamType', 'ExecType', 'DialogType', 'Handle', 'KvDataTypes', - 'NominateResult', 'MapChange', 'MenuStyle', 'MenuAction', - 'MenuSource', 'RegexError', 'SDKCallType', 'SDKLibrary', - 'SDKFuncConfSource', 'SDKType', 'SDKPassMethod', 'RayType', - 'TraceEntityFilter', 'ListenOverride', 'SortOrder', 'SortType', - 'SortFunc2D', 'APLRes', 'FeatureType', 'FeatureStatus', - 'SMCResult', 'SMCError', 'TFClassType', 'TFTeam', 'TFCond', - 'TFResourceType', 'Timer', 'TopMenuAction', 'TopMenuObjectType', - 'TopMenuPosition', 'TopMenuObject', 'UserMsg')) + SM_TYPES = {'Action', 'bool', 'Float', 'Plugin', 'String', 'any', + 'AdminFlag', 'OverrideType', 'OverrideRule', 'ImmunityType', + 'GroupId', 'AdminId', 'AdmAccessMode', 'AdminCachePart', + 'CookieAccess', 'CookieMenu', 'CookieMenuAction', 'NetFlow', + 'ConVarBounds', 'QueryCookie', 'ReplySource', + 'ConVarQueryResult', 'ConVarQueryFinished', 'Function', + 'Action', 'Identity', 'PluginStatus', 'PluginInfo', 'DBResult', + 'DBBindType', 'DBPriority', 'PropType', 'PropFieldType', + 'MoveType', 'RenderMode', 'RenderFx', 'EventHookMode', + 'EventHook', 'FileType', 'FileTimeMode', 'PathType', + 'ParamType', 'ExecType', 'DialogType', 'Handle', 'KvDataTypes', + 'NominateResult', 'MapChange', 'MenuStyle', 'MenuAction', + 'MenuSource', 'RegexError', 'SDKCallType', 'SDKLibrary', + 'SDKFuncConfSource', 'SDKType', 'SDKPassMethod', 'RayType', + 'TraceEntityFilter', 'ListenOverride', 'SortOrder', 'SortType', + 'SortFunc2D', 'APLRes', 'FeatureType', 'FeatureStatus', + 'SMCResult', 'SMCError', 'TFClassType', 'TFTeam', 'TFCond', + 'TFResourceType', 'Timer', 'TopMenuAction', 'TopMenuObjectType', + 'TopMenuPosition', 'TopMenuObject', 'UserMsg'} def __init__(self, **options): self.smhighlighting = get_bool_opt(options, @@ -148,7 +146,7 @@ class PawnLexer(RegexLexer): tokens = { 'root': [ # preprocessor directives: without whitespace - ('^#if\s+0', Comment.Preproc, 'if0'), + (r'^#if\s+0', Comment.Preproc, 'if0'), ('^#', Comment.Preproc, 'macro'), # or with whitespace ('^' + _ws1 + r'#if\s+0', Comment.Preproc, 'if0'), @@ -166,7 +164,6 @@ class PawnLexer(RegexLexer): (r'0x[0-9a-fA-F]+[LlUu]*', Number.Hex), (r'0[0-7]+[LlUu]*', Number.Oct), (r'\d+[LlUu]*', Number.Integer), - (r'\*/', Error), (r'[~!%^&*+=|?:<>/-]', Operator), (r'[()\[\],.;]', Punctuation), (r'(switch|case|default|const|new|static|char|continue|break|' @@ -174,7 +171,7 @@ class PawnLexer(RegexLexer): r'public|return|sizeof|tagof|state|goto)\b', Keyword), (r'(bool|Float)\b', Keyword.Type), (r'(true|false)\b', Keyword.Constant), - ('[a-zA-Z_]\w*', Name), + (r'[a-zA-Z_]\w*', Name), ], 'string': [ (r'"', String, '#pop'), @@ -197,3 +194,9 @@ class PawnLexer(RegexLexer): (r'.*?\n', Comment), ] } + + def analyse_text(text): + """This is basically C. There is a keyword which doesn't exist in C + though and is nearly unique to this language.""" + if 'tagof' in text: + return 0.01 diff --git a/src/syntax_highlighting/libs/pygments/lexers/perl.py b/src/syntax_highlighting/libs/pygments/lexers/perl.py index 4d5ab3b..bac325b 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/perl.py +++ b/src/syntax_highlighting/libs/pygments/lexers/perl.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.perl ~~~~~~~~~~~~~~~~~~~~ - Lexers for Perl and related languages. + Lexers for Perl, Raku and related languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -22,12 +21,12 @@ class PerlLexer(RegexLexer): """ - For `Perl `_ source code. + For `Perl `_ source code. """ name = 'Perl' aliases = ['perl', 'pl'] - filenames = ['*.pl', '*.pm', '*.t'] + filenames = ['*.pl', '*.pm', '*.t', '*.perl'] mimetypes = ['text/x-perl', 'application/x-perl'] flags = re.DOTALL | re.MULTILINE @@ -208,97 +207,205 @@ class PerlLexer(RegexLexer): def analyse_text(text): if shebang_matches(text, r'perl'): return True - if re.search('(?:my|our)\s+[$@%(]', text): - return 0.9 + + result = 0 + + if re.search(r'(?:my|our)\s+[$@%(]', text): + result += 0.9 + + if ':=' in text: + # := is not valid Perl, but it appears in unicon, so we should + # become less confident if we think we found Perl with := + result /= 2 + + return result class Perl6Lexer(ExtendedRegexLexer): """ - For `Perl 6 `_ source code. + For `Raku `_ (a.k.a. Perl 6) source code. .. versionadded:: 2.0 """ name = 'Perl6' - aliases = ['perl6', 'pl6'] + aliases = ['perl6', 'pl6', 'raku'] filenames = ['*.pl', '*.pm', '*.nqp', '*.p6', '*.6pl', '*.p6l', '*.pl6', - '*.6pm', '*.p6m', '*.pm6', '*.t'] + '*.6pm', '*.p6m', '*.pm6', '*.t', '*.raku', '*.rakumod', + '*.rakutest', '*.rakudoc'] mimetypes = ['text/x-perl6', 'application/x-perl6'] flags = re.MULTILINE | re.DOTALL | re.UNICODE - PERL6_IDENTIFIER_RANGE = "['\w:-]" + PERL6_IDENTIFIER_RANGE = r"['\w:-]" PERL6_KEYWORDS = ( - 'BEGIN', 'CATCH', 'CHECK', 'CONTROL', 'END', 'ENTER', 'FIRST', 'INIT', - 'KEEP', 'LAST', 'LEAVE', 'NEXT', 'POST', 'PRE', 'START', 'TEMP', - 'UNDO', 'as', 'assoc', 'async', 'augment', 'binary', 'break', 'but', - 'cached', 'category', 'class', 'constant', 'contend', 'continue', - 'copy', 'deep', 'default', 'defequiv', 'defer', 'die', 'do', 'else', - 'elsif', 'enum', 'equiv', 'exit', 'export', 'fail', 'fatal', 'for', - 'gather', 'given', 'goto', 'grammar', 'handles', 'has', 'if', 'inline', - 'irs', 'is', 'last', 'leave', 'let', 'lift', 'loop', 'looser', 'macro', - 'make', 'maybe', 'method', 'module', 'multi', 'my', 'next', 'of', - 'ofs', 'only', 'oo', 'ors', 'our', 'package', 'parsed', 'prec', - 'proto', 'readonly', 'redo', 'ref', 'regex', 'reparsed', 'repeat', - 'require', 'required', 'return', 'returns', 'role', 'rule', 'rw', - 'self', 'slang', 'state', 'sub', 'submethod', 'subset', 'supersede', - 'take', 'temp', 'tighter', 'token', 'trusts', 'try', 'unary', - 'unless', 'until', 'use', 'warn', 'when', 'where', 'while', 'will', + #Phasers + 'BEGIN','CATCH','CHECK','CLOSE','CONTROL','DOC','END','ENTER','FIRST', + 'INIT','KEEP','LAST','LEAVE','NEXT','POST','PRE','QUIT','UNDO', + #Keywords + 'anon','augment','but','class','constant','default','does','else', + 'elsif','enum','for','gather','given','grammar','has','if','import', + 'is','let','loop','made','make','method','module','multi','my','need', + 'orwith','our','proceed','proto','repeat','require','return', + 'return-rw','returns','role','rule','state','sub','submethod','subset', + 'succeed','supersede','token','try','unit','unless','until','use', + 'when','while','with','without', + #Traits + 'export','native','repr','required','rw','symbol', ) PERL6_BUILTINS = ( - 'ACCEPTS', 'HOW', 'REJECTS', 'VAR', 'WHAT', 'WHENCE', 'WHERE', 'WHICH', - 'WHO', 'abs', 'acos', 'acosec', 'acosech', 'acosh', 'acotan', 'acotanh', - 'all', 'any', 'approx', 'arity', 'asec', 'asech', 'asin', 'asinh', - 'assuming', 'atan', 'atan2', 'atanh', 'attr', 'bless', 'body', 'by', - 'bytes', 'caller', 'callsame', 'callwith', 'can', 'capitalize', 'cat', - 'ceiling', 'chars', 'chmod', 'chomp', 'chop', 'chr', 'chroot', - 'circumfix', 'cis', 'classify', 'clone', 'close', 'cmp_ok', 'codes', - 'comb', 'connect', 'contains', 'context', 'cos', 'cosec', 'cosech', - 'cosh', 'cotan', 'cotanh', 'count', 'defined', 'delete', 'diag', - 'dies_ok', 'does', 'e', 'each', 'eager', 'elems', 'end', 'eof', 'eval', - 'eval_dies_ok', 'eval_elsewhere', 'eval_lives_ok', 'evalfile', 'exists', - 'exp', 'first', 'flip', 'floor', 'flunk', 'flush', 'fmt', 'force_todo', - 'fork', 'from', 'getc', 'gethost', 'getlogin', 'getpeername', 'getpw', - 'gmtime', 'graphs', 'grep', 'hints', 'hyper', 'im', 'index', 'infix', - 'invert', 'is_approx', 'is_deeply', 'isa', 'isa_ok', 'isnt', 'iterator', - 'join', 'key', 'keys', 'kill', 'kv', 'lastcall', 'lazy', 'lc', 'lcfirst', - 'like', 'lines', 'link', 'lives_ok', 'localtime', 'log', 'log10', 'map', - 'max', 'min', 'minmax', 'name', 'new', 'nextsame', 'nextwith', 'nfc', - 'nfd', 'nfkc', 'nfkd', 'nok_error', 'nonce', 'none', 'normalize', 'not', - 'nothing', 'ok', 'once', 'one', 'open', 'opendir', 'operator', 'ord', - 'p5chomp', 'p5chop', 'pack', 'pair', 'pairs', 'pass', 'perl', 'pi', - 'pick', 'plan', 'plan_ok', 'polar', 'pop', 'pos', 'postcircumfix', - 'postfix', 'pred', 'prefix', 'print', 'printf', 'push', 'quasi', - 'quotemeta', 'rand', 're', 'read', 'readdir', 'readline', 'reduce', - 'reverse', 'rewind', 'rewinddir', 'rindex', 'roots', 'round', - 'roundrobin', 'run', 'runinstead', 'sameaccent', 'samecase', 'say', - 'sec', 'sech', 'sech', 'seek', 'shape', 'shift', 'sign', 'signature', - 'sin', 'sinh', 'skip', 'skip_rest', 'sleep', 'slurp', 'sort', 'splice', - 'split', 'sprintf', 'sqrt', 'srand', 'strand', 'subst', 'substr', 'succ', - 'sum', 'symlink', 'tan', 'tanh', 'throws_ok', 'time', 'times', 'to', - 'todo', 'trim', 'trim_end', 'trim_start', 'true', 'truncate', 'uc', - 'ucfirst', 'undef', 'undefine', 'uniq', 'unlike', 'unlink', 'unpack', - 'unpolar', 'unshift', 'unwrap', 'use_ok', 'value', 'values', 'vec', - 'version_lt', 'void', 'wait', 'want', 'wrap', 'write', 'zip', + 'ACCEPTS','abs','abs2rel','absolute','accept','accessed','acos', + 'acosec','acosech','acosh','acotan','acotanh','acquire','act','action', + 'actions','add','add_attribute','add_enum_value','add_fallback', + 'add_method','add_parent','add_private_method','add_role','add_trustee', + 'adverb','after','all','allocate','allof','allowed','alternative-names', + 'annotations','antipair','antipairs','any','anyof','app_lifetime', + 'append','arch','archname','args','arity','Array','asec','asech','asin', + 'asinh','ASSIGN-KEY','ASSIGN-POS','assuming','ast','at','atan','atan2', + 'atanh','AT-KEY','atomic-assign','atomic-dec-fetch','atomic-fetch', + 'atomic-fetch-add','atomic-fetch-dec','atomic-fetch-inc', + 'atomic-fetch-sub','atomic-inc-fetch','AT-POS','attributes','auth', + 'await','backtrace','Bag','BagHash','bail-out','base','basename', + 'base-repeating','batch','BIND-KEY','BIND-POS','bind-stderr', + 'bind-stdin','bind-stdout','bind-udp','bits','bless','block','Bool', + 'bool-only','bounds','break','Bridge','broken','BUILD','build-date', + 'bytes','cache','callframe','calling-package','CALL-ME','callsame', + 'callwith','can','cancel','candidates','cando','can-ok','canonpath', + 'caps','caption','Capture','cas','catdir','categorize','categorize-list', + 'catfile','catpath','cause','ceiling','cglobal','changed','Channel', + 'chars','chdir','child','child-name','child-typename','chmod','chomp', + 'chop','chr','chrs','chunks','cis','classify','classify-list','cleanup', + 'clone','close','closed','close-stdin','cmp-ok','code','codes','collate', + 'column','comb','combinations','command','comment','compiler','Complex', + 'compose','compose_type','composer','condition','config', + 'configure_destroy','configure_type_checking','conj','connect', + 'constraints','construct','contains','contents','copy','cos','cosec', + 'cosech','cosh','cotan','cotanh','count','count-only','cpu-cores', + 'cpu-usage','CREATE','create_type','cross','cue','curdir','curupdir','d', + 'Date','DateTime','day','daycount','day-of-month','day-of-week', + 'day-of-year','days-in-month','declaration','decode','decoder','deepmap', + 'default','defined','DEFINITE','delayed','DELETE-KEY','DELETE-POS', + 'denominator','desc','DESTROY','destroyers','devnull','diag', + 'did-you-mean','die','dies-ok','dir','dirname','dir-sep','DISTROnames', + 'do','does','does-ok','done','done-testing','duckmap','dynamic','e', + 'eager','earlier','elems','emit','enclosing','encode','encoder', + 'encoding','end','ends-with','enum_from_value','enum_value_list', + 'enum_values','enums','eof','EVAL','eval-dies-ok','EVALFILE', + 'eval-lives-ok','exception','excludes-max','excludes-min','EXISTS-KEY', + 'EXISTS-POS','exit','exitcode','exp','expected','explicitly-manage', + 'expmod','extension','f','fail','fails-like','fc','feature','file', + 'filename','find_method','find_method_qualified','finish','first','flat', + 'flatmap','flip','floor','flunk','flush','fmt','format','formatter', + 'freeze','from','from-list','from-loop','from-posix','full', + 'full-barrier','get','get_value','getc','gist','got','grab','grabpairs', + 'grep','handle','handled','handles','hardware','has_accessor','Hash', + 'head','headers','hh-mm-ss','hidden','hides','hour','how','hyper','id', + 'illegal','im','in','indent','index','indices','indir','infinite', + 'infix','infix:<+>','infix:<->','install_method_cache','Instant', + 'instead','Int','int-bounds','interval','in-timezone','invalid-str', + 'invert','invocant','IO','IO::Notification.watch-path','is_trusted', + 'is_type','isa','is-absolute','isa-ok','is-approx','is-deeply', + 'is-hidden','is-initial-thread','is-int','is-lazy','is-leap-year', + 'isNaN','isnt','is-prime','is-relative','is-routine','is-setting', + 'is-win','item','iterator','join','keep','kept','KERNELnames','key', + 'keyof','keys','kill','kv','kxxv','l','lang','last','lastcall','later', + 'lazy','lc','leading','level','like','line','lines','link','List', + 'listen','live','lives-ok','local','lock','log','log10','lookup','lsb', + 'made','MAIN','make','Map','match','max','maxpairs','merge','message', + 'method','method_table','methods','migrate','min','minmax','minpairs', + 'minute','misplaced','Mix','MixHash','mkdir','mode','modified','month', + 'move','mro','msb','multi','multiness','my','name','named','named_names', + 'narrow','nativecast','native-descriptor','nativesizeof','new','new_type', + 'new-from-daycount','new-from-pairs','next','nextcallee','next-handle', + 'nextsame','nextwith','NFC','NFD','NFKC','NFKD','nl-in','nl-out', + 'nodemap','nok','none','norm','not','note','now','nude','Num', + 'numerator','Numeric','of','offset','offset-in-hours','offset-in-minutes', + 'ok','old','on-close','one','on-switch','open','opened','operation', + 'optional','ord','ords','orig','os-error','osname','out-buffer','pack', + 'package','package-kind','package-name','packages','pair','pairs', + 'pairup','parameter','params','parent','parent-name','parents','parse', + 'parse-base','parsefile','parse-names','parts','pass','path','path-sep', + 'payload','peer-host','peer-port','periods','perl','permutations','phaser', + 'pick','pickpairs','pid','placeholder','plan','plus','polar','poll', + 'polymod','pop','pos','positional','posix','postfix','postmatch', + 'precomp-ext','precomp-target','pred','prefix','prematch','prepend', + 'print','printf','print-nl','print-to','private','private_method_table', + 'proc','produce','Promise','prompt','protect','pull-one','push', + 'push-all','push-at-least','push-exactly','push-until-lazy','put', + 'qualifier-type','quit','r','race','radix','rand','range','Rat','raw', + 're','read','readchars','readonly','ready','Real','reallocate','reals', + 'reason','rebless','receive','recv','redispatcher','redo','reduce', + 'rel2abs','relative','release','rename','repeated','replacement', + 'report','reserved','resolve','restore','result','resume','rethrow', + 'reverse','right','rindex','rmdir','role','roles_to_compose','rolish', + 'roll','rootdir','roots','rotate','rotor','round','roundrobin', + 'routine-type','run','rwx','s','samecase','samemark','samewith','say', + 'schedule-on','scheduler','scope','sec','sech','second','seek','self', + 'send','Set','set_hidden','set_name','set_package','set_rw','set_value', + 'SetHash','set-instruments','setup_finalization','shape','share','shell', + 'shift','sibling','sigil','sign','signal','signals','signature','sin', + 'sinh','sink','sink-all','skip','skip-at-least','skip-at-least-pull-one', + 'skip-one','skip-rest','sleep','sleep-timer','sleep-until','Slip','slurp', + 'slurp-rest','slurpy','snap','snapper','so','socket-host','socket-port', + 'sort','source','source-package','spawn','SPEC','splice','split', + 'splitdir','splitpath','sprintf','spurt','sqrt','squish','srand','stable', + 'start','started','starts-with','status','stderr','stdout','Str', + 'sub_signature','subbuf','subbuf-rw','subname','subparse','subst', + 'subst-mutate','substr','substr-eq','substr-rw','subtest','succ','sum', + 'Supply','symlink','t','tail','take','take-rw','tan','tanh','tap', + 'target','target-name','tc','tclc','tell','then','throttle','throw', + 'throws-like','timezone','tmpdir','to','today','todo','toggle','to-posix', + 'total','trailing','trans','tree','trim','trim-leading','trim-trailing', + 'truncate','truncated-to','trusts','try_acquire','trying','twigil','type', + 'type_captures','typename','uc','udp','uncaught_handler','unimatch', + 'uniname','uninames','uniparse','uniprop','uniprops','unique','unival', + 'univals','unlike','unlink','unlock','unpack','unpolar','unshift', + 'unwrap','updir','USAGE','use-ok','utc','val','value','values','VAR', + 'variable','verbose-config','version','VMnames','volume','vow','w','wait', + 'warn','watch','watch-path','week','weekday-of-month','week-number', + 'week-year','WHAT','when','WHERE','WHEREFORE','WHICH','WHO', + 'whole-second','WHY','wordcase','words','workaround','wrap','write', + 'write-to','x','yada','year','yield','yyyy-mm-dd','z','zip','zip-latest', + ) PERL6_BUILTIN_CLASSES = ( - 'Abstraction', 'Any', 'AnyChar', 'Array', 'Associative', 'Bag', 'Bit', - 'Blob', 'Block', 'Bool', 'Buf', 'Byte', 'Callable', 'Capture', 'Char', 'Class', - 'Code', 'Codepoint', 'Comparator', 'Complex', 'Decreasing', 'Exception', - 'Failure', 'False', 'Grammar', 'Grapheme', 'Hash', 'IO', 'Increasing', - 'Int', 'Junction', 'KeyBag', 'KeyExtractor', 'KeyHash', 'KeySet', - 'KitchenSink', 'List', 'Macro', 'Mapping', 'Match', 'Matcher', 'Method', - 'Module', 'Num', 'Object', 'Ordered', 'Ordering', 'OrderingPair', - 'Package', 'Pair', 'Positional', 'Proxy', 'Range', 'Rat', 'Regex', - 'Role', 'Routine', 'Scalar', 'Seq', 'Set', 'Signature', 'Str', 'StrLen', - 'StrPos', 'Sub', 'Submethod', 'True', 'UInt', 'Undef', 'Version', 'Void', - 'Whatever', 'bit', 'bool', 'buf', 'buf1', 'buf16', 'buf2', 'buf32', - 'buf4', 'buf64', 'buf8', 'complex', 'int', 'int1', 'int16', 'int2', - 'int32', 'int4', 'int64', 'int8', 'num', 'rat', 'rat1', 'rat16', 'rat2', - 'rat32', 'rat4', 'rat64', 'rat8', 'uint', 'uint1', 'uint16', 'uint2', - 'uint32', 'uint4', 'uint64', 'uint8', 'utf16', 'utf32', 'utf8', + #Booleans + 'False','True', + #Classes + 'Any','Array','Associative','AST','atomicint','Attribute','Backtrace', + 'Backtrace::Frame','Bag','Baggy','BagHash','Blob','Block','Bool','Buf', + 'Callable','CallFrame','Cancellation','Capture','CArray','Channel','Code', + 'compiler','Complex','ComplexStr','Cool','CurrentThreadScheduler', + 'Cursor','Date','Dateish','DateTime','Distro','Duration','Encoding', + 'Exception','Failure','FatRat','Grammar','Hash','HyperWhatever','Instant', + 'Int','int16','int32','int64','int8','IntStr','IO','IO::ArgFiles', + 'IO::CatHandle','IO::Handle','IO::Notification','IO::Path', + 'IO::Path::Cygwin','IO::Path::QNX','IO::Path::Unix','IO::Path::Win32', + 'IO::Pipe','IO::Socket','IO::Socket::Async','IO::Socket::INET','IO::Spec', + 'IO::Spec::Cygwin','IO::Spec::QNX','IO::Spec::Unix','IO::Spec::Win32', + 'IO::Special','Iterable','Iterator','Junction','Kernel','Label','List', + 'Lock','Lock::Async','long','longlong','Macro','Map','Match', + 'Metamodel::AttributeContainer','Metamodel::C3MRO','Metamodel::ClassHOW', + 'Metamodel::EnumHOW','Metamodel::Finalization','Metamodel::MethodContainer', + 'Metamodel::MROBasedMethodDispatch','Metamodel::MultipleInheritance', + 'Metamodel::Naming','Metamodel::Primitives','Metamodel::PrivateMethodContainer', + 'Metamodel::RoleContainer','Metamodel::Trusting','Method','Mix','MixHash', + 'Mixy','Mu','NFC','NFD','NFKC','NFKD','Nil','Num','num32','num64', + 'Numeric','NumStr','ObjAt','Order','Pair','Parameter','Perl','Pod::Block', + 'Pod::Block::Code','Pod::Block::Comment','Pod::Block::Declarator', + 'Pod::Block::Named','Pod::Block::Para','Pod::Block::Table','Pod::Heading', + 'Pod::Item','Pointer','Positional','PositionalBindFailover','Proc', + 'Proc::Async','Promise','Proxy','PseudoStash','QuantHash','Range','Rat', + 'Rational','RatStr','Real','Regex','Routine','Scalar','Scheduler', + 'Semaphore','Seq','Set','SetHash','Setty','Signature','size_t','Slip', + 'Stash','Str','StrDistance','Stringy','Sub','Submethod','Supplier', + 'Supplier::Preserving','Supply','Systemic','Tap','Telemetry', + 'Telemetry::Instrument::Thread','Telemetry::Instrument::Usage', + 'Telemetry::Period','Telemetry::Sampler','Thread','ThreadPoolScheduler', + 'UInt','uint16','uint32','uint64','uint8','Uni','utf8','Variable', + 'Version','VM','Whatever','WhateverCode','WrapHandle' ) PERL6_OPERATORS = ( @@ -311,76 +418,76 @@ class Perl6Lexer(ExtendedRegexLexer): '~', '&', '^', 'but', 'does', '<=>', '..', '..^', '^..', '^..^', '!=', '==', '<', '<=', '>', '>=', '~~', '===', '!eqv', '&&', '||', '^^', '//', 'min', 'max', '??', '!!', 'ff', 'fff', 'so', - 'not', '<==', '==>', '<<==', '==>>', + 'not', '<==', '==>', '<<==', '==>>','unicmp', ) # Perl 6 has a *lot* of possible bracketing characters # this list was lifted from STD.pm6 (https://github.com/perl6/std) PERL6_BRACKETS = { - u'\u0028': u'\u0029', u'\u003c': u'\u003e', u'\u005b': u'\u005d', - u'\u007b': u'\u007d', u'\u00ab': u'\u00bb', u'\u0f3a': u'\u0f3b', - u'\u0f3c': u'\u0f3d', u'\u169b': u'\u169c', u'\u2018': u'\u2019', - u'\u201a': u'\u2019', u'\u201b': u'\u2019', u'\u201c': u'\u201d', - u'\u201e': u'\u201d', u'\u201f': u'\u201d', u'\u2039': u'\u203a', - u'\u2045': u'\u2046', u'\u207d': u'\u207e', u'\u208d': u'\u208e', - u'\u2208': u'\u220b', u'\u2209': u'\u220c', u'\u220a': u'\u220d', - u'\u2215': u'\u29f5', u'\u223c': u'\u223d', u'\u2243': u'\u22cd', - u'\u2252': u'\u2253', u'\u2254': u'\u2255', u'\u2264': u'\u2265', - u'\u2266': u'\u2267', u'\u2268': u'\u2269', u'\u226a': u'\u226b', - u'\u226e': u'\u226f', u'\u2270': u'\u2271', u'\u2272': u'\u2273', - u'\u2274': u'\u2275', u'\u2276': u'\u2277', u'\u2278': u'\u2279', - u'\u227a': u'\u227b', u'\u227c': u'\u227d', u'\u227e': u'\u227f', - u'\u2280': u'\u2281', u'\u2282': u'\u2283', u'\u2284': u'\u2285', - u'\u2286': u'\u2287', u'\u2288': u'\u2289', u'\u228a': u'\u228b', - u'\u228f': u'\u2290', u'\u2291': u'\u2292', u'\u2298': u'\u29b8', - u'\u22a2': u'\u22a3', u'\u22a6': u'\u2ade', u'\u22a8': u'\u2ae4', - u'\u22a9': u'\u2ae3', u'\u22ab': u'\u2ae5', u'\u22b0': u'\u22b1', - u'\u22b2': u'\u22b3', u'\u22b4': u'\u22b5', u'\u22b6': u'\u22b7', - u'\u22c9': u'\u22ca', u'\u22cb': u'\u22cc', u'\u22d0': u'\u22d1', - u'\u22d6': u'\u22d7', u'\u22d8': u'\u22d9', u'\u22da': u'\u22db', - u'\u22dc': u'\u22dd', u'\u22de': u'\u22df', u'\u22e0': u'\u22e1', - u'\u22e2': u'\u22e3', u'\u22e4': u'\u22e5', u'\u22e6': u'\u22e7', - u'\u22e8': u'\u22e9', u'\u22ea': u'\u22eb', u'\u22ec': u'\u22ed', - u'\u22f0': u'\u22f1', u'\u22f2': u'\u22fa', u'\u22f3': u'\u22fb', - u'\u22f4': u'\u22fc', u'\u22f6': u'\u22fd', u'\u22f7': u'\u22fe', - u'\u2308': u'\u2309', u'\u230a': u'\u230b', u'\u2329': u'\u232a', - u'\u23b4': u'\u23b5', u'\u2768': u'\u2769', u'\u276a': u'\u276b', - u'\u276c': u'\u276d', u'\u276e': u'\u276f', u'\u2770': u'\u2771', - u'\u2772': u'\u2773', u'\u2774': u'\u2775', u'\u27c3': u'\u27c4', - u'\u27c5': u'\u27c6', u'\u27d5': u'\u27d6', u'\u27dd': u'\u27de', - u'\u27e2': u'\u27e3', u'\u27e4': u'\u27e5', u'\u27e6': u'\u27e7', - u'\u27e8': u'\u27e9', u'\u27ea': u'\u27eb', u'\u2983': u'\u2984', - u'\u2985': u'\u2986', u'\u2987': u'\u2988', u'\u2989': u'\u298a', - u'\u298b': u'\u298c', u'\u298d': u'\u298e', u'\u298f': u'\u2990', - u'\u2991': u'\u2992', u'\u2993': u'\u2994', u'\u2995': u'\u2996', - u'\u2997': u'\u2998', u'\u29c0': u'\u29c1', u'\u29c4': u'\u29c5', - u'\u29cf': u'\u29d0', u'\u29d1': u'\u29d2', u'\u29d4': u'\u29d5', - u'\u29d8': u'\u29d9', u'\u29da': u'\u29db', u'\u29f8': u'\u29f9', - u'\u29fc': u'\u29fd', u'\u2a2b': u'\u2a2c', u'\u2a2d': u'\u2a2e', - u'\u2a34': u'\u2a35', u'\u2a3c': u'\u2a3d', u'\u2a64': u'\u2a65', - u'\u2a79': u'\u2a7a', u'\u2a7d': u'\u2a7e', u'\u2a7f': u'\u2a80', - u'\u2a81': u'\u2a82', u'\u2a83': u'\u2a84', u'\u2a8b': u'\u2a8c', - u'\u2a91': u'\u2a92', u'\u2a93': u'\u2a94', u'\u2a95': u'\u2a96', - u'\u2a97': u'\u2a98', u'\u2a99': u'\u2a9a', u'\u2a9b': u'\u2a9c', - u'\u2aa1': u'\u2aa2', u'\u2aa6': u'\u2aa7', u'\u2aa8': u'\u2aa9', - u'\u2aaa': u'\u2aab', u'\u2aac': u'\u2aad', u'\u2aaf': u'\u2ab0', - u'\u2ab3': u'\u2ab4', u'\u2abb': u'\u2abc', u'\u2abd': u'\u2abe', - u'\u2abf': u'\u2ac0', u'\u2ac1': u'\u2ac2', u'\u2ac3': u'\u2ac4', - u'\u2ac5': u'\u2ac6', u'\u2acd': u'\u2ace', u'\u2acf': u'\u2ad0', - u'\u2ad1': u'\u2ad2', u'\u2ad3': u'\u2ad4', u'\u2ad5': u'\u2ad6', - u'\u2aec': u'\u2aed', u'\u2af7': u'\u2af8', u'\u2af9': u'\u2afa', - u'\u2e02': u'\u2e03', u'\u2e04': u'\u2e05', u'\u2e09': u'\u2e0a', - u'\u2e0c': u'\u2e0d', u'\u2e1c': u'\u2e1d', u'\u2e20': u'\u2e21', - u'\u3008': u'\u3009', u'\u300a': u'\u300b', u'\u300c': u'\u300d', - u'\u300e': u'\u300f', u'\u3010': u'\u3011', u'\u3014': u'\u3015', - u'\u3016': u'\u3017', u'\u3018': u'\u3019', u'\u301a': u'\u301b', - u'\u301d': u'\u301e', u'\ufd3e': u'\ufd3f', u'\ufe17': u'\ufe18', - u'\ufe35': u'\ufe36', u'\ufe37': u'\ufe38', u'\ufe39': u'\ufe3a', - u'\ufe3b': u'\ufe3c', u'\ufe3d': u'\ufe3e', u'\ufe3f': u'\ufe40', - u'\ufe41': u'\ufe42', u'\ufe43': u'\ufe44', u'\ufe47': u'\ufe48', - u'\ufe59': u'\ufe5a', u'\ufe5b': u'\ufe5c', u'\ufe5d': u'\ufe5e', - u'\uff08': u'\uff09', u'\uff1c': u'\uff1e', u'\uff3b': u'\uff3d', - u'\uff5b': u'\uff5d', u'\uff5f': u'\uff60', u'\uff62': u'\uff63', + '\u0028': '\u0029', '\u003c': '\u003e', '\u005b': '\u005d', + '\u007b': '\u007d', '\u00ab': '\u00bb', '\u0f3a': '\u0f3b', + '\u0f3c': '\u0f3d', '\u169b': '\u169c', '\u2018': '\u2019', + '\u201a': '\u2019', '\u201b': '\u2019', '\u201c': '\u201d', + '\u201e': '\u201d', '\u201f': '\u201d', '\u2039': '\u203a', + '\u2045': '\u2046', '\u207d': '\u207e', '\u208d': '\u208e', + '\u2208': '\u220b', '\u2209': '\u220c', '\u220a': '\u220d', + '\u2215': '\u29f5', '\u223c': '\u223d', '\u2243': '\u22cd', + '\u2252': '\u2253', '\u2254': '\u2255', '\u2264': '\u2265', + '\u2266': '\u2267', '\u2268': '\u2269', '\u226a': '\u226b', + '\u226e': '\u226f', '\u2270': '\u2271', '\u2272': '\u2273', + '\u2274': '\u2275', '\u2276': '\u2277', '\u2278': '\u2279', + '\u227a': '\u227b', '\u227c': '\u227d', '\u227e': '\u227f', + '\u2280': '\u2281', '\u2282': '\u2283', '\u2284': '\u2285', + '\u2286': '\u2287', '\u2288': '\u2289', '\u228a': '\u228b', + '\u228f': '\u2290', '\u2291': '\u2292', '\u2298': '\u29b8', + '\u22a2': '\u22a3', '\u22a6': '\u2ade', '\u22a8': '\u2ae4', + '\u22a9': '\u2ae3', '\u22ab': '\u2ae5', '\u22b0': '\u22b1', + '\u22b2': '\u22b3', '\u22b4': '\u22b5', '\u22b6': '\u22b7', + '\u22c9': '\u22ca', '\u22cb': '\u22cc', '\u22d0': '\u22d1', + '\u22d6': '\u22d7', '\u22d8': '\u22d9', '\u22da': '\u22db', + '\u22dc': '\u22dd', '\u22de': '\u22df', '\u22e0': '\u22e1', + '\u22e2': '\u22e3', '\u22e4': '\u22e5', '\u22e6': '\u22e7', + '\u22e8': '\u22e9', '\u22ea': '\u22eb', '\u22ec': '\u22ed', + '\u22f0': '\u22f1', '\u22f2': '\u22fa', '\u22f3': '\u22fb', + '\u22f4': '\u22fc', '\u22f6': '\u22fd', '\u22f7': '\u22fe', + '\u2308': '\u2309', '\u230a': '\u230b', '\u2329': '\u232a', + '\u23b4': '\u23b5', '\u2768': '\u2769', '\u276a': '\u276b', + '\u276c': '\u276d', '\u276e': '\u276f', '\u2770': '\u2771', + '\u2772': '\u2773', '\u2774': '\u2775', '\u27c3': '\u27c4', + '\u27c5': '\u27c6', '\u27d5': '\u27d6', '\u27dd': '\u27de', + '\u27e2': '\u27e3', '\u27e4': '\u27e5', '\u27e6': '\u27e7', + '\u27e8': '\u27e9', '\u27ea': '\u27eb', '\u2983': '\u2984', + '\u2985': '\u2986', '\u2987': '\u2988', '\u2989': '\u298a', + '\u298b': '\u298c', '\u298d': '\u298e', '\u298f': '\u2990', + '\u2991': '\u2992', '\u2993': '\u2994', '\u2995': '\u2996', + '\u2997': '\u2998', '\u29c0': '\u29c1', '\u29c4': '\u29c5', + '\u29cf': '\u29d0', '\u29d1': '\u29d2', '\u29d4': '\u29d5', + '\u29d8': '\u29d9', '\u29da': '\u29db', '\u29f8': '\u29f9', + '\u29fc': '\u29fd', '\u2a2b': '\u2a2c', '\u2a2d': '\u2a2e', + '\u2a34': '\u2a35', '\u2a3c': '\u2a3d', '\u2a64': '\u2a65', + '\u2a79': '\u2a7a', '\u2a7d': '\u2a7e', '\u2a7f': '\u2a80', + '\u2a81': '\u2a82', '\u2a83': '\u2a84', '\u2a8b': '\u2a8c', + '\u2a91': '\u2a92', '\u2a93': '\u2a94', '\u2a95': '\u2a96', + '\u2a97': '\u2a98', '\u2a99': '\u2a9a', '\u2a9b': '\u2a9c', + '\u2aa1': '\u2aa2', '\u2aa6': '\u2aa7', '\u2aa8': '\u2aa9', + '\u2aaa': '\u2aab', '\u2aac': '\u2aad', '\u2aaf': '\u2ab0', + '\u2ab3': '\u2ab4', '\u2abb': '\u2abc', '\u2abd': '\u2abe', + '\u2abf': '\u2ac0', '\u2ac1': '\u2ac2', '\u2ac3': '\u2ac4', + '\u2ac5': '\u2ac6', '\u2acd': '\u2ace', '\u2acf': '\u2ad0', + '\u2ad1': '\u2ad2', '\u2ad3': '\u2ad4', '\u2ad5': '\u2ad6', + '\u2aec': '\u2aed', '\u2af7': '\u2af8', '\u2af9': '\u2afa', + '\u2e02': '\u2e03', '\u2e04': '\u2e05', '\u2e09': '\u2e0a', + '\u2e0c': '\u2e0d', '\u2e1c': '\u2e1d', '\u2e20': '\u2e21', + '\u3008': '\u3009', '\u300a': '\u300b', '\u300c': '\u300d', + '\u300e': '\u300f', '\u3010': '\u3011', '\u3014': '\u3015', + '\u3016': '\u3017', '\u3018': '\u3019', '\u301a': '\u301b', + '\u301d': '\u301e', '\ufd3e': '\ufd3f', '\ufe17': '\ufe18', + '\ufe35': '\ufe36', '\ufe37': '\ufe38', '\ufe39': '\ufe3a', + '\ufe3b': '\ufe3c', '\ufe3d': '\ufe3e', '\ufe3f': '\ufe40', + '\ufe41': '\ufe42', '\ufe43': '\ufe44', '\ufe47': '\ufe48', + '\ufe59': '\ufe5a', '\ufe5b': '\ufe5c', '\ufe5d': '\ufe5e', + '\uff08': '\uff09', '\uff1c': '\uff1e', '\uff3b': '\uff3d', + '\uff5b': '\uff5d', '\uff5f': '\uff60', '\uff62': '\uff63', } def _build_word_match(words, boundary_regex_fragment=None, prefix='', suffix=''): @@ -489,13 +596,13 @@ def embedded_perl6_callback(lexer, match, context): 'common': [ (r'#[`|=](?P(?P[' + ''.join(PERL6_BRACKETS) + r'])(?P=first_char)*)', brackets_callback(Comment.Multiline)), - (r'#[^\n]*$', Comment.Singleline), + (r'#[^\n]*$', Comment.Single), (r'^(\s*)=begin\s+(\w+)\b.*?^\1=end\s+\2', Comment.Multiline), (r'^(\s*)=for.*?\n\s*?\n', Comment.Multiline), (r'^=.*?\n\s*?\n', Comment.Multiline), (r'(regex|token|rule)(\s*' + PERL6_IDENTIFIER_RANGE + '+:sym)', bygroups(Keyword, Name), 'token-sym-brackets'), - (r'(regex|token|rule)(?!' + PERL6_IDENTIFIER_RANGE + ')(\s*' + PERL6_IDENTIFIER_RANGE + '+)?', + (r'(regex|token|rule)(?!' + PERL6_IDENTIFIER_RANGE + r')(\s*' + PERL6_IDENTIFIER_RANGE + '+)?', bygroups(Keyword, Name), 'pre-token'), # deal with a special case in the Perl 6 grammar (role q { ... }) (r'(role)(\s+)(q)(\s*)', bygroups(Keyword, Text, Name, Text)), @@ -504,11 +611,11 @@ def embedded_perl6_callback(lexer, match, context): Name.Builtin), (_build_word_match(PERL6_BUILTINS, PERL6_IDENTIFIER_RANGE), Name.Builtin), # copied from PerlLexer - (r'[$@%&][.^:?=!~]?' + PERL6_IDENTIFIER_RANGE + u'+(?:<<.*?>>|<.*?>|«.*?»)*', + (r'[$@%&][.^:?=!~]?' + PERL6_IDENTIFIER_RANGE + '+(?:<<.*?>>|<.*?>|«.*?»)*', Name.Variable), (r'\$[!/](?:<<.*?>>|<.*?>|«.*?»)*', Name.Variable.Global), (r'::\?\w+', Name.Variable.Global), - (r'[$@%&]\*' + PERL6_IDENTIFIER_RANGE + u'+(?:<<.*?>>|<.*?>|«.*?»)*', + (r'[$@%&]\*' + PERL6_IDENTIFIER_RANGE + '+(?:<<.*?>>|<.*?>|«.*?»)*', Name.Variable.Global), (r'\$(?:<.*?>)+', Name.Variable), (r'(?:q|qq|Q)[a-zA-Z]?\s*(?P:[\w\s:]+)?\s*(?P(?P[^0-9a-zA-Z:\s])' @@ -558,7 +665,7 @@ def embedded_perl6_callback(lexer, match, context): # make sure that '#' characters in quotes aren't treated as comments (r"(?my|our)\s+)?(?:module|class|role|enum|grammar)', line) + class_decl = re.match(r'^\s*(?:(?Pmy|our)\s+)?(?:module|class|role|enum|grammar)', line) if class_decl: if saw_perl_decl or class_decl.group('scope') is not None: return True @@ -613,8 +720,12 @@ def strip_pod(lines): continue break + if ':=' in text: + # Same logic as above for PerlLexer + rating /= 2 + return rating def __init__(self, **options): - super(Perl6Lexer, self).__init__(**options) + super().__init__(**options) self.encoding = options.get('encoding', 'utf-8') diff --git a/src/syntax_highlighting/libs/pygments/lexers/php.py b/src/syntax_highlighting/libs/pygments/lexers/php.py index f618b5f..3ba299a 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/php.py +++ b/src/syntax_highlighting/libs/pygments/lexers/php.py @@ -1,23 +1,24 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.php ~~~~~~~~~~~~~~~~~~~ Lexers for PHP and related languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ import re -from pygments.lexer import RegexLexer, include, bygroups, default, using, \ - this, words +from pygments.lexer import Lexer, RegexLexer, include, bygroups, default, \ + using, this, words, do_insertions from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ - Number, Punctuation, Other -from pygments.util import get_bool_opt, get_list_opt, iteritems + Number, Punctuation, Other, Generic +from pygments.util import get_bool_opt, get_list_opt, shebang_matches -__all__ = ['ZephirLexer', 'PhpLexer'] +__all__ = ['ZephirLexer', 'PsyshConsoleLexer', 'PhpLexer'] + +line_re = re.compile('.*?\n') class ZephirLexer(RegexLexer): @@ -49,13 +50,14 @@ class ZephirLexer(RegexLexer): include('commentsandwhitespace'), (r'/(\\.|[^[/\\\n]|\[(\\.|[^\]\\\n])*])+/' r'([gim]+\b|\B)', String.Regex, '#pop'), + (r'/', Operator, '#pop'), default('#pop') ], 'badregex': [ (r'\n', Text, '#pop') ], 'root': [ - (r'^(?=\s|/|)', + (r'([a-z\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]' + r'[\w$\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]*)' + r'(\s*)(:-|-->)', bygroups(Name.Function, Text, Operator)), # function defn - (u'([a-z\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]' - u'[\w$\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]*)' - u'(\\s*)(\\()', + (r'([a-z\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]' + r'[\w$\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]*)' + r'(\s*)(\()', bygroups(Name.Function, Text, Punctuation)), - (u'[a-z\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]' - u'[\w$\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]*', + (r'[a-z\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]' + r'[\w$\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]*', String.Atom), # atom, characters # This one includes ! - (u'[#&*+\\-./:<=>?@\\\\^~\u00a1-\u00bf\u2010-\u303f]+', + (r'[#&*+\-./:<=>?@\\^~\u00a1-\u00bf\u2010-\u303f]+', String.Atom), # atom, graphics (r'[A-Z_]\w*', Name.Variable), - (u'\\s+|[\u2000-\u200f\ufff0-\ufffe\uffef]', Text), + (r'\s+|[\u2000-\u200f\ufff0-\ufffe\uffef]', Text), ], 'nested-comment': [ (r'\*/', Comment.Multiline, '#pop'), @@ -108,19 +106,19 @@ class LogtalkLexer(RegexLexer): (r'\n', Text), (r'\s+', Text), # Numbers - (r"0'.", Number), + (r"0'[\\]?.", Number), (r'0b[01]+', Number.Bin), (r'0o[0-7]+', Number.Oct), (r'0x[0-9a-fA-F]+', Number.Hex), (r'\d+\.?\d*((e|E)(\+|-)?\d+)?', Number), # Variables - (r'([A-Z_]\w*)', Name.Variable), + (r'([A-Z_][a-zA-Z0-9_]*)', Name.Variable), # Event handlers (r'(after|before)(?=[(])', Keyword), # Message forwarding handler (r'forward(?=[(])', Keyword), # Execution-context methods - (r'(parameter|this|se(lf|nder))(?=[(])', Keyword), + (r'(context|parameter|this|se(lf|nder))(?=[(])', Keyword), # Reflection (r'(current_predicate|predicate_property)(?=[(])', Keyword), # DCGs and term expansion @@ -136,20 +134,23 @@ class LogtalkLexer(RegexLexer): # Events (r'(current_event|(abolish|define)_events)(?=[(])', Keyword), # Flags - (r'(current|set)_logtalk_flag(?=[(])', Keyword), + (r'(create|current|set)_logtalk_flag(?=[(])', Keyword), # Compiling, loading, and library paths - (r'logtalk_(compile|l(ibrary_path|oad|oad_context)|make)(?=[(])', Keyword), + (r'logtalk_(compile|l(ibrary_path|oad|oad_context)|make(_target_action)?)(?=[(])', Keyword), (r'\blogtalk_make\b', Keyword), # Database (r'(clause|retract(all)?)(?=[(])', Keyword), (r'a(bolish|ssert(a|z))(?=[(])', Keyword), # Control constructs (r'(ca(ll|tch)|throw)(?=[(])', Keyword), - (r'(fa(il|lse)|true)\b', Keyword), + (r'(fa(il|lse)|true|(instantiation|system)_error)\b', Keyword), + (r'(type|domain|existence|permission|representation|evaluation|resource|syntax)_error(?=[(])', Keyword), # All solutions (r'((bag|set)of|f(ind|or)all)(?=[(])', Keyword), - # Multi-threading meta-predicates - (r'threaded(_(call|once|ignore|exit|peek|wait|notify))?(?=[(])', Keyword), + # Multi-threading predicates + (r'threaded(_(ca(ll|ncel)|once|ignore|exit|peek|wait|notify))?(?=[(])', Keyword), + # Engine predicates + (r'threaded_engine(_(create|destroy|self|next|next_reified|yield|post|fetch))?(?=[(])', Keyword), # Term unification (r'(subsumes_term|unify_with_occurs_check)(?=[(])', Keyword), # Term creation and decomposition @@ -161,8 +162,7 @@ class LogtalkLexer(RegexLexer): # Other arithmetic functors (r'(cos|a(cos|sin|tan|tan2)|exp|log|s(in|qrt)|xor)(?=[(])', Keyword), # Term testing - (r'(var|atom(ic)?|integer|float|c(allable|ompound)|n(onvar|umber)|' - r'ground|acyclic_term)(?=[(])', Keyword), + (r'(var|atom(ic)?|integer|float|c(allable|ompound)|n(onvar|umber)|ground|acyclic_term)(?=[(])', Keyword), # Term comparison (r'compare(?=[(])', Keyword), # Stream selection and control @@ -226,11 +226,11 @@ class LogtalkLexer(RegexLexer): # Existential quantifier (r'\^', Operator), # Strings - (r'"(\\\\|\\"|[^"])*"', String), - # Ponctuation + (r'"(\\\\|\\[^\\]|[^"\\])*"', String), + # Punctuation (r'[()\[\],.|]', Text), # Atoms - (r"[a-z]\w*", Text), + (r"[a-z][a-zA-Z0-9_]*", Text), (r"'", String, 'quoted_atom'), ], @@ -245,44 +245,43 @@ class LogtalkLexer(RegexLexer): 'directive': [ # Conditional compilation directives (r'(el)?if(?=[(])', Keyword, 'root'), - (r'(e(lse|ndif))[.]', Keyword, 'root'), + (r'(e(lse|ndif))(?=[.])', Keyword, 'root'), # Entity directives (r'(category|object|protocol)(?=[(])', Keyword, 'entityrelations'), - (r'(end_(category|object|protocol))[.]', Keyword, 'root'), + (r'(end_(category|object|protocol))(?=[.])', Keyword, 'root'), # Predicate scope directives (r'(public|protected|private)(?=[(])', Keyword, 'root'), # Other directives (r'e(n(coding|sure_loaded)|xport)(?=[(])', Keyword, 'root'), (r'in(clude|itialization|fo)(?=[(])', Keyword, 'root'), - (r'(built_in|dynamic|synchronized|threaded)[.]', Keyword, 'root'), - (r'(alias|d(ynamic|iscontiguous)|m(eta_(non_terminal|predicate)|ode|ultifile)|' - r's(et_(logtalk|prolog)_flag|ynchronized))(?=[(])', Keyword, 'root'), + (r'(built_in|dynamic|synchronized|threaded)(?=[.])', Keyword, 'root'), + (r'(alias|d(ynamic|iscontiguous)|m(eta_(non_terminal|predicate)|ode|ultifile)|s(et_(logtalk|prolog)_flag|ynchronized))(?=[(])', Keyword, 'root'), (r'op(?=[(])', Keyword, 'root'), (r'(c(alls|oinductive)|module|reexport|use(s|_module))(?=[(])', Keyword, 'root'), - (r'[a-z]\w*(?=[(])', Text, 'root'), - (r'[a-z]\w*[.]', Text, 'root'), + (r'[a-z][a-zA-Z0-9_]*(?=[(])', Text, 'root'), + (r'[a-z][a-zA-Z0-9_]*(?=[.])', Text, 'root'), ], 'entityrelations': [ (r'(complements|extends|i(nstantiates|mp(lements|orts))|specializes)(?=[(])', Keyword), # Numbers - (r"0'.", Number), + (r"0'[\\]?.", Number), (r'0b[01]+', Number.Bin), (r'0o[0-7]+', Number.Oct), (r'0x[0-9a-fA-F]+', Number.Hex), (r'\d+\.?\d*((e|E)(\+|-)?\d+)?', Number), # Variables - (r'([A-Z_]\w*)', Name.Variable), + (r'([A-Z_][a-zA-Z0-9_]*)', Name.Variable), # Atoms - (r"[a-z]\w*", Text), + (r"[a-z][a-zA-Z0-9_]*", Text), (r"'", String, 'quoted_atom'), # Strings - (r'"(\\\\|\\"|[^"])*"', String), + (r'"(\\\\|\\[^\\]|[^"\\])*"', String), # End of entity-opening directive (r'([)]\.)', Text, 'root'), # Scope operator (r'(::)', Operator), - # Ponctuation + # Punctuation (r'[()\[\],.|]', Text), # Comments (r'%.*?\n', Comment), @@ -300,7 +299,7 @@ def analyse_text(text): return 1.0 elif ':- category(' in text: return 1.0 - elif re.search('^:-\s[a-z]', text, re.M): + elif re.search(r'^:-\s[a-z]', text, re.M): return 0.9 else: return 0.0 diff --git a/src/syntax_highlighting/libs/pygments/lexers/promql.py b/src/syntax_highlighting/libs/pygments/lexers/promql.py new file mode 100644 index 0000000..b9646d4 --- /dev/null +++ b/src/syntax_highlighting/libs/pygments/lexers/promql.py @@ -0,0 +1,182 @@ +""" + pygments.lexers.promql + ~~~~~~~~~~~~~~~~~~~~~~ + + Lexer for Prometheus Query Language. + + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer, bygroups, default, words +from pygments.token import ( + Comment, + Keyword, + Name, + Number, + Operator, + Punctuation, + String, + Whitespace, +) + +__all__ = ["PromQLLexer"] + + +class PromQLLexer(RegexLexer): + """ + For `PromQL `_ queries. + + For details about the grammar see: + https://github.com/prometheus/prometheus/tree/master/promql/parser + + .. versionadded: 2.7 + """ + + name = "PromQL" + aliases = ["promql"] + filenames = ["*.promql"] + + base_keywords = ( + words( + ( + "bool", + "by", + "group_left", + "group_right", + "ignoring", + "offset", + "on", + "without", + ), + suffix=r"\b", + ), + Keyword, + ) + + aggregator_keywords = ( + words( + ( + "sum", + "min", + "max", + "avg", + "group", + "stddev", + "stdvar", + "count", + "count_values", + "bottomk", + "topk", + "quantile", + ), + suffix=r"\b", + ), + Keyword, + ) + + function_keywords = ( + words( + ( + "abs", + "absent", + "absent_over_time", + "avg_over_time", + "ceil", + "changes", + "clamp_max", + "clamp_min", + "count_over_time", + "day_of_month", + "day_of_week", + "days_in_month", + "delta", + "deriv", + "exp", + "floor", + "histogram_quantile", + "holt_winters", + "hour", + "idelta", + "increase", + "irate", + "label_join", + "label_replace", + "ln", + "log10", + "log2", + "max_over_time", + "min_over_time", + "minute", + "month", + "predict_linear", + "quantile_over_time", + "rate", + "resets", + "round", + "scalar", + "sort", + "sort_desc", + "sqrt", + "stddev_over_time", + "stdvar_over_time", + "sum_over_time", + "time", + "timestamp", + "vector", + "year", + ), + suffix=r"\b", + ), + Keyword.Reserved, + ) + + tokens = { + "root": [ + (r"\n", Whitespace), + (r"\s+", Whitespace), + (r",", Punctuation), + # Keywords + base_keywords, + aggregator_keywords, + function_keywords, + # Offsets + (r"[1-9][0-9]*[smhdwy]", String), + # Numbers + (r"-?[0-9]+\.[0-9]+", Number.Float), + (r"-?[0-9]+", Number.Integer), + # Comments + (r"#.*?$", Comment.Single), + # Operators + (r"(\+|\-|\*|\/|\%|\^)", Operator), + (r"==|!=|>=|<=|<|>", Operator), + (r"and|or|unless", Operator.Word), + # Metrics + (r"[_a-zA-Z][a-zA-Z0-9_]+", Name.Variable), + # Params + (r'(["\'])(.*?)(["\'])', bygroups(Punctuation, String, Punctuation)), + # Other states + (r"\(", Operator, "function"), + (r"\)", Operator), + (r"\{", Punctuation, "labels"), + (r"\[", Punctuation, "range"), + ], + "labels": [ + (r"\}", Punctuation, "#pop"), + (r"\n", Whitespace), + (r"\s+", Whitespace), + (r",", Punctuation), + (r'([_a-zA-Z][a-zA-Z0-9_]*?)(\s*?)(=~|!=|=|!~)(\s*?)("|\')(.*?)("|\')', + bygroups(Name.Label, Whitespace, Operator, Whitespace, + Punctuation, String, Punctuation)), + ], + "range": [ + (r"\]", Punctuation, "#pop"), + (r"[1-9][0-9]*[smhdwy]", String), + ], + "function": [ + (r"\)", Operator, "#pop"), + (r"\(", Operator, "#push"), + default("#pop"), + ], + } diff --git a/src/syntax_highlighting/libs/pygments/lexers/python.py b/src/syntax_highlighting/libs/pygments/lexers/python.py index 390eafe..9db7a2d 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/python.py +++ b/src/syntax_highlighting/libs/pygments/lexers/python.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.python ~~~~~~~~~~~~~~~~~~~~~~ Lexers for Python and related languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -19,21 +18,387 @@ from pygments import unistring as uni __all__ = ['PythonLexer', 'PythonConsoleLexer', 'PythonTracebackLexer', - 'Python3Lexer', 'Python3TracebackLexer', 'CythonLexer', - 'DgLexer', 'NumPyLexer'] + 'Python2Lexer', 'Python2TracebackLexer', + 'CythonLexer', 'DgLexer', 'NumPyLexer'] line_re = re.compile('.*?\n') class PythonLexer(RegexLexer): """ - For `Python `_ source code. + For `Python `_ source code (version 3.x). + + .. versionadded:: 0.10 + + .. versionchanged:: 2.5 + This is now the default ``PythonLexer``. It is still available as the + alias ``Python3Lexer``. """ name = 'Python' - aliases = ['python', 'py', 'sage'] - filenames = ['*.py', '*.pyw', '*.sc', 'SConstruct', 'SConscript', '*.tac', '*.sage'] - mimetypes = ['text/x-python', 'application/x-python'] + aliases = ['python', 'py', 'sage', 'python3', 'py3'] + filenames = [ + '*.py', + '*.pyw', + # Jython + '*.jy', + # Sage + '*.sage', + # SCons + '*.sc', + 'SConstruct', + 'SConscript', + # Skylark/Starlark (used by Bazel, Buck, and Pants) + '*.bzl', + 'BUCK', + 'BUILD', + 'BUILD.bazel', + 'WORKSPACE', + # Twisted Application infrastructure + '*.tac', + ] + mimetypes = ['text/x-python', 'application/x-python', + 'text/x-python3', 'application/x-python3'] + + flags = re.MULTILINE | re.UNICODE + + uni_name = "[%s][%s]*" % (uni.xid_start, uni.xid_continue) + + def innerstring_rules(ttype): + return [ + # the old style '%s' % (...) string formatting (still valid in Py3) + (r'%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?' + '[hlL]?[E-GXc-giorsaux%]', String.Interpol), + # the new style '{}'.format(...) string formatting + (r'\{' + r'((\w+)((\.\w+)|(\[[^\]]+\]))*)?' # field name + r'(\![sra])?' # conversion + r'(\:(.?[<>=\^])?[-+ ]?#?0?(\d+)?,?(\.\d+)?[E-GXb-gnosx%]?)?' + r'\}', String.Interpol), + + # backslashes, quotes and formatting signs must be parsed one at a time + (r'[^\\\'"%{\n]+', ttype), + (r'[\'"\\]', ttype), + # unhandled string formatting sign + (r'%|(\{{1,2})', ttype) + # newlines are an error (use "nl" state) + ] + + def fstring_rules(ttype): + return [ + # Assuming that a '}' is the closing brace after format specifier. + # Sadly, this means that we won't detect syntax error. But it's + # more important to parse correct syntax correctly, than to + # highlight invalid syntax. + (r'\}', String.Interpol), + (r'\{', String.Interpol, 'expr-inside-fstring'), + # backslashes, quotes and formatting signs must be parsed one at a time + (r'[^\\\'"{}\n]+', ttype), + (r'[\'"\\]', ttype), + # newlines are an error (use "nl" state) + ] + + tokens = { + 'root': [ + (r'\n', Text), + (r'^(\s*)([rRuUbB]{,2})("""(?:.|\n)*?""")', + bygroups(Text, String.Affix, String.Doc)), + (r"^(\s*)([rRuUbB]{,2})('''(?:.|\n)*?''')", + bygroups(Text, String.Affix, String.Doc)), + (r'\A#!.+$', Comment.Hashbang), + (r'#.*$', Comment.Single), + (r'\\\n', Text), + (r'\\', Text), + include('keywords'), + (r'(def)((?:\s|\\\s)+)', bygroups(Keyword, Text), 'funcname'), + (r'(class)((?:\s|\\\s)+)', bygroups(Keyword, Text), 'classname'), + (r'(from)((?:\s|\\\s)+)', bygroups(Keyword.Namespace, Text), + 'fromimport'), + (r'(import)((?:\s|\\\s)+)', bygroups(Keyword.Namespace, Text), + 'import'), + include('expr'), + ], + 'expr': [ + # raw f-strings + ('(?i)(rf|fr)(""")', + bygroups(String.Affix, String.Double), + combined('rfstringescape', 'tdqf')), + ("(?i)(rf|fr)(''')", + bygroups(String.Affix, String.Single), + combined('rfstringescape', 'tsqf')), + ('(?i)(rf|fr)(")', + bygroups(String.Affix, String.Double), + combined('rfstringescape', 'dqf')), + ("(?i)(rf|fr)(')", + bygroups(String.Affix, String.Single), + combined('rfstringescape', 'sqf')), + # non-raw f-strings + ('([fF])(""")', bygroups(String.Affix, String.Double), + combined('fstringescape', 'tdqf')), + ("([fF])(''')", bygroups(String.Affix, String.Single), + combined('fstringescape', 'tsqf')), + ('([fF])(")', bygroups(String.Affix, String.Double), + combined('fstringescape', 'dqf')), + ("([fF])(')", bygroups(String.Affix, String.Single), + combined('fstringescape', 'sqf')), + # raw strings + ('(?i)(rb|br|r)(""")', + bygroups(String.Affix, String.Double), 'tdqs'), + ("(?i)(rb|br|r)(''')", + bygroups(String.Affix, String.Single), 'tsqs'), + ('(?i)(rb|br|r)(")', + bygroups(String.Affix, String.Double), 'dqs'), + ("(?i)(rb|br|r)(')", + bygroups(String.Affix, String.Single), 'sqs'), + # non-raw strings + ('([uUbB]?)(""")', bygroups(String.Affix, String.Double), + combined('stringescape', 'tdqs')), + ("([uUbB]?)(''')", bygroups(String.Affix, String.Single), + combined('stringescape', 'tsqs')), + ('([uUbB]?)(")', bygroups(String.Affix, String.Double), + combined('stringescape', 'dqs')), + ("([uUbB]?)(')", bygroups(String.Affix, String.Single), + combined('stringescape', 'sqs')), + (r'[^\S\n]+', Text), + include('numbers'), + (r'!=|==|<<|>>|:=|[-~+/*%=<>&^|.]', Operator), + (r'[]{}:(),;[]', Punctuation), + (r'(in|is|and|or|not)\b', Operator.Word), + include('expr-keywords'), + include('builtins'), + include('magicfuncs'), + include('magicvars'), + include('name'), + ], + 'expr-inside-fstring': [ + (r'[{([]', Punctuation, 'expr-inside-fstring-inner'), + # without format specifier + (r'(=\s*)?' # debug (https://bugs.python.org/issue36817) + r'(\![sraf])?' # conversion + r'\}', String.Interpol, '#pop'), + # with format specifier + # we'll catch the remaining '}' in the outer scope + (r'(=\s*)?' # debug (https://bugs.python.org/issue36817) + r'(\![sraf])?' # conversion + r':', String.Interpol, '#pop'), + (r'\s+', Text), # allow new lines + include('expr'), + ], + 'expr-inside-fstring-inner': [ + (r'[{([]', Punctuation, 'expr-inside-fstring-inner'), + (r'[])}]', Punctuation, '#pop'), + (r'\s+', Text), # allow new lines + include('expr'), + ], + 'expr-keywords': [ + # Based on https://docs.python.org/3/reference/expressions.html + (words(( + 'async for', 'await', 'else', 'for', 'if', 'lambda', + 'yield', 'yield from'), suffix=r'\b'), + Keyword), + (words(('True', 'False', 'None'), suffix=r'\b'), Keyword.Constant), + ], + 'keywords': [ + (words(( + 'assert', 'async', 'await', 'break', 'continue', 'del', 'elif', + 'else', 'except', 'finally', 'for', 'global', 'if', 'lambda', + 'pass', 'raise', 'nonlocal', 'return', 'try', 'while', 'yield', + 'yield from', 'as', 'with'), suffix=r'\b'), + Keyword), + (words(('True', 'False', 'None'), suffix=r'\b'), Keyword.Constant), + ], + 'builtins': [ + (words(( + '__import__', 'abs', 'all', 'any', 'bin', 'bool', 'bytearray', + 'bytes', 'chr', 'classmethod', 'compile', 'complex', + 'delattr', 'dict', 'dir', 'divmod', 'enumerate', 'eval', 'filter', + 'float', 'format', 'frozenset', 'getattr', 'globals', 'hasattr', + 'hash', 'hex', 'id', 'input', 'int', 'isinstance', 'issubclass', + 'iter', 'len', 'list', 'locals', 'map', 'max', 'memoryview', + 'min', 'next', 'object', 'oct', 'open', 'ord', 'pow', 'print', + 'property', 'range', 'repr', 'reversed', 'round', 'set', 'setattr', + 'slice', 'sorted', 'staticmethod', 'str', 'sum', 'super', 'tuple', + 'type', 'vars', 'zip'), prefix=r'(?`_ source code. + + .. versionchanged:: 2.5 + This class has been renamed from ``PythonLexer``. ``PythonLexer`` now + refers to the Python 3 variant. File name patterns like ``*.py`` have + been moved to Python 3 as well. + """ + + name = 'Python 2.x' + aliases = ['python2', 'py2'] + filenames = [] # now taken over by PythonLexer (3.x) + mimetypes = ['text/x-python2', 'application/x-python2'] def innerstring_rules(ttype): return [ @@ -124,15 +489,15 @@ def innerstring_rules(ttype): 'Exception', 'FloatingPointError', 'FutureWarning', 'GeneratorExit', 'IOError', 'ImportError', 'ImportWarning', 'IndentationError', 'IndexError', 'KeyError', 'KeyboardInterrupt', 'LookupError', - 'MemoryError', 'NameError', 'NotImplemented', 'NotImplementedError', - 'OSError', 'OverflowError', 'OverflowWarning', 'PendingDeprecationWarning', - 'ReferenceError', 'RuntimeError', 'RuntimeWarning', 'StandardError', - 'StopIteration', 'SyntaxError', 'SyntaxWarning', 'SystemError', - 'SystemExit', 'TabError', 'TypeError', 'UnboundLocalError', - 'UnicodeDecodeError', 'UnicodeEncodeError', 'UnicodeError', - 'UnicodeTranslateError', 'UnicodeWarning', 'UserWarning', - 'ValueError', 'VMSError', 'Warning', 'WindowsError', - 'ZeroDivisionError'), prefix=r'(?`_ source code (version 3.0). - - .. versionadded:: 0.10 - """ - - name = 'Python 3' - aliases = ['python3', 'py3'] - filenames = [] # Nothing until Python 3 gets widespread - mimetypes = ['text/x-python3', 'application/x-python3'] - - flags = re.MULTILINE | re.UNICODE - - uni_name = "[%s][%s]*" % (uni.xid_start, uni.xid_continue) - - def innerstring_rules(ttype): - return [ - # the old style '%s' % (...) string formatting (still valid in Py3) - (r'%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?' - '[hlL]?[E-GXc-giorsux%]', String.Interpol), - # the new style '{}'.format(...) string formatting - (r'\{' - '((\w+)((\.\w+)|(\[[^\]]+\]))*)?' # field name - '(\![sra])?' # conversion - '(\:(.?[<>=\^])?[-+ ]?#?0?(\d+)?,?(\.\d+)?[E-GXb-gnosx%]?)?' - '\}', String.Interpol), - - # backslashes, quotes and formatting signs must be parsed one at a time - (r'[^\\\'"%{\n]+', ttype), - (r'[\'"\\]', ttype), - # unhandled string formatting sign - (r'%|(\{{1,2})', ttype) - # newlines are an error (use "nl" state) - ] - - tokens = PythonLexer.tokens.copy() - tokens['keywords'] = [ - (words(( - 'assert', 'async', 'await', 'break', 'continue', 'del', 'elif', - 'else', 'except', 'finally', 'for', 'global', 'if', 'lambda', 'pass', - 'raise', 'nonlocal', 'return', 'try', 'while', 'yield', 'yield from', - 'as', 'with'), suffix=r'\b'), - Keyword), - (words(( - 'True', 'False', 'None'), suffix=r'\b'), - Keyword.Constant), - ] - tokens['builtins'] = [ - (words(( - '__import__', 'abs', 'all', 'any', 'bin', 'bool', 'bytearray', 'bytes', - 'chr', 'classmethod', 'cmp', 'compile', 'complex', 'delattr', 'dict', - 'dir', 'divmod', 'enumerate', 'eval', 'filter', 'float', 'format', - 'frozenset', 'getattr', 'globals', 'hasattr', 'hash', 'hex', 'id', - 'input', 'int', 'isinstance', 'issubclass', 'iter', 'len', 'list', - 'locals', 'map', 'max', 'memoryview', 'min', 'next', 'object', 'oct', - 'open', 'ord', 'pow', 'print', 'property', 'range', 'repr', 'reversed', - 'round', 'set', 'setattr', 'slice', 'sorted', 'staticmethod', 'str', - 'sum', 'super', 'tuple', 'type', 'vars', 'zip'), prefix=r'(?>> ') or line.startswith(u'... '): + if line.startswith('>>> ') or line.startswith('... '): tb = 0 insertions.append((len(curcode), [(0, Generic.Prompt, line[:4])])) curcode += line[4:] - elif line.rstrip() == u'...' and not tb: + elif line.rstrip() == '...' and not tb: # only a new >>> prompt can end an exception block # otherwise an ellipsis in place of the traceback frames # will be mishandled insertions.append((len(curcode), - [(0, Generic.Prompt, u'...')])) + [(0, Generic.Prompt, '...')])) curcode += line[3:] else: if curcode: - for item in do_insertions( - insertions, pylexer.get_tokens_unprocessed(curcode)): - yield item + yield from do_insertions( + insertions, pylexer.get_tokens_unprocessed(curcode)) curcode = '' insertions = [] - if (line.startswith(u'Traceback (most recent call last):') or - re.match(u' File "[^"]+", line \\d+\\n$', line)): + if (line.startswith('Traceback (most recent call last):') or + re.match(' File "[^"]+", line \\d+\\n$', line)): tb = 1 curtb = line tbindex = match.start() @@ -472,7 +679,7 @@ def get_tokens_unprocessed(self, text): yield match.start(), Name.Class, line elif tb: curtb += line - if not (line.startswith(' ') or line.strip() == u'...'): + if not (line.startswith(' ') or line.strip() == '...'): tb = 0 for i, t, v in tblexer.get_tokens_unprocessed(curtb): yield tbindex+i, t, v @@ -480,9 +687,8 @@ def get_tokens_unprocessed(self, text): else: yield match.start(), Generic.Output, line if curcode: - for item in do_insertions(insertions, - pylexer.get_tokens_unprocessed(curcode)): - yield item + yield from do_insertions(insertions, + pylexer.get_tokens_unprocessed(curcode)) if curtb: for i, t, v in tblexer.get_tokens_unprocessed(curtb): yield tbindex+i, t, v @@ -490,23 +696,28 @@ def get_tokens_unprocessed(self, text): class PythonTracebackLexer(RegexLexer): """ - For Python tracebacks. + For Python 3.x tracebacks, with support for chained exceptions. - .. versionadded:: 0.7 + .. versionadded:: 1.0 + + .. versionchanged:: 2.5 + This is now the default ``PythonTracebackLexer``. It is still available + as the alias ``Python3TracebackLexer``. """ name = 'Python Traceback' - aliases = ['pytb'] - filenames = ['*.pytb'] - mimetypes = ['text/x-python-traceback'] + aliases = ['pytb', 'py3tb'] + filenames = ['*.pytb', '*.py3tb'] + mimetypes = ['text/x-python-traceback', 'text/x-python3-traceback'] tokens = { 'root': [ - # Cover both (most recent call last) and (innermost last) - # The optional ^C allows us to catch keyboard interrupt signals. - (r'^(\^C)?(Traceback.*\n)', - bygroups(Text, Generic.Traceback), 'intb'), - # SyntaxError starts with this. + (r'\n', Text), + (r'^Traceback \(most recent call last\):\n', Generic.Traceback, 'intb'), + (r'^During handling of the above exception, another ' + r'exception occurred:\n\n', Generic.Traceback), + (r'^The above exception was the direct cause of the ' + r'following exception:\n\n', Generic.Traceback), (r'^(?= File "[^"]+", line \d+)', Generic.Traceback, 'intb'), (r'^.*\n', Other), ], @@ -516,38 +727,54 @@ class PythonTracebackLexer(RegexLexer): (r'^( File )("[^"]+")(, line )(\d+)(\n)', bygroups(Text, Name.Builtin, Text, Number, Text)), (r'^( )(.+)(\n)', - bygroups(Text, using(PythonLexer), Text)), + bygroups(Text, using(PythonLexer), Text), 'markers'), (r'^([ \t]*)(\.\.\.)(\n)', bygroups(Text, Comment, Text)), # for doctests... (r'^([^:]+)(: )(.+)(\n)', bygroups(Generic.Error, Text, Name, Text), '#pop'), - (r'^([a-zA-Z_]\w*)(:?\n)', + (r'^([a-zA-Z_][\w.]*)(:?\n)', bygroups(Generic.Error, Text), '#pop') ], + 'markers': [ + # Either `PEP 657 ` + # error locations in Python 3.11+, or single-caret markers + # for syntax errors before that. + (r'^( {4,})(\^+)(\n)', + bygroups(Text, Punctuation.Marker, Text), + '#pop'), + default('#pop'), + ], } -class Python3TracebackLexer(RegexLexer): +Python3TracebackLexer = PythonTracebackLexer + + +class Python2TracebackLexer(RegexLexer): """ - For Python 3.0 tracebacks, with support for chained exceptions. + For Python tracebacks. - .. versionadded:: 1.0 + .. versionadded:: 0.7 + + .. versionchanged:: 2.5 + This class has been renamed from ``PythonTracebackLexer``. + ``PythonTracebackLexer`` now refers to the Python 3 variant. """ - name = 'Python 3.0 Traceback' - aliases = ['py3tb'] - filenames = ['*.py3tb'] - mimetypes = ['text/x-python3-traceback'] + name = 'Python 2.x Traceback' + aliases = ['py2tb'] + filenames = ['*.py2tb'] + mimetypes = ['text/x-python2-traceback'] tokens = { 'root': [ - (r'\n', Text), - (r'^Traceback \(most recent call last\):\n', Generic.Traceback, 'intb'), - (r'^During handling of the above exception, another ' - r'exception occurred:\n\n', Generic.Traceback), - (r'^The above exception was the direct cause of the ' - r'following exception:\n\n', Generic.Traceback), + # Cover both (most recent call last) and (innermost last) + # The optional ^C allows us to catch keyboard interrupt signals. + (r'^(\^C)?(Traceback.*\n)', + bygroups(Text, Generic.Traceback), 'intb'), + # SyntaxError starts with this. (r'^(?= File "[^"]+", line \d+)', Generic.Traceback, 'intb'), + (r'^.*\n', Other), ], 'intb': [ (r'^( File )("[^"]+")(, line )(\d+)(, in )(.+)(\n)', @@ -555,7 +782,7 @@ class Python3TracebackLexer(RegexLexer): (r'^( File )("[^"]+")(, line )(\d+)(\n)', bygroups(Text, Name.Builtin, Text, Number, Text)), (r'^( )(.+)(\n)', - bygroups(Text, using(Python3Lexer), Text)), + bygroups(Text, using(Python2Lexer), Text), 'marker'), (r'^([ \t]*)(\.\.\.)(\n)', bygroups(Text, Comment, Text)), # for doctests... (r'^([^:]+)(: )(.+)(\n)', @@ -563,6 +790,11 @@ class Python3TracebackLexer(RegexLexer): (r'^([a-zA-Z_]\w*)(:?\n)', bygroups(Generic.Error, Text), '#pop') ], + 'marker': [ + # For syntax errors. + (r'( {4,})(\^)', bygroups(Text, Punctuation.Marker), '#pop'), + default('#pop'), + ], } @@ -618,7 +850,7 @@ class CythonLexer(RegexLexer): ], 'keywords': [ (words(( - 'assert', 'break', 'by', 'continue', 'ctypedef', 'del', 'elif', + 'assert', 'async', 'await', 'break', 'by', 'continue', 'ctypedef', 'del', 'elif', 'else', 'except', 'except?', 'exec', 'finally', 'for', 'fused', 'gil', 'global', 'if', 'include', 'lambda', 'nogil', 'pass', 'print', 'raise', 'return', 'try', 'while', 'yield', 'as', 'with'), suffix=r'\b'), @@ -627,14 +859,14 @@ class CythonLexer(RegexLexer): ], 'builtins': [ (words(( - '__import__', 'abs', 'all', 'any', 'apply', 'basestring', 'bin', + '__import__', 'abs', 'all', 'any', 'apply', 'basestring', 'bin', 'bint', 'bool', 'buffer', 'bytearray', 'bytes', 'callable', 'chr', 'classmethod', 'cmp', 'coerce', 'compile', 'complex', 'delattr', 'dict', 'dir', 'divmod', 'enumerate', 'eval', 'execfile', 'exit', 'file', 'filter', 'float', 'frozenset', 'getattr', 'globals', 'hasattr', 'hash', 'hex', 'id', 'input', 'int', 'intern', 'isinstance', 'issubclass', 'iter', 'len', 'list', 'locals', 'long', 'map', 'max', - 'min', 'next', 'object', 'oct', 'open', 'ord', 'pow', 'property', + 'min', 'next', 'object', 'oct', 'open', 'ord', 'pow', 'property', 'Py_ssize_t', 'range', 'raw_input', 'reduce', 'reload', 'repr', 'reversed', 'round', 'set', 'setattr', 'slice', 'sorted', 'staticmethod', 'str', 'sum', 'super', 'tuple', 'type', 'unichr', 'unicode', 'unsigned', @@ -671,10 +903,10 @@ class CythonLexer(RegexLexer): ], 'name': [ (r'@\w+', Name.Decorator), - ('[a-zA-Z_]\w*', Name), + (r'[a-zA-Z_]\w*', Name), ], 'funcname': [ - ('[a-zA-Z_]\w*', Name.Function, '#pop') + (r'[a-zA-Z_]\w*', Name.Function, '#pop') ], 'cdef': [ (r'(public|readonly|extern|api|inline)\b', Keyword.Reserved), @@ -691,7 +923,7 @@ class CythonLexer(RegexLexer): (r'.', Text), ], 'classname': [ - ('[a-zA-Z_]\w*', Name.Class, '#pop') + (r'[a-zA-Z_]\w*', Name.Class, '#pop') ], 'import': [ (r'(\s+)(as)(\s+)', bygroups(Text, Keyword, Text)), @@ -858,7 +1090,7 @@ class NumPyLexer(PythonLexer): mimetypes = [] filenames = [] - EXTRA_KEYWORDS = set(( + EXTRA_KEYWORDS = { 'abs', 'absolute', 'accumulate', 'add', 'alen', 'all', 'allclose', 'alltrue', 'alterdot', 'amax', 'amin', 'angle', 'any', 'append', 'apply_along_axis', 'apply_over_axes', 'arange', 'arccos', 'arccosh', @@ -923,7 +1155,7 @@ class NumPyLexer(PythonLexer): 'typename', 'uniform', 'union1d', 'unique', 'unique1d', 'unravel_index', 'unwrap', 'vander', 'var', 'vdot', 'vectorize', 'view', 'vonmises', 'vsplit', 'vstack', 'weibull', 'where', 'who', 'zeros', 'zeros_like' - )) + } def get_tokens_unprocessed(self, text): for index, token, value in \ @@ -934,6 +1166,7 @@ def get_tokens_unprocessed(self, text): yield index, token, value def analyse_text(text): - return (shebang_matches(text, r'pythonw?(2(\.\d)?)?') or - 'import ' in text[:1000]) \ - and ('import numpy' in text or 'from numpy import' in text) + ltext = text[:1000] + return (shebang_matches(text, r'pythonw?(3(\.\d)?)?') or + 'import ' in ltext) \ + and ('import numpy' in ltext or 'from numpy import' in ltext) diff --git a/src/syntax_highlighting/libs/pygments/lexers/qvt.py b/src/syntax_highlighting/libs/pygments/lexers/qvt.py index f496d60..72817f0 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/qvt.py +++ b/src/syntax_highlighting/libs/pygments/lexers/qvt.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.qvt ~~~~~~~~~~~~~~~~~~~ Lexer for QVT Operational language. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -126,7 +125,7 @@ class QVToLexer(RegexLexer): (r'[^\\\'"\n]+', String), # quotes, percents and backslashes must be parsed one at a time (r'[\'"\\]', String), - ], + ], 'stringescape': [ (r'\\([\\btnfr"\']|u[0-3][0-7]{2}|u[0-7]{1,2})', String.Escape) ], @@ -134,15 +133,15 @@ class QVToLexer(RegexLexer): (r'"', String, '#pop'), (r'\\\\|\\"', String.Escape), include('strings') - ], + ], 'sqs': [ # single-quoted string (r"'", String, '#pop'), (r"\\\\|\\'", String.Escape), include('strings') - ], + ], 'name': [ - ('[a-zA-Z_]\w*', Name), - ], + (r'[a-zA-Z_]\w*', Name), + ], # numbers: excerpt taken from the python lexer 'numbers': [ (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float), diff --git a/src/syntax_highlighting/libs/pygments/lexers/r.py b/src/syntax_highlighting/libs/pygments/lexers/r.py index dce6196..643fb3a 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/r.py +++ b/src/syntax_highlighting/libs/pygments/lexers/r.py @@ -1,17 +1,16 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.r ~~~~~~~~~~~~~~~~~ Lexers for the R/S languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ import re -from pygments.lexer import Lexer, RegexLexer, include, words, do_insertions +from pygments.lexer import Lexer, RegexLexer, include, do_insertions, bygroups from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ Number, Punctuation, Generic @@ -49,9 +48,8 @@ def get_tokens_unprocessed(self, text): # If we have stored prompt lines, need to process them first. if current_code_block: # Weave together the prompts and highlight code. - for item in do_insertions( - insertions, slexer.get_tokens_unprocessed(current_code_block)): - yield item + yield from do_insertions( + insertions, slexer.get_tokens_unprocessed(current_code_block)) # Reset vars for next code block. current_code_block = '' insertions = [] @@ -62,9 +60,8 @@ def get_tokens_unprocessed(self, text): # process the last code block. This is neither elegant nor DRY so # should be changed. if current_code_block: - for item in do_insertions( - insertions, slexer.get_tokens_unprocessed(current_code_block)): - yield item + yield from do_insertions( + insertions, slexer.get_tokens_unprocessed(current_code_block)) class SLexer(RegexLexer): @@ -80,286 +77,25 @@ class SLexer(RegexLexer): mimetypes = ['text/S-plus', 'text/S', 'text/x-r-source', 'text/x-r', 'text/x-R', 'text/x-r-history', 'text/x-r-profile'] - builtins_base = ( - 'Arg', 'Conj', 'Cstack_info', 'Encoding', 'FALSE', - 'Filter', 'Find', 'I', 'ISOdate', 'ISOdatetime', 'Im', 'Inf', - 'La.svd', 'Map', 'Math.Date', 'Math.POSIXt', 'Math.data.frame', - 'Math.difftime', 'Math.factor', 'Mod', 'NA_character_', - 'NA_complex_', 'NA_real_', 'NCOL', 'NROW', 'NULLNA_integer_', 'NaN', - 'Negate', 'NextMethod', 'Ops.Date', 'Ops.POSIXt', 'Ops.data.frame', - 'Ops.difftime', 'Ops.factor', 'Ops.numeric_version', 'Ops.ordered', - 'Position', 'R.Version', 'R.home', 'R.version', 'R.version.string', - 'RNGkind', 'RNGversion', 'R_system_version', 'Re', 'Recall', - 'Reduce', 'Summary.Date', 'Summary.POSIXct', 'Summary.POSIXlt', - 'Summary.data.frame', 'Summary.difftime', 'Summary.factor', - 'Summary.numeric_version', 'Summary.ordered', 'Sys.Date', - 'Sys.chmod', 'Sys.getenv', 'Sys.getlocale', 'Sys.getpid', - 'Sys.glob', 'Sys.info', 'Sys.localeconv', 'Sys.readlink', - 'Sys.setFileTime', 'Sys.setenv', 'Sys.setlocale', 'Sys.sleep', - 'Sys.time', 'Sys.timezone', 'Sys.umask', 'Sys.unsetenv', - 'Sys.which', 'TRUE', 'UseMethod', 'Vectorize', 'abbreviate', 'abs', - 'acos', 'acosh', 'addNA', 'addTaskCallback', 'agrep', 'alist', - 'all', 'all.equal', 'all.equal.POSIXct', 'all.equal.character', - 'all.equal.default', 'all.equal.factor', 'all.equal.formula', - 'all.equal.language', 'all.equal.list', 'all.equal.numeric', - 'all.equal.raw', 'all.names', 'all.vars', 'any', 'anyDuplicated', - 'anyDuplicated.array', 'anyDuplicated.data.frame', - 'anyDuplicated.default', 'anyDuplicated.matrix', 'aperm', - 'aperm.default', 'aperm.table', 'append', 'apply', 'args', - 'arrayInd', 'as.Date', 'as.Date.POSIXct', 'as.Date.POSIXlt', - 'as.Date.character', 'as.Date.date', 'as.Date.dates', - 'as.Date.default', 'as.Date.factor', 'as.Date.numeric', - 'as.POSIXct', 'as.POSIXct.Date', 'as.POSIXct.POSIXlt', - 'as.POSIXct.date', 'as.POSIXct.dates', 'as.POSIXct.default', - 'as.POSIXct.numeric', 'as.POSIXlt', 'as.POSIXlt.Date', - 'as.POSIXlt.POSIXct', 'as.POSIXlt.character', 'as.POSIXlt.date', - 'as.POSIXlt.dates', 'as.POSIXlt.default', 'as.POSIXlt.factor', - 'as.POSIXlt.numeric', 'as.array', 'as.array.default', 'as.call', - 'as.character', 'as.character.Date', 'as.character.POSIXt', - 'as.character.condition', 'as.character.default', - 'as.character.error', 'as.character.factor', 'as.character.hexmode', - 'as.character.numeric_version', 'as.character.octmode', - 'as.character.srcref', 'as.complex', 'as.data.frame', - 'as.data.frame.AsIs', 'as.data.frame.Date', 'as.data.frame.POSIXct', - 'as.data.frame.POSIXlt', 'as.data.frame.array', - 'as.data.frame.character', 'as.data.frame.complex', - 'as.data.frame.data.frame', 'as.data.frame.default', - 'as.data.frame.difftime', 'as.data.frame.factor', - 'as.data.frame.integer', 'as.data.frame.list', - 'as.data.frame.logical', 'as.data.frame.matrix', - 'as.data.frame.model.matrix', 'as.data.frame.numeric', - 'as.data.frame.numeric_version', 'as.data.frame.ordered', - 'as.data.frame.raw', 'as.data.frame.table', 'as.data.frame.ts', - 'as.data.frame.vector', 'as.difftime', 'as.double', - 'as.double.POSIXlt', 'as.double.difftime', 'as.environment', - 'as.expression', 'as.expression.default', 'as.factor', - 'as.function', 'as.function.default', 'as.hexmode', 'as.integer', - 'as.list', 'as.list.Date', 'as.list.POSIXct', 'as.list.data.frame', - 'as.list.default', 'as.list.environment', 'as.list.factor', - 'as.list.function', 'as.list.numeric_version', 'as.logical', - 'as.logical.factor', 'as.matrix', 'as.matrix.POSIXlt', - 'as.matrix.data.frame', 'as.matrix.default', 'as.matrix.noquote', - 'as.name', 'as.null', 'as.null.default', 'as.numeric', - 'as.numeric_version', 'as.octmode', 'as.ordered', - 'as.package_version', 'as.pairlist', 'as.qr', 'as.raw', 'as.single', - 'as.single.default', 'as.symbol', 'as.table', 'as.table.default', - 'as.vector', 'as.vector.factor', 'asNamespace', 'asS3', 'asS4', - 'asin', 'asinh', 'assign', 'atan', 'atan2', 'atanh', - 'attachNamespace', 'attr', 'attr.all.equal', 'attributes', - 'autoload', 'autoloader', 'backsolve', 'baseenv', 'basename', - 'besselI', 'besselJ', 'besselK', 'besselY', 'beta', - 'bindingIsActive', 'bindingIsLocked', 'bindtextdomain', 'bitwAnd', - 'bitwNot', 'bitwOr', 'bitwShiftL', 'bitwShiftR', 'bitwXor', 'body', - 'bquote', 'browser', 'browserCondition', 'browserSetDebug', - 'browserText', 'builtins', 'by', 'by.data.frame', 'by.default', - 'bzfile', 'c.Date', 'c.POSIXct', 'c.POSIXlt', 'c.noquote', - 'c.numeric_version', 'call', 'callCC', 'capabilities', 'casefold', - 'cat', 'category', 'cbind', 'cbind.data.frame', 'ceiling', - 'char.expand', 'charToRaw', 'charmatch', 'chartr', 'check_tzones', - 'chol', 'chol.default', 'chol2inv', 'choose', 'class', - 'clearPushBack', 'close', 'close.connection', 'close.srcfile', - 'close.srcfilealias', 'closeAllConnections', 'col', 'colMeans', - 'colSums', 'colnames', 'commandArgs', 'comment', 'computeRestarts', - 'conditionCall', 'conditionCall.condition', 'conditionMessage', - 'conditionMessage.condition', 'conflicts', 'contributors', 'cos', - 'cosh', 'crossprod', 'cummax', 'cummin', 'cumprod', 'cumsum', 'cut', - 'cut.Date', 'cut.POSIXt', 'cut.default', 'dQuote', 'data.class', - 'data.matrix', 'date', 'debug', 'debugonce', - 'default.stringsAsFactors', 'delayedAssign', 'deparse', 'det', - 'determinant', 'determinant.matrix', 'dget', 'diag', 'diff', - 'diff.Date', 'diff.POSIXt', 'diff.default', 'difftime', 'digamma', - 'dim', 'dim.data.frame', 'dimnames', 'dimnames.data.frame', 'dir', - 'dir.create', 'dirname', 'do.call', 'dput', 'drop', 'droplevels', - 'droplevels.data.frame', 'droplevels.factor', 'dump', 'duplicated', - 'duplicated.POSIXlt', 'duplicated.array', 'duplicated.data.frame', - 'duplicated.default', 'duplicated.matrix', - 'duplicated.numeric_version', 'dyn.load', 'dyn.unload', 'eapply', - 'eigen', 'else', 'emptyenv', 'enc2native', 'enc2utf8', - 'encodeString', 'enquote', 'env.profile', 'environment', - 'environmentIsLocked', 'environmentName', 'eval', 'eval.parent', - 'evalq', 'exists', 'exp', 'expand.grid', 'expm1', 'expression', - 'factor', 'factorial', 'fifo', 'file', 'file.access', 'file.append', - 'file.choose', 'file.copy', 'file.create', 'file.exists', - 'file.info', 'file.link', 'file.path', 'file.remove', 'file.rename', - 'file.show', 'file.symlink', 'find.package', 'findInterval', - 'findPackageEnv', 'findRestart', 'floor', 'flush', - 'flush.connection', 'force', 'formals', 'format', - 'format.AsIs', 'format.Date', 'format.POSIXct', 'format.POSIXlt', - 'format.data.frame', 'format.default', 'format.difftime', - 'format.factor', 'format.hexmode', 'format.info', - 'format.libraryIQR', 'format.numeric_version', 'format.octmode', - 'format.packageInfo', 'format.pval', 'format.summaryDefault', - 'formatC', 'formatDL', 'forwardsolve', 'gamma', 'gc', 'gc.time', - 'gcinfo', 'gctorture', 'gctorture2', 'get', 'getAllConnections', - 'getCallingDLL', 'getCallingDLLe', 'getConnection', - 'getDLLRegisteredRoutines', 'getDLLRegisteredRoutines.DLLInfo', - 'getDLLRegisteredRoutines.character', 'getElement', - 'getExportedValue', 'getHook', 'getLoadedDLLs', 'getNamespace', - 'getNamespaceExports', 'getNamespaceImports', 'getNamespaceInfo', - 'getNamespaceName', 'getNamespaceUsers', 'getNamespaceVersion', - 'getNativeSymbolInfo', 'getOption', 'getRversion', 'getSrcLines', - 'getTaskCallbackNames', 'geterrmessage', 'gettext', 'gettextf', - 'getwd', 'gl', 'globalenv', 'gregexpr', 'grep', 'grepRaw', 'grepl', - 'gsub', 'gzcon', 'gzfile', 'head', 'iconv', 'iconvlist', - 'icuSetCollate', 'identical', 'identity', 'ifelse', 'importIntoEnv', - 'in', 'inherits', 'intToBits', 'intToUtf8', 'interaction', 'interactive', - 'intersect', 'inverse.rle', 'invisible', 'invokeRestart', - 'invokeRestartInteractively', 'is.R', 'is.array', 'is.atomic', - 'is.call', 'is.character', 'is.complex', 'is.data.frame', - 'is.double', 'is.element', 'is.environment', 'is.expression', - 'is.factor', 'is.finite', 'is.function', 'is.infinite', - 'is.integer', 'is.language', 'is.list', 'is.loaded', 'is.logical', - 'is.matrix', 'is.na', 'is.na.POSIXlt', 'is.na.data.frame', - 'is.na.numeric_version', 'is.name', 'is.nan', 'is.null', - 'is.numeric', 'is.numeric.Date', 'is.numeric.POSIXt', - 'is.numeric.difftime', 'is.numeric_version', 'is.object', - 'is.ordered', 'is.package_version', 'is.pairlist', 'is.primitive', - 'is.qr', 'is.raw', 'is.recursive', 'is.single', 'is.symbol', - 'is.table', 'is.unsorted', 'is.vector', 'isBaseNamespace', - 'isIncomplete', 'isNamespace', 'isOpen', 'isRestart', 'isS4', - 'isSeekable', 'isSymmetric', 'isSymmetric.matrix', 'isTRUE', - 'isatty', 'isdebugged', 'jitter', 'julian', 'julian.Date', - 'julian.POSIXt', 'kappa', 'kappa.default', 'kappa.lm', 'kappa.qr', - 'kronecker', 'l10n_info', 'labels', 'labels.default', 'lapply', - 'lazyLoad', 'lazyLoadDBexec', 'lazyLoadDBfetch', 'lbeta', 'lchoose', - 'length', 'length.POSIXlt', 'letters', 'levels', 'levels.default', - 'lfactorial', 'lgamma', 'library.dynam', 'library.dynam.unload', - 'licence', 'license', 'list.dirs', 'list.files', 'list2env', 'load', - 'loadNamespace', 'loadedNamespaces', 'loadingNamespaceInfo', - 'local', 'lockBinding', 'lockEnvironment', 'log', 'log10', 'log1p', - 'log2', 'logb', 'lower.tri', 'ls', 'make.names', 'make.unique', - 'makeActiveBinding', 'mapply', 'margin.table', 'mat.or.vec', - 'match', 'match.arg', 'match.call', 'match.fun', 'max', 'max.col', - 'mean', 'mean.Date', 'mean.POSIXct', 'mean.POSIXlt', 'mean.default', - 'mean.difftime', 'mem.limits', 'memCompress', 'memDecompress', - 'memory.profile', 'merge', 'merge.data.frame', 'merge.default', - 'message', 'mget', 'min', 'missing', 'mode', 'month.abb', - 'month.name', 'months', 'months.Date', 'months.POSIXt', - 'months.abb', 'months.nameletters', 'names', 'names.POSIXlt', - 'namespaceExport', 'namespaceImport', 'namespaceImportClasses', - 'namespaceImportFrom', 'namespaceImportMethods', 'nargs', 'nchar', - 'ncol', 'new.env', 'ngettext', 'nlevels', 'noquote', 'norm', - 'normalizePath', 'nrow', 'numeric_version', 'nzchar', 'objects', - 'oldClass', 'on.exit', 'open', 'open.connection', 'open.srcfile', - 'open.srcfilealias', 'open.srcfilecopy', 'options', 'order', - 'ordered', 'outer', 'packBits', 'packageEvent', - 'packageHasNamespace', 'packageStartupMessage', 'package_version', - 'pairlist', 'parent.env', 'parent.frame', 'parse', - 'parseNamespaceFile', 'paste', 'paste0', 'path.expand', - 'path.package', 'pipe', 'pmatch', 'pmax', 'pmax.int', 'pmin', - 'pmin.int', 'polyroot', 'pos.to.env', 'pretty', 'pretty.default', - 'prettyNum', 'print', 'print.AsIs', 'print.DLLInfo', - 'print.DLLInfoList', 'print.DLLRegisteredRoutines', 'print.Date', - 'print.NativeRoutineList', 'print.POSIXct', 'print.POSIXlt', - 'print.by', 'print.condition', 'print.connection', - 'print.data.frame', 'print.default', 'print.difftime', - 'print.factor', 'print.function', 'print.hexmode', - 'print.libraryIQR', 'print.listof', 'print.noquote', - 'print.numeric_version', 'print.octmode', 'print.packageInfo', - 'print.proc_time', 'print.restart', 'print.rle', - 'print.simple.list', 'print.srcfile', 'print.srcref', - 'print.summary.table', 'print.summaryDefault', 'print.table', - 'print.warnings', 'prmatrix', 'proc.time', 'prod', 'prop.table', - 'provideDimnames', 'psigamma', 'pushBack', 'pushBackLength', 'q', - 'qr', 'qr.Q', 'qr.R', 'qr.X', 'qr.coef', 'qr.default', 'qr.fitted', - 'qr.qty', 'qr.qy', 'qr.resid', 'qr.solve', 'quarters', - 'quarters.Date', 'quarters.POSIXt', 'quit', 'quote', 'range', - 'range.default', 'rank', 'rapply', 'raw', 'rawConnection', - 'rawConnectionValue', 'rawShift', 'rawToBits', 'rawToChar', 'rbind', - 'rbind.data.frame', 'rcond', 'read.dcf', 'readBin', 'readChar', - 'readLines', 'readRDS', 'readRenviron', 'readline', 'reg.finalizer', - 'regexec', 'regexpr', 'registerS3method', 'registerS3methods', - 'regmatches', 'remove', 'removeTaskCallback', 'rep', 'rep.Date', - 'rep.POSIXct', 'rep.POSIXlt', 'rep.factor', 'rep.int', - 'rep.numeric_version', 'rep_len', 'replace', 'replicate', - 'requireNamespace', 'restartDescription', 'restartFormals', - 'retracemem', 'rev', 'rev.default', 'rle', 'rm', 'round', - 'round.Date', 'round.POSIXt', 'row', 'row.names', - 'row.names.data.frame', 'row.names.default', 'rowMeans', 'rowSums', - 'rownames', 'rowsum', 'rowsum.data.frame', 'rowsum.default', - 'sQuote', 'sample', 'sample.int', 'sapply', 'save', 'save.image', - 'saveRDS', 'scale', 'scale.default', 'scan', 'search', - 'searchpaths', 'seek', 'seek.connection', 'seq', 'seq.Date', - 'seq.POSIXt', 'seq.default', 'seq.int', 'seq_along', 'seq_len', - 'sequence', 'serialize', 'set.seed', 'setHook', 'setNamespaceInfo', - 'setSessionTimeLimit', 'setTimeLimit', 'setdiff', 'setequal', - 'setwd', 'shQuote', 'showConnections', 'sign', 'signalCondition', - 'signif', 'simpleCondition', 'simpleError', 'simpleMessage', - 'simpleWarning', 'simplify2array', 'sin', 'single', - 'sinh', 'sink', 'sink.number', 'slice.index', 'socketConnection', - 'socketSelect', 'solve', 'solve.default', 'solve.qr', 'sort', - 'sort.POSIXlt', 'sort.default', 'sort.int', 'sort.list', 'split', - 'split.Date', 'split.POSIXct', 'split.data.frame', 'split.default', - 'sprintf', 'sqrt', 'srcfile', 'srcfilealias', 'srcfilecopy', - 'srcref', 'standardGeneric', 'stderr', 'stdin', 'stdout', 'stop', - 'stopifnot', 'storage.mode', 'strftime', 'strptime', 'strsplit', - 'strtoi', 'strtrim', 'structure', 'strwrap', 'sub', 'subset', - 'subset.data.frame', 'subset.default', 'subset.matrix', - 'substitute', 'substr', 'substring', 'sum', 'summary', - 'summary.Date', 'summary.POSIXct', 'summary.POSIXlt', - 'summary.connection', 'summary.data.frame', 'summary.default', - 'summary.factor', 'summary.matrix', 'summary.proc_time', - 'summary.srcfile', 'summary.srcref', 'summary.table', - 'suppressMessages', 'suppressPackageStartupMessages', - 'suppressWarnings', 'svd', 'sweep', 'sys.call', 'sys.calls', - 'sys.frame', 'sys.frames', 'sys.function', 'sys.load.image', - 'sys.nframe', 'sys.on.exit', 'sys.parent', 'sys.parents', - 'sys.save.image', 'sys.source', 'sys.status', 'system', - 'system.file', 'system.time', 'system2', 't', 't.data.frame', - 't.default', 'table', 'tabulate', 'tail', 'tan', 'tanh', 'tapply', - 'taskCallbackManager', 'tcrossprod', 'tempdir', 'tempfile', - 'testPlatformEquivalence', 'textConnection', 'textConnectionValue', - 'toString', 'toString.default', 'tolower', 'topenv', 'toupper', - 'trace', 'traceback', 'tracemem', 'tracingState', 'transform', - 'transform.data.frame', 'transform.default', 'trigamma', 'trunc', - 'trunc.Date', 'trunc.POSIXt', 'truncate', 'truncate.connection', - 'try', 'tryCatch', 'typeof', 'unclass', 'undebug', 'union', - 'unique', 'unique.POSIXlt', 'unique.array', 'unique.data.frame', - 'unique.default', 'unique.matrix', 'unique.numeric_version', - 'units', 'units.difftime', 'unix.time', 'unlink', 'unlist', - 'unloadNamespace', 'unlockBinding', 'unname', 'unserialize', - 'unsplit', 'untrace', 'untracemem', 'unz', 'upper.tri', 'url', - 'utf8ToInt', 'vapply', 'version', 'warning', 'warnings', 'weekdays', - 'weekdays.Date', 'weekdays.POSIXt', 'which', 'which.max', - 'which.min', 'with', 'with.default', 'withCallingHandlers', - 'withRestarts', 'withVisible', 'within', 'within.data.frame', - 'within.list', 'write', 'write.dcf', 'writeBin', 'writeChar', - 'writeLines', 'xor', 'xor.hexmode', 'xor.octmode', - 'xpdrows.data.frame', 'xtfrm', 'xtfrm.AsIs', 'xtfrm.Date', - 'xtfrm.POSIXct', 'xtfrm.POSIXlt', 'xtfrm.Surv', 'xtfrm.default', - 'xtfrm.difftime', 'xtfrm.factor', 'xtfrm.numeric_version', 'xzfile', - 'zapsmall' - ) - + valid_name = r'`[^`\\]*(?:\\.[^`\\]*)*`|(?:[a-zA-Z]|\.[A-Za-z_.])[\w.]*|\.' tokens = { 'comments': [ (r'#.*$', Comment.Single), ], 'valid_name': [ - (r'[a-zA-Z][\w.]*', Text), - # can begin with ., but not if that is followed by a digit - (r'\.[a-zA-Z_][\w.]*', Text), + (valid_name, Name), ], 'punctuation': [ (r'\[{1,2}|\]{1,2}|\(|\)|;|,', Punctuation), ], 'keywords': [ - (words(builtins_base, suffix=r'(?![\w. =])'), - Keyword.Pseudo), (r'(if|else|for|while|repeat|in|next|break|return|switch|function)' r'(?![\w.])', Keyword.Reserved), - (r'(array|category|character|complex|double|function|integer|list|' - r'logical|matrix|numeric|vector|data.frame|c)' - r'(?![\w.])', - Keyword.Type), - (r'(library|require|attach|detach|source)' - r'(?![\w.])', - Keyword.Namespace) ], 'operators': [ (r'<>?|-|==|<=|>=|<|>|&&?|!=|\|\|?|\?', Operator), - (r'\*|\+|\^|/|!|%[^%]*%|=|~|\$|@|:{1,3}', Operator) + (r'\*|\+|\^|/|!|%[^%]*%|=|~|\$|@|:{1,3}', Operator), ], 'builtin_symbols': [ (r'(NULL|NA(_(integer|real|complex|character)_)?|' @@ -379,17 +115,18 @@ class SLexer(RegexLexer): include('comments'), # whitespaces (r'\s+', Text), - (r'`.*?`', String.Backtick), (r'\'', String, 'string_squote'), (r'\"', String, 'string_dquote'), include('builtin_symbols'), + include('valid_name'), include('numbers'), include('keywords'), include('punctuation'), include('operators'), - include('valid_name'), ], 'root': [ + # calls: + (r'(%s)\s*(?=\()' % valid_name, Name.Function), include('statements'), # blocks: (r'\{|\}', Punctuation), @@ -421,7 +158,7 @@ class RdLexer(RegexLexer): This is a very minimal implementation, highlighting little more than the macros. A description of Rd syntax is found in `Writing R Extensions `_ - and `Parsing Rd files `_. + and `Parsing Rd files `_. .. versionadded:: 1.6 """ diff --git a/src/syntax_highlighting/libs/pygments/lexers/rdf.py b/src/syntax_highlighting/libs/pygments/lexers/rdf.py index d0f8778..bd7a4f6 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/rdf.py +++ b/src/syntax_highlighting/libs/pygments/lexers/rdf.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.rdf ~~~~~~~~~~~~~~~~~~~ Lexers for semantic web and RDF query languages and markup. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -15,12 +14,12 @@ from pygments.token import Keyword, Punctuation, String, Number, Operator, Generic, \ Whitespace, Name, Literal, Comment, Text -__all__ = ['SparqlLexer', 'TurtleLexer'] +__all__ = ['SparqlLexer', 'TurtleLexer', 'ShExCLexer'] class SparqlLexer(RegexLexer): """ - Lexer for `SPARQL `_ query language. + Lexer for `SPARQL `_ query language. .. versionadded:: 2.0 """ @@ -31,27 +30,27 @@ class SparqlLexer(RegexLexer): # character group definitions :: - PN_CHARS_BASE_GRP = (u'a-zA-Z' - u'\u00c0-\u00d6' - u'\u00d8-\u00f6' - u'\u00f8-\u02ff' - u'\u0370-\u037d' - u'\u037f-\u1fff' - u'\u200c-\u200d' - u'\u2070-\u218f' - u'\u2c00-\u2fef' - u'\u3001-\ud7ff' - u'\uf900-\ufdcf' - u'\ufdf0-\ufffd') + PN_CHARS_BASE_GRP = ('a-zA-Z' + '\u00c0-\u00d6' + '\u00d8-\u00f6' + '\u00f8-\u02ff' + '\u0370-\u037d' + '\u037f-\u1fff' + '\u200c-\u200d' + '\u2070-\u218f' + '\u2c00-\u2fef' + '\u3001-\ud7ff' + '\uf900-\ufdcf' + '\ufdf0-\ufffd') PN_CHARS_U_GRP = (PN_CHARS_BASE_GRP + '_') PN_CHARS_GRP = (PN_CHARS_U_GRP + r'\-' + r'0-9' + - u'\u00b7' + - u'\u0300-\u036f' + - u'\u203f-\u2040') + '\u00b7' + + '\u0300-\u036f' + + '\u203f-\u2040') HEX_GRP = '0-9A-Fa-f' @@ -76,8 +75,8 @@ class SparqlLexer(RegexLexer): PN_PREFIX = PN_CHARS_BASE + '(?:[' + PN_CHARS_GRP + '.]*' + PN_CHARS + ')?' - VARNAME = u'[0-9' + PN_CHARS_U_GRP + '][' + PN_CHARS_U_GRP + \ - u'0-9\u00b7\u0300-\u036f\u203f-\u2040]*' + VARNAME = '[0-9' + PN_CHARS_U_GRP + '][' + PN_CHARS_U_GRP + \ + '0-9\u00b7\u0300-\u036f\u203f-\u2040]*' PERCENT = '%' + HEX + HEX @@ -97,12 +96,12 @@ class SparqlLexer(RegexLexer): 'root': [ (r'\s+', Text), # keywords :: - (r'((?i)select|construct|describe|ask|where|filter|group\s+by|minus|' + (r'(?i)(select|construct|describe|ask|where|filter|group\s+by|minus|' r'distinct|reduced|from\s+named|from|order\s+by|desc|asc|limit|' - r'offset|bindings|load|clear|drop|create|add|move|copy|' - r'insert\s+data|delete\s+data|delete\s+where|delete|insert|' + r'offset|values|bindings|load|into|clear|drop|create|add|move|copy|' + r'insert\s+data|delete\s+data|delete\s+where|with|delete|insert|' r'using\s+named|using|graph|default|named|all|optional|service|' - r'silent|bind|union|not\s+in|in|as|having|to|prefix|base)\b', Keyword), + r'silent|bind|undef|union|not\s+in|in|as|having|to|prefix|base)\b', Keyword), (r'(a)\b', Keyword), # IRIs :: ('(' + IRIREF + ')', Name.Label), @@ -111,13 +110,13 @@ class SparqlLexer(RegexLexer): # # variables :: ('[?$]' + VARNAME, Name.Variable), # prefixed names :: - (r'(' + PN_PREFIX + ')?(\:)(' + PN_LOCAL + ')?', + (r'(' + PN_PREFIX + r')?(\:)(' + PN_LOCAL + r')?', bygroups(Name.Namespace, Punctuation, Name.Tag)), # function names :: - (r'((?i)str|lang|langmatches|datatype|bound|iri|uri|bnode|rand|abs|' + (r'(?i)(str|lang|langmatches|datatype|bound|iri|uri|bnode|rand|abs|' r'ceil|floor|round|concat|strlen|ucase|lcase|encode_for_uri|' r'contains|strstarts|strends|strbefore|strafter|year|month|day|' - r'hours|minutes|seconds|timezone|tz|now|md5|sha1|sha256|sha384|' + r'hours|minutes|seconds|timezone|tz|now|uuid|struuid|md5|sha1|sha256|sha384|' r'sha512|coalesce|if|strlang|strdt|sameterm|isiri|isuri|isblank|' r'isliteral|isnumeric|regex|substr|replace|exists|not\s+exists|' r'count|sum|min|max|avg|sample|group_concat|separator)\b', @@ -125,7 +124,7 @@ class SparqlLexer(RegexLexer): # boolean literals :: (r'(true|false)', Keyword.Constant), # double literals :: - (r'[+\-]?(\d+\.\d*' + EXPONENT + '|\.?\d+' + EXPONENT + ')', Number.Float), + (r'[+\-]?(\d+\.\d*' + EXPONENT + r'|\.?\d+' + EXPONENT + ')', Number.Float), # decimal literals :: (r'[+\-]?(\d+\.\d*|\.\d+)', Number.Float), # integer literals :: @@ -187,19 +186,61 @@ class TurtleLexer(RegexLexer): filenames = ['*.ttl'] mimetypes = ['text/turtle', 'application/x-turtle'] - flags = re.IGNORECASE + # character group definitions :: + PN_CHARS_BASE_GRP = ('a-zA-Z' + '\u00c0-\u00d6' + '\u00d8-\u00f6' + '\u00f8-\u02ff' + '\u0370-\u037d' + '\u037f-\u1fff' + '\u200c-\u200d' + '\u2070-\u218f' + '\u2c00-\u2fef' + '\u3001-\ud7ff' + '\uf900-\ufdcf' + '\ufdf0-\ufffd') + + PN_CHARS_U_GRP = (PN_CHARS_BASE_GRP + '_') + + PN_CHARS_GRP = (PN_CHARS_U_GRP + + r'\-' + + r'0-9' + + '\u00b7' + + '\u0300-\u036f' + + '\u203f-\u2040') + + PN_CHARS = '[' + PN_CHARS_GRP + ']' + + PN_CHARS_BASE = '[' + PN_CHARS_BASE_GRP + ']' + + PN_PREFIX = PN_CHARS_BASE + '(?:[' + PN_CHARS_GRP + '.]*' + PN_CHARS + ')?' + + HEX_GRP = '0-9A-Fa-f' + + HEX = '[' + HEX_GRP + ']' + + PERCENT = '%' + HEX + HEX + + PN_LOCAL_ESC_CHARS_GRP = r' _~.\-!$&"()*+,;=/?#@%' + + PN_LOCAL_ESC_CHARS = '[' + PN_LOCAL_ESC_CHARS_GRP + ']' + + PN_LOCAL_ESC = r'\\' + PN_LOCAL_ESC_CHARS + + PLX = '(?:' + PERCENT + ')|(?:' + PN_LOCAL_ESC + ')' + + PN_LOCAL = ('(?:[' + PN_CHARS_U_GRP + ':0-9' + ']|' + PLX + ')' + + '(?:(?:[' + PN_CHARS_GRP + '.:]|' + PLX + ')*(?:[' + + PN_CHARS_GRP + ':]|' + PLX + '))?') patterns = { - 'PNAME_NS': r'((?:[a-z][\w-]*)?\:)', # Simplified character range + 'PNAME_NS': r'((?:[a-zA-Z][\w-]*)?\:)', # Simplified character range 'IRIREF': r'(<[^<>"{}|^`\\\x00-\x20]*>)' } - # PNAME_NS PN_LOCAL (with simplified character range) - patterns['PrefixedName'] = r'%(PNAME_NS)s([a-z][\w-]*)' % patterns - tokens = { 'root': [ - (r'\s+', Whitespace), + (r'\s+', Text), # Base / prefix (r'(@base|BASE)(\s+)%(IRIREF)s(\s*)(\.?)' % patterns, @@ -216,8 +257,8 @@ class TurtleLexer(RegexLexer): (r'%(IRIREF)s' % patterns, Name.Variable), # PrefixedName - (r'%(PrefixedName)s' % patterns, - bygroups(Name.Namespace, Name.Tag)), + (r'(' + PN_PREFIX + r')?(\:)(' + PN_LOCAL + r')?', + bygroups(Name.Namespace, Punctuation, Name.Tag)), # Comment (r'#[^\n]+', Comment), @@ -257,14 +298,165 @@ class TurtleLexer(RegexLexer): (r'.', String, '#pop'), ], 'end-of-string': [ - (r'(@)([a-z]+(:?-[a-z0-9]+)*)', + (r'(@)([a-zA-Z]+(?:-[a-zA-Z0-9]+)*)', bygroups(Operator, Generic.Emph), '#pop:2'), (r'(\^\^)%(IRIREF)s' % patterns, bygroups(Operator, Generic.Emph), '#pop:2'), - (r'(\^\^)%(PrefixedName)s' % patterns, - bygroups(Operator, Generic.Emph, Generic.Emph), '#pop:2'), default('#pop:2'), ], } + + # Turtle and Tera Term macro files share the same file extension + # but each has a recognizable and distinct syntax. + def analyse_text(text): + for t in ('@base ', 'BASE ', '@prefix ', 'PREFIX '): + if re.search(r'^\s*%s' % t, text): + return 0.80 + + +class ShExCLexer(RegexLexer): + """ + Lexer for `ShExC `_ shape expressions language syntax. + """ + name = 'ShExC' + aliases = ['shexc', 'shex'] + filenames = ['*.shex'] + mimetypes = ['text/shex'] + + # character group definitions :: + + PN_CHARS_BASE_GRP = ('a-zA-Z' + '\u00c0-\u00d6' + '\u00d8-\u00f6' + '\u00f8-\u02ff' + '\u0370-\u037d' + '\u037f-\u1fff' + '\u200c-\u200d' + '\u2070-\u218f' + '\u2c00-\u2fef' + '\u3001-\ud7ff' + '\uf900-\ufdcf' + '\ufdf0-\ufffd') + + PN_CHARS_U_GRP = (PN_CHARS_BASE_GRP + '_') + + PN_CHARS_GRP = (PN_CHARS_U_GRP + + r'\-' + + r'0-9' + + '\u00b7' + + '\u0300-\u036f' + + '\u203f-\u2040') + + HEX_GRP = '0-9A-Fa-f' + + PN_LOCAL_ESC_CHARS_GRP = r"_~.\-!$&'()*+,;=/?#@%" + + # terminal productions :: + + PN_CHARS_BASE = '[' + PN_CHARS_BASE_GRP + ']' + + PN_CHARS_U = '[' + PN_CHARS_U_GRP + ']' + + PN_CHARS = '[' + PN_CHARS_GRP + ']' + + HEX = '[' + HEX_GRP + ']' + + PN_LOCAL_ESC_CHARS = '[' + PN_LOCAL_ESC_CHARS_GRP + ']' + + UCHAR_NO_BACKSLASH = '(?:u' + HEX + '{4}|U' + HEX + '{8})' + + UCHAR = r'\\' + UCHAR_NO_BACKSLASH + + IRIREF = r'<(?:[^\x00-\x20<>"{}|^`\\]|' + UCHAR + ')*>' + + BLANK_NODE_LABEL = '_:[0-9' + PN_CHARS_U_GRP + '](?:[' + PN_CHARS_GRP + \ + '.]*' + PN_CHARS + ')?' + + PN_PREFIX = PN_CHARS_BASE + '(?:[' + PN_CHARS_GRP + '.]*' + PN_CHARS + ')?' + + PERCENT = '%' + HEX + HEX + + PN_LOCAL_ESC = r'\\' + PN_LOCAL_ESC_CHARS + + PLX = '(?:' + PERCENT + ')|(?:' + PN_LOCAL_ESC + ')' + + PN_LOCAL = ('(?:[' + PN_CHARS_U_GRP + ':0-9' + ']|' + PLX + ')' + + '(?:(?:[' + PN_CHARS_GRP + '.:]|' + PLX + ')*(?:[' + + PN_CHARS_GRP + ':]|' + PLX + '))?') + + EXPONENT = r'[eE][+-]?\d+' + + # Lexer token definitions :: + + tokens = { + 'root': [ + (r'\s+', Text), + # keywords :: + (r'(?i)(base|prefix|start|external|' + r'literal|iri|bnode|nonliteral|length|minlength|maxlength|' + r'mininclusive|minexclusive|maxinclusive|maxexclusive|' + r'totaldigits|fractiondigits|' + r'closed|extra)\b', Keyword), + (r'(a)\b', Keyword), + # IRIs :: + ('(' + IRIREF + ')', Name.Label), + # blank nodes :: + ('(' + BLANK_NODE_LABEL + ')', Name.Label), + # prefixed names :: + (r'(' + PN_PREFIX + r')?(\:)(' + PN_LOCAL + ')?', + bygroups(Name.Namespace, Punctuation, Name.Tag)), + # boolean literals :: + (r'(true|false)', Keyword.Constant), + # double literals :: + (r'[+\-]?(\d+\.\d*' + EXPONENT + r'|\.?\d+' + EXPONENT + ')', Number.Float), + # decimal literals :: + (r'[+\-]?(\d+\.\d*|\.\d+)', Number.Float), + # integer literals :: + (r'[+\-]?\d+', Number.Integer), + # operators :: + (r'[@|$&=*+?^\-~]', Operator), + # operator keywords :: + (r'(?i)(and|or|not)\b', Operator.Word), + # punctuation characters :: + (r'[(){}.;,:^\[\]]', Punctuation), + # line comments :: + (r'#[^\n]*', Comment), + # strings :: + (r'"""', String, 'triple-double-quoted-string'), + (r'"', String, 'single-double-quoted-string'), + (r"'''", String, 'triple-single-quoted-string'), + (r"'", String, 'single-single-quoted-string'), + ], + 'triple-double-quoted-string': [ + (r'"""', String, 'end-of-string'), + (r'[^\\]+', String), + (r'\\', String, 'string-escape'), + ], + 'single-double-quoted-string': [ + (r'"', String, 'end-of-string'), + (r'[^"\\\n]+', String), + (r'\\', String, 'string-escape'), + ], + 'triple-single-quoted-string': [ + (r"'''", String, 'end-of-string'), + (r'[^\\]+', String), + (r'\\', String.Escape, 'string-escape'), + ], + 'single-single-quoted-string': [ + (r"'", String, 'end-of-string'), + (r"[^'\\\n]+", String), + (r'\\', String, 'string-escape'), + ], + 'string-escape': [ + (UCHAR_NO_BACKSLASH, String.Escape, '#pop'), + (r'.', String.Escape, '#pop'), + ], + 'end-of-string': [ + (r'(@)([a-zA-Z]+(?:-[a-zA-Z0-9]+)*)', + bygroups(Operator, Name.Function), '#pop:2'), + (r'\^\^', Operator, '#pop:2'), + default('#pop:2'), + ], + } diff --git a/src/syntax_highlighting/libs/pygments/lexers/rebol.py b/src/syntax_highlighting/libs/pygments/lexers/rebol.py index f3d0020..57480a1 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/rebol.py +++ b/src/syntax_highlighting/libs/pygments/lexers/rebol.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.rebol ~~~~~~~~~~~~~~~~~~~~~ Lexers for the REBOL and related languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -102,12 +101,12 @@ def word_callback(lexer, match): yield match.start(), Generic.Heading, word elif re.match("to-.*", word): yield match.start(), Keyword, word - elif re.match('(\+|-|\*|/|//|\*\*|and|or|xor|=\?|=|==|<>|<|>|<=|>=)$', + elif re.match(r'(\+|-|\*|/|//|\*\*|and|or|xor|=\?|=|==|<>|<|>|<=|>=)$', word): yield match.start(), Operator, word - elif re.match(".*\?$", word): + elif re.match(r".*\?$", word): yield match.start(), Keyword, word - elif re.match(".*\!$", word): + elif re.match(r".*\!$", word): yield match.start(), Keyword.Type, word elif re.match("'.*", word): yield match.start(), Name.Variable.Instance, word # lit-word @@ -239,7 +238,7 @@ def analyse_text(text): if re.match(r'^\s*REBOL\s*\[', text, re.IGNORECASE): # The code starts with REBOL header return 1.0 - elif re.search(r'\s*REBOL\s*[', text, re.IGNORECASE): + elif re.search(r'\s*REBOL\s*\[', text, re.IGNORECASE): # The code contains REBOL header but also some text before it return 0.5 @@ -297,10 +296,10 @@ def word_callback(lexer, match): yield match.start(), Keyword.Namespace, word elif re.match("to-.*", word): yield match.start(), Keyword, word - elif re.match('(\+|-\*\*|-|\*\*|//|/|\*|and|or|xor|=\?|===|==|=|<>|<=|>=|' - '<<<|>>>|<<|>>|<|>%)$', word): + elif re.match(r'(\+|-\*\*|-|\*\*|//|/|\*|and|or|xor|=\?|===|==|=|<>|<=|>=|' + r'<<<|>>>|<<|>>|<|>%)$', word): yield match.start(), Operator, word - elif re.match(".*\!$", word): + elif re.match(r".*\!$", word): yield match.start(), Keyword.Type, word elif re.match("'.*", word): yield match.start(), Name.Variable.Instance, word # lit-word diff --git a/src/syntax_highlighting/libs/pygments/lexers/resource.py b/src/syntax_highlighting/libs/pygments/lexers/resource.py index f749490..3ed176a 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/resource.py +++ b/src/syntax_highlighting/libs/pygments/lexers/resource.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.resource ~~~~~~~~~~~~~~~~~~~~~~~~ Lexer for resource definition files. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -25,8 +24,8 @@ class ResourceLexer(RegexLexer): .. versionadded:: 2.0 """ name = 'ResourceBundle' - aliases = ['resource', 'resourcebundle'] - filenames = ['*.txt'] + aliases = ['resourcebundle', 'resource'] + filenames = [] _types = (':table', ':array', ':string', ':bin', ':import', ':intvector', ':int', ':alias') diff --git a/src/syntax_highlighting/libs/pygments/lexers/ride.py b/src/syntax_highlighting/libs/pygments/lexers/ride.py new file mode 100644 index 0000000..07cc1ef --- /dev/null +++ b/src/syntax_highlighting/libs/pygments/lexers/ride.py @@ -0,0 +1,138 @@ +""" + pygments.lexers.ride + ~~~~~~~~~~~~~~~~~~~~ + + Lexer for the Ride programming language. + + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer, words, include +from pygments.token import Comment, Keyword, Name, Number, Punctuation, String, Text + +__all__ = ['RideLexer'] + + +class RideLexer(RegexLexer): + """ + For `Ride `_ + source code. + + .. versionadded:: 2.6 + """ + + name = 'Ride' + aliases = ['ride'] + filenames = ['*.ride'] + mimetypes = ['text/x-ride'] + + validName = r'[a-zA-Z_][a-zA-Z0-9_\']*' + + builtinOps = ( + '||', '|', '>=', '>', '==', '!', + '=', '<=', '<', '::', ':+', ':', '!=', '/', + '.', '=>', '-', '+', '*', '&&', '%', '++', + ) + + globalVariablesName = ( + 'NOALG', 'MD5', 'SHA1', 'SHA224', 'SHA256', 'SHA384', 'SHA512', + 'SHA3224', 'SHA3256', 'SHA3384', 'SHA3512', 'nil', 'this', 'unit', + 'height', 'lastBlock', 'Buy', 'Sell', 'CEILING', 'FLOOR', 'DOWN', + 'HALFDOWN', 'HALFEVEN', 'HALFUP', 'UP', + ) + + typesName = ( + 'Unit', 'Int', 'Boolean', 'ByteVector', 'String', 'Address', 'Alias', + 'Transfer', 'AssetPair', 'DataEntry', 'Order', 'Transaction', + 'GenesisTransaction', 'PaymentTransaction', 'ReissueTransaction', + 'BurnTransaction', 'MassTransferTransaction', 'ExchangeTransaction', + 'TransferTransaction', 'SetAssetScriptTransaction', + 'InvokeScriptTransaction', 'IssueTransaction', 'LeaseTransaction', + 'LeaseCancelTransaction', 'CreateAliasTransaction', + 'SetScriptTransaction', 'SponsorFeeTransaction', 'DataTransaction', + 'WriteSet', 'AttachedPayment', 'ScriptTransfer', 'TransferSet', + 'ScriptResult', 'Invocation', 'Asset', 'BlockInfo', 'Issue', 'Reissue', + 'Burn', 'NoAlg', 'Md5', 'Sha1', 'Sha224', 'Sha256', 'Sha384', 'Sha512', + 'Sha3224', 'Sha3256', 'Sha3384', 'Sha3512', 'BinaryEntry', + 'BooleanEntry', 'IntegerEntry', 'StringEntry', 'List', 'Ceiling', + 'Down', 'Floor', 'HalfDown', 'HalfEven', 'HalfUp', 'Up', + ) + + functionsName = ( + 'fraction', 'size', 'toBytes', 'take', 'drop', 'takeRight', 'dropRight', + 'toString', 'isDefined', 'extract', 'throw', 'getElement', 'value', + 'cons', 'toUtf8String', 'toInt', 'indexOf', 'lastIndexOf', 'split', + 'parseInt', 'parseIntValue', 'keccak256', 'blake2b256', 'sha256', + 'sigVerify', 'toBase58String', 'fromBase58String', 'toBase64String', + 'fromBase64String', 'transactionById', 'transactionHeightById', + 'getInteger', 'getBoolean', 'getBinary', 'getString', + 'addressFromPublicKey', 'addressFromString', 'addressFromRecipient', + 'assetBalance', 'wavesBalance', 'getIntegerValue', 'getBooleanValue', + 'getBinaryValue', 'getStringValue', 'addressFromStringValue', + 'assetInfo', 'rsaVerify', 'checkMerkleProof', 'median', + 'valueOrElse', 'valueOrErrorMessage', 'contains', 'log', 'pow', + 'toBase16String', 'fromBase16String', 'blockInfoByHeight', + 'transferTransactionById', + ) + + reservedWords = words(( + 'match', 'case', 'else', 'func', 'if', + 'let', 'then', '@Callable', '@Verifier', + ), suffix=r'\b') + + tokens = { + 'root': [ + # Comments + (r'#.*', Comment.Single), + # Whitespace + (r'\s+', Text), + # Strings + (r'"', String, 'doublequote'), + (r'utf8\'', String, 'utf8quote'), + (r'base(58|64|16)\'', String, 'singlequote'), + # Keywords + (reservedWords, Keyword.Reserved), + (r'\{-#.*?#-\}', Keyword.Reserved), + (r'FOLD<\d+>', Keyword.Reserved), + # Types + (words(typesName), Keyword.Type), + # Main + # (specialName, Keyword.Reserved), + # Prefix Operators + (words(builtinOps, prefix=r'\(', suffix=r'\)'), Name.Function), + # Infix Operators + (words(builtinOps), Name.Function), + (words(globalVariablesName), Name.Function), + (words(functionsName), Name.Function), + # Numbers + include('numbers'), + # Variable Names + (validName, Name.Variable), + # Parens + (r'[,()\[\]{}]', Punctuation), + ], + + 'doublequote': [ + (r'\\u[0-9a-fA-F]{4}', String.Escape), + (r'\\[nrfvb\\"]', String.Escape), + (r'[^"]', String), + (r'"', String, '#pop'), + ], + + 'utf8quote': [ + (r'\\u[0-9a-fA-F]{4}', String.Escape), + (r'\\[nrfvb\\\']', String.Escape), + (r'[^\']', String), + (r'\'', String, '#pop'), + ], + + 'singlequote': [ + (r'[^\']', String), + (r'\'', String, '#pop'), + ], + + 'numbers': [ + (r'_?\d+', Number.Integer), + ], + } diff --git a/src/syntax_highlighting/libs/pygments/lexers/rnc.py b/src/syntax_highlighting/libs/pygments/lexers/rnc.py index 2f2aacd..cc8950a 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/rnc.py +++ b/src/syntax_highlighting/libs/pygments/lexers/rnc.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.rnc ~~~~~~~~~~~~~~~~~~~ Lexer for Relax-NG Compact syntax - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -24,7 +23,7 @@ class RNCCompactLexer(RegexLexer): """ name = 'Relax-NG Compact' - aliases = ['rnc', 'rng-compact'] + aliases = ['rng-compact', 'rnc'] filenames = ['*.rnc'] tokens = { diff --git a/src/syntax_highlighting/libs/pygments/lexers/roboconf.py b/src/syntax_highlighting/libs/pygments/lexers/roboconf.py index 8c7df83..4380113 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/roboconf.py +++ b/src/syntax_highlighting/libs/pygments/lexers/roboconf.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.roboconf ~~~~~~~~~~~~~~~~~~~~~~~~ Lexers for Roboconf DSL. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/src/syntax_highlighting/libs/pygments/lexers/robotframework.py b/src/syntax_highlighting/libs/pygments/lexers/robotframework.py index e868127..305cc26 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/robotframework.py +++ b/src/syntax_highlighting/libs/pygments/lexers/robotframework.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.robotframework ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Lexer for Robot Framework. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -27,7 +26,6 @@ from pygments.lexer import Lexer from pygments.token import Token -from pygments.util import text_type __all__ = ['RobotFrameworkLexer'] @@ -64,7 +62,7 @@ class RobotFrameworkLexer(Lexer): """ name = 'RobotFramework' aliases = ['robotframework'] - filenames = ['*.txt', '*.robot'] + filenames = ['*.robot'] mimetypes = ['text/x-robotframework'] def __init__(self, **options): @@ -80,11 +78,11 @@ def get_tokens_unprocessed(self, text): for value, token in row_tokenizer.tokenize(row): for value, token in var_tokenizer.tokenize(value, token): if value: - yield index, token, text_type(value) + yield index, token, str(value) index += len(value) -class VariableTokenizer(object): +class VariableTokenizer: def tokenize(self, string, token): var = VariableSplitter(string, identifiers='$@%&') @@ -99,19 +97,16 @@ def _tokenize(self, var, string, orig_token): before = string[:var.start] yield before, orig_token yield var.identifier + '{', SYNTAX - for value, token in self.tokenize(var.base, VARIABLE): - yield value, token + yield from self.tokenize(var.base, VARIABLE) yield '}', SYNTAX if var.index: yield '[', SYNTAX - for value, token in self.tokenize(var.index, VARIABLE): - yield value, token + yield from self.tokenize(var.index, VARIABLE) yield ']', SYNTAX - for value, token in self.tokenize(string[var.end:], orig_token): - yield value, token + yield from self.tokenize(string[var.end:], orig_token) -class RowTokenizer(object): +class RowTokenizer: def __init__(self): self._table = UnknownTable() @@ -124,6 +119,7 @@ def __init__(self): 'metadata': settings, 'variables': variables, 'variable': variables, 'testcases': testcases, 'testcase': testcases, + 'tasks': testcases, 'task': testcases, 'keywords': keywords, 'keyword': keywords, 'userkeywords': keywords, 'userkeyword': keywords} @@ -138,9 +134,8 @@ def tokenize(self, row): elif index == 0 and value.startswith('*'): self._table = self._start_table(value) heading = True - for value, token in self._tokenize(value, index, commented, - separator, heading): - yield value, token + yield from self._tokenize(value, index, commented, + separator, heading) self._table.end_row() def _start_table(self, header): @@ -155,25 +150,22 @@ def _tokenize(self, value, index, commented, separator, heading): elif heading: yield value, HEADING else: - for value, token in self._table.tokenize(value, index): - yield value, token + yield from self._table.tokenize(value, index) -class RowSplitter(object): +class RowSplitter: _space_splitter = re.compile('( {2,})') - _pipe_splitter = re.compile('((?:^| +)\|(?: +|$))') + _pipe_splitter = re.compile(r'((?:^| +)\|(?: +|$))') def split(self, row): splitter = (row.startswith('| ') and self._split_from_pipes or self._split_from_spaces) - for value in splitter(row): - yield value + yield from splitter(row) yield '\n' def _split_from_spaces(self, row): yield '' # Start with (pseudo)separator similarly as with pipes - for value in self._space_splitter.split(row): - yield value + yield from self._space_splitter.split(row) def _split_from_pipes(self, row): _, separator, rest = self._pipe_splitter.split(row, 1) @@ -185,7 +177,7 @@ def _split_from_pipes(self, row): yield rest -class Tokenizer(object): +class Tokenizer: _tokens = None def __init__(self): @@ -216,11 +208,11 @@ class Comment(Tokenizer): class Setting(Tokenizer): _tokens = (SETTING, ARGUMENT) _keyword_settings = ('suitesetup', 'suiteprecondition', 'suiteteardown', - 'suitepostcondition', 'testsetup', 'testprecondition', - 'testteardown', 'testpostcondition', 'testtemplate') + 'suitepostcondition', 'testsetup', 'tasksetup', 'testprecondition', + 'testteardown','taskteardown', 'testpostcondition', 'testtemplate', 'tasktemplate') _import_settings = ('library', 'resource', 'variables') _other_settings = ('documentation', 'metadata', 'forcetags', 'defaulttags', - 'testtimeout') + 'testtimeout','tasktimeout') _custom_tokenizer = None def __init__(self, template_setter=None): @@ -292,7 +284,7 @@ def _tokenize(self, value, index): return GherkinTokenizer().tokenize(value, KEYWORD) -class GherkinTokenizer(object): +class GherkinTokenizer: _gherkin_prefix = re.compile('^(Given|When|Then|And) ', re.IGNORECASE) def tokenize(self, value, token): @@ -320,7 +312,7 @@ def _tokenize(self, value, index): return token -class _Table(object): +class _Table: _tokenizer_class = None def __init__(self, prev_tokenizer=None): @@ -333,8 +325,7 @@ def tokenize(self, value, index): self._tokenizer = self._prev_tokenizer yield value, SYNTAX else: - for value_and_token in self._tokenize(value, index): - yield value_and_token + yield from self._tokenize(value, index) self._prev_values_on_row.append(value) def _continues(self, value, index): diff --git a/src/syntax_highlighting/libs/pygments/lexers/ruby.py b/src/syntax_highlighting/libs/pygments/lexers/ruby.py index fe750f1..2c3e7bd 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/ruby.py +++ b/src/syntax_highlighting/libs/pygments/lexers/ruby.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.ruby ~~~~~~~~~~~~~~~~~~~~ Lexers for Ruby and related languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -34,7 +33,7 @@ class RubyLexer(ExtendedRegexLexer): """ name = 'Ruby' - aliases = ['rb', 'ruby', 'duby'] + aliases = ['ruby', 'rb', 'duby'] filenames = ['*.rb', '*.rbw', 'Rakefile', '*.rake', '*.gemspec', '*.rbx', '*.duby', 'Gemfile'] mimetypes = ['text/x-ruby', 'application/x-ruby'] @@ -43,23 +42,25 @@ class RubyLexer(ExtendedRegexLexer): def heredoc_callback(self, match, ctx): # okay, this is the hardest part of parsing Ruby... - # match: 1 = <<-?, 2 = quote? 3 = name 4 = quote? 5 = rest of line + # match: 1 = <<[-~]?, 2 = quote? 3 = name 4 = quote? 5 = rest of line start = match.start(1) - yield start, Operator, match.group(1) # <<-? + yield start, Operator, match.group(1) # <<[-~]? yield match.start(2), String.Heredoc, match.group(2) # quote ", ', ` yield match.start(3), String.Delimiter, match.group(3) # heredoc name yield match.start(4), String.Heredoc, match.group(4) # quote again heredocstack = ctx.__dict__.setdefault('heredocstack', []) outermost = not bool(heredocstack) - heredocstack.append((match.group(1) == '<<-', match.group(3))) + heredocstack.append((match.group(1) in ('<<-', '<<~'), match.group(3))) ctx.pos = match.start(5) ctx.end = match.end(5) - # this may find other heredocs - for i, t, v in self.get_tokens_unprocessed(context=ctx): - yield i, t, v + # this may find other heredocs, so limit the recursion depth + if len(heredocstack) < 100: + yield from self.get_tokens_unprocessed(context=ctx) + else: + yield ctx.pos, String.Heredoc, match.group(5) ctx.pos = match.end() if outermost: @@ -108,17 +109,18 @@ def intp_string_callback(self, match, ctx): # easy ones (r'\:@{0,2}[a-zA-Z_]\w*[!?]?', String.Symbol), (words(RUBY_OPERATORS, prefix=r'\:@{0,2}'), String.Symbol), - (r":'(\\\\|\\'|[^'])*'", String.Symbol), - (r"'(\\\\|\\'|[^'])*'", String.Single), + (r":'(\\\\|\\[^\\]|[^'\\])*'", String.Symbol), (r':"', String.Symbol, 'simple-sym'), (r'([a-zA-Z_]\w*)(:)(?!:)', bygroups(String.Symbol, Punctuation)), # Since Ruby 1.9 - (r'"', String.Double, 'simple-string'), + (r'"', String.Double, 'simple-string-double'), + (r"'", String.Single, 'simple-string-single'), (r'(?~!:])|' @@ -327,9 +329,13 @@ def intp_string_callback(self, match, ctx): ], 'funcname': [ (r'\(', Punctuation, 'defexpr'), - (r'(?:([a-zA-Z_]\w*)(\.))?' - r'([a-zA-Z_]\w*[!?]?|\*\*?|[-+]@?|' - r'[/%&|^`~]|\[\]=?|<<|>>|<=?>|>=?|===?)', + (r'(?:([a-zA-Z_]\w*)(\.))?' # optional scope name, like "self." + r'(' + r'[a-zA-Z\u0080-\uffff][a-zA-Z0-9_\u0080-\uffff]*[!?=]?' # method name + r'|!=|!~|=~|\*\*?|[-+!~]@?|[/%&|^]|<=>|<[<=]?|>[>=]?|===?' # or operator override + r'|\[\]=?' # or element reference/assignment override + r'|`' # or the undocumented backtick override + r')', bygroups(Name.Class, Operator, Name.Function), '#pop'), default('#pop') ], @@ -403,8 +409,8 @@ class RubyConsoleLexer(Lexer): aliases = ['rbcon', 'irb'] mimetypes = ['text/x-ruby-shellsession'] - _prompt_re = re.compile('irb\([a-zA-Z_]\w*\):\d{3}:\d+[>*"\'] ' - '|>> |\?> ') + _prompt_re = re.compile(r'irb\([a-zA-Z_]\w*\):\d{3}:\d+[>*"\'] ' + r'|>> |\?> ') def get_tokens_unprocessed(self, text): rblexer = RubyLexer(**self.options) @@ -421,16 +427,14 @@ def get_tokens_unprocessed(self, text): curcode += line[end:] else: if curcode: - for item in do_insertions( - insertions, rblexer.get_tokens_unprocessed(curcode)): - yield item + yield from do_insertions( + insertions, rblexer.get_tokens_unprocessed(curcode)) curcode = '' insertions = [] yield match.start(), Generic.Output, line if curcode: - for item in do_insertions( - insertions, rblexer.get_tokens_unprocessed(curcode)): - yield item + yield from do_insertions( + insertions, rblexer.get_tokens_unprocessed(curcode)) class FancyLexer(RegexLexer): @@ -451,26 +455,26 @@ class FancyLexer(RegexLexer): tokens = { # copied from PerlLexer: 'balanced-regex': [ - (r'/(\\\\|\\/|[^/])*/[egimosx]*', String.Regex, '#pop'), - (r'!(\\\\|\\!|[^!])*![egimosx]*', String.Regex, '#pop'), + (r'/(\\\\|\\[^\\]|[^/\\])*/[egimosx]*', String.Regex, '#pop'), + (r'!(\\\\|\\[^\\]|[^!\\])*![egimosx]*', String.Regex, '#pop'), (r'\\(\\\\|[^\\])*\\[egimosx]*', String.Regex, '#pop'), - (r'\{(\\\\|\\\}|[^}])*\}[egimosx]*', String.Regex, '#pop'), - (r'<(\\\\|\\>|[^>])*>[egimosx]*', String.Regex, '#pop'), - (r'\[(\\\\|\\\]|[^\]])*\][egimosx]*', String.Regex, '#pop'), - (r'\((\\\\|\\\)|[^)])*\)[egimosx]*', String.Regex, '#pop'), - (r'@(\\\\|\\@|[^@])*@[egimosx]*', String.Regex, '#pop'), - (r'%(\\\\|\\%|[^%])*%[egimosx]*', String.Regex, '#pop'), - (r'\$(\\\\|\\\$|[^$])*\$[egimosx]*', String.Regex, '#pop'), + (r'\{(\\\\|\\[^\\]|[^}\\])*\}[egimosx]*', String.Regex, '#pop'), + (r'<(\\\\|\\[^\\]|[^>\\])*>[egimosx]*', String.Regex, '#pop'), + (r'\[(\\\\|\\[^\\]|[^\]\\])*\][egimosx]*', String.Regex, '#pop'), + (r'\((\\\\|\\[^\\]|[^)\\])*\)[egimosx]*', String.Regex, '#pop'), + (r'@(\\\\|\\[^\\]|[^@\\])*@[egimosx]*', String.Regex, '#pop'), + (r'%(\\\\|\\[^\\]|[^%\\])*%[egimosx]*', String.Regex, '#pop'), + (r'\$(\\\\|\\[^\\]|[^$\\])*\$[egimosx]*', String.Regex, '#pop'), ], 'root': [ (r'\s+', Text), # balanced delimiters (copied from PerlLexer): - (r's\{(\\\\|\\\}|[^}])*\}\s*', String.Regex, 'balanced-regex'), - (r's<(\\\\|\\>|[^>])*>\s*', String.Regex, 'balanced-regex'), - (r's\[(\\\\|\\\]|[^\]])*\]\s*', String.Regex, 'balanced-regex'), - (r's\((\\\\|\\\)|[^)])*\)\s*', String.Regex, 'balanced-regex'), - (r'm?/(\\\\|\\/|[^/\n])*/[gcimosx]*', String.Regex), + (r's\{(\\\\|\\[^\\]|[^}\\])*\}\s*', String.Regex, 'balanced-regex'), + (r's<(\\\\|\\[^\\]|[^>\\])*>\s*', String.Regex, 'balanced-regex'), + (r's\[(\\\\|\\[^\\]|[^\]\\])*\]\s*', String.Regex, 'balanced-regex'), + (r's\((\\\\|\\[^\\]|[^)\\])*\)\s*', String.Regex, 'balanced-regex'), + (r'm?/(\\\\|\\[^\\]|[^///\n])*/[gcimosx]*', String.Regex), (r'm(?=[/!\\{<\[(@%$])', String.Regex, 'balanced-regex'), # Comments @@ -478,9 +482,9 @@ class FancyLexer(RegexLexer): # Symbols (r'\'([^\'\s\[\](){}]+|\[\])', String.Symbol), # Multi-line DoubleQuotedString - (r'"""(\\\\|\\"|[^"])*"""', String), + (r'"""(\\\\|\\[^\\]|[^\\])*?"""', String), # DoubleQuotedString - (r'"(\\\\|\\"|[^"])*"', String), + (r'"(\\\\|\\[^\\]|[^"\\])*"', String), # keywords (r'(def|class|try|catch|finally|retry|return|return_local|match|' r'case|->|=>)\b', Keyword), @@ -498,11 +502,11 @@ class FancyLexer(RegexLexer): (r'[a-zA-Z](\w|[-+?!=*/^><%])*:', Name.Function), # operators, must be below functions (r'[-+*/~,<>=&!?%^\[\].$]+', Operator), - ('[A-Z]\w*', Name.Constant), - ('@[a-zA-Z_]\w*', Name.Variable.Instance), - ('@@[a-zA-Z_]\w*', Name.Variable.Class), + (r'[A-Z]\w*', Name.Constant), + (r'@[a-zA-Z_]\w*', Name.Variable.Instance), + (r'@@[a-zA-Z_]\w*', Name.Variable.Class), ('@@?', Operator), - ('[a-zA-Z_]\w*', Name), + (r'[a-zA-Z_]\w*', Name), # numbers - / checks are necessary to avoid mismarking regexes, # see comment in RubyLexer (r'(0[oO]?[0-7]+(?:_[0-7]+)*)(\s*)([/?])?', diff --git a/src/syntax_highlighting/libs/pygments/lexers/rust.py b/src/syntax_highlighting/libs/pygments/lexers/rust.py index 6914f54..d01f73e 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/rust.py +++ b/src/syntax_highlighting/libs/pygments/lexers/rust.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.rust ~~~~~~~~~~~~~~~~~~~~ Lexers for the Rust language. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -18,48 +17,56 @@ class RustLexer(RegexLexer): """ - Lexer for the Rust programming language (version 1.10). + Lexer for the Rust programming language (version 1.47). .. versionadded:: 1.6 """ name = 'Rust' filenames = ['*.rs', '*.rs.in'] - aliases = ['rust'] - mimetypes = ['text/rust'] - - keyword_types = ( - words(('u8', 'u16', 'u32', 'u64', 'i8', 'i16', 'i32', 'i64', - 'usize', 'isize', 'f32', 'f64', 'str', 'bool'), - suffix=r'\b'), - Keyword.Type) - - builtin_types = (words(( - # Reexported core operators - 'Copy', 'Send', 'Sized', 'Sync', - 'Drop', 'Fn', 'FnMut', 'FnOnce', - - # Reexported types and traits - 'Box', - 'ToOwned', - 'Clone', + aliases = ['rust', 'rs'] + mimetypes = ['text/rust', 'text/x-rust'] + + keyword_types = (words(( + 'u8', 'u16', 'u32', 'u64', 'u128', 'i8', 'i16', 'i32', 'i64', 'i128', + 'usize', 'isize', 'f32', 'f64', 'char', 'str', 'bool', + ), suffix=r'\b'), Keyword.Type) + + builtin_funcs_types = (words(( + 'Copy', 'Send', 'Sized', 'Sync', 'Unpin', + 'Drop', 'Fn', 'FnMut', 'FnOnce', 'drop', + 'Box', 'ToOwned', 'Clone', 'PartialEq', 'PartialOrd', 'Eq', 'Ord', - 'AsRef', 'AsMut', 'Into', 'From', - 'Default', - 'Iterator', 'Extend', 'IntoIterator', - 'DoubleEndedIterator', 'ExactSizeIterator', - 'Option', - 'Some', 'None', - 'Result', - 'Ok', 'Err', - 'SliceConcatExt', - 'String', 'ToString', - 'Vec'), suffix=r'\b'), - Name.Builtin) + 'AsRef', 'AsMut', 'Into', 'From', 'Default', + 'Iterator', 'Extend', 'IntoIterator', 'DoubleEndedIterator', + 'ExactSizeIterator', + 'Option', 'Some', 'None', + 'Result', 'Ok', 'Err', + 'String', 'ToString', 'Vec', + ), suffix=r'\b'), Name.Builtin) + + builtin_macros = (words(( + 'asm', 'assert', 'assert_eq', 'assert_ne', 'cfg', 'column', + 'compile_error', 'concat', 'concat_idents', 'dbg', 'debug_assert', + 'debug_assert_eq', 'debug_assert_ne', 'env', 'eprint', 'eprintln', + 'file', 'format', 'format_args', 'format_args_nl', 'global_asm', + 'include', 'include_bytes', 'include_str', + 'is_aarch64_feature_detected', + 'is_arm_feature_detected', + 'is_mips64_feature_detected', + 'is_mips_feature_detected', + 'is_powerpc64_feature_detected', + 'is_powerpc_feature_detected', + 'is_x86_feature_detected', + 'line', 'llvm_asm', 'log_syntax', 'macro_rules', 'matches', + 'module_path', 'option_env', 'panic', 'print', 'println', 'stringify', + 'thread_local', 'todo', 'trace_macros', 'unimplemented', 'unreachable', + 'vec', 'write', 'writeln', + ), suffix=r'!'), Name.Function.Magic) tokens = { 'root': [ # rust allows a file to start with a shebang, but if the first line - # starts with #![ then it’s not a shebang but a crate attribute. + # starts with #![ then it's not a shebang but a crate attribute. (r'#![^[\r\n].*$', Comment.Preproc), default('base'), ], @@ -77,76 +84,79 @@ class RustLexer(RegexLexer): # Macro parameters (r"""\$([a-zA-Z_]\w*|\(,?|\),?|,?)""", Comment.Preproc), # Keywords - (words(( - 'as', 'box', 'const', 'crate', 'else', 'extern', - 'for', 'if', 'impl', 'in', 'loop', 'match', 'move', - 'mut', 'pub', 'ref', 'return', 'static', 'super', - 'trait', 'unsafe', 'use', 'where', 'while'), suffix=r'\b'), - Keyword), - (words(('abstract', 'alignof', 'become', 'do', 'final', 'macro', - 'offsetof', 'override', 'priv', 'proc', 'pure', 'sizeof', - 'typeof', 'unsized', 'virtual', 'yield'), suffix=r'\b'), - Keyword.Reserved), + (words(('as', 'async', 'await', 'box', 'const', 'crate', 'dyn', + 'else', 'extern', 'for', 'if', 'impl', 'in', 'loop', + 'match', 'move', 'mut', 'pub', 'ref', 'return', 'static', + 'super', 'trait', 'unsafe', 'use', 'where', 'while'), + suffix=r'\b'), Keyword), + (words(('abstract', 'become', 'do', 'final', 'macro', 'override', + 'priv', 'typeof', 'try', 'unsized', 'virtual', 'yield'), + suffix=r'\b'), Keyword.Reserved), (r'(true|false)\b', Keyword.Constant), + (r'self\b', Name.Builtin.Pseudo), (r'mod\b', Keyword, 'modname'), (r'let\b', Keyword.Declaration), (r'fn\b', Keyword, 'funcname'), (r'(struct|enum|type|union)\b', Keyword, 'typename'), (r'(default)(\s+)(type|fn)\b', bygroups(Keyword, Text, Keyword)), keyword_types, - (r'self\b', Name.Builtin.Pseudo), - # Prelude (taken from Rust’s src/libstd/prelude.rs) - builtin_types, + (r'[sS]elf\b', Name.Builtin.Pseudo), + # Prelude (taken from Rust's src/libstd/prelude.rs) + builtin_funcs_types, + builtin_macros, # Path seperators, so types don't catch them. (r'::\b', Text), # Types in positions. (r'(?::|->)', Text, 'typename'), # Labels - (r'(break|continue)(\s*)(\'[A-Za-z_]\w*)?', + (r'(break|continue)(\b\s*)(\'[A-Za-z_]\w*)?', bygroups(Keyword, Text.Whitespace, Name.Label)), - # Character Literal + + # Character literals (r"""'(\\['"\\nrt]|\\x[0-7][0-9a-fA-F]|\\0""" r"""|\\u\{[0-9a-fA-F]{1,6}\}|.)'""", String.Char), (r"""b'(\\['"\\nrt]|\\x[0-9a-fA-F]{2}|\\0""" r"""|\\u\{[0-9a-fA-F]{1,6}\}|.)'""", String.Char), - # Binary Literal + + # Binary literals (r'0b[01_]+', Number.Bin, 'number_lit'), - # Octal Literal + # Octal literals (r'0o[0-7_]+', Number.Oct, 'number_lit'), - # Hexadecimal Literal + # Hexadecimal literals (r'0[xX][0-9a-fA-F_]+', Number.Hex, 'number_lit'), - # Decimal Literal + # Decimal literals (r'[0-9][0-9_]*(\.[0-9_]+[eE][+\-]?[0-9_]+|' r'\.[0-9_]*(?!\.)|[eE][+\-]?[0-9_]+)', Number.Float, 'number_lit'), (r'[0-9][0-9_]*', Number.Integer, 'number_lit'), - # String Literal + + # String literals (r'b"', String, 'bytestring'), (r'"', String, 'string'), - (r'b?r(#*)".*?"\1', String), + (r'(?s)b?r(#*)".*?"\1', String), - # Lifetime - (r"""'static""", Name.Builtin), - (r"""'[a-zA-Z_]\w*""", Name.Attribute), + # Lifetime names + (r"'", Operator, 'lifetime'), # Operators and Punctuation + (r'\.\.=?', Operator), (r'[{}()\[\],.;]', Punctuation), (r'[+\-*/%&|<>^!~@=:?]', Operator), - # Identifier + # Identifiers (r'[a-zA-Z_]\w*', Name), + # Raw identifiers + (r'r#[a-zA-Z_]\w*', Name), # Attributes (r'#!?\[', Comment.Preproc, 'attribute['), - # Macros - (r'([A-Za-z_]\w*)(!)(\s*)([A-Za-z_]\w*)?(\s*)(\{)', - bygroups(Comment.Preproc, Punctuation, Whitespace, Name, - Whitespace, Punctuation), 'macro{'), - (r'([A-Za-z_]\w*)(!)(\s*)([A-Za-z_]\w*)?(\()', - bygroups(Comment.Preproc, Punctuation, Whitespace, Name, - Punctuation), 'macro('), + + # Misc + # Lone hashes: not used in Rust syntax, but allowed in macro + # arguments, most famously for quote::quote!() + (r'#', Text), ], 'comment': [ (r'[^*/]+', Comment.Multiline), @@ -173,11 +183,17 @@ class RustLexer(RegexLexer): 'typename': [ (r'\s+', Text), (r'&', Keyword.Pseudo), - builtin_types, + (r"'", Operator, 'lifetime'), + builtin_funcs_types, keyword_types, (r'[a-zA-Z_]\w*', Name.Class, '#pop'), default('#pop'), ], + 'lifetime': [ + (r"(static|_)", Name.Builtin), + (r"[a-zA-Z_]+\w*", Name.Attribute), + default('#pop'), + ], 'number_lit': [ (r'[ui](8|16|32|64|size)', Keyword, '#pop'), (r'f(32|64)', Keyword, '#pop'), @@ -194,27 +210,13 @@ class RustLexer(RegexLexer): (r"""\\x[89a-fA-F][0-9a-fA-F]""", String.Escape), include('string'), ], - 'macro{': [ - (r'\{', Operator, '#push'), - (r'\}', Operator, '#pop'), - ], - 'macro(': [ - (r'\(', Operator, '#push'), - (r'\)', Operator, '#pop'), - ], 'attribute_common': [ (r'"', String, 'string'), (r'\[', Comment.Preproc, 'attribute['), - (r'\(', Comment.Preproc, 'attribute('), ], 'attribute[': [ include('attribute_common'), - (r'\];?', Comment.Preproc, '#pop'), - (r'[^"\]]+', Comment.Preproc), - ], - 'attribute(': [ - include('attribute_common'), - (r'\);?', Comment.Preproc, '#pop'), - (r'[^")]+', Comment.Preproc), + (r'\]', Comment.Preproc, '#pop'), + (r'[^"\]\[]+', Comment.Preproc), ], } diff --git a/src/syntax_highlighting/libs/pygments/lexers/sas.py b/src/syntax_highlighting/libs/pygments/lexers/sas.py index 3747ed9..7d7f9d3 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/sas.py +++ b/src/syntax_highlighting/libs/pygments/lexers/sas.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.sas ~~~~~~~~~~~~~~~~~~~ Lexer for SAS. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/src/syntax_highlighting/libs/pygments/lexers/scdoc.py b/src/syntax_highlighting/libs/pygments/lexers/scdoc.py new file mode 100644 index 0000000..48b0682 --- /dev/null +++ b/src/syntax_highlighting/libs/pygments/lexers/scdoc.py @@ -0,0 +1,82 @@ +""" + pygments.lexers.scdoc + ~~~~~~~~~~~~~~~~~~~~~ + + Lexer for scdoc, a simple man page generator. + + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re + +from pygments.lexer import RegexLexer, include, bygroups, \ + using, this +from pygments.token import Text, Comment, Keyword, String, \ + Generic + + +__all__ = ['ScdocLexer'] + + +class ScdocLexer(RegexLexer): + """ + `scdoc` is a simple man page generator for POSIX systems written in C99. + https://git.sr.ht/~sircmpwn/scdoc + + .. versionadded:: 2.5 + """ + name = 'scdoc' + aliases = ['scdoc', 'scd'] + filenames = ['*.scd', '*.scdoc'] + flags = re.MULTILINE + + tokens = { + 'root': [ + # comment + (r'^(;.+\n)', bygroups(Comment)), + + # heading with pound prefix + (r'^(#)([^#].+\n)', bygroups(Generic.Heading, Text)), + (r'^(#{2})(.+\n)', bygroups(Generic.Subheading, Text)), + # bulleted lists + (r'^(\s*)([*-])(\s)(.+\n)', + bygroups(Text, Keyword, Text, using(this, state='inline'))), + # numbered lists + (r'^(\s*)(\.+\.)( .+\n)', + bygroups(Text, Keyword, using(this, state='inline'))), + # quote + (r'^(\s*>\s)(.+\n)', bygroups(Keyword, Generic.Emph)), + # text block + (r'^(```\n)([\w\W]*?)(^```$)', bygroups(String, Text, String)), + + include('inline'), + ], + 'inline': [ + # escape + (r'\\.', Text), + # underlines + (r'(\s)(_[^_]+_)(\W|\n)', bygroups(Text, Generic.Emph, Text)), + # bold + (r'(\s)(\*[^*]+\*)(\W|\n)', bygroups(Text, Generic.Strong, Text)), + # inline code + (r'`[^`]+`', String.Backtick), + + # general text, must come last! + (r'[^\\\s]+', Text), + (r'.', Text), + ], + } + + def analyse_text(text): + """This is very similar to markdown, save for the escape characters + needed for * and _.""" + result = 0 + + if '\\*' in text: + result += 0.01 + + if '\\_' in text: + result += 0.01 + + return result diff --git a/src/syntax_highlighting/libs/pygments/lexers/scripting.py b/src/syntax_highlighting/libs/pygments/lexers/scripting.py index b3af606..47d066f 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/scripting.py +++ b/src/syntax_highlighting/libs/pygments/lexers/scripting.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.scripting ~~~~~~~~~~~~~~~~~~~~~~~~~ Lexer for scripting and embedded languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -15,11 +14,11 @@ words from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ Number, Punctuation, Error, Whitespace, Other -from pygments.util import get_bool_opt, get_list_opt, iteritems +from pygments.util import get_bool_opt, get_list_opt __all__ = ['LuaLexer', 'MoonScriptLexer', 'ChaiscriptLexer', 'LSLLexer', 'AppleScriptLexer', 'RexxLexer', 'MOOCodeLexer', 'HybrisLexer', - 'EasytrieveLexer', 'JclLexer'] + 'EasytrieveLexer', 'JclLexer', 'MiniScriptLexer'] class LuaLexer(RegexLexer): @@ -104,7 +103,7 @@ class LuaLexer(RegexLexer): (r'%s(?=%s*[.:])' % (_name, _s), Name.Class), (_name, Name.Function, '#pop'), # inline function - ('\(', Punctuation, '#pop'), + (r'\(', Punctuation, '#pop'), ], 'goto': [ @@ -142,7 +141,7 @@ def __init__(self, **options): self._functions = set() if self.func_name_highlighting: from pygments.lexers._lua_builtins import MODULES - for mod, func in iteritems(MODULES): + for mod, func in MODULES.items(): if mod not in self.disabled_modules: self._functions.update(func) RegexLexer.__init__(self, **options) @@ -157,12 +156,11 @@ def get_tokens_unprocessed(self, text): elif '.' in value: a, b = value.split('.') yield index, Name, a - yield index + len(a), Punctuation, u'.' + yield index + len(a), Punctuation, '.' yield index + len(a) + 1, Name, b continue yield index, token, value - class MoonScriptLexer(LuaLexer): """ For `MoonScript `_ source code. @@ -170,9 +168,9 @@ class MoonScriptLexer(LuaLexer): .. versionadded:: 1.5 """ - name = "MoonScript" - aliases = ["moon", "moonscript"] - filenames = ["*.moon"] + name = 'MoonScript' + aliases = ['moonscript', 'moon'] + filenames = ['*.moon'] mimetypes = ['text/x-moonscript', 'application/x-moonscript'] tokens = { @@ -239,7 +237,7 @@ class ChaiscriptLexer(RegexLexer): """ name = 'ChaiScript' - aliases = ['chai', 'chaiscript'] + aliases = ['chaiscript', 'chai'] filenames = ['*.chai'] mimetypes = ['text/x-chaiscript', 'application/x-chaiscript'] @@ -284,7 +282,7 @@ class ChaiscriptLexer(RegexLexer): (r'0x[0-9a-fA-F]+', Number.Hex), (r'[0-9]+', Number.Integer), (r'"', String.Double, 'dqstring'), - (r"'(\\\\|\\'|[^'])*'", String.Single), + (r"'(\\\\|\\[^\\]|[^'\\])*'", String.Single), ], 'dqstring': [ (r'\$\{[^"}]+?\}', String.Interpol), @@ -660,18 +658,18 @@ class AppleScriptLexer(RegexLexer): tokens = { 'root': [ (r'\s+', Text), - (u'¬\\n', String.Escape), + (r'¬\n', String.Escape), (r"'s\s+", Text), # This is a possessive, consider moving (r'(--|#).*?$', Comment), (r'\(\*', Comment.Multiline, 'comment'), (r'[(){}!,.:]', Punctuation), - (u'(«)([^»]+)(»)', + (r'(«)([^»]+)(»)', bygroups(Text, Name.Builtin, Text)), (r'\b((?:considering|ignoring)\s*)' r'(application responses|case|diacriticals|hyphens|' r'numeric strings|punctuation|white space)', bygroups(Keyword, Name.Builtin)), - (u'(-|\\*|\\+|&|≠|>=?|<=?|=|≥|≤|/|÷|\\^)', Operator), + (r'(-|\*|\+|&|≠|>=?|<=?|=|≥|≤|/|÷|\^)', Operator), (r"\b(%s)\b" % '|'.join(Operators), Operator.Word), (r'^(\s*(?:on|end)\s+)' r'(%s)' % '|'.join(StudioEvents[::-1]), @@ -690,14 +688,14 @@ class AppleScriptLexer(RegexLexer): (r'\b(%s)s?\b' % '|'.join(StudioClasses), Name.Builtin), (r'\b(%s)\b' % '|'.join(StudioCommands), Name.Builtin), (r'\b(%s)\b' % '|'.join(References), Name.Builtin), - (r'"(\\\\|\\"|[^"])*"', String.Double), + (r'"(\\\\|\\[^\\]|[^"\\])*"', String.Double), (r'\b(%s)\b' % Identifiers, Name.Variable), (r'[-+]?(\d+\.\d*|\d*\.\d+)(E[-+][0-9]+)?', Number.Float), (r'[-+]?\d+', Number.Integer), ], 'comment': [ - ('\(\*', Comment.Multiline, '#push'), - ('\*\)', Comment.Multiline, '#pop'), + (r'\(\*', Comment.Multiline, '#push'), + (r'\*\)', Comment.Multiline, '#pop'), ('[^*(]+', Comment.Multiline), ('[*(]', Comment.Multiline), ], @@ -834,7 +832,7 @@ class MOOCodeLexer(RegexLexer): # Numbers (r'(0|[1-9][0-9_]*)', Number.Integer), # Strings - (r'"(\\\\|\\"|[^"])*"', String), + (r'"(\\\\|\\[^\\]|[^"\\])*"', String), # exceptions (r'(E_PERM|E_DIV)', Name.Exception), # db-refs @@ -925,7 +923,7 @@ class HybrisLexer(RegexLexer): 'Runnable', 'CGI', 'ClientSocket', 'Socket', 'ServerSocket', 'File', 'Console', 'Directory', 'Exception'), suffix=r'\b'), Keyword.Type), - (r'"(\\\\|\\"|[^"])*"', String), + (r'"(\\\\|\\[^\\]|[^"\\])*"', String), (r"'\\.'|'[^\\]'|'\\u[0-9a-f]{4}'", String.Char), (r'(\.)([a-zA-Z_]\w*)', bygroups(Operator, Name.Attribute)), @@ -945,6 +943,15 @@ class HybrisLexer(RegexLexer): ], } + def analyse_text(text): + """public method and private method don't seem to be quite common + elsewhere.""" + result = 0 + if re.search(r'\b(?:public|private)\s+method\b', text): + result += 0.01 + return result + + class EasytrieveLexer(RegexLexer): """ @@ -977,7 +984,7 @@ class EasytrieveLexer(RegexLexer): _DELIMITER_PATTERN = '[' + _DELIMITERS + ']' _DELIMITER_PATTERN_CAPTURE = '(' + _DELIMITER_PATTERN + ')' _NON_DELIMITER_OR_COMMENT_PATTERN = '[^' + _DELIMITERS_OR_COMENT + ']' - _OPERATORS_PATTERN = u'[.+\\-/=\\[\\](){}<>;,&%¬]' + _OPERATORS_PATTERN = '[.+\\-/=\\[\\](){}<>;,&%¬]' _KEYWORDS = [ 'AFTER-BREAK', 'AFTER-LINE', 'AFTER-SCREEN', 'AIM', 'AND', 'ATTR', 'BEFORE', 'BEFORE-BREAK', 'BEFORE-LINE', 'BEFORE-SCREEN', 'BUSHU', @@ -1220,3 +1227,57 @@ def analyse_text(text): result = 1.0 assert 0.0 <= result <= 1.0 return result + + +class MiniScriptLexer(RegexLexer): + """ + For `MiniScript `_ source code. + + .. versionadded:: 2.6 + """ + + name = 'MiniScript' + aliases = ['miniscript', 'ms'] + filenames = ['*.ms'] + mimetypes = ['text/x-minicript', 'application/x-miniscript'] + + tokens = { + 'root': [ + (r'#!(.*?)$', Comment.Preproc), + default('base'), + ], + 'base': [ + ('//.*$', Comment.Single), + (r'(?i)(\d*\.\d+|\d+\.\d*)(e[+-]?\d+)?', Number), + (r'(?i)\d+e[+-]?\d+', Number), + (r'\d+', Number), + (r'\n', Text), + (r'[^\S\n]+', Text), + (r'"', String, 'string_double'), + (r'(==|!=|<=|>=|[=+\-*/%^<>.:])', Operator), + (r'[;,\[\]{}()]', Punctuation), + (words(( + 'break', 'continue', 'else', 'end', 'for', 'function', 'if', + 'in', 'isa', 'then', 'repeat', 'return', 'while'), suffix=r'\b'), + Keyword), + (words(( + 'abs', 'acos', 'asin', 'atan', 'ceil', 'char', 'cos', 'floor', + 'log', 'round', 'rnd', 'pi', 'sign', 'sin', 'sqrt', 'str', 'tan', + 'hasIndex', 'indexOf', 'len', 'val', 'code', 'remove', 'lower', + 'upper', 'replace', 'split', 'indexes', 'values', 'join', 'sum', + 'sort', 'shuffle', 'push', 'pop', 'pull', 'range', + 'print', 'input', 'time', 'wait', 'locals', 'globals', 'outer', + 'yield'), suffix=r'\b'), + Name.Builtin), + (r'(true|false|null)\b', Keyword.Constant), + (r'(and|or|not|new)\b', Operator.Word), + (r'(self|super|__isa)\b', Name.Builtin.Pseudo), + (r'[a-zA-Z_]\w*', Name.Variable) + ], + 'string_double': [ + (r'[^"\n]+', String), + (r'""', String), + (r'"', String, '#pop'), + (r'\n', Text, '#pop'), # Stray linefeed also terminates strings. + ] + } diff --git a/src/syntax_highlighting/libs/pygments/lexers/sgf.py b/src/syntax_highlighting/libs/pygments/lexers/sgf.py new file mode 100644 index 0000000..1098a92 --- /dev/null +++ b/src/syntax_highlighting/libs/pygments/lexers/sgf.py @@ -0,0 +1,60 @@ +""" + pygments.lexers.sgf + ~~~~~~~~~~~~~~~~~~~ + + Lexer for Smart Game Format (sgf) file format. + + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer, bygroups +from pygments.token import Name, Literal, String, Text, Punctuation + +__all__ = ["SmartGameFormatLexer"] + + +class SmartGameFormatLexer(RegexLexer): + """ + Lexer for Smart Game Format (sgf) file format. + + The format is used to store game records of board games for two players + (mainly Go game). + For more information about the definition of the format, see: + https://www.red-bean.com/sgf/ + + .. versionadded:: 2.4 + """ + name = 'SmartGameFormat' + aliases = ['sgf'] + filenames = ['*.sgf'] + + tokens = { + 'root': [ + (r'[\s():;]', Punctuation), + # tokens: + (r'(A[BW]|AE|AN|AP|AR|AS|[BW]L|BM|[BW]R|[BW]S|[BW]T|CA|CH|CP|CR|' + r'DD|DM|DO|DT|EL|EV|EX|FF|FG|G[BW]|GC|GM|GN|HA|HO|ID|IP|IT|IY|KM|' + r'KO|LB|LN|LT|L|MA|MN|M|N|OB|OM|ON|OP|OT|OV|P[BW]|PC|PL|PM|RE|RG|' + r'RO|RU|SO|SC|SE|SI|SL|SO|SQ|ST|SU|SZ|T[BW]|TC|TE|TM|TR|UC|US|VW|' + r'V|[BW]|C)', + Name.Builtin), + # number: + (r'(\[)([0-9.]+)(\])', + bygroups(Punctuation, Literal.Number, Punctuation)), + # date: + (r'(\[)([0-9]{4}-[0-9]{2}-[0-9]{2})(\])', + bygroups(Punctuation, Literal.Date, Punctuation)), + # point: + (r'(\[)([a-z]{2})(\])', + bygroups(Punctuation, String, Punctuation)), + # double points: + (r'(\[)([a-z]{2})(:)([a-z]{2})(\])', + bygroups(Punctuation, String, Punctuation, String, Punctuation)), + + (r'(\[)([\w\s#()+,\-.:?]+)(\])', + bygroups(Punctuation, String, Punctuation)), + (r'(\[)(\s.*)(\])', + bygroups(Punctuation, Text, Punctuation)), + ], + } diff --git a/src/syntax_highlighting/libs/pygments/lexers/shell.py b/src/syntax_highlighting/libs/pygments/lexers/shell.py index ceb6f14..713b50a 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/shell.py +++ b/src/syntax_highlighting/libs/pygments/lexers/shell.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.shell ~~~~~~~~~~~~~~~~~~~~~ Lexers for various shells. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -19,8 +18,9 @@ __all__ = ['BashLexer', 'BashSessionLexer', 'TcshLexer', 'BatchLexer', - 'MSDOSSessionLexer', 'PowerShellLexer', - 'PowerShellSessionLexer', 'TcshSessionLexer', 'FishShellLexer'] + 'SlurmBashLexer', 'MSDOSSessionLexer', 'PowerShellLexer', + 'PowerShellSessionLexer', 'TcshSessionLexer', 'FishShellLexer', + 'ExeclineLexer'] line_re = re.compile('.*?\n') @@ -38,7 +38,7 @@ class BashLexer(RegexLexer): '*.exheres-0', '*.exlib', '*.zsh', '.bashrc', 'bashrc', '.bash_*', 'bash_*', 'zshrc', '.zshrc', 'PKGBUILD'] - mimetypes = ['application/x-sh', 'application/x-shellscript'] + mimetypes = ['application/x-sh', 'application/x-shellscript', 'text/x-shellscript'] tokens = { 'root': [ @@ -56,7 +56,7 @@ class BashLexer(RegexLexer): (r'\$', Text), ], 'basic': [ - (r'\b(if|fi|else|while|do|done|for|then|return|function|case|' + (r'\b(if|fi|else|while|in|do|done|for|then|return|function|case|' r'select|continue|until|esac|elif)(\s*)\b', bygroups(Keyword, Text)), (r'\b(alias|bg|bind|break|builtin|caller|cd|command|compgen|' @@ -76,7 +76,7 @@ class BashLexer(RegexLexer): (r'&&|\|\|', Operator), ], 'data': [ - (r'(?s)\$?"(\\\\|\\[0-7]+|\\.|[^"\\$])*"', String.Double), + (r'(?s)\$?"(\\.|[^"\\$])*"', String.Double), (r'"', String.Double, 'string'), (r"(?s)\$'(\\\\|\\[0-7]+|\\.|[^'\\])*'", String.Single), (r"(?s)'.*?'", String.Single), @@ -126,12 +126,38 @@ def analyse_text(text): return 0.2 +class SlurmBashLexer(BashLexer): + """ + Lexer for (ba|k|z|)sh Slurm scripts. + + .. versionadded:: 2.4 + """ + + name = 'Slurm' + aliases = ['slurm', 'sbatch'] + filenames = ['*.sl'] + mimetypes = [] + EXTRA_KEYWORDS = {'srun'} + + def get_tokens_unprocessed(self, text): + for index, token, value in BashLexer.get_tokens_unprocessed(self, text): + if token is Text and value in self.EXTRA_KEYWORDS: + yield index, Name.Builtin, value + elif token is Comment.Single and 'SBATCH' in value: + yield index, Keyword.Pseudo, value + else: + yield index, token, value + + class ShellSessionBaseLexer(Lexer): """ - Base lexer for simplistic shell sessions. + Base lexer for shell sessions. .. versionadded:: 2.1 """ + + _venv = re.compile(r'^(\([^)]*\))(\s*)') + def get_tokens_unprocessed(self, text): innerlexer = self._innerLexerCls(**self.options) @@ -142,11 +168,20 @@ def get_tokens_unprocessed(self, text): for match in line_re.finditer(text): line = match.group() - m = re.match(self._ps1rgx, line) - if backslash_continuation: - curcode += line - backslash_continuation = curcode.endswith('\\\n') - elif m: + + venv_match = self._venv.match(line) + if venv_match: + venv = venv_match.group(1) + venv_whitespace = venv_match.group(2) + insertions.append((len(curcode), + [(0, Generic.Prompt.VirtualEnv, venv)])) + if venv_whitespace: + insertions.append((len(curcode), + [(0, Text, venv_whitespace)])) + line = line[venv_match.end():] + + m = self._ps1rgx.match(line) + if m: # To support output lexers (say diff output), the output # needs to be broken by prompts whenever the output lexer # changes. @@ -157,10 +192,13 @@ def get_tokens_unprocessed(self, text): [(0, Generic.Prompt, m.group(1))])) curcode += m.group(2) backslash_continuation = curcode.endswith('\\\n') - elif line.startswith(self._ps2): - insertions.append((len(curcode), - [(0, Generic.Prompt, line[:len(self._ps2)])])) - curcode += line[len(self._ps2):] + elif backslash_continuation: + if line.startswith(self._ps2): + insertions.append((len(curcode), + [(0, Generic.Prompt, line[:len(self._ps2)])])) + curcode += line[len(self._ps2):] + else: + curcode += line backslash_continuation = curcode.endswith('\\\n') else: if insertions: @@ -178,7 +216,8 @@ def get_tokens_unprocessed(self, text): class BashSessionLexer(ShellSessionBaseLexer): """ - Lexer for simplistic shell sessions. + Lexer for Bash shell sessions, i.e. command lines, including a + prompt, interspersed with output. .. versionadded:: 1.1 """ @@ -189,10 +228,10 @@ class BashSessionLexer(ShellSessionBaseLexer): mimetypes = ['application/x-shell-session', 'application/x-sh-session'] _innerLexerCls = BashLexer - _ps1rgx = \ + _ps1rgx = re.compile( r'^((?:(?:\[.*?\])|(?:\(\S+\))?(?:| |sh\S*?|\w+\S+[@:]\S+(?:\s+\S+)' \ - r'?|\[\S+[@:][^\n]+\].+))\s*[$#%])(.*\n?)' - _ps2 = '>' + r'?|\[\S+[@:][^\n]+\].+))\s*[$#%]\s*)(.*\n?)') + _ps2 = '> ' class BatchLexer(RegexLexer): @@ -202,7 +241,7 @@ class BatchLexer(RegexLexer): .. versionadded:: 0.7 """ name = 'Batchfile' - aliases = ['bat', 'batch', 'dosbatch', 'winbatch'] + aliases = ['batch', 'bat', 'dosbatch', 'winbatch'] filenames = ['*.bat', '*.cmd'] mimetypes = ['application/x-dos-batch'] @@ -211,14 +250,14 @@ class BatchLexer(RegexLexer): _nl = r'\n\x1a' _punct = r'&<>|' _ws = r'\t\v\f\r ,;=\xa0' + _nlws = r'\s\x1a\xa0,;=' _space = r'(?:(?:(?:\^[%s])?[%s])+)' % (_nl, _ws) _keyword_terminator = (r'(?=(?:\^[%s]?)?[%s+./:[\\\]]|[%s%s(])' % (_nl, _ws, _nl, _punct)) _token_terminator = r'(?=\^?[%s]|[%s%s])' % (_ws, _punct, _nl) _start_label = r'((?:(?<=^[^:])|^[^:]?)[%s]*)(:)' % _ws - _label = r'(?:(?:[^%s%s%s+:^]|\^[%s]?[\w\W])*)' % (_nl, _punct, _ws, _nl) - _label_compound = (r'(?:(?:[^%s%s%s+:^)]|\^[%s]?[^)])*)' % - (_nl, _punct, _ws, _nl)) + _label = r'(?:(?:[^%s%s+:^]|\^[%s]?[\w\W])*)' % (_nlws, _punct, _nl) + _label_compound = r'(?:(?:[^%s%s+:^)]|\^[%s]?[^)])*)' % (_nlws, _punct, _nl) _number = r'(?:-?(?:0[0-7]+|0x[\da-f]+|\d+)%s)' % _token_terminator _opword = r'(?:equ|geq|gtr|leq|lss|neq)' _string = r'(?:"[^%s"]*(?:"|(?=[%s])))' % (_nl, _nl) @@ -228,9 +267,8 @@ class BatchLexer(RegexLexer): r'(?:\^?![^!:%s]+(?::(?:~(?:-?\d+)?(?:,(?:-?\d+)?)?|(?:' r'[^!%s^]|\^[^!%s])[^=%s]*=(?:[^!%s^]|\^[^!%s])*)?)?\^?!))' % (_nl, _nl, _nl, _nl, _nl, _nl, _nl, _nl, _nl, _nl, _nl, _nl)) - _core_token = r'(?:(?:(?:\^[%s]?)?[^"%s%s%s])+)' % (_nl, _nl, _punct, _ws) - _core_token_compound = r'(?:(?:(?:\^[%s]?)?[^"%s%s%s)])+)' % (_nl, _nl, - _punct, _ws) + _core_token = r'(?:(?:(?:\^[%s]?)?[^"%s%s])+)' % (_nl, _nlws, _punct) + _core_token_compound = r'(?:(?:(?:\^[%s]?)?[^"%s%s)])+)' % (_nl, _nlws, _punct) _token = r'(?:[%s]+|%s)' % (_punct, _core_token) _token_compound = r'(?:[%s]+|%s)' % (_punct, _core_token_compound) _stoken = (r'(?:[%s]+|(?:%s|%s|%s)+)' % @@ -341,7 +379,8 @@ def _make_follow_state(compound, _label=_label, return state def _make_arithmetic_state(compound, _nl=_nl, _punct=_punct, - _string=_string, _variable=_variable, _ws=_ws): + _string=_string, _variable=_variable, + _ws=_ws, _nlws=_nlws): op = r'=+\-*/!~' state = [] if compound: @@ -352,8 +391,8 @@ def _make_arithmetic_state(compound, _nl=_nl, _punct=_punct, (r'\d+', Number.Integer), (r'[(),]+', Punctuation), (r'([%s]|%%|\^\^)+' % op, Operator), - (r'(%s|%s|(\^[%s]?)?[^()%s%%^"%s%s%s]|\^[%s%s]?%s)+' % - (_string, _variable, _nl, op, _nl, _punct, _ws, _nl, _ws, + (r'(%s|%s|(\^[%s]?)?[^()%s%%\^"%s%s]|\^[%s]?%s)+' % + (_string, _variable, _nl, op, _nlws, _punct, _nlws, r'[^)]' if compound else r'[\w\W]'), using(this, state='variable')), (r'(?=[\x00|&])', Text, '#pop'), @@ -387,15 +426,15 @@ def _make_redirect_state(compound, _core_token_compound=_core_token_compound, _nl=_nl, _punct=_punct, _stoken=_stoken, _string=_string, _space=_space, - _variable=_variable, _ws=_ws): + _variable=_variable, _nlws=_nlws): stoken_compound = (r'(?:[%s]+|(?:%s|%s|%s)+)' % (_punct, _string, _variable, _core_token_compound)) return [ - (r'((?:(?<=[%s%s])\d)?)(>>?&|<&)([%s%s]*)(\d)' % - (_nl, _ws, _nl, _ws), + (r'((?:(?<=[%s])\d)?)(>>?&|<&)([%s]*)(\d)' % + (_nlws, _nlws), bygroups(Number.Integer, Punctuation, Text, Number.Integer)), - (r'((?:(?<=[%s%s])(?>?|<)(%s?%s)' % - (_nl, _ws, _nl, _space, stoken_compound if compound else _stoken), + (r'((?:(?<=[%s])(?>?|<)(%s?%s)' % + (_nlws, _nl, _space, stoken_compound if compound else _stoken), bygroups(Number.Integer, Punctuation, using(this, state='text'))) ] @@ -434,7 +473,7 @@ def _make_redirect_state(compound, 'text': [ (r'"', String.Double, 'string'), include('variable-or-escape'), - (r'[^"%%^%s%s%s\d)]+|.' % (_nl, _punct, _ws), Text) + (r'[^"%%^%s%s\d)]+|.' % (_nlws, _punct), Text) ], 'variable': [ (r'"', String.Double, 'string'), @@ -455,13 +494,13 @@ def _make_redirect_state(compound, include('follow') ], 'for/f': [ - (r'(")((?:%s|[^"])*?")([%s%s]*)(\))' % (_variable, _nl, _ws), + (r'(")((?:%s|[^"])*?")([%s]*)(\))' % (_variable, _nlws), bygroups(String.Double, using(this, state='string'), Text, Punctuation)), (r'"', String.Double, ('#pop', 'for2', 'string')), - (r"('(?:%%%%|%s|[\w\W])*?')([%s%s]*)(\))" % (_variable, _nl, _ws), + (r"('(?:%%%%|%s|[\w\W])*?')([%s]*)(\))" % (_variable, _nlws), bygroups(using(this, state='sqstring'), Text, Punctuation)), - (r'(`(?:%%%%|%s|[\w\W])*?`)([%s%s]*)(\))' % (_variable, _nl, _ws), + (r'(`(?:%%%%|%s|[\w\W])*?`)([%s]*)(\))' % (_variable, _nlws), bygroups(using(this, state='bqstring'), Text, Punctuation)), include('for2') ], @@ -507,7 +546,8 @@ def _make_redirect_state(compound, class MSDOSSessionLexer(ShellSessionBaseLexer): """ - Lexer for simplistic MSDOS sessions. + Lexer for MS DOS shell sessions, i.e. command lines, including a + prompt, interspersed with output. .. versionadded:: 2.1 """ @@ -518,7 +558,7 @@ class MSDOSSessionLexer(ShellSessionBaseLexer): mimetypes = [] _innerLexerCls = BatchLexer - _ps1rgx = r'^([^>]+>)(.*\n?)' + _ps1rgx = re.compile(r'^([^>]*>)(.*\n?)') _ps2 = 'More? ' @@ -592,7 +632,8 @@ class TcshLexer(RegexLexer): class TcshSessionLexer(ShellSessionBaseLexer): """ - Lexer for Tcsh sessions. + Lexer for Tcsh sessions, i.e. command lines, including a + prompt, interspersed with output. .. versionadded:: 2.1 """ @@ -603,7 +644,7 @@ class TcshSessionLexer(ShellSessionBaseLexer): mimetypes = [] _innerLexerCls = TcshLexer - _ps1rgx = r'^([^>]+>)(.*\n?)' + _ps1rgx = re.compile(r'^([^>]+>)(.*\n?)') _ps2 = '? ' @@ -614,7 +655,7 @@ class PowerShellLexer(RegexLexer): .. versionadded:: 1.5 """ name = 'PowerShell' - aliases = ['powershell', 'posh', 'ps1', 'psm1'] + aliases = ['powershell', 'pwsh', 'posh', 'ps1', 'psm1'] filenames = ['*.ps1', '*.psm1'] mimetypes = ['text/x-powershell'] @@ -638,13 +679,29 @@ class PowerShellLexer(RegexLexer): 'wildcard').split() verbs = ( - 'write where wait use update unregister undo trace test tee take ' - 'suspend stop start split sort skip show set send select scroll resume ' - 'restore restart resolve resize reset rename remove register receive ' - 'read push pop ping out new move measure limit join invoke import ' - 'group get format foreach export expand exit enter enable disconnect ' - 'disable debug cxnew copy convertto convertfrom convert connect ' - 'complete compare clear checkpoint aggregate add').split() + 'write where watch wait use update unregister unpublish unprotect ' + 'unlock uninstall undo unblock trace test tee take sync switch ' + 'suspend submit stop step start split sort skip show set send select ' + 'search scroll save revoke resume restore restart resolve resize ' + 'reset request repair rename remove register redo receive read push ' + 'publish protect pop ping out optimize open new move mount merge ' + 'measure lock limit join invoke install initialize import hide group ' + 'grant get format foreach find export expand exit enter enable edit ' + 'dismount disconnect disable deny debug cxnew copy convertto ' + 'convertfrom convert connect confirm compress complete compare close ' + 'clear checkpoint block backup assert approve aggregate add').split() + + aliases_ = ( + 'ac asnp cat cd cfs chdir clc clear clhy cli clp cls clv cnsn ' + 'compare copy cp cpi cpp curl cvpa dbp del diff dir dnsn ebp echo epal ' + 'epcsv epsn erase etsn exsn fc fhx fl foreach ft fw gal gbp gc gci gcm ' + 'gcs gdr ghy gi gjb gl gm gmo gp gps gpv group gsn gsnp gsv gu gv gwmi ' + 'h history icm iex ihy ii ipal ipcsv ipmo ipsn irm ise iwmi iwr kill lp ' + 'ls man md measure mi mount move mp mv nal ndr ni nmo npssc nsn nv ogv ' + 'oh popd ps pushd pwd r rbp rcjb rcsn rd rdr ren ri rjb rm rmdir rmo ' + 'rni rnp rp rsn rsnp rujb rv rvpa rwmi sajb sal saps sasv sbp sc select ' + 'set shcm si sl sleep sls sort sp spjb spps spsv start sujb sv swmi tee ' + 'trcm type wget where wjb write').split() commenthelp = ( 'component description example externalhelp forwardhelpcategory ' @@ -672,10 +729,11 @@ class PowerShellLexer(RegexLexer): (r'(%s)\b' % '|'.join(keywords), Keyword), (r'-(%s)\b' % '|'.join(operators), Operator), (r'(%s)-[a-z_]\w*\b' % '|'.join(verbs), Name.Builtin), + (r'(%s)\s' % '|'.join(aliases_), Name.Builtin), (r'\[[a-z_\[][\w. `,\[\]]*\]', Name.Constant), # .net [type]s (r'-[a-z_]\w*', Name), (r'\w+', Name), - (r'[.,;@{}\[\]$()=+*/\\&%!~?^`|<>-]|::', Punctuation), + (r'[.,;:@{}\[\]$()=+*/\\&%!~?^`|<>-]', Punctuation), ], 'child': [ (r'\)', Punctuation, '#pop'), @@ -706,18 +764,19 @@ class PowerShellLexer(RegexLexer): class PowerShellSessionLexer(ShellSessionBaseLexer): """ - Lexer for simplistic Windows PowerShell sessions. + Lexer for PowerShell sessions, i.e. command lines, including a + prompt, interspersed with output. .. versionadded:: 2.1 """ name = 'PowerShell Session' - aliases = ['ps1con'] + aliases = ['pwsh-session', 'ps1con'] filenames = [] mimetypes = [] _innerLexerCls = PowerShellLexer - _ps1rgx = r'^(PS [^>]+> )(.*\n?)' + _ps1rgx = re.compile(r'^((?:\[[^]]+\]: )?PS[^>]*> ?)(.*\n?)') _ps2 = '>> ' @@ -792,3 +851,62 @@ class FishShellLexer(RegexLexer): include('root'), ], } + +class ExeclineLexer(RegexLexer): + """ + Lexer for Laurent Bercot's execline language + (https://skarnet.org/software/execline). + + .. versionadded:: 2.7 + """ + + name = 'execline' + aliases = ['execline'] + filenames = ['*.exec'] + + tokens = { + 'root': [ + include('basic'), + include('data'), + include('interp') + ], + 'interp': [ + (r'\$\{', String.Interpol, 'curly'), + (r'\$[\w@#]+', Name.Variable), # user variable + (r'\$', Text), + ], + 'basic': [ + (r'\b(background|backtick|cd|define|dollarat|elgetopt|' + r'elgetpositionals|elglob|emptyenv|envfile|exec|execlineb|' + r'exit|export|fdblock|fdclose|fdmove|fdreserve|fdswap|' + r'forbacktickx|foreground|forstdin|forx|getcwd|getpid|heredoc|' + r'homeof|if|ifelse|ifte|ifthenelse|importas|loopwhilex|' + r'multidefine|multisubstitute|pipeline|piperw|posix-cd|' + r'redirfd|runblock|shift|trap|tryexec|umask|unexport|wait|' + r'withstdinas)\b', Name.Builtin), + (r'\A#!.+\n', Comment.Hashbang), + (r'#.*\n', Comment.Single), + (r'[{}]', Operator) + ], + 'data': [ + (r'(?s)"(\\.|[^"\\$])*"', String.Double), + (r'"', String.Double, 'string'), + (r'\s+', Text), + (r'[^\s{}$"\\]+', Text) + ], + 'string': [ + (r'"', String.Double, '#pop'), + (r'(?s)(\\\\|\\.|[^"\\$])+', String.Double), + include('interp'), + ], + 'curly': [ + (r'\}', String.Interpol, '#pop'), + (r'[\w#@]+', Name.Variable), + include('root') + ] + + } + + def analyse_text(text): + if shebang_matches(text, r'execlineb'): + return 1 diff --git a/src/syntax_highlighting/libs/pygments/lexers/sieve.py b/src/syntax_highlighting/libs/pygments/lexers/sieve.py new file mode 100644 index 0000000..6fa33d3 --- /dev/null +++ b/src/syntax_highlighting/libs/pygments/lexers/sieve.py @@ -0,0 +1,68 @@ +""" + pygments.lexers.sieve + ~~~~~~~~~~~~~~~~~~~~~ + + Lexer for Sieve file format. + + https://tools.ietf.org/html/rfc5228 + https://tools.ietf.org/html/rfc5173 + https://tools.ietf.org/html/rfc5229 + https://tools.ietf.org/html/rfc5230 + https://tools.ietf.org/html/rfc5232 + https://tools.ietf.org/html/rfc5235 + https://tools.ietf.org/html/rfc5429 + https://tools.ietf.org/html/rfc8580 + + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer, bygroups +from pygments.token import Comment, Name, Literal, String, Text, Punctuation, Keyword + +__all__ = ["SieveLexer"] + + +class SieveLexer(RegexLexer): + """ + Lexer for sieve format. + """ + name = 'Sieve' + filenames = ['*.siv', '*.sieve'] + aliases = ['sieve'] + + tokens = { + 'root': [ + (r'\s+', Text), + (r'[();,{}\[\]]', Punctuation), + # import: + (r'(?i)require', + Keyword.Namespace), + # tags: + (r'(?i)(:)(addresses|all|contains|content|create|copy|comparator|count|days|detail|domain|fcc|flags|from|handle|importance|is|localpart|length|lowerfirst|lower|matches|message|mime|options|over|percent|quotewildcard|raw|regex|specialuse|subject|text|under|upperfirst|upper|value)', + bygroups(Name.Tag, Name.Tag)), + # tokens: + (r'(?i)(address|addflag|allof|anyof|body|discard|elsif|else|envelope|ereject|exists|false|fileinto|if|hasflag|header|keep|notify_method_capability|notify|not|redirect|reject|removeflag|setflag|size|spamtest|stop|string|true|vacation|virustest)', + Name.Builtin), + (r'(?i)set', + Keyword.Declaration), + # number: + (r'([0-9.]+)([kmgKMG])?', + bygroups(Literal.Number, Literal.Number)), + # comment: + (r'#.*$', + Comment.Single), + (r'/\*.*\*/', + Comment.Multiline), + # string: + (r'"[^"]*?"', + String), + # text block: + (r'text:', + Name.Tag, 'text'), + ], + 'text': [ + (r'[^.].*?\n', String), + (r'^\.', Punctuation, "#pop"), + ] + } diff --git a/src/syntax_highlighting/libs/pygments/lexers/slash.py b/src/syntax_highlighting/libs/pygments/lexers/slash.py new file mode 100644 index 0000000..df0e23d --- /dev/null +++ b/src/syntax_highlighting/libs/pygments/lexers/slash.py @@ -0,0 +1,184 @@ +""" + pygments.lexers.slash + ~~~~~~~~~~~~~~~~~~~~~ + + Lexer for the `Slash `_ programming + language. + + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import ExtendedRegexLexer, bygroups, DelegatingLexer +from pygments.token import Name, Number, String, Comment, Punctuation, \ + Other, Keyword, Operator, Whitespace + +__all__ = ['SlashLexer'] + + +class SlashLanguageLexer(ExtendedRegexLexer): + _nkw = r'(?=[^a-zA-Z_0-9])' + + def move_state(new_state): + return ("#pop", new_state) + + def right_angle_bracket(lexer, match, ctx): + if len(ctx.stack) > 1 and ctx.stack[-2] == "string": + ctx.stack.pop() + yield match.start(), String.Interpol, '}' + ctx.pos = match.end() + pass + + tokens = { + "root": [ + (r"<%=", Comment.Preproc, move_state("slash")), + (r"<%!!", Comment.Preproc, move_state("slash")), + (r"<%#.*?%>", Comment.Multiline), + (r"<%", Comment.Preproc, move_state("slash")), + (r".|\n", Other), + ], + "string": [ + (r"\\", String.Escape, move_state("string_e")), + (r"\"", String, move_state("slash")), + (r"#\{", String.Interpol, "slash"), + (r'.|\n', String), + ], + "string_e": [ + (r'n', String.Escape, move_state("string")), + (r't', String.Escape, move_state("string")), + (r'r', String.Escape, move_state("string")), + (r'e', String.Escape, move_state("string")), + (r'x[a-fA-F0-9]{2}', String.Escape, move_state("string")), + (r'.', String.Escape, move_state("string")), + ], + "regexp": [ + (r'}[a-z]*', String.Regex, move_state("slash")), + (r'\\(.|\n)', String.Regex), + (r'{', String.Regex, "regexp_r"), + (r'.|\n', String.Regex), + ], + "regexp_r": [ + (r'}[a-z]*', String.Regex, "#pop"), + (r'\\(.|\n)', String.Regex), + (r'{', String.Regex, "regexp_r"), + ], + "slash": [ + (r"%>", Comment.Preproc, move_state("root")), + (r"\"", String, move_state("string")), + (r"'[a-zA-Z0-9_]+", String), + (r'%r{', String.Regex, move_state("regexp")), + (r'/\*.*?\*/', Comment.Multiline), + (r"(#|//).*?\n", Comment.Single), + (r'-?[0-9]+e[+-]?[0-9]+', Number.Float), + (r'-?[0-9]+\.[0-9]+(e[+-]?[0-9]+)?', Number.Float), + (r'-?[0-9]+', Number.Integer), + (r'nil'+_nkw, Name.Builtin), + (r'true'+_nkw, Name.Builtin), + (r'false'+_nkw, Name.Builtin), + (r'self'+_nkw, Name.Builtin), + (r'(class)(\s+)([A-Z][a-zA-Z0-9_\']*)', + bygroups(Keyword, Whitespace, Name.Class)), + (r'class'+_nkw, Keyword), + (r'extends'+_nkw, Keyword), + (r'(def)(\s+)(self)(\s*)(\.)(\s*)([a-z_][a-zA-Z0-9_\']*=?|<<|>>|==|<=>|<=|<|>=|>|\+|-(self)?|~(self)?|\*|/|%|^|&&|&|\||\[\]=?)', + bygroups(Keyword, Whitespace, Name.Builtin, Whitespace, Punctuation, Whitespace, Name.Function)), + (r'(def)(\s+)([a-z_][a-zA-Z0-9_\']*=?|<<|>>|==|<=>|<=|<|>=|>|\+|-(self)?|~(self)?|\*|/|%|^|&&|&|\||\[\]=?)', + bygroups(Keyword, Whitespace, Name.Function)), + (r'def'+_nkw, Keyword), + (r'if'+_nkw, Keyword), + (r'elsif'+_nkw, Keyword), + (r'else'+_nkw, Keyword), + (r'unless'+_nkw, Keyword), + (r'for'+_nkw, Keyword), + (r'in'+_nkw, Keyword), + (r'while'+_nkw, Keyword), + (r'until'+_nkw, Keyword), + (r'and'+_nkw, Keyword), + (r'or'+_nkw, Keyword), + (r'not'+_nkw, Keyword), + (r'lambda'+_nkw, Keyword), + (r'try'+_nkw, Keyword), + (r'catch'+_nkw, Keyword), + (r'return'+_nkw, Keyword), + (r'next'+_nkw, Keyword), + (r'last'+_nkw, Keyword), + (r'throw'+_nkw, Keyword), + (r'use'+_nkw, Keyword), + (r'switch'+_nkw, Keyword), + (r'\\', Keyword), + (r'λ', Keyword), + (r'__FILE__'+_nkw, Name.Builtin.Pseudo), + (r'__LINE__'+_nkw, Name.Builtin.Pseudo), + (r'[A-Z][a-zA-Z0-9_\']*'+_nkw, Name.Constant), + (r'[a-z_][a-zA-Z0-9_\']*'+_nkw, Name), + (r'@[a-z_][a-zA-Z0-9_\']*'+_nkw, Name.Variable.Instance), + (r'@@[a-z_][a-zA-Z0-9_\']*'+_nkw, Name.Variable.Class), + (r'\(', Punctuation), + (r'\)', Punctuation), + (r'\[', Punctuation), + (r'\]', Punctuation), + (r'\{', Punctuation), + (r'\}', right_angle_bracket), + (r';', Punctuation), + (r',', Punctuation), + (r'<<=', Operator), + (r'>>=', Operator), + (r'<<', Operator), + (r'>>', Operator), + (r'==', Operator), + (r'!=', Operator), + (r'=>', Operator), + (r'=', Operator), + (r'<=>', Operator), + (r'<=', Operator), + (r'>=', Operator), + (r'<', Operator), + (r'>', Operator), + (r'\+\+', Operator), + (r'\+=', Operator), + (r'-=', Operator), + (r'\*\*=', Operator), + (r'\*=', Operator), + (r'\*\*', Operator), + (r'\*', Operator), + (r'/=', Operator), + (r'\+', Operator), + (r'-', Operator), + (r'/', Operator), + (r'%=', Operator), + (r'%', Operator), + (r'^=', Operator), + (r'&&=', Operator), + (r'&=', Operator), + (r'&&', Operator), + (r'&', Operator), + (r'\|\|=', Operator), + (r'\|=', Operator), + (r'\|\|', Operator), + (r'\|', Operator), + (r'!', Operator), + (r'\.\.\.', Operator), + (r'\.\.', Operator), + (r'\.', Operator), + (r'::', Operator), + (r':', Operator), + (r'(\s|\n)+', Whitespace), + (r'[a-z_][a-zA-Z0-9_\']*', Name.Variable), + ], + } + + +class SlashLexer(DelegatingLexer): + """ + Lexer for the Slash programming language. + + .. versionadded:: 2.4 + """ + + name = 'Slash' + aliases = ['slash'] + filenames = ['*.sla'] + + def __init__(self, **options): + from pygments.lexers.web import HtmlLexer + super().__init__(HtmlLexer, SlashLanguageLexer, **options) diff --git a/src/syntax_highlighting/libs/pygments/lexers/smalltalk.py b/src/syntax_highlighting/libs/pygments/lexers/smalltalk.py index 79078b6..ebb3311 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/smalltalk.py +++ b/src/syntax_highlighting/libs/pygments/lexers/smalltalk.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.smalltalk ~~~~~~~~~~~~~~~~~~~~~~~~~ Lexers for Smalltalk and related languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -139,7 +138,7 @@ class SmalltalkLexer(RegexLexer): class NewspeakLexer(RegexLexer): """ - For `Newspeak ` syntax. + For `Newspeak `_ syntax. .. versionadded:: 1.1 """ diff --git a/src/syntax_highlighting/libs/pygments/lexers/smithy.py b/src/syntax_highlighting/libs/pygments/lexers/smithy.py new file mode 100644 index 0000000..0f0a912 --- /dev/null +++ b/src/syntax_highlighting/libs/pygments/lexers/smithy.py @@ -0,0 +1,79 @@ +""" + pygments.lexers.smithy + ~~~~~~~~~~~~~~~~~~~~~~ + + Lexers for the Smithy IDL. + + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re + +from pygments.lexer import RegexLexer, bygroups, words +from pygments.token import Text, Comment, Keyword, Name, String, \ + Number, Whitespace, Punctuation + +__all__ = ['SmithyLexer'] + + +class SmithyLexer(RegexLexer): + """ + For Smithy IDL + + .. versionadded:: 2.10 + """ + name = 'Smithy' + filenames = ['*.smithy'] + aliases = ['smithy'] + + flags = re.MULTILINE | re.UNICODE + unquoted = r'[A-Za-z0-9_\.#$-]+' + identifier = r"[A-Za-z0-9_\.#$-]+" + + simple_shapes = ( + 'use', 'byte', 'short', 'integer', 'long', 'float', 'document', + 'double', 'bigInteger', 'bigDecimal', 'boolean', 'blob', 'string', + 'timestamp', + ) + + aggregate_shapes = ( + 'apply', 'list', 'map', 'set', 'structure', 'union', 'resource', + 'operation', 'service', 'trait' + ) + + tokens = { + 'root': [ + (r'///.*$', Comment.Multiline), + (r'//.*$', Comment), + (r'@[0-9a-zA-Z\.#-]*', Name.Decorator), + (r'(=)', Name.Decorator), + (r'^(\$version)(:)(.+)', + bygroups(Keyword.Declaration, Name.Decorator, Name.Class)), + (r'^(namespace)(\s+' + identifier + r')\b', + bygroups(Keyword.Declaration, Name.Class)), + (words(simple_shapes, + prefix=r'^', suffix=r'(\s+' + identifier + r')\b'), + bygroups(Keyword.Declaration, Name.Class)), + (words(aggregate_shapes, + prefix=r'^', suffix=r'(\s+' + identifier + r')'), + bygroups(Keyword.Declaration, Name.Class)), + (r'^(metadata)(\s+.+)(\s*)(=)', + bygroups(Keyword.Declaration, Name.Class, Whitespace, Name.Decorator)), + (r"(true|false|null)", Keyword.Constant), + (r"(-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?)", Number), + (identifier + ":", Name.Label), + (identifier, Name.Variable.Class), + (r'\[', Text, "#push"), + (r'\]', Text, "#pop"), + (r'\(', Text, "#push"), + (r'\)', Text, "#pop"), + (r'\{', Text, "#push"), + (r'\}', Text, "#pop"), + (r'"{3}(\\\\|\n|\\")*"{3}', String.Doc), + (r'"(\\\\|\n|\\"|[^"])*"', String.Double), + (r"'(\\\\|\n|\\'|[^'])*'", String.Single), + (r'[:,]+', Punctuation), + (r'\s+', Whitespace), + ] + } diff --git a/src/syntax_highlighting/libs/pygments/lexers/smv.py b/src/syntax_highlighting/libs/pygments/lexers/smv.py index 380a3b7..a4cbf94 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/smv.py +++ b/src/syntax_highlighting/libs/pygments/lexers/smv.py @@ -1,17 +1,16 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.smv ~~~~~~~~~~~~~~~~~~~ Lexers for the SMV languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ from pygments.lexer import RegexLexer, words -from pygments.token import Comment, Generic, Keyword, Name, Number, \ - Operator, Punctuation, Text +from pygments.token import Comment, Keyword, Name, Number, Operator, \ + Punctuation, Text __all__ = ['NuSMVLexer'] @@ -69,7 +68,7 @@ class NuSMVLexer(RegexLexer): (r'\-?\d+\b', Number.Integer), (r'0[su][bB]\d*_[01_]+', Number.Bin), (r'0[su][oO]\d*_[0-7_]+', Number.Oct), - (r'0[su][dD]\d*_[\d_]+', Number.Dec), + (r'0[su][dD]\d*_[\d_]+', Number.Decimal), (r'0[su][hH]\d*_[\da-fA-F_]+', Number.Hex), # Whitespace, punctuation and the rest diff --git a/src/syntax_highlighting/libs/pygments/lexers/snobol.py b/src/syntax_highlighting/libs/pygments/lexers/snobol.py index f6e12fd..b5719c3 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/snobol.py +++ b/src/syntax_highlighting/libs/pygments/lexers/snobol.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.snobol ~~~~~~~~~~~~~~~~~~~~~~ Lexers for the SNOBOL language. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/src/syntax_highlighting/libs/pygments/lexers/solidity.py b/src/syntax_highlighting/libs/pygments/lexers/solidity.py new file mode 100644 index 0000000..0c42586 --- /dev/null +++ b/src/syntax_highlighting/libs/pygments/lexers/solidity.py @@ -0,0 +1,91 @@ +""" + pygments.lexers.solidity + ~~~~~~~~~~~~~~~~~~~~~~~~ + + Lexers for Solidity. + + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re + +from pygments.lexer import RegexLexer, bygroups, include, words +from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ + Number, Punctuation, Whitespace + +__all__ = ['SolidityLexer'] + + +class SolidityLexer(RegexLexer): + """ + For Solidity source code. + + .. versionadded:: 2.5 + """ + + name = 'Solidity' + aliases = ['solidity'] + filenames = ['*.sol'] + mimetypes = [] + + flags = re.MULTILINE | re.UNICODE + + datatype = ( + r'\b(address|bool|(?:(?:bytes|hash|int|string|uint)(?:8|16|24|32|40|48|56|64' + r'|72|80|88|96|104|112|120|128|136|144|152|160|168|176|184|192|200|208' + r'|216|224|232|240|248|256)?))\b' + ) + + tokens = { + 'root': [ + include('whitespace'), + include('comments'), + (r'\bpragma\s+solidity\b', Keyword, 'pragma'), + (r'\b(contract)(\s+)([a-zA-Z_]\w*)', + bygroups(Keyword, Whitespace, Name.Entity)), + (datatype + r'(\s+)((?:external|public|internal|private)\s+)?' + + r'([a-zA-Z_]\w*)', + bygroups(Keyword.Type, Whitespace, Keyword, Name.Variable)), + (r'\b(enum|event|function|struct)(\s+)([a-zA-Z_]\w*)', + bygroups(Keyword.Type, Whitespace, Name.Variable)), + (r'\b(msg|block|tx)\.([A-Za-z_][a-zA-Z0-9_]*)\b', Keyword), + (words(( + 'block', 'break', 'constant', 'constructor', 'continue', + 'contract', 'do', 'else', 'external', 'false', 'for', + 'function', 'if', 'import', 'inherited', 'internal', 'is', + 'library', 'mapping', 'memory', 'modifier', 'msg', 'new', + 'payable', 'private', 'public', 'require', 'return', + 'returns', 'struct', 'suicide', 'throw', 'this', 'true', + 'tx', 'var', 'while'), prefix=r'\b', suffix=r'\b'), + Keyword.Type), + (words(('keccak256',), prefix=r'\b', suffix=r'\b'), Name.Builtin), + (datatype, Keyword.Type), + include('constants'), + (r'[a-zA-Z_]\w*', Text), + (r'[!<=>+*/-]', Operator), + (r'[.;:{}(),\[\]]', Punctuation) + ], + 'comments': [ + (r'//(\n|[\w\W]*?[^\\]\n)', Comment.Single), + (r'/(\\\n)?[*][\w\W]*?[*](\\\n)?/', Comment.Multiline), + (r'/(\\\n)?[*][\w\W]*', Comment.Multiline) + ], + 'constants': [ + (r'("(\\"|.)*?")', String.Double), + (r"('(\\'|.)*?')", String.Single), + (r'\b0[xX][0-9a-fA-F]+\b', Number.Hex), + (r'\b\d+\b', Number.Decimal), + ], + 'pragma': [ + include('whitespace'), + include('comments'), + (r'(\^|>=|<)(\s*)(\d+\.\d+\.\d+)', + bygroups(Operator, Whitespace, Keyword)), + (r';', Punctuation, '#pop') + ], + 'whitespace': [ + (r'\s+', Whitespace), + (r'\n', Whitespace) + ] + } diff --git a/src/syntax_highlighting/libs/pygments/lexers/special.py b/src/syntax_highlighting/libs/pygments/lexers/special.py index 6e076b0..bff6652 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/special.py +++ b/src/syntax_highlighting/libs/pygments/lexers/special.py @@ -1,22 +1,22 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.special ~~~~~~~~~~~~~~~~~~~~~~~ Special lexers. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ +import ast import re from pygments.lexer import Lexer -from pygments.token import Token, Error, Text -from pygments.util import get_choice_opt, text_type, BytesIO +from pygments.token import Token, Error, Text, Generic +from pygments.util import get_choice_opt -__all__ = ['TextLexer', 'RawTokenLexer'] +__all__ = ['TextLexer', 'OutputLexer', 'RawTokenLexer'] class TextLexer(Lexer): @@ -35,16 +35,28 @@ def get_tokens_unprocessed(self, text): def analyse_text(text): return TextLexer.priority + +class OutputLexer(Lexer): + """ + Simple lexer that highlights everything as ``Token.Generic.Output``. + + .. versionadded:: 2.10 + """ + name = 'Text output' + aliases = ['output'] + + def get_tokens_unprocessed(self, text): + yield 0, Generic.Output, text + + _ttype_cache = {} -line_re = re.compile(b'.*?\n') +line_re = re.compile('.*?\n') class RawTokenLexer(Lexer): """ - Recreate a token stream formatted with the `RawTokenFormatter`. This - lexer raises exceptions during parsing if the token stream in the - file is malformed. + Recreate a token stream formatted with the `RawTokenFormatter`. Additional options accepted: @@ -53,7 +65,7 @@ class RawTokenLexer(Lexer): the given compression algorithm before lexing (default: ``""``). """ name = 'Raw token data' - aliases = ['raw'] + aliases = [] filenames = [] mimetypes = ['application/x-pygments-tokens'] @@ -63,20 +75,23 @@ def __init__(self, **options): Lexer.__init__(self, **options) def get_tokens(self, text): - if isinstance(text, text_type): - # raw token stream never has any non-ASCII characters - text = text.encode('ascii') - if self.compress == 'gz': - import gzip - gzipfile = gzip.GzipFile('', 'rb', 9, BytesIO(text)) - text = gzipfile.read() - elif self.compress == 'bz2': - import bz2 - text = bz2.decompress(text) - - # do not call Lexer.get_tokens() because we do not want Unicode - # decoding to occur, and stripping is not optional. - text = text.strip(b'\n') + b'\n' + if self.compress: + if isinstance(text, str): + text = text.encode('latin1') + try: + if self.compress == 'gz': + import gzip + text = gzip.decompress(text) + elif self.compress == 'bz2': + import bz2 + text = bz2.decompress(text) + except OSError: + yield Error, text.decode('latin1') + if isinstance(text, bytes): + text = text.decode('latin1') + + # do not call Lexer.get_tokens() because stripping is not optional. + text = text.strip('\n') + '\n' for i, t, v in self.get_tokens_unprocessed(text): yield t, v @@ -84,11 +99,7 @@ def get_tokens_unprocessed(self, text): length = 0 for match in line_re.finditer(text): try: - ttypestr, val = match.group().split(b'\t', 1) - except ValueError: - val = match.group().decode('ascii', 'replace') - ttype = Error - else: + ttypestr, val = match.group().rstrip().split('\t', 1) ttype = _ttype_cache.get(ttypestr) if not ttype: ttype = Token @@ -98,6 +109,11 @@ def get_tokens_unprocessed(self, text): raise ValueError('malformed token name') ttype = getattr(ttype, ttype_) _ttype_cache[ttypestr] = ttype - val = val[2:-2].decode('unicode-escape') + val = ast.literal_eval(val) + if not isinstance(val, str): + raise ValueError('expected str') + except (SyntaxError, ValueError): + val = match.group() + ttype = Error yield length, ttype, val length += len(val) diff --git a/src/syntax_highlighting/libs/pygments/lexers/sql.py b/src/syntax_highlighting/libs/pygments/lexers/sql.py index 7507c0f..0611ab7 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/sql.py +++ b/src/syntax_highlighting/libs/pygments/lexers/sql.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.sql ~~~~~~~~~~~~~~~~~~~ @@ -34,20 +33,26 @@ The ``tests/examplefiles`` contains a few test files with data to be parsed by these lexers. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ import re from pygments.lexer import Lexer, RegexLexer, do_insertions, bygroups, words -from pygments.token import Punctuation, Whitespace, Error, \ - Text, Comment, Operator, Keyword, Name, String, Number, Generic +from pygments.token import Punctuation, Whitespace, Text, Comment, Operator, \ + Keyword, Name, String, Number, Generic, Literal from pygments.lexers import get_lexer_by_name, ClassNotFound -from pygments.util import iteritems from pygments.lexers._postgres_builtins import KEYWORDS, DATATYPES, \ PSEUDO_TYPES, PLPGSQL_KEYWORDS +from pygments.lexers._mysql_builtins import \ + MYSQL_CONSTANTS, \ + MYSQL_DATATYPES, \ + MYSQL_FUNCTIONS, \ + MYSQL_KEYWORDS, \ + MYSQL_OPTIMIZER_HINTS + from pygments.lexers import _tsql_builtins @@ -59,7 +64,14 @@ language_re = re.compile(r"\s+LANGUAGE\s+'?(\w+)'?", re.IGNORECASE) -do_re = re.compile(r'\bDO\b', re.IGNORECASE) +do_re = re.compile(r'\bDO\b', re.IGNORECASE) + +# Regular expressions for analyse_text() +name_between_bracket_re = re.compile(r'\[[a-zA-Z_]\w*\]') +name_between_backtick_re = re.compile(r'`[a-zA-Z_]\w*`') +tsql_go_re = re.compile(r'\bgo\b', re.IGNORECASE) +tsql_declare_re = re.compile(r'\bdeclare\s+@', re.IGNORECASE) +tsql_variable_re = re.compile(r'@[a-zA-Z_]\w*\b') def language_callback(lexer, match): @@ -68,29 +80,28 @@ def language_callback(lexer, match): The lexer is chosen looking for a nearby LANGUAGE or assumed as plpgsql if inside a DO statement and no LANGUAGE has been found. """ - l = None + lx = None m = language_re.match(lexer.text[match.end():match.end()+100]) if m is not None: - l = lexer._get_lexer(m.group(1)) + lx = lexer._get_lexer(m.group(1)) else: m = list(language_re.finditer( lexer.text[max(0, match.start()-100):match.start()])) if m: - l = lexer._get_lexer(m[-1].group(1)) + lx = lexer._get_lexer(m[-1].group(1)) else: m = list(do_re.finditer( lexer.text[max(0, match.start()-25):match.start()])) if m: - l = lexer._get_lexer('plpgsql') - + lx = lexer._get_lexer('plpgsql') + # 1 = $, 2 = delimiter, 3 = $ yield (match.start(1), String, match.group(1)) yield (match.start(2), String.Delimiter, match.group(2)) yield (match.start(3), String, match.group(3)) # 4 = string contents - if l: - for x in l.get_tokens_unprocessed(match.group(4)): - yield x + if lx: + yield from lx.get_tokens_unprocessed(match.group(4)) else: yield (match.start(4), String, match.group(4)) # 5 = $, 6 = delimiter, 7 = $ @@ -99,7 +110,7 @@ def language_callback(lexer, match): yield (match.start(7), String, match.group(7)) -class PostgresBase(object): +class PostgresBase: """Base class for Postgres-related lexers. This is implemented as a mixin to avoid the Lexer metaclass kicking in. @@ -111,9 +122,7 @@ class PostgresBase(object): def get_tokens_unprocessed(self, text, *args): # Have a copy of the entire text to be used by `language_callback`. self.text = text - for x in super(PostgresBase, self).get_tokens_unprocessed( - text, *args): - yield x + yield from super().get_tokens_unprocessed(text, *args) def _get_lexer(self, lang): if lang.lower() == 'sql': @@ -127,9 +136,9 @@ def _get_lexer(self, lang): if lang.startswith('pl') and lang.endswith('u'): tries.append(lang[2:-1]) - for l in tries: + for lx in tries: try: - return get_lexer_by_name(l, **self.options) + return get_lexer_by_name(lx, **self.options) except ClassNotFound: pass else: @@ -155,9 +164,9 @@ class PostgresLexer(PostgresBase, RegexLexer): (r'\s+', Text), (r'--.*\n?', Comment.Single), (r'/\*', Comment.Multiline, 'multiline-comments'), - (r'(' + '|'.join(s.replace(" ", "\s+") - for s in DATATYPES + PSEUDO_TYPES) - + r')\b', Name.Builtin), + (r'(' + '|'.join(s.replace(" ", r"\s+") + for s in DATATYPES + PSEUDO_TYPES) + r')\b', + Name.Builtin), (words(KEYWORDS, suffix=r'\b'), Keyword), (r'[+*/<>=~!@#%^&|`?-]+', Operator), (r'::', Operator), # cast @@ -205,7 +214,7 @@ class PlPgsqlLexer(PostgresBase, RegexLexer): mimetypes = ['text/x-plpgsql'] flags = re.IGNORECASE - tokens = dict((k, l[:]) for (k, l) in iteritems(PostgresLexer.tokens)) + tokens = {k: l[:] for (k, l) in PostgresLexer.tokens.items()} # extend the keywords list for i, pattern in enumerate(tokens['root']): @@ -239,7 +248,7 @@ class PsqlRegexLexer(PostgresBase, RegexLexer): aliases = [] # not public flags = re.IGNORECASE - tokens = dict((k, l[:]) for (k, l) in iteritems(PostgresLexer.tokens)) + tokens = {k: l[:] for (k, l) in PostgresLexer.tokens.items()} tokens['root'].append( (r'\\[^\s]+', Keyword.Pseudo, 'psql-command')) @@ -253,6 +262,7 @@ class PsqlRegexLexer(PostgresBase, RegexLexer): (r"[^\s]+", String.Symbol), ] + re_prompt = re.compile(r'^(\S.*?)??[=\-\(\$\'\"][#>]') re_psql_command = re.compile(r'\s*\\') re_end_command = re.compile(r';\s*(--.*?)?$') @@ -263,7 +273,7 @@ class PsqlRegexLexer(PostgresBase, RegexLexer): r'FATAL|HINT|DETAIL|CONTEXT|LINE [0-9]+):)(.*?\n)') -class lookahead(object): +class lookahead: """Wrap an iterator and allow pushing back an item.""" def __init__(self, x): self.iter = iter(x) @@ -308,19 +318,11 @@ def get_tokens_unprocessed(self, data): # and continue until the end of command is detected curcode = '' insertions = [] - while 1: - try: - line = next(lines) - except StopIteration: - # allow the emission of partially collected items - # the repl loop will be broken below - break - + for line in lines: # Identify a shell prompt in case of psql commandline example if line.startswith('$') and not curcode: lexer = get_lexer_by_name('console', **self.options) - for x in lexer.get_tokens_unprocessed(line): - yield x + yield from lexer.get_tokens_unprocessed(line) break # Identify a psql prompt @@ -340,14 +342,12 @@ def get_tokens_unprocessed(self, data): break # Emit the combined stream of command and prompt(s) - for item in do_insertions(insertions, - sql.get_tokens_unprocessed(curcode)): - yield item + yield from do_insertions(insertions, + sql.get_tokens_unprocessed(curcode)) # Emit the output lines out_token = Generic.Output - while 1: - line = next(lines) + for line in lines: mprompt = re_prompt.match(line) if mprompt is not None: # push the line back to have it processed by the prompt @@ -363,6 +363,8 @@ def get_tokens_unprocessed(self, data): yield (mmsg.start(2), out_token, mmsg.group(2)) else: yield (0, out_token, line) + else: + return class SqlLexer(RegexLexer): @@ -383,86 +385,99 @@ class SqlLexer(RegexLexer): (r'--.*\n?', Comment.Single), (r'/\*', Comment.Multiline, 'multiline-comments'), (words(( - 'ABORT', 'ABS', 'ABSOLUTE', 'ACCESS', 'ADA', 'ADD', 'ADMIN', 'AFTER', 'AGGREGATE', - 'ALIAS', 'ALL', 'ALLOCATE', 'ALTER', 'ANALYSE', 'ANALYZE', 'AND', 'ANY', 'ARE', 'AS', - 'ASC', 'ASENSITIVE', 'ASSERTION', 'ASSIGNMENT', 'ASYMMETRIC', 'AT', 'ATOMIC', - 'AUTHORIZATION', 'AVG', 'BACKWARD', 'BEFORE', 'BEGIN', 'BETWEEN', 'BITVAR', - 'BIT_LENGTH', 'BOTH', 'BREADTH', 'BY', 'C', 'CACHE', 'CALL', 'CALLED', 'CARDINALITY', - 'CASCADE', 'CASCADED', 'CASE', 'CAST', 'CATALOG', 'CATALOG_NAME', 'CHAIN', + 'ABORT', 'ABS', 'ABSOLUTE', 'ACCESS', 'ADA', 'ADD', 'ADMIN', 'AFTER', + 'AGGREGATE', 'ALIAS', 'ALL', 'ALLOCATE', 'ALTER', 'ANALYSE', 'ANALYZE', + 'AND', 'ANY', 'ARE', 'AS', 'ASC', 'ASENSITIVE', 'ASSERTION', 'ASSIGNMENT', + 'ASYMMETRIC', 'AT', 'ATOMIC', 'AUTHORIZATION', 'AVG', 'BACKWARD', + 'BEFORE', 'BEGIN', 'BETWEEN', 'BITVAR', 'BIT_LENGTH', 'BOTH', 'BREADTH', + 'BY', 'C', 'CACHE', 'CALL', 'CALLED', 'CARDINALITY', 'CASCADE', + 'CASCADED', 'CASE', 'CAST', 'CATALOG', 'CATALOG_NAME', 'CHAIN', 'CHARACTERISTICS', 'CHARACTER_LENGTH', 'CHARACTER_SET_CATALOG', 'CHARACTER_SET_NAME', 'CHARACTER_SET_SCHEMA', 'CHAR_LENGTH', 'CHECK', - 'CHECKED', 'CHECKPOINT', 'CLASS', 'CLASS_ORIGIN', 'CLOB', 'CLOSE', 'CLUSTER', - 'COALSECE', 'COBOL', 'COLLATE', 'COLLATION', 'COLLATION_CATALOG', - 'COLLATION_NAME', 'COLLATION_SCHEMA', 'COLUMN', 'COLUMN_NAME', - 'COMMAND_FUNCTION', 'COMMAND_FUNCTION_CODE', 'COMMENT', 'COMMIT', - 'COMMITTED', 'COMPLETION', 'CONDITION_NUMBER', 'CONNECT', 'CONNECTION', - 'CONNECTION_NAME', 'CONSTRAINT', 'CONSTRAINTS', 'CONSTRAINT_CATALOG', - 'CONSTRAINT_NAME', 'CONSTRAINT_SCHEMA', 'CONSTRUCTOR', 'CONTAINS', - 'CONTINUE', 'CONVERSION', 'CONVERT', 'COPY', 'CORRESPONTING', 'COUNT', - 'CREATE', 'CREATEDB', 'CREATEUSER', 'CROSS', 'CUBE', 'CURRENT', 'CURRENT_DATE', - 'CURRENT_PATH', 'CURRENT_ROLE', 'CURRENT_TIME', 'CURRENT_TIMESTAMP', - 'CURRENT_USER', 'CURSOR', 'CURSOR_NAME', 'CYCLE', 'DATA', 'DATABASE', + 'CHECKED', 'CHECKPOINT', 'CLASS', 'CLASS_ORIGIN', 'CLOB', 'CLOSE', + 'CLUSTER', 'COALESCE', 'COBOL', 'COLLATE', 'COLLATION', + 'COLLATION_CATALOG', 'COLLATION_NAME', 'COLLATION_SCHEMA', 'COLUMN', + 'COLUMN_NAME', 'COMMAND_FUNCTION', 'COMMAND_FUNCTION_CODE', 'COMMENT', + 'COMMIT', 'COMMITTED', 'COMPLETION', 'CONDITION_NUMBER', 'CONNECT', + 'CONNECTION', 'CONNECTION_NAME', 'CONSTRAINT', 'CONSTRAINTS', + 'CONSTRAINT_CATALOG', 'CONSTRAINT_NAME', 'CONSTRAINT_SCHEMA', + 'CONSTRUCTOR', 'CONTAINS', 'CONTINUE', 'CONVERSION', 'CONVERT', + 'COPY', 'CORRESPONDING', 'COUNT', 'CREATE', 'CREATEDB', 'CREATEUSER', + 'CROSS', 'CUBE', 'CURRENT', 'CURRENT_DATE', 'CURRENT_PATH', + 'CURRENT_ROLE', 'CURRENT_TIME', 'CURRENT_TIMESTAMP', 'CURRENT_USER', + 'CURSOR', 'CURSOR_NAME', 'CYCLE', 'DATA', 'DATABASE', 'DATETIME_INTERVAL_CODE', 'DATETIME_INTERVAL_PRECISION', 'DAY', - 'DEALLOCATE', 'DECLARE', 'DEFAULT', 'DEFAULTS', 'DEFERRABLE', 'DEFERRED', - 'DEFINED', 'DEFINER', 'DELETE', 'DELIMITER', 'DELIMITERS', 'DEREF', 'DESC', - 'DESCRIBE', 'DESCRIPTOR', 'DESTROY', 'DESTRUCTOR', 'DETERMINISTIC', - 'DIAGNOSTICS', 'DICTIONARY', 'DISCONNECT', 'DISPATCH', 'DISTINCT', 'DO', - 'DOMAIN', 'DROP', 'DYNAMIC', 'DYNAMIC_FUNCTION', 'DYNAMIC_FUNCTION_CODE', 'EACH', - 'ELSE', 'ELSIF', 'ENCODING', 'ENCRYPTED', 'END', 'END-EXEC', 'EQUALS', 'ESCAPE', 'EVERY', - 'EXCEPTION', 'EXCEPT', 'EXCLUDING', 'EXCLUSIVE', 'EXEC', 'EXECUTE', 'EXISTING', - 'EXISTS', 'EXPLAIN', 'EXTERNAL', 'EXTRACT', 'FALSE', 'FETCH', 'FINAL', 'FIRST', 'FOR', - 'FORCE', 'FOREIGN', 'FORTRAN', 'FORWARD', 'FOUND', 'FREE', 'FREEZE', 'FROM', 'FULL', - 'FUNCTION', 'G', 'GENERAL', 'GENERATED', 'GET', 'GLOBAL', 'GO', 'GOTO', 'GRANT', 'GRANTED', - 'GROUP', 'GROUPING', 'HANDLER', 'HAVING', 'HIERARCHY', 'HOLD', 'HOST', 'IDENTITY', 'IF', - 'IGNORE', 'ILIKE', 'IMMEDIATE', 'IMMUTABLE', 'IMPLEMENTATION', 'IMPLICIT', 'IN', - 'INCLUDING', 'INCREMENT', 'INDEX', 'INDITCATOR', 'INFIX', 'INHERITS', 'INITIALIZE', - 'INITIALLY', 'INNER', 'INOUT', 'INPUT', 'INSENSITIVE', 'INSERT', 'INSTANTIABLE', - 'INSTEAD', 'INTERSECT', 'INTO', 'INVOKER', 'IS', 'ISNULL', 'ISOLATION', 'ITERATE', 'JOIN', - 'KEY', 'KEY_MEMBER', 'KEY_TYPE', 'LANCOMPILER', 'LANGUAGE', 'LARGE', 'LAST', - 'LATERAL', 'LEADING', 'LEFT', 'LENGTH', 'LESS', 'LEVEL', 'LIKE', 'LIMIT', 'LISTEN', 'LOAD', - 'LOCAL', 'LOCALTIME', 'LOCALTIMESTAMP', 'LOCATION', 'LOCATOR', 'LOCK', 'LOWER', - 'MAP', 'MATCH', 'MAX', 'MAXVALUE', 'MESSAGE_LENGTH', 'MESSAGE_OCTET_LENGTH', - 'MESSAGE_TEXT', 'METHOD', 'MIN', 'MINUTE', 'MINVALUE', 'MOD', 'MODE', 'MODIFIES', - 'MODIFY', 'MONTH', 'MORE', 'MOVE', 'MUMPS', 'NAMES', 'NATIONAL', 'NATURAL', 'NCHAR', - 'NCLOB', 'NEW', 'NEXT', 'NO', 'NOCREATEDB', 'NOCREATEUSER', 'NONE', 'NOT', 'NOTHING', - 'NOTIFY', 'NOTNULL', 'NULL', 'NULLABLE', 'NULLIF', 'OBJECT', 'OCTET_LENGTH', 'OF', 'OFF', - 'OFFSET', 'OIDS', 'OLD', 'ON', 'ONLY', 'OPEN', 'OPERATION', 'OPERATOR', 'OPTION', 'OPTIONS', - 'OR', 'ORDER', 'ORDINALITY', 'OUT', 'OUTER', 'OUTPUT', 'OVERLAPS', 'OVERLAY', 'OVERRIDING', - 'OWNER', 'PAD', 'PARAMETER', 'PARAMETERS', 'PARAMETER_MODE', 'PARAMATER_NAME', - 'PARAMATER_ORDINAL_POSITION', 'PARAMETER_SPECIFIC_CATALOG', - 'PARAMETER_SPECIFIC_NAME', 'PARAMATER_SPECIFIC_SCHEMA', 'PARTIAL', - 'PASCAL', 'PENDANT', 'PLACING', 'PLI', 'POSITION', 'POSTFIX', 'PRECISION', 'PREFIX', - 'PREORDER', 'PREPARE', 'PRESERVE', 'PRIMARY', 'PRIOR', 'PRIVILEGES', 'PROCEDURAL', - 'PROCEDURE', 'PUBLIC', 'READ', 'READS', 'RECHECK', 'RECURSIVE', 'REF', 'REFERENCES', - 'REFERENCING', 'REINDEX', 'RELATIVE', 'RENAME', 'REPEATABLE', 'REPLACE', 'RESET', - 'RESTART', 'RESTRICT', 'RESULT', 'RETURN', 'RETURNED_LENGTH', - 'RETURNED_OCTET_LENGTH', 'RETURNED_SQLSTATE', 'RETURNS', 'REVOKE', 'RIGHT', - 'ROLE', 'ROLLBACK', 'ROLLUP', 'ROUTINE', 'ROUTINE_CATALOG', 'ROUTINE_NAME', - 'ROUTINE_SCHEMA', 'ROW', 'ROWS', 'ROW_COUNT', 'RULE', 'SAVE_POINT', 'SCALE', 'SCHEMA', - 'SCHEMA_NAME', 'SCOPE', 'SCROLL', 'SEARCH', 'SECOND', 'SECURITY', 'SELECT', 'SELF', - 'SENSITIVE', 'SERIALIZABLE', 'SERVER_NAME', 'SESSION', 'SESSION_USER', 'SET', - 'SETOF', 'SETS', 'SHARE', 'SHOW', 'SIMILAR', 'SIMPLE', 'SIZE', 'SOME', 'SOURCE', 'SPACE', - 'SPECIFIC', 'SPECIFICTYPE', 'SPECIFIC_NAME', 'SQL', 'SQLCODE', 'SQLERROR', - 'SQLEXCEPTION', 'SQLSTATE', 'SQLWARNINIG', 'STABLE', 'START', 'STATE', 'STATEMENT', - 'STATIC', 'STATISTICS', 'STDIN', 'STDOUT', 'STORAGE', 'STRICT', 'STRUCTURE', 'STYPE', - 'SUBCLASS_ORIGIN', 'SUBLIST', 'SUBSTRING', 'SUM', 'SYMMETRIC', 'SYSID', 'SYSTEM', - 'SYSTEM_USER', 'TABLE', 'TABLE_NAME', ' TEMP', 'TEMPLATE', 'TEMPORARY', 'TERMINATE', - 'THAN', 'THEN', 'TIMESTAMP', 'TIMEZONE_HOUR', 'TIMEZONE_MINUTE', 'TO', 'TOAST', - 'TRAILING', 'TRANSATION', 'TRANSACTIONS_COMMITTED', - 'TRANSACTIONS_ROLLED_BACK', 'TRANSATION_ACTIVE', 'TRANSFORM', - 'TRANSFORMS', 'TRANSLATE', 'TRANSLATION', 'TREAT', 'TRIGGER', 'TRIGGER_CATALOG', - 'TRIGGER_NAME', 'TRIGGER_SCHEMA', 'TRIM', 'TRUE', 'TRUNCATE', 'TRUSTED', 'TYPE', - 'UNCOMMITTED', 'UNDER', 'UNENCRYPTED', 'UNION', 'UNIQUE', 'UNKNOWN', 'UNLISTEN', - 'UNNAMED', 'UNNEST', 'UNTIL', 'UPDATE', 'UPPER', 'USAGE', 'USER', - 'USER_DEFINED_TYPE_CATALOG', 'USER_DEFINED_TYPE_NAME', - 'USER_DEFINED_TYPE_SCHEMA', 'USING', 'VACUUM', 'VALID', 'VALIDATOR', 'VALUES', - 'VARIABLE', 'VERBOSE', 'VERSION', 'VIEW', 'VOLATILE', 'WHEN', 'WHENEVER', 'WHERE', - 'WITH', 'WITHOUT', 'WORK', 'WRITE', 'YEAR', 'ZONE'), suffix=r'\b'), + 'DEALLOCATE', 'DECLARE', 'DEFAULT', 'DEFAULTS', 'DEFERRABLE', + 'DEFERRED', 'DEFINED', 'DEFINER', 'DELETE', 'DELIMITER', 'DELIMITERS', + 'DEREF', 'DESC', 'DESCRIBE', 'DESCRIPTOR', 'DESTROY', 'DESTRUCTOR', + 'DETERMINISTIC', 'DIAGNOSTICS', 'DICTIONARY', 'DISCONNECT', 'DISPATCH', + 'DISTINCT', 'DO', 'DOMAIN', 'DROP', 'DYNAMIC', 'DYNAMIC_FUNCTION', + 'DYNAMIC_FUNCTION_CODE', 'EACH', 'ELSE', 'ELSIF', 'ENCODING', + 'ENCRYPTED', 'END', 'END-EXEC', 'EQUALS', 'ESCAPE', 'EVERY', 'EXCEPTION', + 'EXCEPT', 'EXCLUDING', 'EXCLUSIVE', 'EXEC', 'EXECUTE', 'EXISTING', + 'EXISTS', 'EXPLAIN', 'EXTERNAL', 'EXTRACT', 'FALSE', 'FETCH', 'FINAL', + 'FIRST', 'FOR', 'FORCE', 'FOREIGN', 'FORTRAN', 'FORWARD', 'FOUND', 'FREE', + 'FREEZE', 'FROM', 'FULL', 'FUNCTION', 'G', 'GENERAL', 'GENERATED', 'GET', + 'GLOBAL', 'GO', 'GOTO', 'GRANT', 'GRANTED', 'GROUP', 'GROUPING', + 'HANDLER', 'HAVING', 'HIERARCHY', 'HOLD', 'HOST', 'IDENTITY', 'IF', + 'IGNORE', 'ILIKE', 'IMMEDIATE', 'IMMEDIATELY', 'IMMUTABLE', 'IMPLEMENTATION', 'IMPLICIT', + 'IN', 'INCLUDING', 'INCREMENT', 'INDEX', 'INDITCATOR', 'INFIX', + 'INHERITS', 'INITIALIZE', 'INITIALLY', 'INNER', 'INOUT', 'INPUT', + 'INSENSITIVE', 'INSERT', 'INSTANTIABLE', 'INSTEAD', 'INTERSECT', 'INTO', + 'INVOKER', 'IS', 'ISNULL', 'ISOLATION', 'ITERATE', 'JOIN', 'KEY', + 'KEY_MEMBER', 'KEY_TYPE', 'LANCOMPILER', 'LANGUAGE', 'LARGE', 'LAST', + 'LATERAL', 'LEADING', 'LEFT', 'LENGTH', 'LESS', 'LEVEL', 'LIKE', 'LIMIT', + 'LISTEN', 'LOAD', 'LOCAL', 'LOCALTIME', 'LOCALTIMESTAMP', 'LOCATION', + 'LOCATOR', 'LOCK', 'LOWER', 'MAP', 'MATCH', 'MAX', 'MAXVALUE', + 'MESSAGE_LENGTH', 'MESSAGE_OCTET_LENGTH', 'MESSAGE_TEXT', 'METHOD', 'MIN', + 'MINUTE', 'MINVALUE', 'MOD', 'MODE', 'MODIFIES', 'MODIFY', 'MONTH', + 'MORE', 'MOVE', 'MUMPS', 'NAMES', 'NATIONAL', 'NATURAL', 'NCHAR', 'NCLOB', + 'NEW', 'NEXT', 'NO', 'NOCREATEDB', 'NOCREATEUSER', 'NONE', 'NOT', + 'NOTHING', 'NOTIFY', 'NOTNULL', 'NULL', 'NULLABLE', 'NULLIF', 'OBJECT', + 'OCTET_LENGTH', 'OF', 'OFF', 'OFFSET', 'OIDS', 'OLD', 'ON', 'ONLY', + 'OPEN', 'OPERATION', 'OPERATOR', 'OPTION', 'OPTIONS', 'OR', 'ORDER', + 'ORDINALITY', 'OUT', 'OUTER', 'OUTPUT', 'OVERLAPS', 'OVERLAY', + 'OVERRIDING', 'OWNER', 'PAD', 'PARAMETER', 'PARAMETERS', 'PARAMETER_MODE', + 'PARAMETER_NAME', 'PARAMETER_ORDINAL_POSITION', + 'PARAMETER_SPECIFIC_CATALOG', 'PARAMETER_SPECIFIC_NAME', + 'PARAMETER_SPECIFIC_SCHEMA', 'PARTIAL', 'PASCAL', 'PENDANT', 'PERIOD', 'PLACING', + 'PLI', 'POSITION', 'POSTFIX', 'PRECEEDS', 'PRECISION', 'PREFIX', 'PREORDER', + 'PREPARE', 'PRESERVE', 'PRIMARY', 'PRIOR', 'PRIVILEGES', 'PROCEDURAL', + 'PROCEDURE', 'PUBLIC', 'READ', 'READS', 'RECHECK', 'RECURSIVE', 'REF', + 'REFERENCES', 'REFERENCING', 'REINDEX', 'RELATIVE', 'RENAME', + 'REPEATABLE', 'REPLACE', 'RESET', 'RESTART', 'RESTRICT', 'RESULT', + 'RETURN', 'RETURNED_LENGTH', 'RETURNED_OCTET_LENGTH', 'RETURNED_SQLSTATE', + 'RETURNS', 'REVOKE', 'RIGHT', 'ROLE', 'ROLLBACK', 'ROLLUP', 'ROUTINE', + 'ROUTINE_CATALOG', 'ROUTINE_NAME', 'ROUTINE_SCHEMA', 'ROW', 'ROWS', + 'ROW_COUNT', 'RULE', 'SAVE_POINT', 'SCALE', 'SCHEMA', 'SCHEMA_NAME', + 'SCOPE', 'SCROLL', 'SEARCH', 'SECOND', 'SECURITY', 'SELECT', 'SELF', + 'SENSITIVE', 'SERIALIZABLE', 'SERVER_NAME', 'SESSION', 'SESSION_USER', + 'SET', 'SETOF', 'SETS', 'SHARE', 'SHOW', 'SIMILAR', 'SIMPLE', 'SIZE', + 'SOME', 'SOURCE', 'SPACE', 'SPECIFIC', 'SPECIFICTYPE', 'SPECIFIC_NAME', + 'SQL', 'SQLCODE', 'SQLERROR', 'SQLEXCEPTION', 'SQLSTATE', 'SQLWARNINIG', + 'STABLE', 'START', 'STATE', 'STATEMENT', 'STATIC', 'STATISTICS', 'STDIN', + 'STDOUT', 'STORAGE', 'STRICT', 'STRUCTURE', 'STYPE', 'SUBCLASS_ORIGIN', + 'SUBLIST', 'SUBSTRING', 'SUCCEEDS', 'SUM', 'SYMMETRIC', 'SYSID', 'SYSTEM', + 'SYSTEM_USER', 'TABLE', 'TABLE_NAME', ' TEMP', 'TEMPLATE', 'TEMPORARY', + 'TERMINATE', 'THAN', 'THEN', 'TIME', 'TIMESTAMP', 'TIMEZONE_HOUR', + 'TIMEZONE_MINUTE', 'TO', 'TOAST', 'TRAILING', 'TRANSACTION', + 'TRANSACTIONS_COMMITTED', 'TRANSACTIONS_ROLLED_BACK', 'TRANSACTION_ACTIVE', + 'TRANSFORM', 'TRANSFORMS', 'TRANSLATE', 'TRANSLATION', 'TREAT', 'TRIGGER', + 'TRIGGER_CATALOG', 'TRIGGER_NAME', 'TRIGGER_SCHEMA', 'TRIM', 'TRUE', + 'TRUNCATE', 'TRUSTED', 'TYPE', 'UNCOMMITTED', 'UNDER', 'UNENCRYPTED', + 'UNION', 'UNIQUE', 'UNKNOWN', 'UNLISTEN', 'UNNAMED', 'UNNEST', 'UNTIL', + 'UPDATE', 'UPPER', 'USAGE', 'USER', 'USER_DEFINED_TYPE_CATALOG', + 'USER_DEFINED_TYPE_NAME', 'USER_DEFINED_TYPE_SCHEMA', 'USING', 'VACUUM', + 'VALID', 'VALIDATOR', 'VALUES', 'VARIABLE', 'VERBOSE', + 'VERSION', 'VERSIONS', 'VERSIONING', 'VIEW', + 'VOLATILE', 'WHEN', 'WHENEVER', 'WHERE', 'WITH', 'WITHOUT', 'WORK', + 'WRITE', 'YEAR', 'ZONE'), suffix=r'\b'), Keyword), (words(( - 'ARRAY', 'BIGINT', 'BINARY', 'BIT', 'BLOB', 'BOOLEAN', 'CHAR', 'CHARACTER', 'DATE', - 'DEC', 'DECIMAL', 'FLOAT', 'INT', 'INTEGER', 'INTERVAL', 'NUMBER', 'NUMERIC', 'REAL', - 'SERIAL', 'SMALLINT', 'VARCHAR', 'VARYING', 'INT8', 'SERIAL8', 'TEXT'), suffix=r'\b'), + 'ARRAY', 'BIGINT', 'BINARY', 'BIT', 'BLOB', 'BOOLEAN', 'CHAR', + 'CHARACTER', 'DATE', 'DEC', 'DECIMAL', 'FLOAT', 'INT', 'INTEGER', + 'INTERVAL', 'NUMBER', 'NUMERIC', 'REAL', 'SERIAL', 'SMALLINT', + 'VARCHAR', 'VARYING', 'INT8', 'SERIAL8', 'TEXT'), suffix=r'\b'), Name.Builtin), (r'[+*/<>=~!@#%^&|`?-]', Operator), (r'[0-9]+', Number.Integer), @@ -499,7 +514,7 @@ class TransactSqlLexer(RegexLexer): tokens = { 'root': [ (r'\s+', Whitespace), - (r'--(?m).*?$\n?', Comment.Single), + (r'--.*?$\n?', Comment.Single), (r'/\*', Comment.Multiline, 'multiline-comments'), (words(_tsql_builtins.OPERATORS), Operator), (words(_tsql_builtins.OPERATOR_WORDS, suffix=r'\b'), Operator.Word), @@ -536,10 +551,42 @@ class TransactSqlLexer(RegexLexer): ] } + def analyse_text(text): + rating = 0 + if tsql_declare_re.search(text): + # Found T-SQL variable declaration. + rating = 1.0 + else: + name_between_backtick_count = len( + name_between_backtick_re.findall(text)) + name_between_bracket_count = len( + name_between_bracket_re.findall(text)) + # We need to check if there are any names using + # backticks or brackets, as otherwise both are 0 + # and 0 >= 2 * 0, so we would always assume it's true + dialect_name_count = name_between_backtick_count + name_between_bracket_count + if dialect_name_count >= 1 and \ + name_between_bracket_count >= 2 * name_between_backtick_count: + # Found at least twice as many [name] as `name`. + rating += 0.5 + elif name_between_bracket_count > name_between_backtick_count: + rating += 0.2 + elif name_between_bracket_count > 0: + rating += 0.1 + if tsql_variable_re.search(text) is not None: + rating += 0.1 + if tsql_go_re.search(text) is not None: + rating += 0.1 + return rating + class MySqlLexer(RegexLexer): - """ - Special lexer for MySQL. + """The Oracle MySQL lexer. + + This lexer does not attempt to maintain strict compatibility with + MariaDB syntax or keywords. Although MySQL and MariaDB's common code + history suggests there may be significant overlap between the two, + compatibility between the two is not a target for this lexer. """ name = 'MySQL' @@ -550,65 +597,171 @@ class MySqlLexer(RegexLexer): tokens = { 'root': [ (r'\s+', Text), - (r'(#|--\s+).*\n?', Comment.Single), - (r'/\*', Comment.Multiline, 'multiline-comments'), - (r'[0-9]+', Number.Integer), - (r'[0-9]*\.[0-9]+(e[+-][0-9]+)', Number.Float), - (r"'(\\\\|\\'|''|[^'])*'", String.Single), - (r'"(\\\\|\\"|""|[^"])*"', String.Double), - (r"`(\\\\|\\`|``|[^`])*`", String.Symbol), - (r'[+*/<>=~!@#%^&|`?-]', Operator), - (r'\b(tinyint|smallint|mediumint|int|integer|bigint|date|' - r'datetime|time|bit|bool|tinytext|mediumtext|longtext|text|' - r'tinyblob|mediumblob|longblob|blob|float|double|double\s+' - r'precision|real|numeric|dec|decimal|timestamp|year|char|' - r'varchar|varbinary|varcharacter|enum|set)(\b\s*)(\()?', - bygroups(Keyword.Type, Text, Punctuation)), - (r'\b(add|all|alter|analyze|and|as|asc|asensitive|before|between|' - r'bigint|binary|blob|both|by|call|cascade|case|change|char|' - r'character|check|collate|column|condition|constraint|continue|' - r'convert|create|cross|current_date|current_time|' - r'current_timestamp|current_user|cursor|database|databases|' - r'day_hour|day_microsecond|day_minute|day_second|dec|decimal|' - r'declare|default|delayed|delete|desc|describe|deterministic|' - r'distinct|distinctrow|div|double|drop|dual|each|else|elseif|' - r'enclosed|escaped|exists|exit|explain|fetch|flush|float|float4|' - r'float8|for|force|foreign|from|fulltext|grant|group|having|' - r'high_priority|hour_microsecond|hour_minute|hour_second|if|' - r'ignore|in|index|infile|inner|inout|insensitive|insert|int|' - r'int1|int2|int3|int4|int8|integer|interval|into|is|iterate|' - r'join|key|keys|kill|leading|leave|left|like|limit|lines|load|' - r'localtime|localtimestamp|lock|long|loop|low_priority|match|' - r'minute_microsecond|minute_second|mod|modifies|natural|' - r'no_write_to_binlog|not|numeric|on|optimize|option|optionally|' - r'or|order|out|outer|outfile|precision|primary|procedure|purge|' - r'raid0|read|reads|real|references|regexp|release|rename|repeat|' - r'replace|require|restrict|return|revoke|right|rlike|schema|' - r'schemas|second_microsecond|select|sensitive|separator|set|' - r'show|smallint|soname|spatial|specific|sql|sql_big_result|' - r'sql_calc_found_rows|sql_small_result|sqlexception|sqlstate|' - r'sqlwarning|ssl|starting|straight_join|table|terminated|then|' - r'to|trailing|trigger|undo|union|unique|unlock|unsigned|update|' - r'usage|use|using|utc_date|utc_time|utc_timestamp|values|' - r'varying|when|where|while|with|write|x509|xor|year_month|' - r'zerofill)\b', Keyword), - # TODO: this list is not complete - (r'\b(auto_increment|engine|charset|tables)\b', Keyword.Pseudo), - (r'(true|false|null)', Name.Constant), - (r'([a-z_]\w*)(\s*)(\()', + + # Comments + (r'(?:#|--\s+).*', Comment.Single), + (r'/\*\+', Comment.Special, 'optimizer-hints'), + (r'/\*', Comment.Multiline, 'multiline-comment'), + + # Hexadecimal literals + (r"x'([0-9a-f]{2})+'", Number.Hex), # MySQL requires paired hex characters in this form. + (r'0x[0-9a-f]+', Number.Hex), + + # Binary literals + (r"b'[01]+'", Number.Bin), + (r'0b[01]+', Number.Bin), + + # Numeric literals + (r'[0-9]+\.[0-9]*(e[+-]?[0-9]+)?', Number.Float), # Mandatory integer, optional fraction and exponent + (r'[0-9]*\.[0-9]+(e[+-]?[0-9]+)?', Number.Float), # Mandatory fraction, optional integer and exponent + (r'[0-9]+e[+-]?[0-9]+', Number.Float), # Exponents with integer significands are still floats + (r'[0-9]+(?=[^0-9a-z$_\u0080-\uffff])', Number.Integer), # Integers that are not in a schema object name + + # Date literals + (r"\{\s*d\s*(?P['\"])\s*\d{2}(\d{2})?.?\d{2}.?\d{2}\s*(?P=quote)\s*\}", + Literal.Date), + + # Time literals + (r"\{\s*t\s*(?P['\"])\s*(?:\d+\s+)?\d{1,2}.?\d{1,2}.?\d{1,2}(\.\d*)?\s*(?P=quote)\s*\}", + Literal.Date), + + # Timestamp literals + ( + r"\{\s*ts\s*(?P['\"])\s*" + r"\d{2}(?:\d{2})?.?\d{2}.?\d{2}" # Date part + r"\s+" # Whitespace between date and time + r"\d{1,2}.?\d{1,2}.?\d{1,2}(\.\d*)?" # Time part + r"\s*(?P=quote)\s*\}", + Literal.Date + ), + + # String literals + (r"'", String.Single, 'single-quoted-string'), + (r'"', String.Double, 'double-quoted-string'), + + # Variables + (r'@@(?:global\.|persist\.|persist_only\.|session\.)?[a-z_]+', Name.Variable), + (r'@[a-z0-9_$.]+', Name.Variable), + (r"@'", Name.Variable, 'single-quoted-variable'), + (r'@"', Name.Variable, 'double-quoted-variable'), + (r"@`", Name.Variable, 'backtick-quoted-variable'), + (r'\?', Name.Variable), # For demonstrating prepared statements + + # Operators + (r'[!%&*+/:<=>^|~-]+', Operator), + + # Exceptions; these words tokenize differently in different contexts. + (r'\b(set)(?!\s*\()', Keyword), + (r'\b(character)(\s+)(set)\b', bygroups(Keyword, Text, Keyword)), + # In all other known cases, "SET" is tokenized by MYSQL_DATATYPES. + + (words(MYSQL_CONSTANTS, prefix=r'\b', suffix=r'\b'), Name.Constant), + (words(MYSQL_DATATYPES, prefix=r'\b', suffix=r'\b'), Keyword.Type), + (words(MYSQL_KEYWORDS, prefix=r'\b', suffix=r'\b'), Keyword), + (words(MYSQL_FUNCTIONS, prefix=r'\b', suffix=r'\b(\s*)(\()'), bygroups(Name.Function, Text, Punctuation)), - (r'[a-z_]\w*', Name), - (r'@[a-z0-9]*[._]*[a-z0-9]*', Name.Variable), - (r'[;:()\[\],.]', Punctuation) + + # Schema object names + # + # Note: Although the first regex supports unquoted all-numeric + # identifiers, this will not be a problem in practice because + # numeric literals have already been handled above. + # + ('[0-9a-z$_\u0080-\uffff]+', Name), + (r'`', Name.Quoted, 'schema-object-name'), + + # Punctuation + (r'[(),.;]', Punctuation), ], - 'multiline-comments': [ - (r'/\*', Comment.Multiline, 'multiline-comments'), + + # Multiline comment substates + # --------------------------- + + 'optimizer-hints': [ + (r'[^*a-z]+', Comment.Special), + (r'\*/', Comment.Special, '#pop'), + (words(MYSQL_OPTIMIZER_HINTS, suffix=r'\b'), Comment.Preproc), + ('[a-z]+', Comment.Special), + (r'\*', Comment.Special), + ], + + 'multiline-comment': [ + (r'[^*]+', Comment.Multiline), (r'\*/', Comment.Multiline, '#pop'), - (r'[^/*]+', Comment.Multiline), - (r'[/*]', Comment.Multiline) - ] + (r'\*', Comment.Multiline), + ], + + # String substates + # ---------------- + + 'single-quoted-string': [ + (r"[^'\\]+", String.Single), + (r"''", String.Escape), + (r"""\\[0'"bnrtZ\\%_]""", String.Escape), + (r"'", String.Single, '#pop'), + ], + + 'double-quoted-string': [ + (r'[^"\\]+', String.Double), + (r'""', String.Escape), + (r"""\\[0'"bnrtZ\\%_]""", String.Escape), + (r'"', String.Double, '#pop'), + ], + + # Variable substates + # ------------------ + + 'single-quoted-variable': [ + (r"[^']+", Name.Variable), + (r"''", Name.Variable), + (r"'", Name.Variable, '#pop'), + ], + + 'double-quoted-variable': [ + (r'[^"]+', Name.Variable), + (r'""', Name.Variable), + (r'"', Name.Variable, '#pop'), + ], + + 'backtick-quoted-variable': [ + (r'[^`]+', Name.Variable), + (r'``', Name.Variable), + (r'`', Name.Variable, '#pop'), + ], + + # Schema object name substates + # ---------------------------- + # + # "Name.Quoted" and "Name.Quoted.Escape" are non-standard but + # formatters will style them as "Name" by default but add + # additional styles based on the token name. This gives users + # flexibility to add custom styles as desired. + # + 'schema-object-name': [ + (r'[^`]+', Name.Quoted), + (r'``', Name.Quoted.Escape), + (r'`', Name.Quoted, '#pop'), + ], } + def analyse_text(text): + rating = 0 + name_between_backtick_count = len( + name_between_backtick_re.findall(text)) + name_between_bracket_count = len( + name_between_bracket_re.findall(text)) + # Same logic as above in the TSQL analysis + dialect_name_count = name_between_backtick_count + name_between_bracket_count + if dialect_name_count >= 1 and \ + name_between_backtick_count >= 2 * name_between_bracket_count: + # Found at least twice as many `name` as [name]. + rating += 0.5 + elif name_between_backtick_count > name_between_bracket_count: + rating += 0.2 + elif name_between_backtick_count > 0: + rating += 0.1 + return rating + class SqliteConsoleLexer(Lexer): """ @@ -635,9 +788,8 @@ def get_tokens_unprocessed(self, data): curcode += line[8:] else: if curcode: - for item in do_insertions(insertions, - sql.get_tokens_unprocessed(curcode)): - yield item + yield from do_insertions(insertions, + sql.get_tokens_unprocessed(curcode)) curcode = '' insertions = [] if line.startswith('SQL error: '): @@ -645,9 +797,8 @@ def get_tokens_unprocessed(self, data): else: yield (match.start(), Generic.Output, line) if curcode: - for item in do_insertions(insertions, - sql.get_tokens_unprocessed(curcode)): - yield item + yield from do_insertions(insertions, + sql.get_tokens_unprocessed(curcode)) class RqlLexer(RegexLexer): diff --git a/src/syntax_highlighting/libs/pygments/lexers/stata.py b/src/syntax_highlighting/libs/pygments/lexers/stata.py index a015a23..4ec6cf4 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/stata.py +++ b/src/syntax_highlighting/libs/pygments/lexers/stata.py @@ -1,15 +1,15 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.stata ~~~~~~~~~~~~~~~~~~~~~ Lexer for Stata - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ -from pygments.lexer import RegexLexer, include, words +import re +from pygments.lexer import RegexLexer, default, include, words from pygments.token import Comment, Keyword, Name, Number, \ String, Text, Operator @@ -26,63 +26,125 @@ class StataLexer(RegexLexer): """ # Syntax based on # - http://fmwww.bc.edu/RePEc/bocode/s/synlightlist.ado - # - http://github.com/isagalaev/highlight.js/blob/master/src/languages/stata.js - # - http://github.com/jpitblado/vim-stata/blob/master/syntax/stata.vim + # - https://github.com/isagalaev/highlight.js/blob/master/src/languages/stata.js + # - https://github.com/jpitblado/vim-stata/blob/master/syntax/stata.vim name = 'Stata' aliases = ['stata', 'do'] filenames = ['*.do', '*.ado'] mimetypes = ['text/x-stata', 'text/stata', 'application/x-stata'] + flags = re.MULTILINE | re.DOTALL tokens = { 'root': [ include('comments'), - include('vars-strings'), + include('strings'), + include('macros'), include('numbers'), include('keywords'), + include('operators'), + include('format'), (r'.', Text), ], - # Global and local macros; regular and special strings - 'vars-strings': [ - (r'\$[\w{]', Name.Variable.Global, 'var_validglobal'), - (r'`\w{0,31}\'', Name.Variable), - (r'"', String, 'string_dquote'), - (r'`"', String, 'string_mquote'), - ], - # For either string type, highlight macros as macros - 'string_dquote': [ - (r'"', String, '#pop'), - (r'\\\\|\\"|\\\n', String.Escape), - (r'\$', Name.Variable.Global, 'var_validglobal'), - (r'`', Name.Variable, 'var_validlocal'), - (r'[^$`"\\]+', String), - (r'[$"\\]', String), - ], - 'string_mquote': [ + # Comments are a complicated beast in Stata because they can be + # nested and there are a few corner cases with that. See: + # - github.com/kylebarron/language-stata/issues/90 + # - statalist.org/forums/forum/general-stata-discussion/general/1448244 + 'comments': [ + (r'(^//|(?<=\s)//)(?!/)', Comment.Single, 'comments-double-slash'), + (r'^\s*\*', Comment.Single, 'comments-star'), + (r'/\*', Comment.Multiline, 'comments-block'), + (r'(^///|(?<=\s)///)', Comment.Special, 'comments-triple-slash') + ], + 'comments-block': [ + (r'/\*', Comment.Multiline, '#push'), + # this ends and restarts a comment block. but need to catch this so + # that it doesn\'t start _another_ level of comment blocks + (r'\*/\*', Comment.Multiline), + (r'(\*/\s+\*(?!/)[^\n]*)|(\*/)', Comment.Multiline, '#pop'), + # Match anything else as a character inside the comment + (r'.', Comment.Multiline), + ], + 'comments-star': [ + (r'///.*?\n', Comment.Single, + ('#pop', 'comments-triple-slash')), + (r'(^//|(?<=\s)//)(?!/)', Comment.Single, + ('#pop', 'comments-double-slash')), + (r'/\*', Comment.Multiline, 'comments-block'), + (r'.(?=\n)', Comment.Single, '#pop'), + (r'.', Comment.Single), + ], + 'comments-triple-slash': [ + (r'\n', Comment.Special, '#pop'), + # A // breaks out of a comment for the rest of the line + (r'//.*?(?=\n)', Comment.Single, '#pop'), + (r'.', Comment.Special), + ], + 'comments-double-slash': [ + (r'\n', Text, '#pop'), + (r'.', Comment.Single), + ], + # `"compound string"' and regular "string"; note the former are + # nested. + 'strings': [ + (r'`"', String, 'string-compound'), + (r'(?>|<=|>=|&&|\|\||\*\*|[-+~!*/%<>&^|?:]', Operator), ], 'data': [ - (r'\s+', Text), + (r'\s+', Whitespace), (r'0x[a-fA-F0-9]+', Number.Hex), (r'0[0-7]+', Number.Oct), (r'\d+\.\d+', Number.Float), (r'\d+', Number.Integer), (r'\$([\w.:-]+)', Name.Variable), - (r'([\w.:-]+)', Text), + (r'([\w.,@:-]+)', Text), ], 'params': [ (r';', Keyword, '#pop'), diff --git a/src/syntax_highlighting/libs/pygments/lexers/teal.py b/src/syntax_highlighting/libs/pygments/lexers/teal.py new file mode 100644 index 0000000..393000a --- /dev/null +++ b/src/syntax_highlighting/libs/pygments/lexers/teal.py @@ -0,0 +1,88 @@ +# -*- coding: utf-8 -*- +""" + pygments.lexers.teal + ~~~~~~~~~~~~~~~~~~~~ + + Lexer for TEAL. + + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer, bygroups, include, words +from pygments.token import Comment, Name, Number, String, Text, Keyword + +__all__ = ['TealLexer'] + +class TealLexer(RegexLexer): + """ + For the `Transaction Execution Approval Language (TEAL) + ` + + For more information about the grammar, see: + https://github.com/algorand/go-algorand/blob/master/data/transactions/logic/assembler.go + + .. versionadded:: 2.9 + """ + name = 'teal' + aliases = ['teal'] + filenames = ['*.teal'] + + keywords = words({ + 'Sender', 'Fee', 'FirstValid', 'FirstValidTime', 'LastValid', 'Note', + 'Lease', 'Receiver', 'Amount', 'CloseRemainderTo', 'VotePK', + 'SelectionPK', 'VoteFirst', 'VoteLast', 'VoteKeyDilution', 'Type', + 'TypeEnum', 'XferAsset', 'AssetAmount', 'AssetSender', 'AssetReceiver', + 'AssetCloseTo', 'GroupIndex', 'TxID', 'ApplicationID', 'OnCompletion', + 'ApplicationArgs', 'NumAppArgs', 'Accounts', 'NumAccounts', + 'ApprovalProgram', 'ClearStateProgram', 'RekeyTo', 'ConfigAsset', + 'ConfigAssetTotal', 'ConfigAssetDecimals', 'ConfigAssetDefaultFrozen', + 'ConfigAssetUnitName', 'ConfigAssetName', 'ConfigAssetURL', + 'ConfigAssetMetadataHash', 'ConfigAssetManager', 'ConfigAssetReserve', + 'ConfigAssetFreeze', 'ConfigAssetClawback', 'FreezeAsset', + 'FreezeAssetAccount', 'FreezeAssetFrozen', + 'NoOp', 'OptIn', 'CloseOut', 'ClearState', 'UpdateApplication', + 'DeleteApplication', + 'MinTxnFee', 'MinBalance', 'MaxTxnLife', 'ZeroAddress', 'GroupSize', + 'LogicSigVersion', 'Round', 'LatestTimestamp', 'CurrentApplicationID', + 'AssetBalance', 'AssetFrozen', + 'AssetTotal', 'AssetDecimals', 'AssetDefaultFrozen', 'AssetUnitName', + 'AssetName', 'AssetURL', 'AssetMetadataHash', 'AssetManager', + 'AssetReserve', 'AssetFreeze', 'AssetClawback', + }, suffix = r'\b') + + identifier = r'[^ \t\n]+(?=\/\/)|[^ \t\n]+' + newline = r'\r?\n' + tokens = { + 'root': [ + include('whitespace'), + # pragmas match specifically on the space character + (r'^#pragma .*' + newline, Comment.Directive), + # labels must be followed by a space, + # but anything after that is ignored + ('(' + identifier + ':' + ')' + '([ \t].*)', + bygroups(Name.Label, Comment.Single)), + (identifier, Name.Function, 'function-args'), + ], + 'function-args': [ + include('whitespace'), + (r'"', String, 'string'), + (r'(b(?:ase)?(?:32|64) ?)(\(?[a-zA-Z0-9+/=]+\)?)', + bygroups(String.Affix, String.Other)), + (r'[A-Z2-7]{58}', Number), # address + (r'0x[\da-fA-F]+', Number.Hex), + (r'\d+', Number.Integer), + (keywords, Keyword), + (identifier, Name.Attributes), # branch targets + (newline, Text, '#pop'), + ], + 'string': [ + (r'\\(?:["nrt\\]|x\d\d)', String.Escape), + (r'[^\\\"\n]+', String), + (r'"', String, '#pop'), + ], + 'whitespace': [ + (r'[ \t]+', Text), + (r'//[^\n]+', Comment.Single), + ], + } diff --git a/src/syntax_highlighting/libs/pygments/lexers/templates.py b/src/syntax_highlighting/libs/pygments/lexers/templates.py index 83c57db..548e14a 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/templates.py +++ b/src/syntax_highlighting/libs/pygments/lexers/templates.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.templates ~~~~~~~~~~~~~~~~~~~~~~~~~ Lexers for various template engines' markup. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -179,21 +178,21 @@ class SmartyLexer(RegexLexer): (r'(true|false|null)\b', Keyword.Constant), (r"[0-9](\.[0-9]*)?(eE[+-][0-9])?[flFLdD]?|" r"0[xX][0-9a-fA-F]+[Ll]?", Number), - (r'"(\\\\|\\"|[^"])*"', String.Double), - (r"'(\\\\|\\'|[^'])*'", String.Single), + (r'"(\\\\|\\[^\\]|[^"\\])*"', String.Double), + (r"'(\\\\|\\[^\\]|[^'\\])*'", String.Single), (r'[a-zA-Z_]\w*', Name.Attribute) ] } def analyse_text(text): rv = 0.0 - if re.search('\{if\s+.*?\}.*?\{/if\}', text): + if re.search(r'\{if\s+.*?\}.*?\{/if\}', text): rv += 0.15 - if re.search('\{include\s+file=.*?\}', text): + if re.search(r'\{include\s+file=.*?\}', text): rv += 0.15 - if re.search('\{foreach\s+.*?\}.*?\{/foreach\}', text): + if re.search(r'\{foreach\s+.*?\}.*?\{/foreach\}', text): rv += 0.15 - if re.search('\{\$.*?\}', text): + if re.search(r'\{\$.*?\}', text): rv += 0.01 return rv @@ -226,7 +225,7 @@ class VelocityLexer(RegexLexer): 'directiveparams'), (r'(#\{?)(' + identifier + r')(\}|\b)', bygroups(Comment.Preproc, Name.Function, Comment.Preproc)), - (r'\$\{?', Punctuation, 'variable') + (r'\$!?\{?', Punctuation, 'variable') ], 'variable': [ (identifier, Name.Variable), @@ -249,11 +248,11 @@ class VelocityLexer(RegexLexer): (r'\]', Operator, '#pop') ], 'funcparams': [ - (r'\$\{?', Punctuation, 'variable'), + (r'\$!?\{?', Punctuation, 'variable'), (r'\s+', Text), (r'[,:]', Punctuation), - (r'"(\\\\|\\"|[^"])*"', String.Double), - (r"'(\\\\|\\'|[^'])*'", String.Single), + (r'"(\\\\|\\[^\\]|[^"\\])*"', String.Double), + (r"'(\\\\|\\[^\\]|[^'\\])*'", String.Single), (r"0[xX][0-9a-fA-F]+[Ll]?", Number), (r"\b[0-9]+\b", Number), (r'(true|false|null)\b', Keyword.Constant), @@ -268,13 +267,13 @@ class VelocityLexer(RegexLexer): def analyse_text(text): rv = 0.0 - if re.search(r'#\{?macro\}?\(.*?\).*?#\{?end\}?', text): + if re.search(r'#\{?macro\}?\(.*?\).*?#\{?end\}?', text, re.DOTALL): rv += 0.25 - if re.search(r'#\{?if\}?\(.+?\).*?#\{?end\}?', text): + if re.search(r'#\{?if\}?\(.+?\).*?#\{?end\}?', text, re.DOTALL): rv += 0.15 - if re.search(r'#\{?foreach\}?\(.+?\).*?#\{?end\}?', text): + if re.search(r'#\{?foreach\}?\(.+?\).*?#\{?end\}?', text, re.DOTALL): rv += 0.15 - if re.search(r'\$\{?[a-zA-Z_]\w*(\([^)]*\))?' + if re.search(r'\$!?\{?[a-zA-Z_]\w*(\([^)]*\))?' r'(\.\w+(\([^)]*\))?)*\}?', text): rv += 0.01 return rv @@ -293,8 +292,7 @@ class VelocityHtmlLexer(DelegatingLexer): mimetypes = ['text/html+velocity'] def __init__(self, **options): - super(VelocityHtmlLexer, self).__init__(HtmlLexer, VelocityLexer, - **options) + super().__init__(HtmlLexer, VelocityLexer, **options) class VelocityXmlLexer(DelegatingLexer): @@ -310,8 +308,7 @@ class VelocityXmlLexer(DelegatingLexer): mimetypes = ['application/xml+velocity'] def __init__(self, **options): - super(VelocityXmlLexer, self).__init__(XmlLexer, VelocityLexer, - **options) + super().__init__(XmlLexer, VelocityLexer, **options) def analyse_text(text): rv = VelocityLexer.analyse_text(text) - 0.01 @@ -323,7 +320,7 @@ def analyse_text(text): class DjangoLexer(RegexLexer): """ Generic `django `_ - and `jinja `_ template lexer. + and `jinja `_ template lexer. It just highlights django/jinja code between the preprocessor directives, other data is left untouched by the lexer. @@ -340,7 +337,7 @@ class DjangoLexer(RegexLexer): (r'[^{]+', Other), (r'\{\{', Comment.Preproc, 'var'), # jinja/django comments - (r'\{[*#].*?[*#]\}', Comment), + (r'\{#.*?#\}', Comment), # django comments (r'(\{%)(-?\s*)(comment)(\s*-?)(%\})(.*?)' r'(\{%)(-?\s*)(endcomment)(\s*-?)(%\})', @@ -373,9 +370,9 @@ class DjangoLexer(RegexLexer): (r'(loop|block|super|forloop)\b', Name.Builtin), (r'[a-zA-Z_][\w-]*', Name.Variable), (r'\.\w+', Name.Variable), - (r':?"(\\\\|\\"|[^"])*"', String.Double), - (r":?'(\\\\|\\'|[^'])*'", String.Single), - (r'([{}()\[\]+\-*/,:~]|[><=]=?)', Operator), + (r':?"(\\\\|\\[^\\]|[^"\\])*"', String.Double), + (r":?'(\\\\|\\[^\\]|[^'\\])*'", String.Single), + (r'([{}()\[\]+\-*/%,:~]|[><=]=?|!=)', Operator), (r"[0-9](\.[0-9]*)?(eE[+-][0-9])?[flFLdD]?|" r"0[xX][0-9a-fA-F]+[Ll]?", Number), ], @@ -421,18 +418,18 @@ class MyghtyLexer(RegexLexer): tokens = { 'root': [ (r'\s+', Text), - (r'(<%(?:def|method))(\s*)(.*?)(>)(.*?)()(?s)', + (r'(?s)(<%(?:def|method))(\s*)(.*?)(>)(.*?)()', bygroups(Name.Tag, Text, Name.Function, Name.Tag, using(this), Name.Tag)), - (r'(<%\w+)(.*?)(>)(.*?)()(?s)', + (r'(?s)(<%\w+)(.*?)(>)(.*?)()', bygroups(Name.Tag, Name.Function, Name.Tag, using(PythonLexer), Name.Tag)), (r'(<&[^|])(.*?)(,.*?)?(&>)', bygroups(Name.Tag, Name.Function, using(PythonLexer), Name.Tag)), - (r'(<&\|)(.*?)(,.*?)?(&>)(?s)', + (r'(?s)(<&\|)(.*?)(,.*?)?(&>)', bygroups(Name.Tag, Name.Function, using(PythonLexer), Name.Tag)), (r'', Name.Tag), - (r'(<%!?)(.*?)(%>)(?s)', + (r'(?s)(<%!?)(.*?)(%>)', bygroups(Name.Tag, using(PythonLexer), Name.Tag)), (r'(?<=^)#[^\n]*(\n|\Z)', Comment), (r'(?<=^)(%)([^\n]*)(\n|\Z)', @@ -464,8 +461,7 @@ class MyghtyHtmlLexer(DelegatingLexer): mimetypes = ['text/html+myghty'] def __init__(self, **options): - super(MyghtyHtmlLexer, self).__init__(HtmlLexer, MyghtyLexer, - **options) + super().__init__(HtmlLexer, MyghtyLexer, **options) class MyghtyXmlLexer(DelegatingLexer): @@ -481,8 +477,7 @@ class MyghtyXmlLexer(DelegatingLexer): mimetypes = ['application/xml+myghty'] def __init__(self, **options): - super(MyghtyXmlLexer, self).__init__(XmlLexer, MyghtyLexer, - **options) + super().__init__(XmlLexer, MyghtyLexer, **options) class MyghtyJavascriptLexer(DelegatingLexer): @@ -494,14 +489,13 @@ class MyghtyJavascriptLexer(DelegatingLexer): """ name = 'JavaScript+Myghty' - aliases = ['js+myghty', 'javascript+myghty'] + aliases = ['javascript+myghty', 'js+myghty'] mimetypes = ['application/x-javascript+myghty', 'text/x-javascript+myghty', 'text/javascript+mygthy'] def __init__(self, **options): - super(MyghtyJavascriptLexer, self).__init__(JavascriptLexer, - MyghtyLexer, **options) + super().__init__(JavascriptLexer, MyghtyLexer, **options) class MyghtyCssLexer(DelegatingLexer): @@ -517,8 +511,7 @@ class MyghtyCssLexer(DelegatingLexer): mimetypes = ['text/css+myghty'] def __init__(self, **options): - super(MyghtyCssLexer, self).__init__(CssLexer, MyghtyLexer, - **options) + super().__init__(CssLexer, MyghtyLexer, **options) class MasonLexer(RegexLexer): @@ -538,20 +531,19 @@ class MasonLexer(RegexLexer): tokens = { 'root': [ (r'\s+', Text), - (r'(<%doc>)(.*?)()(?s)', + (r'(?s)(<%doc>)(.*?)()', bygroups(Name.Tag, Comment.Multiline, Name.Tag)), - (r'(<%(?:def|method))(\s*)(.*?)(>)(.*?)()(?s)', + (r'(?s)(<%(?:def|method))(\s*)(.*?)(>)(.*?)()', bygroups(Name.Tag, Text, Name.Function, Name.Tag, using(this), Name.Tag)), - (r'(<%\w+)(.*?)(>)(.*?)()(?s)', - bygroups(Name.Tag, Name.Function, Name.Tag, - using(PerlLexer), Name.Tag)), - (r'(<&[^|])(.*?)(,.*?)?(&>)(?s)', + (r'(?s)(<%(\w+)(.*?)(>))(.*?)()', + bygroups(Name.Tag, None, None, None, using(PerlLexer), Name.Tag)), + (r'(?s)(<&[^|])(.*?)(,.*?)?(&>)', bygroups(Name.Tag, Name.Function, using(PerlLexer), Name.Tag)), - (r'(<&\|)(.*?)(,.*?)?(&>)(?s)', + (r'(?s)(<&\|)(.*?)(,.*?)?(&>)', bygroups(Name.Tag, Name.Function, using(PerlLexer), Name.Tag)), (r'', Name.Tag), - (r'(<%!?)(.*?)(%>)(?s)', + (r'(?s)(<%!?)(.*?)(%>)', bygroups(Name.Tag, using(PerlLexer), Name.Tag)), (r'(?<=^)#[^\n]*(\n|\Z)', Comment), (r'(?<=^)(%)([^\n]*)(\n|\Z)', @@ -571,7 +563,7 @@ class MasonLexer(RegexLexer): def analyse_text(text): result = 0.0 - if re.search(r'', text) is not None: + if re.search(r'', text) is not None: result = 1.0 elif re.search(r'<&.+&>', text, re.DOTALL) is not None: result = 0.11 @@ -596,18 +588,18 @@ class MakoLexer(RegexLexer): tokens = { 'root': [ (r'(\s*)(%)(\s*end(?:\w+))(\n|\Z)', - bygroups(Text, Comment.Preproc, Keyword, Other)), + bygroups(Text.Whitespace, Comment.Preproc, Keyword, Other)), (r'(\s*)(%)([^\n]*)(\n|\Z)', - bygroups(Text, Comment.Preproc, using(PythonLexer), Other)), + bygroups(Text.Whitespace, Comment.Preproc, using(PythonLexer), Other)), (r'(\s*)(##[^\n]*)(\n|\Z)', - bygroups(Text, Comment.Preproc, Other)), - (r'(?s)<%doc>.*?', Comment.Preproc), + bygroups(Text.Whitespace, Comment.Single, Text.Whitespace)), + (r'(?s)<%doc>.*?', Comment.Multiline), (r'(<%)([\w.:]+)', bygroups(Comment.Preproc, Name.Builtin), 'tag'), (r'()', bygroups(Comment.Preproc, Name.Builtin, Comment.Preproc)), (r'<%(?=([\w.:]+))', Comment.Preproc, 'ondeftags'), - (r'(<%(?:!?))(.*?)(%>)(?s)', + (r'(?s)(<%(?:!?))(.*?)(%>)', bygroups(Comment.Preproc, using(PythonLexer), Comment.Preproc)), (r'(\$\{)(.*?)(\})', bygroups(Comment.Preproc, using(PythonLexer), Comment.Preproc)), @@ -659,8 +651,7 @@ class MakoHtmlLexer(DelegatingLexer): mimetypes = ['text/html+mako'] def __init__(self, **options): - super(MakoHtmlLexer, self).__init__(HtmlLexer, MakoLexer, - **options) + super().__init__(HtmlLexer, MakoLexer, **options) class MakoXmlLexer(DelegatingLexer): @@ -676,8 +667,7 @@ class MakoXmlLexer(DelegatingLexer): mimetypes = ['application/xml+mako'] def __init__(self, **options): - super(MakoXmlLexer, self).__init__(XmlLexer, MakoLexer, - **options) + super().__init__(XmlLexer, MakoLexer, **options) class MakoJavascriptLexer(DelegatingLexer): @@ -689,14 +679,13 @@ class MakoJavascriptLexer(DelegatingLexer): """ name = 'JavaScript+Mako' - aliases = ['js+mako', 'javascript+mako'] + aliases = ['javascript+mako', 'js+mako'] mimetypes = ['application/x-javascript+mako', 'text/x-javascript+mako', 'text/javascript+mako'] def __init__(self, **options): - super(MakoJavascriptLexer, self).__init__(JavascriptLexer, - MakoLexer, **options) + super().__init__(JavascriptLexer, MakoLexer, **options) class MakoCssLexer(DelegatingLexer): @@ -712,8 +701,7 @@ class MakoCssLexer(DelegatingLexer): mimetypes = ['text/css+mako'] def __init__(self, **options): - super(MakoCssLexer, self).__init__(CssLexer, MakoLexer, - **options) + super().__init__(CssLexer, MakoLexer, **options) # Genshi and Cheetah lexers courtesy of Matt Good. @@ -759,7 +747,7 @@ class CheetahLexer(RegexLexer): # TODO support other Python syntax like $foo['bar'] (r'(\$)([a-zA-Z_][\w.]*\w)', bygroups(Comment.Preproc, using(CheetahPythonLexer))), - (r'(\$\{!?)(.*?)(\})(?s)', + (r'(?s)(\$\{!?)(.*?)(\})', bygroups(Comment.Preproc, using(CheetahPythonLexer), Comment.Preproc)), (r'''(?sx) @@ -786,8 +774,7 @@ class CheetahHtmlLexer(DelegatingLexer): mimetypes = ['text/html+cheetah', 'text/html+spitfire'] def __init__(self, **options): - super(CheetahHtmlLexer, self).__init__(HtmlLexer, CheetahLexer, - **options) + super().__init__(HtmlLexer, CheetahLexer, **options) class CheetahXmlLexer(DelegatingLexer): @@ -801,8 +788,7 @@ class CheetahXmlLexer(DelegatingLexer): mimetypes = ['application/xml+cheetah', 'application/xml+spitfire'] def __init__(self, **options): - super(CheetahXmlLexer, self).__init__(XmlLexer, CheetahLexer, - **options) + super().__init__(XmlLexer, CheetahLexer, **options) class CheetahJavascriptLexer(DelegatingLexer): @@ -812,8 +798,8 @@ class CheetahJavascriptLexer(DelegatingLexer): """ name = 'JavaScript+Cheetah' - aliases = ['js+cheetah', 'javascript+cheetah', - 'js+spitfire', 'javascript+spitfire'] + aliases = ['javascript+cheetah', 'js+cheetah', + 'javascript+spitfire', 'js+spitfire'] mimetypes = ['application/x-javascript+cheetah', 'text/x-javascript+cheetah', 'text/javascript+cheetah', @@ -822,8 +808,7 @@ class CheetahJavascriptLexer(DelegatingLexer): 'text/javascript+spitfire'] def __init__(self, **options): - super(CheetahJavascriptLexer, self).__init__(JavascriptLexer, - CheetahLexer, **options) + super().__init__(JavascriptLexer, CheetahLexer, **options) class GenshiTextLexer(RegexLexer): @@ -937,14 +922,13 @@ class HtmlGenshiLexer(DelegatingLexer): mimetypes = ['text/html+genshi'] def __init__(self, **options): - super(HtmlGenshiLexer, self).__init__(HtmlLexer, GenshiMarkupLexer, - **options) + super().__init__(HtmlLexer, GenshiMarkupLexer, **options) def analyse_text(text): rv = 0.0 - if re.search('\$\{.*?\}', text) is not None: + if re.search(r'\$\{.*?\}', text) is not None: rv += 0.2 - if re.search('py:(.*?)=["\']', text) is not None: + if re.search(r'py:(.*?)=["\']', text) is not None: rv += 0.2 return rv + HtmlLexer.analyse_text(text) - 0.01 @@ -962,14 +946,13 @@ class GenshiLexer(DelegatingLexer): mimetypes = ['application/x-genshi', 'application/x-kid'] def __init__(self, **options): - super(GenshiLexer, self).__init__(XmlLexer, GenshiMarkupLexer, - **options) + super().__init__(XmlLexer, GenshiMarkupLexer, **options) def analyse_text(text): rv = 0.0 - if re.search('\$\{.*?\}', text) is not None: + if re.search(r'\$\{.*?\}', text) is not None: rv += 0.2 - if re.search('py:(.*?)=["\']', text) is not None: + if re.search(r'py:(.*?)=["\']', text) is not None: rv += 0.2 return rv + XmlLexer.analyse_text(text) - 0.01 @@ -988,9 +971,7 @@ class JavascriptGenshiLexer(DelegatingLexer): 'text/javascript+genshi'] def __init__(self, **options): - super(JavascriptGenshiLexer, self).__init__(JavascriptLexer, - GenshiTextLexer, - **options) + super().__init__(JavascriptLexer, GenshiTextLexer, **options) def analyse_text(text): return GenshiLexer.analyse_text(text) - 0.05 @@ -1007,8 +988,7 @@ class CssGenshiLexer(DelegatingLexer): mimetypes = ['text/css+genshi'] def __init__(self, **options): - super(CssGenshiLexer, self).__init__(CssLexer, GenshiTextLexer, - **options) + super().__init__(CssLexer, GenshiTextLexer, **options) def analyse_text(text): return GenshiLexer.analyse_text(text) - 0.05 @@ -1029,7 +1009,7 @@ class RhtmlLexer(DelegatingLexer): mimetypes = ['text/html+ruby'] def __init__(self, **options): - super(RhtmlLexer, self).__init__(HtmlLexer, ErbLexer, **options) + super().__init__(HtmlLexer, ErbLexer, **options) def analyse_text(text): rv = ErbLexer.analyse_text(text) - 0.01 @@ -1046,12 +1026,12 @@ class XmlErbLexer(DelegatingLexer): """ name = 'XML+Ruby' - aliases = ['xml+erb', 'xml+ruby'] + aliases = ['xml+ruby', 'xml+erb'] alias_filenames = ['*.xml'] mimetypes = ['application/xml+ruby'] def __init__(self, **options): - super(XmlErbLexer, self).__init__(XmlLexer, ErbLexer, **options) + super().__init__(XmlLexer, ErbLexer, **options) def analyse_text(text): rv = ErbLexer.analyse_text(text) - 0.01 @@ -1066,12 +1046,12 @@ class CssErbLexer(DelegatingLexer): """ name = 'CSS+Ruby' - aliases = ['css+erb', 'css+ruby'] + aliases = ['css+ruby', 'css+erb'] alias_filenames = ['*.css'] mimetypes = ['text/css+ruby'] def __init__(self, **options): - super(CssErbLexer, self).__init__(CssLexer, ErbLexer, **options) + super().__init__(CssLexer, ErbLexer, **options) def analyse_text(text): return ErbLexer.analyse_text(text) - 0.05 @@ -1084,15 +1064,14 @@ class JavascriptErbLexer(DelegatingLexer): """ name = 'JavaScript+Ruby' - aliases = ['js+erb', 'javascript+erb', 'js+ruby', 'javascript+ruby'] + aliases = ['javascript+ruby', 'js+ruby', 'javascript+erb', 'js+erb'] alias_filenames = ['*.js'] mimetypes = ['application/x-javascript+ruby', 'text/x-javascript+ruby', 'text/javascript+ruby'] def __init__(self, **options): - super(JavascriptErbLexer, self).__init__(JavascriptLexer, ErbLexer, - **options) + super().__init__(JavascriptLexer, ErbLexer, **options) def analyse_text(text): return ErbLexer.analyse_text(text) - 0.05 @@ -1115,7 +1094,7 @@ class HtmlPhpLexer(DelegatingLexer): 'application/x-httpd-php4', 'application/x-httpd-php5'] def __init__(self, **options): - super(HtmlPhpLexer, self).__init__(HtmlLexer, PhpLexer, **options) + super().__init__(HtmlLexer, PhpLexer, **options) def analyse_text(text): rv = PhpLexer.analyse_text(text) - 0.01 @@ -1135,7 +1114,7 @@ class XmlPhpLexer(DelegatingLexer): mimetypes = ['application/xml+php'] def __init__(self, **options): - super(XmlPhpLexer, self).__init__(XmlLexer, PhpLexer, **options) + super().__init__(XmlLexer, PhpLexer, **options) def analyse_text(text): rv = PhpLexer.analyse_text(text) - 0.01 @@ -1155,7 +1134,7 @@ class CssPhpLexer(DelegatingLexer): mimetypes = ['text/css+php'] def __init__(self, **options): - super(CssPhpLexer, self).__init__(CssLexer, PhpLexer, **options) + super().__init__(CssLexer, PhpLexer, **options) def analyse_text(text): return PhpLexer.analyse_text(text) - 0.05 @@ -1168,15 +1147,14 @@ class JavascriptPhpLexer(DelegatingLexer): """ name = 'JavaScript+PHP' - aliases = ['js+php', 'javascript+php'] + aliases = ['javascript+php', 'js+php'] alias_filenames = ['*.js'] mimetypes = ['application/x-javascript+php', 'text/x-javascript+php', 'text/javascript+php'] def __init__(self, **options): - super(JavascriptPhpLexer, self).__init__(JavascriptLexer, PhpLexer, - **options) + super().__init__(JavascriptLexer, PhpLexer, **options) def analyse_text(text): return PhpLexer.analyse_text(text) @@ -1196,7 +1174,7 @@ class HtmlSmartyLexer(DelegatingLexer): mimetypes = ['text/html+smarty'] def __init__(self, **options): - super(HtmlSmartyLexer, self).__init__(HtmlLexer, SmartyLexer, **options) + super().__init__(HtmlLexer, SmartyLexer, **options) def analyse_text(text): rv = SmartyLexer.analyse_text(text) - 0.01 @@ -1217,7 +1195,7 @@ class XmlSmartyLexer(DelegatingLexer): mimetypes = ['application/xml+smarty'] def __init__(self, **options): - super(XmlSmartyLexer, self).__init__(XmlLexer, SmartyLexer, **options) + super().__init__(XmlLexer, SmartyLexer, **options) def analyse_text(text): rv = SmartyLexer.analyse_text(text) - 0.01 @@ -1238,7 +1216,7 @@ class CssSmartyLexer(DelegatingLexer): mimetypes = ['text/css+smarty'] def __init__(self, **options): - super(CssSmartyLexer, self).__init__(CssLexer, SmartyLexer, **options) + super().__init__(CssLexer, SmartyLexer, **options) def analyse_text(text): return SmartyLexer.analyse_text(text) - 0.05 @@ -1251,15 +1229,14 @@ class JavascriptSmartyLexer(DelegatingLexer): """ name = 'JavaScript+Smarty' - aliases = ['js+smarty', 'javascript+smarty'] + aliases = ['javascript+smarty', 'js+smarty'] alias_filenames = ['*.js', '*.tpl'] mimetypes = ['application/x-javascript+smarty', 'text/x-javascript+smarty', 'text/javascript+smarty'] def __init__(self, **options): - super(JavascriptSmartyLexer, self).__init__(JavascriptLexer, SmartyLexer, - **options) + super().__init__(JavascriptLexer, SmartyLexer, **options) def analyse_text(text): return SmartyLexer.analyse_text(text) - 0.05 @@ -1279,7 +1256,7 @@ class HtmlDjangoLexer(DelegatingLexer): mimetypes = ['text/html+django', 'text/html+jinja'] def __init__(self, **options): - super(HtmlDjangoLexer, self).__init__(HtmlLexer, DjangoLexer, **options) + super().__init__(HtmlLexer, DjangoLexer, **options) def analyse_text(text): rv = DjangoLexer.analyse_text(text) - 0.01 @@ -1300,7 +1277,7 @@ class XmlDjangoLexer(DelegatingLexer): mimetypes = ['application/xml+django', 'application/xml+jinja'] def __init__(self, **options): - super(XmlDjangoLexer, self).__init__(XmlLexer, DjangoLexer, **options) + super().__init__(XmlLexer, DjangoLexer, **options) def analyse_text(text): rv = DjangoLexer.analyse_text(text) - 0.01 @@ -1321,7 +1298,7 @@ class CssDjangoLexer(DelegatingLexer): mimetypes = ['text/css+django', 'text/css+jinja'] def __init__(self, **options): - super(CssDjangoLexer, self).__init__(CssLexer, DjangoLexer, **options) + super().__init__(CssLexer, DjangoLexer, **options) def analyse_text(text): return DjangoLexer.analyse_text(text) - 0.05 @@ -1334,8 +1311,8 @@ class JavascriptDjangoLexer(DelegatingLexer): """ name = 'JavaScript+Django/Jinja' - aliases = ['js+django', 'javascript+django', - 'js+jinja', 'javascript+jinja'] + aliases = ['javascript+django', 'js+django', + 'javascript+jinja', 'js+jinja'] alias_filenames = ['*.js'] mimetypes = ['application/x-javascript+django', 'application/x-javascript+jinja', @@ -1345,8 +1322,7 @@ class JavascriptDjangoLexer(DelegatingLexer): 'text/javascript+jinja'] def __init__(self, **options): - super(JavascriptDjangoLexer, self).__init__(JavascriptLexer, DjangoLexer, - **options) + super().__init__(JavascriptLexer, DjangoLexer, **options) def analyse_text(text): return DjangoLexer.analyse_text(text) - 0.05 @@ -1389,7 +1365,7 @@ class JspLexer(DelegatingLexer): mimetypes = ['application/x-jsp'] def __init__(self, **options): - super(JspLexer, self).__init__(XmlLexer, JspRootLexer, **options) + super().__init__(XmlLexer, JspRootLexer, **options) def analyse_text(text): rv = JavaLexer.analyse_text(text) - 0.01 @@ -1428,7 +1404,7 @@ class EvoqueLexer(RegexLexer): # see doc for handling first name arg: /directives/evoque/ # + minor inconsistency: the "name" in e.g. $overlay{name=site_base} # should be using(PythonLexer), not passed out as String - (r'(\$)(evoque|overlay)(\{(%)?)(\s*[#\w\-"\'.]+[^=,%}]+?)?' + (r'(\$)(evoque|overlay)(\{(%)?)(\s*[#\w\-"\'.]+)?' r'(.*?)((?(4)%)\})', bygroups(Punctuation, Name.Builtin, Punctuation, None, String, using(PythonLexer), Punctuation)), @@ -1452,6 +1428,10 @@ class EvoqueLexer(RegexLexer): ], } + def analyse_text(text): + """Evoque templates use $evoque, which is unique.""" + if '$evoque' in text: + return 1 class EvoqueHtmlLexer(DelegatingLexer): """ @@ -1466,8 +1446,10 @@ class EvoqueHtmlLexer(DelegatingLexer): mimetypes = ['text/html+evoque'] def __init__(self, **options): - super(EvoqueHtmlLexer, self).__init__(HtmlLexer, EvoqueLexer, - **options) + super().__init__(HtmlLexer, EvoqueLexer, **options) + + def analyse_text(text): + return EvoqueLexer.analyse_text(text) class EvoqueXmlLexer(DelegatingLexer): @@ -1483,8 +1465,10 @@ class EvoqueXmlLexer(DelegatingLexer): mimetypes = ['application/xml+evoque'] def __init__(self, **options): - super(EvoqueXmlLexer, self).__init__(XmlLexer, EvoqueLexer, - **options) + super().__init__(XmlLexer, EvoqueLexer, **options) + + def analyse_text(text): + return EvoqueLexer.analyse_text(text) class ColdfusionLexer(RegexLexer): @@ -1591,8 +1575,7 @@ class ColdfusionHtmlLexer(DelegatingLexer): mimetypes = ['application/x-coldfusion'] def __init__(self, **options): - super(ColdfusionHtmlLexer, self).__init__(HtmlLexer, ColdfusionMarkupLexer, - **options) + super().__init__(HtmlLexer, ColdfusionMarkupLexer, **options) class ColdfusionCFCLexer(DelegatingLexer): @@ -1607,8 +1590,7 @@ class ColdfusionCFCLexer(DelegatingLexer): mimetypes = [] def __init__(self, **options): - super(ColdfusionCFCLexer, self).__init__(ColdfusionHtmlLexer, ColdfusionLexer, - **options) + super().__init__(ColdfusionHtmlLexer, ColdfusionLexer, **options) class SspLexer(DelegatingLexer): @@ -1623,11 +1605,11 @@ class SspLexer(DelegatingLexer): mimetypes = ['application/x-ssp'] def __init__(self, **options): - super(SspLexer, self).__init__(XmlLexer, JspRootLexer, **options) + super().__init__(XmlLexer, JspRootLexer, **options) def analyse_text(text): rv = 0.0 - if re.search('val \w+\s*:', text): + if re.search(r'val \w+\s*:', text): rv += 0.6 if looks_like_xml(text): rv += 0.2 @@ -1670,8 +1652,7 @@ class TeaTemplateLexer(DelegatingLexer): mimetypes = ['text/x-tea'] def __init__(self, **options): - super(TeaTemplateLexer, self).__init__(XmlLexer, - TeaTemplateRootLexer, **options) + super().__init__(XmlLexer, TeaTemplateRootLexer, **options) def analyse_text(text): rv = TeaLangLexer.analyse_text(text) - 0.01 @@ -1701,7 +1682,7 @@ class LassoHtmlLexer(DelegatingLexer): 'application/x-httpd-lasso[89]'] def __init__(self, **options): - super(LassoHtmlLexer, self).__init__(HtmlLexer, LassoLexer, **options) + super().__init__(HtmlLexer, LassoLexer, **options) def analyse_text(text): rv = LassoLexer.analyse_text(text) - 0.01 @@ -1725,7 +1706,7 @@ class LassoXmlLexer(DelegatingLexer): mimetypes = ['application/xml+lasso'] def __init__(self, **options): - super(LassoXmlLexer, self).__init__(XmlLexer, LassoLexer, **options) + super().__init__(XmlLexer, LassoLexer, **options) def analyse_text(text): rv = LassoLexer.analyse_text(text) - 0.01 @@ -1749,11 +1730,11 @@ class LassoCssLexer(DelegatingLexer): def __init__(self, **options): options['requiredelimiters'] = True - super(LassoCssLexer, self).__init__(CssLexer, LassoLexer, **options) + super().__init__(CssLexer, LassoLexer, **options) def analyse_text(text): rv = LassoLexer.analyse_text(text) - 0.05 - if re.search(r'\w+:.+?;', text): + if re.search(r'\w+:[^;]+;', text): rv += 0.1 if 'padding:' in text: rv += 0.1 @@ -1769,7 +1750,7 @@ class LassoJavascriptLexer(DelegatingLexer): """ name = 'JavaScript+Lasso' - aliases = ['js+lasso', 'javascript+lasso'] + aliases = ['javascript+lasso', 'js+lasso'] alias_filenames = ['*.js'] mimetypes = ['application/x-javascript+lasso', 'text/x-javascript+lasso', @@ -1777,8 +1758,7 @@ class LassoJavascriptLexer(DelegatingLexer): def __init__(self, **options): options['requiredelimiters'] = True - super(LassoJavascriptLexer, self).__init__(JavascriptLexer, LassoLexer, - **options) + super().__init__(JavascriptLexer, LassoLexer, **options) def analyse_text(text): rv = LassoLexer.analyse_text(text) - 0.05 @@ -1802,27 +1782,27 @@ class HandlebarsLexer(RegexLexer): 'root': [ (r'[^{]+', Other), + # Comment start {{! }} or {{!-- (r'\{\{!.*\}\}', Comment), + # HTML Escaping open {{{expression (r'(\{\{\{)(\s*)', bygroups(Comment.Special, Text), 'tag'), + + # {{blockOpen {{#blockOpen {{/blockClose with optional tilde ~ + (r'(\{\{)([#~/]+)([^\s}]*)', + bygroups(Comment.Preproc, Number.Attribute, Number.Attribute), 'tag'), (r'(\{\{)(\s*)', bygroups(Comment.Preproc, Text), 'tag'), ], 'tag': [ (r'\s+', Text), + # HTML Escaping close }}} (r'\}\}\}', Comment.Special, '#pop'), - (r'\}\}', Comment.Preproc, '#pop'), - - # Handlebars - (r'([#/]*)(each|if|unless|else|with|log|in(line)?)', bygroups(Keyword, - Keyword)), - (r'#\*inline', Keyword), - - # General {{#block}} - (r'([#/])([\w-]+)', bygroups(Name.Function, Name.Function)), + # blockClose}}, includes optional tilde ~ + (r'(~?)(\}\})', bygroups(Number, Comment.Preproc), '#pop'), # {{opt=something}} - (r'([\w-]+)(=)', bygroups(Name.Attribute, Operator)), + (r'([^\s}]+)(=)', bygroups(Name.Attribute, Operator)), # Partials {{> ...}} (r'(>)(\s*)(@partial-block)', bygroups(Keyword, Text, Keyword)), @@ -1845,7 +1825,7 @@ class HandlebarsLexer(RegexLexer): include('generic'), ], 'variable': [ - (r'[a-zA-Z][\w-]*', Name.Variable), + (r'[()/@a-zA-Z][\w-]*', Name.Variable), (r'\.[\w-]+', Name.Variable), (r'(this\/|\.\/|(\.\.\/)+)[\w-]+', Name.Variable), ], @@ -1853,8 +1833,8 @@ class HandlebarsLexer(RegexLexer): include('variable'), # borrowed from DjangoLexer - (r':?"(\\\\|\\"|[^"])*"', String.Double), - (r":?'(\\\\|\\'|[^'])*'", String.Single), + (r':?"(\\\\|\\[^\\]|[^"\\])*"', String.Double), + (r":?'(\\\\|\\[^\\]|[^'\\])*'", String.Single), (r"[0-9](\.[0-9]*)?(eE[+-][0-9])?[flFLdD]?|" r"0[xX][0-9a-fA-F]+[Ll]?", Number), ] @@ -1875,7 +1855,7 @@ class HandlebarsHtmlLexer(DelegatingLexer): mimetypes = ['text/html+handlebars', 'text/x-handlebars-template'] def __init__(self, **options): - super(HandlebarsHtmlLexer, self).__init__(HtmlLexer, HandlebarsLexer, **options) + super().__init__(HtmlLexer, HandlebarsLexer, **options) class YamlJinjaLexer(DelegatingLexer): @@ -1894,7 +1874,7 @@ class YamlJinjaLexer(DelegatingLexer): mimetypes = ['text/x-yaml+jinja', 'text/x-sls'] def __init__(self, **options): - super(YamlJinjaLexer, self).__init__(YamlLexer, DjangoLexer, **options) + super().__init__(YamlLexer, DjangoLexer, **options) class LiquidLexer(RegexLexer): @@ -1955,7 +1935,7 @@ class LiquidLexer(RegexLexer): 'output': [ include('whitespace'), - ('\}\}', Punctuation, '#pop'), # end of output + (r'\}\}', Punctuation, '#pop'), # end of output (r'\|', Punctuation, 'filters') ], @@ -2166,8 +2146,8 @@ class TwigLexer(RegexLexer): (_ident_inner, Name.Variable), (r'\.' + _ident_inner, Name.Variable), (r'\.[0-9]+', Number), - (r':?"(\\\\|\\"|[^"])*"', String.Double), - (r":?'(\\\\|\\'|[^'])*'", String.Single), + (r':?"(\\\\|\\[^\\]|[^"\\])*"', String.Double), + (r":?'(\\\\|\\[^\\]|[^'\\])*'", String.Single), (r'([{}()\[\]+\-*/,:~%]|\.\.|\?|:|\*\*|\/\/|!=|[><=]=?)', Operator), (r"[0-9](\.[0-9]*)?(eE[+-][0-9])?[flFLdD]?|" r"0[xX][0-9a-fA-F]+[Ll]?", Number), @@ -2200,7 +2180,7 @@ class TwigHtmlLexer(DelegatingLexer): mimetypes = ['text/html+twig'] def __init__(self, **options): - super(TwigHtmlLexer, self).__init__(HtmlLexer, TwigLexer, **options) + super().__init__(HtmlLexer, TwigLexer, **options) class Angular2Lexer(RegexLexer): @@ -2235,9 +2215,9 @@ class Angular2Lexer(RegexLexer): # *ngIf="..."; #f="ngForm" (r'([*#])([\w:.-]+)(\s*)(=)(\s*)', - bygroups(Punctuation, Name.Attribute, Punctuation, Operator), 'attr'), + bygroups(Punctuation, Name.Attribute, Text, Operator, Text), 'attr'), (r'([*#])([\w:.-]+)(\s*)', - bygroups(Punctuation, Name.Attribute, Punctuation)), + bygroups(Punctuation, Name.Attribute, Text)), ], 'ngExpression': [ @@ -2246,8 +2226,8 @@ class Angular2Lexer(RegexLexer): # Literals (r':?(true|false)', String.Boolean), - (r':?"(\\\\|\\"|[^"])*"', String.Double), - (r":?'(\\\\|\\'|[^'])*'", String.Single), + (r':?"(\\\\|\\[^\\]|[^"\\])*"', String.Double), + (r":?'(\\\\|\\[^\\]|[^'\\])*'", String.Single), (r"[0-9](\.[0-9]*)?(eE[+-][0-9])?[flFLdD]?|" r"0[xX][0-9a-fA-F]+[Ll]?", Number), @@ -2280,4 +2260,4 @@ class Angular2HtmlLexer(DelegatingLexer): filenames = ['*.ng2'] def __init__(self, **options): - super(Angular2HtmlLexer, self).__init__(HtmlLexer, Angular2Lexer, **options) + super().__init__(HtmlLexer, Angular2Lexer, **options) diff --git a/src/syntax_highlighting/libs/pygments/lexers/teraterm.py b/src/syntax_highlighting/libs/pygments/lexers/teraterm.py new file mode 100644 index 0000000..feb552d --- /dev/null +++ b/src/syntax_highlighting/libs/pygments/lexers/teraterm.py @@ -0,0 +1,334 @@ +""" + pygments.lexers.teraterm + ~~~~~~~~~~~~~~~~~~~~~~~~ + + Lexer for Tera Term macro files. + + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re + +from pygments.lexer import RegexLexer, include, bygroups +from pygments.token import Text, Comment, Operator, Name, String, \ + Number, Keyword + +__all__ = ['TeraTermLexer'] + + +class TeraTermLexer(RegexLexer): + """ + For `Tera Term `_ macro source code. + + .. versionadded:: 2.4 + """ + name = 'Tera Term macro' + aliases = ['teratermmacro', 'teraterm', 'ttl'] + filenames = ['*.ttl'] + mimetypes = ['text/x-teratermmacro'] + + tokens = { + 'root': [ + include('comments'), + include('labels'), + include('commands'), + include('builtin-variables'), + include('user-variables'), + include('operators'), + include('numeric-literals'), + include('string-literals'), + include('all-whitespace'), + (r'\S', Text), + ], + 'comments': [ + (r';[^\r\n]*', Comment.Single), + (r'/\*', Comment.Multiline, 'in-comment'), + ], + 'in-comment': [ + (r'\*/', Comment.Multiline, '#pop'), + (r'[^*/]+', Comment.Multiline), + (r'[*/]', Comment.Multiline) + ], + 'labels': [ + (r'(?i)^(\s*)(:[a-z0-9_]+)', bygroups(Text, Name.Label)), + ], + 'commands': [ + ( + r'(?i)\b(' + r'basename|' + r'beep|' + r'bplusrecv|' + r'bplussend|' + r'break|' + r'bringupbox|' + # 'call' is handled separately. + r'callmenu|' + r'changedir|' + r'checksum16|' + r'checksum16file|' + r'checksum32|' + r'checksum32file|' + r'checksum8|' + r'checksum8file|' + r'clearscreen|' + r'clipb2var|' + r'closesbox|' + r'closett|' + r'code2str|' + r'connect|' + r'continue|' + r'crc16|' + r'crc16file|' + r'crc32|' + r'crc32file|' + r'cygconnect|' + r'delpassword|' + r'dirname|' + r'dirnamebox|' + r'disconnect|' + r'dispstr|' + r'do|' + r'else|' + r'elseif|' + r'enablekeyb|' + r'end|' + r'endif|' + r'enduntil|' + r'endwhile|' + r'exec|' + r'execcmnd|' + r'exit|' + r'expandenv|' + r'fileclose|' + r'fileconcat|' + r'filecopy|' + r'filecreate|' + r'filedelete|' + r'filelock|' + r'filemarkptr|' + r'filenamebox|' + r'fileopen|' + r'fileread|' + r'filereadln|' + r'filerename|' + r'filesearch|' + r'fileseek|' + r'fileseekback|' + r'filestat|' + r'filestrseek|' + r'filestrseek2|' + r'filetruncate|' + r'fileunlock|' + r'filewrite|' + r'filewriteln|' + r'findclose|' + r'findfirst|' + r'findnext|' + r'flushrecv|' + r'foldercreate|' + r'folderdelete|' + r'foldersearch|' + r'for|' + r'getdate|' + r'getdir|' + r'getenv|' + r'getfileattr|' + r'gethostname|' + r'getipv4addr|' + r'getipv6addr|' + r'getmodemstatus|' + r'getpassword|' + r'getspecialfolder|' + r'gettime|' + r'gettitle|' + r'getttdir|' + r'getver|' + # 'goto' is handled separately. + r'if|' + r'ifdefined|' + r'include|' + r'inputbox|' + r'int2str|' + r'intdim|' + r'ispassword|' + r'kmtfinish|' + r'kmtget|' + r'kmtrecv|' + r'kmtsend|' + r'listbox|' + r'loadkeymap|' + r'logautoclosemode|' + r'logclose|' + r'loginfo|' + r'logopen|' + r'logpause|' + r'logrotate|' + r'logstart|' + r'logwrite|' + r'loop|' + r'makepath|' + r'messagebox|' + r'mpause|' + r'next|' + r'passwordbox|' + r'pause|' + r'quickvanrecv|' + r'quickvansend|' + r'random|' + r'recvln|' + r'regexoption|' + r'restoresetup|' + r'return|' + r'rotateleft|' + r'rotateright|' + r'scprecv|' + r'scpsend|' + r'send|' + r'sendbreak|' + r'sendbroadcast|' + r'sendfile|' + r'sendkcode|' + r'sendln|' + r'sendlnbroadcast|' + r'sendlnmulticast|' + r'sendmulticast|' + r'setbaud|' + r'setdate|' + r'setdebug|' + r'setdir|' + r'setdlgpos|' + r'setdtr|' + r'setecho|' + r'setenv|' + r'setexitcode|' + r'setfileattr|' + r'setflowctrl|' + r'setmulticastname|' + r'setpassword|' + r'setrts|' + r'setspeed|' + r'setsync|' + r'settime|' + r'settitle|' + r'show|' + r'showtt|' + r'sprintf|' + r'sprintf2|' + r'statusbox|' + r'str2code|' + r'str2int|' + r'strcompare|' + r'strconcat|' + r'strcopy|' + r'strdim|' + r'strinsert|' + r'strjoin|' + r'strlen|' + r'strmatch|' + r'strremove|' + r'strreplace|' + r'strscan|' + r'strspecial|' + r'strsplit|' + r'strtrim|' + r'testlink|' + r'then|' + r'tolower|' + r'toupper|' + r'unlink|' + r'until|' + r'uptime|' + r'var2clipb|' + r'wait|' + r'wait4all|' + r'waitevent|' + r'waitln|' + r'waitn|' + r'waitrecv|' + r'waitregex|' + r'while|' + r'xmodemrecv|' + r'xmodemsend|' + r'yesnobox|' + r'ymodemrecv|' + r'ymodemsend|' + r'zmodemrecv|' + r'zmodemsend' + r')\b', + Keyword, + ), + (r'(?i)(call|goto)([ \t]+)([a-z0-9_]+)', + bygroups(Keyword, Text, Name.Label)), + ], + 'builtin-variables': [ + ( + r'(?i)(' + r'groupmatchstr1|' + r'groupmatchstr2|' + r'groupmatchstr3|' + r'groupmatchstr4|' + r'groupmatchstr5|' + r'groupmatchstr6|' + r'groupmatchstr7|' + r'groupmatchstr8|' + r'groupmatchstr9|' + r'inputstr|' + r'matchstr|' + r'mtimeout|' + r'param1|' + r'param2|' + r'param3|' + r'param4|' + r'param5|' + r'param6|' + r'param7|' + r'param8|' + r'param9|' + r'paramcnt|' + r'params|' + r'result|' + r'timeout' + r')\b', + Name.Builtin + ), + ], + 'user-variables': [ + (r'(?i)[a-z_][a-z0-9_]*', Name.Variable), + ], + 'numeric-literals': [ + (r'(-?)([0-9]+)', bygroups(Operator, Number.Integer)), + (r'(?i)\$[0-9a-f]+', Number.Hex), + ], + 'string-literals': [ + (r'(?i)#(?:[0-9]+|\$[0-9a-f]+)', String.Char), + (r"'", String.Single, 'in-single-string'), + (r'"', String.Double, 'in-double-string'), + ], + 'in-general-string': [ + (r'\\[\\nt]', String.Escape), # Only three escapes are supported. + (r'.', String), + ], + 'in-single-string': [ + (r"'", String.Single, '#pop'), + include('in-general-string'), + ], + 'in-double-string': [ + (r'"', String.Double, '#pop'), + include('in-general-string'), + ], + 'operators': [ + (r'and|not|or|xor', Operator.Word), + (r'[!%&*+<=>^~\|\/-]+', Operator), + (r'[()]', String.Symbol), + ], + 'all-whitespace': [ + (r'\s+', Text), + ], + } + + # Turtle and Tera Term macro files share the same file extension + # but each has a recognizable and distinct syntax. + def analyse_text(text): + if re.search(TeraTermLexer.tokens['commands'][0][0], text): + return 0.01 diff --git a/src/syntax_highlighting/libs/pygments/lexers/testing.py b/src/syntax_highlighting/libs/pygments/lexers/testing.py index 1e0795b..9111a1c 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/testing.py +++ b/src/syntax_highlighting/libs/pygments/lexers/testing.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.testing ~~~~~~~~~~~~~~~~~~~~~~~ Lexers for testing languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -17,19 +16,19 @@ class GherkinLexer(RegexLexer): """ - For `Gherkin ` syntax. + For `Gherkin ` syntax. .. versionadded:: 1.2 """ name = 'Gherkin' - aliases = ['cucumber', 'gherkin'] + aliases = ['gherkin', 'cucumber'] filenames = ['*.feature'] mimetypes = ['text/x-gherkin'] - feature_keywords = u'^(기능|機能|功能|フィーチャ|خاصية|תכונה|Функціонал|Функционалност|Функционал|Фича|Особина|Могућност|Özellik|Właściwość|Tính năng|Trajto|Savybė|Požiadavka|Požadavek|Osobina|Ominaisuus|Omadus|OH HAI|Mogućnost|Mogucnost|Jellemző|Fīča|Funzionalità|Funktionalität|Funkcionalnost|Funkcionalitāte|Funcționalitate|Functionaliteit|Functionalitate|Funcionalitat|Funcionalidade|Fonctionnalité|Fitur|Feature|Egenskap|Egenskab|Crikey|Característica|Arwedd)(:)(.*)$' - feature_element_keywords = u'^(\\s*)(시나리오 개요|시나리오|배경|背景|場景大綱|場景|场景大纲|场景|劇本大綱|劇本|剧本大纲|剧本|テンプレ|シナリオテンプレート|シナリオテンプレ|シナリオアウトライン|シナリオ|سيناريو مخطط|سيناريو|الخلفية|תרחיש|תבנית תרחיש|רקע|Тарих|Сценарій|Сценарио|Сценарий структураси|Сценарий|Структура сценарію|Структура сценарија|Структура сценария|Скица|Рамка на сценарий|Пример|Предыстория|Предистория|Позадина|Передумова|Основа|Концепт|Контекст|Założenia|Wharrimean is|Tình huống|The thing of it is|Tausta|Taust|Tapausaihio|Tapaus|Szenariogrundriss|Szenario|Szablon scenariusza|Stsenaarium|Struktura scenarija|Skica|Skenario konsep|Skenario|Situācija|Senaryo taslağı|Senaryo|Scénář|Scénario|Schema dello scenario|Scenārijs pēc parauga|Scenārijs|Scenár|Scenaro|Scenariusz|Scenariul de şablon|Scenariul de sablon|Scenariu|Scenario Outline|Scenario Amlinellol|Scenario|Scenarijus|Scenarijaus šablonas|Scenarij|Scenarie|Rerefons|Raamstsenaarium|Primer|Pozadí|Pozadina|Pozadie|Plan du scénario|Plan du Scénario|Osnova scénáře|Osnova|Náčrt Scénáře|Náčrt Scenáru|Mate|MISHUN SRSLY|MISHUN|Kịch bản|Konturo de la scenaro|Kontext|Konteksts|Kontekstas|Kontekst|Koncept|Khung tình huống|Khung kịch bản|Háttér|Grundlage|Geçmiş|Forgatókönyv vázlat|Forgatókönyv|Fono|Esquema do Cenário|Esquema do Cenario|Esquema del escenario|Esquema de l\'escenari|Escenario|Escenari|Dis is what went down|Dasar|Contexto|Contexte|Contesto|Condiţii|Conditii|Cenário|Cenario|Cefndir|Bối cảnh|Blokes|Bakgrunn|Bakgrund|Baggrund|Background|B4|Antecedents|Antecedentes|All y\'all|Achtergrond|Abstrakt Scenario|Abstract Scenario)(:)(.*)$' - examples_keywords = u'^(\\s*)(예|例子|例|サンプル|امثلة|דוגמאות|Сценарији|Примери|Приклади|Мисоллар|Значения|Örnekler|Voorbeelden|Variantai|Tapaukset|Scenarios|Scenariji|Scenarijai|Příklady|Példák|Príklady|Przykłady|Primjeri|Primeri|Piemēri|Pavyzdžiai|Paraugs|Juhtumid|Exemplos|Exemples|Exemplele|Exempel|Examples|Esempi|Enghreifftiau|Ekzemploj|Eksempler|Ejemplos|EXAMPLZ|Dữ liệu|Contoh|Cobber|Beispiele)(:)(.*)$' - step_keywords = u'^(\\s*)(하지만|조건|먼저|만일|만약|단|그리고|그러면|那麼|那么|而且|當|当|前提|假設|假设|假如|假定|但是|但し|並且|并且|同時|同时|もし|ならば|ただし|しかし|かつ|و |متى |لكن |عندما |ثم |بفرض |اذاً |כאשר |וגם |בהינתן |אזי |אז |אבל |Якщо |Унда |То |Припустимо, що |Припустимо |Онда |Но |Нехай |Лекин |Когато |Када |Кад |К тому же |И |Задато |Задати |Задате |Если |Допустим |Дадено |Ва |Бирок |Аммо |Али |Але |Агар |А |І |Și |És |Zatati |Zakładając |Zadato |Zadate |Zadano |Zadani |Zadan |Youse know when youse got |Youse know like when |Yna |Ya know how |Ya gotta |Y |Wun |Wtedy |When y\'all |When |Wenn |WEN |Và |Ve |Und |Un |Thì |Then y\'all |Then |Tapi |Tak |Tada |Tad |Så |Stel |Soit |Siis |Si |Sed |Se |Quando |Quand |Quan |Pryd |Pokud |Pokiaľ |Però |Pero |Pak |Oraz |Onda |Ond |Oletetaan |Og |Och |O zaman |Når |När |Niin |Nhưng |N |Mutta |Men |Mas |Maka |Majd |Mais |Maar |Ma |Lorsque |Lorsqu\'|Kun |Kuid |Kui |Khi |Keď |Ketika |Když |Kaj |Kai |Kada |Kad |Jeżeli |Ja |Ir |I CAN HAZ |I |Ha |Givun |Givet |Given y\'all |Given |Gitt |Gegeven |Gegeben sei |Fakat |Eğer ki |Etant donné |Et |Então |Entonces |Entao |En |Eeldades |E |Duota |Dun |Donitaĵo |Donat |Donada |Do |Diyelim ki |Dengan |Den youse gotta |De |Dato |Dar |Dann |Dan |Dado |Dacă |Daca |DEN |Când |Cuando |Cho |Cept |Cand |Cal |But y\'all |But |Buh |Biết |Bet |BUT |Atès |Atunci |Atesa |Anrhegedig a |Angenommen |And y\'all |And |An |Ama |Als |Alors |Allora |Ali |Aleshores |Ale |Akkor |Aber |AN |A také |A |\* )' + feature_keywords = '^(기능|機能|功能|フィーチャ|خاصية|תכונה|Функціонал|Функционалност|Функционал|Фича|Особина|Могућност|Özellik|Właściwość|Tính năng|Trajto|Savybė|Požiadavka|Požadavek|Osobina|Ominaisuus|Omadus|OH HAI|Mogućnost|Mogucnost|Jellemző|Fīča|Funzionalità|Funktionalität|Funkcionalnost|Funkcionalitāte|Funcționalitate|Functionaliteit|Functionalitate|Funcionalitat|Funcionalidade|Fonctionnalité|Fitur|Feature|Egenskap|Egenskab|Crikey|Característica|Arwedd)(:)(.*)$' + feature_element_keywords = '^(\\s*)(시나리오 개요|시나리오|배경|背景|場景大綱|場景|场景大纲|场景|劇本大綱|劇本|剧本大纲|剧本|テンプレ|シナリオテンプレート|シナリオテンプレ|シナリオアウトライン|シナリオ|سيناريو مخطط|سيناريو|الخلفية|תרחיש|תבנית תרחיש|רקע|Тарих|Сценарій|Сценарио|Сценарий структураси|Сценарий|Структура сценарію|Структура сценарија|Структура сценария|Скица|Рамка на сценарий|Пример|Предыстория|Предистория|Позадина|Передумова|Основа|Концепт|Контекст|Założenia|Wharrimean is|Tình huống|The thing of it is|Tausta|Taust|Tapausaihio|Tapaus|Szenariogrundriss|Szenario|Szablon scenariusza|Stsenaarium|Struktura scenarija|Skica|Skenario konsep|Skenario|Situācija|Senaryo taslağı|Senaryo|Scénář|Scénario|Schema dello scenario|Scenārijs pēc parauga|Scenārijs|Scenár|Scenaro|Scenariusz|Scenariul de şablon|Scenariul de sablon|Scenariu|Scenario Outline|Scenario Amlinellol|Scenario|Scenarijus|Scenarijaus šablonas|Scenarij|Scenarie|Rerefons|Raamstsenaarium|Primer|Pozadí|Pozadina|Pozadie|Plan du scénario|Plan du Scénario|Osnova scénáře|Osnova|Náčrt Scénáře|Náčrt Scenáru|Mate|MISHUN SRSLY|MISHUN|Kịch bản|Konturo de la scenaro|Kontext|Konteksts|Kontekstas|Kontekst|Koncept|Khung tình huống|Khung kịch bản|Háttér|Grundlage|Geçmiş|Forgatókönyv vázlat|Forgatókönyv|Fono|Esquema do Cenário|Esquema do Cenario|Esquema del escenario|Esquema de l\'escenari|Escenario|Escenari|Dis is what went down|Dasar|Contexto|Contexte|Contesto|Condiţii|Conditii|Cenário|Cenario|Cefndir|Bối cảnh|Blokes|Bakgrunn|Bakgrund|Baggrund|Background|B4|Antecedents|Antecedentes|All y\'all|Achtergrond|Abstrakt Scenario|Abstract Scenario)(:)(.*)$' + examples_keywords = '^(\\s*)(예|例子|例|サンプル|امثلة|דוגמאות|Сценарији|Примери|Приклади|Мисоллар|Значения|Örnekler|Voorbeelden|Variantai|Tapaukset|Scenarios|Scenariji|Scenarijai|Příklady|Példák|Príklady|Przykłady|Primjeri|Primeri|Piemēri|Pavyzdžiai|Paraugs|Juhtumid|Exemplos|Exemples|Exemplele|Exempel|Examples|Esempi|Enghreifftiau|Ekzemploj|Eksempler|Ejemplos|EXAMPLZ|Dữ liệu|Contoh|Cobber|Beispiele)(:)(.*)$' + step_keywords = '^(\\s*)(하지만|조건|먼저|만일|만약|단|그리고|그러면|那麼|那么|而且|當|当|前提|假設|假设|假如|假定|但是|但し|並且|并且|同時|同时|もし|ならば|ただし|しかし|かつ|و |متى |لكن |عندما |ثم |بفرض |اذاً |כאשר |וגם |בהינתן |אזי |אז |אבל |Якщо |Унда |То |Припустимо, що |Припустимо |Онда |Но |Нехай |Лекин |Когато |Када |Кад |К тому же |И |Задато |Задати |Задате |Если |Допустим |Дадено |Ва |Бирок |Аммо |Али |Але |Агар |А |І |Și |És |Zatati |Zakładając |Zadato |Zadate |Zadano |Zadani |Zadan |Youse know when youse got |Youse know like when |Yna |Ya know how |Ya gotta |Y |Wun |Wtedy |When y\'all |When |Wenn |WEN |Và |Ve |Und |Un |Thì |Then y\'all |Then |Tapi |Tak |Tada |Tad |Så |Stel |Soit |Siis |Si |Sed |Se |Quando |Quand |Quan |Pryd |Pokud |Pokiaľ |Però |Pero |Pak |Oraz |Onda |Ond |Oletetaan |Og |Och |O zaman |Når |När |Niin |Nhưng |N |Mutta |Men |Mas |Maka |Majd |Mais |Maar |Ma |Lorsque |Lorsqu\'|Kun |Kuid |Kui |Khi |Keď |Ketika |Když |Kaj |Kai |Kada |Kad |Jeżeli |Ja |Ir |I CAN HAZ |I |Ha |Givun |Givet |Given y\'all |Given |Gitt |Gegeven |Gegeben sei |Fakat |Eğer ki |Etant donné |Et |Então |Entonces |Entao |En |Eeldades |E |Duota |Dun |Donitaĵo |Donat |Donada |Do |Diyelim ki |Dengan |Den youse gotta |De |Dato |Dar |Dann |Dan |Dado |Dacă |Daca |DEN |Când |Cuando |Cho |Cept |Cand |Cal |But y\'all |But |Buh |Biết |Bet |BUT |Atès |Atunci |Atesa |Anrhegedig a |Angenommen |And y\'all |And |An |Ama |Als |Alors |Allora |Ali |Aleshores |Ale |Akkor |Aber |AN |A také |A |\\* )' tokens = { 'comments': [ diff --git a/src/syntax_highlighting/libs/pygments/lexers/text.py b/src/syntax_highlighting/libs/pygments/lexers/text.py index 9b3b5fe..68e0659 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/text.py +++ b/src/syntax_highlighting/libs/pygments/lexers/text.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.text ~~~~~~~~~~~~~~~~~~~~ Lexers for non-source code file types. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -18,6 +17,7 @@ from pygments.lexers.installers import DebianControlLexer, SourcesListLexer from pygments.lexers.make import MakefileLexer, BaseMakefileLexer, CMakeLexer from pygments.lexers.haxe import HxmlLexer +from pygments.lexers.sgf import SmartGameFormatLexer from pygments.lexers.diff import DiffLexer, DarcsPatchLexer from pygments.lexers.data import YamlLexer from pygments.lexers.textfmts import IrcLogsLexer, GettextLexer, HttpLexer diff --git a/src/syntax_highlighting/libs/pygments/lexers/textedit.py b/src/syntax_highlighting/libs/pygments/lexers/textedit.py index e8856db..24fbe2e 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/textedit.py +++ b/src/syntax_highlighting/libs/pygments/lexers/textedit.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.textedit ~~~~~~~~~~~~~~~~~~~~~~~~ Lexers for languages related to text processing. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -69,8 +68,8 @@ class AwkLexer(RegexLexer): (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float), (r'0x[0-9a-fA-F]+', Number.Hex), (r'[0-9]+', Number.Integer), - (r'"(\\\\|\\"|[^"])*"', String.Double), - (r"'(\\\\|\\'|[^'])*'", String.Single), + (r'"(\\\\|\\[^\\]|[^"\\])*"', String.Double), + (r"'(\\\\|\\[^\\]|[^'\\])*'", String.Single), ] } @@ -102,9 +101,9 @@ class VimLexer(RegexLexer): (r'[ \t]+', Text), # TODO: regexes can have other delims - (r'/(\\\\|\\/|[^\n/])*/', String.Regex), - (r'"(\\\\|\\"|[^\n"])*"', String.Double), - (r"'(''|[^\n'])*'", String.Single), + (r'/[^/\\\n]*(?:\\[\s\S][^/\\\n]*)*/', String.Regex), + (r'"[^"\\\n]*(?:\\[\s\S][^"\\\n]*)*"', String.Double), + (r"'[^\n']*(?:''[^\n']*)*'", String.Single), # Who decided that doublequote was a good comment character?? (r'(?<=\s)"[^\-:.%#=*].*', Comment), diff --git a/src/syntax_highlighting/libs/pygments/lexers/textfmts.py b/src/syntax_highlighting/libs/pygments/lexers/textfmts.py index bb8124e..62d300a 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/textfmts.py +++ b/src/syntax_highlighting/libs/pygments/lexers/textfmts.py @@ -1,22 +1,23 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.textfmts ~~~~~~~~~~~~~~~~~~~~~~~~ Lexers for various text formats. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ import re -from pygments.lexer import RegexLexer, bygroups +from pygments.lexers import guess_lexer, get_lexer_by_name +from pygments.lexer import RegexLexer, bygroups, default, include from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ - Number, Generic, Literal + Number, Generic, Literal, Punctuation from pygments.util import ClassNotFound -__all__ = ['IrcLogsLexer', 'TodotxtLexer', 'HttpLexer', 'GettextLexer'] +__all__ = ['IrcLogsLexer', 'TodotxtLexer', 'HttpLexer', 'GettextLexer', + 'NotmuchLexer', 'KernelLogLexer'] class IrcLogsLexer(RegexLexer): @@ -173,13 +174,13 @@ def content_callback(self, match): tokens = { 'root': [ (r'(GET|POST|PUT|DELETE|HEAD|OPTIONS|TRACE|PATCH)( +)([^ ]+)( +)' - r'(HTTP)(/)(1\.[01])(\r?\n|\Z)', + r'(HTTP)(/)(1\.[01]|2(?:\.0)?|3)(\r?\n|\Z)', bygroups(Name.Function, Text, Name.Namespace, Text, Keyword.Reserved, Operator, Number, Text), 'headers'), - (r'(HTTP)(/)(1\.[01])( +)(\d{3})( +)([^\r\n]+)(\r?\n|\Z)', - bygroups(Keyword.Reserved, Operator, Number, Text, Number, - Text, Name.Exception, Text), + (r'(HTTP)(/)(1\.[01]|2(?:\.0)?|3)( +)(\d{3})(?:( +)([^\r\n]*))?(\r?\n|\Z)', + bygroups(Keyword.Reserved, Operator, Number, Text, Number, Text, + Name.Exception, Text), 'headers'), ], 'headers': [ @@ -266,7 +267,7 @@ class TodotxtLexer(RegexLexer): # 5. Leading project (project_regex, Project, 'incomplete'), # 6. Non-whitespace catch-all - ('\S+', IncompleteTaskText, 'incomplete'), + (r'\S+', IncompleteTaskText, 'incomplete'), ], # Parse a complete task @@ -277,9 +278,9 @@ class TodotxtLexer(RegexLexer): (context_regex, Context), (project_regex, Project), # Tokenize non-whitespace text - ('\S+', CompleteTaskText), + (r'\S+', CompleteTaskText), # Tokenize whitespace not containing a newline - ('\s+', CompleteTaskText), + (r'\s+', CompleteTaskText), ], # Parse an incomplete task @@ -290,8 +291,139 @@ class TodotxtLexer(RegexLexer): (context_regex, Context), (project_regex, Project), # Tokenize non-whitespace text - ('\S+', IncompleteTaskText), + (r'\S+', IncompleteTaskText), # Tokenize whitespace not containing a newline - ('\s+', IncompleteTaskText), + (r'\s+', IncompleteTaskText), ], } + + +class NotmuchLexer(RegexLexer): + """ + For `Notmuch `_ email text format. + + .. versionadded:: 2.5 + + Additional options accepted: + + `body_lexer` + If given, highlight the contents of the message body with the specified + lexer, else guess it according to the body content (default: ``None``). + """ + + name = 'Notmuch' + aliases = ['notmuch'] + + def _highlight_code(self, match): + code = match.group(1) + + try: + if self.body_lexer: + lexer = get_lexer_by_name(self.body_lexer) + else: + lexer = guess_lexer(code.strip()) + except ClassNotFound: + lexer = get_lexer_by_name('text') + + yield from lexer.get_tokens_unprocessed(code) + + tokens = { + 'root': [ + (r'\fmessage\{\s*', Keyword, ('message', 'message-attr')), + ], + 'message-attr': [ + (r'(\s*id:\s*)(\S+)', bygroups(Name.Attribute, String)), + (r'(\s*(?:depth|match|excluded):\s*)(\d+)', + bygroups(Name.Attribute, Number.Integer)), + (r'(\s*filename:\s*)(.+\n)', + bygroups(Name.Attribute, String)), + default('#pop'), + ], + 'message': [ + (r'\fmessage\}\n', Keyword, '#pop'), + (r'\fheader\{\n', Keyword, 'header'), + (r'\fbody\{\n', Keyword, 'body'), + ], + 'header': [ + (r'\fheader\}\n', Keyword, '#pop'), + (r'((?:Subject|From|To|Cc|Date):\s*)(.*\n)', + bygroups(Name.Attribute, String)), + (r'(.*)(\s*\(.*\))(\s*\(.*\)\n)', + bygroups(Generic.Strong, Literal, Name.Tag)), + ], + 'body': [ + (r'\fpart\{\n', Keyword, 'part'), + (r'\f(part|attachment)\{\s*', Keyword, ('part', 'part-attr')), + (r'\fbody\}\n', Keyword, '#pop'), + ], + 'part-attr': [ + (r'(ID:\s*)(\d+)', bygroups(Name.Attribute, Number.Integer)), + (r'(,\s*)((?:Filename|Content-id):\s*)([^,]+)', + bygroups(Punctuation, Name.Attribute, String)), + (r'(,\s*)(Content-type:\s*)(.+\n)', + bygroups(Punctuation, Name.Attribute, String)), + default('#pop'), + ], + 'part': [ + (r'\f(?:part|attachment)\}\n', Keyword, '#pop'), + (r'\f(?:part|attachment)\{\s*', Keyword, ('#push', 'part-attr')), + (r'^Non-text part: .*\n', Comment), + (r'(?s)(.*?(?=\f(?:part|attachment)\}\n))', _highlight_code), + ], + } + + def analyse_text(text): + return 1.0 if text.startswith('\fmessage{') else 0.0 + + def __init__(self, **options): + self.body_lexer = options.get('body_lexer', None) + RegexLexer.__init__(self, **options) + + +class KernelLogLexer(RegexLexer): + """ + For Linux Kernel log ("dmesg") output. + + .. versionadded:: 2.6 + """ + name = 'Kernel log' + aliases = ['kmsg', 'dmesg'] + filenames = ['*.kmsg', '*.dmesg'] + + tokens = { + 'root': [ + (r'^[^:]+:debug : (?=\[)', Text, 'debug'), + (r'^[^:]+:info : (?=\[)', Text, 'info'), + (r'^[^:]+:warn : (?=\[)', Text, 'warn'), + (r'^[^:]+:notice: (?=\[)', Text, 'warn'), + (r'^[^:]+:err : (?=\[)', Text, 'error'), + (r'^[^:]+:crit : (?=\[)', Text, 'error'), + (r'^(?=\[)', Text, 'unknown'), + ], + 'unknown': [ + (r'^(?=.+(warning|notice|audit|deprecated))', Text, 'warn'), + (r'^(?=.+(error|critical|fail|Bug))', Text, 'error'), + default('info'), + ], + 'base': [ + (r'\[[0-9. ]+\] ', Number), + (r'(?<=\] ).+?:', Keyword), + (r'\n', Text, '#pop'), + ], + 'debug': [ + include('base'), + (r'.+\n', Comment, '#pop') + ], + 'info': [ + include('base'), + (r'.+\n', Text, '#pop') + ], + 'warn': [ + include('base'), + (r'.+\n', Generic.Strong, '#pop') + ], + 'error': [ + include('base'), + (r'.+\n', Generic.Error, '#pop') + ] + } diff --git a/src/syntax_highlighting/libs/pygments/lexers/theorem.py b/src/syntax_highlighting/libs/pygments/lexers/theorem.py index e84a398..ec55a32 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/theorem.py +++ b/src/syntax_highlighting/libs/pygments/lexers/theorem.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.theorem ~~~~~~~~~~~~~~~~~~~~~~~ Lexers for theorem-proving languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -30,6 +29,8 @@ class CoqLexer(RegexLexer): filenames = ['*.v'] mimetypes = ['text/x-coq'] + flags = re.UNICODE + keywords1 = ( # Vernacular commands 'Section', 'Module', 'End', 'Require', 'Import', 'Export', 'Variable', @@ -42,6 +43,7 @@ class CoqLexer(RegexLexer): 'Canonical', 'Coercion', 'Theorem', 'Lemma', 'Corollary', 'Proposition', 'Fact', 'Remark', 'Example', 'Proof', 'Goal', 'Save', 'Qed', 'Defined', 'Hint', 'Resolve', 'Rewrite', 'View', 'Search', + 'Abort', 'Admitted', 'Show', 'Print', 'Printing', 'All', 'Graph', 'Projections', 'inside', 'outside', 'Check', 'Global', 'Instance', 'Class', 'Existing', 'Universe', 'Polymorphic', 'Monomorphic', 'Context' @@ -54,7 +56,7 @@ class CoqLexer(RegexLexer): ) keywords3 = ( # Sorts - 'Type', 'Prop', + 'Type', 'Prop', 'SProp', ) keywords4 = ( # Tactics @@ -93,12 +95,12 @@ class CoqLexer(RegexLexer): '<->', '=', '>', '>]', r'>\}', r'\?', r'\?\?', r'\[', r'\[<', r'\[>', r'\[\|', ']', '_', '`', r'\{', r'\{<', r'\|', r'\|]', r'\}', '~', '=>', r'/\\', r'\\/', r'\{\|', r'\|\}', - u'Π', u'λ', + # 'Π', 'Σ', # Not defined in the standard library + 'λ', '¬', '∧', '∨', '∀', '∃', '→', '↔', '≠', '≤', '≥', ) operators = r'[!$%&*+\./:<=>?@^|~-]' prefix_syms = r'[!?~]' infix_syms = r'[=<>@^|&+\*/$%-]' - primitives = ('unit', 'nat', 'bool', 'string', 'ascii', 'list') tokens = { 'root': [ @@ -115,7 +117,6 @@ class CoqLexer(RegexLexer): (r'\b([A-Z][\w\']*)', Name), (r'(%s)' % '|'.join(keyopts[::-1]), Operator), (r'(%s|%s)?%s' % (infix_syms, prefix_syms, operators), Operator), - (r'\b(%s)\b' % '|'.join(primitives), Keyword.Type), (r"[^\W\d][\w']*", Name), @@ -125,14 +126,15 @@ class CoqLexer(RegexLexer): (r'0[bB][01][01_]*', Number.Bin), (r'-?\d[\d_]*(.[\d_]*)?([eE][+\-]?\d[\d_]*)', Number.Float), - (r"'(?:(\\[\\\"'ntbr ])|(\\[0-9]{3})|(\\x[0-9a-fA-F]{2}))'", - String.Char), + (r"'(?:(\\[\\\"'ntbr ])|(\\[0-9]{3})|(\\x[0-9a-fA-F]{2}))'", String.Char), + (r"'.'", String.Char), (r"'", Keyword), # a stray quote is another syntax element (r'"', String.Double, 'string'), (r'[~?][a-z][\w\']*:', Name), + (r'\S', Name.Builtin.Pseudo), ], 'comment': [ (r'[^(*)]+', Comment), @@ -156,8 +158,8 @@ class CoqLexer(RegexLexer): } def analyse_text(text): - if text.startswith('(*'): - return True + if 'Qed' in text and 'Proof' in text: + return 1 class IsabelleLexer(RegexLexer): @@ -390,69 +392,87 @@ class LeanLexer(RegexLexer): flags = re.MULTILINE | re.UNICODE - keywords1 = ( - 'import', 'abbreviation', 'opaque_hint', 'tactic_hint', 'definition', - 'renaming', 'inline', 'hiding', 'exposing', 'parameter', 'parameters', - 'conjecture', 'hypothesis', 'lemma', 'corollary', 'variable', 'variables', - 'theorem', 'axiom', 'inductive', 'structure', 'universe', 'alias', - 'help', 'options', 'precedence', 'postfix', 'prefix', 'calc_trans', - 'calc_subst', 'calc_refl', 'infix', 'infixl', 'infixr', 'notation', 'eval', - 'check', 'exit', 'coercion', 'end', 'private', 'using', 'namespace', - 'including', 'instance', 'section', 'context', 'protected', 'expose', - 'export', 'set_option', 'add_rewrite', 'extends', 'open', 'example', - 'constant', 'constants', 'print', 'opaque', 'reducible', 'irreducible', - ) - - keywords2 = ( - 'forall', 'fun', 'Pi', 'obtain', 'from', 'have', 'show', 'assume', - 'take', 'let', 'if', 'else', 'then', 'by', 'in', 'with', 'begin', - 'proof', 'qed', 'calc', 'match', - ) - - keywords3 = ( - # Sorts - 'Type', 'Prop', - ) - - operators = ( - u'!=', u'#', u'&', u'&&', u'*', u'+', u'-', u'/', u'@', u'!', u'`', - u'-.', u'->', u'.', u'..', u'...', u'::', u':>', u';', u';;', u'<', - u'<-', u'=', u'==', u'>', u'_', u'|', u'||', u'~', u'=>', u'<=', u'>=', - u'/\\', u'\\/', u'∀', u'Π', u'λ', u'↔', u'∧', u'∨', u'≠', u'≤', u'≥', - u'¬', u'⁻¹', u'⬝', u'▸', u'→', u'∃', u'ℕ', u'ℤ', u'≈', u'×', u'⌞', - u'⌟', u'≡', u'⟨', u'⟩', - ) - - punctuation = (u'(', u')', u':', u'{', u'}', u'[', u']', u'⦃', u'⦄', - u':=', u',') - tokens = { 'root': [ (r'\s+', Text), + (r'/--', String.Doc, 'docstring'), (r'/-', Comment, 'comment'), (r'--.*?$', Comment.Single), - (words(keywords1, prefix=r'\b', suffix=r'\b'), Keyword.Namespace), - (words(keywords2, prefix=r'\b', suffix=r'\b'), Keyword), - (words(keywords3, prefix=r'\b', suffix=r'\b'), Keyword.Type), - (words(operators), Name.Builtin.Pseudo), - (words(punctuation), Operator), - (u"[A-Za-z_\u03b1-\u03ba\u03bc-\u03fb\u1f00-\u1ffe\u2100-\u214f]" - u"[A-Za-z_'\u03b1-\u03ba\u03bc-\u03fb\u1f00-\u1ffe\u2070-\u2079" - u"\u207f-\u2089\u2090-\u209c\u2100-\u214f0-9]*", Name), + (words(( + 'import', 'renaming', 'hiding', + 'namespace', + 'local', + 'private', 'protected', 'section', + 'include', 'omit', 'section', + 'protected', 'export', + 'open', + 'attribute', + ), prefix=r'\b', suffix=r'\b'), Keyword.Namespace), + (words(( + 'lemma', 'theorem', 'def', 'definition', 'example', + 'axiom', 'axioms', 'constant', 'constants', + 'universe', 'universes', + 'inductive', 'coinductive', 'structure', 'extends', + 'class', 'instance', + 'abbreviation', + + 'noncomputable theory', + + 'noncomputable', 'mutual', 'meta', + + 'attribute', + + 'parameter', 'parameters', + 'variable', 'variables', + + 'reserve', 'precedence', + 'postfix', 'prefix', 'notation', 'infix', 'infixl', 'infixr', + + 'begin', 'by', 'end', + + 'set_option', + 'run_cmd', + ), prefix=r'\b', suffix=r'\b'), Keyword.Declaration), + (r'@\[[^\]]*\]', Keyword.Declaration), + (words(( + 'forall', 'fun', 'Pi', 'from', 'have', 'show', 'assume', 'suffices', + 'let', 'if', 'else', 'then', 'in', 'with', 'calc', 'match', + 'do' + ), prefix=r'\b', suffix=r'\b'), Keyword), + (words(('sorry', 'admit'), prefix=r'\b', suffix=r'\b'), Generic.Error), + (words(('Sort', 'Prop', 'Type'), prefix=r'\b', suffix=r'\b'), Keyword.Type), + (words(( + '#eval', '#check', '#reduce', '#exit', + '#print', '#help', + ), suffix=r'\b'), Keyword), + (words(( + '(', ')', ':', '{', '}', '[', ']', '⟨', '⟩', '‹', '›', '⦃', '⦄', ':=', ',', + )), Operator), + (r'[A-Za-z_\u03b1-\u03ba\u03bc-\u03fb\u1f00-\u1ffe\u2100-\u214f]' + r'[.A-Za-z_\'\u03b1-\u03ba\u03bc-\u03fb\u1f00-\u1ffe\u2070-\u2079' + r'\u207f-\u2089\u2090-\u209c\u2100-\u214f0-9]*', Name), + (r'0x[A-Za-z0-9]+', Number.Integer), + (r'0b[01]+', Number.Integer), (r'\d+', Number.Integer), (r'"', String.Double, 'string'), - (r'[~?][a-z][\w\']*:', Name.Variable) + (r"'(?:(\\[\\\"'nt])|(\\x[0-9a-fA-F]{2})|(\\u[0-9a-fA-F]{4})|.)'", String.Char), + (r'[~?][a-z][\w\']*:', Name.Variable), + (r'\S', Name.Builtin.Pseudo), ], 'comment': [ - # Multiline Comments (r'[^/-]', Comment.Multiline), (r'/-', Comment.Multiline, '#push'), (r'-/', Comment.Multiline, '#pop'), (r'[/-]', Comment.Multiline) ], + 'docstring': [ + (r'[^/-]', String.Doc), + (r'-/', String.Doc, '#pop'), + (r'[/-]', String.Doc) + ], 'string': [ (r'[^\\"]+', String.Double), - (r'\\[n"\\]', String.Escape), + (r"(?:(\\[\\\"'nt])|(\\x[0-9a-fA-F]{2})|(\\u[0-9a-fA-F]{4}))", String.Escape), ('"', String.Double, '#pop'), ], } diff --git a/src/syntax_highlighting/libs/pygments/lexers/thingsdb.py b/src/syntax_highlighting/libs/pygments/lexers/thingsdb.py new file mode 100644 index 0000000..ffb1825 --- /dev/null +++ b/src/syntax_highlighting/libs/pygments/lexers/thingsdb.py @@ -0,0 +1,118 @@ +# -*- coding: utf-8 -*- +""" + pygments.lexers.thingsdb + ~~~~~~~~~~~~~~~~~~~~~~ + + Lexers for the ThingsDB language. + + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re +from pygments.lexer import RegexLexer, include, bygroups +from pygments.token import Comment, Keyword, Name, Number, String, Text, \ + Operator, Punctuation, Whitespace + +__all__ = ['ThingsDBLexer'] + + +class ThingsDBLexer(RegexLexer): + """ + Lexer for the ThingsDB programming language. + + .. versionadded:: 2.9 + """ + name = 'ThingsDB' + aliases = ['ti', 'thingsdb'] + filenames = ['*.ti'] + + tokens = { + 'root': [ + include('expression'), + ], + 'expression': [ + include('comments'), + include('whitespace'), + + # numbers + (r'[-+]?0b[01]+', Number.Bin), + (r'[-+]?0o[0-8]+', Number.Oct), + (r'([-+]?0x[0-9a-fA-F]+)', Number.Hex), + (r'[-+]?[0-9]+', Number.Integer), + (r'[-+]?((inf|nan)([^0-9A-Za-z_]|$)|[0-9]*\.[0-9]+(e[+-][0-9]+)?)', + Number.Float), + + # strings + (r'(?:"(?:[^"]*)")+', String.Double), + (r"(?:'(?:[^']*)')+", String.Single), + + # literals + (r'(true|false|nil)\b', Keyword.Constant), + + # regular expressions + (r'(/[^/\\]*(?:\\.[^/\\]*)*/i?)', String.Regex), + + # thing id's + (r'#[0-9]+', Comment.Preproc), + + # name, assignments and functions + include('names'), + + (r'[(){}\[\],;]', Punctuation), + (r'[+\-*/%&|<>^!~@=:?]', Operator), + ], + 'names': [ + (r'(\.)' + r'(add|call|contains|del|endswith|extend|filter|find|findindex|' + r'get|has|id|indexof|keys|len|lower|map|pop|push|remove|set|sort|' + r'splice|startswith|test|unwrap|upper|values|wrap)' + r'(\()', + bygroups(Name.Function, Name.Function, Punctuation), 'arguments'), + (r'(array|assert|assert_err|auth_err|backup_info|backups_info|' + r'bad_data_err|bool|closure|collection_info|collections_info|' + r'counters|deep|del_backup|del_collection|del_expired|del_node|' + r'del_procedure|del_token|del_type|del_user|err|float|' + r'forbidden_err|grant|int|isarray|isascii|isbool|isbytes|iserr|' + r'isfloat|isinf|isint|islist|isnan|isnil|israw|isset|isstr|' + r'isthing|istuple|isutf8|lookup_err|max_quota_err|mod_type|new|' + r'new_backup|new_collection|new_node|new_procedure|new_token|' + r'new_type|new_user|node_err|node_info|nodes_info|now|' + r'num_arguments_err|operation_err|overflow_err|procedure_doc|' + r'procedure_info|procedures_info|raise|refs|rename_collection|' + r'rename_user|reset_counters|return|revoke|run|set_log_level|set|' + r'set_quota|set_type|shutdown|str|syntax_err|thing|try|type|' + r'type_err|type_count|type_info|types_info|user_info|users_info|' + r'value_err|wse|zero_div_err)' + r'(\()', + bygroups(Name.Function, Punctuation), + 'arguments'), + (r'(\.[A-Za-z_][0-9A-Za-z_]*)' + r'(\s*)(=)', + bygroups(Name.Attribute, Text, Operator)), + (r'\.[A-Za-z_][0-9A-Za-z_]*', Name.Attribute), + (r'([A-Za-z_][0-9A-Za-z_]*)(\s*)(=)', + bygroups(Name.Variable, Text, Operator)), + (r'[A-Za-z_][0-9A-Za-z_]*', Name.Variable), + ], + 'whitespace': [ + (r'\n', Whitespace), + (r'\s+', Whitespace), + ], + 'comments': [ + (r'//(.*?)\n', Comment.Single), + (r'/\*', Comment.Multiline, 'comment'), + ], + 'comment': [ + (r'[^*/]+', Comment.Multiline), + (r'/\*', Comment.Multiline, '#push'), + (r'\*/', Comment.Multiline, '#pop'), + (r'[*/]', Comment.Multiline), + ], + 'arguments': [ + include('expression'), + (',', Punctuation), + (r'\(', Punctuation, '#push'), + (r'\)', Punctuation, '#pop'), + ], + } diff --git a/src/syntax_highlighting/libs/pygments/lexers/tnt.py b/src/syntax_highlighting/libs/pygments/lexers/tnt.py new file mode 100644 index 0000000..e6e7196 --- /dev/null +++ b/src/syntax_highlighting/libs/pygments/lexers/tnt.py @@ -0,0 +1,272 @@ +""" + pygments.lexers.tnt + ~~~~~~~~~~~~~~~~~~~ + + Lexer for Typographic Number Theory. + + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re + +from pygments.lexer import Lexer +from pygments.token import Text, Comment, Operator, Keyword, Name, Number, \ + Punctuation, Error + +__all__ = ['TNTLexer'] + + +class TNTLexer(Lexer): + """ + Lexer for Typographic Number Theory, as described in the book + Gödel, Escher, Bach, by Douglas R. Hofstadter, + or as summarized here: + https://github.com/Kenny2github/language-tnt/blob/master/README.md#summary-of-tnt + + .. versionadded:: 2.7 + """ + + name = 'Typographic Number Theory' + aliases = ['tnt'] + filenames = ['*.tnt'] + + cur = [] + + LOGIC = set('⊃→]&∧^|∨Vv') + OPERATORS = set('+.⋅*') + VARIABLES = set('abcde') + PRIMES = set("'′") + NEGATORS = set('~!') + QUANTIFIERS = set('AE∀∃') + NUMBERS = set('0123456789') + WHITESPACE = set('\t \v\n') + + RULES = re.compile('''(?xi) + joining | separation | double-tilde | fantasy\\ rule + | carry[- ]over(?:\\ of)?(?:\\ line)?\\ ([0-9]+) | detachment + | contrapositive | De\\ Morgan | switcheroo + | specification | generalization | interchange + | existence | symmetry | transitivity + | add\\ S | drop\\ S | induction + | axiom\\ ([1-5]) | premise | push | pop + ''') + LINENOS = re.compile(r'(?:[0-9]+)(?:(?:, ?|,? and )(?:[0-9]+))*') + COMMENT = re.compile(r'\[[^\n\]]+\]') + + def __init__(self, *args, **kwargs): + Lexer.__init__(self, *args, **kwargs) + self.cur = [] + + def whitespace(self, start, text, required=False): + """Tokenize whitespace.""" + end = start + try: + while text[end] in self.WHITESPACE: + end += 1 + except IndexError: + end = len(text) + if required and end == start: + raise AssertionError + if end != start: + self.cur.append((start, Text, text[start:end])) + return end + + def variable(self, start, text): + """Tokenize a variable.""" + if text[start] not in self.VARIABLES: + raise AssertionError + end = start+1 + while text[end] in self.PRIMES: + end += 1 + self.cur.append((start, Name.Variable, text[start:end])) + return end + + def term(self, start, text): + """Tokenize a term.""" + if text[start] == 'S': # S...S(...) or S...0 + end = start+1 + while text[end] == 'S': + end += 1 + self.cur.append((start, Number.Integer, text[start:end])) + return self.term(end, text) + if text[start] == '0': # the singleton 0 + self.cur.append((start, Number.Integer, text[start])) + return start+1 + if text[start] in self.VARIABLES: # a''... + return self.variable(start, text) + if text[start] == '(': # (...+...) + self.cur.append((start, Punctuation, text[start])) + start = self.term(start+1, text) + if text[start] not in self.OPERATORS: + raise AssertionError + self.cur.append((start, Operator, text[start])) + start = self.term(start+1, text) + if text[start] != ')': + raise AssertionError + self.cur.append((start, Punctuation, text[start])) + return start+1 + raise AssertionError # no matches + + def formula(self, start, text): + """Tokenize a formula.""" + if text[start] in self.NEGATORS: # ~<...> + end = start+1 + while text[end] in self.NEGATORS: + end += 1 + self.cur.append((start, Operator, text[start:end])) + return self.formula(end, text) + if text[start] in self.QUANTIFIERS: # Aa:<...> + self.cur.append((start, Keyword.Declaration, text[start])) + start = self.variable(start+1, text) + if text[start] != ':': + raise AssertionError + self.cur.append((start, Punctuation, text[start])) + return self.formula(start+1, text) + if text[start] == '<': # <...&...> + self.cur.append((start, Punctuation, text[start])) + start = self.formula(start+1, text) + if text[start] not in self.LOGIC: + raise AssertionError + self.cur.append((start, Operator, text[start])) + start = self.formula(start+1, text) + if text[start] != '>': + raise AssertionError + self.cur.append((start, Punctuation, text[start])) + return start+1 + # ...=... + start = self.term(start, text) + if text[start] != '=': + raise AssertionError + self.cur.append((start, Operator, text[start])) + start = self.term(start+1, text) + return start + + def rule(self, start, text): + """Tokenize a rule.""" + match = self.RULES.match(text, start) + if match is None: + raise AssertionError + groups = sorted(match.regs[1:]) # exclude whole match + for group in groups: + if group[0] >= 0: # this group matched + self.cur.append((start, Keyword, text[start:group[0]])) + self.cur.append((group[0], Number.Integer, + text[group[0]:group[1]])) + if group[1] != match.end(): + self.cur.append((group[1], Keyword, + text[group[1]:match.end()])) + break + else: + self.cur.append((start, Keyword, text[start:match.end()])) + return match.end() + + def lineno(self, start, text): + """Tokenize a line referral.""" + end = start + while text[end] not in self.NUMBERS: + end += 1 + self.cur.append((start, Punctuation, text[start])) + self.cur.append((start+1, Text, text[start+1:end])) + start = end + match = self.LINENOS.match(text, start) + if match is None: + raise AssertionError + if text[match.end()] != ')': + raise AssertionError + self.cur.append((match.start(), Number.Integer, match.group(0))) + self.cur.append((match.end(), Punctuation, text[match.end()])) + return match.end() + 1 + + def error_till_line_end(self, start, text): + """Mark everything from ``start`` to the end of the line as Error.""" + end = start + try: + while text[end] != '\n': # there's whitespace in rules + end += 1 + except IndexError: + end = len(text) + if end != start: + self.cur.append((start, Error, text[start:end])) + end = self.whitespace(end, text) + return end + + def get_tokens_unprocessed(self, text): + """Returns a list of TNT tokens.""" + self.cur = [] + start = end = self.whitespace(0, text) + while start <= end < len(text): + try: + # try line number + while text[end] in self.NUMBERS: + end += 1 + if end != start: # actual number present + self.cur.append((start, Number.Integer, text[start:end])) + # whitespace is required after a line number + orig = len(self.cur) + try: + start = end = self.whitespace(end, text, True) + except AssertionError: + del self.cur[orig:] + start = end = self.error_till_line_end(end, text) + continue + # at this point it could be a comment + match = self.COMMENT.match(text, start) + if match is not None: + self.cur.append((start, Comment, text[start:match.end()])) + start = end = match.end() + # anything after the closing bracket is invalid + start = end = self.error_till_line_end(start, text) + # do not attempt to process the rest + continue + del match + if text[start] in '[]': # fantasy push or pop + self.cur.append((start, Keyword, text[start])) + start += 1 + end += 1 + else: + # one formula, possibly containing subformulae + orig = len(self.cur) + try: + start = end = self.formula(start, text) + except (AssertionError, RecursionError): # not well-formed + del self.cur[orig:] + while text[end] not in self.WHITESPACE: + end += 1 + self.cur.append((start, Error, text[start:end])) + start = end + # skip whitespace after formula + orig = len(self.cur) + try: + start = end = self.whitespace(end, text, True) + except AssertionError: + del self.cur[orig:] + start = end = self.error_till_line_end(start, text) + continue + # rule proving this formula a theorem + orig = len(self.cur) + try: + start = end = self.rule(start, text) + except AssertionError: + del self.cur[orig:] + start = end = self.error_till_line_end(start, text) + continue + # skip whitespace after rule + start = end = self.whitespace(end, text) + # line marker + if text[start] == '(': + orig = len(self.cur) + try: + start = end = self.lineno(start, text) + except AssertionError: + del self.cur[orig:] + start = end = self.error_till_line_end(start, text) + continue + start = end = self.whitespace(start, text) + except IndexError: + try: + del self.cur[orig:] + except NameError: + pass # if orig was never defined, fine + self.error_till_line_end(start, text) + return self.cur diff --git a/src/syntax_highlighting/libs/pygments/lexers/trafficscript.py b/src/syntax_highlighting/libs/pygments/lexers/trafficscript.py index 4254228..e796a27 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/trafficscript.py +++ b/src/syntax_highlighting/libs/pygments/lexers/trafficscript.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.trafficscript ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Lexer for RiverBed's TrafficScript (RTS) language. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -24,7 +23,7 @@ class RtsLexer(RegexLexer): .. versionadded:: 2.1 """ name = 'TrafficScript' - aliases = ['rts','trafficscript'] + aliases = ['trafficscript', 'rts'] filenames = ['*.rts'] tokens = { diff --git a/src/syntax_highlighting/libs/pygments/lexers/typoscript.py b/src/syntax_highlighting/libs/pygments/lexers/typoscript.py index e358af0..b2e4299 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/typoscript.py +++ b/src/syntax_highlighting/libs/pygments/lexers/typoscript.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.typoscript ~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -14,7 +13,7 @@ `TypoScriptHtmlDataLexer` Lexer that highlights markers, constants and registers within html tags. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -108,14 +107,11 @@ class TypoScriptLexer(RegexLexer): name = 'TypoScript' aliases = ['typoscript'] - filenames = ['*.ts', '*.txt'] + filenames = ['*.typoscript'] mimetypes = ['text/x-typoscript'] flags = re.DOTALL | re.MULTILINE - # Slightly higher than TypeScript (which is 0). - priority = 0.1 - tokens = { 'root': [ include('comment'), @@ -132,7 +128,7 @@ class TypoScriptLexer(RegexLexer): ], 'keywords': [ # Conditions - (r'(\[)(?i)(browser|compatVersion|dayofmonth|dayofweek|dayofyear|' + (r'(?i)(\[)(browser|compatVersion|dayofmonth|dayofweek|dayofyear|' r'device|ELSE|END|GLOBAL|globalString|globalVar|hostname|hour|IP|' r'language|loginUser|loginuser|minute|month|page|PIDinRootline|' r'PIDupinRootline|system|treeLevel|useragent|userFunc|usergroup|' @@ -172,7 +168,7 @@ class TypoScriptLexer(RegexLexer): 'html': [ (r'<\S[^\n>]*>', using(TypoScriptHtmlDataLexer)), (r'&[^;\n]*;', String), - (r'(_CSS_DEFAULT_STYLE)(\s*)(\()(?s)(.*(?=\n\)))', + (r'(?s)(_CSS_DEFAULT_STYLE)(\s*)(\()(.*(?=\n\)))', bygroups(Name.Class, Text, String.Symbol, using(TypoScriptCssDataLexer))), ], 'literal': [ @@ -220,7 +216,3 @@ class TypoScriptLexer(RegexLexer): (r'[\w"\-!/&;]+', Text), ], } - - def analyse_text(text): - if '@|>>@|\.>|->|===|~===|\*\*|\+\+|--|\.|~==|~=|<=|>=|==|' + r'=|<<=|<<|>>=|>>|:=:|:=|->|<->|\+:=|\|', Operator), + (r'"(?:[^\\"]|\\.)*"', String), + (r"'(?:[^\\']|\\.)*'", String.Character), + (r'[*<>+=/&!?@~\\-]', Operator), + (r'\^', Operator), + (r'(\w+)(\s*|[(,])', bygroups(Name, using(this))), + (r"[\[\]]", Punctuation), + (r"<>|=>|[()|:;,.'`{}%&?]", Punctuation), + (r'\n+', Text), + ], + 'numbers': [ + (r'\b([+-]?([2-9]|[12][0-9]|3[0-6])[rR][0-9a-zA-Z]+)\b', Number.Hex), + (r'[+-]?[0-9]*\.([0-9]*)([Ee][+-]?[0-9]*)?', Number.Float), + (r'\b([+-]?[0-9]+[KMGTPkmgtp]?)\b', Number.Integer), + ], + 'subprogram': [ + (r'\(', Punctuation, ('#pop', 'formal_part')), + (r';', Punctuation, '#pop'), + (r'"[^"]+"|\w+', Name.Function), + include('root'), + ], + 'type_def': [ + (r'\(', Punctuation, 'formal_part'), + ], + 'formal_part': [ + (r'\)', Punctuation, '#pop'), + (r'\w+', Name.Variable), + (r',', Punctuation), + (r'(:string|:integer|:real)\b', Keyword.Reserved), + include('root'), + ], + } + + +class IconLexer(RegexLexer): + """ + Lexer for Icon. + + .. versionadded:: 1.6 + """ + name = 'Icon' + aliases = ['icon'] + filenames = ['*.icon', '*.ICON'] + mimetypes = [] + flags = re.MULTILINE + + tokens = { + 'root': [ + (r'[^\S\n]+', Text), + (r'#.*?\n', Comment.Single), + (r'[^\S\n]+', Text), + (r'class|method|procedure', Keyword.Declaration, 'subprogram'), + (r'(record)(\s+)(\w+)', + bygroups(Keyword.Declaration, Text, Keyword.Type), 'type_def'), + (r'(#line|\$C|\$Cend|\$define|\$else|\$endif|\$error|\$ifdef|' + r'\$ifndef|\$include|\$line|\$undef)\b', Keyword.PreProc), + (r'(&null|&fail)\b', Keyword.Constant), + (r'&allocated|&ascii|&clock|&collections|&column|&col|&control|' + r'&cset|¤t|&dateline|&date|&digits|&dump|' + r'&errno|&errornumber|&errortext|&errorvalue|&error|&errout|' + r'&eventcode|&eventvalue|&eventsource|&e|' + r'&features|&file|&host|&input|&interval|&lcase|&letters|' + r'&level|&line|&ldrag|&lpress|&lrelease|' + r'&main|&mdrag|&meta|&mpress|&mrelease|&now|&output|' + r'&phi|&pick|&pi|&pos|&progname|' + r'&random|&rdrag|®ions|&resize|&row|&rpress|&rrelease|' + r'&shift|&source|&storage|&subject|' + r'&time|&trace|&ucase|&version|' + r'&window|&x|&y', Keyword.Reserved), + (r'(by|of|not|to)\b', Keyword.Reserved), + (r'(global|local|static)\b', Keyword.Reserved), + (r'link', Keyword.Declaration), + (words(( + 'break', 'case', 'create', 'default', 'end', 'all', + 'do', 'else', 'every', 'fail', 'if', 'initial', + 'invocable', 'next', + 'repeat', 'return', 'suspend', + 'then', 'until', 'while'), prefix=r'\b', suffix=r'\b'), + Keyword.Reserved), + (words(( + 'abs', 'acos', 'Active', 'Alert', 'any', + 'args', 'array', 'asin', 'atan', 'atanh', 'Attrib', + 'bal', 'Bg', + 'callout', 'center', 'char', 'chdir', 'chmod', 'chown', 'chroot', + 'Clip', 'Clone', 'close', 'cofail', 'collect', + 'Color', 'ColorValue', 'condvar', 'copy', + 'CopyArea', 'cos', 'Couple', 'crypt', 'cset', 'ctime', + 'delay', 'delete', 'detab', 'display', 'DrawArc', + 'DrawCircle', 'DrawCube', 'DrawCurve', 'DrawCylinder', + 'DrawDisk', 'DrawImage', 'DrawLine', 'DrawPoint', 'DrawPolygon', + 'DrawRectangle', 'DrawSegment', 'DrawSphere', 'DrawString', + 'DrawTorus', 'dtor', + 'entab', 'EraseArea', 'errorclear', 'Event', 'eventmask', + 'EvGet', 'EvSend', 'exec', 'exit', 'exp', 'Eye', + 'fcntl', 'fdup', 'fetch', 'Fg', 'fieldnames', + 'FillArc', 'FillCircle', 'FillPolygon', + 'FillRectangle', 'find', 'flock', 'flush', 'Font', + 'FreeColor', 'FreeSpace', 'function', + 'get', 'getch', 'getche', 'getenv', + 'GetSpace', 'gettimeofday', + 'getuid', 'globalnames', 'GotoRC', 'GotoXY', 'gtime', 'hardlink', + 'iand', 'icom', 'IdentityMatrix', 'image', 'InPort', 'insert', + 'Int86', 'integer', 'ioctl', 'ior', 'ishift', 'istate', 'ixor', + 'kbhit', 'key', 'keyword', 'kill', + 'left', 'Len', 'list', 'load', 'loadfunc', 'localnames', + 'lock', 'log', 'Lower', 'lstat', + 'many', 'map', 'match', 'MatrixMode', 'max', 'member', + 'membernames', 'methodnames', 'methods', 'min', 'mkdir', 'move', + 'MultMatrix', 'mutex', + 'name', 'NewColor', 'Normals', 'numeric', + 'open', 'opencl', 'oprec', 'ord', 'OutPort', + 'PaletteChars', 'PaletteColor', 'PaletteKey', 'paramnames', + 'parent', 'Pattern', 'Peek', 'Pending', 'pipe', 'Pixel', + 'Poke', 'pop', 'PopMatrix', 'Pos', 'pos', + 'proc', 'pull', 'push', 'PushMatrix', 'PushRotate', 'PushScale', + 'PushTranslate', 'put', + 'QueryPointer', + 'Raise', 'read', 'ReadImage', 'readlink', 'reads', 'ready', + 'real', 'receive', 'Refresh', 'Rem', 'remove', 'rename', + 'repl', 'reverse', 'right', 'rmdir', 'Rotate', 'Rpos', + 'rtod', 'runerr', + 'save', 'Scale', 'seek', 'select', 'send', 'seq', + 'serial', 'set', 'setenv', + 'setuid', 'signal', 'sin', 'sort', 'sortf', + 'spawn', 'sql', 'sqrt', 'stat', 'staticnames', 'stop', + 'string', 'structure', 'Swi', + 'symlink', 'sys_errstr', 'system', 'syswrite', + 'tab', 'table', 'tan', + 'Texcoord', 'Texture', 'TextWidth', 'Translate', + 'trap', 'trim', 'truncate', 'trylock', 'type', + 'umask', 'Uncouple', 'unlock', 'upto', 'utime', + 'variable', + 'wait', 'WAttrib', 'WDefault', 'WFlush', 'where', + 'WinAssociate', 'WinButton', 'WinColorDialog', 'WindowContents', + 'WinEditRegion', 'WinFontDialog', 'WinMenuBar', 'WinOpenDialog', + 'WinPlayMedia', 'WinSaveDialog', 'WinScrollBar', 'WinSelectDialog', + 'write', 'WriteImage', 'writes', 'WSection', + 'WSync'), prefix=r'\b', suffix=r'\b'), + Name.Function), + include('numbers'), + (r'===|~===|\*\*|\+\+|--|\.|==|~==|<=|>=|=|~=|<<=|<<|>>=|>>|' + r':=:|:=|<->|<-|\+:=|\|\||\|', Operator), + (r'"(?:[^\\"]|\\.)*"', String), + (r"'(?:[^\\']|\\.)*'", String.Character), + (r'[*<>+=/&!?@~\\-]', Operator), + (r'(\w+)(\s*|[(,])', bygroups(Name, using(this))), + (r"[\[\]]", Punctuation), + (r"<>|=>|[()|:;,.'`{}%\^&?]", Punctuation), + (r'\n+', Text), + ], + 'numbers': [ + (r'\b([+-]?([2-9]|[12][0-9]|3[0-6])[rR][0-9a-zA-Z]+)\b', Number.Hex), + (r'[+-]?[0-9]*\.([0-9]*)([Ee][+-]?[0-9]*)?', Number.Float), + (r'\b([+-]?[0-9]+[KMGTPkmgtp]?)\b', Number.Integer), + ], + 'subprogram': [ + (r'\(', Punctuation, ('#pop', 'formal_part')), + (r';', Punctuation, '#pop'), + (r'"[^"]+"|\w+', Name.Function), + include('root'), + ], + 'type_def': [ + (r'\(', Punctuation, 'formal_part'), + ], + 'formal_part': [ + (r'\)', Punctuation, '#pop'), + (r'\w+', Name.Variable), + (r',', Punctuation), + (r'(:string|:integer|:real)\b', Keyword.Reserved), + include('root'), + ], + } + + +class UcodeLexer(RegexLexer): + """ + Lexer for Icon ucode files. + + .. versionadded:: 2.4 + """ + name = 'ucode' + aliases = ['ucode'] + filenames = ['*.u', '*.u1', '*.u2'] + mimetypes = [] + flags = re.MULTILINE + + tokens = { + 'root': [ + (r'(#.*\n)', Comment), + (words(( + 'con', 'declend', 'end', + 'global', + 'impl', 'invocable', + 'lab', 'link', 'local', + 'record', + 'uid', 'unions', + 'version'), + prefix=r'\b', suffix=r'\b'), + Name.Function), + (words(( + 'colm', 'filen', 'line', 'synt'), + prefix=r'\b', suffix=r'\b'), + Comment), + (words(( + 'asgn', + 'bang', 'bscan', + 'cat', 'ccase', 'chfail', + 'coact', 'cofail', 'compl', + 'coret', 'create', 'cset', + 'diff', 'div', 'dup', + 'efail', 'einit', 'end', 'eqv', 'eret', + 'error', 'escan', 'esusp', + 'field', + 'goto', + 'init', 'int', 'inter', + 'invoke', + 'keywd', + 'lconcat', 'lexeq', 'lexge', + 'lexgt', 'lexle', 'lexlt', 'lexne', + 'limit', 'llist', 'lsusp', + 'mark', 'mark0', 'minus', 'mod', 'mult', + 'neg', 'neqv', 'nonnull', 'noop', 'null', + 'number', 'numeq', 'numge', 'numgt', + 'numle', 'numlt', 'numne', + 'pfail', 'plus', 'pnull', 'pop', 'power', + 'pret', 'proc', 'psusp', 'push1', 'pushn1', + 'random', 'rasgn', 'rcv', 'rcvbk', 'real', + 'refresh', 'rswap', + 'sdup', 'sect', 'size', 'snd', 'sndbk', + 'str', 'subsc', 'swap', + 'tabmat', 'tally', 'toby', 'trace', + 'unmark', + 'value', 'var'), prefix=r'\b', suffix=r'\b'), + Keyword.Declaration), + (words(( + 'any', + 'case', + 'endcase', 'endevery', 'endif', + 'endifelse', 'endrepeat', 'endsuspend', + 'enduntil', 'endwhile', 'every', + 'if', 'ifelse', + 'repeat', + 'suspend', + 'until', + 'while'), + prefix=r'\b', suffix=r'\b'), + Name.Constant), + (r'\d+(\s*|\.$|$)', Number.Integer), + (r'[+-]?\d*\.\d+(E[-+]?\d+)?', Number.Float), + (r'[+-]?\d+\.\d*(E[-+]?\d+)?', Number.Float), + (r"(<>|=>|[()|:;,.'`]|[{}]|[%^]|[&?])", Punctuation), + (r'\s+\b', Text), + (r'[\w-]+', Text), + ], + } + + def analyse_text(text): + """endsuspend and endrepeat are unique to this language, and + \\self, /self doesn't seem to get used anywhere else either.""" + result = 0 + + if 'endsuspend' in text: + result += 0.1 + + if 'endrepeat' in text: + result += 0.1 + + if ':=' in text: + result += 0.01 + + if 'procedure' in text and 'end' in text: + result += 0.01 + + # This seems quite unique to unicon -- doesn't appear in any other + # example source we have (A quick search reveals that \SELF appears in + # Perl/Raku code) + if r'\self' in text and r'/self' in text: + result += 0.5 + + return result diff --git a/src/syntax_highlighting/libs/pygments/lexers/urbi.py b/src/syntax_highlighting/libs/pygments/lexers/urbi.py index 7aaba90..d9c1c9f 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/urbi.py +++ b/src/syntax_highlighting/libs/pygments/lexers/urbi.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.urbi ~~~~~~~~~~~~~~~~~~~~ Lexers for UrbiScript language. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -117,11 +116,11 @@ def blob_callback(lexer, match, ctx): ], 'string.double': [ (r'((?:\\\\|\\"|[^"])*?)(\\B\((\d+)\)\()', blob_callback), - (r'(\\\\|\\"|[^"])*?"', String.Double, '#pop'), + (r'(\\\\|\\[^\\]|[^"\\])*?"', String.Double, '#pop'), ], 'string.single': [ (r"((?:\\\\|\\'|[^'])*?)(\\B\((\d+)\)\()", blob_callback), - (r"(\\\\|\\'|[^'])*?'", String.Single, '#pop'), + (r"(\\\\|\\[^\\]|[^'\\])*?'", String.Single, '#pop'), ], # from http://pygments.org/docs/lexerdevelopment/#changing-states 'comment': [ @@ -131,3 +130,16 @@ def blob_callback(lexer, match, ctx): (r'[*/]', Comment.Multiline), ] } + + def analyse_text(text): + """This is fairly similar to C and others, but freezeif and + waituntil are unique keywords.""" + result = 0 + + if 'freezeif' in text: + result += 0.05 + + if 'waituntil' in text: + result += 0.05 + + return result diff --git a/src/syntax_highlighting/libs/pygments/lexers/usd.py b/src/syntax_highlighting/libs/pygments/lexers/usd.py new file mode 100644 index 0000000..2b4ace8 --- /dev/null +++ b/src/syntax_highlighting/libs/pygments/lexers/usd.py @@ -0,0 +1,89 @@ +""" + pygments.lexers.usd + ~~~~~~~~~~~~~~~~~~~ + + The module that parses Pixar's Universal Scene Description file format. + + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer, bygroups +from pygments.lexer import words as words_ +from pygments.lexers._usd_builtins import COMMON_ATTRIBUTES, KEYWORDS, \ + OPERATORS, SPECIAL_NAMES, TYPES +from pygments.token import Comment, Keyword, Name, Number, Operator, \ + Punctuation, String, Text, Whitespace + +__all__ = ["UsdLexer"] + + +def _keywords(words, type_): + return [(words_(words, prefix=r"\b", suffix=r"\b"), type_)] + + +_TYPE = r"(\w+(?:\[\])?)" +_BASE_ATTRIBUTE = r"(\w+(?:\:\w+)*)(?:(\.)(timeSamples))?" +_WHITESPACE = r"([ \t]+)" + + +class UsdLexer(RegexLexer): + """ + A lexer that parses Pixar's Universal Scene Description file format. + + .. versionadded:: 2.6 + """ + + name = "USD" + aliases = ["usd", "usda"] + filenames = ["*.usd", "*.usda"] + + tokens = { + "root": [ + (r"(custom){_WHITESPACE}(uniform)(\s+){}(\s+){}(\s*)(=)".format( + _TYPE, _BASE_ATTRIBUTE, _WHITESPACE=_WHITESPACE), + bygroups(Keyword.Token, Whitespace, Keyword.Token, Whitespace, + Keyword.Type, Whitespace, Name.Attribute, Text, + Name.Keyword.Tokens, Whitespace, Operator)), + (r"(custom){_WHITESPACE}{}(\s+){}(\s*)(=)".format( + _TYPE, _BASE_ATTRIBUTE, _WHITESPACE=_WHITESPACE), + bygroups(Keyword.Token, Whitespace, Keyword.Type, Whitespace, + Name.Attribute, Text, Name.Keyword.Tokens, Whitespace, + Operator)), + (r"(uniform){_WHITESPACE}{}(\s+){}(\s*)(=)".format( + _TYPE, _BASE_ATTRIBUTE, _WHITESPACE=_WHITESPACE), + bygroups(Keyword.Token, Whitespace, Keyword.Type, Whitespace, + Name.Attribute, Text, Name.Keyword.Tokens, Whitespace, + Operator)), + (r"{}{_WHITESPACE}{}(\s*)(=)".format( + _TYPE, _BASE_ATTRIBUTE, _WHITESPACE=_WHITESPACE), + bygroups(Keyword.Type, Whitespace, Name.Attribute, Text, + Name.Keyword.Tokens, Whitespace, Operator)), + ] + + _keywords(KEYWORDS, Keyword.Tokens) + + _keywords(SPECIAL_NAMES, Name.Builtins) + + _keywords(COMMON_ATTRIBUTES, Name.Attribute) + + [(r"\b\w+:[\w:]+\b", Name.Attribute)] + + _keywords(OPERATORS, Operator) + # more attributes + [(type_ + r"\[\]", Keyword.Type) for type_ in TYPES] + + _keywords(TYPES, Keyword.Type) + + [ + (r"[(){}\[\]]", Punctuation), + ("#.*?$", Comment.Single), + (",", Punctuation), + (";", Punctuation), # ";"s are allowed to combine separate metadata lines + ("=", Operator), + (r"[-]*([0-9]*[.])?[0-9]+(?:e[+-]*\d+)?", Number), + (r"'''(?:.|\n)*?'''", String), + (r'"""(?:.|\n)*?"""', String), + (r"'.*?'", String), + (r'".*?"', String), + (r"<(\.\./)*([\w/]+|[\w/]+\.\w+[\w:]*)>", Name.Namespace), + (r"@.*?@", String.Interpol), + (r'\(.*"[.\\n]*".*\)', String.Doc), + (r"\A#usda .+$", Comment.Hashbang), + (r"\s+", Whitespace), + (r"\w+", Text), + (r"[_:.]+", Punctuation), + ], + } diff --git a/src/syntax_highlighting/libs/pygments/lexers/varnish.py b/src/syntax_highlighting/libs/pygments/lexers/varnish.py index 4452142..618049b 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/varnish.py +++ b/src/syntax_highlighting/libs/pygments/lexers/varnish.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.varnish ~~~~~~~~~~~~~~~~~~~~~~~ Lexers for Varnish configuration - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -36,7 +35,7 @@ def analyse_text(text): # Skip over comments and blank lines # This is accurate enough that returning 0.9 is reasonable. # Almost no VCL files start without some comments. - elif '\nvcl 4\.0;' in text[:1000]: + elif '\nvcl 4.0;' in text[:1000]: return 0.9 tokens = { @@ -61,7 +60,7 @@ def analyse_text(text): bygroups(Name.Attribute, Operator, Name.Variable.Global, Punctuation)), (r'(\.probe)(\s*=\s*)(\{)', bygroups(Name.Attribute, Operator, Punctuation), 'probe'), - (r'(\.\w+\b)(\s*=\s*)([^;]*)(\s*;)', + (r'(\.\w+\b)(\s*=\s*)([^;\s]*)(\s*;)', bygroups(Name.Attribute, Operator, using(this), Punctuation)), (r'\{', Punctuation, '#push'), (r'\}', Punctuation, '#pop'), @@ -120,7 +119,7 @@ def analyse_text(text): r'([a-zA-Z_]\w*)' r'(\s*\(.*\))', bygroups(Name.Function, Punctuation, Name.Function, using(this))), - ('[a-zA-Z_]\w*', Name), + (r'[a-zA-Z_]\w*', Name), ], 'comment': [ (r'[^*/]+', Comment.Multiline), diff --git a/src/syntax_highlighting/libs/pygments/lexers/verification.py b/src/syntax_highlighting/libs/pygments/lexers/verification.py index 5322e17..2d473ae 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/verification.py +++ b/src/syntax_highlighting/libs/pygments/lexers/verification.py @@ -1,17 +1,16 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.verification ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Lexer for Intermediate Verification Languages (IVLs). - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ from pygments.lexer import RegexLexer, include, words from pygments.token import Comment, Operator, Keyword, Name, Number, \ - Punctuation, Whitespace + Punctuation, Text, Generic __all__ = ['BoogieLexer', 'SilverLexer'] @@ -29,8 +28,9 @@ class BoogieLexer(RegexLexer): tokens = { 'root': [ # Whitespace and Comments - (r'\n', Whitespace), - (r'\s+', Whitespace), + (r'\n', Text), + (r'\s+', Text), + (r'\\\n', Text), # line continuation (r'//[/!](.*?)\n', Comment.Doc), (r'//(.*?)\n', Comment.Single), (r'/\*', Comment.Multiline, 'comment'), @@ -45,6 +45,7 @@ class BoogieLexer(RegexLexer): (words(('bool', 'int', 'ref'), suffix=r'\b'), Keyword.Type), include('numbers'), (r"(>=|<=|:=|!=|==>|&&|\|\||[+/\-=>*<\[\]])", Operator), + (r'\{.*?\}', Generic.Emph), #triggers (r"([{}():;,.])", Punctuation), # Identifier (r'[a-zA-Z_]\w*', Name), @@ -74,8 +75,9 @@ class SilverLexer(RegexLexer): tokens = { 'root': [ # Whitespace and Comments - (r'\n', Whitespace), - (r'\s+', Whitespace), + (r'\n', Text), + (r'\s+', Text), + (r'\\\n', Text), # line continuation (r'//[/!](.*?)\n', Comment.Doc), (r'//(.*?)\n', Comment.Single), (r'/\*', Comment.Multiline, 'comment'), @@ -83,18 +85,18 @@ class SilverLexer(RegexLexer): (words(( 'result', 'true', 'false', 'null', 'method', 'function', 'predicate', 'program', 'domain', 'axiom', 'var', 'returns', - 'field', 'define', 'requires', 'ensures', 'invariant', - 'fold', 'unfold', 'inhale', 'exhale', 'new', 'assert', + 'field', 'define', 'fold', 'unfold', 'inhale', 'exhale', 'new', 'assert', 'assume', 'goto', 'while', 'if', 'elseif', 'else', 'fresh', 'constraining', 'Seq', 'Set', 'Multiset', 'union', 'intersection', 'setminus', 'subset', 'unfolding', 'in', 'old', 'forall', 'exists', 'acc', 'wildcard', 'write', 'none', 'epsilon', 'perm', 'unique', 'apply', 'package', 'folding', 'label', 'forperm'), suffix=r'\b'), Keyword), - (words(('Int', 'Perm', 'Bool', 'Ref'), suffix=r'\b'), Keyword.Type), + (words(('requires', 'ensures', 'invariant'), suffix=r'\b'), Name.Decorator), + (words(('Int', 'Perm', 'Bool', 'Ref', 'Rational'), suffix=r'\b'), Keyword.Type), include('numbers'), - (r'[!%&*+=|?:<>/\-\[\]]', Operator), + (r'\{.*?\}', Generic.Emph), #triggers (r'([{}():;,.])', Punctuation), # Identifier (r'[\w$]\w*', Name), diff --git a/src/syntax_highlighting/libs/pygments/lexers/web.py b/src/syntax_highlighting/libs/pygments/lexers/web.py index 6e9c4f9..a186048 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/web.py +++ b/src/syntax_highlighting/libs/pygments/lexers/web.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.web ~~~~~~~~~~~~~~~~~~~ Just export previously exported lexers. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/src/syntax_highlighting/libs/pygments/lexers/webassembly.py b/src/syntax_highlighting/libs/pygments/lexers/webassembly.py new file mode 100644 index 0000000..d162a46 --- /dev/null +++ b/src/syntax_highlighting/libs/pygments/lexers/webassembly.py @@ -0,0 +1,120 @@ +# -*- coding: utf-8 -*- +""" + pygments.lexers.webassembly + ~~~~~~~~~~~~~~~~~~~ + + Lexers for the WebAssembly text format. + + The grammar can be found at https://github.com/WebAssembly/spec/blob/master/interpreter/README.md + and https://webassembly.github.io/spec/core/text/. + + + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer, words, bygroups, default +from pygments.token import Text, Comment, Operator, Keyword, String, Number, Punctuation, Literal, Error, Name + +__all__ = ['WatLexer'] + +keywords = ( + 'module', 'import', 'func', 'funcref', 'start', 'param', 'local', 'type', + 'result', 'export', 'memory', 'global', 'mut', 'data', 'table', 'elem', + 'if', 'then', 'else', 'end', 'block', 'loop' +) + +builtins = ( + 'unreachable', 'nop', 'block', 'loop', 'if', 'else', 'end', 'br', 'br_if', + 'br_table', 'return', 'call', 'call_indirect', 'drop', 'select', + 'local.get', 'local.set', 'local.tee', 'global.get', 'global.set', + 'i32.load', 'i64.load', 'f32.load', 'f64.load', 'i32.load8_s', + 'i32.load8_u', 'i32.load16_s', 'i32.load16_u', 'i64.load8_s', + 'i64.load8_u', 'i64.load16_s', 'i64.load16_u', 'i64.load32_s', + 'i64.load32_u', 'i32.store', 'i64.store', 'f32.store', 'f64.store', + 'i32.store8', 'i32.store16', 'i64.store8', 'i64.store16', 'i64.store32', + 'memory.size', 'memory.grow', 'i32.const', 'i64.const', 'f32.const', + 'f64.const', 'i32.eqz', 'i32.eq', 'i32.ne', 'i32.lt_s', 'i32.lt_u', + 'i32.gt_s', 'i32.gt_u', 'i32.le_s', 'i32.le_u', 'i32.ge_s', 'i32.ge_u', + 'i64.eqz', 'i64.eq', 'i64.ne', 'i64.lt_s', 'i64.lt_u', 'i64.gt_s', + 'i64.gt_u', 'i64.le_s', 'i64.le_u', 'i64.ge_s', 'i64.ge_u', 'f32.eq', + 'f32.ne', 'f32.lt', 'f32.gt', 'f32.le', 'f32.ge', 'f64.eq', 'f64.ne', + 'f64.lt', 'f64.gt', 'f64.le', 'f64.ge', 'i32.clz', 'i32.ctz', 'i32.popcnt', + 'i32.add', 'i32.sub', 'i32.mul', 'i32.div_s', 'i32.div_u', 'i32.rem_s', + 'i32.rem_u', 'i32.and', 'i32.or', 'i32.xor', 'i32.shl', 'i32.shr_s', + 'i32.shr_u', 'i32.rotl', 'i32.rotr', 'i64.clz', 'i64.ctz', 'i64.popcnt', + 'i64.add', 'i64.sub', 'i64.mul', 'i64.div_s', 'i64.div_u', 'i64.rem_s', + 'i64.rem_u', 'i64.and', 'i64.or', 'i64.xor', 'i64.shl', 'i64.shr_s', + 'i64.shr_u', 'i64.rotl', 'i64.rotr', 'f32.abs', 'f32.neg', 'f32.ceil', + 'f32.floor', 'f32.trunc', 'f32.nearest', 'f32.sqrt', 'f32.add', 'f32.sub', + 'f32.mul', 'f32.div', 'f32.min', 'f32.max', 'f32.copysign', 'f64.abs', + 'f64.neg', 'f64.ceil', 'f64.floor', 'f64.trunc', 'f64.nearest', 'f64.sqrt', + 'f64.add', 'f64.sub', 'f64.mul', 'f64.div', 'f64.min', 'f64.max', + 'f64.copysign', 'i32.wrap_i64', 'i32.trunc_f32_s', 'i32.trunc_f32_u', + 'i32.trunc_f64_s', 'i32.trunc_f64_u', 'i64.extend_i32_s', + 'i64.extend_i32_u', 'i64.trunc_f32_s', 'i64.trunc_f32_u', + 'i64.trunc_f64_s', 'i64.trunc_f64_u', 'f32.convert_i32_s', + 'f32.convert_i32_u', 'f32.convert_i64_s', 'f32.convert_i64_u', + 'f32.demote_f64', 'f64.convert_i32_s', 'f64.convert_i32_u', + 'f64.convert_i64_s', 'f64.convert_i64_u', 'f64.promote_f32', + 'i32.reinterpret_f32', 'i64.reinterpret_f64', 'f32.reinterpret_i32', + 'f64.reinterpret_i64', +) + + +class WatLexer(RegexLexer): + """Lexer for the `WebAssembly text format `_. + + .. versionadded:: 2.9 + """ + + name = 'WebAssembly' + aliases = ['wast', 'wat'] + filenames = ['*.wat', '*.wast'] + + tokens = { + 'root': [ + (words(keywords, suffix=r'(?=[^a-z_\.])'), Keyword), + (words(builtins), Name.Builtin, 'arguments'), + (words(['i32', 'i64', 'f32', 'f64']), Keyword.Type), + (r'\$[A-Za-z0-9!#$%&\'*+./:<=>?@\\^_`|~-]+', Name.Variable), # yes, all of the are valid in identifiers + (r';;.*?$', Comment.Single), + (r'\(;', Comment.Multiline, 'nesting_comment'), + (r'[+-]?0x[\dA-Fa-f](_?[\dA-Fa-f])*(.([\dA-Fa-f](_?[\dA-Fa-f])*)?)?([pP][+-]?[\dA-Fa-f](_?[\dA-Fa-f])*)?', Number.Float), + (r'[+-]?\d.\d(_?\d)*[eE][+-]?\d(_?\d)*', Number.Float), + (r'[+-]?\d.\d(_?\d)*', Number.Float), + (r'[+-]?\d.[eE][+-]?\d(_?\d)*', Number.Float), + (r'[+-]?(inf|nan:0x[\dA-Fa-f](_?[\dA-Fa-f])*|nan)', Number.Float), + (r'[+-]?0x[\dA-Fa-f](_?[\dA-Fa-f])*', Number.Hex), + (r'[+-]?\d(_?\d)*', Number.Integer), + (r'[\(\)]', Punctuation), + (r'"', String.Double, 'string'), + (r'\s+', Text), + ], + 'nesting_comment': [ + (r'\(;', Comment.Multiline, '#push'), + (r';\)', Comment.Multiline, '#pop'), + (r'[^;(]+', Comment.Multiline), + (r'[;(]', Comment.Multiline), + ], + 'string': [ + (r'\\[\dA-Fa-f][\dA-Fa-f]', String.Escape), # must have exactly two hex digits + (r'\\t', String.Escape), + (r'\\n', String.Escape), + (r'\\r', String.Escape), + (r'\\"', String.Escape), + (r"\\'", String.Escape), + (r'\\u\{[\dA-Fa-f](_?[\dA-Fa-f])*\}', String.Escape), + (r'\\\\', String.Escape), + (r'"', String.Double, '#pop'), + (r'[^"\\]+', String.Double), + ], + 'arguments': [ + (r'\s+', Text), + (r'(offset)(=)(0x[\dA-Fa-f](_?[\dA-Fa-f])*)', bygroups(Keyword, Operator, Number.Hex)), + (r'(offset)(=)(\d(_?\d)*)', bygroups(Keyword, Operator, Number.Integer)), + (r'(align)(=)(0x[\dA-Fa-f](_?[\dA-Fa-f])*)', bygroups(Keyword, Operator, Number.Hex)), + (r'(align)(=)(\d(_?\d)*)', bygroups(Keyword, Operator, Number.Integer)), + default('#pop'), + ] + } diff --git a/src/syntax_highlighting/libs/pygments/lexers/webidl.py b/src/syntax_highlighting/libs/pygments/lexers/webidl.py new file mode 100644 index 0000000..d805484 --- /dev/null +++ b/src/syntax_highlighting/libs/pygments/lexers/webidl.py @@ -0,0 +1,298 @@ +""" + pygments.lexers.webidl + ~~~~~~~~~~~~~~~~~~~~~~ + + Lexers for Web IDL, including some extensions. + + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer, default, include, words +from pygments.token import Comment, Keyword, Name, Number, Punctuation, \ + String, Text + +__all__ = ['WebIDLLexer'] + +_builtin_types = ( + # primitive types + 'byte', 'octet', 'boolean', + r'(?:unsigned\s+)?(?:short|long(?:\s+long)?)', + r'(?:unrestricted\s+)?(?:float|double)', + # string types + 'DOMString', 'ByteString', 'USVString', + # exception types + 'Error', 'DOMException', + # typed array types + 'Uint8Array', 'Uint16Array', 'Uint32Array', 'Uint8ClampedArray', + 'Float32Array', 'Float64Array', + # buffer source types + 'ArrayBuffer', 'DataView', 'Int8Array', 'Int16Array', 'Int32Array', + # other + 'any', 'void', 'object', 'RegExp', +) +_identifier = r'_?[A-Za-z][a-zA-Z0-9_-]*' +_keyword_suffix = r'(?![\w-])' +_string = r'"[^"]*"' + + +class WebIDLLexer(RegexLexer): + """ + For Web IDL. + + .. versionadded:: 2.6 + """ + + name = 'Web IDL' + aliases = ['webidl'] + filenames = ['*.webidl'] + + tokens = { + 'common': [ + (r'\s+', Text), + (r'(?s)/\*.*?\*/', Comment.Multiline), + (r'//.*', Comment.Single), + (r'^#.*', Comment.Preproc), + ], + 'root': [ + include('common'), + (r'\[', Punctuation, 'extended_attributes'), + (r'partial' + _keyword_suffix, Keyword), + (r'typedef' + _keyword_suffix, Keyword, ('typedef', 'type')), + (r'interface' + _keyword_suffix, Keyword, 'interface_rest'), + (r'enum' + _keyword_suffix, Keyword, 'enum_rest'), + (r'callback' + _keyword_suffix, Keyword, 'callback_rest'), + (r'dictionary' + _keyword_suffix, Keyword, 'dictionary_rest'), + (r'namespace' + _keyword_suffix, Keyword, 'namespace_rest'), + (_identifier, Name.Class, 'implements_rest'), + ], + 'extended_attributes': [ + include('common'), + (r',', Punctuation), + (_identifier, Name.Decorator), + (r'=', Punctuation, 'extended_attribute_rest'), + (r'\(', Punctuation, 'argument_list'), + (r'\]', Punctuation, '#pop'), + ], + 'extended_attribute_rest': [ + include('common'), + (_identifier, Name, 'extended_attribute_named_rest'), + (_string, String), + (r'\(', Punctuation, 'identifier_list'), + default('#pop'), + ], + 'extended_attribute_named_rest': [ + include('common'), + (r'\(', Punctuation, 'argument_list'), + default('#pop'), + ], + 'argument_list': [ + include('common'), + (r'\)', Punctuation, '#pop'), + default('argument'), + ], + 'argument': [ + include('common'), + (r'optional' + _keyword_suffix, Keyword), + (r'\[', Punctuation, 'extended_attributes'), + (r',', Punctuation, '#pop'), + (r'\)', Punctuation, '#pop:2'), + default(('argument_rest', 'type')) + ], + 'argument_rest': [ + include('common'), + (_identifier, Name.Variable), + (r'\.\.\.', Punctuation), + (r'=', Punctuation, 'default_value'), + default('#pop'), + ], + 'identifier_list': [ + include('common'), + (_identifier, Name.Class), + (r',', Punctuation), + (r'\)', Punctuation, '#pop'), + ], + 'type': [ + include('common'), + (r'(?:' + r'|'.join(_builtin_types) + r')' + _keyword_suffix, + Keyword.Type, 'type_null'), + (words(('sequence', 'Promise', 'FrozenArray'), + suffix=_keyword_suffix), Keyword.Type, 'type_identifier'), + (_identifier, Name.Class, 'type_identifier'), + (r'\(', Punctuation, 'union_type'), + ], + 'union_type': [ + include('common'), + (r'or' + _keyword_suffix, Keyword), + (r'\)', Punctuation, ('#pop', 'type_null')), + default('type'), + ], + 'type_identifier': [ + (r'<', Punctuation, 'type_list'), + default(('#pop', 'type_null')) + ], + 'type_null': [ + (r'\?', Punctuation), + default('#pop:2'), + ], + 'default_value': [ + include('common'), + include('const_value'), + (_string, String, '#pop'), + (r'\[\s*\]', Punctuation, '#pop'), + ], + 'const_value': [ + include('common'), + (words(('true', 'false', '-Infinity', 'Infinity', 'NaN', 'null'), + suffix=_keyword_suffix), Keyword.Constant, '#pop'), + (r'-?(?:(?:[0-9]+\.[0-9]*|[0-9]*\.[0-9]+)(?:[Ee][+-]?[0-9]+)?' + + r'|[0-9]+[Ee][+-]?[0-9]+)', Number.Float, '#pop'), + (r'-?[1-9][0-9]*', Number.Integer, '#pop'), + (r'-?0[Xx][0-9A-Fa-f]+', Number.Hex, '#pop'), + (r'-?0[0-7]*', Number.Oct, '#pop'), + ], + 'typedef': [ + include('common'), + (_identifier, Name.Class), + (r';', Punctuation, '#pop'), + ], + 'namespace_rest': [ + include('common'), + (_identifier, Name.Namespace), + (r'\{', Punctuation, 'namespace_body'), + (r';', Punctuation, '#pop'), + ], + 'namespace_body': [ + include('common'), + (r'\[', Punctuation, 'extended_attributes'), + (r'readonly' + _keyword_suffix, Keyword), + (r'attribute' + _keyword_suffix, + Keyword, ('attribute_rest', 'type')), + (r'const' + _keyword_suffix, Keyword, ('const_rest', 'type')), + (r'\}', Punctuation, '#pop'), + default(('operation_rest', 'type')), + ], + 'interface_rest': [ + include('common'), + (_identifier, Name.Class), + (r':', Punctuation), + (r'\{', Punctuation, 'interface_body'), + (r';', Punctuation, '#pop'), + ], + 'interface_body': [ + (words(('iterable', 'maplike', 'setlike'), suffix=_keyword_suffix), + Keyword, 'iterable_maplike_setlike_rest'), + (words(('setter', 'getter', 'creator', 'deleter', 'legacycaller', + 'inherit', 'static', 'stringifier', 'jsonifier'), + suffix=_keyword_suffix), Keyword), + (r'serializer' + _keyword_suffix, Keyword, 'serializer_rest'), + (r';', Punctuation), + include('namespace_body'), + ], + 'attribute_rest': [ + include('common'), + (_identifier, Name.Variable), + (r';', Punctuation, '#pop'), + ], + 'const_rest': [ + include('common'), + (_identifier, Name.Constant), + (r'=', Punctuation, 'const_value'), + (r';', Punctuation, '#pop'), + ], + 'operation_rest': [ + include('common'), + (r';', Punctuation, '#pop'), + default('operation'), + ], + 'operation': [ + include('common'), + (_identifier, Name.Function), + (r'\(', Punctuation, 'argument_list'), + (r';', Punctuation, '#pop:2'), + ], + 'iterable_maplike_setlike_rest': [ + include('common'), + (r'<', Punctuation, 'type_list'), + (r';', Punctuation, '#pop'), + ], + 'type_list': [ + include('common'), + (r',', Punctuation), + (r'>', Punctuation, '#pop'), + default('type'), + ], + 'serializer_rest': [ + include('common'), + (r'=', Punctuation, 'serialization_pattern'), + (r';', Punctuation, '#pop'), + default('operation'), + ], + 'serialization_pattern': [ + include('common'), + (_identifier, Name.Variable, '#pop'), + (r'\{', Punctuation, 'serialization_pattern_map'), + (r'\[', Punctuation, 'serialization_pattern_list'), + ], + 'serialization_pattern_map': [ + include('common'), + (words(('getter', 'inherit', 'attribute'), + suffix=_keyword_suffix), Keyword), + (r',', Punctuation), + (_identifier, Name.Variable), + (r'\}', Punctuation, '#pop:2'), + ], + 'serialization_pattern_list': [ + include('common'), + (words(('getter', 'attribute'), suffix=_keyword_suffix), Keyword), + (r',', Punctuation), + (_identifier, Name.Variable), + (r']', Punctuation, '#pop:2'), + ], + 'enum_rest': [ + include('common'), + (_identifier, Name.Class), + (r'\{', Punctuation, 'enum_body'), + (r';', Punctuation, '#pop'), + ], + 'enum_body': [ + include('common'), + (_string, String), + (r',', Punctuation), + (r'\}', Punctuation, '#pop'), + ], + 'callback_rest': [ + include('common'), + (r'interface' + _keyword_suffix, + Keyword, ('#pop', 'interface_rest')), + (_identifier, Name.Class), + (r'=', Punctuation, ('operation', 'type')), + (r';', Punctuation, '#pop'), + ], + 'dictionary_rest': [ + include('common'), + (_identifier, Name.Class), + (r':', Punctuation), + (r'\{', Punctuation, 'dictionary_body'), + (r';', Punctuation, '#pop'), + ], + 'dictionary_body': [ + include('common'), + (r'\[', Punctuation, 'extended_attributes'), + (r'required' + _keyword_suffix, Keyword), + (r'\}', Punctuation, '#pop'), + default(('dictionary_item', 'type')), + ], + 'dictionary_item': [ + include('common'), + (_identifier, Name.Variable), + (r'=', Punctuation, 'default_value'), + (r';', Punctuation, '#pop'), + ], + 'implements_rest': [ + include('common'), + (r'implements' + _keyword_suffix, Keyword), + (_identifier, Name.Class), + (r';', Punctuation, '#pop'), + ], + } diff --git a/src/syntax_highlighting/libs/pygments/lexers/webmisc.py b/src/syntax_highlighting/libs/pygments/lexers/webmisc.py index 712c824..b32005e 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/webmisc.py +++ b/src/syntax_highlighting/libs/pygments/lexers/webmisc.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.webmisc ~~~~~~~~~~~~~~~~~~~~~~~ Lexers for misc. web stuff. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -15,7 +14,6 @@ default, using from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ Number, Punctuation, Literal -from pygments.util import unirange from pygments.lexers.css import _indentation, _starts_block from pygments.lexers.html import HtmlLexer @@ -74,15 +72,15 @@ class XQueryLexer(ExtendedRegexLexer): # FIX UNICODE LATER # ncnamestartchar = ( - # ur"[A-Z]|_|[a-z]|[\u00C0-\u00D6]|[\u00D8-\u00F6]|[\u00F8-\u02FF]|" - # ur"[\u0370-\u037D]|[\u037F-\u1FFF]|[\u200C-\u200D]|[\u2070-\u218F]|" - # ur"[\u2C00-\u2FEF]|[\u3001-\uD7FF]|[\uF900-\uFDCF]|[\uFDF0-\uFFFD]|" - # ur"[\u10000-\uEFFFF]" + # r"[A-Z]|_|[a-z]|[\u00C0-\u00D6]|[\u00D8-\u00F6]|[\u00F8-\u02FF]|" + # r"[\u0370-\u037D]|[\u037F-\u1FFF]|[\u200C-\u200D]|[\u2070-\u218F]|" + # r"[\u2C00-\u2FEF]|[\u3001-\uD7FF]|[\uF900-\uFDCF]|[\uFDF0-\uFFFD]|" + # r"[\u10000-\uEFFFF]" # ) ncnamestartchar = r"(?:[A-Z]|_|[a-z])" # FIX UNICODE LATER - # ncnamechar = ncnamestartchar + (ur"|-|\.|[0-9]|\u00B7|[\u0300-\u036F]|" - # ur"[\u203F-\u2040]") + # ncnamechar = ncnamestartchar + (r"|-|\.|[0-9]|\u00B7|[\u0300-\u036F]|" + # r"[\u203F-\u2040]") ncnamechar = r"(?:" + ncnamestartchar + r"|-|\.|[0-9])" ncname = "(?:%s+%s*)" % (ncnamestartchar, ncnamechar) pitarget_namestartchar = r"(?:[A-KN-WYZ]|_|:|[a-kn-wyz])" @@ -99,14 +97,14 @@ class XQueryLexer(ExtendedRegexLexer): stringsingle = r"(?:'(?:" + entityref + r"|" + charref + r"|''|[^&'])*')" # FIX UNICODE LATER - # elementcontentchar = (ur'\t|\r|\n|[\u0020-\u0025]|[\u0028-\u003b]|' - # ur'[\u003d-\u007a]|\u007c|[\u007e-\u007F]') + # elementcontentchar = (r'\t|\r|\n|[\u0020-\u0025]|[\u0028-\u003b]|' + # r'[\u003d-\u007a]|\u007c|[\u007e-\u007F]') elementcontentchar = r'[A-Za-z]|\s|\d|[!"#$%()*+,\-./:;=?@\[\\\]^_\'`|~]' - # quotattrcontentchar = (ur'\t|\r|\n|[\u0020-\u0021]|[\u0023-\u0025]|' - # ur'[\u0027-\u003b]|[\u003d-\u007a]|\u007c|[\u007e-\u007F]') + # quotattrcontentchar = (r'\t|\r|\n|[\u0020-\u0021]|[\u0023-\u0025]|' + # r'[\u0027-\u003b]|[\u003d-\u007a]|\u007c|[\u007e-\u007F]') quotattrcontentchar = r'[A-Za-z]|\s|\d|[!#$%()*+,\-./:;=?@\[\\\]^_\'`|~]' - # aposattrcontentchar = (ur'\t|\r|\n|[\u0020-\u0025]|[\u0028-\u003b]|' - # ur'[\u003d-\u007a]|\u007c|[\u007e-\u007F]') + # aposattrcontentchar = (r'\t|\r|\n|[\u0020-\u0025]|[\u0028-\u003b]|' + # r'[\u003d-\u007a]|\u007c|[\u007e-\u007F]') aposattrcontentchar = r'[A-Za-z]|\s|\d|[!"#$%()*+,\-./:;=?@\[\\\]^_`|~]' # CHAR elements - fix the above elementcontentchar, quotattrcontentchar, @@ -129,7 +127,8 @@ def operator_root_callback(lexer, match, ctx): def popstate_tag_callback(lexer, match, ctx): yield match.start(), Name.Tag, match.group(1) - ctx.stack.append(lexer.xquery_parse_state.pop()) + if lexer.xquery_parse_state: + ctx.stack.append(lexer.xquery_parse_state.pop()) ctx.pos = match.end() def popstate_xmlcomment_callback(lexer, match, ctx): @@ -158,6 +157,9 @@ def popstate_callback(lexer, match, ctx): # state stack if len(lexer.xquery_parse_state) == 0: ctx.stack.pop() + if not ctx.stack: + # make sure we have at least the root state on invalid inputs + ctx.stack = ['root'] elif len(ctx.stack) > 1: ctx.stack.append(lexer.xquery_parse_state.pop()) else: @@ -438,7 +440,7 @@ def pushstate_operator_callback(lexer, match, ctx): ], 'varname': [ (r'\(:', Comment, 'comment'), - (r'(' + qname + ')(\()?', bygroups(Name, Punctuation), 'operator'), + (r'(' + qname + r')(\()?', bygroups(Name, Punctuation), 'operator'), ], 'singletype': [ include('whitespace'), @@ -515,8 +517,8 @@ def pushstate_operator_callback(lexer, match, ctx): 'xml_comment': [ (r'(-->)', popstate_xmlcomment_callback), (r'[^-]{1,2}', Literal), - (u'\\t|\\r|\\n|[\u0020-\uD7FF]|[\uE000-\uFFFD]|' + - unirange(0x10000, 0x10ffff), Literal), + (r'\t|\r|\n|[\u0020-\uD7FF]|[\uE000-\uFFFD]|[\U00010000-\U0010FFFF]', + Literal), ], 'processing_instruction': [ (r'\s+', Text, 'processing_instruction_content'), @@ -525,13 +527,13 @@ def pushstate_operator_callback(lexer, match, ctx): ], 'processing_instruction_content': [ (r'\?>', String.Doc, '#pop'), - (u'\\t|\\r|\\n|[\u0020-\uD7FF]|[\uE000-\uFFFD]|' + - unirange(0x10000, 0x10ffff), Literal), + (r'\t|\r|\n|[\u0020-\uD7FF]|[\uE000-\uFFFD]|[\U00010000-\U0010FFFF]', + Literal), ], 'cdata_section': [ (r']]>', String.Doc, '#pop'), - (u'\\t|\\r|\\n|[\u0020-\uD7FF]|[\uE000-\uFFFD]|' + - unirange(0x10000, 0x10ffff), Literal), + (r'\t|\r|\n|[\u0020-\uD7FF]|[\uE000-\uFFFD]|[\U00010000-\U0010FFFF]', + Literal), ], 'start_tag': [ include('whitespace'), @@ -600,8 +602,8 @@ def pushstate_operator_callback(lexer, match, ctx): ], 'pragmacontents': [ (r'#\)', Punctuation, 'operator'), - (u'\\t|\\r|\\n|[\u0020-\uD7FF]|[\uE000-\uFFFD]|' + - unirange(0x10000, 0x10ffff), Literal), + (r'\t|\r|\n|[\u0020-\uD7FF]|[\uE000-\uFFFD]|[\U00010000-\U0010FFFF]', + Literal), (r'(\s+)', Text), ], 'occurrenceindicator': [ @@ -643,9 +645,9 @@ def pushstate_operator_callback(lexer, match, ctx): bygroups(Keyword.Declaration, Text, Keyword.Declaration, Text, Keyword.Declaration), 'operator'), (r'(declare)(\s+)(context)(\s+)(item)', bygroups(Keyword.Declaration, Text, Keyword.Declaration, Text, Keyword.Declaration), 'operator'), - (ncname + ':\*', Name, 'operator'), - ('\*:'+ncname, Name.Tag, 'operator'), - ('\*', Name.Tag, 'operator'), + (ncname + r':\*', Name, 'operator'), + (r'\*:'+ncname, Name.Tag, 'operator'), + (r'\*', Name.Tag, 'operator'), (stringdouble, String.Double, 'operator'), (stringsingle, String.Single, 'operator'), @@ -661,7 +663,8 @@ def pushstate_operator_callback(lexer, match, ctx): # NAMESPACE KEYWORD (r'(declare)(\s+)(default)(\s+)(element|function)', - bygroups(Keyword.Declaration, Text, Keyword.Declaration, Text, Keyword.Declaration), 'namespacekeyword'), + bygroups(Keyword.Declaration, Text, Keyword.Declaration, Text, Keyword.Declaration), + 'namespacekeyword'), (r'(import)(\s+)(schema|module)', bygroups(Keyword.Pseudo, Text, Keyword.Pseudo), 'namespacekeyword'), (r'(declare)(\s+)(copy-namespaces)', @@ -854,14 +857,14 @@ class QmlLexer(RegexLexer): (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float), (r'0x[0-9a-fA-F]+', Number.Hex), (r'[0-9]+', Number.Integer), - (r'"(\\\\|\\"|[^"])*"', String.Double), - (r"'(\\\\|\\'|[^'])*'", String.Single), + (r'"(\\\\|\\[^\\]|[^"\\])*"', String.Double), + (r"'(\\\\|\\[^\\]|[^'\\])*'", String.Single), ] } class CirruLexer(RegexLexer): - """ + r""" Syntax rules of Cirru can be found at: http://cirru.org/ diff --git a/src/syntax_highlighting/libs/pygments/lexers/whiley.py b/src/syntax_highlighting/libs/pygments/lexers/whiley.py index 0d0e8ab..82b100b 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/whiley.py +++ b/src/syntax_highlighting/libs/pygments/lexers/whiley.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.whiley ~~~~~~~~~~~~~~~~~~~~~~ Lexers for the Whiley language. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -97,11 +96,11 @@ class WhileyLexer(RegexLexer): # operators and punctuation (r'[{}()\[\],.;]', Punctuation), - (u'[+\\-*/%&|<>^!~@=:?' + (r'[+\-*/%&|<>^!~@=:?' # unicode operators - u'\u2200\u2203\u2205\u2282\u2286\u2283\u2287' - u'\u222A\u2229\u2264\u2265\u2208\u2227\u2228' - u']', Operator), + r'\u2200\u2203\u2205\u2282\u2286\u2283\u2287' + r'\u222A\u2229\u2264\u2265\u2208\u2227\u2228' + r']', Operator), # identifier (r'[a-zA-Z_]\w*', Name), diff --git a/src/syntax_highlighting/libs/pygments/lexers/x10.py b/src/syntax_highlighting/libs/pygments/lexers/x10.py index 1c63326..5c01fe5 100644 --- a/src/syntax_highlighting/libs/pygments/lexers/x10.py +++ b/src/syntax_highlighting/libs/pygments/lexers/x10.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.lexers.x10 ~~~~~~~~~~~~~~~~~~~ Lexers for the X10 programming language. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -62,7 +61,7 @@ class X10Lexer(RegexLexer): (r'\b(%s)\b' % '|'.join(types), Keyword.Type), (r'\b(%s)\b' % '|'.join(values), Keyword.Constant), (r'\b(%s)\b' % '|'.join(modifiers), Keyword.Declaration), - (r'"(\\\\|\\"|[^"])*"', String), + (r'"(\\\\|\\[^\\]|[^"\\])*"', String), (r"'\\.'|'[^\\]'|'\\u[0-9a-fA-F]{4}'", String.Char), (r'.', Text) ], diff --git a/src/syntax_highlighting/libs/pygments/lexers/xorg.py b/src/syntax_highlighting/libs/pygments/lexers/xorg.py new file mode 100644 index 0000000..490b7c1 --- /dev/null +++ b/src/syntax_highlighting/libs/pygments/lexers/xorg.py @@ -0,0 +1,36 @@ +""" + pygments.lexers.xorg + ~~~~~~~~~~~~~~~~~~~~ + + Lexers for Xorg configs. + + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer, bygroups +from pygments.token import Comment, String, Name, Text + +__all__ = ['XorgLexer'] + + +class XorgLexer(RegexLexer): + """Lexer for xorg.conf file.""" + name = 'Xorg' + aliases = ['xorg.conf'] + filenames = ['xorg.conf'] + mimetypes = [] + + tokens = { + 'root': [ + (r'\s+', Text), + (r'#.*$', Comment), + + (r'((?:Sub)?Section)(\s+)("\w+")', + bygroups(String.Escape, Text, String.Escape)), + (r'(End(?:Sub)?Section)', String.Escape), + + (r'(\w+)(\s+)([^\n#]+)', + bygroups(Name.Builtin, Text, Name.Constant)), + ], + } diff --git a/src/syntax_highlighting/libs/pygments/lexers/yang.py b/src/syntax_highlighting/libs/pygments/lexers/yang.py new file mode 100644 index 0000000..4452df2 --- /dev/null +++ b/src/syntax_highlighting/libs/pygments/lexers/yang.py @@ -0,0 +1,103 @@ +""" + pygments.lexers.yang + ~~~~~~~~~~~~~~~~~~~~ + + Lexer for the YANG 1.1 modeling language. See :rfc:`7950`. + + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import (RegexLexer, bygroups, words) +from pygments.token import (Text, Token, Name, String, Comment, + Number) + +__all__ = ['YangLexer'] + +class YangLexer(RegexLexer): + """ + Lexer for `YANG `_, based on RFC7950 + + .. versionadded:: 2.7 + """ + name = 'YANG' + aliases = ['yang'] + filenames = ['*.yang'] + mimetypes = ['application/yang'] + + #Keywords from RFC7950 ; oriented at BNF style + TOP_STMTS_KEYWORDS = ("module", "submodule") + MODULE_HEADER_STMT_KEYWORDS = ("belongs-to", "namespace", "prefix", "yang-version") + META_STMT_KEYWORDS = ("contact", "description", "organization", + "reference", "revision") + LINKAGE_STMTS_KEYWORDS = ("import", "include", "revision-date") + BODY_STMT_KEYWORDS = ("action", "argument", "augment", "deviation", + "extension", "feature", "grouping", "identity", + "if-feature", "input", "notification", "output", + "rpc", "typedef") + DATA_DEF_STMT_KEYWORDS = ("anydata", "anyxml", "case", "choice", + "config", "container", "deviate", "leaf", + "leaf-list", "list", "must", "presence", + "refine", "uses", "when") + TYPE_STMT_KEYWORDS = ("base", "bit", "default", "enum", "error-app-tag", + "error-message", "fraction-digits", "length", + "max-elements", "min-elements", "modifier", + "ordered-by", "path", "pattern", "position", + "range", "require-instance", "status", "type", + "units", "value", "yin-element") + LIST_STMT_KEYWORDS = ("key", "mandatory", "unique") + + #RFC7950 other keywords + CONSTANTS_KEYWORDS = ("add", "current", "delete", "deprecated", "false", + "invert-match", "max", "min", "not-supported", + "obsolete", "replace", "true", "unbounded", "user") + + #RFC7950 Built-In Types + TYPES = ("binary", "bits", "boolean", "decimal64", "empty", "enumeration", + "identityref", "instance-identifier", "int16", "int32", "int64", + "int8", "leafref", "string", "uint16", "uint32", "uint64", + "uint8", "union") + + suffix_re_pattern = r'(?=[^\w\-:])' + + tokens = { + 'comments': [ + (r'[^*/]', Comment), + (r'/\*', Comment, '#push'), + (r'\*/', Comment, '#pop'), + (r'[*/]', Comment), + ], + "root": [ + (r'\s+', Text.Whitespace), + (r'[{};]+', Token.Punctuation), + (r'(?`_ source code. + + grammar: https://ziglang.org/documentation/master/#Grammar + """ + name = 'Zig' + aliases = ['zig'] + filenames = ['*.zig'] + mimetypes = ['text/zig'] + + type_keywords = ( + words(('bool', 'f16', 'f32', 'f64', 'f128', 'void', 'noreturn', 'type', + 'anyerror', 'promise', 'i0', 'u0', 'isize', 'usize', 'comptime_int', + 'comptime_float', 'c_short', 'c_ushort', 'c_int', 'c_uint', 'c_long', + 'c_ulong', 'c_longlong', 'c_ulonglong', 'c_longdouble', 'c_void' + 'i8', 'u8', 'i16', 'u16', 'i32', 'u32', 'i64', 'u64', 'i128', + 'u128'), suffix=r'\b'), + Keyword.Type) + + storage_keywords = ( + words(('const', 'var', 'extern', 'packed', 'export', 'pub', 'noalias', + 'inline', 'comptime', 'nakedcc', 'stdcallcc', 'volatile', 'allowzero', + 'align', 'linksection', 'threadlocal'), suffix=r'\b'), + Keyword.Reserved) + + structure_keywords = ( + words(('struct', 'enum', 'union', 'error'), suffix=r'\b'), + Keyword) + + statement_keywords = ( + words(('break', 'return', 'continue', 'asm', 'defer', 'errdefer', + 'unreachable', 'try', 'catch', 'async', 'await', 'suspend', + 'resume', 'cancel'), suffix=r'\b'), + Keyword) + + conditional_keywords = ( + words(('if', 'else', 'switch', 'and', 'or', 'orelse'), suffix=r'\b'), + Keyword) + + repeat_keywords = ( + words(('while', 'for'), suffix=r'\b'), + Keyword) + + other_keywords = ( + words(('fn', 'usingnamespace', 'test'), suffix=r'\b'), + Keyword) + + constant_keywords = ( + words(('true', 'false', 'null', 'undefined'), suffix=r'\b'), + Keyword.Constant) + + tokens = { + 'root': [ + (r'\n', Whitespace), + (r'\s+', Whitespace), + (r'//.*?\n', Comment.Single), + + # Keywords + statement_keywords, + storage_keywords, + structure_keywords, + repeat_keywords, + type_keywords, + constant_keywords, + conditional_keywords, + other_keywords, + + # Floats + (r'0x[0-9a-fA-F]+\.[0-9a-fA-F]+([pP][\-+]?[0-9a-fA-F]+)?', Number.Float), + (r'0x[0-9a-fA-F]+\.?[pP][\-+]?[0-9a-fA-F]+', Number.Float), + (r'[0-9]+\.[0-9]+([eE][-+]?[0-9]+)?', Number.Float), + (r'[0-9]+\.?[eE][-+]?[0-9]+', Number.Float), + + # Integers + (r'0b[01]+', Number.Bin), + (r'0o[0-7]+', Number.Oct), + (r'0x[0-9a-fA-F]+', Number.Hex), + (r'[0-9]+', Number.Integer), + + # Identifier + (r'@[a-zA-Z_]\w*', Name.Builtin), + (r'[a-zA-Z_]\w*', Name), + + # Characters + (r'\'\\\'\'', String.Escape), + (r'\'\\(x[a-fA-F0-9]{2}|u[a-fA-F0-9]{4}|U[a-fA-F0-9]{6}|[nr\\t\'"])\'', + String.Escape), + (r'\'[^\\\']\'', String), + + # Strings + (r'\\\\[^\n]*', String.Heredoc), + (r'c\\\\[^\n]*', String.Heredoc), + (r'c?"', String, 'string'), + + # Operators, Punctuation + (r'[+%=><|^!?/\-*&~:]', Operator), + (r'[{}()\[\],.;]', Punctuation) + ], + 'string': [ + (r'\\(x[a-fA-F0-9]{2}|u[a-fA-F0-9]{4}|U[a-fA-F0-9]{6}|[nr\\t\'"])', + String.Escape), + (r'[^\\"\n]+', String), + (r'"', String, '#pop') + ] + } diff --git a/src/syntax_highlighting/libs/pygments/modeline.py b/src/syntax_highlighting/libs/pygments/modeline.py index 9f8d5da..047d86d 100644 --- a/src/syntax_highlighting/libs/pygments/modeline.py +++ b/src/syntax_highlighting/libs/pygments/modeline.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.modeline ~~~~~~~~~~~~~~~~~ A simple modeline parser (based on pymodeline). - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/src/syntax_highlighting/libs/pygments/plugin.py b/src/syntax_highlighting/libs/pygments/plugin.py index 7987d64..b1085b7 100644 --- a/src/syntax_highlighting/libs/pygments/plugin.py +++ b/src/syntax_highlighting/libs/pygments/plugin.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ pygments.plugin ~~~~~~~~~~~~~~~ @@ -32,7 +31,7 @@ yourfilter = yourfilter:YourFilter - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ LEXER_ENTRY_POINT = 'pygments.lexers' @@ -40,14 +39,16 @@ STYLE_ENTRY_POINT = 'pygments.styles' FILTER_ENTRY_POINT = 'pygments.filters' + def iter_entry_points(group_name): try: import pkg_resources - except ImportError: + except (ImportError, OSError): return [] return pkg_resources.iter_entry_points(group_name) + def find_plugin_lexers(): for entrypoint in iter_entry_points(LEXER_ENTRY_POINT): yield entrypoint.load() diff --git a/src/syntax_highlighting/libs/pygments/regexopt.py b/src/syntax_highlighting/libs/pygments/regexopt.py index dcfae2f..cb2c8e2 100644 --- a/src/syntax_highlighting/libs/pygments/regexopt.py +++ b/src/syntax_highlighting/libs/pygments/regexopt.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ pygments.regexopt ~~~~~~~~~~~~~~~~~ @@ -6,7 +5,7 @@ An algorithm that generates optimized regexes for matching long lists of literal strings. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -16,7 +15,7 @@ from itertools import groupby from operator import itemgetter -CS_ESCAPE = re.compile(r'[\^\\\-\]]') +CS_ESCAPE = re.compile(r'[\[\^\\\-\]]') FIRST_ELEMENT = itemgetter(0) diff --git a/src/syntax_highlighting/libs/pygments/scanner.py b/src/syntax_highlighting/libs/pygments/scanner.py index 3350ac8..5f32a22 100644 --- a/src/syntax_highlighting/libs/pygments/scanner.py +++ b/src/syntax_highlighting/libs/pygments/scanner.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ pygments.scanner ~~~~~~~~~~~~~~~~ @@ -12,7 +11,7 @@ Have a look at the `DelphiLexer` to get an idea of how to use this scanner. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ import re @@ -25,7 +24,7 @@ class EndOfText(RuntimeError): """ -class Scanner(object): +class Scanner: """ Simple scanner diff --git a/src/syntax_highlighting/libs/pygments/sphinxext.py b/src/syntax_highlighting/libs/pygments/sphinxext.py index f962f8c..644eb8c 100644 --- a/src/syntax_highlighting/libs/pygments/sphinxext.py +++ b/src/syntax_highlighting/libs/pygments/sphinxext.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ pygments.sphinxext ~~~~~~~~~~~~~~~~~~ @@ -6,17 +5,15 @@ Sphinx extension to generate automatic documentation of lexers, formatters and filters. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ -from __future__ import print_function - import sys from docutils import nodes from docutils.statemachine import ViewList -from sphinx.util.compat import Directive +from docutils.parsers.rst import Directive from sphinx.util.nodes import nested_parse_with_titles diff --git a/src/syntax_highlighting/libs/pygments/style.py b/src/syntax_highlighting/libs/pygments/style.py index 879c4e0..8ed0594 100644 --- a/src/syntax_highlighting/libs/pygments/style.py +++ b/src/syntax_highlighting/libs/pygments/style.py @@ -1,37 +1,56 @@ -# -*- coding: utf-8 -*- """ pygments.style ~~~~~~~~~~~~~~ Basic style object. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ from pygments.token import Token, STANDARD_TYPES -from pygments.util import add_metaclass -# Default mapping of #ansixxx to RGB colors. +# Default mapping of ansixxx to RGB colors. _ansimap = { # dark - '#ansiblack': '000000', - '#ansidarkred': '7f0000', - '#ansidarkgreen': '007f00', - '#ansibrown': '7f7fe0', - '#ansidarkblue': '00007f', - '#ansipurple': '7f007f', - '#ansiteal': '007f7f', - '#ansilightgray': 'e5e5e5', + 'ansiblack': '000000', + 'ansired': '7f0000', + 'ansigreen': '007f00', + 'ansiyellow': '7f7fe0', + 'ansiblue': '00007f', + 'ansimagenta': '7f007f', + 'ansicyan': '007f7f', + 'ansigray': 'e5e5e5', # normal - '#ansidarkgray': '555555', - '#ansired': 'ff0000', - '#ansigreen': '00ff00', - '#ansiyellow': 'ffff00', - '#ansiblue': '0000ff', - '#ansifuchsia': 'ff00ff', - '#ansiturquoise': '00ffff', - '#ansiwhite': 'ffffff', + 'ansibrightblack': '555555', + 'ansibrightred': 'ff0000', + 'ansibrightgreen': '00ff00', + 'ansibrightyellow': 'ffff00', + 'ansibrightblue': '0000ff', + 'ansibrightmagenta': 'ff00ff', + 'ansibrightcyan': '00ffff', + 'ansiwhite': 'ffffff', +} +# mapping of deprecated #ansixxx colors to new color names +_deprecated_ansicolors = { + # dark + '#ansiblack': 'ansiblack', + '#ansidarkred': 'ansired', + '#ansidarkgreen': 'ansigreen', + '#ansibrown': 'ansiyellow', + '#ansidarkblue': 'ansiblue', + '#ansipurple': 'ansimagenta', + '#ansiteal': 'ansicyan', + '#ansilightgray': 'ansigray', + # normal + '#ansidarkgray': 'ansibrightblack', + '#ansired': 'ansibrightred', + '#ansigreen': 'ansibrightgreen', + '#ansiyellow': 'ansibrightyellow', + '#ansiblue': 'ansibrightblue', + '#ansifuchsia': 'ansibrightmagenta', + '#ansiturquoise': 'ansibrightcyan', + '#ansiwhite': 'ansiwhite', } ansicolors = set(_ansimap) @@ -52,9 +71,11 @@ def colorformat(text): if len(col) == 6: return col elif len(col) == 3: - return col[0]*2 + col[1]*2 + col[2]*2 + return col[0] * 2 + col[1] * 2 + col[2] * 2 elif text == '': return '' + elif text.startswith('var') or text.startswith('calc'): + return text assert False, "wrong color format %r" % text _styles = obj._styles = {} @@ -106,11 +127,15 @@ def style_for_token(cls, token): t = cls._styles[token] ansicolor = bgansicolor = None color = t[0] - if color.startswith('#ansi'): + if color in _deprecated_ansicolors: + color = _deprecated_ansicolors[color] + if color in ansicolors: ansicolor = color color = _ansimap[color] bgcolor = t[4] - if bgcolor.startswith('#ansi'): + if bgcolor in _deprecated_ansicolors: + bgcolor = _deprecated_ansicolors[bgcolor] + if bgcolor in ansicolors: bgansicolor = bgcolor bgcolor = _ansimap[bgcolor] @@ -142,8 +167,7 @@ def __len__(cls): return len(cls._styles) -@add_metaclass(StyleMeta) -class Style(object): +class Style(metaclass=StyleMeta): #: overall background color (``None`` means transparent) background_color = '#ffffff' @@ -151,5 +175,17 @@ class Style(object): #: highlight background color highlight_color = '#ffffcc' + #: line number font color + line_number_color = 'inherit' + + #: line number background color + line_number_background_color = 'transparent' + + #: special line number font color + line_number_special_color = '#000000' + + #: special line number background color + line_number_special_background_color = '#ffffc0' + #: Style definitions for individual token types. styles = {} diff --git a/src/syntax_highlighting/libs/pygments/styles/__init__.py b/src/syntax_highlighting/libs/pygments/styles/__init__.py index 839a9b7..50f6cb6 100644 --- a/src/syntax_highlighting/libs/pygments/styles/__init__.py +++ b/src/syntax_highlighting/libs/pygments/styles/__init__.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.styles ~~~~~~~~~~~~~~~ Contains built-in styles. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -22,6 +21,7 @@ 'autumn': 'autumn::AutumnStyle', 'murphy': 'murphy::MurphyStyle', 'manni': 'manni::ManniStyle', + 'material': 'material::MaterialStyle', 'monokai': 'monokai::MonokaiStyle', 'perldoc': 'perldoc::PerldocStyle', 'pastie': 'pastie::PastieStyle', @@ -44,6 +44,16 @@ 'arduino': 'arduino::ArduinoStyle', 'rainbow_dash': 'rainbow_dash::RainbowDashStyle', 'abap': 'abap::AbapStyle', + 'solarized-dark': 'solarized::SolarizedDarkStyle', + 'solarized-light': 'solarized::SolarizedLightStyle', + 'sas': 'sas::SasStyle', + 'stata': 'stata_light::StataLightStyle', + 'stata-light': 'stata_light::StataLightStyle', + 'stata-dark': 'stata_dark::StataDarkStyle', + 'inkpot': 'inkpot::InkPotStyle', + 'zenburn': 'zenburn::ZenburnStyle', + 'gruvbox-dark': 'gruvbox::GruvboxDarkStyle', + 'gruvbox-light': 'gruvbox::GruvboxLightStyle', } @@ -72,9 +82,8 @@ def get_style_by_name(name): def get_all_styles(): - """Return an generator for all styles by name, + """Return a generator for all styles by name, both builtin and plugin.""" - for name in STYLE_MAP: - yield name + yield from STYLE_MAP for name, _ in find_plugin_styles(): yield name diff --git a/src/syntax_highlighting/libs/pygments/styles/abap.py b/src/syntax_highlighting/libs/pygments/styles/abap.py index 91286a3..1e51e05 100644 --- a/src/syntax_highlighting/libs/pygments/styles/abap.py +++ b/src/syntax_highlighting/libs/pygments/styles/abap.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.styles.abap ~~~~~~~~~~~~~~~~~~~~ ABAP workbench like style. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/src/syntax_highlighting/libs/pygments/styles/algol.py b/src/syntax_highlighting/libs/pygments/styles/algol.py index 16461e0..f293b0e 100644 --- a/src/syntax_highlighting/libs/pygments/styles/algol.py +++ b/src/syntax_highlighting/libs/pygments/styles/algol.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ pygments.styles.algol ~~~~~~~~~~~~~~~~~~~~~ @@ -26,7 +25,7 @@ [1] `Revised Report on the Algorithmic Language Algol-60 ` - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/src/syntax_highlighting/libs/pygments/styles/algol_nu.py b/src/syntax_highlighting/libs/pygments/styles/algol_nu.py index 366ae21..fe28466 100644 --- a/src/syntax_highlighting/libs/pygments/styles/algol_nu.py +++ b/src/syntax_highlighting/libs/pygments/styles/algol_nu.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ pygments.styles.algol_nu ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -26,7 +25,7 @@ [1] `Revised Report on the Algorithmic Language Algol-60 ` - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/src/syntax_highlighting/libs/pygments/styles/arduino.py b/src/syntax_highlighting/libs/pygments/styles/arduino.py index 57e3809..4dfe0f3 100644 --- a/src/syntax_highlighting/libs/pygments/styles/arduino.py +++ b/src/syntax_highlighting/libs/pygments/styles/arduino.py @@ -1,17 +1,16 @@ -# -*- coding: utf-8 -*- """ pygments.styles.arduino ~~~~~~~~~~~~~~~~~~~~~~~ Arduino® Syntax highlighting style. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ from pygments.style import Style from pygments.token import Keyword, Name, Comment, String, Error, \ - Number, Operator, Generic, Whitespace + Number, Operator, Generic, Whitespace class ArduinoStyle(Style): diff --git a/src/syntax_highlighting/libs/pygments/styles/autumn.py b/src/syntax_highlighting/libs/pygments/styles/autumn.py index 71b93b1..85fd898 100644 --- a/src/syntax_highlighting/libs/pygments/styles/autumn.py +++ b/src/syntax_highlighting/libs/pygments/styles/autumn.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.styles.autumn ~~~~~~~~~~~~~~~~~~~~~~ A colorful style, inspired by the terminal highlighting style. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/src/syntax_highlighting/libs/pygments/styles/borland.py b/src/syntax_highlighting/libs/pygments/styles/borland.py index 0d13d1a..427e149 100644 --- a/src/syntax_highlighting/libs/pygments/styles/borland.py +++ b/src/syntax_highlighting/libs/pygments/styles/borland.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.styles.borland ~~~~~~~~~~~~~~~~~~~~~~~ Style similar to the style used in the Borland IDEs. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/src/syntax_highlighting/libs/pygments/styles/bw.py b/src/syntax_highlighting/libs/pygments/styles/bw.py index f0a6b14..1b38538 100644 --- a/src/syntax_highlighting/libs/pygments/styles/bw.py +++ b/src/syntax_highlighting/libs/pygments/styles/bw.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.styles.bw ~~~~~~~~~~~~~~~~~~ Simple black/white only style. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/src/syntax_highlighting/libs/pygments/styles/colorful.py b/src/syntax_highlighting/libs/pygments/styles/colorful.py index bfc0b50..a67b8e3 100644 --- a/src/syntax_highlighting/libs/pygments/styles/colorful.py +++ b/src/syntax_highlighting/libs/pygments/styles/colorful.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.styles.colorful ~~~~~~~~~~~~~~~~~~~~~~~~ A colorful style, inspired by CodeRay. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/src/syntax_highlighting/libs/pygments/styles/default.py b/src/syntax_highlighting/libs/pygments/styles/default.py index 6b9bd44..414de2f 100644 --- a/src/syntax_highlighting/libs/pygments/styles/default.py +++ b/src/syntax_highlighting/libs/pygments/styles/default.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.styles.default ~~~~~~~~~~~~~~~~~~~~~~~ The default highlighting style. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/src/syntax_highlighting/libs/pygments/styles/emacs.py b/src/syntax_highlighting/libs/pygments/styles/emacs.py index af15f30..e9cbc33 100644 --- a/src/syntax_highlighting/libs/pygments/styles/emacs.py +++ b/src/syntax_highlighting/libs/pygments/styles/emacs.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.styles.emacs ~~~~~~~~~~~~~~~~~~~~~ A highlighting style for Pygments, inspired by Emacs. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/src/syntax_highlighting/libs/pygments/styles/friendly.py b/src/syntax_highlighting/libs/pygments/styles/friendly.py index b2d1c0c..82f2479 100644 --- a/src/syntax_highlighting/libs/pygments/styles/friendly.py +++ b/src/syntax_highlighting/libs/pygments/styles/friendly.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.styles.friendly ~~~~~~~~~~~~~~~~~~~~~~~~ A modern style based on the VIM pyte theme. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -21,6 +20,7 @@ class FriendlyStyle(Style): background_color = "#f0f0f0" default_style = "" + line_number_color = "#666666" styles = { Whitespace: "#bbbbbb", diff --git a/src/syntax_highlighting/libs/pygments/styles/fruity.py b/src/syntax_highlighting/libs/pygments/styles/fruity.py index 1bbe031..2446915 100644 --- a/src/syntax_highlighting/libs/pygments/styles/fruity.py +++ b/src/syntax_highlighting/libs/pygments/styles/fruity.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.styles.fruity ~~~~~~~~~~~~~~~~~~~~~~ pygments version of my "fruity" vim theme. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/src/syntax_highlighting/libs/pygments/styles/gruvbox.py b/src/syntax_highlighting/libs/pygments/styles/gruvbox.py new file mode 100644 index 0000000..ddf05ea --- /dev/null +++ b/src/syntax_highlighting/libs/pygments/styles/gruvbox.py @@ -0,0 +1,107 @@ +""" + pygments.styles.gruvbox + ~~~~~~~~~~~~~~~~~~~~~~~ + + pygments version of the "gruvbox" vim theme. + https://github.com/morhetz/gruvbox + + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.style import Style +from pygments.token import Keyword, Name, Comment, String, Error, \ + Number, Operator, Generic + + +class GruvboxDarkStyle(Style): + """ + Pygments version of the "gruvbox" dark vim theme. + """ + + background_color = '#282828' + highlight_color = '#ebdbb2' + + styles = { + Comment: 'italic #928374', + Comment.PreProc: '#8ec07c', + Comment.Special: 'bold italic #ebdbb2', + + Keyword: '#fb4934', + Operator.Word: '#fb4934', + + String: '#b8bb26', + String.Escape: '#fe8019', + + Number: '#d3869b', + + Name.Builtin: '#fe8019', + Name.Variable: '#83a598', + Name.Constant: '#d3869b', + Name.Class: '#8ec07c', + Name.Function: '#8ec07c', + Name.Namespace: '#8ec07c', + Name.Exception: '#fb4934', + Name.Tag: '#8ec07c', + Name.Attribute: '#fabd2f', + Name.Decorator: '#fb4934', + + Generic.Heading: 'bold #ebdbb2', + Generic.Subheading: 'underline #ebdbb2', + Generic.Deleted: 'bg:#fb4934 #282828', + Generic.Inserted: 'bg:#b8bb26 #282828', + Generic.Error: '#fb4934', + Generic.Emph: 'italic', + Generic.Strong: 'bold', + Generic.Prompt: '#a89984', + Generic.Output: '#f2e5bc', + Generic.Traceback: '#fb4934', + + Error: 'bg:#fb4934 #282828' + } + +class GruvboxLightStyle(Style): + """ + Pygments version of the "gruvbox" Light vim theme. + """ + + background_color = '#fbf1c7' + highlight_color = '#3c3836' + + styles = { + Comment: 'italic #928374', + Comment.PreProc: '#427b58', + Comment.Special: 'bold italic #3c3836', + + Keyword: '#9d0006', + Operator.Word: '#9d0006', + + String: '#79740e', + String.Escape: '#af3a03', + + Number: '#8f3f71', + + Name.Builtin: '#af3a03', + Name.Variable: '#076678', + Name.Constant: '#8f3f71', + Name.Class: '#427b58', + Name.Function: '#427b58', + Name.Namespace: '#427b58', + Name.Exception: '#9d0006', + Name.Tag: '#427b58', + Name.Attribute: '#b57614', + Name.Decorator: '#9d0006', + + Generic.Heading: 'bold #3c3836', + Generic.Subheading: 'underline #3c3836', + Generic.Deleted: 'bg:#9d0006 #fbf1c7', + Generic.Inserted: 'bg:#79740e #fbf1c7', + Generic.Error: '#9d0006', + Generic.Emph: 'italic', + Generic.Strong: 'bold', + Generic.Prompt: '#7c6f64', + Generic.Output: '#32302f', + Generic.Traceback: '#9d0006', + + Error: 'bg:#9d0006 #fbf1c7' + } diff --git a/src/syntax_highlighting/libs/pygments/styles/igor.py b/src/syntax_highlighting/libs/pygments/styles/igor.py index d4620a4..278e9da 100644 --- a/src/syntax_highlighting/libs/pygments/styles/igor.py +++ b/src/syntax_highlighting/libs/pygments/styles/igor.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.styles.igor ~~~~~~~~~~~~~~~~~~~~ Igor Pro default style. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/src/syntax_highlighting/libs/pygments/styles/inkpot.py b/src/syntax_highlighting/libs/pygments/styles/inkpot.py new file mode 100644 index 0000000..d2eff2c --- /dev/null +++ b/src/syntax_highlighting/libs/pygments/styles/inkpot.py @@ -0,0 +1,66 @@ +""" + pygments.styles.inkpot + ~~~~~~~~~~~~~~~~~~~~~~ + + A highlighting style for Pygments, inspired by the Inkpot theme for VIM. + + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.style import Style +from pygments.token import Text, Other, Keyword, Name, Comment, String, \ + Error, Number, Operator, Generic, Whitespace, Punctuation + + +class InkPotStyle(Style): + background_color = "#1e1e27" + default_style = "" + styles = { + Text: "#cfbfad", + Other: "#cfbfad", + Whitespace: "#434357", + Comment: "#cd8b00", + Comment.Preproc: "#409090", + Comment.PreprocFile: "bg:#404040 #ffcd8b", + Comment.Special: "#808bed", + + Keyword: "#808bed", + Keyword.Pseudo: "nobold", + Keyword.Type: "#ff8bff", + + Operator: "#666666", + + Punctuation: "#cfbfad", + + Name: "#cfbfad", + Name.Attribute: "#cfbfad", + Name.Builtin.Pseudo: '#ffff00', + Name.Builtin: "#808bed", + Name.Class: "#ff8bff", + Name.Constant: "#409090", + Name.Decorator: "#409090", + Name.Exception: "#ff0000", + Name.Function: "#c080d0", + Name.Label: "#808bed", + Name.Namespace: "#ff0000", + Name.Variable: "#cfbfad", + + String: "bg:#404040 #ffcd8b", + String.Doc: "#808bed", + + Number: "#f0ad6d", + + Generic.Heading: "bold #000080", + Generic.Subheading: "bold #800080", + Generic.Deleted: "#A00000", + Generic.Inserted: "#00A000", + Generic.Error: "#FF0000", + Generic.Emph: "italic", + Generic.Strong: "bold", + Generic.Prompt: "bold #000080", + Generic.Output: "#888", + Generic.Traceback: "#04D", + + Error: "bg:#6e2e2e #ffffff" + } diff --git a/src/syntax_highlighting/libs/pygments/styles/lovelace.py b/src/syntax_highlighting/libs/pygments/styles/lovelace.py index 861f778..ec8d2a9 100644 --- a/src/syntax_highlighting/libs/pygments/styles/lovelace.py +++ b/src/syntax_highlighting/libs/pygments/styles/lovelace.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ pygments.styles.lovelace ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -9,7 +8,7 @@ A desaturated, somewhat subdued style created for the Lovelace interactive learning environment. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/src/syntax_highlighting/libs/pygments/styles/manni.py b/src/syntax_highlighting/libs/pygments/styles/manni.py index f0a325a..167dceb 100644 --- a/src/syntax_highlighting/libs/pygments/styles/manni.py +++ b/src/syntax_highlighting/libs/pygments/styles/manni.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ pygments.styles.manni ~~~~~~~~~~~~~~~~~~~~~ @@ -8,7 +7,7 @@ This is a port of the style used in the `php port`_ of pygments by Manni. The style is called 'default' there. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/src/syntax_highlighting/libs/pygments/styles/material.py b/src/syntax_highlighting/libs/pygments/styles/material.py new file mode 100644 index 0000000..c7012fe --- /dev/null +++ b/src/syntax_highlighting/libs/pygments/styles/material.py @@ -0,0 +1,118 @@ +""" + pygments.styles.material + ~~~~~~~~~~~~~~~~~~~~~~~~ + + Mimic the Material theme color scheme. + + https://github.com/material-theme/vsc-material-theme + + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.style import Style +from pygments.token import Keyword, Name, Comment, String, Escape, \ + Error, Text, Number, Operator, Generic, Punctuation, Literal + +class MaterialStyle(Style): + """ + This style mimics the Material Theme color scheme. + """ + dark_teal = '#263238' + white= '#FFFFFF' + black= '#000000' + red= '#FF5370' + orange= '#F78C6C' + yellow= '#FFCB6B' + green= '#C3E88D' + cyan= '#89DDFF' + blue= '#82AAFF' + paleblue= '#B2CCD6' + purple= '#C792EA' + brown= '#C17E70' + pink= '#F07178' + violet= '#BB80B3' + foreground = '#EEFFFF' + faded = '#546E7A' + + default_style = "" + background_color = dark_teal + highlight_color = '#2C3B41' + line_number_color = '#37474F' + line_number_background_color = dark_teal + line_number_special_color = '#607A86' + line_number_special_background_color = dark_teal + + styles = { + Text: foreground, + Escape: cyan, + Error: red, + + Keyword: violet, + Keyword.Constant: cyan, + Keyword.Declaration: violet, + Keyword.Namespace: 'italic ' + cyan, + Keyword.Pseudo: cyan, + Keyword.Type: violet, + + Name: foreground, + Name.Attribute: violet, + Name.Builtin: blue, + Name.Builtin.Pseudo: cyan, + Name.Class: yellow, + Name.Constant: foreground, + Name.Decorator: blue, + Name.Entity: cyan, + Name.Exception: yellow, + Name.Function: blue, + Name.Function.Magic: blue, + Name.Label: blue, + Name.Property: yellow, + Name.Namespace: yellow, + Name.Other: foreground, + Name.Tag: red, + Name.Variable: cyan, + Name.Variable.Class: cyan, + Name.Variable.Global: cyan, + Name.Variable.Instance: cyan, + Name.Variable.Magic: blue, + + Literal: green, + Literal.Date: green, + + String: green, + String.Affix: violet, + String.Backtick: green, + String.Char: green, + String.Delimiter: foreground, + String.Doc: 'italic ' + faded, + String.Double: green, + String.Escape: foreground, + String.Heredoc: green, + String.Interpol: cyan, + String.Other: green, + String.Regex: cyan, + String.Single: green, + String.Symbol: cyan, + + Number: orange, + + Operator: cyan, + Operator.Word: 'italic ' + cyan, + + Punctuation: cyan, + + Comment: 'italic ' + faded, + + Generic: foreground, + Generic.Deleted: red, + Generic.Emph: cyan, + Generic.Error: red, + Generic.Heading: green, + Generic.Inserted: green, + Generic.Output: faded, + Generic.Prompt: yellow, + Generic.Strong: red, + Generic.Subheading: cyan, + Generic.Traceback: red, + } diff --git a/src/syntax_highlighting/libs/pygments/styles/monokai.py b/src/syntax_highlighting/libs/pygments/styles/monokai.py index 337e2f8..9d69ec0 100644 --- a/src/syntax_highlighting/libs/pygments/styles/monokai.py +++ b/src/syntax_highlighting/libs/pygments/styles/monokai.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ pygments.styles.monokai ~~~~~~~~~~~~~~~~~~~~~~~ @@ -7,7 +6,7 @@ http://www.monokai.nl/blog/2006/07/15/textmate-color-theme/ - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -92,14 +91,15 @@ class MonokaiStyle(Style): String.Single: "", # class: 's1' String.Symbol: "", # class: 'ss' + Generic: "", # class: 'g' Generic.Deleted: "#f92672", # class: 'gd', Generic.Emph: "italic", # class: 'ge' Generic.Error: "", # class: 'gr' Generic.Heading: "", # class: 'gh' Generic.Inserted: "#a6e22e", # class: 'gi' - Generic.Output: "", # class: 'go' - Generic.Prompt: "", # class: 'gp' + Generic.Output: "#66d9ef", # class: 'go' + Generic.Prompt: "bold #f92672", # class: 'gp' Generic.Strong: "bold", # class: 'gs' Generic.Subheading: "#75715e", # class: 'gu' Generic.Traceback: "", # class: 'gt' diff --git a/src/syntax_highlighting/libs/pygments/styles/murphy.py b/src/syntax_highlighting/libs/pygments/styles/murphy.py index c827006..9115c2d 100644 --- a/src/syntax_highlighting/libs/pygments/styles/murphy.py +++ b/src/syntax_highlighting/libs/pygments/styles/murphy.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.styles.murphy ~~~~~~~~~~~~~~~~~~~~~~ Murphy's style from CodeRay. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/src/syntax_highlighting/libs/pygments/styles/native.py b/src/syntax_highlighting/libs/pygments/styles/native.py index 921a58d..9c2f2c1 100644 --- a/src/syntax_highlighting/libs/pygments/styles/native.py +++ b/src/syntax_highlighting/libs/pygments/styles/native.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.styles.native ~~~~~~~~~~~~~~~~~~~~~~ pygments version of my "native" vim theme. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -21,6 +20,7 @@ class NativeStyle(Style): background_color = '#202020' highlight_color = '#404040' + line_number_color = '#aaaaaa' styles = { Token: '#d0d0d0', diff --git a/src/syntax_highlighting/libs/pygments/styles/paraiso_dark.py b/src/syntax_highlighting/libs/pygments/styles/paraiso_dark.py index 5f334bb..2556139 100644 --- a/src/syntax_highlighting/libs/pygments/styles/paraiso_dark.py +++ b/src/syntax_highlighting/libs/pygments/styles/paraiso_dark.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ pygments.styles.paraiso_dark ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -9,7 +8,7 @@ Created with Base16 Builder by Chris Kempson (https://github.com/chriskempson/base16-builder). - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/src/syntax_highlighting/libs/pygments/styles/paraiso_light.py b/src/syntax_highlighting/libs/pygments/styles/paraiso_light.py index a811281..608d173 100644 --- a/src/syntax_highlighting/libs/pygments/styles/paraiso_light.py +++ b/src/syntax_highlighting/libs/pygments/styles/paraiso_light.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ pygments.styles.paraiso_light ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -9,7 +8,7 @@ Created with Base16 Builder by Chris Kempson (https://github.com/chriskempson/base16-builder). - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/src/syntax_highlighting/libs/pygments/styles/pastie.py b/src/syntax_highlighting/libs/pygments/styles/pastie.py index d614290..14ec700 100644 --- a/src/syntax_highlighting/libs/pygments/styles/pastie.py +++ b/src/syntax_highlighting/libs/pygments/styles/pastie.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ pygments.styles.pastie ~~~~~~~~~~~~~~~~~~~~~~ @@ -7,7 +6,7 @@ .. _pastie: http://pastie.caboo.se/ - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/src/syntax_highlighting/libs/pygments/styles/perldoc.py b/src/syntax_highlighting/libs/pygments/styles/perldoc.py index 24af2df..5eae210 100644 --- a/src/syntax_highlighting/libs/pygments/styles/perldoc.py +++ b/src/syntax_highlighting/libs/pygments/styles/perldoc.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ pygments.styles.perldoc ~~~~~~~~~~~~~~~~~~~~~~~ @@ -7,7 +6,7 @@ .. _perldoc: http://perldoc.perl.org/ - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/src/syntax_highlighting/libs/pygments/styles/rainbow_dash.py b/src/syntax_highlighting/libs/pygments/styles/rainbow_dash.py index 7cf5c9d..3996332 100644 --- a/src/syntax_highlighting/libs/pygments/styles/rainbow_dash.py +++ b/src/syntax_highlighting/libs/pygments/styles/rainbow_dash.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ pygments.styles.rainbow_dash ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -7,7 +6,7 @@ .. _theme: http://sanssecours.github.io/Rainbow-Dash.tmbundle - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/src/syntax_highlighting/libs/pygments/styles/rrt.py b/src/syntax_highlighting/libs/pygments/styles/rrt.py index 96f9490..455f1d9 100644 --- a/src/syntax_highlighting/libs/pygments/styles/rrt.py +++ b/src/syntax_highlighting/libs/pygments/styles/rrt.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.styles.rrt ~~~~~~~~~~~~~~~~~~~ pygments "rrt" theme, based on Zap and Emacs defaults. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/src/syntax_highlighting/libs/pygments/styles/sas.py b/src/syntax_highlighting/libs/pygments/styles/sas.py index 78686fc..f620353 100644 --- a/src/syntax_highlighting/libs/pygments/styles/sas.py +++ b/src/syntax_highlighting/libs/pygments/styles/sas.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ pygments.styles.sas ~~~~~~~~~~~~~~~~~~~ @@ -7,7 +6,7 @@ meant to be a complete style. It's merely meant to mimic SAS' program editor syntax highlighting. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/src/syntax_highlighting/libs/pygments/styles/solarized.py b/src/syntax_highlighting/libs/pygments/styles/solarized.py new file mode 100644 index 0000000..1e051eb --- /dev/null +++ b/src/syntax_highlighting/libs/pygments/styles/solarized.py @@ -0,0 +1,136 @@ +""" + pygments.styles.solarized + ~~~~~~~~~~~~~~~~~~~~~~~~~ + + Solarized by Camil Staps + + A Pygments style for the Solarized themes (licensed under MIT). + See: https://github.com/altercation/solarized + + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.style import Style +from pygments.token import Comment, Error, Generic, Keyword, Name, Number, \ + Operator, String, Token + + +def make_style(colors): + return { + Token: colors['base0'], + + Comment: 'italic ' + colors['base01'], + Comment.Hashbang: colors['base01'], + Comment.Multiline: colors['base01'], + Comment.Preproc: 'noitalic ' + colors['magenta'], + Comment.PreprocFile: 'noitalic ' + colors['base01'], + + Keyword: colors['green'], + Keyword.Constant: colors['cyan'], + Keyword.Declaration: colors['cyan'], + Keyword.Namespace: colors['orange'], + Keyword.Type: colors['yellow'], + + Operator: colors['base01'], + Operator.Word: colors['green'], + + Name.Builtin: colors['blue'], + Name.Builtin.Pseudo: colors['blue'], + Name.Class: colors['blue'], + Name.Constant: colors['blue'], + Name.Decorator: colors['blue'], + Name.Entity: colors['blue'], + Name.Exception: colors['blue'], + Name.Function: colors['blue'], + Name.Function.Magic: colors['blue'], + Name.Label: colors['blue'], + Name.Namespace: colors['blue'], + Name.Tag: colors['blue'], + Name.Variable: colors['blue'], + Name.Variable.Global:colors['blue'], + Name.Variable.Magic: colors['blue'], + + String: colors['cyan'], + String.Doc: colors['base01'], + String.Regex: colors['orange'], + + Number: colors['cyan'], + + Generic: colors['base0'], + Generic.Deleted: colors['red'], + Generic.Emph: 'italic', + Generic.Error: colors['red'], + Generic.Heading: 'bold', + Generic.Subheading: 'underline', + Generic.Inserted: colors['green'], + Generic.Output: colors['base0'], + Generic.Prompt: 'bold ' + colors['blue'], + Generic.Strong: 'bold', + Generic.Traceback: colors['blue'], + + Error: 'bg:' + colors['red'], + } + + +DARK_COLORS = { + 'base03': '#002b36', + 'base02': '#073642', + 'base01': '#586e75', + 'base00': '#657b83', + 'base0': '#839496', + 'base1': '#93a1a1', + 'base2': '#eee8d5', + 'base3': '#fdf6e3', + 'yellow': '#b58900', + 'orange': '#cb4b16', + 'red': '#dc322f', + 'magenta': '#d33682', + 'violet': '#6c71c4', + 'blue': '#268bd2', + 'cyan': '#2aa198', + 'green': '#859900', +} + +LIGHT_COLORS = { + 'base3': '#002b36', + 'base2': '#073642', + 'base1': '#586e75', + 'base0': '#657b83', + 'base00': '#839496', + 'base01': '#93a1a1', + 'base02': '#eee8d5', + 'base03': '#fdf6e3', + 'yellow': '#b58900', + 'orange': '#cb4b16', + 'red': '#dc322f', + 'magenta': '#d33682', + 'violet': '#6c71c4', + 'blue': '#268bd2', + 'cyan': '#2aa198', + 'green': '#859900', +} + + +class SolarizedDarkStyle(Style): + """ + The solarized style, dark. + """ + + styles = make_style(DARK_COLORS) + background_color = DARK_COLORS['base03'] + highlight_color = DARK_COLORS['base02'] + line_number_color = DARK_COLORS['base01'] + line_number_background_color = DARK_COLORS['base02'] + + +class SolarizedLightStyle(SolarizedDarkStyle): + """ + The solarized style, light. + """ + + styles = make_style(LIGHT_COLORS) + background_color = LIGHT_COLORS['base03'] + highlight_color = LIGHT_COLORS['base02'] + line_number_color = LIGHT_COLORS['base01'] + line_number_background_color = LIGHT_COLORS['base02'] diff --git a/src/syntax_highlighting/libs/pygments/styles/stata_dark.py b/src/syntax_highlighting/libs/pygments/styles/stata_dark.py new file mode 100644 index 0000000..073f013 --- /dev/null +++ b/src/syntax_highlighting/libs/pygments/styles/stata_dark.py @@ -0,0 +1,40 @@ +""" + pygments.styles.stata_dark + ~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Dark style inspired by Stata's do-file editor. Note this is not + meant to be a complete style, just for Stata's file formats. + + + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.style import Style +from pygments.token import Keyword, Name, Comment, String, Error, \ + Number, Operator, Whitespace, Generic, Text + + +class StataDarkStyle(Style): + + default_style = '' + + background_color = "#232629" + highlight_color = "#49483e" + + styles = { + Whitespace: '#bbbbbb', + Error: 'bg:#e3d2d2 #a61717', + Text: '#cccccc', + String: '#51cc99', + Number: '#4FB8CC', + Operator: '', + Name.Function: '#6a6aff', + Name.Other: '#e2828e', + Keyword: 'bold #7686bb', + Keyword.Constant: '', + Comment: 'italic #777777', + Name.Variable: 'bold #7AB4DB', + Name.Variable.Global: 'bold #BE646C', + Generic.Prompt: '#ffffff', + } diff --git a/src/syntax_highlighting/libs/pygments/styles/stata.py b/src/syntax_highlighting/libs/pygments/styles/stata_light.py similarity index 52% rename from src/syntax_highlighting/libs/pygments/styles/stata.py rename to src/syntax_highlighting/libs/pygments/styles/stata_light.py index 2b5f5ed..9dead70 100644 --- a/src/syntax_highlighting/libs/pygments/styles/stata.py +++ b/src/syntax_highlighting/libs/pygments/styles/stata_light.py @@ -1,40 +1,38 @@ -# -*- coding: utf-8 -*- """ - pygments.styles.stata - ~~~~~~~~~~~~~~~~~~~~~ + pygments.styles.stata_light + ~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Style inspired by Stata's do-file editor. Note this is not meant - to be a complete style. It's merely meant to mimic Stata's do file - editor syntax highlighting. + Light Style inspired by Stata's do-file editor. Note this is not + meant to be a complete style, just for Stata's file formats. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ from pygments.style import Style from pygments.token import Keyword, Name, Comment, String, Error, \ - Number, Operator, Whitespace + Number, Operator, Whitespace, Text -class StataStyle(Style): +class StataLightStyle(Style): """ - Style inspired by Stata's do-file editor. Note this is not meant - to be a complete style. It's merely meant to mimic Stata's do file - editor syntax highlighting. + Light mode style inspired by Stata's do-file editor. This is not + meant to be a complete style, just for use with Stata. """ default_style = '' - styles = { + Text: '#111111', Whitespace: '#bbbbbb', - Comment: 'italic #008800', + Error: 'bg:#e3d2d2 #a61717', String: '#7a2424', Number: '#2c2cff', Operator: '', + Name.Function: '#2c2cff', + Name.Other: '#be646c', Keyword: 'bold #353580', Keyword.Constant: '', - Name.Function: '#2c2cff', + Comment: 'italic #008800', Name.Variable: 'bold #35baba', Name.Variable.Global: 'bold #b5565e', - Error: 'bg:#e3d2d2 #a61717' } diff --git a/src/syntax_highlighting/libs/pygments/styles/tango.py b/src/syntax_highlighting/libs/pygments/styles/tango.py index 2abc8c6..694e277 100644 --- a/src/syntax_highlighting/libs/pygments/styles/tango.py +++ b/src/syntax_highlighting/libs/pygments/styles/tango.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ pygments.styles.tango ~~~~~~~~~~~~~~~~~~~~~ @@ -33,7 +32,7 @@ have been chosen to have the same style. Similarly, keywords (Keyword.*), and Operator.Word (and, or, in) have been assigned the same style. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/src/syntax_highlighting/libs/pygments/styles/trac.py b/src/syntax_highlighting/libs/pygments/styles/trac.py index aff39fd..b2a1fd4 100644 --- a/src/syntax_highlighting/libs/pygments/styles/trac.py +++ b/src/syntax_highlighting/libs/pygments/styles/trac.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.styles.trac ~~~~~~~~~~~~~~~~~~~~ Port of the default trac highlighter design. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/src/syntax_highlighting/libs/pygments/styles/vim.py b/src/syntax_highlighting/libs/pygments/styles/vim.py index 888088b..f498606 100644 --- a/src/syntax_highlighting/libs/pygments/styles/vim.py +++ b/src/syntax_highlighting/libs/pygments/styles/vim.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.styles.vim ~~~~~~~~~~~~~~~~~~~ A highlighting style for Pygments, inspired by vim. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/src/syntax_highlighting/libs/pygments/styles/vs.py b/src/syntax_highlighting/libs/pygments/styles/vs.py index bc3ed2b..a504f91 100644 --- a/src/syntax_highlighting/libs/pygments/styles/vs.py +++ b/src/syntax_highlighting/libs/pygments/styles/vs.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.styles.vs ~~~~~~~~~~~~~~~~~~ Simple style with MS Visual Studio colors. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/src/syntax_highlighting/libs/pygments/styles/xcode.py b/src/syntax_highlighting/libs/pygments/styles/xcode.py index 64bfcf0..fd8621e 100644 --- a/src/syntax_highlighting/libs/pygments/styles/xcode.py +++ b/src/syntax_highlighting/libs/pygments/styles/xcode.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.styles.xcode ~~~~~~~~~~~~~~~~~~~~~ Style similar to the `Xcode` default theme. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/src/syntax_highlighting/libs/pygments/styles/zenburn.py b/src/syntax_highlighting/libs/pygments/styles/zenburn.py new file mode 100644 index 0000000..8797df7 --- /dev/null +++ b/src/syntax_highlighting/libs/pygments/styles/zenburn.py @@ -0,0 +1,80 @@ +""" + pygments.styles.zenburn + ~~~~~~~~~~~~~~~~~~~~~~~ + + Low contrast color scheme Zenburn. + + See: https://kippura.org/zenburnpage/ + https://github.com/jnurmine/Zenburn + + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.style import Style +from pygments.token import ( + Token, Name, Operator, Keyword, Generic, Comment, Number, String, Literal, + Punctuation, Error, +) + + +class ZenburnStyle(Style): + """ + Low contrast Zenburn style. + """ + + default_style = "" + background_color = '#3f3f3f' + highlight_color = '#484848' + line_number_color = '#5d6262' + line_number_background_color = '#353535' + line_number_special_color = '#7a8080' + line_number_special_background_color = '#353535' + styles = { + Token: '#dcdccc', + Error: '#e37170 bold', + + Keyword: '#efdcbc', + Keyword.Type: '#dfdfbf bold', + Keyword.Constant: '#dca3a3', + Keyword.Declaration: '#f0dfaf', + Keyword.Namespace: '#f0dfaf', + + Name: '#dcdccc', + Name.Tag: '#e89393 bold', + Name.Entity: '#cfbfaf', + Name.Constant: '#dca3a3', + Name.Class: '#efef8f', + Name.Function: '#efef8f', + Name.Builtin: '#efef8f', + Name.Builtin.Pseudo: '#dcdccc', + Name.Attribute: '#efef8f', + Name.Exception: '#c3bf9f bold', + + Literal: '#9fafaf', + + String: '#cc9393', + String.Doc: '#7f9f7f', + String.Interpol: '#dca3a3 bold', + + Number: '#8cd0d3', + Number.Float: '#c0bed1', + + Operator: '#f0efd0', + + Punctuation: '#f0efd0', + + Comment: '#7f9f7f italic', + Comment.Preproc: '#dfaf8f bold', + Comment.PreprocFile: '#cc9393', + Comment.Special: '#dfdfdf bold', + + Generic: '#ecbcbc bold', + Generic.Emph: '#ffffff bold', + Generic.Output: '#5b605e bold', + Generic.Heading: '#efefef bold', + Generic.Deleted: '#c3bf9f bg:#313c36', + Generic.Inserted: '#709080 bg:#313c36 bold', + Generic.Traceback: '#80d4aa bg:#2f2f2f bold', + Generic.Subheading: '#efefef bold', + } diff --git a/src/syntax_highlighting/libs/pygments/token.py b/src/syntax_highlighting/libs/pygments/token.py index 43f73c8..9013acb 100644 --- a/src/syntax_highlighting/libs/pygments/token.py +++ b/src/syntax_highlighting/libs/pygments/token.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- """ pygments.token ~~~~~~~~~~~~~~ Basic token types and the standard tokens. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/src/syntax_highlighting/libs/pygments/unistring.py b/src/syntax_highlighting/libs/pygments/unistring.py index 6096d11..4db9c7c 100644 --- a/src/syntax_highlighting/libs/pygments/unistring.py +++ b/src/syntax_highlighting/libs/pygments/unistring.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ pygments.unistring ~~~~~~~~~~~~~~~~~~ @@ -8,128 +7,89 @@ Inspired by chartypes_create.py from the MoinMoin project. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ import sys -Cc = u'\x00-\x1f\x7f-\x9f' +Cc = '\x00-\x1f\x7f-\x9f' -Cf = u'\xad\u0600-\u0604\u061c\u06dd\u070f\u180e\u200b-\u200f\u202a-\u202e\u2060-\u2064\u2066-\u206f\ufeff\ufff9-\ufffb' +Cf = '\xad\u0600-\u0605\u061c\u06dd\u070f\u08e2\u180e\u200b-\u200f\u202a-\u202e\u2060-\u2064\u2066-\u206f\ufeff\ufff9-\ufffb\U000110bd\U000110cd\U0001bca0-\U0001bca3\U0001d173-\U0001d17a\U000e0001\U000e0020-\U000e007f' -Cn = u'\u0378-\u0379\u037f-\u0383\u038b\u038d\u03a2\u0528-\u0530\u0557-\u0558\u0560\u0588\u058b-\u058e\u0590\u05c8-\u05cf\u05eb-\u05ef\u05f5-\u05ff\u0605\u061d\u070e\u074b-\u074c\u07b2-\u07bf\u07fb-\u07ff\u082e-\u082f\u083f\u085c-\u085d\u085f-\u089f\u08a1\u08ad-\u08e3\u08ff\u0978\u0980\u0984\u098d-\u098e\u0991-\u0992\u09a9\u09b1\u09b3-\u09b5\u09ba-\u09bb\u09c5-\u09c6\u09c9-\u09ca\u09cf-\u09d6\u09d8-\u09db\u09de\u09e4-\u09e5\u09fc-\u0a00\u0a04\u0a0b-\u0a0e\u0a11-\u0a12\u0a29\u0a31\u0a34\u0a37\u0a3a-\u0a3b\u0a3d\u0a43-\u0a46\u0a49-\u0a4a\u0a4e-\u0a50\u0a52-\u0a58\u0a5d\u0a5f-\u0a65\u0a76-\u0a80\u0a84\u0a8e\u0a92\u0aa9\u0ab1\u0ab4\u0aba-\u0abb\u0ac6\u0aca\u0ace-\u0acf\u0ad1-\u0adf\u0ae4-\u0ae5\u0af2-\u0b00\u0b04\u0b0d-\u0b0e\u0b11-\u0b12\u0b29\u0b31\u0b34\u0b3a-\u0b3b\u0b45-\u0b46\u0b49-\u0b4a\u0b4e-\u0b55\u0b58-\u0b5b\u0b5e\u0b64-\u0b65\u0b78-\u0b81\u0b84\u0b8b-\u0b8d\u0b91\u0b96-\u0b98\u0b9b\u0b9d\u0ba0-\u0ba2\u0ba5-\u0ba7\u0bab-\u0bad\u0bba-\u0bbd\u0bc3-\u0bc5\u0bc9\u0bce-\u0bcf\u0bd1-\u0bd6\u0bd8-\u0be5\u0bfb-\u0c00\u0c04\u0c0d\u0c11\u0c29\u0c34\u0c3a-\u0c3c\u0c45\u0c49\u0c4e-\u0c54\u0c57\u0c5a-\u0c5f\u0c64-\u0c65\u0c70-\u0c77\u0c80-\u0c81\u0c84\u0c8d\u0c91\u0ca9\u0cb4\u0cba-\u0cbb\u0cc5\u0cc9\u0cce-\u0cd4\u0cd7-\u0cdd\u0cdf\u0ce4-\u0ce5\u0cf0\u0cf3-\u0d01\u0d04\u0d0d\u0d11\u0d3b-\u0d3c\u0d45\u0d49\u0d4f-\u0d56\u0d58-\u0d5f\u0d64-\u0d65\u0d76-\u0d78\u0d80-\u0d81\u0d84\u0d97-\u0d99\u0db2\u0dbc\u0dbe-\u0dbf\u0dc7-\u0dc9\u0dcb-\u0dce\u0dd5\u0dd7\u0de0-\u0df1\u0df5-\u0e00\u0e3b-\u0e3e\u0e5c-\u0e80\u0e83\u0e85-\u0e86\u0e89\u0e8b-\u0e8c\u0e8e-\u0e93\u0e98\u0ea0\u0ea4\u0ea6\u0ea8-\u0ea9\u0eac\u0eba\u0ebe-\u0ebf\u0ec5\u0ec7\u0ece-\u0ecf\u0eda-\u0edb\u0ee0-\u0eff\u0f48\u0f6d-\u0f70\u0f98\u0fbd\u0fcd\u0fdb-\u0fff\u10c6\u10c8-\u10cc\u10ce-\u10cf\u1249\u124e-\u124f\u1257\u1259\u125e-\u125f\u1289\u128e-\u128f\u12b1\u12b6-\u12b7\u12bf\u12c1\u12c6-\u12c7\u12d7\u1311\u1316-\u1317\u135b-\u135c\u137d-\u137f\u139a-\u139f\u13f5-\u13ff\u169d-\u169f\u16f1-\u16ff\u170d\u1715-\u171f\u1737-\u173f\u1754-\u175f\u176d\u1771\u1774-\u177f\u17de-\u17df\u17ea-\u17ef\u17fa-\u17ff\u180f\u181a-\u181f\u1878-\u187f\u18ab-\u18af\u18f6-\u18ff\u191d-\u191f\u192c-\u192f\u193c-\u193f\u1941-\u1943\u196e-\u196f\u1975-\u197f\u19ac-\u19af\u19ca-\u19cf\u19db-\u19dd\u1a1c-\u1a1d\u1a5f\u1a7d-\u1a7e\u1a8a-\u1a8f\u1a9a-\u1a9f\u1aae-\u1aff\u1b4c-\u1b4f\u1b7d-\u1b7f\u1bf4-\u1bfb\u1c38-\u1c3a\u1c4a-\u1c4c\u1c80-\u1cbf\u1cc8-\u1ccf\u1cf7-\u1cff\u1de7-\u1dfb\u1f16-\u1f17\u1f1e-\u1f1f\u1f46-\u1f47\u1f4e-\u1f4f\u1f58\u1f5a\u1f5c\u1f5e\u1f7e-\u1f7f\u1fb5\u1fc5\u1fd4-\u1fd5\u1fdc\u1ff0-\u1ff1\u1ff5\u1fff\u2065\u2072-\u2073\u208f\u209d-\u209f\u20bb-\u20cf\u20f1-\u20ff\u218a-\u218f\u23f4-\u23ff\u2427-\u243f\u244b-\u245f\u2700\u2b4d-\u2b4f\u2b5a-\u2bff\u2c2f\u2c5f\u2cf4-\u2cf8\u2d26\u2d28-\u2d2c\u2d2e-\u2d2f\u2d68-\u2d6e\u2d71-\u2d7e\u2d97-\u2d9f\u2da7\u2daf\u2db7\u2dbf\u2dc7\u2dcf\u2dd7\u2ddf\u2e3c-\u2e7f\u2e9a\u2ef4-\u2eff\u2fd6-\u2fef\u2ffc-\u2fff\u3040\u3097-\u3098\u3100-\u3104\u312e-\u3130\u318f\u31bb-\u31bf\u31e4-\u31ef\u321f\u32ff\u4db6-\u4dbf\u9fcd-\u9fff\ua48d-\ua48f\ua4c7-\ua4cf\ua62c-\ua63f\ua698-\ua69e\ua6f8-\ua6ff\ua78f\ua794-\ua79f\ua7ab-\ua7f7\ua82c-\ua82f\ua83a-\ua83f\ua878-\ua87f\ua8c5-\ua8cd\ua8da-\ua8df\ua8fc-\ua8ff\ua954-\ua95e\ua97d-\ua97f\ua9ce\ua9da-\ua9dd\ua9e0-\ua9ff\uaa37-\uaa3f\uaa4e-\uaa4f\uaa5a-\uaa5b\uaa7c-\uaa7f\uaac3-\uaada\uaaf7-\uab00\uab07-\uab08\uab0f-\uab10\uab17-\uab1f\uab27\uab2f-\uabbf\uabee-\uabef\uabfa-\uabff\ud7a4-\ud7af\ud7c7-\ud7ca\ud7fc-\ud7ff\ufa6e-\ufa6f\ufada-\ufaff\ufb07-\ufb12\ufb18-\ufb1c\ufb37\ufb3d\ufb3f\ufb42\ufb45\ufbc2-\ufbd2\ufd40-\ufd4f\ufd90-\ufd91\ufdc8-\ufdef\ufdfe-\ufdff\ufe1a-\ufe1f\ufe27-\ufe2f\ufe53\ufe67\ufe6c-\ufe6f\ufe75\ufefd-\ufefe\uff00\uffbf-\uffc1\uffc8-\uffc9\uffd0-\uffd1\uffd8-\uffd9\uffdd-\uffdf\uffe7\uffef-\ufff8\ufffe-\uffff' +Cn = '\u0378-\u0379\u0380-\u0383\u038b\u038d\u03a2\u0530\u0557-\u0558\u058b-\u058c\u0590\u05c8-\u05cf\u05eb-\u05ee\u05f5-\u05ff\u061d\u070e\u074b-\u074c\u07b2-\u07bf\u07fb-\u07fc\u082e-\u082f\u083f\u085c-\u085d\u085f\u086b-\u089f\u08b5\u08be-\u08d2\u0984\u098d-\u098e\u0991-\u0992\u09a9\u09b1\u09b3-\u09b5\u09ba-\u09bb\u09c5-\u09c6\u09c9-\u09ca\u09cf-\u09d6\u09d8-\u09db\u09de\u09e4-\u09e5\u09ff-\u0a00\u0a04\u0a0b-\u0a0e\u0a11-\u0a12\u0a29\u0a31\u0a34\u0a37\u0a3a-\u0a3b\u0a3d\u0a43-\u0a46\u0a49-\u0a4a\u0a4e-\u0a50\u0a52-\u0a58\u0a5d\u0a5f-\u0a65\u0a77-\u0a80\u0a84\u0a8e\u0a92\u0aa9\u0ab1\u0ab4\u0aba-\u0abb\u0ac6\u0aca\u0ace-\u0acf\u0ad1-\u0adf\u0ae4-\u0ae5\u0af2-\u0af8\u0b00\u0b04\u0b0d-\u0b0e\u0b11-\u0b12\u0b29\u0b31\u0b34\u0b3a-\u0b3b\u0b45-\u0b46\u0b49-\u0b4a\u0b4e-\u0b55\u0b58-\u0b5b\u0b5e\u0b64-\u0b65\u0b78-\u0b81\u0b84\u0b8b-\u0b8d\u0b91\u0b96-\u0b98\u0b9b\u0b9d\u0ba0-\u0ba2\u0ba5-\u0ba7\u0bab-\u0bad\u0bba-\u0bbd\u0bc3-\u0bc5\u0bc9\u0bce-\u0bcf\u0bd1-\u0bd6\u0bd8-\u0be5\u0bfb-\u0bff\u0c0d\u0c11\u0c29\u0c3a-\u0c3c\u0c45\u0c49\u0c4e-\u0c54\u0c57\u0c5b-\u0c5f\u0c64-\u0c65\u0c70-\u0c77\u0c8d\u0c91\u0ca9\u0cb4\u0cba-\u0cbb\u0cc5\u0cc9\u0cce-\u0cd4\u0cd7-\u0cdd\u0cdf\u0ce4-\u0ce5\u0cf0\u0cf3-\u0cff\u0d04\u0d0d\u0d11\u0d45\u0d49\u0d50-\u0d53\u0d64-\u0d65\u0d80-\u0d81\u0d84\u0d97-\u0d99\u0db2\u0dbc\u0dbe-\u0dbf\u0dc7-\u0dc9\u0dcb-\u0dce\u0dd5\u0dd7\u0de0-\u0de5\u0df0-\u0df1\u0df5-\u0e00\u0e3b-\u0e3e\u0e5c-\u0e80\u0e83\u0e85-\u0e86\u0e89\u0e8b-\u0e8c\u0e8e-\u0e93\u0e98\u0ea0\u0ea4\u0ea6\u0ea8-\u0ea9\u0eac\u0eba\u0ebe-\u0ebf\u0ec5\u0ec7\u0ece-\u0ecf\u0eda-\u0edb\u0ee0-\u0eff\u0f48\u0f6d-\u0f70\u0f98\u0fbd\u0fcd\u0fdb-\u0fff\u10c6\u10c8-\u10cc\u10ce-\u10cf\u1249\u124e-\u124f\u1257\u1259\u125e-\u125f\u1289\u128e-\u128f\u12b1\u12b6-\u12b7\u12bf\u12c1\u12c6-\u12c7\u12d7\u1311\u1316-\u1317\u135b-\u135c\u137d-\u137f\u139a-\u139f\u13f6-\u13f7\u13fe-\u13ff\u169d-\u169f\u16f9-\u16ff\u170d\u1715-\u171f\u1737-\u173f\u1754-\u175f\u176d\u1771\u1774-\u177f\u17de-\u17df\u17ea-\u17ef\u17fa-\u17ff\u180f\u181a-\u181f\u1879-\u187f\u18ab-\u18af\u18f6-\u18ff\u191f\u192c-\u192f\u193c-\u193f\u1941-\u1943\u196e-\u196f\u1975-\u197f\u19ac-\u19af\u19ca-\u19cf\u19db-\u19dd\u1a1c-\u1a1d\u1a5f\u1a7d-\u1a7e\u1a8a-\u1a8f\u1a9a-\u1a9f\u1aae-\u1aaf\u1abf-\u1aff\u1b4c-\u1b4f\u1b7d-\u1b7f\u1bf4-\u1bfb\u1c38-\u1c3a\u1c4a-\u1c4c\u1c89-\u1c8f\u1cbb-\u1cbc\u1cc8-\u1ccf\u1cfa-\u1cff\u1dfa\u1f16-\u1f17\u1f1e-\u1f1f\u1f46-\u1f47\u1f4e-\u1f4f\u1f58\u1f5a\u1f5c\u1f5e\u1f7e-\u1f7f\u1fb5\u1fc5\u1fd4-\u1fd5\u1fdc\u1ff0-\u1ff1\u1ff5\u1fff\u2065\u2072-\u2073\u208f\u209d-\u209f\u20c0-\u20cf\u20f1-\u20ff\u218c-\u218f\u2427-\u243f\u244b-\u245f\u2b74-\u2b75\u2b96-\u2b97\u2bc9\u2bff\u2c2f\u2c5f\u2cf4-\u2cf8\u2d26\u2d28-\u2d2c\u2d2e-\u2d2f\u2d68-\u2d6e\u2d71-\u2d7e\u2d97-\u2d9f\u2da7\u2daf\u2db7\u2dbf\u2dc7\u2dcf\u2dd7\u2ddf\u2e4f-\u2e7f\u2e9a\u2ef4-\u2eff\u2fd6-\u2fef\u2ffc-\u2fff\u3040\u3097-\u3098\u3100-\u3104\u3130\u318f\u31bb-\u31bf\u31e4-\u31ef\u321f\u32ff\u4db6-\u4dbf\u9ff0-\u9fff\ua48d-\ua48f\ua4c7-\ua4cf\ua62c-\ua63f\ua6f8-\ua6ff\ua7ba-\ua7f6\ua82c-\ua82f\ua83a-\ua83f\ua878-\ua87f\ua8c6-\ua8cd\ua8da-\ua8df\ua954-\ua95e\ua97d-\ua97f\ua9ce\ua9da-\ua9dd\ua9ff\uaa37-\uaa3f\uaa4e-\uaa4f\uaa5a-\uaa5b\uaac3-\uaada\uaaf7-\uab00\uab07-\uab08\uab0f-\uab10\uab17-\uab1f\uab27\uab2f\uab66-\uab6f\uabee-\uabef\uabfa-\uabff\ud7a4-\ud7af\ud7c7-\ud7ca\ud7fc-\ud7ff\ufa6e-\ufa6f\ufada-\ufaff\ufb07-\ufb12\ufb18-\ufb1c\ufb37\ufb3d\ufb3f\ufb42\ufb45\ufbc2-\ufbd2\ufd40-\ufd4f\ufd90-\ufd91\ufdc8-\ufdef\ufdfe-\ufdff\ufe1a-\ufe1f\ufe53\ufe67\ufe6c-\ufe6f\ufe75\ufefd-\ufefe\uff00\uffbf-\uffc1\uffc8-\uffc9\uffd0-\uffd1\uffd8-\uffd9\uffdd-\uffdf\uffe7\uffef-\ufff8\ufffe-\uffff\U0001000c\U00010027\U0001003b\U0001003e\U0001004e-\U0001004f\U0001005e-\U0001007f\U000100fb-\U000100ff\U00010103-\U00010106\U00010134-\U00010136\U0001018f\U0001019c-\U0001019f\U000101a1-\U000101cf\U000101fe-\U0001027f\U0001029d-\U0001029f\U000102d1-\U000102df\U000102fc-\U000102ff\U00010324-\U0001032c\U0001034b-\U0001034f\U0001037b-\U0001037f\U0001039e\U000103c4-\U000103c7\U000103d6-\U000103ff\U0001049e-\U0001049f\U000104aa-\U000104af\U000104d4-\U000104d7\U000104fc-\U000104ff\U00010528-\U0001052f\U00010564-\U0001056e\U00010570-\U000105ff\U00010737-\U0001073f\U00010756-\U0001075f\U00010768-\U000107ff\U00010806-\U00010807\U00010809\U00010836\U00010839-\U0001083b\U0001083d-\U0001083e\U00010856\U0001089f-\U000108a6\U000108b0-\U000108df\U000108f3\U000108f6-\U000108fa\U0001091c-\U0001091e\U0001093a-\U0001093e\U00010940-\U0001097f\U000109b8-\U000109bb\U000109d0-\U000109d1\U00010a04\U00010a07-\U00010a0b\U00010a14\U00010a18\U00010a36-\U00010a37\U00010a3b-\U00010a3e\U00010a49-\U00010a4f\U00010a59-\U00010a5f\U00010aa0-\U00010abf\U00010ae7-\U00010aea\U00010af7-\U00010aff\U00010b36-\U00010b38\U00010b56-\U00010b57\U00010b73-\U00010b77\U00010b92-\U00010b98\U00010b9d-\U00010ba8\U00010bb0-\U00010bff\U00010c49-\U00010c7f\U00010cb3-\U00010cbf\U00010cf3-\U00010cf9\U00010d28-\U00010d2f\U00010d3a-\U00010e5f\U00010e7f-\U00010eff\U00010f28-\U00010f2f\U00010f5a-\U00010fff\U0001104e-\U00011051\U00011070-\U0001107e\U000110c2-\U000110cc\U000110ce-\U000110cf\U000110e9-\U000110ef\U000110fa-\U000110ff\U00011135\U00011147-\U0001114f\U00011177-\U0001117f\U000111ce-\U000111cf\U000111e0\U000111f5-\U000111ff\U00011212\U0001123f-\U0001127f\U00011287\U00011289\U0001128e\U0001129e\U000112aa-\U000112af\U000112eb-\U000112ef\U000112fa-\U000112ff\U00011304\U0001130d-\U0001130e\U00011311-\U00011312\U00011329\U00011331\U00011334\U0001133a\U00011345-\U00011346\U00011349-\U0001134a\U0001134e-\U0001134f\U00011351-\U00011356\U00011358-\U0001135c\U00011364-\U00011365\U0001136d-\U0001136f\U00011375-\U000113ff\U0001145a\U0001145c\U0001145f-\U0001147f\U000114c8-\U000114cf\U000114da-\U0001157f\U000115b6-\U000115b7\U000115de-\U000115ff\U00011645-\U0001164f\U0001165a-\U0001165f\U0001166d-\U0001167f\U000116b8-\U000116bf\U000116ca-\U000116ff\U0001171b-\U0001171c\U0001172c-\U0001172f\U00011740-\U000117ff\U0001183c-\U0001189f\U000118f3-\U000118fe\U00011900-\U000119ff\U00011a48-\U00011a4f\U00011a84-\U00011a85\U00011aa3-\U00011abf\U00011af9-\U00011bff\U00011c09\U00011c37\U00011c46-\U00011c4f\U00011c6d-\U00011c6f\U00011c90-\U00011c91\U00011ca8\U00011cb7-\U00011cff\U00011d07\U00011d0a\U00011d37-\U00011d39\U00011d3b\U00011d3e\U00011d48-\U00011d4f\U00011d5a-\U00011d5f\U00011d66\U00011d69\U00011d8f\U00011d92\U00011d99-\U00011d9f\U00011daa-\U00011edf\U00011ef9-\U00011fff\U0001239a-\U000123ff\U0001246f\U00012475-\U0001247f\U00012544-\U00012fff\U0001342f-\U000143ff\U00014647-\U000167ff\U00016a39-\U00016a3f\U00016a5f\U00016a6a-\U00016a6d\U00016a70-\U00016acf\U00016aee-\U00016aef\U00016af6-\U00016aff\U00016b46-\U00016b4f\U00016b5a\U00016b62\U00016b78-\U00016b7c\U00016b90-\U00016e3f\U00016e9b-\U00016eff\U00016f45-\U00016f4f\U00016f7f-\U00016f8e\U00016fa0-\U00016fdf\U00016fe2-\U00016fff\U000187f2-\U000187ff\U00018af3-\U0001afff\U0001b11f-\U0001b16f\U0001b2fc-\U0001bbff\U0001bc6b-\U0001bc6f\U0001bc7d-\U0001bc7f\U0001bc89-\U0001bc8f\U0001bc9a-\U0001bc9b\U0001bca4-\U0001cfff\U0001d0f6-\U0001d0ff\U0001d127-\U0001d128\U0001d1e9-\U0001d1ff\U0001d246-\U0001d2df\U0001d2f4-\U0001d2ff\U0001d357-\U0001d35f\U0001d379-\U0001d3ff\U0001d455\U0001d49d\U0001d4a0-\U0001d4a1\U0001d4a3-\U0001d4a4\U0001d4a7-\U0001d4a8\U0001d4ad\U0001d4ba\U0001d4bc\U0001d4c4\U0001d506\U0001d50b-\U0001d50c\U0001d515\U0001d51d\U0001d53a\U0001d53f\U0001d545\U0001d547-\U0001d549\U0001d551\U0001d6a6-\U0001d6a7\U0001d7cc-\U0001d7cd\U0001da8c-\U0001da9a\U0001daa0\U0001dab0-\U0001dfff\U0001e007\U0001e019-\U0001e01a\U0001e022\U0001e025\U0001e02b-\U0001e7ff\U0001e8c5-\U0001e8c6\U0001e8d7-\U0001e8ff\U0001e94b-\U0001e94f\U0001e95a-\U0001e95d\U0001e960-\U0001ec70\U0001ecb5-\U0001edff\U0001ee04\U0001ee20\U0001ee23\U0001ee25-\U0001ee26\U0001ee28\U0001ee33\U0001ee38\U0001ee3a\U0001ee3c-\U0001ee41\U0001ee43-\U0001ee46\U0001ee48\U0001ee4a\U0001ee4c\U0001ee50\U0001ee53\U0001ee55-\U0001ee56\U0001ee58\U0001ee5a\U0001ee5c\U0001ee5e\U0001ee60\U0001ee63\U0001ee65-\U0001ee66\U0001ee6b\U0001ee73\U0001ee78\U0001ee7d\U0001ee7f\U0001ee8a\U0001ee9c-\U0001eea0\U0001eea4\U0001eeaa\U0001eebc-\U0001eeef\U0001eef2-\U0001efff\U0001f02c-\U0001f02f\U0001f094-\U0001f09f\U0001f0af-\U0001f0b0\U0001f0c0\U0001f0d0\U0001f0f6-\U0001f0ff\U0001f10d-\U0001f10f\U0001f16c-\U0001f16f\U0001f1ad-\U0001f1e5\U0001f203-\U0001f20f\U0001f23c-\U0001f23f\U0001f249-\U0001f24f\U0001f252-\U0001f25f\U0001f266-\U0001f2ff\U0001f6d5-\U0001f6df\U0001f6ed-\U0001f6ef\U0001f6fa-\U0001f6ff\U0001f774-\U0001f77f\U0001f7d9-\U0001f7ff\U0001f80c-\U0001f80f\U0001f848-\U0001f84f\U0001f85a-\U0001f85f\U0001f888-\U0001f88f\U0001f8ae-\U0001f8ff\U0001f90c-\U0001f90f\U0001f93f\U0001f971-\U0001f972\U0001f977-\U0001f979\U0001f97b\U0001f9a3-\U0001f9af\U0001f9ba-\U0001f9bf\U0001f9c3-\U0001f9cf\U0001fa00-\U0001fa5f\U0001fa6e-\U0001ffff\U0002a6d7-\U0002a6ff\U0002b735-\U0002b73f\U0002b81e-\U0002b81f\U0002cea2-\U0002ceaf\U0002ebe1-\U0002f7ff\U0002fa1e-\U000e0000\U000e0002-\U000e001f\U000e0080-\U000e00ff\U000e01f0-\U000effff\U000ffffe-\U000fffff\U0010fffe-\U0010ffff' -Co = u'\ue000-\uf8ff' +Co = '\ue000-\uf8ff\U000f0000-\U000ffffd\U00100000-\U0010fffd' -try: - Cs = eval(r"u'\ud800-\udbff\\\udc00\udc01-\udfff'") -except UnicodeDecodeError: - Cs = '' # Jython can't handle isolated surrogates +Cs = '\ud800-\udbff\\\udc00\udc01-\udfff' -Ll = u'a-z\xb5\xdf-\xf6\xf8-\xff\u0101\u0103\u0105\u0107\u0109\u010b\u010d\u010f\u0111\u0113\u0115\u0117\u0119\u011b\u011d\u011f\u0121\u0123\u0125\u0127\u0129\u012b\u012d\u012f\u0131\u0133\u0135\u0137-\u0138\u013a\u013c\u013e\u0140\u0142\u0144\u0146\u0148-\u0149\u014b\u014d\u014f\u0151\u0153\u0155\u0157\u0159\u015b\u015d\u015f\u0161\u0163\u0165\u0167\u0169\u016b\u016d\u016f\u0171\u0173\u0175\u0177\u017a\u017c\u017e-\u0180\u0183\u0185\u0188\u018c-\u018d\u0192\u0195\u0199-\u019b\u019e\u01a1\u01a3\u01a5\u01a8\u01aa-\u01ab\u01ad\u01b0\u01b4\u01b6\u01b9-\u01ba\u01bd-\u01bf\u01c6\u01c9\u01cc\u01ce\u01d0\u01d2\u01d4\u01d6\u01d8\u01da\u01dc-\u01dd\u01df\u01e1\u01e3\u01e5\u01e7\u01e9\u01eb\u01ed\u01ef-\u01f0\u01f3\u01f5\u01f9\u01fb\u01fd\u01ff\u0201\u0203\u0205\u0207\u0209\u020b\u020d\u020f\u0211\u0213\u0215\u0217\u0219\u021b\u021d\u021f\u0221\u0223\u0225\u0227\u0229\u022b\u022d\u022f\u0231\u0233-\u0239\u023c\u023f-\u0240\u0242\u0247\u0249\u024b\u024d\u024f-\u0293\u0295-\u02af\u0371\u0373\u0377\u037b-\u037d\u0390\u03ac-\u03ce\u03d0-\u03d1\u03d5-\u03d7\u03d9\u03db\u03dd\u03df\u03e1\u03e3\u03e5\u03e7\u03e9\u03eb\u03ed\u03ef-\u03f3\u03f5\u03f8\u03fb-\u03fc\u0430-\u045f\u0461\u0463\u0465\u0467\u0469\u046b\u046d\u046f\u0471\u0473\u0475\u0477\u0479\u047b\u047d\u047f\u0481\u048b\u048d\u048f\u0491\u0493\u0495\u0497\u0499\u049b\u049d\u049f\u04a1\u04a3\u04a5\u04a7\u04a9\u04ab\u04ad\u04af\u04b1\u04b3\u04b5\u04b7\u04b9\u04bb\u04bd\u04bf\u04c2\u04c4\u04c6\u04c8\u04ca\u04cc\u04ce-\u04cf\u04d1\u04d3\u04d5\u04d7\u04d9\u04db\u04dd\u04df\u04e1\u04e3\u04e5\u04e7\u04e9\u04eb\u04ed\u04ef\u04f1\u04f3\u04f5\u04f7\u04f9\u04fb\u04fd\u04ff\u0501\u0503\u0505\u0507\u0509\u050b\u050d\u050f\u0511\u0513\u0515\u0517\u0519\u051b\u051d\u051f\u0521\u0523\u0525\u0527\u0561-\u0587\u1d00-\u1d2b\u1d6b-\u1d77\u1d79-\u1d9a\u1e01\u1e03\u1e05\u1e07\u1e09\u1e0b\u1e0d\u1e0f\u1e11\u1e13\u1e15\u1e17\u1e19\u1e1b\u1e1d\u1e1f\u1e21\u1e23\u1e25\u1e27\u1e29\u1e2b\u1e2d\u1e2f\u1e31\u1e33\u1e35\u1e37\u1e39\u1e3b\u1e3d\u1e3f\u1e41\u1e43\u1e45\u1e47\u1e49\u1e4b\u1e4d\u1e4f\u1e51\u1e53\u1e55\u1e57\u1e59\u1e5b\u1e5d\u1e5f\u1e61\u1e63\u1e65\u1e67\u1e69\u1e6b\u1e6d\u1e6f\u1e71\u1e73\u1e75\u1e77\u1e79\u1e7b\u1e7d\u1e7f\u1e81\u1e83\u1e85\u1e87\u1e89\u1e8b\u1e8d\u1e8f\u1e91\u1e93\u1e95-\u1e9d\u1e9f\u1ea1\u1ea3\u1ea5\u1ea7\u1ea9\u1eab\u1ead\u1eaf\u1eb1\u1eb3\u1eb5\u1eb7\u1eb9\u1ebb\u1ebd\u1ebf\u1ec1\u1ec3\u1ec5\u1ec7\u1ec9\u1ecb\u1ecd\u1ecf\u1ed1\u1ed3\u1ed5\u1ed7\u1ed9\u1edb\u1edd\u1edf\u1ee1\u1ee3\u1ee5\u1ee7\u1ee9\u1eeb\u1eed\u1eef\u1ef1\u1ef3\u1ef5\u1ef7\u1ef9\u1efb\u1efd\u1eff-\u1f07\u1f10-\u1f15\u1f20-\u1f27\u1f30-\u1f37\u1f40-\u1f45\u1f50-\u1f57\u1f60-\u1f67\u1f70-\u1f7d\u1f80-\u1f87\u1f90-\u1f97\u1fa0-\u1fa7\u1fb0-\u1fb4\u1fb6-\u1fb7\u1fbe\u1fc2-\u1fc4\u1fc6-\u1fc7\u1fd0-\u1fd3\u1fd6-\u1fd7\u1fe0-\u1fe7\u1ff2-\u1ff4\u1ff6-\u1ff7\u210a\u210e-\u210f\u2113\u212f\u2134\u2139\u213c-\u213d\u2146-\u2149\u214e\u2184\u2c30-\u2c5e\u2c61\u2c65-\u2c66\u2c68\u2c6a\u2c6c\u2c71\u2c73-\u2c74\u2c76-\u2c7b\u2c81\u2c83\u2c85\u2c87\u2c89\u2c8b\u2c8d\u2c8f\u2c91\u2c93\u2c95\u2c97\u2c99\u2c9b\u2c9d\u2c9f\u2ca1\u2ca3\u2ca5\u2ca7\u2ca9\u2cab\u2cad\u2caf\u2cb1\u2cb3\u2cb5\u2cb7\u2cb9\u2cbb\u2cbd\u2cbf\u2cc1\u2cc3\u2cc5\u2cc7\u2cc9\u2ccb\u2ccd\u2ccf\u2cd1\u2cd3\u2cd5\u2cd7\u2cd9\u2cdb\u2cdd\u2cdf\u2ce1\u2ce3-\u2ce4\u2cec\u2cee\u2cf3\u2d00-\u2d25\u2d27\u2d2d\ua641\ua643\ua645\ua647\ua649\ua64b\ua64d\ua64f\ua651\ua653\ua655\ua657\ua659\ua65b\ua65d\ua65f\ua661\ua663\ua665\ua667\ua669\ua66b\ua66d\ua681\ua683\ua685\ua687\ua689\ua68b\ua68d\ua68f\ua691\ua693\ua695\ua697\ua723\ua725\ua727\ua729\ua72b\ua72d\ua72f-\ua731\ua733\ua735\ua737\ua739\ua73b\ua73d\ua73f\ua741\ua743\ua745\ua747\ua749\ua74b\ua74d\ua74f\ua751\ua753\ua755\ua757\ua759\ua75b\ua75d\ua75f\ua761\ua763\ua765\ua767\ua769\ua76b\ua76d\ua76f\ua771-\ua778\ua77a\ua77c\ua77f\ua781\ua783\ua785\ua787\ua78c\ua78e\ua791\ua793\ua7a1\ua7a3\ua7a5\ua7a7\ua7a9\ua7fa\ufb00-\ufb06\ufb13-\ufb17\uff41-\uff5a' +Ll = 'a-z\xb5\xdf-\xf6\xf8-\xff\u0101\u0103\u0105\u0107\u0109\u010b\u010d\u010f\u0111\u0113\u0115\u0117\u0119\u011b\u011d\u011f\u0121\u0123\u0125\u0127\u0129\u012b\u012d\u012f\u0131\u0133\u0135\u0137-\u0138\u013a\u013c\u013e\u0140\u0142\u0144\u0146\u0148-\u0149\u014b\u014d\u014f\u0151\u0153\u0155\u0157\u0159\u015b\u015d\u015f\u0161\u0163\u0165\u0167\u0169\u016b\u016d\u016f\u0171\u0173\u0175\u0177\u017a\u017c\u017e-\u0180\u0183\u0185\u0188\u018c-\u018d\u0192\u0195\u0199-\u019b\u019e\u01a1\u01a3\u01a5\u01a8\u01aa-\u01ab\u01ad\u01b0\u01b4\u01b6\u01b9-\u01ba\u01bd-\u01bf\u01c6\u01c9\u01cc\u01ce\u01d0\u01d2\u01d4\u01d6\u01d8\u01da\u01dc-\u01dd\u01df\u01e1\u01e3\u01e5\u01e7\u01e9\u01eb\u01ed\u01ef-\u01f0\u01f3\u01f5\u01f9\u01fb\u01fd\u01ff\u0201\u0203\u0205\u0207\u0209\u020b\u020d\u020f\u0211\u0213\u0215\u0217\u0219\u021b\u021d\u021f\u0221\u0223\u0225\u0227\u0229\u022b\u022d\u022f\u0231\u0233-\u0239\u023c\u023f-\u0240\u0242\u0247\u0249\u024b\u024d\u024f-\u0293\u0295-\u02af\u0371\u0373\u0377\u037b-\u037d\u0390\u03ac-\u03ce\u03d0-\u03d1\u03d5-\u03d7\u03d9\u03db\u03dd\u03df\u03e1\u03e3\u03e5\u03e7\u03e9\u03eb\u03ed\u03ef-\u03f3\u03f5\u03f8\u03fb-\u03fc\u0430-\u045f\u0461\u0463\u0465\u0467\u0469\u046b\u046d\u046f\u0471\u0473\u0475\u0477\u0479\u047b\u047d\u047f\u0481\u048b\u048d\u048f\u0491\u0493\u0495\u0497\u0499\u049b\u049d\u049f\u04a1\u04a3\u04a5\u04a7\u04a9\u04ab\u04ad\u04af\u04b1\u04b3\u04b5\u04b7\u04b9\u04bb\u04bd\u04bf\u04c2\u04c4\u04c6\u04c8\u04ca\u04cc\u04ce-\u04cf\u04d1\u04d3\u04d5\u04d7\u04d9\u04db\u04dd\u04df\u04e1\u04e3\u04e5\u04e7\u04e9\u04eb\u04ed\u04ef\u04f1\u04f3\u04f5\u04f7\u04f9\u04fb\u04fd\u04ff\u0501\u0503\u0505\u0507\u0509\u050b\u050d\u050f\u0511\u0513\u0515\u0517\u0519\u051b\u051d\u051f\u0521\u0523\u0525\u0527\u0529\u052b\u052d\u052f\u0560-\u0588\u10d0-\u10fa\u10fd-\u10ff\u13f8-\u13fd\u1c80-\u1c88\u1d00-\u1d2b\u1d6b-\u1d77\u1d79-\u1d9a\u1e01\u1e03\u1e05\u1e07\u1e09\u1e0b\u1e0d\u1e0f\u1e11\u1e13\u1e15\u1e17\u1e19\u1e1b\u1e1d\u1e1f\u1e21\u1e23\u1e25\u1e27\u1e29\u1e2b\u1e2d\u1e2f\u1e31\u1e33\u1e35\u1e37\u1e39\u1e3b\u1e3d\u1e3f\u1e41\u1e43\u1e45\u1e47\u1e49\u1e4b\u1e4d\u1e4f\u1e51\u1e53\u1e55\u1e57\u1e59\u1e5b\u1e5d\u1e5f\u1e61\u1e63\u1e65\u1e67\u1e69\u1e6b\u1e6d\u1e6f\u1e71\u1e73\u1e75\u1e77\u1e79\u1e7b\u1e7d\u1e7f\u1e81\u1e83\u1e85\u1e87\u1e89\u1e8b\u1e8d\u1e8f\u1e91\u1e93\u1e95-\u1e9d\u1e9f\u1ea1\u1ea3\u1ea5\u1ea7\u1ea9\u1eab\u1ead\u1eaf\u1eb1\u1eb3\u1eb5\u1eb7\u1eb9\u1ebb\u1ebd\u1ebf\u1ec1\u1ec3\u1ec5\u1ec7\u1ec9\u1ecb\u1ecd\u1ecf\u1ed1\u1ed3\u1ed5\u1ed7\u1ed9\u1edb\u1edd\u1edf\u1ee1\u1ee3\u1ee5\u1ee7\u1ee9\u1eeb\u1eed\u1eef\u1ef1\u1ef3\u1ef5\u1ef7\u1ef9\u1efb\u1efd\u1eff-\u1f07\u1f10-\u1f15\u1f20-\u1f27\u1f30-\u1f37\u1f40-\u1f45\u1f50-\u1f57\u1f60-\u1f67\u1f70-\u1f7d\u1f80-\u1f87\u1f90-\u1f97\u1fa0-\u1fa7\u1fb0-\u1fb4\u1fb6-\u1fb7\u1fbe\u1fc2-\u1fc4\u1fc6-\u1fc7\u1fd0-\u1fd3\u1fd6-\u1fd7\u1fe0-\u1fe7\u1ff2-\u1ff4\u1ff6-\u1ff7\u210a\u210e-\u210f\u2113\u212f\u2134\u2139\u213c-\u213d\u2146-\u2149\u214e\u2184\u2c30-\u2c5e\u2c61\u2c65-\u2c66\u2c68\u2c6a\u2c6c\u2c71\u2c73-\u2c74\u2c76-\u2c7b\u2c81\u2c83\u2c85\u2c87\u2c89\u2c8b\u2c8d\u2c8f\u2c91\u2c93\u2c95\u2c97\u2c99\u2c9b\u2c9d\u2c9f\u2ca1\u2ca3\u2ca5\u2ca7\u2ca9\u2cab\u2cad\u2caf\u2cb1\u2cb3\u2cb5\u2cb7\u2cb9\u2cbb\u2cbd\u2cbf\u2cc1\u2cc3\u2cc5\u2cc7\u2cc9\u2ccb\u2ccd\u2ccf\u2cd1\u2cd3\u2cd5\u2cd7\u2cd9\u2cdb\u2cdd\u2cdf\u2ce1\u2ce3-\u2ce4\u2cec\u2cee\u2cf3\u2d00-\u2d25\u2d27\u2d2d\ua641\ua643\ua645\ua647\ua649\ua64b\ua64d\ua64f\ua651\ua653\ua655\ua657\ua659\ua65b\ua65d\ua65f\ua661\ua663\ua665\ua667\ua669\ua66b\ua66d\ua681\ua683\ua685\ua687\ua689\ua68b\ua68d\ua68f\ua691\ua693\ua695\ua697\ua699\ua69b\ua723\ua725\ua727\ua729\ua72b\ua72d\ua72f-\ua731\ua733\ua735\ua737\ua739\ua73b\ua73d\ua73f\ua741\ua743\ua745\ua747\ua749\ua74b\ua74d\ua74f\ua751\ua753\ua755\ua757\ua759\ua75b\ua75d\ua75f\ua761\ua763\ua765\ua767\ua769\ua76b\ua76d\ua76f\ua771-\ua778\ua77a\ua77c\ua77f\ua781\ua783\ua785\ua787\ua78c\ua78e\ua791\ua793-\ua795\ua797\ua799\ua79b\ua79d\ua79f\ua7a1\ua7a3\ua7a5\ua7a7\ua7a9\ua7af\ua7b5\ua7b7\ua7b9\ua7fa\uab30-\uab5a\uab60-\uab65\uab70-\uabbf\ufb00-\ufb06\ufb13-\ufb17\uff41-\uff5a\U00010428-\U0001044f\U000104d8-\U000104fb\U00010cc0-\U00010cf2\U000118c0-\U000118df\U00016e60-\U00016e7f\U0001d41a-\U0001d433\U0001d44e-\U0001d454\U0001d456-\U0001d467\U0001d482-\U0001d49b\U0001d4b6-\U0001d4b9\U0001d4bb\U0001d4bd-\U0001d4c3\U0001d4c5-\U0001d4cf\U0001d4ea-\U0001d503\U0001d51e-\U0001d537\U0001d552-\U0001d56b\U0001d586-\U0001d59f\U0001d5ba-\U0001d5d3\U0001d5ee-\U0001d607\U0001d622-\U0001d63b\U0001d656-\U0001d66f\U0001d68a-\U0001d6a5\U0001d6c2-\U0001d6da\U0001d6dc-\U0001d6e1\U0001d6fc-\U0001d714\U0001d716-\U0001d71b\U0001d736-\U0001d74e\U0001d750-\U0001d755\U0001d770-\U0001d788\U0001d78a-\U0001d78f\U0001d7aa-\U0001d7c2\U0001d7c4-\U0001d7c9\U0001d7cb\U0001e922-\U0001e943' -Lm = u'\u02b0-\u02c1\u02c6-\u02d1\u02e0-\u02e4\u02ec\u02ee\u0374\u037a\u0559\u0640\u06e5-\u06e6\u07f4-\u07f5\u07fa\u081a\u0824\u0828\u0971\u0e46\u0ec6\u10fc\u17d7\u1843\u1aa7\u1c78-\u1c7d\u1d2c-\u1d6a\u1d78\u1d9b-\u1dbf\u2071\u207f\u2090-\u209c\u2c7c-\u2c7d\u2d6f\u2e2f\u3005\u3031-\u3035\u303b\u309d-\u309e\u30fc-\u30fe\ua015\ua4f8-\ua4fd\ua60c\ua67f\ua717-\ua71f\ua770\ua788\ua7f8-\ua7f9\ua9cf\uaa70\uaadd\uaaf3-\uaaf4\uff70\uff9e-\uff9f' +Lm = '\u02b0-\u02c1\u02c6-\u02d1\u02e0-\u02e4\u02ec\u02ee\u0374\u037a\u0559\u0640\u06e5-\u06e6\u07f4-\u07f5\u07fa\u081a\u0824\u0828\u0971\u0e46\u0ec6\u10fc\u17d7\u1843\u1aa7\u1c78-\u1c7d\u1d2c-\u1d6a\u1d78\u1d9b-\u1dbf\u2071\u207f\u2090-\u209c\u2c7c-\u2c7d\u2d6f\u2e2f\u3005\u3031-\u3035\u303b\u309d-\u309e\u30fc-\u30fe\ua015\ua4f8-\ua4fd\ua60c\ua67f\ua69c-\ua69d\ua717-\ua71f\ua770\ua788\ua7f8-\ua7f9\ua9cf\ua9e6\uaa70\uaadd\uaaf3-\uaaf4\uab5c-\uab5f\uff70\uff9e-\uff9f\U00016b40-\U00016b43\U00016f93-\U00016f9f\U00016fe0-\U00016fe1' -Lo = u'\xaa\xba\u01bb\u01c0-\u01c3\u0294\u05d0-\u05ea\u05f0-\u05f2\u0620-\u063f\u0641-\u064a\u066e-\u066f\u0671-\u06d3\u06d5\u06ee-\u06ef\u06fa-\u06fc\u06ff\u0710\u0712-\u072f\u074d-\u07a5\u07b1\u07ca-\u07ea\u0800-\u0815\u0840-\u0858\u08a0\u08a2-\u08ac\u0904-\u0939\u093d\u0950\u0958-\u0961\u0972-\u0977\u0979-\u097f\u0985-\u098c\u098f-\u0990\u0993-\u09a8\u09aa-\u09b0\u09b2\u09b6-\u09b9\u09bd\u09ce\u09dc-\u09dd\u09df-\u09e1\u09f0-\u09f1\u0a05-\u0a0a\u0a0f-\u0a10\u0a13-\u0a28\u0a2a-\u0a30\u0a32-\u0a33\u0a35-\u0a36\u0a38-\u0a39\u0a59-\u0a5c\u0a5e\u0a72-\u0a74\u0a85-\u0a8d\u0a8f-\u0a91\u0a93-\u0aa8\u0aaa-\u0ab0\u0ab2-\u0ab3\u0ab5-\u0ab9\u0abd\u0ad0\u0ae0-\u0ae1\u0b05-\u0b0c\u0b0f-\u0b10\u0b13-\u0b28\u0b2a-\u0b30\u0b32-\u0b33\u0b35-\u0b39\u0b3d\u0b5c-\u0b5d\u0b5f-\u0b61\u0b71\u0b83\u0b85-\u0b8a\u0b8e-\u0b90\u0b92-\u0b95\u0b99-\u0b9a\u0b9c\u0b9e-\u0b9f\u0ba3-\u0ba4\u0ba8-\u0baa\u0bae-\u0bb9\u0bd0\u0c05-\u0c0c\u0c0e-\u0c10\u0c12-\u0c28\u0c2a-\u0c33\u0c35-\u0c39\u0c3d\u0c58-\u0c59\u0c60-\u0c61\u0c85-\u0c8c\u0c8e-\u0c90\u0c92-\u0ca8\u0caa-\u0cb3\u0cb5-\u0cb9\u0cbd\u0cde\u0ce0-\u0ce1\u0cf1-\u0cf2\u0d05-\u0d0c\u0d0e-\u0d10\u0d12-\u0d3a\u0d3d\u0d4e\u0d60-\u0d61\u0d7a-\u0d7f\u0d85-\u0d96\u0d9a-\u0db1\u0db3-\u0dbb\u0dbd\u0dc0-\u0dc6\u0e01-\u0e30\u0e32-\u0e33\u0e40-\u0e45\u0e81-\u0e82\u0e84\u0e87-\u0e88\u0e8a\u0e8d\u0e94-\u0e97\u0e99-\u0e9f\u0ea1-\u0ea3\u0ea5\u0ea7\u0eaa-\u0eab\u0ead-\u0eb0\u0eb2-\u0eb3\u0ebd\u0ec0-\u0ec4\u0edc-\u0edf\u0f00\u0f40-\u0f47\u0f49-\u0f6c\u0f88-\u0f8c\u1000-\u102a\u103f\u1050-\u1055\u105a-\u105d\u1061\u1065-\u1066\u106e-\u1070\u1075-\u1081\u108e\u10d0-\u10fa\u10fd-\u1248\u124a-\u124d\u1250-\u1256\u1258\u125a-\u125d\u1260-\u1288\u128a-\u128d\u1290-\u12b0\u12b2-\u12b5\u12b8-\u12be\u12c0\u12c2-\u12c5\u12c8-\u12d6\u12d8-\u1310\u1312-\u1315\u1318-\u135a\u1380-\u138f\u13a0-\u13f4\u1401-\u166c\u166f-\u167f\u1681-\u169a\u16a0-\u16ea\u1700-\u170c\u170e-\u1711\u1720-\u1731\u1740-\u1751\u1760-\u176c\u176e-\u1770\u1780-\u17b3\u17dc\u1820-\u1842\u1844-\u1877\u1880-\u18a8\u18aa\u18b0-\u18f5\u1900-\u191c\u1950-\u196d\u1970-\u1974\u1980-\u19ab\u19c1-\u19c7\u1a00-\u1a16\u1a20-\u1a54\u1b05-\u1b33\u1b45-\u1b4b\u1b83-\u1ba0\u1bae-\u1baf\u1bba-\u1be5\u1c00-\u1c23\u1c4d-\u1c4f\u1c5a-\u1c77\u1ce9-\u1cec\u1cee-\u1cf1\u1cf5-\u1cf6\u2135-\u2138\u2d30-\u2d67\u2d80-\u2d96\u2da0-\u2da6\u2da8-\u2dae\u2db0-\u2db6\u2db8-\u2dbe\u2dc0-\u2dc6\u2dc8-\u2dce\u2dd0-\u2dd6\u2dd8-\u2dde\u3006\u303c\u3041-\u3096\u309f\u30a1-\u30fa\u30ff\u3105-\u312d\u3131-\u318e\u31a0-\u31ba\u31f0-\u31ff\u3400-\u4db5\u4e00-\u9fcc\ua000-\ua014\ua016-\ua48c\ua4d0-\ua4f7\ua500-\ua60b\ua610-\ua61f\ua62a-\ua62b\ua66e\ua6a0-\ua6e5\ua7fb-\ua801\ua803-\ua805\ua807-\ua80a\ua80c-\ua822\ua840-\ua873\ua882-\ua8b3\ua8f2-\ua8f7\ua8fb\ua90a-\ua925\ua930-\ua946\ua960-\ua97c\ua984-\ua9b2\uaa00-\uaa28\uaa40-\uaa42\uaa44-\uaa4b\uaa60-\uaa6f\uaa71-\uaa76\uaa7a\uaa80-\uaaaf\uaab1\uaab5-\uaab6\uaab9-\uaabd\uaac0\uaac2\uaadb-\uaadc\uaae0-\uaaea\uaaf2\uab01-\uab06\uab09-\uab0e\uab11-\uab16\uab20-\uab26\uab28-\uab2e\uabc0-\uabe2\uac00-\ud7a3\ud7b0-\ud7c6\ud7cb-\ud7fb\uf900-\ufa6d\ufa70-\ufad9\ufb1d\ufb1f-\ufb28\ufb2a-\ufb36\ufb38-\ufb3c\ufb3e\ufb40-\ufb41\ufb43-\ufb44\ufb46-\ufbb1\ufbd3-\ufd3d\ufd50-\ufd8f\ufd92-\ufdc7\ufdf0-\ufdfb\ufe70-\ufe74\ufe76-\ufefc\uff66-\uff6f\uff71-\uff9d\uffa0-\uffbe\uffc2-\uffc7\uffca-\uffcf\uffd2-\uffd7\uffda-\uffdc' +Lo = '\xaa\xba\u01bb\u01c0-\u01c3\u0294\u05d0-\u05ea\u05ef-\u05f2\u0620-\u063f\u0641-\u064a\u066e-\u066f\u0671-\u06d3\u06d5\u06ee-\u06ef\u06fa-\u06fc\u06ff\u0710\u0712-\u072f\u074d-\u07a5\u07b1\u07ca-\u07ea\u0800-\u0815\u0840-\u0858\u0860-\u086a\u08a0-\u08b4\u08b6-\u08bd\u0904-\u0939\u093d\u0950\u0958-\u0961\u0972-\u0980\u0985-\u098c\u098f-\u0990\u0993-\u09a8\u09aa-\u09b0\u09b2\u09b6-\u09b9\u09bd\u09ce\u09dc-\u09dd\u09df-\u09e1\u09f0-\u09f1\u09fc\u0a05-\u0a0a\u0a0f-\u0a10\u0a13-\u0a28\u0a2a-\u0a30\u0a32-\u0a33\u0a35-\u0a36\u0a38-\u0a39\u0a59-\u0a5c\u0a5e\u0a72-\u0a74\u0a85-\u0a8d\u0a8f-\u0a91\u0a93-\u0aa8\u0aaa-\u0ab0\u0ab2-\u0ab3\u0ab5-\u0ab9\u0abd\u0ad0\u0ae0-\u0ae1\u0af9\u0b05-\u0b0c\u0b0f-\u0b10\u0b13-\u0b28\u0b2a-\u0b30\u0b32-\u0b33\u0b35-\u0b39\u0b3d\u0b5c-\u0b5d\u0b5f-\u0b61\u0b71\u0b83\u0b85-\u0b8a\u0b8e-\u0b90\u0b92-\u0b95\u0b99-\u0b9a\u0b9c\u0b9e-\u0b9f\u0ba3-\u0ba4\u0ba8-\u0baa\u0bae-\u0bb9\u0bd0\u0c05-\u0c0c\u0c0e-\u0c10\u0c12-\u0c28\u0c2a-\u0c39\u0c3d\u0c58-\u0c5a\u0c60-\u0c61\u0c80\u0c85-\u0c8c\u0c8e-\u0c90\u0c92-\u0ca8\u0caa-\u0cb3\u0cb5-\u0cb9\u0cbd\u0cde\u0ce0-\u0ce1\u0cf1-\u0cf2\u0d05-\u0d0c\u0d0e-\u0d10\u0d12-\u0d3a\u0d3d\u0d4e\u0d54-\u0d56\u0d5f-\u0d61\u0d7a-\u0d7f\u0d85-\u0d96\u0d9a-\u0db1\u0db3-\u0dbb\u0dbd\u0dc0-\u0dc6\u0e01-\u0e30\u0e32-\u0e33\u0e40-\u0e45\u0e81-\u0e82\u0e84\u0e87-\u0e88\u0e8a\u0e8d\u0e94-\u0e97\u0e99-\u0e9f\u0ea1-\u0ea3\u0ea5\u0ea7\u0eaa-\u0eab\u0ead-\u0eb0\u0eb2-\u0eb3\u0ebd\u0ec0-\u0ec4\u0edc-\u0edf\u0f00\u0f40-\u0f47\u0f49-\u0f6c\u0f88-\u0f8c\u1000-\u102a\u103f\u1050-\u1055\u105a-\u105d\u1061\u1065-\u1066\u106e-\u1070\u1075-\u1081\u108e\u1100-\u1248\u124a-\u124d\u1250-\u1256\u1258\u125a-\u125d\u1260-\u1288\u128a-\u128d\u1290-\u12b0\u12b2-\u12b5\u12b8-\u12be\u12c0\u12c2-\u12c5\u12c8-\u12d6\u12d8-\u1310\u1312-\u1315\u1318-\u135a\u1380-\u138f\u1401-\u166c\u166f-\u167f\u1681-\u169a\u16a0-\u16ea\u16f1-\u16f8\u1700-\u170c\u170e-\u1711\u1720-\u1731\u1740-\u1751\u1760-\u176c\u176e-\u1770\u1780-\u17b3\u17dc\u1820-\u1842\u1844-\u1878\u1880-\u1884\u1887-\u18a8\u18aa\u18b0-\u18f5\u1900-\u191e\u1950-\u196d\u1970-\u1974\u1980-\u19ab\u19b0-\u19c9\u1a00-\u1a16\u1a20-\u1a54\u1b05-\u1b33\u1b45-\u1b4b\u1b83-\u1ba0\u1bae-\u1baf\u1bba-\u1be5\u1c00-\u1c23\u1c4d-\u1c4f\u1c5a-\u1c77\u1ce9-\u1cec\u1cee-\u1cf1\u1cf5-\u1cf6\u2135-\u2138\u2d30-\u2d67\u2d80-\u2d96\u2da0-\u2da6\u2da8-\u2dae\u2db0-\u2db6\u2db8-\u2dbe\u2dc0-\u2dc6\u2dc8-\u2dce\u2dd0-\u2dd6\u2dd8-\u2dde\u3006\u303c\u3041-\u3096\u309f\u30a1-\u30fa\u30ff\u3105-\u312f\u3131-\u318e\u31a0-\u31ba\u31f0-\u31ff\u3400-\u4db5\u4e00-\u9fef\ua000-\ua014\ua016-\ua48c\ua4d0-\ua4f7\ua500-\ua60b\ua610-\ua61f\ua62a-\ua62b\ua66e\ua6a0-\ua6e5\ua78f\ua7f7\ua7fb-\ua801\ua803-\ua805\ua807-\ua80a\ua80c-\ua822\ua840-\ua873\ua882-\ua8b3\ua8f2-\ua8f7\ua8fb\ua8fd-\ua8fe\ua90a-\ua925\ua930-\ua946\ua960-\ua97c\ua984-\ua9b2\ua9e0-\ua9e4\ua9e7-\ua9ef\ua9fa-\ua9fe\uaa00-\uaa28\uaa40-\uaa42\uaa44-\uaa4b\uaa60-\uaa6f\uaa71-\uaa76\uaa7a\uaa7e-\uaaaf\uaab1\uaab5-\uaab6\uaab9-\uaabd\uaac0\uaac2\uaadb-\uaadc\uaae0-\uaaea\uaaf2\uab01-\uab06\uab09-\uab0e\uab11-\uab16\uab20-\uab26\uab28-\uab2e\uabc0-\uabe2\uac00-\ud7a3\ud7b0-\ud7c6\ud7cb-\ud7fb\uf900-\ufa6d\ufa70-\ufad9\ufb1d\ufb1f-\ufb28\ufb2a-\ufb36\ufb38-\ufb3c\ufb3e\ufb40-\ufb41\ufb43-\ufb44\ufb46-\ufbb1\ufbd3-\ufd3d\ufd50-\ufd8f\ufd92-\ufdc7\ufdf0-\ufdfb\ufe70-\ufe74\ufe76-\ufefc\uff66-\uff6f\uff71-\uff9d\uffa0-\uffbe\uffc2-\uffc7\uffca-\uffcf\uffd2-\uffd7\uffda-\uffdc\U00010000-\U0001000b\U0001000d-\U00010026\U00010028-\U0001003a\U0001003c-\U0001003d\U0001003f-\U0001004d\U00010050-\U0001005d\U00010080-\U000100fa\U00010280-\U0001029c\U000102a0-\U000102d0\U00010300-\U0001031f\U0001032d-\U00010340\U00010342-\U00010349\U00010350-\U00010375\U00010380-\U0001039d\U000103a0-\U000103c3\U000103c8-\U000103cf\U00010450-\U0001049d\U00010500-\U00010527\U00010530-\U00010563\U00010600-\U00010736\U00010740-\U00010755\U00010760-\U00010767\U00010800-\U00010805\U00010808\U0001080a-\U00010835\U00010837-\U00010838\U0001083c\U0001083f-\U00010855\U00010860-\U00010876\U00010880-\U0001089e\U000108e0-\U000108f2\U000108f4-\U000108f5\U00010900-\U00010915\U00010920-\U00010939\U00010980-\U000109b7\U000109be-\U000109bf\U00010a00\U00010a10-\U00010a13\U00010a15-\U00010a17\U00010a19-\U00010a35\U00010a60-\U00010a7c\U00010a80-\U00010a9c\U00010ac0-\U00010ac7\U00010ac9-\U00010ae4\U00010b00-\U00010b35\U00010b40-\U00010b55\U00010b60-\U00010b72\U00010b80-\U00010b91\U00010c00-\U00010c48\U00010d00-\U00010d23\U00010f00-\U00010f1c\U00010f27\U00010f30-\U00010f45\U00011003-\U00011037\U00011083-\U000110af\U000110d0-\U000110e8\U00011103-\U00011126\U00011144\U00011150-\U00011172\U00011176\U00011183-\U000111b2\U000111c1-\U000111c4\U000111da\U000111dc\U00011200-\U00011211\U00011213-\U0001122b\U00011280-\U00011286\U00011288\U0001128a-\U0001128d\U0001128f-\U0001129d\U0001129f-\U000112a8\U000112b0-\U000112de\U00011305-\U0001130c\U0001130f-\U00011310\U00011313-\U00011328\U0001132a-\U00011330\U00011332-\U00011333\U00011335-\U00011339\U0001133d\U00011350\U0001135d-\U00011361\U00011400-\U00011434\U00011447-\U0001144a\U00011480-\U000114af\U000114c4-\U000114c5\U000114c7\U00011580-\U000115ae\U000115d8-\U000115db\U00011600-\U0001162f\U00011644\U00011680-\U000116aa\U00011700-\U0001171a\U00011800-\U0001182b\U000118ff\U00011a00\U00011a0b-\U00011a32\U00011a3a\U00011a50\U00011a5c-\U00011a83\U00011a86-\U00011a89\U00011a9d\U00011ac0-\U00011af8\U00011c00-\U00011c08\U00011c0a-\U00011c2e\U00011c40\U00011c72-\U00011c8f\U00011d00-\U00011d06\U00011d08-\U00011d09\U00011d0b-\U00011d30\U00011d46\U00011d60-\U00011d65\U00011d67-\U00011d68\U00011d6a-\U00011d89\U00011d98\U00011ee0-\U00011ef2\U00012000-\U00012399\U00012480-\U00012543\U00013000-\U0001342e\U00014400-\U00014646\U00016800-\U00016a38\U00016a40-\U00016a5e\U00016ad0-\U00016aed\U00016b00-\U00016b2f\U00016b63-\U00016b77\U00016b7d-\U00016b8f\U00016f00-\U00016f44\U00016f50\U00017000-\U000187f1\U00018800-\U00018af2\U0001b000-\U0001b11e\U0001b170-\U0001b2fb\U0001bc00-\U0001bc6a\U0001bc70-\U0001bc7c\U0001bc80-\U0001bc88\U0001bc90-\U0001bc99\U0001e800-\U0001e8c4\U0001ee00-\U0001ee03\U0001ee05-\U0001ee1f\U0001ee21-\U0001ee22\U0001ee24\U0001ee27\U0001ee29-\U0001ee32\U0001ee34-\U0001ee37\U0001ee39\U0001ee3b\U0001ee42\U0001ee47\U0001ee49\U0001ee4b\U0001ee4d-\U0001ee4f\U0001ee51-\U0001ee52\U0001ee54\U0001ee57\U0001ee59\U0001ee5b\U0001ee5d\U0001ee5f\U0001ee61-\U0001ee62\U0001ee64\U0001ee67-\U0001ee6a\U0001ee6c-\U0001ee72\U0001ee74-\U0001ee77\U0001ee79-\U0001ee7c\U0001ee7e\U0001ee80-\U0001ee89\U0001ee8b-\U0001ee9b\U0001eea1-\U0001eea3\U0001eea5-\U0001eea9\U0001eeab-\U0001eebb\U00020000-\U0002a6d6\U0002a700-\U0002b734\U0002b740-\U0002b81d\U0002b820-\U0002cea1\U0002ceb0-\U0002ebe0\U0002f800-\U0002fa1d' -Lt = u'\u01c5\u01c8\u01cb\u01f2\u1f88-\u1f8f\u1f98-\u1f9f\u1fa8-\u1faf\u1fbc\u1fcc\u1ffc' +Lt = '\u01c5\u01c8\u01cb\u01f2\u1f88-\u1f8f\u1f98-\u1f9f\u1fa8-\u1faf\u1fbc\u1fcc\u1ffc' -Lu = u'A-Z\xc0-\xd6\xd8-\xde\u0100\u0102\u0104\u0106\u0108\u010a\u010c\u010e\u0110\u0112\u0114\u0116\u0118\u011a\u011c\u011e\u0120\u0122\u0124\u0126\u0128\u012a\u012c\u012e\u0130\u0132\u0134\u0136\u0139\u013b\u013d\u013f\u0141\u0143\u0145\u0147\u014a\u014c\u014e\u0150\u0152\u0154\u0156\u0158\u015a\u015c\u015e\u0160\u0162\u0164\u0166\u0168\u016a\u016c\u016e\u0170\u0172\u0174\u0176\u0178-\u0179\u017b\u017d\u0181-\u0182\u0184\u0186-\u0187\u0189-\u018b\u018e-\u0191\u0193-\u0194\u0196-\u0198\u019c-\u019d\u019f-\u01a0\u01a2\u01a4\u01a6-\u01a7\u01a9\u01ac\u01ae-\u01af\u01b1-\u01b3\u01b5\u01b7-\u01b8\u01bc\u01c4\u01c7\u01ca\u01cd\u01cf\u01d1\u01d3\u01d5\u01d7\u01d9\u01db\u01de\u01e0\u01e2\u01e4\u01e6\u01e8\u01ea\u01ec\u01ee\u01f1\u01f4\u01f6-\u01f8\u01fa\u01fc\u01fe\u0200\u0202\u0204\u0206\u0208\u020a\u020c\u020e\u0210\u0212\u0214\u0216\u0218\u021a\u021c\u021e\u0220\u0222\u0224\u0226\u0228\u022a\u022c\u022e\u0230\u0232\u023a-\u023b\u023d-\u023e\u0241\u0243-\u0246\u0248\u024a\u024c\u024e\u0370\u0372\u0376\u0386\u0388-\u038a\u038c\u038e-\u038f\u0391-\u03a1\u03a3-\u03ab\u03cf\u03d2-\u03d4\u03d8\u03da\u03dc\u03de\u03e0\u03e2\u03e4\u03e6\u03e8\u03ea\u03ec\u03ee\u03f4\u03f7\u03f9-\u03fa\u03fd-\u042f\u0460\u0462\u0464\u0466\u0468\u046a\u046c\u046e\u0470\u0472\u0474\u0476\u0478\u047a\u047c\u047e\u0480\u048a\u048c\u048e\u0490\u0492\u0494\u0496\u0498\u049a\u049c\u049e\u04a0\u04a2\u04a4\u04a6\u04a8\u04aa\u04ac\u04ae\u04b0\u04b2\u04b4\u04b6\u04b8\u04ba\u04bc\u04be\u04c0-\u04c1\u04c3\u04c5\u04c7\u04c9\u04cb\u04cd\u04d0\u04d2\u04d4\u04d6\u04d8\u04da\u04dc\u04de\u04e0\u04e2\u04e4\u04e6\u04e8\u04ea\u04ec\u04ee\u04f0\u04f2\u04f4\u04f6\u04f8\u04fa\u04fc\u04fe\u0500\u0502\u0504\u0506\u0508\u050a\u050c\u050e\u0510\u0512\u0514\u0516\u0518\u051a\u051c\u051e\u0520\u0522\u0524\u0526\u0531-\u0556\u10a0-\u10c5\u10c7\u10cd\u1e00\u1e02\u1e04\u1e06\u1e08\u1e0a\u1e0c\u1e0e\u1e10\u1e12\u1e14\u1e16\u1e18\u1e1a\u1e1c\u1e1e\u1e20\u1e22\u1e24\u1e26\u1e28\u1e2a\u1e2c\u1e2e\u1e30\u1e32\u1e34\u1e36\u1e38\u1e3a\u1e3c\u1e3e\u1e40\u1e42\u1e44\u1e46\u1e48\u1e4a\u1e4c\u1e4e\u1e50\u1e52\u1e54\u1e56\u1e58\u1e5a\u1e5c\u1e5e\u1e60\u1e62\u1e64\u1e66\u1e68\u1e6a\u1e6c\u1e6e\u1e70\u1e72\u1e74\u1e76\u1e78\u1e7a\u1e7c\u1e7e\u1e80\u1e82\u1e84\u1e86\u1e88\u1e8a\u1e8c\u1e8e\u1e90\u1e92\u1e94\u1e9e\u1ea0\u1ea2\u1ea4\u1ea6\u1ea8\u1eaa\u1eac\u1eae\u1eb0\u1eb2\u1eb4\u1eb6\u1eb8\u1eba\u1ebc\u1ebe\u1ec0\u1ec2\u1ec4\u1ec6\u1ec8\u1eca\u1ecc\u1ece\u1ed0\u1ed2\u1ed4\u1ed6\u1ed8\u1eda\u1edc\u1ede\u1ee0\u1ee2\u1ee4\u1ee6\u1ee8\u1eea\u1eec\u1eee\u1ef0\u1ef2\u1ef4\u1ef6\u1ef8\u1efa\u1efc\u1efe\u1f08-\u1f0f\u1f18-\u1f1d\u1f28-\u1f2f\u1f38-\u1f3f\u1f48-\u1f4d\u1f59\u1f5b\u1f5d\u1f5f\u1f68-\u1f6f\u1fb8-\u1fbb\u1fc8-\u1fcb\u1fd8-\u1fdb\u1fe8-\u1fec\u1ff8-\u1ffb\u2102\u2107\u210b-\u210d\u2110-\u2112\u2115\u2119-\u211d\u2124\u2126\u2128\u212a-\u212d\u2130-\u2133\u213e-\u213f\u2145\u2183\u2c00-\u2c2e\u2c60\u2c62-\u2c64\u2c67\u2c69\u2c6b\u2c6d-\u2c70\u2c72\u2c75\u2c7e-\u2c80\u2c82\u2c84\u2c86\u2c88\u2c8a\u2c8c\u2c8e\u2c90\u2c92\u2c94\u2c96\u2c98\u2c9a\u2c9c\u2c9e\u2ca0\u2ca2\u2ca4\u2ca6\u2ca8\u2caa\u2cac\u2cae\u2cb0\u2cb2\u2cb4\u2cb6\u2cb8\u2cba\u2cbc\u2cbe\u2cc0\u2cc2\u2cc4\u2cc6\u2cc8\u2cca\u2ccc\u2cce\u2cd0\u2cd2\u2cd4\u2cd6\u2cd8\u2cda\u2cdc\u2cde\u2ce0\u2ce2\u2ceb\u2ced\u2cf2\ua640\ua642\ua644\ua646\ua648\ua64a\ua64c\ua64e\ua650\ua652\ua654\ua656\ua658\ua65a\ua65c\ua65e\ua660\ua662\ua664\ua666\ua668\ua66a\ua66c\ua680\ua682\ua684\ua686\ua688\ua68a\ua68c\ua68e\ua690\ua692\ua694\ua696\ua722\ua724\ua726\ua728\ua72a\ua72c\ua72e\ua732\ua734\ua736\ua738\ua73a\ua73c\ua73e\ua740\ua742\ua744\ua746\ua748\ua74a\ua74c\ua74e\ua750\ua752\ua754\ua756\ua758\ua75a\ua75c\ua75e\ua760\ua762\ua764\ua766\ua768\ua76a\ua76c\ua76e\ua779\ua77b\ua77d-\ua77e\ua780\ua782\ua784\ua786\ua78b\ua78d\ua790\ua792\ua7a0\ua7a2\ua7a4\ua7a6\ua7a8\ua7aa\uff21-\uff3a' +Lu = 'A-Z\xc0-\xd6\xd8-\xde\u0100\u0102\u0104\u0106\u0108\u010a\u010c\u010e\u0110\u0112\u0114\u0116\u0118\u011a\u011c\u011e\u0120\u0122\u0124\u0126\u0128\u012a\u012c\u012e\u0130\u0132\u0134\u0136\u0139\u013b\u013d\u013f\u0141\u0143\u0145\u0147\u014a\u014c\u014e\u0150\u0152\u0154\u0156\u0158\u015a\u015c\u015e\u0160\u0162\u0164\u0166\u0168\u016a\u016c\u016e\u0170\u0172\u0174\u0176\u0178-\u0179\u017b\u017d\u0181-\u0182\u0184\u0186-\u0187\u0189-\u018b\u018e-\u0191\u0193-\u0194\u0196-\u0198\u019c-\u019d\u019f-\u01a0\u01a2\u01a4\u01a6-\u01a7\u01a9\u01ac\u01ae-\u01af\u01b1-\u01b3\u01b5\u01b7-\u01b8\u01bc\u01c4\u01c7\u01ca\u01cd\u01cf\u01d1\u01d3\u01d5\u01d7\u01d9\u01db\u01de\u01e0\u01e2\u01e4\u01e6\u01e8\u01ea\u01ec\u01ee\u01f1\u01f4\u01f6-\u01f8\u01fa\u01fc\u01fe\u0200\u0202\u0204\u0206\u0208\u020a\u020c\u020e\u0210\u0212\u0214\u0216\u0218\u021a\u021c\u021e\u0220\u0222\u0224\u0226\u0228\u022a\u022c\u022e\u0230\u0232\u023a-\u023b\u023d-\u023e\u0241\u0243-\u0246\u0248\u024a\u024c\u024e\u0370\u0372\u0376\u037f\u0386\u0388-\u038a\u038c\u038e-\u038f\u0391-\u03a1\u03a3-\u03ab\u03cf\u03d2-\u03d4\u03d8\u03da\u03dc\u03de\u03e0\u03e2\u03e4\u03e6\u03e8\u03ea\u03ec\u03ee\u03f4\u03f7\u03f9-\u03fa\u03fd-\u042f\u0460\u0462\u0464\u0466\u0468\u046a\u046c\u046e\u0470\u0472\u0474\u0476\u0478\u047a\u047c\u047e\u0480\u048a\u048c\u048e\u0490\u0492\u0494\u0496\u0498\u049a\u049c\u049e\u04a0\u04a2\u04a4\u04a6\u04a8\u04aa\u04ac\u04ae\u04b0\u04b2\u04b4\u04b6\u04b8\u04ba\u04bc\u04be\u04c0-\u04c1\u04c3\u04c5\u04c7\u04c9\u04cb\u04cd\u04d0\u04d2\u04d4\u04d6\u04d8\u04da\u04dc\u04de\u04e0\u04e2\u04e4\u04e6\u04e8\u04ea\u04ec\u04ee\u04f0\u04f2\u04f4\u04f6\u04f8\u04fa\u04fc\u04fe\u0500\u0502\u0504\u0506\u0508\u050a\u050c\u050e\u0510\u0512\u0514\u0516\u0518\u051a\u051c\u051e\u0520\u0522\u0524\u0526\u0528\u052a\u052c\u052e\u0531-\u0556\u10a0-\u10c5\u10c7\u10cd\u13a0-\u13f5\u1c90-\u1cba\u1cbd-\u1cbf\u1e00\u1e02\u1e04\u1e06\u1e08\u1e0a\u1e0c\u1e0e\u1e10\u1e12\u1e14\u1e16\u1e18\u1e1a\u1e1c\u1e1e\u1e20\u1e22\u1e24\u1e26\u1e28\u1e2a\u1e2c\u1e2e\u1e30\u1e32\u1e34\u1e36\u1e38\u1e3a\u1e3c\u1e3e\u1e40\u1e42\u1e44\u1e46\u1e48\u1e4a\u1e4c\u1e4e\u1e50\u1e52\u1e54\u1e56\u1e58\u1e5a\u1e5c\u1e5e\u1e60\u1e62\u1e64\u1e66\u1e68\u1e6a\u1e6c\u1e6e\u1e70\u1e72\u1e74\u1e76\u1e78\u1e7a\u1e7c\u1e7e\u1e80\u1e82\u1e84\u1e86\u1e88\u1e8a\u1e8c\u1e8e\u1e90\u1e92\u1e94\u1e9e\u1ea0\u1ea2\u1ea4\u1ea6\u1ea8\u1eaa\u1eac\u1eae\u1eb0\u1eb2\u1eb4\u1eb6\u1eb8\u1eba\u1ebc\u1ebe\u1ec0\u1ec2\u1ec4\u1ec6\u1ec8\u1eca\u1ecc\u1ece\u1ed0\u1ed2\u1ed4\u1ed6\u1ed8\u1eda\u1edc\u1ede\u1ee0\u1ee2\u1ee4\u1ee6\u1ee8\u1eea\u1eec\u1eee\u1ef0\u1ef2\u1ef4\u1ef6\u1ef8\u1efa\u1efc\u1efe\u1f08-\u1f0f\u1f18-\u1f1d\u1f28-\u1f2f\u1f38-\u1f3f\u1f48-\u1f4d\u1f59\u1f5b\u1f5d\u1f5f\u1f68-\u1f6f\u1fb8-\u1fbb\u1fc8-\u1fcb\u1fd8-\u1fdb\u1fe8-\u1fec\u1ff8-\u1ffb\u2102\u2107\u210b-\u210d\u2110-\u2112\u2115\u2119-\u211d\u2124\u2126\u2128\u212a-\u212d\u2130-\u2133\u213e-\u213f\u2145\u2183\u2c00-\u2c2e\u2c60\u2c62-\u2c64\u2c67\u2c69\u2c6b\u2c6d-\u2c70\u2c72\u2c75\u2c7e-\u2c80\u2c82\u2c84\u2c86\u2c88\u2c8a\u2c8c\u2c8e\u2c90\u2c92\u2c94\u2c96\u2c98\u2c9a\u2c9c\u2c9e\u2ca0\u2ca2\u2ca4\u2ca6\u2ca8\u2caa\u2cac\u2cae\u2cb0\u2cb2\u2cb4\u2cb6\u2cb8\u2cba\u2cbc\u2cbe\u2cc0\u2cc2\u2cc4\u2cc6\u2cc8\u2cca\u2ccc\u2cce\u2cd0\u2cd2\u2cd4\u2cd6\u2cd8\u2cda\u2cdc\u2cde\u2ce0\u2ce2\u2ceb\u2ced\u2cf2\ua640\ua642\ua644\ua646\ua648\ua64a\ua64c\ua64e\ua650\ua652\ua654\ua656\ua658\ua65a\ua65c\ua65e\ua660\ua662\ua664\ua666\ua668\ua66a\ua66c\ua680\ua682\ua684\ua686\ua688\ua68a\ua68c\ua68e\ua690\ua692\ua694\ua696\ua698\ua69a\ua722\ua724\ua726\ua728\ua72a\ua72c\ua72e\ua732\ua734\ua736\ua738\ua73a\ua73c\ua73e\ua740\ua742\ua744\ua746\ua748\ua74a\ua74c\ua74e\ua750\ua752\ua754\ua756\ua758\ua75a\ua75c\ua75e\ua760\ua762\ua764\ua766\ua768\ua76a\ua76c\ua76e\ua779\ua77b\ua77d-\ua77e\ua780\ua782\ua784\ua786\ua78b\ua78d\ua790\ua792\ua796\ua798\ua79a\ua79c\ua79e\ua7a0\ua7a2\ua7a4\ua7a6\ua7a8\ua7aa-\ua7ae\ua7b0-\ua7b4\ua7b6\ua7b8\uff21-\uff3a\U00010400-\U00010427\U000104b0-\U000104d3\U00010c80-\U00010cb2\U000118a0-\U000118bf\U00016e40-\U00016e5f\U0001d400-\U0001d419\U0001d434-\U0001d44d\U0001d468-\U0001d481\U0001d49c\U0001d49e-\U0001d49f\U0001d4a2\U0001d4a5-\U0001d4a6\U0001d4a9-\U0001d4ac\U0001d4ae-\U0001d4b5\U0001d4d0-\U0001d4e9\U0001d504-\U0001d505\U0001d507-\U0001d50a\U0001d50d-\U0001d514\U0001d516-\U0001d51c\U0001d538-\U0001d539\U0001d53b-\U0001d53e\U0001d540-\U0001d544\U0001d546\U0001d54a-\U0001d550\U0001d56c-\U0001d585\U0001d5a0-\U0001d5b9\U0001d5d4-\U0001d5ed\U0001d608-\U0001d621\U0001d63c-\U0001d655\U0001d670-\U0001d689\U0001d6a8-\U0001d6c0\U0001d6e2-\U0001d6fa\U0001d71c-\U0001d734\U0001d756-\U0001d76e\U0001d790-\U0001d7a8\U0001d7ca\U0001e900-\U0001e921' -Mc = u'\u0903\u093b\u093e-\u0940\u0949-\u094c\u094e-\u094f\u0982-\u0983\u09be-\u09c0\u09c7-\u09c8\u09cb-\u09cc\u09d7\u0a03\u0a3e-\u0a40\u0a83\u0abe-\u0ac0\u0ac9\u0acb-\u0acc\u0b02-\u0b03\u0b3e\u0b40\u0b47-\u0b48\u0b4b-\u0b4c\u0b57\u0bbe-\u0bbf\u0bc1-\u0bc2\u0bc6-\u0bc8\u0bca-\u0bcc\u0bd7\u0c01-\u0c03\u0c41-\u0c44\u0c82-\u0c83\u0cbe\u0cc0-\u0cc4\u0cc7-\u0cc8\u0cca-\u0ccb\u0cd5-\u0cd6\u0d02-\u0d03\u0d3e-\u0d40\u0d46-\u0d48\u0d4a-\u0d4c\u0d57\u0d82-\u0d83\u0dcf-\u0dd1\u0dd8-\u0ddf\u0df2-\u0df3\u0f3e-\u0f3f\u0f7f\u102b-\u102c\u1031\u1038\u103b-\u103c\u1056-\u1057\u1062-\u1064\u1067-\u106d\u1083-\u1084\u1087-\u108c\u108f\u109a-\u109c\u17b6\u17be-\u17c5\u17c7-\u17c8\u1923-\u1926\u1929-\u192b\u1930-\u1931\u1933-\u1938\u19b0-\u19c0\u19c8-\u19c9\u1a19-\u1a1a\u1a55\u1a57\u1a61\u1a63-\u1a64\u1a6d-\u1a72\u1b04\u1b35\u1b3b\u1b3d-\u1b41\u1b43-\u1b44\u1b82\u1ba1\u1ba6-\u1ba7\u1baa\u1bac-\u1bad\u1be7\u1bea-\u1bec\u1bee\u1bf2-\u1bf3\u1c24-\u1c2b\u1c34-\u1c35\u1ce1\u1cf2-\u1cf3\u302e-\u302f\ua823-\ua824\ua827\ua880-\ua881\ua8b4-\ua8c3\ua952-\ua953\ua983\ua9b4-\ua9b5\ua9ba-\ua9bb\ua9bd-\ua9c0\uaa2f-\uaa30\uaa33-\uaa34\uaa4d\uaa7b\uaaeb\uaaee-\uaaef\uaaf5\uabe3-\uabe4\uabe6-\uabe7\uabe9-\uabea\uabec' +Mc = '\u0903\u093b\u093e-\u0940\u0949-\u094c\u094e-\u094f\u0982-\u0983\u09be-\u09c0\u09c7-\u09c8\u09cb-\u09cc\u09d7\u0a03\u0a3e-\u0a40\u0a83\u0abe-\u0ac0\u0ac9\u0acb-\u0acc\u0b02-\u0b03\u0b3e\u0b40\u0b47-\u0b48\u0b4b-\u0b4c\u0b57\u0bbe-\u0bbf\u0bc1-\u0bc2\u0bc6-\u0bc8\u0bca-\u0bcc\u0bd7\u0c01-\u0c03\u0c41-\u0c44\u0c82-\u0c83\u0cbe\u0cc0-\u0cc4\u0cc7-\u0cc8\u0cca-\u0ccb\u0cd5-\u0cd6\u0d02-\u0d03\u0d3e-\u0d40\u0d46-\u0d48\u0d4a-\u0d4c\u0d57\u0d82-\u0d83\u0dcf-\u0dd1\u0dd8-\u0ddf\u0df2-\u0df3\u0f3e-\u0f3f\u0f7f\u102b-\u102c\u1031\u1038\u103b-\u103c\u1056-\u1057\u1062-\u1064\u1067-\u106d\u1083-\u1084\u1087-\u108c\u108f\u109a-\u109c\u17b6\u17be-\u17c5\u17c7-\u17c8\u1923-\u1926\u1929-\u192b\u1930-\u1931\u1933-\u1938\u1a19-\u1a1a\u1a55\u1a57\u1a61\u1a63-\u1a64\u1a6d-\u1a72\u1b04\u1b35\u1b3b\u1b3d-\u1b41\u1b43-\u1b44\u1b82\u1ba1\u1ba6-\u1ba7\u1baa\u1be7\u1bea-\u1bec\u1bee\u1bf2-\u1bf3\u1c24-\u1c2b\u1c34-\u1c35\u1ce1\u1cf2-\u1cf3\u1cf7\u302e-\u302f\ua823-\ua824\ua827\ua880-\ua881\ua8b4-\ua8c3\ua952-\ua953\ua983\ua9b4-\ua9b5\ua9ba-\ua9bb\ua9bd-\ua9c0\uaa2f-\uaa30\uaa33-\uaa34\uaa4d\uaa7b\uaa7d\uaaeb\uaaee-\uaaef\uaaf5\uabe3-\uabe4\uabe6-\uabe7\uabe9-\uabea\uabec\U00011000\U00011002\U00011082\U000110b0-\U000110b2\U000110b7-\U000110b8\U0001112c\U00011145-\U00011146\U00011182\U000111b3-\U000111b5\U000111bf-\U000111c0\U0001122c-\U0001122e\U00011232-\U00011233\U00011235\U000112e0-\U000112e2\U00011302-\U00011303\U0001133e-\U0001133f\U00011341-\U00011344\U00011347-\U00011348\U0001134b-\U0001134d\U00011357\U00011362-\U00011363\U00011435-\U00011437\U00011440-\U00011441\U00011445\U000114b0-\U000114b2\U000114b9\U000114bb-\U000114be\U000114c1\U000115af-\U000115b1\U000115b8-\U000115bb\U000115be\U00011630-\U00011632\U0001163b-\U0001163c\U0001163e\U000116ac\U000116ae-\U000116af\U000116b6\U00011720-\U00011721\U00011726\U0001182c-\U0001182e\U00011838\U00011a39\U00011a57-\U00011a58\U00011a97\U00011c2f\U00011c3e\U00011ca9\U00011cb1\U00011cb4\U00011d8a-\U00011d8e\U00011d93-\U00011d94\U00011d96\U00011ef5-\U00011ef6\U00016f51-\U00016f7e\U0001d165-\U0001d166\U0001d16d-\U0001d172' -Me = u'\u0488-\u0489\u20dd-\u20e0\u20e2-\u20e4\ua670-\ua672' +Me = '\u0488-\u0489\u1abe\u20dd-\u20e0\u20e2-\u20e4\ua670-\ua672' -Mn = u'\u0300-\u036f\u0483-\u0487\u0591-\u05bd\u05bf\u05c1-\u05c2\u05c4-\u05c5\u05c7\u0610-\u061a\u064b-\u065f\u0670\u06d6-\u06dc\u06df-\u06e4\u06e7-\u06e8\u06ea-\u06ed\u0711\u0730-\u074a\u07a6-\u07b0\u07eb-\u07f3\u0816-\u0819\u081b-\u0823\u0825-\u0827\u0829-\u082d\u0859-\u085b\u08e4-\u08fe\u0900-\u0902\u093a\u093c\u0941-\u0948\u094d\u0951-\u0957\u0962-\u0963\u0981\u09bc\u09c1-\u09c4\u09cd\u09e2-\u09e3\u0a01-\u0a02\u0a3c\u0a41-\u0a42\u0a47-\u0a48\u0a4b-\u0a4d\u0a51\u0a70-\u0a71\u0a75\u0a81-\u0a82\u0abc\u0ac1-\u0ac5\u0ac7-\u0ac8\u0acd\u0ae2-\u0ae3\u0b01\u0b3c\u0b3f\u0b41-\u0b44\u0b4d\u0b56\u0b62-\u0b63\u0b82\u0bc0\u0bcd\u0c3e-\u0c40\u0c46-\u0c48\u0c4a-\u0c4d\u0c55-\u0c56\u0c62-\u0c63\u0cbc\u0cbf\u0cc6\u0ccc-\u0ccd\u0ce2-\u0ce3\u0d41-\u0d44\u0d4d\u0d62-\u0d63\u0dca\u0dd2-\u0dd4\u0dd6\u0e31\u0e34-\u0e3a\u0e47-\u0e4e\u0eb1\u0eb4-\u0eb9\u0ebb-\u0ebc\u0ec8-\u0ecd\u0f18-\u0f19\u0f35\u0f37\u0f39\u0f71-\u0f7e\u0f80-\u0f84\u0f86-\u0f87\u0f8d-\u0f97\u0f99-\u0fbc\u0fc6\u102d-\u1030\u1032-\u1037\u1039-\u103a\u103d-\u103e\u1058-\u1059\u105e-\u1060\u1071-\u1074\u1082\u1085-\u1086\u108d\u109d\u135d-\u135f\u1712-\u1714\u1732-\u1734\u1752-\u1753\u1772-\u1773\u17b4-\u17b5\u17b7-\u17bd\u17c6\u17c9-\u17d3\u17dd\u180b-\u180d\u18a9\u1920-\u1922\u1927-\u1928\u1932\u1939-\u193b\u1a17-\u1a18\u1a1b\u1a56\u1a58-\u1a5e\u1a60\u1a62\u1a65-\u1a6c\u1a73-\u1a7c\u1a7f\u1b00-\u1b03\u1b34\u1b36-\u1b3a\u1b3c\u1b42\u1b6b-\u1b73\u1b80-\u1b81\u1ba2-\u1ba5\u1ba8-\u1ba9\u1bab\u1be6\u1be8-\u1be9\u1bed\u1bef-\u1bf1\u1c2c-\u1c33\u1c36-\u1c37\u1cd0-\u1cd2\u1cd4-\u1ce0\u1ce2-\u1ce8\u1ced\u1cf4\u1dc0-\u1de6\u1dfc-\u1dff\u20d0-\u20dc\u20e1\u20e5-\u20f0\u2cef-\u2cf1\u2d7f\u2de0-\u2dff\u302a-\u302d\u3099-\u309a\ua66f\ua674-\ua67d\ua69f\ua6f0-\ua6f1\ua802\ua806\ua80b\ua825-\ua826\ua8c4\ua8e0-\ua8f1\ua926-\ua92d\ua947-\ua951\ua980-\ua982\ua9b3\ua9b6-\ua9b9\ua9bc\uaa29-\uaa2e\uaa31-\uaa32\uaa35-\uaa36\uaa43\uaa4c\uaab0\uaab2-\uaab4\uaab7-\uaab8\uaabe-\uaabf\uaac1\uaaec-\uaaed\uaaf6\uabe5\uabe8\uabed\ufb1e\ufe00-\ufe0f\ufe20-\ufe26' +Mn = '\u0300-\u036f\u0483-\u0487\u0591-\u05bd\u05bf\u05c1-\u05c2\u05c4-\u05c5\u05c7\u0610-\u061a\u064b-\u065f\u0670\u06d6-\u06dc\u06df-\u06e4\u06e7-\u06e8\u06ea-\u06ed\u0711\u0730-\u074a\u07a6-\u07b0\u07eb-\u07f3\u07fd\u0816-\u0819\u081b-\u0823\u0825-\u0827\u0829-\u082d\u0859-\u085b\u08d3-\u08e1\u08e3-\u0902\u093a\u093c\u0941-\u0948\u094d\u0951-\u0957\u0962-\u0963\u0981\u09bc\u09c1-\u09c4\u09cd\u09e2-\u09e3\u09fe\u0a01-\u0a02\u0a3c\u0a41-\u0a42\u0a47-\u0a48\u0a4b-\u0a4d\u0a51\u0a70-\u0a71\u0a75\u0a81-\u0a82\u0abc\u0ac1-\u0ac5\u0ac7-\u0ac8\u0acd\u0ae2-\u0ae3\u0afa-\u0aff\u0b01\u0b3c\u0b3f\u0b41-\u0b44\u0b4d\u0b56\u0b62-\u0b63\u0b82\u0bc0\u0bcd\u0c00\u0c04\u0c3e-\u0c40\u0c46-\u0c48\u0c4a-\u0c4d\u0c55-\u0c56\u0c62-\u0c63\u0c81\u0cbc\u0cbf\u0cc6\u0ccc-\u0ccd\u0ce2-\u0ce3\u0d00-\u0d01\u0d3b-\u0d3c\u0d41-\u0d44\u0d4d\u0d62-\u0d63\u0dca\u0dd2-\u0dd4\u0dd6\u0e31\u0e34-\u0e3a\u0e47-\u0e4e\u0eb1\u0eb4-\u0eb9\u0ebb-\u0ebc\u0ec8-\u0ecd\u0f18-\u0f19\u0f35\u0f37\u0f39\u0f71-\u0f7e\u0f80-\u0f84\u0f86-\u0f87\u0f8d-\u0f97\u0f99-\u0fbc\u0fc6\u102d-\u1030\u1032-\u1037\u1039-\u103a\u103d-\u103e\u1058-\u1059\u105e-\u1060\u1071-\u1074\u1082\u1085-\u1086\u108d\u109d\u135d-\u135f\u1712-\u1714\u1732-\u1734\u1752-\u1753\u1772-\u1773\u17b4-\u17b5\u17b7-\u17bd\u17c6\u17c9-\u17d3\u17dd\u180b-\u180d\u1885-\u1886\u18a9\u1920-\u1922\u1927-\u1928\u1932\u1939-\u193b\u1a17-\u1a18\u1a1b\u1a56\u1a58-\u1a5e\u1a60\u1a62\u1a65-\u1a6c\u1a73-\u1a7c\u1a7f\u1ab0-\u1abd\u1b00-\u1b03\u1b34\u1b36-\u1b3a\u1b3c\u1b42\u1b6b-\u1b73\u1b80-\u1b81\u1ba2-\u1ba5\u1ba8-\u1ba9\u1bab-\u1bad\u1be6\u1be8-\u1be9\u1bed\u1bef-\u1bf1\u1c2c-\u1c33\u1c36-\u1c37\u1cd0-\u1cd2\u1cd4-\u1ce0\u1ce2-\u1ce8\u1ced\u1cf4\u1cf8-\u1cf9\u1dc0-\u1df9\u1dfb-\u1dff\u20d0-\u20dc\u20e1\u20e5-\u20f0\u2cef-\u2cf1\u2d7f\u2de0-\u2dff\u302a-\u302d\u3099-\u309a\ua66f\ua674-\ua67d\ua69e-\ua69f\ua6f0-\ua6f1\ua802\ua806\ua80b\ua825-\ua826\ua8c4-\ua8c5\ua8e0-\ua8f1\ua8ff\ua926-\ua92d\ua947-\ua951\ua980-\ua982\ua9b3\ua9b6-\ua9b9\ua9bc\ua9e5\uaa29-\uaa2e\uaa31-\uaa32\uaa35-\uaa36\uaa43\uaa4c\uaa7c\uaab0\uaab2-\uaab4\uaab7-\uaab8\uaabe-\uaabf\uaac1\uaaec-\uaaed\uaaf6\uabe5\uabe8\uabed\ufb1e\ufe00-\ufe0f\ufe20-\ufe2f\U000101fd\U000102e0\U00010376-\U0001037a\U00010a01-\U00010a03\U00010a05-\U00010a06\U00010a0c-\U00010a0f\U00010a38-\U00010a3a\U00010a3f\U00010ae5-\U00010ae6\U00010d24-\U00010d27\U00010f46-\U00010f50\U00011001\U00011038-\U00011046\U0001107f-\U00011081\U000110b3-\U000110b6\U000110b9-\U000110ba\U00011100-\U00011102\U00011127-\U0001112b\U0001112d-\U00011134\U00011173\U00011180-\U00011181\U000111b6-\U000111be\U000111c9-\U000111cc\U0001122f-\U00011231\U00011234\U00011236-\U00011237\U0001123e\U000112df\U000112e3-\U000112ea\U00011300-\U00011301\U0001133b-\U0001133c\U00011340\U00011366-\U0001136c\U00011370-\U00011374\U00011438-\U0001143f\U00011442-\U00011444\U00011446\U0001145e\U000114b3-\U000114b8\U000114ba\U000114bf-\U000114c0\U000114c2-\U000114c3\U000115b2-\U000115b5\U000115bc-\U000115bd\U000115bf-\U000115c0\U000115dc-\U000115dd\U00011633-\U0001163a\U0001163d\U0001163f-\U00011640\U000116ab\U000116ad\U000116b0-\U000116b5\U000116b7\U0001171d-\U0001171f\U00011722-\U00011725\U00011727-\U0001172b\U0001182f-\U00011837\U00011839-\U0001183a\U00011a01-\U00011a0a\U00011a33-\U00011a38\U00011a3b-\U00011a3e\U00011a47\U00011a51-\U00011a56\U00011a59-\U00011a5b\U00011a8a-\U00011a96\U00011a98-\U00011a99\U00011c30-\U00011c36\U00011c38-\U00011c3d\U00011c3f\U00011c92-\U00011ca7\U00011caa-\U00011cb0\U00011cb2-\U00011cb3\U00011cb5-\U00011cb6\U00011d31-\U00011d36\U00011d3a\U00011d3c-\U00011d3d\U00011d3f-\U00011d45\U00011d47\U00011d90-\U00011d91\U00011d95\U00011d97\U00011ef3-\U00011ef4\U00016af0-\U00016af4\U00016b30-\U00016b36\U00016f8f-\U00016f92\U0001bc9d-\U0001bc9e\U0001d167-\U0001d169\U0001d17b-\U0001d182\U0001d185-\U0001d18b\U0001d1aa-\U0001d1ad\U0001d242-\U0001d244\U0001da00-\U0001da36\U0001da3b-\U0001da6c\U0001da75\U0001da84\U0001da9b-\U0001da9f\U0001daa1-\U0001daaf\U0001e000-\U0001e006\U0001e008-\U0001e018\U0001e01b-\U0001e021\U0001e023-\U0001e024\U0001e026-\U0001e02a\U0001e8d0-\U0001e8d6\U0001e944-\U0001e94a\U000e0100-\U000e01ef' -Nd = u'0-9\u0660-\u0669\u06f0-\u06f9\u07c0-\u07c9\u0966-\u096f\u09e6-\u09ef\u0a66-\u0a6f\u0ae6-\u0aef\u0b66-\u0b6f\u0be6-\u0bef\u0c66-\u0c6f\u0ce6-\u0cef\u0d66-\u0d6f\u0e50-\u0e59\u0ed0-\u0ed9\u0f20-\u0f29\u1040-\u1049\u1090-\u1099\u17e0-\u17e9\u1810-\u1819\u1946-\u194f\u19d0-\u19d9\u1a80-\u1a89\u1a90-\u1a99\u1b50-\u1b59\u1bb0-\u1bb9\u1c40-\u1c49\u1c50-\u1c59\ua620-\ua629\ua8d0-\ua8d9\ua900-\ua909\ua9d0-\ua9d9\uaa50-\uaa59\uabf0-\uabf9\uff10-\uff19' +Nd = '0-9\u0660-\u0669\u06f0-\u06f9\u07c0-\u07c9\u0966-\u096f\u09e6-\u09ef\u0a66-\u0a6f\u0ae6-\u0aef\u0b66-\u0b6f\u0be6-\u0bef\u0c66-\u0c6f\u0ce6-\u0cef\u0d66-\u0d6f\u0de6-\u0def\u0e50-\u0e59\u0ed0-\u0ed9\u0f20-\u0f29\u1040-\u1049\u1090-\u1099\u17e0-\u17e9\u1810-\u1819\u1946-\u194f\u19d0-\u19d9\u1a80-\u1a89\u1a90-\u1a99\u1b50-\u1b59\u1bb0-\u1bb9\u1c40-\u1c49\u1c50-\u1c59\ua620-\ua629\ua8d0-\ua8d9\ua900-\ua909\ua9d0-\ua9d9\ua9f0-\ua9f9\uaa50-\uaa59\uabf0-\uabf9\uff10-\uff19\U000104a0-\U000104a9\U00010d30-\U00010d39\U00011066-\U0001106f\U000110f0-\U000110f9\U00011136-\U0001113f\U000111d0-\U000111d9\U000112f0-\U000112f9\U00011450-\U00011459\U000114d0-\U000114d9\U00011650-\U00011659\U000116c0-\U000116c9\U00011730-\U00011739\U000118e0-\U000118e9\U00011c50-\U00011c59\U00011d50-\U00011d59\U00011da0-\U00011da9\U00016a60-\U00016a69\U00016b50-\U00016b59\U0001d7ce-\U0001d7ff\U0001e950-\U0001e959' -Nl = u'\u16ee-\u16f0\u2160-\u2182\u2185-\u2188\u3007\u3021-\u3029\u3038-\u303a\ua6e6-\ua6ef' +Nl = '\u16ee-\u16f0\u2160-\u2182\u2185-\u2188\u3007\u3021-\u3029\u3038-\u303a\ua6e6-\ua6ef\U00010140-\U00010174\U00010341\U0001034a\U000103d1-\U000103d5\U00012400-\U0001246e' -No = u'\xb2-\xb3\xb9\xbc-\xbe\u09f4-\u09f9\u0b72-\u0b77\u0bf0-\u0bf2\u0c78-\u0c7e\u0d70-\u0d75\u0f2a-\u0f33\u1369-\u137c\u17f0-\u17f9\u19da\u2070\u2074-\u2079\u2080-\u2089\u2150-\u215f\u2189\u2460-\u249b\u24ea-\u24ff\u2776-\u2793\u2cfd\u3192-\u3195\u3220-\u3229\u3248-\u324f\u3251-\u325f\u3280-\u3289\u32b1-\u32bf\ua830-\ua835' +No = '\xb2-\xb3\xb9\xbc-\xbe\u09f4-\u09f9\u0b72-\u0b77\u0bf0-\u0bf2\u0c78-\u0c7e\u0d58-\u0d5e\u0d70-\u0d78\u0f2a-\u0f33\u1369-\u137c\u17f0-\u17f9\u19da\u2070\u2074-\u2079\u2080-\u2089\u2150-\u215f\u2189\u2460-\u249b\u24ea-\u24ff\u2776-\u2793\u2cfd\u3192-\u3195\u3220-\u3229\u3248-\u324f\u3251-\u325f\u3280-\u3289\u32b1-\u32bf\ua830-\ua835\U00010107-\U00010133\U00010175-\U00010178\U0001018a-\U0001018b\U000102e1-\U000102fb\U00010320-\U00010323\U00010858-\U0001085f\U00010879-\U0001087f\U000108a7-\U000108af\U000108fb-\U000108ff\U00010916-\U0001091b\U000109bc-\U000109bd\U000109c0-\U000109cf\U000109d2-\U000109ff\U00010a40-\U00010a48\U00010a7d-\U00010a7e\U00010a9d-\U00010a9f\U00010aeb-\U00010aef\U00010b58-\U00010b5f\U00010b78-\U00010b7f\U00010ba9-\U00010baf\U00010cfa-\U00010cff\U00010e60-\U00010e7e\U00010f1d-\U00010f26\U00010f51-\U00010f54\U00011052-\U00011065\U000111e1-\U000111f4\U0001173a-\U0001173b\U000118ea-\U000118f2\U00011c5a-\U00011c6c\U00016b5b-\U00016b61\U00016e80-\U00016e96\U0001d2e0-\U0001d2f3\U0001d360-\U0001d378\U0001e8c7-\U0001e8cf\U0001ec71-\U0001ecab\U0001ecad-\U0001ecaf\U0001ecb1-\U0001ecb4\U0001f100-\U0001f10c' -Pc = u'_\u203f-\u2040\u2054\ufe33-\ufe34\ufe4d-\ufe4f\uff3f' +Pc = '_\u203f-\u2040\u2054\ufe33-\ufe34\ufe4d-\ufe4f\uff3f' -Pd = u'\\-\u058a\u05be\u1400\u1806\u2010-\u2015\u2e17\u2e1a\u2e3a-\u2e3b\u301c\u3030\u30a0\ufe31-\ufe32\ufe58\ufe63\uff0d' +Pd = '\\-\u058a\u05be\u1400\u1806\u2010-\u2015\u2e17\u2e1a\u2e3a-\u2e3b\u2e40\u301c\u3030\u30a0\ufe31-\ufe32\ufe58\ufe63\uff0d' -Pe = u')\\]}\u0f3b\u0f3d\u169c\u2046\u207e\u208e\u2309\u230b\u232a\u2769\u276b\u276d\u276f\u2771\u2773\u2775\u27c6\u27e7\u27e9\u27eb\u27ed\u27ef\u2984\u2986\u2988\u298a\u298c\u298e\u2990\u2992\u2994\u2996\u2998\u29d9\u29db\u29fd\u2e23\u2e25\u2e27\u2e29\u3009\u300b\u300d\u300f\u3011\u3015\u3017\u3019\u301b\u301e-\u301f\ufd3f\ufe18\ufe36\ufe38\ufe3a\ufe3c\ufe3e\ufe40\ufe42\ufe44\ufe48\ufe5a\ufe5c\ufe5e\uff09\uff3d\uff5d\uff60\uff63' +Pe = ')\\]}\u0f3b\u0f3d\u169c\u2046\u207e\u208e\u2309\u230b\u232a\u2769\u276b\u276d\u276f\u2771\u2773\u2775\u27c6\u27e7\u27e9\u27eb\u27ed\u27ef\u2984\u2986\u2988\u298a\u298c\u298e\u2990\u2992\u2994\u2996\u2998\u29d9\u29db\u29fd\u2e23\u2e25\u2e27\u2e29\u3009\u300b\u300d\u300f\u3011\u3015\u3017\u3019\u301b\u301e-\u301f\ufd3e\ufe18\ufe36\ufe38\ufe3a\ufe3c\ufe3e\ufe40\ufe42\ufe44\ufe48\ufe5a\ufe5c\ufe5e\uff09\uff3d\uff5d\uff60\uff63' -Pf = u'\xbb\u2019\u201d\u203a\u2e03\u2e05\u2e0a\u2e0d\u2e1d\u2e21' +Pf = '\xbb\u2019\u201d\u203a\u2e03\u2e05\u2e0a\u2e0d\u2e1d\u2e21' -Pi = u'\xab\u2018\u201b-\u201c\u201f\u2039\u2e02\u2e04\u2e09\u2e0c\u2e1c\u2e20' +Pi = '\xab\u2018\u201b-\u201c\u201f\u2039\u2e02\u2e04\u2e09\u2e0c\u2e1c\u2e20' -Po = u"!-#%-'*,.-/:-;?-@\\\\\xa1\xa7\xb6-\xb7\xbf\u037e\u0387\u055a-\u055f\u0589\u05c0\u05c3\u05c6\u05f3-\u05f4\u0609-\u060a\u060c-\u060d\u061b\u061e-\u061f\u066a-\u066d\u06d4\u0700-\u070d\u07f7-\u07f9\u0830-\u083e\u085e\u0964-\u0965\u0970\u0af0\u0df4\u0e4f\u0e5a-\u0e5b\u0f04-\u0f12\u0f14\u0f85\u0fd0-\u0fd4\u0fd9-\u0fda\u104a-\u104f\u10fb\u1360-\u1368\u166d-\u166e\u16eb-\u16ed\u1735-\u1736\u17d4-\u17d6\u17d8-\u17da\u1800-\u1805\u1807-\u180a\u1944-\u1945\u1a1e-\u1a1f\u1aa0-\u1aa6\u1aa8-\u1aad\u1b5a-\u1b60\u1bfc-\u1bff\u1c3b-\u1c3f\u1c7e-\u1c7f\u1cc0-\u1cc7\u1cd3\u2016-\u2017\u2020-\u2027\u2030-\u2038\u203b-\u203e\u2041-\u2043\u2047-\u2051\u2053\u2055-\u205e\u2cf9-\u2cfc\u2cfe-\u2cff\u2d70\u2e00-\u2e01\u2e06-\u2e08\u2e0b\u2e0e-\u2e16\u2e18-\u2e19\u2e1b\u2e1e-\u2e1f\u2e2a-\u2e2e\u2e30-\u2e39\u3001-\u3003\u303d\u30fb\ua4fe-\ua4ff\ua60d-\ua60f\ua673\ua67e\ua6f2-\ua6f7\ua874-\ua877\ua8ce-\ua8cf\ua8f8-\ua8fa\ua92e-\ua92f\ua95f\ua9c1-\ua9cd\ua9de-\ua9df\uaa5c-\uaa5f\uaade-\uaadf\uaaf0-\uaaf1\uabeb\ufe10-\ufe16\ufe19\ufe30\ufe45-\ufe46\ufe49-\ufe4c\ufe50-\ufe52\ufe54-\ufe57\ufe5f-\ufe61\ufe68\ufe6a-\ufe6b\uff01-\uff03\uff05-\uff07\uff0a\uff0c\uff0e-\uff0f\uff1a-\uff1b\uff1f-\uff20\uff3c\uff61\uff64-\uff65" +Po = "!-#%-'*,.-/:-;?-@\\\\\xa1\xa7\xb6-\xb7\xbf\u037e\u0387\u055a-\u055f\u0589\u05c0\u05c3\u05c6\u05f3-\u05f4\u0609-\u060a\u060c-\u060d\u061b\u061e-\u061f\u066a-\u066d\u06d4\u0700-\u070d\u07f7-\u07f9\u0830-\u083e\u085e\u0964-\u0965\u0970\u09fd\u0a76\u0af0\u0c84\u0df4\u0e4f\u0e5a-\u0e5b\u0f04-\u0f12\u0f14\u0f85\u0fd0-\u0fd4\u0fd9-\u0fda\u104a-\u104f\u10fb\u1360-\u1368\u166d-\u166e\u16eb-\u16ed\u1735-\u1736\u17d4-\u17d6\u17d8-\u17da\u1800-\u1805\u1807-\u180a\u1944-\u1945\u1a1e-\u1a1f\u1aa0-\u1aa6\u1aa8-\u1aad\u1b5a-\u1b60\u1bfc-\u1bff\u1c3b-\u1c3f\u1c7e-\u1c7f\u1cc0-\u1cc7\u1cd3\u2016-\u2017\u2020-\u2027\u2030-\u2038\u203b-\u203e\u2041-\u2043\u2047-\u2051\u2053\u2055-\u205e\u2cf9-\u2cfc\u2cfe-\u2cff\u2d70\u2e00-\u2e01\u2e06-\u2e08\u2e0b\u2e0e-\u2e16\u2e18-\u2e19\u2e1b\u2e1e-\u2e1f\u2e2a-\u2e2e\u2e30-\u2e39\u2e3c-\u2e3f\u2e41\u2e43-\u2e4e\u3001-\u3003\u303d\u30fb\ua4fe-\ua4ff\ua60d-\ua60f\ua673\ua67e\ua6f2-\ua6f7\ua874-\ua877\ua8ce-\ua8cf\ua8f8-\ua8fa\ua8fc\ua92e-\ua92f\ua95f\ua9c1-\ua9cd\ua9de-\ua9df\uaa5c-\uaa5f\uaade-\uaadf\uaaf0-\uaaf1\uabeb\ufe10-\ufe16\ufe19\ufe30\ufe45-\ufe46\ufe49-\ufe4c\ufe50-\ufe52\ufe54-\ufe57\ufe5f-\ufe61\ufe68\ufe6a-\ufe6b\uff01-\uff03\uff05-\uff07\uff0a\uff0c\uff0e-\uff0f\uff1a-\uff1b\uff1f-\uff20\uff3c\uff61\uff64-\uff65\U00010100-\U00010102\U0001039f\U000103d0\U0001056f\U00010857\U0001091f\U0001093f\U00010a50-\U00010a58\U00010a7f\U00010af0-\U00010af6\U00010b39-\U00010b3f\U00010b99-\U00010b9c\U00010f55-\U00010f59\U00011047-\U0001104d\U000110bb-\U000110bc\U000110be-\U000110c1\U00011140-\U00011143\U00011174-\U00011175\U000111c5-\U000111c8\U000111cd\U000111db\U000111dd-\U000111df\U00011238-\U0001123d\U000112a9\U0001144b-\U0001144f\U0001145b\U0001145d\U000114c6\U000115c1-\U000115d7\U00011641-\U00011643\U00011660-\U0001166c\U0001173c-\U0001173e\U0001183b\U00011a3f-\U00011a46\U00011a9a-\U00011a9c\U00011a9e-\U00011aa2\U00011c41-\U00011c45\U00011c70-\U00011c71\U00011ef7-\U00011ef8\U00012470-\U00012474\U00016a6e-\U00016a6f\U00016af5\U00016b37-\U00016b3b\U00016b44\U00016e97-\U00016e9a\U0001bc9f\U0001da87-\U0001da8b\U0001e95e-\U0001e95f" -Ps = u'(\\[{\u0f3a\u0f3c\u169b\u201a\u201e\u2045\u207d\u208d\u2308\u230a\u2329\u2768\u276a\u276c\u276e\u2770\u2772\u2774\u27c5\u27e6\u27e8\u27ea\u27ec\u27ee\u2983\u2985\u2987\u2989\u298b\u298d\u298f\u2991\u2993\u2995\u2997\u29d8\u29da\u29fc\u2e22\u2e24\u2e26\u2e28\u3008\u300a\u300c\u300e\u3010\u3014\u3016\u3018\u301a\u301d\ufd3e\ufe17\ufe35\ufe37\ufe39\ufe3b\ufe3d\ufe3f\ufe41\ufe43\ufe47\ufe59\ufe5b\ufe5d\uff08\uff3b\uff5b\uff5f\uff62' +Ps = '(\\[{\u0f3a\u0f3c\u169b\u201a\u201e\u2045\u207d\u208d\u2308\u230a\u2329\u2768\u276a\u276c\u276e\u2770\u2772\u2774\u27c5\u27e6\u27e8\u27ea\u27ec\u27ee\u2983\u2985\u2987\u2989\u298b\u298d\u298f\u2991\u2993\u2995\u2997\u29d8\u29da\u29fc\u2e22\u2e24\u2e26\u2e28\u2e42\u3008\u300a\u300c\u300e\u3010\u3014\u3016\u3018\u301a\u301d\ufd3f\ufe17\ufe35\ufe37\ufe39\ufe3b\ufe3d\ufe3f\ufe41\ufe43\ufe47\ufe59\ufe5b\ufe5d\uff08\uff3b\uff5b\uff5f\uff62' -Sc = u'$\xa2-\xa5\u058f\u060b\u09f2-\u09f3\u09fb\u0af1\u0bf9\u0e3f\u17db\u20a0-\u20ba\ua838\ufdfc\ufe69\uff04\uffe0-\uffe1\uffe5-\uffe6' +Sc = '$\xa2-\xa5\u058f\u060b\u07fe-\u07ff\u09f2-\u09f3\u09fb\u0af1\u0bf9\u0e3f\u17db\u20a0-\u20bf\ua838\ufdfc\ufe69\uff04\uffe0-\uffe1\uffe5-\uffe6\U0001ecb0' -Sk = u'\\^`\xa8\xaf\xb4\xb8\u02c2-\u02c5\u02d2-\u02df\u02e5-\u02eb\u02ed\u02ef-\u02ff\u0375\u0384-\u0385\u1fbd\u1fbf-\u1fc1\u1fcd-\u1fcf\u1fdd-\u1fdf\u1fed-\u1fef\u1ffd-\u1ffe\u309b-\u309c\ua700-\ua716\ua720-\ua721\ua789-\ua78a\ufbb2-\ufbc1\uff3e\uff40\uffe3' +Sk = '\\^`\xa8\xaf\xb4\xb8\u02c2-\u02c5\u02d2-\u02df\u02e5-\u02eb\u02ed\u02ef-\u02ff\u0375\u0384-\u0385\u1fbd\u1fbf-\u1fc1\u1fcd-\u1fcf\u1fdd-\u1fdf\u1fed-\u1fef\u1ffd-\u1ffe\u309b-\u309c\ua700-\ua716\ua720-\ua721\ua789-\ua78a\uab5b\ufbb2-\ufbc1\uff3e\uff40\uffe3\U0001f3fb-\U0001f3ff' -Sm = u'+<->|~\xac\xb1\xd7\xf7\u03f6\u0606-\u0608\u2044\u2052\u207a-\u207c\u208a-\u208c\u2118\u2140-\u2144\u214b\u2190-\u2194\u219a-\u219b\u21a0\u21a3\u21a6\u21ae\u21ce-\u21cf\u21d2\u21d4\u21f4-\u22ff\u2320-\u2321\u237c\u239b-\u23b3\u23dc-\u23e1\u25b7\u25c1\u25f8-\u25ff\u266f\u27c0-\u27c4\u27c7-\u27e5\u27f0-\u27ff\u2900-\u2982\u2999-\u29d7\u29dc-\u29fb\u29fe-\u2aff\u2b30-\u2b44\u2b47-\u2b4c\ufb29\ufe62\ufe64-\ufe66\uff0b\uff1c-\uff1e\uff5c\uff5e\uffe2\uffe9-\uffec' +Sm = '+<->|~\xac\xb1\xd7\xf7\u03f6\u0606-\u0608\u2044\u2052\u207a-\u207c\u208a-\u208c\u2118\u2140-\u2144\u214b\u2190-\u2194\u219a-\u219b\u21a0\u21a3\u21a6\u21ae\u21ce-\u21cf\u21d2\u21d4\u21f4-\u22ff\u2320-\u2321\u237c\u239b-\u23b3\u23dc-\u23e1\u25b7\u25c1\u25f8-\u25ff\u266f\u27c0-\u27c4\u27c7-\u27e5\u27f0-\u27ff\u2900-\u2982\u2999-\u29d7\u29dc-\u29fb\u29fe-\u2aff\u2b30-\u2b44\u2b47-\u2b4c\ufb29\ufe62\ufe64-\ufe66\uff0b\uff1c-\uff1e\uff5c\uff5e\uffe2\uffe9-\uffec\U0001d6c1\U0001d6db\U0001d6fb\U0001d715\U0001d735\U0001d74f\U0001d76f\U0001d789\U0001d7a9\U0001d7c3\U0001eef0-\U0001eef1' -So = u'\xa6\xa9\xae\xb0\u0482\u060e-\u060f\u06de\u06e9\u06fd-\u06fe\u07f6\u09fa\u0b70\u0bf3-\u0bf8\u0bfa\u0c7f\u0d79\u0f01-\u0f03\u0f13\u0f15-\u0f17\u0f1a-\u0f1f\u0f34\u0f36\u0f38\u0fbe-\u0fc5\u0fc7-\u0fcc\u0fce-\u0fcf\u0fd5-\u0fd8\u109e-\u109f\u1390-\u1399\u1940\u19de-\u19ff\u1b61-\u1b6a\u1b74-\u1b7c\u2100-\u2101\u2103-\u2106\u2108-\u2109\u2114\u2116-\u2117\u211e-\u2123\u2125\u2127\u2129\u212e\u213a-\u213b\u214a\u214c-\u214d\u214f\u2195-\u2199\u219c-\u219f\u21a1-\u21a2\u21a4-\u21a5\u21a7-\u21ad\u21af-\u21cd\u21d0-\u21d1\u21d3\u21d5-\u21f3\u2300-\u2307\u230c-\u231f\u2322-\u2328\u232b-\u237b\u237d-\u239a\u23b4-\u23db\u23e2-\u23f3\u2400-\u2426\u2440-\u244a\u249c-\u24e9\u2500-\u25b6\u25b8-\u25c0\u25c2-\u25f7\u2600-\u266e\u2670-\u26ff\u2701-\u2767\u2794-\u27bf\u2800-\u28ff\u2b00-\u2b2f\u2b45-\u2b46\u2b50-\u2b59\u2ce5-\u2cea\u2e80-\u2e99\u2e9b-\u2ef3\u2f00-\u2fd5\u2ff0-\u2ffb\u3004\u3012-\u3013\u3020\u3036-\u3037\u303e-\u303f\u3190-\u3191\u3196-\u319f\u31c0-\u31e3\u3200-\u321e\u322a-\u3247\u3250\u3260-\u327f\u328a-\u32b0\u32c0-\u32fe\u3300-\u33ff\u4dc0-\u4dff\ua490-\ua4c6\ua828-\ua82b\ua836-\ua837\ua839\uaa77-\uaa79\ufdfd\uffe4\uffe8\uffed-\uffee\ufffc-\ufffd' +So = '\xa6\xa9\xae\xb0\u0482\u058d-\u058e\u060e-\u060f\u06de\u06e9\u06fd-\u06fe\u07f6\u09fa\u0b70\u0bf3-\u0bf8\u0bfa\u0c7f\u0d4f\u0d79\u0f01-\u0f03\u0f13\u0f15-\u0f17\u0f1a-\u0f1f\u0f34\u0f36\u0f38\u0fbe-\u0fc5\u0fc7-\u0fcc\u0fce-\u0fcf\u0fd5-\u0fd8\u109e-\u109f\u1390-\u1399\u1940\u19de-\u19ff\u1b61-\u1b6a\u1b74-\u1b7c\u2100-\u2101\u2103-\u2106\u2108-\u2109\u2114\u2116-\u2117\u211e-\u2123\u2125\u2127\u2129\u212e\u213a-\u213b\u214a\u214c-\u214d\u214f\u218a-\u218b\u2195-\u2199\u219c-\u219f\u21a1-\u21a2\u21a4-\u21a5\u21a7-\u21ad\u21af-\u21cd\u21d0-\u21d1\u21d3\u21d5-\u21f3\u2300-\u2307\u230c-\u231f\u2322-\u2328\u232b-\u237b\u237d-\u239a\u23b4-\u23db\u23e2-\u2426\u2440-\u244a\u249c-\u24e9\u2500-\u25b6\u25b8-\u25c0\u25c2-\u25f7\u2600-\u266e\u2670-\u2767\u2794-\u27bf\u2800-\u28ff\u2b00-\u2b2f\u2b45-\u2b46\u2b4d-\u2b73\u2b76-\u2b95\u2b98-\u2bc8\u2bca-\u2bfe\u2ce5-\u2cea\u2e80-\u2e99\u2e9b-\u2ef3\u2f00-\u2fd5\u2ff0-\u2ffb\u3004\u3012-\u3013\u3020\u3036-\u3037\u303e-\u303f\u3190-\u3191\u3196-\u319f\u31c0-\u31e3\u3200-\u321e\u322a-\u3247\u3250\u3260-\u327f\u328a-\u32b0\u32c0-\u32fe\u3300-\u33ff\u4dc0-\u4dff\ua490-\ua4c6\ua828-\ua82b\ua836-\ua837\ua839\uaa77-\uaa79\ufdfd\uffe4\uffe8\uffed-\uffee\ufffc-\ufffd\U00010137-\U0001013f\U00010179-\U00010189\U0001018c-\U0001018e\U00010190-\U0001019b\U000101a0\U000101d0-\U000101fc\U00010877-\U00010878\U00010ac8\U0001173f\U00016b3c-\U00016b3f\U00016b45\U0001bc9c\U0001d000-\U0001d0f5\U0001d100-\U0001d126\U0001d129-\U0001d164\U0001d16a-\U0001d16c\U0001d183-\U0001d184\U0001d18c-\U0001d1a9\U0001d1ae-\U0001d1e8\U0001d200-\U0001d241\U0001d245\U0001d300-\U0001d356\U0001d800-\U0001d9ff\U0001da37-\U0001da3a\U0001da6d-\U0001da74\U0001da76-\U0001da83\U0001da85-\U0001da86\U0001ecac\U0001f000-\U0001f02b\U0001f030-\U0001f093\U0001f0a0-\U0001f0ae\U0001f0b1-\U0001f0bf\U0001f0c1-\U0001f0cf\U0001f0d1-\U0001f0f5\U0001f110-\U0001f16b\U0001f170-\U0001f1ac\U0001f1e6-\U0001f202\U0001f210-\U0001f23b\U0001f240-\U0001f248\U0001f250-\U0001f251\U0001f260-\U0001f265\U0001f300-\U0001f3fa\U0001f400-\U0001f6d4\U0001f6e0-\U0001f6ec\U0001f6f0-\U0001f6f9\U0001f700-\U0001f773\U0001f780-\U0001f7d8\U0001f800-\U0001f80b\U0001f810-\U0001f847\U0001f850-\U0001f859\U0001f860-\U0001f887\U0001f890-\U0001f8ad\U0001f900-\U0001f90b\U0001f910-\U0001f93e\U0001f940-\U0001f970\U0001f973-\U0001f976\U0001f97a\U0001f97c-\U0001f9a2\U0001f9b0-\U0001f9b9\U0001f9c0-\U0001f9c2\U0001f9d0-\U0001f9ff\U0001fa60-\U0001fa6d' -Zl = u'\u2028' +Zl = '\u2028' -Zp = u'\u2029' +Zp = '\u2029' -Zs = u' \xa0\u1680\u2000-\u200a\u202f\u205f\u3000' +Zs = ' \xa0\u1680\u2000-\u200a\u202f\u205f\u3000' -xid_continue = u'0-9A-Z_a-z\xaa\xb5\xb7\xba\xc0-\xd6\xd8-\xf6\xf8-\u02c1\u02c6-\u02d1\u02e0-\u02e4\u02ec\u02ee\u0300-\u0374\u0376-\u0377\u037b-\u037d\u0386-\u038a\u038c\u038e-\u03a1\u03a3-\u03f5\u03f7-\u0481\u0483-\u0487\u048a-\u0527\u0531-\u0556\u0559\u0561-\u0587\u0591-\u05bd\u05bf\u05c1-\u05c2\u05c4-\u05c5\u05c7\u05d0-\u05ea\u05f0-\u05f2\u0610-\u061a\u0620-\u0669\u066e-\u06d3\u06d5-\u06dc\u06df-\u06e8\u06ea-\u06fc\u06ff\u0710-\u074a\u074d-\u07b1\u07c0-\u07f5\u07fa\u0800-\u082d\u0840-\u085b\u08a0\u08a2-\u08ac\u08e4-\u08fe\u0900-\u0963\u0966-\u096f\u0971-\u0977\u0979-\u097f\u0981-\u0983\u0985-\u098c\u098f-\u0990\u0993-\u09a8\u09aa-\u09b0\u09b2\u09b6-\u09b9\u09bc-\u09c4\u09c7-\u09c8\u09cb-\u09ce\u09d7\u09dc-\u09dd\u09df-\u09e3\u09e6-\u09f1\u0a01-\u0a03\u0a05-\u0a0a\u0a0f-\u0a10\u0a13-\u0a28\u0a2a-\u0a30\u0a32-\u0a33\u0a35-\u0a36\u0a38-\u0a39\u0a3c\u0a3e-\u0a42\u0a47-\u0a48\u0a4b-\u0a4d\u0a51\u0a59-\u0a5c\u0a5e\u0a66-\u0a75\u0a81-\u0a83\u0a85-\u0a8d\u0a8f-\u0a91\u0a93-\u0aa8\u0aaa-\u0ab0\u0ab2-\u0ab3\u0ab5-\u0ab9\u0abc-\u0ac5\u0ac7-\u0ac9\u0acb-\u0acd\u0ad0\u0ae0-\u0ae3\u0ae6-\u0aef\u0b01-\u0b03\u0b05-\u0b0c\u0b0f-\u0b10\u0b13-\u0b28\u0b2a-\u0b30\u0b32-\u0b33\u0b35-\u0b39\u0b3c-\u0b44\u0b47-\u0b48\u0b4b-\u0b4d\u0b56-\u0b57\u0b5c-\u0b5d\u0b5f-\u0b63\u0b66-\u0b6f\u0b71\u0b82-\u0b83\u0b85-\u0b8a\u0b8e-\u0b90\u0b92-\u0b95\u0b99-\u0b9a\u0b9c\u0b9e-\u0b9f\u0ba3-\u0ba4\u0ba8-\u0baa\u0bae-\u0bb9\u0bbe-\u0bc2\u0bc6-\u0bc8\u0bca-\u0bcd\u0bd0\u0bd7\u0be6-\u0bef\u0c01-\u0c03\u0c05-\u0c0c\u0c0e-\u0c10\u0c12-\u0c28\u0c2a-\u0c33\u0c35-\u0c39\u0c3d-\u0c44\u0c46-\u0c48\u0c4a-\u0c4d\u0c55-\u0c56\u0c58-\u0c59\u0c60-\u0c63\u0c66-\u0c6f\u0c82-\u0c83\u0c85-\u0c8c\u0c8e-\u0c90\u0c92-\u0ca8\u0caa-\u0cb3\u0cb5-\u0cb9\u0cbc-\u0cc4\u0cc6-\u0cc8\u0cca-\u0ccd\u0cd5-\u0cd6\u0cde\u0ce0-\u0ce3\u0ce6-\u0cef\u0cf1-\u0cf2\u0d02-\u0d03\u0d05-\u0d0c\u0d0e-\u0d10\u0d12-\u0d3a\u0d3d-\u0d44\u0d46-\u0d48\u0d4a-\u0d4e\u0d57\u0d60-\u0d63\u0d66-\u0d6f\u0d7a-\u0d7f\u0d82-\u0d83\u0d85-\u0d96\u0d9a-\u0db1\u0db3-\u0dbb\u0dbd\u0dc0-\u0dc6\u0dca\u0dcf-\u0dd4\u0dd6\u0dd8-\u0ddf\u0df2-\u0df3\u0e01-\u0e3a\u0e40-\u0e4e\u0e50-\u0e59\u0e81-\u0e82\u0e84\u0e87-\u0e88\u0e8a\u0e8d\u0e94-\u0e97\u0e99-\u0e9f\u0ea1-\u0ea3\u0ea5\u0ea7\u0eaa-\u0eab\u0ead-\u0eb9\u0ebb-\u0ebd\u0ec0-\u0ec4\u0ec6\u0ec8-\u0ecd\u0ed0-\u0ed9\u0edc-\u0edf\u0f00\u0f18-\u0f19\u0f20-\u0f29\u0f35\u0f37\u0f39\u0f3e-\u0f47\u0f49-\u0f6c\u0f71-\u0f84\u0f86-\u0f97\u0f99-\u0fbc\u0fc6\u1000-\u1049\u1050-\u109d\u10a0-\u10c5\u10c7\u10cd\u10d0-\u10fa\u10fc-\u1248\u124a-\u124d\u1250-\u1256\u1258\u125a-\u125d\u1260-\u1288\u128a-\u128d\u1290-\u12b0\u12b2-\u12b5\u12b8-\u12be\u12c0\u12c2-\u12c5\u12c8-\u12d6\u12d8-\u1310\u1312-\u1315\u1318-\u135a\u135d-\u135f\u1369-\u1371\u1380-\u138f\u13a0-\u13f4\u1401-\u166c\u166f-\u167f\u1681-\u169a\u16a0-\u16ea\u16ee-\u16f0\u1700-\u170c\u170e-\u1714\u1720-\u1734\u1740-\u1753\u1760-\u176c\u176e-\u1770\u1772-\u1773\u1780-\u17d3\u17d7\u17dc-\u17dd\u17e0-\u17e9\u180b-\u180d\u1810-\u1819\u1820-\u1877\u1880-\u18aa\u18b0-\u18f5\u1900-\u191c\u1920-\u192b\u1930-\u193b\u1946-\u196d\u1970-\u1974\u1980-\u19ab\u19b0-\u19c9\u19d0-\u19da\u1a00-\u1a1b\u1a20-\u1a5e\u1a60-\u1a7c\u1a7f-\u1a89\u1a90-\u1a99\u1aa7\u1b00-\u1b4b\u1b50-\u1b59\u1b6b-\u1b73\u1b80-\u1bf3\u1c00-\u1c37\u1c40-\u1c49\u1c4d-\u1c7d\u1cd0-\u1cd2\u1cd4-\u1cf6\u1d00-\u1de6\u1dfc-\u1f15\u1f18-\u1f1d\u1f20-\u1f45\u1f48-\u1f4d\u1f50-\u1f57\u1f59\u1f5b\u1f5d\u1f5f-\u1f7d\u1f80-\u1fb4\u1fb6-\u1fbc\u1fbe\u1fc2-\u1fc4\u1fc6-\u1fcc\u1fd0-\u1fd3\u1fd6-\u1fdb\u1fe0-\u1fec\u1ff2-\u1ff4\u1ff6-\u1ffc\u203f-\u2040\u2054\u2071\u207f\u2090-\u209c\u20d0-\u20dc\u20e1\u20e5-\u20f0\u2102\u2107\u210a-\u2113\u2115\u2118-\u211d\u2124\u2126\u2128\u212a-\u2139\u213c-\u213f\u2145-\u2149\u214e\u2160-\u2188\u2c00-\u2c2e\u2c30-\u2c5e\u2c60-\u2ce4\u2ceb-\u2cf3\u2d00-\u2d25\u2d27\u2d2d\u2d30-\u2d67\u2d6f\u2d7f-\u2d96\u2da0-\u2da6\u2da8-\u2dae\u2db0-\u2db6\u2db8-\u2dbe\u2dc0-\u2dc6\u2dc8-\u2dce\u2dd0-\u2dd6\u2dd8-\u2dde\u2de0-\u2dff\u3005-\u3007\u3021-\u302f\u3031-\u3035\u3038-\u303c\u3041-\u3096\u3099-\u309a\u309d-\u309f\u30a1-\u30fa\u30fc-\u30ff\u3105-\u312d\u3131-\u318e\u31a0-\u31ba\u31f0-\u31ff\u3400-\u4db5\u4e00-\u9fcc\ua000-\ua48c\ua4d0-\ua4fd\ua500-\ua60c\ua610-\ua62b\ua640-\ua66f\ua674-\ua67d\ua67f-\ua697\ua69f-\ua6f1\ua717-\ua71f\ua722-\ua788\ua78b-\ua78e\ua790-\ua793\ua7a0-\ua7aa\ua7f8-\ua827\ua840-\ua873\ua880-\ua8c4\ua8d0-\ua8d9\ua8e0-\ua8f7\ua8fb\ua900-\ua92d\ua930-\ua953\ua960-\ua97c\ua980-\ua9c0\ua9cf-\ua9d9\uaa00-\uaa36\uaa40-\uaa4d\uaa50-\uaa59\uaa60-\uaa76\uaa7a-\uaa7b\uaa80-\uaac2\uaadb-\uaadd\uaae0-\uaaef\uaaf2-\uaaf6\uab01-\uab06\uab09-\uab0e\uab11-\uab16\uab20-\uab26\uab28-\uab2e\uabc0-\uabea\uabec-\uabed\uabf0-\uabf9\uac00-\ud7a3\ud7b0-\ud7c6\ud7cb-\ud7fb\uf900-\ufa6d\ufa70-\ufad9\ufb00-\ufb06\ufb13-\ufb17\ufb1d-\ufb28\ufb2a-\ufb36\ufb38-\ufb3c\ufb3e\ufb40-\ufb41\ufb43-\ufb44\ufb46-\ufbb1\ufbd3-\ufc5d\ufc64-\ufd3d\ufd50-\ufd8f\ufd92-\ufdc7\ufdf0-\ufdf9\ufe00-\ufe0f\ufe20-\ufe26\ufe33-\ufe34\ufe4d-\ufe4f\ufe71\ufe73\ufe77\ufe79\ufe7b\ufe7d\ufe7f-\ufefc\uff10-\uff19\uff21-\uff3a\uff3f\uff41-\uff5a\uff66-\uffbe\uffc2-\uffc7\uffca-\uffcf\uffd2-\uffd7\uffda-\uffdc' +xid_continue = '0-9A-Z_a-z\xaa\xb5\xb7\xba\xc0-\xd6\xd8-\xf6\xf8-\u02c1\u02c6-\u02d1\u02e0-\u02e4\u02ec\u02ee\u0300-\u0374\u0376-\u0377\u037b-\u037d\u037f\u0386-\u038a\u038c\u038e-\u03a1\u03a3-\u03f5\u03f7-\u0481\u0483-\u0487\u048a-\u052f\u0531-\u0556\u0559\u0560-\u0588\u0591-\u05bd\u05bf\u05c1-\u05c2\u05c4-\u05c5\u05c7\u05d0-\u05ea\u05ef-\u05f2\u0610-\u061a\u0620-\u0669\u066e-\u06d3\u06d5-\u06dc\u06df-\u06e8\u06ea-\u06fc\u06ff\u0710-\u074a\u074d-\u07b1\u07c0-\u07f5\u07fa\u07fd\u0800-\u082d\u0840-\u085b\u0860-\u086a\u08a0-\u08b4\u08b6-\u08bd\u08d3-\u08e1\u08e3-\u0963\u0966-\u096f\u0971-\u0983\u0985-\u098c\u098f-\u0990\u0993-\u09a8\u09aa-\u09b0\u09b2\u09b6-\u09b9\u09bc-\u09c4\u09c7-\u09c8\u09cb-\u09ce\u09d7\u09dc-\u09dd\u09df-\u09e3\u09e6-\u09f1\u09fc\u09fe\u0a01-\u0a03\u0a05-\u0a0a\u0a0f-\u0a10\u0a13-\u0a28\u0a2a-\u0a30\u0a32-\u0a33\u0a35-\u0a36\u0a38-\u0a39\u0a3c\u0a3e-\u0a42\u0a47-\u0a48\u0a4b-\u0a4d\u0a51\u0a59-\u0a5c\u0a5e\u0a66-\u0a75\u0a81-\u0a83\u0a85-\u0a8d\u0a8f-\u0a91\u0a93-\u0aa8\u0aaa-\u0ab0\u0ab2-\u0ab3\u0ab5-\u0ab9\u0abc-\u0ac5\u0ac7-\u0ac9\u0acb-\u0acd\u0ad0\u0ae0-\u0ae3\u0ae6-\u0aef\u0af9-\u0aff\u0b01-\u0b03\u0b05-\u0b0c\u0b0f-\u0b10\u0b13-\u0b28\u0b2a-\u0b30\u0b32-\u0b33\u0b35-\u0b39\u0b3c-\u0b44\u0b47-\u0b48\u0b4b-\u0b4d\u0b56-\u0b57\u0b5c-\u0b5d\u0b5f-\u0b63\u0b66-\u0b6f\u0b71\u0b82-\u0b83\u0b85-\u0b8a\u0b8e-\u0b90\u0b92-\u0b95\u0b99-\u0b9a\u0b9c\u0b9e-\u0b9f\u0ba3-\u0ba4\u0ba8-\u0baa\u0bae-\u0bb9\u0bbe-\u0bc2\u0bc6-\u0bc8\u0bca-\u0bcd\u0bd0\u0bd7\u0be6-\u0bef\u0c00-\u0c0c\u0c0e-\u0c10\u0c12-\u0c28\u0c2a-\u0c39\u0c3d-\u0c44\u0c46-\u0c48\u0c4a-\u0c4d\u0c55-\u0c56\u0c58-\u0c5a\u0c60-\u0c63\u0c66-\u0c6f\u0c80-\u0c83\u0c85-\u0c8c\u0c8e-\u0c90\u0c92-\u0ca8\u0caa-\u0cb3\u0cb5-\u0cb9\u0cbc-\u0cc4\u0cc6-\u0cc8\u0cca-\u0ccd\u0cd5-\u0cd6\u0cde\u0ce0-\u0ce3\u0ce6-\u0cef\u0cf1-\u0cf2\u0d00-\u0d03\u0d05-\u0d0c\u0d0e-\u0d10\u0d12-\u0d44\u0d46-\u0d48\u0d4a-\u0d4e\u0d54-\u0d57\u0d5f-\u0d63\u0d66-\u0d6f\u0d7a-\u0d7f\u0d82-\u0d83\u0d85-\u0d96\u0d9a-\u0db1\u0db3-\u0dbb\u0dbd\u0dc0-\u0dc6\u0dca\u0dcf-\u0dd4\u0dd6\u0dd8-\u0ddf\u0de6-\u0def\u0df2-\u0df3\u0e01-\u0e3a\u0e40-\u0e4e\u0e50-\u0e59\u0e81-\u0e82\u0e84\u0e87-\u0e88\u0e8a\u0e8d\u0e94-\u0e97\u0e99-\u0e9f\u0ea1-\u0ea3\u0ea5\u0ea7\u0eaa-\u0eab\u0ead-\u0eb9\u0ebb-\u0ebd\u0ec0-\u0ec4\u0ec6\u0ec8-\u0ecd\u0ed0-\u0ed9\u0edc-\u0edf\u0f00\u0f18-\u0f19\u0f20-\u0f29\u0f35\u0f37\u0f39\u0f3e-\u0f47\u0f49-\u0f6c\u0f71-\u0f84\u0f86-\u0f97\u0f99-\u0fbc\u0fc6\u1000-\u1049\u1050-\u109d\u10a0-\u10c5\u10c7\u10cd\u10d0-\u10fa\u10fc-\u1248\u124a-\u124d\u1250-\u1256\u1258\u125a-\u125d\u1260-\u1288\u128a-\u128d\u1290-\u12b0\u12b2-\u12b5\u12b8-\u12be\u12c0\u12c2-\u12c5\u12c8-\u12d6\u12d8-\u1310\u1312-\u1315\u1318-\u135a\u135d-\u135f\u1369-\u1371\u1380-\u138f\u13a0-\u13f5\u13f8-\u13fd\u1401-\u166c\u166f-\u167f\u1681-\u169a\u16a0-\u16ea\u16ee-\u16f8\u1700-\u170c\u170e-\u1714\u1720-\u1734\u1740-\u1753\u1760-\u176c\u176e-\u1770\u1772-\u1773\u1780-\u17d3\u17d7\u17dc-\u17dd\u17e0-\u17e9\u180b-\u180d\u1810-\u1819\u1820-\u1878\u1880-\u18aa\u18b0-\u18f5\u1900-\u191e\u1920-\u192b\u1930-\u193b\u1946-\u196d\u1970-\u1974\u1980-\u19ab\u19b0-\u19c9\u19d0-\u19da\u1a00-\u1a1b\u1a20-\u1a5e\u1a60-\u1a7c\u1a7f-\u1a89\u1a90-\u1a99\u1aa7\u1ab0-\u1abd\u1b00-\u1b4b\u1b50-\u1b59\u1b6b-\u1b73\u1b80-\u1bf3\u1c00-\u1c37\u1c40-\u1c49\u1c4d-\u1c7d\u1c80-\u1c88\u1c90-\u1cba\u1cbd-\u1cbf\u1cd0-\u1cd2\u1cd4-\u1cf9\u1d00-\u1df9\u1dfb-\u1f15\u1f18-\u1f1d\u1f20-\u1f45\u1f48-\u1f4d\u1f50-\u1f57\u1f59\u1f5b\u1f5d\u1f5f-\u1f7d\u1f80-\u1fb4\u1fb6-\u1fbc\u1fbe\u1fc2-\u1fc4\u1fc6-\u1fcc\u1fd0-\u1fd3\u1fd6-\u1fdb\u1fe0-\u1fec\u1ff2-\u1ff4\u1ff6-\u1ffc\u203f-\u2040\u2054\u2071\u207f\u2090-\u209c\u20d0-\u20dc\u20e1\u20e5-\u20f0\u2102\u2107\u210a-\u2113\u2115\u2118-\u211d\u2124\u2126\u2128\u212a-\u2139\u213c-\u213f\u2145-\u2149\u214e\u2160-\u2188\u2c00-\u2c2e\u2c30-\u2c5e\u2c60-\u2ce4\u2ceb-\u2cf3\u2d00-\u2d25\u2d27\u2d2d\u2d30-\u2d67\u2d6f\u2d7f-\u2d96\u2da0-\u2da6\u2da8-\u2dae\u2db0-\u2db6\u2db8-\u2dbe\u2dc0-\u2dc6\u2dc8-\u2dce\u2dd0-\u2dd6\u2dd8-\u2dde\u2de0-\u2dff\u3005-\u3007\u3021-\u302f\u3031-\u3035\u3038-\u303c\u3041-\u3096\u3099-\u309a\u309d-\u309f\u30a1-\u30fa\u30fc-\u30ff\u3105-\u312f\u3131-\u318e\u31a0-\u31ba\u31f0-\u31ff\u3400-\u4db5\u4e00-\u9fef\ua000-\ua48c\ua4d0-\ua4fd\ua500-\ua60c\ua610-\ua62b\ua640-\ua66f\ua674-\ua67d\ua67f-\ua6f1\ua717-\ua71f\ua722-\ua788\ua78b-\ua7b9\ua7f7-\ua827\ua840-\ua873\ua880-\ua8c5\ua8d0-\ua8d9\ua8e0-\ua8f7\ua8fb\ua8fd-\ua92d\ua930-\ua953\ua960-\ua97c\ua980-\ua9c0\ua9cf-\ua9d9\ua9e0-\ua9fe\uaa00-\uaa36\uaa40-\uaa4d\uaa50-\uaa59\uaa60-\uaa76\uaa7a-\uaac2\uaadb-\uaadd\uaae0-\uaaef\uaaf2-\uaaf6\uab01-\uab06\uab09-\uab0e\uab11-\uab16\uab20-\uab26\uab28-\uab2e\uab30-\uab5a\uab5c-\uab65\uab70-\uabea\uabec-\uabed\uabf0-\uabf9\uac00-\ud7a3\ud7b0-\ud7c6\ud7cb-\ud7fb\uf900-\ufa6d\ufa70-\ufad9\ufb00-\ufb06\ufb13-\ufb17\ufb1d-\ufb28\ufb2a-\ufb36\ufb38-\ufb3c\ufb3e\ufb40-\ufb41\ufb43-\ufb44\ufb46-\ufbb1\ufbd3-\ufc5d\ufc64-\ufd3d\ufd50-\ufd8f\ufd92-\ufdc7\ufdf0-\ufdf9\ufe00-\ufe0f\ufe20-\ufe2f\ufe33-\ufe34\ufe4d-\ufe4f\ufe71\ufe73\ufe77\ufe79\ufe7b\ufe7d\ufe7f-\ufefc\uff10-\uff19\uff21-\uff3a\uff3f\uff41-\uff5a\uff66-\uffbe\uffc2-\uffc7\uffca-\uffcf\uffd2-\uffd7\uffda-\uffdc\U00010000-\U0001000b\U0001000d-\U00010026\U00010028-\U0001003a\U0001003c-\U0001003d\U0001003f-\U0001004d\U00010050-\U0001005d\U00010080-\U000100fa\U00010140-\U00010174\U000101fd\U00010280-\U0001029c\U000102a0-\U000102d0\U000102e0\U00010300-\U0001031f\U0001032d-\U0001034a\U00010350-\U0001037a\U00010380-\U0001039d\U000103a0-\U000103c3\U000103c8-\U000103cf\U000103d1-\U000103d5\U00010400-\U0001049d\U000104a0-\U000104a9\U000104b0-\U000104d3\U000104d8-\U000104fb\U00010500-\U00010527\U00010530-\U00010563\U00010600-\U00010736\U00010740-\U00010755\U00010760-\U00010767\U00010800-\U00010805\U00010808\U0001080a-\U00010835\U00010837-\U00010838\U0001083c\U0001083f-\U00010855\U00010860-\U00010876\U00010880-\U0001089e\U000108e0-\U000108f2\U000108f4-\U000108f5\U00010900-\U00010915\U00010920-\U00010939\U00010980-\U000109b7\U000109be-\U000109bf\U00010a00-\U00010a03\U00010a05-\U00010a06\U00010a0c-\U00010a13\U00010a15-\U00010a17\U00010a19-\U00010a35\U00010a38-\U00010a3a\U00010a3f\U00010a60-\U00010a7c\U00010a80-\U00010a9c\U00010ac0-\U00010ac7\U00010ac9-\U00010ae6\U00010b00-\U00010b35\U00010b40-\U00010b55\U00010b60-\U00010b72\U00010b80-\U00010b91\U00010c00-\U00010c48\U00010c80-\U00010cb2\U00010cc0-\U00010cf2\U00010d00-\U00010d27\U00010d30-\U00010d39\U00010f00-\U00010f1c\U00010f27\U00010f30-\U00010f50\U00011000-\U00011046\U00011066-\U0001106f\U0001107f-\U000110ba\U000110d0-\U000110e8\U000110f0-\U000110f9\U00011100-\U00011134\U00011136-\U0001113f\U00011144-\U00011146\U00011150-\U00011173\U00011176\U00011180-\U000111c4\U000111c9-\U000111cc\U000111d0-\U000111da\U000111dc\U00011200-\U00011211\U00011213-\U00011237\U0001123e\U00011280-\U00011286\U00011288\U0001128a-\U0001128d\U0001128f-\U0001129d\U0001129f-\U000112a8\U000112b0-\U000112ea\U000112f0-\U000112f9\U00011300-\U00011303\U00011305-\U0001130c\U0001130f-\U00011310\U00011313-\U00011328\U0001132a-\U00011330\U00011332-\U00011333\U00011335-\U00011339\U0001133b-\U00011344\U00011347-\U00011348\U0001134b-\U0001134d\U00011350\U00011357\U0001135d-\U00011363\U00011366-\U0001136c\U00011370-\U00011374\U00011400-\U0001144a\U00011450-\U00011459\U0001145e\U00011480-\U000114c5\U000114c7\U000114d0-\U000114d9\U00011580-\U000115b5\U000115b8-\U000115c0\U000115d8-\U000115dd\U00011600-\U00011640\U00011644\U00011650-\U00011659\U00011680-\U000116b7\U000116c0-\U000116c9\U00011700-\U0001171a\U0001171d-\U0001172b\U00011730-\U00011739\U00011800-\U0001183a\U000118a0-\U000118e9\U000118ff\U00011a00-\U00011a3e\U00011a47\U00011a50-\U00011a83\U00011a86-\U00011a99\U00011a9d\U00011ac0-\U00011af8\U00011c00-\U00011c08\U00011c0a-\U00011c36\U00011c38-\U00011c40\U00011c50-\U00011c59\U00011c72-\U00011c8f\U00011c92-\U00011ca7\U00011ca9-\U00011cb6\U00011d00-\U00011d06\U00011d08-\U00011d09\U00011d0b-\U00011d36\U00011d3a\U00011d3c-\U00011d3d\U00011d3f-\U00011d47\U00011d50-\U00011d59\U00011d60-\U00011d65\U00011d67-\U00011d68\U00011d6a-\U00011d8e\U00011d90-\U00011d91\U00011d93-\U00011d98\U00011da0-\U00011da9\U00011ee0-\U00011ef6\U00012000-\U00012399\U00012400-\U0001246e\U00012480-\U00012543\U00013000-\U0001342e\U00014400-\U00014646\U00016800-\U00016a38\U00016a40-\U00016a5e\U00016a60-\U00016a69\U00016ad0-\U00016aed\U00016af0-\U00016af4\U00016b00-\U00016b36\U00016b40-\U00016b43\U00016b50-\U00016b59\U00016b63-\U00016b77\U00016b7d-\U00016b8f\U00016e40-\U00016e7f\U00016f00-\U00016f44\U00016f50-\U00016f7e\U00016f8f-\U00016f9f\U00016fe0-\U00016fe1\U00017000-\U000187f1\U00018800-\U00018af2\U0001b000-\U0001b11e\U0001b170-\U0001b2fb\U0001bc00-\U0001bc6a\U0001bc70-\U0001bc7c\U0001bc80-\U0001bc88\U0001bc90-\U0001bc99\U0001bc9d-\U0001bc9e\U0001d165-\U0001d169\U0001d16d-\U0001d172\U0001d17b-\U0001d182\U0001d185-\U0001d18b\U0001d1aa-\U0001d1ad\U0001d242-\U0001d244\U0001d400-\U0001d454\U0001d456-\U0001d49c\U0001d49e-\U0001d49f\U0001d4a2\U0001d4a5-\U0001d4a6\U0001d4a9-\U0001d4ac\U0001d4ae-\U0001d4b9\U0001d4bb\U0001d4bd-\U0001d4c3\U0001d4c5-\U0001d505\U0001d507-\U0001d50a\U0001d50d-\U0001d514\U0001d516-\U0001d51c\U0001d51e-\U0001d539\U0001d53b-\U0001d53e\U0001d540-\U0001d544\U0001d546\U0001d54a-\U0001d550\U0001d552-\U0001d6a5\U0001d6a8-\U0001d6c0\U0001d6c2-\U0001d6da\U0001d6dc-\U0001d6fa\U0001d6fc-\U0001d714\U0001d716-\U0001d734\U0001d736-\U0001d74e\U0001d750-\U0001d76e\U0001d770-\U0001d788\U0001d78a-\U0001d7a8\U0001d7aa-\U0001d7c2\U0001d7c4-\U0001d7cb\U0001d7ce-\U0001d7ff\U0001da00-\U0001da36\U0001da3b-\U0001da6c\U0001da75\U0001da84\U0001da9b-\U0001da9f\U0001daa1-\U0001daaf\U0001e000-\U0001e006\U0001e008-\U0001e018\U0001e01b-\U0001e021\U0001e023-\U0001e024\U0001e026-\U0001e02a\U0001e800-\U0001e8c4\U0001e8d0-\U0001e8d6\U0001e900-\U0001e94a\U0001e950-\U0001e959\U0001ee00-\U0001ee03\U0001ee05-\U0001ee1f\U0001ee21-\U0001ee22\U0001ee24\U0001ee27\U0001ee29-\U0001ee32\U0001ee34-\U0001ee37\U0001ee39\U0001ee3b\U0001ee42\U0001ee47\U0001ee49\U0001ee4b\U0001ee4d-\U0001ee4f\U0001ee51-\U0001ee52\U0001ee54\U0001ee57\U0001ee59\U0001ee5b\U0001ee5d\U0001ee5f\U0001ee61-\U0001ee62\U0001ee64\U0001ee67-\U0001ee6a\U0001ee6c-\U0001ee72\U0001ee74-\U0001ee77\U0001ee79-\U0001ee7c\U0001ee7e\U0001ee80-\U0001ee89\U0001ee8b-\U0001ee9b\U0001eea1-\U0001eea3\U0001eea5-\U0001eea9\U0001eeab-\U0001eebb\U00020000-\U0002a6d6\U0002a700-\U0002b734\U0002b740-\U0002b81d\U0002b820-\U0002cea1\U0002ceb0-\U0002ebe0\U0002f800-\U0002fa1d\U000e0100-\U000e01ef' -xid_start = u'A-Z_a-z\xaa\xb5\xba\xc0-\xd6\xd8-\xf6\xf8-\u02c1\u02c6-\u02d1\u02e0-\u02e4\u02ec\u02ee\u0370-\u0374\u0376-\u0377\u037b-\u037d\u0386\u0388-\u038a\u038c\u038e-\u03a1\u03a3-\u03f5\u03f7-\u0481\u048a-\u0527\u0531-\u0556\u0559\u0561-\u0587\u05d0-\u05ea\u05f0-\u05f2\u0620-\u064a\u066e-\u066f\u0671-\u06d3\u06d5\u06e5-\u06e6\u06ee-\u06ef\u06fa-\u06fc\u06ff\u0710\u0712-\u072f\u074d-\u07a5\u07b1\u07ca-\u07ea\u07f4-\u07f5\u07fa\u0800-\u0815\u081a\u0824\u0828\u0840-\u0858\u08a0\u08a2-\u08ac\u0904-\u0939\u093d\u0950\u0958-\u0961\u0971-\u0977\u0979-\u097f\u0985-\u098c\u098f-\u0990\u0993-\u09a8\u09aa-\u09b0\u09b2\u09b6-\u09b9\u09bd\u09ce\u09dc-\u09dd\u09df-\u09e1\u09f0-\u09f1\u0a05-\u0a0a\u0a0f-\u0a10\u0a13-\u0a28\u0a2a-\u0a30\u0a32-\u0a33\u0a35-\u0a36\u0a38-\u0a39\u0a59-\u0a5c\u0a5e\u0a72-\u0a74\u0a85-\u0a8d\u0a8f-\u0a91\u0a93-\u0aa8\u0aaa-\u0ab0\u0ab2-\u0ab3\u0ab5-\u0ab9\u0abd\u0ad0\u0ae0-\u0ae1\u0b05-\u0b0c\u0b0f-\u0b10\u0b13-\u0b28\u0b2a-\u0b30\u0b32-\u0b33\u0b35-\u0b39\u0b3d\u0b5c-\u0b5d\u0b5f-\u0b61\u0b71\u0b83\u0b85-\u0b8a\u0b8e-\u0b90\u0b92-\u0b95\u0b99-\u0b9a\u0b9c\u0b9e-\u0b9f\u0ba3-\u0ba4\u0ba8-\u0baa\u0bae-\u0bb9\u0bd0\u0c05-\u0c0c\u0c0e-\u0c10\u0c12-\u0c28\u0c2a-\u0c33\u0c35-\u0c39\u0c3d\u0c58-\u0c59\u0c60-\u0c61\u0c85-\u0c8c\u0c8e-\u0c90\u0c92-\u0ca8\u0caa-\u0cb3\u0cb5-\u0cb9\u0cbd\u0cde\u0ce0-\u0ce1\u0cf1-\u0cf2\u0d05-\u0d0c\u0d0e-\u0d10\u0d12-\u0d3a\u0d3d\u0d4e\u0d60-\u0d61\u0d7a-\u0d7f\u0d85-\u0d96\u0d9a-\u0db1\u0db3-\u0dbb\u0dbd\u0dc0-\u0dc6\u0e01-\u0e30\u0e32\u0e40-\u0e46\u0e81-\u0e82\u0e84\u0e87-\u0e88\u0e8a\u0e8d\u0e94-\u0e97\u0e99-\u0e9f\u0ea1-\u0ea3\u0ea5\u0ea7\u0eaa-\u0eab\u0ead-\u0eb0\u0eb2\u0ebd\u0ec0-\u0ec4\u0ec6\u0edc-\u0edf\u0f00\u0f40-\u0f47\u0f49-\u0f6c\u0f88-\u0f8c\u1000-\u102a\u103f\u1050-\u1055\u105a-\u105d\u1061\u1065-\u1066\u106e-\u1070\u1075-\u1081\u108e\u10a0-\u10c5\u10c7\u10cd\u10d0-\u10fa\u10fc-\u1248\u124a-\u124d\u1250-\u1256\u1258\u125a-\u125d\u1260-\u1288\u128a-\u128d\u1290-\u12b0\u12b2-\u12b5\u12b8-\u12be\u12c0\u12c2-\u12c5\u12c8-\u12d6\u12d8-\u1310\u1312-\u1315\u1318-\u135a\u1380-\u138f\u13a0-\u13f4\u1401-\u166c\u166f-\u167f\u1681-\u169a\u16a0-\u16ea\u16ee-\u16f0\u1700-\u170c\u170e-\u1711\u1720-\u1731\u1740-\u1751\u1760-\u176c\u176e-\u1770\u1780-\u17b3\u17d7\u17dc\u1820-\u1877\u1880-\u18a8\u18aa\u18b0-\u18f5\u1900-\u191c\u1950-\u196d\u1970-\u1974\u1980-\u19ab\u19c1-\u19c7\u1a00-\u1a16\u1a20-\u1a54\u1aa7\u1b05-\u1b33\u1b45-\u1b4b\u1b83-\u1ba0\u1bae-\u1baf\u1bba-\u1be5\u1c00-\u1c23\u1c4d-\u1c4f\u1c5a-\u1c7d\u1ce9-\u1cec\u1cee-\u1cf1\u1cf5-\u1cf6\u1d00-\u1dbf\u1e00-\u1f15\u1f18-\u1f1d\u1f20-\u1f45\u1f48-\u1f4d\u1f50-\u1f57\u1f59\u1f5b\u1f5d\u1f5f-\u1f7d\u1f80-\u1fb4\u1fb6-\u1fbc\u1fbe\u1fc2-\u1fc4\u1fc6-\u1fcc\u1fd0-\u1fd3\u1fd6-\u1fdb\u1fe0-\u1fec\u1ff2-\u1ff4\u1ff6-\u1ffc\u2071\u207f\u2090-\u209c\u2102\u2107\u210a-\u2113\u2115\u2118-\u211d\u2124\u2126\u2128\u212a-\u2139\u213c-\u213f\u2145-\u2149\u214e\u2160-\u2188\u2c00-\u2c2e\u2c30-\u2c5e\u2c60-\u2ce4\u2ceb-\u2cee\u2cf2-\u2cf3\u2d00-\u2d25\u2d27\u2d2d\u2d30-\u2d67\u2d6f\u2d80-\u2d96\u2da0-\u2da6\u2da8-\u2dae\u2db0-\u2db6\u2db8-\u2dbe\u2dc0-\u2dc6\u2dc8-\u2dce\u2dd0-\u2dd6\u2dd8-\u2dde\u3005-\u3007\u3021-\u3029\u3031-\u3035\u3038-\u303c\u3041-\u3096\u309d-\u309f\u30a1-\u30fa\u30fc-\u30ff\u3105-\u312d\u3131-\u318e\u31a0-\u31ba\u31f0-\u31ff\u3400-\u4db5\u4e00-\u9fcc\ua000-\ua48c\ua4d0-\ua4fd\ua500-\ua60c\ua610-\ua61f\ua62a-\ua62b\ua640-\ua66e\ua67f-\ua697\ua6a0-\ua6ef\ua717-\ua71f\ua722-\ua788\ua78b-\ua78e\ua790-\ua793\ua7a0-\ua7aa\ua7f8-\ua801\ua803-\ua805\ua807-\ua80a\ua80c-\ua822\ua840-\ua873\ua882-\ua8b3\ua8f2-\ua8f7\ua8fb\ua90a-\ua925\ua930-\ua946\ua960-\ua97c\ua984-\ua9b2\ua9cf\uaa00-\uaa28\uaa40-\uaa42\uaa44-\uaa4b\uaa60-\uaa76\uaa7a\uaa80-\uaaaf\uaab1\uaab5-\uaab6\uaab9-\uaabd\uaac0\uaac2\uaadb-\uaadd\uaae0-\uaaea\uaaf2-\uaaf4\uab01-\uab06\uab09-\uab0e\uab11-\uab16\uab20-\uab26\uab28-\uab2e\uabc0-\uabe2\uac00-\ud7a3\ud7b0-\ud7c6\ud7cb-\ud7fb\uf900-\ufa6d\ufa70-\ufad9\ufb00-\ufb06\ufb13-\ufb17\ufb1d\ufb1f-\ufb28\ufb2a-\ufb36\ufb38-\ufb3c\ufb3e\ufb40-\ufb41\ufb43-\ufb44\ufb46-\ufbb1\ufbd3-\ufc5d\ufc64-\ufd3d\ufd50-\ufd8f\ufd92-\ufdc7\ufdf0-\ufdf9\ufe71\ufe73\ufe77\ufe79\ufe7b\ufe7d\ufe7f-\ufefc\uff21-\uff3a\uff41-\uff5a\uff66-\uff9d\uffa0-\uffbe\uffc2-\uffc7\uffca-\uffcf\uffd2-\uffd7\uffda-\uffdc' - -if sys.maxunicode > 0xFFFF: - # non-BMP characters, use only on wide Unicode builds - Cf += u'\U000110bd\U0001d173-\U0001d17a\U000e0001\U000e0020-\U000e007f' - - Cn += u'\U0001000c\U00010027\U0001003b\U0001003e\U0001004e-\U0001004f\U0001005e-\U0001007f\U000100fb-\U000100ff\U00010103-\U00010106\U00010134-\U00010136\U0001018b-\U0001018f\U0001019c-\U000101cf\U000101fe-\U0001027f\U0001029d-\U0001029f\U000102d1-\U000102ff\U0001031f\U00010324-\U0001032f\U0001034b-\U0001037f\U0001039e\U000103c4-\U000103c7\U000103d6-\U000103ff\U0001049e-\U0001049f\U000104aa-\U000107ff\U00010806-\U00010807\U00010809\U00010836\U00010839-\U0001083b\U0001083d-\U0001083e\U00010856\U00010860-\U000108ff\U0001091c-\U0001091e\U0001093a-\U0001093e\U00010940-\U0001097f\U000109b8-\U000109bd\U000109c0-\U000109ff\U00010a04\U00010a07-\U00010a0b\U00010a14\U00010a18\U00010a34-\U00010a37\U00010a3b-\U00010a3e\U00010a48-\U00010a4f\U00010a59-\U00010a5f\U00010a80-\U00010aff\U00010b36-\U00010b38\U00010b56-\U00010b57\U00010b73-\U00010b77\U00010b80-\U00010bff\U00010c49-\U00010e5f\U00010e7f-\U00010fff\U0001104e-\U00011051\U00011070-\U0001107f\U000110c2-\U000110cf\U000110e9-\U000110ef\U000110fa-\U000110ff\U00011135\U00011144-\U0001117f\U000111c9-\U000111cf\U000111da-\U0001167f\U000116b8-\U000116bf\U000116ca-\U00011fff\U0001236f-\U000123ff\U00012463-\U0001246f\U00012474-\U00012fff\U0001342f-\U000167ff\U00016a39-\U00016eff\U00016f45-\U00016f4f\U00016f7f-\U00016f8e\U00016fa0-\U0001afff\U0001b002-\U0001cfff\U0001d0f6-\U0001d0ff\U0001d127-\U0001d128\U0001d1de-\U0001d1ff\U0001d246-\U0001d2ff\U0001d357-\U0001d35f\U0001d372-\U0001d3ff\U0001d455\U0001d49d\U0001d4a0-\U0001d4a1\U0001d4a3-\U0001d4a4\U0001d4a7-\U0001d4a8\U0001d4ad\U0001d4ba\U0001d4bc\U0001d4c4\U0001d506\U0001d50b-\U0001d50c\U0001d515\U0001d51d\U0001d53a\U0001d53f\U0001d545\U0001d547-\U0001d549\U0001d551\U0001d6a6-\U0001d6a7\U0001d7cc-\U0001d7cd\U0001d800-\U0001edff\U0001ee04\U0001ee20\U0001ee23\U0001ee25-\U0001ee26\U0001ee28\U0001ee33\U0001ee38\U0001ee3a\U0001ee3c-\U0001ee41\U0001ee43-\U0001ee46\U0001ee48\U0001ee4a\U0001ee4c\U0001ee50\U0001ee53\U0001ee55-\U0001ee56\U0001ee58\U0001ee5a\U0001ee5c\U0001ee5e\U0001ee60\U0001ee63\U0001ee65-\U0001ee66\U0001ee6b\U0001ee73\U0001ee78\U0001ee7d\U0001ee7f\U0001ee8a\U0001ee9c-\U0001eea0\U0001eea4\U0001eeaa\U0001eebc-\U0001eeef\U0001eef2-\U0001efff\U0001f02c-\U0001f02f\U0001f094-\U0001f09f\U0001f0af-\U0001f0b0\U0001f0bf-\U0001f0c0\U0001f0d0\U0001f0e0-\U0001f0ff\U0001f10b-\U0001f10f\U0001f12f\U0001f16c-\U0001f16f\U0001f19b-\U0001f1e5\U0001f203-\U0001f20f\U0001f23b-\U0001f23f\U0001f249-\U0001f24f\U0001f252-\U0001f2ff\U0001f321-\U0001f32f\U0001f336\U0001f37d-\U0001f37f\U0001f394-\U0001f39f\U0001f3c5\U0001f3cb-\U0001f3df\U0001f3f1-\U0001f3ff\U0001f43f\U0001f441\U0001f4f8\U0001f4fd-\U0001f4ff\U0001f53e-\U0001f53f\U0001f544-\U0001f54f\U0001f568-\U0001f5fa\U0001f641-\U0001f644\U0001f650-\U0001f67f\U0001f6c6-\U0001f6ff\U0001f774-\U0001ffff\U0002a6d7-\U0002a6ff\U0002b735-\U0002b73f\U0002b81e-\U0002f7ff\U0002fa1e-\U000e0000\U000e0002-\U000e001f\U000e0080-\U000e00ff\U000e01f0-\U000effff\U000ffffe-\U000fffff\U0010fffe-\U0010ffff' - - Co += u'\U000f0000-\U000ffffd\U00100000-\U0010fffd' - - Ll += u'\U00010428-\U0001044f\U0001d41a-\U0001d433\U0001d44e-\U0001d454\U0001d456-\U0001d467\U0001d482-\U0001d49b\U0001d4b6-\U0001d4b9\U0001d4bb\U0001d4bd-\U0001d4c3\U0001d4c5-\U0001d4cf\U0001d4ea-\U0001d503\U0001d51e-\U0001d537\U0001d552-\U0001d56b\U0001d586-\U0001d59f\U0001d5ba-\U0001d5d3\U0001d5ee-\U0001d607\U0001d622-\U0001d63b\U0001d656-\U0001d66f\U0001d68a-\U0001d6a5\U0001d6c2-\U0001d6da\U0001d6dc-\U0001d6e1\U0001d6fc-\U0001d714\U0001d716-\U0001d71b\U0001d736-\U0001d74e\U0001d750-\U0001d755\U0001d770-\U0001d788\U0001d78a-\U0001d78f\U0001d7aa-\U0001d7c2\U0001d7c4-\U0001d7c9\U0001d7cb' - - Lm += u'\U00016f93-\U00016f9f' - - Lo += u'\U00010000-\U0001000b\U0001000d-\U00010026\U00010028-\U0001003a\U0001003c-\U0001003d\U0001003f-\U0001004d\U00010050-\U0001005d\U00010080-\U000100fa\U00010280-\U0001029c\U000102a0-\U000102d0\U00010300-\U0001031e\U00010330-\U00010340\U00010342-\U00010349\U00010380-\U0001039d\U000103a0-\U000103c3\U000103c8-\U000103cf\U00010450-\U0001049d\U00010800-\U00010805\U00010808\U0001080a-\U00010835\U00010837-\U00010838\U0001083c\U0001083f-\U00010855\U00010900-\U00010915\U00010920-\U00010939\U00010980-\U000109b7\U000109be-\U000109bf\U00010a00\U00010a10-\U00010a13\U00010a15-\U00010a17\U00010a19-\U00010a33\U00010a60-\U00010a7c\U00010b00-\U00010b35\U00010b40-\U00010b55\U00010b60-\U00010b72\U00010c00-\U00010c48\U00011003-\U00011037\U00011083-\U000110af\U000110d0-\U000110e8\U00011103-\U00011126\U00011183-\U000111b2\U000111c1-\U000111c4\U00011680-\U000116aa\U00012000-\U0001236e\U00013000-\U0001342e\U00016800-\U00016a38\U00016f00-\U00016f44\U00016f50\U0001b000-\U0001b001\U0001ee00-\U0001ee03\U0001ee05-\U0001ee1f\U0001ee21-\U0001ee22\U0001ee24\U0001ee27\U0001ee29-\U0001ee32\U0001ee34-\U0001ee37\U0001ee39\U0001ee3b\U0001ee42\U0001ee47\U0001ee49\U0001ee4b\U0001ee4d-\U0001ee4f\U0001ee51-\U0001ee52\U0001ee54\U0001ee57\U0001ee59\U0001ee5b\U0001ee5d\U0001ee5f\U0001ee61-\U0001ee62\U0001ee64\U0001ee67-\U0001ee6a\U0001ee6c-\U0001ee72\U0001ee74-\U0001ee77\U0001ee79-\U0001ee7c\U0001ee7e\U0001ee80-\U0001ee89\U0001ee8b-\U0001ee9b\U0001eea1-\U0001eea3\U0001eea5-\U0001eea9\U0001eeab-\U0001eebb\U00020000-\U0002a6d6\U0002a700-\U0002b734\U0002b740-\U0002b81d\U0002f800-\U0002fa1d' - - Lu += u'\U00010400-\U00010427\U0001d400-\U0001d419\U0001d434-\U0001d44d\U0001d468-\U0001d481\U0001d49c\U0001d49e-\U0001d49f\U0001d4a2\U0001d4a5-\U0001d4a6\U0001d4a9-\U0001d4ac\U0001d4ae-\U0001d4b5\U0001d4d0-\U0001d4e9\U0001d504-\U0001d505\U0001d507-\U0001d50a\U0001d50d-\U0001d514\U0001d516-\U0001d51c\U0001d538-\U0001d539\U0001d53b-\U0001d53e\U0001d540-\U0001d544\U0001d546\U0001d54a-\U0001d550\U0001d56c-\U0001d585\U0001d5a0-\U0001d5b9\U0001d5d4-\U0001d5ed\U0001d608-\U0001d621\U0001d63c-\U0001d655\U0001d670-\U0001d689\U0001d6a8-\U0001d6c0\U0001d6e2-\U0001d6fa\U0001d71c-\U0001d734\U0001d756-\U0001d76e\U0001d790-\U0001d7a8\U0001d7ca' - - Mc += u'\U00011000\U00011002\U00011082\U000110b0-\U000110b2\U000110b7-\U000110b8\U0001112c\U00011182\U000111b3-\U000111b5\U000111bf-\U000111c0\U000116ac\U000116ae-\U000116af\U000116b6\U00016f51-\U00016f7e\U0001d165-\U0001d166\U0001d16d-\U0001d172' - - Mn += u'\U000101fd\U00010a01-\U00010a03\U00010a05-\U00010a06\U00010a0c-\U00010a0f\U00010a38-\U00010a3a\U00010a3f\U00011001\U00011038-\U00011046\U00011080-\U00011081\U000110b3-\U000110b6\U000110b9-\U000110ba\U00011100-\U00011102\U00011127-\U0001112b\U0001112d-\U00011134\U00011180-\U00011181\U000111b6-\U000111be\U000116ab\U000116ad\U000116b0-\U000116b5\U000116b7\U00016f8f-\U00016f92\U0001d167-\U0001d169\U0001d17b-\U0001d182\U0001d185-\U0001d18b\U0001d1aa-\U0001d1ad\U0001d242-\U0001d244\U000e0100-\U000e01ef' - - Nd += u'\U000104a0-\U000104a9\U00011066-\U0001106f\U000110f0-\U000110f9\U00011136-\U0001113f\U000111d0-\U000111d9\U000116c0-\U000116c9\U0001d7ce-\U0001d7ff' - - Nl += u'\U00010140-\U00010174\U00010341\U0001034a\U000103d1-\U000103d5\U00012400-\U00012462' - - No += u'\U00010107-\U00010133\U00010175-\U00010178\U0001018a\U00010320-\U00010323\U00010858-\U0001085f\U00010916-\U0001091b\U00010a40-\U00010a47\U00010a7d-\U00010a7e\U00010b58-\U00010b5f\U00010b78-\U00010b7f\U00010e60-\U00010e7e\U00011052-\U00011065\U0001d360-\U0001d371\U0001f100-\U0001f10a' - - Po += u'\U00010100-\U00010102\U0001039f\U000103d0\U00010857\U0001091f\U0001093f\U00010a50-\U00010a58\U00010a7f\U00010b39-\U00010b3f\U00011047-\U0001104d\U000110bb-\U000110bc\U000110be-\U000110c1\U00011140-\U00011143\U000111c5-\U000111c8\U00012470-\U00012473' - - Sm += u'\U0001d6c1\U0001d6db\U0001d6fb\U0001d715\U0001d735\U0001d74f\U0001d76f\U0001d789\U0001d7a9\U0001d7c3\U0001eef0-\U0001eef1' - - So += u'\U00010137-\U0001013f\U00010179-\U00010189\U00010190-\U0001019b\U000101d0-\U000101fc\U0001d000-\U0001d0f5\U0001d100-\U0001d126\U0001d129-\U0001d164\U0001d16a-\U0001d16c\U0001d183-\U0001d184\U0001d18c-\U0001d1a9\U0001d1ae-\U0001d1dd\U0001d200-\U0001d241\U0001d245\U0001d300-\U0001d356\U0001f000-\U0001f02b\U0001f030-\U0001f093\U0001f0a0-\U0001f0ae\U0001f0b1-\U0001f0be\U0001f0c1-\U0001f0cf\U0001f0d1-\U0001f0df\U0001f110-\U0001f12e\U0001f130-\U0001f16b\U0001f170-\U0001f19a\U0001f1e6-\U0001f202\U0001f210-\U0001f23a\U0001f240-\U0001f248\U0001f250-\U0001f251\U0001f300-\U0001f320\U0001f330-\U0001f335\U0001f337-\U0001f37c\U0001f380-\U0001f393\U0001f3a0-\U0001f3c4\U0001f3c6-\U0001f3ca\U0001f3e0-\U0001f3f0\U0001f400-\U0001f43e\U0001f440\U0001f442-\U0001f4f7\U0001f4f9-\U0001f4fc\U0001f500-\U0001f53d\U0001f540-\U0001f543\U0001f550-\U0001f567\U0001f5fb-\U0001f640\U0001f645-\U0001f64f\U0001f680-\U0001f6c5\U0001f700-\U0001f773' - - xid_continue += u'\U00010000-\U0001000b\U0001000d-\U00010026\U00010028-\U0001003a\U0001003c-\U0001003d\U0001003f-\U0001004d\U00010050-\U0001005d\U00010080-\U000100fa\U00010140-\U00010174\U000101fd\U00010280-\U0001029c\U000102a0-\U000102d0\U00010300-\U0001031e\U00010330-\U0001034a\U00010380-\U0001039d\U000103a0-\U000103c3\U000103c8-\U000103cf\U000103d1-\U000103d5\U00010400-\U0001049d\U000104a0-\U000104a9\U00010800-\U00010805\U00010808\U0001080a-\U00010835\U00010837-\U00010838\U0001083c\U0001083f-\U00010855\U00010900-\U00010915\U00010920-\U00010939\U00010980-\U000109b7\U000109be-\U000109bf\U00010a00-\U00010a03\U00010a05-\U00010a06\U00010a0c-\U00010a13\U00010a15-\U00010a17\U00010a19-\U00010a33\U00010a38-\U00010a3a\U00010a3f\U00010a60-\U00010a7c\U00010b00-\U00010b35\U00010b40-\U00010b55\U00010b60-\U00010b72\U00010c00-\U00010c48\U00011000-\U00011046\U00011066-\U0001106f\U00011080-\U000110ba\U000110d0-\U000110e8\U000110f0-\U000110f9\U00011100-\U00011134\U00011136-\U0001113f\U00011180-\U000111c4\U000111d0-\U000111d9\U00011680-\U000116b7\U000116c0-\U000116c9\U00012000-\U0001236e\U00012400-\U00012462\U00013000-\U0001342e\U00016800-\U00016a38\U00016f00-\U00016f44\U00016f50-\U00016f7e\U00016f8f-\U00016f9f\U0001b000-\U0001b001\U0001d165-\U0001d169\U0001d16d-\U0001d172\U0001d17b-\U0001d182\U0001d185-\U0001d18b\U0001d1aa-\U0001d1ad\U0001d242-\U0001d244\U0001d400-\U0001d454\U0001d456-\U0001d49c\U0001d49e-\U0001d49f\U0001d4a2\U0001d4a5-\U0001d4a6\U0001d4a9-\U0001d4ac\U0001d4ae-\U0001d4b9\U0001d4bb\U0001d4bd-\U0001d4c3\U0001d4c5-\U0001d505\U0001d507-\U0001d50a\U0001d50d-\U0001d514\U0001d516-\U0001d51c\U0001d51e-\U0001d539\U0001d53b-\U0001d53e\U0001d540-\U0001d544\U0001d546\U0001d54a-\U0001d550\U0001d552-\U0001d6a5\U0001d6a8-\U0001d6c0\U0001d6c2-\U0001d6da\U0001d6dc-\U0001d6fa\U0001d6fc-\U0001d714\U0001d716-\U0001d734\U0001d736-\U0001d74e\U0001d750-\U0001d76e\U0001d770-\U0001d788\U0001d78a-\U0001d7a8\U0001d7aa-\U0001d7c2\U0001d7c4-\U0001d7cb\U0001d7ce-\U0001d7ff\U0001ee00-\U0001ee03\U0001ee05-\U0001ee1f\U0001ee21-\U0001ee22\U0001ee24\U0001ee27\U0001ee29-\U0001ee32\U0001ee34-\U0001ee37\U0001ee39\U0001ee3b\U0001ee42\U0001ee47\U0001ee49\U0001ee4b\U0001ee4d-\U0001ee4f\U0001ee51-\U0001ee52\U0001ee54\U0001ee57\U0001ee59\U0001ee5b\U0001ee5d\U0001ee5f\U0001ee61-\U0001ee62\U0001ee64\U0001ee67-\U0001ee6a\U0001ee6c-\U0001ee72\U0001ee74-\U0001ee77\U0001ee79-\U0001ee7c\U0001ee7e\U0001ee80-\U0001ee89\U0001ee8b-\U0001ee9b\U0001eea1-\U0001eea3\U0001eea5-\U0001eea9\U0001eeab-\U0001eebb\U00020000-\U0002a6d6\U0002a700-\U0002b734\U0002b740-\U0002b81d\U0002f800-\U0002fa1d\U000e0100-\U000e01ef' - - xid_start += u'\U00010000-\U0001000b\U0001000d-\U00010026\U00010028-\U0001003a\U0001003c-\U0001003d\U0001003f-\U0001004d\U00010050-\U0001005d\U00010080-\U000100fa\U00010140-\U00010174\U00010280-\U0001029c\U000102a0-\U000102d0\U00010300-\U0001031e\U00010330-\U0001034a\U00010380-\U0001039d\U000103a0-\U000103c3\U000103c8-\U000103cf\U000103d1-\U000103d5\U00010400-\U0001049d\U00010800-\U00010805\U00010808\U0001080a-\U00010835\U00010837-\U00010838\U0001083c\U0001083f-\U00010855\U00010900-\U00010915\U00010920-\U00010939\U00010980-\U000109b7\U000109be-\U000109bf\U00010a00\U00010a10-\U00010a13\U00010a15-\U00010a17\U00010a19-\U00010a33\U00010a60-\U00010a7c\U00010b00-\U00010b35\U00010b40-\U00010b55\U00010b60-\U00010b72\U00010c00-\U00010c48\U00011003-\U00011037\U00011083-\U000110af\U000110d0-\U000110e8\U00011103-\U00011126\U00011183-\U000111b2\U000111c1-\U000111c4\U00011680-\U000116aa\U00012000-\U0001236e\U00012400-\U00012462\U00013000-\U0001342e\U00016800-\U00016a38\U00016f00-\U00016f44\U00016f50\U00016f93-\U00016f9f\U0001b000-\U0001b001\U0001d400-\U0001d454\U0001d456-\U0001d49c\U0001d49e-\U0001d49f\U0001d4a2\U0001d4a5-\U0001d4a6\U0001d4a9-\U0001d4ac\U0001d4ae-\U0001d4b9\U0001d4bb\U0001d4bd-\U0001d4c3\U0001d4c5-\U0001d505\U0001d507-\U0001d50a\U0001d50d-\U0001d514\U0001d516-\U0001d51c\U0001d51e-\U0001d539\U0001d53b-\U0001d53e\U0001d540-\U0001d544\U0001d546\U0001d54a-\U0001d550\U0001d552-\U0001d6a5\U0001d6a8-\U0001d6c0\U0001d6c2-\U0001d6da\U0001d6dc-\U0001d6fa\U0001d6fc-\U0001d714\U0001d716-\U0001d734\U0001d736-\U0001d74e\U0001d750-\U0001d76e\U0001d770-\U0001d788\U0001d78a-\U0001d7a8\U0001d7aa-\U0001d7c2\U0001d7c4-\U0001d7cb\U0001ee00-\U0001ee03\U0001ee05-\U0001ee1f\U0001ee21-\U0001ee22\U0001ee24\U0001ee27\U0001ee29-\U0001ee32\U0001ee34-\U0001ee37\U0001ee39\U0001ee3b\U0001ee42\U0001ee47\U0001ee49\U0001ee4b\U0001ee4d-\U0001ee4f\U0001ee51-\U0001ee52\U0001ee54\U0001ee57\U0001ee59\U0001ee5b\U0001ee5d\U0001ee5f\U0001ee61-\U0001ee62\U0001ee64\U0001ee67-\U0001ee6a\U0001ee6c-\U0001ee72\U0001ee74-\U0001ee77\U0001ee79-\U0001ee7c\U0001ee7e\U0001ee80-\U0001ee89\U0001ee8b-\U0001ee9b\U0001eea1-\U0001eea3\U0001eea5-\U0001eea9\U0001eeab-\U0001eebb\U00020000-\U0002a6d6\U0002a700-\U0002b734\U0002b740-\U0002b81d\U0002f800-\U0002fa1d' +xid_start = 'A-Z_a-z\xaa\xb5\xba\xc0-\xd6\xd8-\xf6\xf8-\u02c1\u02c6-\u02d1\u02e0-\u02e4\u02ec\u02ee\u0370-\u0374\u0376-\u0377\u037b-\u037d\u037f\u0386\u0388-\u038a\u038c\u038e-\u03a1\u03a3-\u03f5\u03f7-\u0481\u048a-\u052f\u0531-\u0556\u0559\u0560-\u0588\u05d0-\u05ea\u05ef-\u05f2\u0620-\u064a\u066e-\u066f\u0671-\u06d3\u06d5\u06e5-\u06e6\u06ee-\u06ef\u06fa-\u06fc\u06ff\u0710\u0712-\u072f\u074d-\u07a5\u07b1\u07ca-\u07ea\u07f4-\u07f5\u07fa\u0800-\u0815\u081a\u0824\u0828\u0840-\u0858\u0860-\u086a\u08a0-\u08b4\u08b6-\u08bd\u0904-\u0939\u093d\u0950\u0958-\u0961\u0971-\u0980\u0985-\u098c\u098f-\u0990\u0993-\u09a8\u09aa-\u09b0\u09b2\u09b6-\u09b9\u09bd\u09ce\u09dc-\u09dd\u09df-\u09e1\u09f0-\u09f1\u09fc\u0a05-\u0a0a\u0a0f-\u0a10\u0a13-\u0a28\u0a2a-\u0a30\u0a32-\u0a33\u0a35-\u0a36\u0a38-\u0a39\u0a59-\u0a5c\u0a5e\u0a72-\u0a74\u0a85-\u0a8d\u0a8f-\u0a91\u0a93-\u0aa8\u0aaa-\u0ab0\u0ab2-\u0ab3\u0ab5-\u0ab9\u0abd\u0ad0\u0ae0-\u0ae1\u0af9\u0b05-\u0b0c\u0b0f-\u0b10\u0b13-\u0b28\u0b2a-\u0b30\u0b32-\u0b33\u0b35-\u0b39\u0b3d\u0b5c-\u0b5d\u0b5f-\u0b61\u0b71\u0b83\u0b85-\u0b8a\u0b8e-\u0b90\u0b92-\u0b95\u0b99-\u0b9a\u0b9c\u0b9e-\u0b9f\u0ba3-\u0ba4\u0ba8-\u0baa\u0bae-\u0bb9\u0bd0\u0c05-\u0c0c\u0c0e-\u0c10\u0c12-\u0c28\u0c2a-\u0c39\u0c3d\u0c58-\u0c5a\u0c60-\u0c61\u0c80\u0c85-\u0c8c\u0c8e-\u0c90\u0c92-\u0ca8\u0caa-\u0cb3\u0cb5-\u0cb9\u0cbd\u0cde\u0ce0-\u0ce1\u0cf1-\u0cf2\u0d05-\u0d0c\u0d0e-\u0d10\u0d12-\u0d3a\u0d3d\u0d4e\u0d54-\u0d56\u0d5f-\u0d61\u0d7a-\u0d7f\u0d85-\u0d96\u0d9a-\u0db1\u0db3-\u0dbb\u0dbd\u0dc0-\u0dc6\u0e01-\u0e30\u0e32\u0e40-\u0e46\u0e81-\u0e82\u0e84\u0e87-\u0e88\u0e8a\u0e8d\u0e94-\u0e97\u0e99-\u0e9f\u0ea1-\u0ea3\u0ea5\u0ea7\u0eaa-\u0eab\u0ead-\u0eb0\u0eb2\u0ebd\u0ec0-\u0ec4\u0ec6\u0edc-\u0edf\u0f00\u0f40-\u0f47\u0f49-\u0f6c\u0f88-\u0f8c\u1000-\u102a\u103f\u1050-\u1055\u105a-\u105d\u1061\u1065-\u1066\u106e-\u1070\u1075-\u1081\u108e\u10a0-\u10c5\u10c7\u10cd\u10d0-\u10fa\u10fc-\u1248\u124a-\u124d\u1250-\u1256\u1258\u125a-\u125d\u1260-\u1288\u128a-\u128d\u1290-\u12b0\u12b2-\u12b5\u12b8-\u12be\u12c0\u12c2-\u12c5\u12c8-\u12d6\u12d8-\u1310\u1312-\u1315\u1318-\u135a\u1380-\u138f\u13a0-\u13f5\u13f8-\u13fd\u1401-\u166c\u166f-\u167f\u1681-\u169a\u16a0-\u16ea\u16ee-\u16f8\u1700-\u170c\u170e-\u1711\u1720-\u1731\u1740-\u1751\u1760-\u176c\u176e-\u1770\u1780-\u17b3\u17d7\u17dc\u1820-\u1878\u1880-\u18a8\u18aa\u18b0-\u18f5\u1900-\u191e\u1950-\u196d\u1970-\u1974\u1980-\u19ab\u19b0-\u19c9\u1a00-\u1a16\u1a20-\u1a54\u1aa7\u1b05-\u1b33\u1b45-\u1b4b\u1b83-\u1ba0\u1bae-\u1baf\u1bba-\u1be5\u1c00-\u1c23\u1c4d-\u1c4f\u1c5a-\u1c7d\u1c80-\u1c88\u1c90-\u1cba\u1cbd-\u1cbf\u1ce9-\u1cec\u1cee-\u1cf1\u1cf5-\u1cf6\u1d00-\u1dbf\u1e00-\u1f15\u1f18-\u1f1d\u1f20-\u1f45\u1f48-\u1f4d\u1f50-\u1f57\u1f59\u1f5b\u1f5d\u1f5f-\u1f7d\u1f80-\u1fb4\u1fb6-\u1fbc\u1fbe\u1fc2-\u1fc4\u1fc6-\u1fcc\u1fd0-\u1fd3\u1fd6-\u1fdb\u1fe0-\u1fec\u1ff2-\u1ff4\u1ff6-\u1ffc\u2071\u207f\u2090-\u209c\u2102\u2107\u210a-\u2113\u2115\u2118-\u211d\u2124\u2126\u2128\u212a-\u2139\u213c-\u213f\u2145-\u2149\u214e\u2160-\u2188\u2c00-\u2c2e\u2c30-\u2c5e\u2c60-\u2ce4\u2ceb-\u2cee\u2cf2-\u2cf3\u2d00-\u2d25\u2d27\u2d2d\u2d30-\u2d67\u2d6f\u2d80-\u2d96\u2da0-\u2da6\u2da8-\u2dae\u2db0-\u2db6\u2db8-\u2dbe\u2dc0-\u2dc6\u2dc8-\u2dce\u2dd0-\u2dd6\u2dd8-\u2dde\u3005-\u3007\u3021-\u3029\u3031-\u3035\u3038-\u303c\u3041-\u3096\u309d-\u309f\u30a1-\u30fa\u30fc-\u30ff\u3105-\u312f\u3131-\u318e\u31a0-\u31ba\u31f0-\u31ff\u3400-\u4db5\u4e00-\u9fef\ua000-\ua48c\ua4d0-\ua4fd\ua500-\ua60c\ua610-\ua61f\ua62a-\ua62b\ua640-\ua66e\ua67f-\ua69d\ua6a0-\ua6ef\ua717-\ua71f\ua722-\ua788\ua78b-\ua7b9\ua7f7-\ua801\ua803-\ua805\ua807-\ua80a\ua80c-\ua822\ua840-\ua873\ua882-\ua8b3\ua8f2-\ua8f7\ua8fb\ua8fd-\ua8fe\ua90a-\ua925\ua930-\ua946\ua960-\ua97c\ua984-\ua9b2\ua9cf\ua9e0-\ua9e4\ua9e6-\ua9ef\ua9fa-\ua9fe\uaa00-\uaa28\uaa40-\uaa42\uaa44-\uaa4b\uaa60-\uaa76\uaa7a\uaa7e-\uaaaf\uaab1\uaab5-\uaab6\uaab9-\uaabd\uaac0\uaac2\uaadb-\uaadd\uaae0-\uaaea\uaaf2-\uaaf4\uab01-\uab06\uab09-\uab0e\uab11-\uab16\uab20-\uab26\uab28-\uab2e\uab30-\uab5a\uab5c-\uab65\uab70-\uabe2\uac00-\ud7a3\ud7b0-\ud7c6\ud7cb-\ud7fb\uf900-\ufa6d\ufa70-\ufad9\ufb00-\ufb06\ufb13-\ufb17\ufb1d\ufb1f-\ufb28\ufb2a-\ufb36\ufb38-\ufb3c\ufb3e\ufb40-\ufb41\ufb43-\ufb44\ufb46-\ufbb1\ufbd3-\ufc5d\ufc64-\ufd3d\ufd50-\ufd8f\ufd92-\ufdc7\ufdf0-\ufdf9\ufe71\ufe73\ufe77\ufe79\ufe7b\ufe7d\ufe7f-\ufefc\uff21-\uff3a\uff41-\uff5a\uff66-\uff9d\uffa0-\uffbe\uffc2-\uffc7\uffca-\uffcf\uffd2-\uffd7\uffda-\uffdc\U00010000-\U0001000b\U0001000d-\U00010026\U00010028-\U0001003a\U0001003c-\U0001003d\U0001003f-\U0001004d\U00010050-\U0001005d\U00010080-\U000100fa\U00010140-\U00010174\U00010280-\U0001029c\U000102a0-\U000102d0\U00010300-\U0001031f\U0001032d-\U0001034a\U00010350-\U00010375\U00010380-\U0001039d\U000103a0-\U000103c3\U000103c8-\U000103cf\U000103d1-\U000103d5\U00010400-\U0001049d\U000104b0-\U000104d3\U000104d8-\U000104fb\U00010500-\U00010527\U00010530-\U00010563\U00010600-\U00010736\U00010740-\U00010755\U00010760-\U00010767\U00010800-\U00010805\U00010808\U0001080a-\U00010835\U00010837-\U00010838\U0001083c\U0001083f-\U00010855\U00010860-\U00010876\U00010880-\U0001089e\U000108e0-\U000108f2\U000108f4-\U000108f5\U00010900-\U00010915\U00010920-\U00010939\U00010980-\U000109b7\U000109be-\U000109bf\U00010a00\U00010a10-\U00010a13\U00010a15-\U00010a17\U00010a19-\U00010a35\U00010a60-\U00010a7c\U00010a80-\U00010a9c\U00010ac0-\U00010ac7\U00010ac9-\U00010ae4\U00010b00-\U00010b35\U00010b40-\U00010b55\U00010b60-\U00010b72\U00010b80-\U00010b91\U00010c00-\U00010c48\U00010c80-\U00010cb2\U00010cc0-\U00010cf2\U00010d00-\U00010d23\U00010f00-\U00010f1c\U00010f27\U00010f30-\U00010f45\U00011003-\U00011037\U00011083-\U000110af\U000110d0-\U000110e8\U00011103-\U00011126\U00011144\U00011150-\U00011172\U00011176\U00011183-\U000111b2\U000111c1-\U000111c4\U000111da\U000111dc\U00011200-\U00011211\U00011213-\U0001122b\U00011280-\U00011286\U00011288\U0001128a-\U0001128d\U0001128f-\U0001129d\U0001129f-\U000112a8\U000112b0-\U000112de\U00011305-\U0001130c\U0001130f-\U00011310\U00011313-\U00011328\U0001132a-\U00011330\U00011332-\U00011333\U00011335-\U00011339\U0001133d\U00011350\U0001135d-\U00011361\U00011400-\U00011434\U00011447-\U0001144a\U00011480-\U000114af\U000114c4-\U000114c5\U000114c7\U00011580-\U000115ae\U000115d8-\U000115db\U00011600-\U0001162f\U00011644\U00011680-\U000116aa\U00011700-\U0001171a\U00011800-\U0001182b\U000118a0-\U000118df\U000118ff\U00011a00\U00011a0b-\U00011a32\U00011a3a\U00011a50\U00011a5c-\U00011a83\U00011a86-\U00011a89\U00011a9d\U00011ac0-\U00011af8\U00011c00-\U00011c08\U00011c0a-\U00011c2e\U00011c40\U00011c72-\U00011c8f\U00011d00-\U00011d06\U00011d08-\U00011d09\U00011d0b-\U00011d30\U00011d46\U00011d60-\U00011d65\U00011d67-\U00011d68\U00011d6a-\U00011d89\U00011d98\U00011ee0-\U00011ef2\U00012000-\U00012399\U00012400-\U0001246e\U00012480-\U00012543\U00013000-\U0001342e\U00014400-\U00014646\U00016800-\U00016a38\U00016a40-\U00016a5e\U00016ad0-\U00016aed\U00016b00-\U00016b2f\U00016b40-\U00016b43\U00016b63-\U00016b77\U00016b7d-\U00016b8f\U00016e40-\U00016e7f\U00016f00-\U00016f44\U00016f50\U00016f93-\U00016f9f\U00016fe0-\U00016fe1\U00017000-\U000187f1\U00018800-\U00018af2\U0001b000-\U0001b11e\U0001b170-\U0001b2fb\U0001bc00-\U0001bc6a\U0001bc70-\U0001bc7c\U0001bc80-\U0001bc88\U0001bc90-\U0001bc99\U0001d400-\U0001d454\U0001d456-\U0001d49c\U0001d49e-\U0001d49f\U0001d4a2\U0001d4a5-\U0001d4a6\U0001d4a9-\U0001d4ac\U0001d4ae-\U0001d4b9\U0001d4bb\U0001d4bd-\U0001d4c3\U0001d4c5-\U0001d505\U0001d507-\U0001d50a\U0001d50d-\U0001d514\U0001d516-\U0001d51c\U0001d51e-\U0001d539\U0001d53b-\U0001d53e\U0001d540-\U0001d544\U0001d546\U0001d54a-\U0001d550\U0001d552-\U0001d6a5\U0001d6a8-\U0001d6c0\U0001d6c2-\U0001d6da\U0001d6dc-\U0001d6fa\U0001d6fc-\U0001d714\U0001d716-\U0001d734\U0001d736-\U0001d74e\U0001d750-\U0001d76e\U0001d770-\U0001d788\U0001d78a-\U0001d7a8\U0001d7aa-\U0001d7c2\U0001d7c4-\U0001d7cb\U0001e800-\U0001e8c4\U0001e900-\U0001e943\U0001ee00-\U0001ee03\U0001ee05-\U0001ee1f\U0001ee21-\U0001ee22\U0001ee24\U0001ee27\U0001ee29-\U0001ee32\U0001ee34-\U0001ee37\U0001ee39\U0001ee3b\U0001ee42\U0001ee47\U0001ee49\U0001ee4b\U0001ee4d-\U0001ee4f\U0001ee51-\U0001ee52\U0001ee54\U0001ee57\U0001ee59\U0001ee5b\U0001ee5d\U0001ee5f\U0001ee61-\U0001ee62\U0001ee64\U0001ee67-\U0001ee6a\U0001ee6c-\U0001ee72\U0001ee74-\U0001ee77\U0001ee79-\U0001ee7c\U0001ee7e\U0001ee80-\U0001ee89\U0001ee8b-\U0001ee9b\U0001eea1-\U0001eea3\U0001eea5-\U0001eea9\U0001eeab-\U0001eebb\U00020000-\U0002a6d6\U0002a700-\U0002b734\U0002b740-\U0002b81d\U0002b820-\U0002cea1\U0002ceb0-\U0002ebe0\U0002f800-\U0002fa1d' cats = ['Cc', 'Cf', 'Cn', 'Co', 'Cs', 'Ll', 'Lm', 'Lo', 'Lt', 'Lu', 'Mc', 'Me', 'Mn', 'Nd', 'Nl', 'No', 'Pc', 'Pd', 'Pe', 'Pf', 'Pi', 'Po', 'Ps', 'Sc', 'Sk', 'Sm', 'So', 'Zl', 'Zp', 'Zs'] -# Generated from unidata 6.3.0 +# Generated from unidata 11.0.0 def combine(*args): - return u''.join(globals()[cat] for cat in args) + return ''.join(globals()[cat] for cat in args) def allexcept(*args): newcats = cats[:] for arg in args: newcats.remove(arg) - return u''.join(globals()[cat] for cat in newcats) + return ''.join(globals()[cat] for cat in newcats) def _handle_runs(char_list): # pragma: no cover @@ -146,18 +106,13 @@ def _handle_runs(char_list): # pragma: no cover if a == b: yield a else: - yield u'%s-%s' % (a, b) + yield '%s-%s' % (a, b) if __name__ == '__main__': # pragma: no cover import unicodedata - # we need Py3 for the determination of the XID_* properties - if sys.version_info[:2] < (3, 3): - raise RuntimeError('this file must be regenerated with Python 3.3+') - - categories_bmp = {'xid_start': [], 'xid_continue': []} - categories_nonbmp = {'xid_start': [], 'xid_continue': []} + categories = {'xid_start': [], 'xid_continue': []} with open(__file__) as fp: content = fp.read() @@ -171,43 +126,26 @@ def _handle_runs(char_list): # pragma: no cover if ord(c) == 0xdc00: # Hack to avoid combining this combining with the preceeding high # surrogate, 0xdbff, when doing a repr. - c = u'\\' + c + c = '\\' + c elif ord(c) in (0x2d, 0x5b, 0x5c, 0x5d, 0x5e): # Escape regex metachars. - c = u'\\' + c - cat_dic = categories_bmp if code < 0x10000 else categories_nonbmp - cat_dic.setdefault(cat, []).append(c) + c = '\\' + c + categories.setdefault(cat, []).append(c) # XID_START and XID_CONTINUE are special categories used for matching # identifiers in Python 3. if c.isidentifier(): - cat_dic['xid_start'].append(c) + categories['xid_start'].append(c) if ('a' + c).isidentifier(): - cat_dic['xid_continue'].append(c) + categories['xid_continue'].append(c) with open(__file__, 'w') as fp: fp.write(header) - for cat in sorted(categories_bmp): - val = u''.join(_handle_runs(categories_bmp[cat])) - if cat == 'Cs': - # Jython can't handle isolated surrogates - fp.write("""\ -try: - Cs = eval(r"u%s") -except UnicodeDecodeError: - Cs = '' # Jython can't handle isolated surrogates\n\n""" % ascii(val)) - else: - fp.write('%s = u%a\n\n' % (cat, val)) - - fp.write('if sys.maxunicode > 0xFFFF:\n') - fp.write(' # non-BMP characters, use only on wide Unicode builds\n') - for cat in sorted(categories_nonbmp): - # no special case for Cs needed, since there are no surrogates - # in the higher planes - val = u''.join(_handle_runs(categories_nonbmp[cat])) - fp.write(' %s += u%a\n\n' % (cat, val)) + for cat in sorted(categories): + val = ''.join(_handle_runs(categories[cat])) + fp.write('%s = %a\n\n' % (cat, val)) - cats = sorted(categories_bmp) + cats = sorted(categories) cats.remove('xid_start') cats.remove('xid_continue') fp.write('cats = %r\n\n' % cats) diff --git a/src/syntax_highlighting/libs/pygments/util.py b/src/syntax_highlighting/libs/pygments/util.py index 4507006..5d6ddc3 100644 --- a/src/syntax_highlighting/libs/pygments/util.py +++ b/src/syntax_highlighting/libs/pygments/util.py @@ -1,21 +1,19 @@ -# -*- coding: utf-8 -*- """ pygments.util ~~~~~~~~~~~~~ Utility functions. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ import re -import sys +from io import TextIOWrapper split_path_re = re.compile(r'[/\\ ]') doctype_lookup_re = re.compile(r''' - (<\?.*?\?>)?\s* > 10), (0xdc00 + (c & 0x3ff))) -def unirange(a, b): - """Returns a regular expression string to match the given non-BMP range.""" - if b < a: - raise ValueError("Bad character range") - if a < 0x10000 or b < 0x10000: - raise ValueError("unirange is only defined for non-BMP ranges") - - if sys.maxunicode > 0xffff: - # wide build - return u'[%s-%s]' % (unichr(a), unichr(b)) - else: - # narrow build stores surrogates, and the 're' module handles them - # (incorrectly) as characters. Since there is still ordering among - # these characters, expand the range to one that it understands. Some - # background in http://bugs.python.org/issue3665 and - # http://bugs.python.org/issue12749 - # - # Additionally, the lower constants are using unichr rather than - # literals because jython [which uses the wide path] can't load this - # file if they are literals. - ah, al = _surrogatepair(a) - bh, bl = _surrogatepair(b) - if ah == bh: - return u'(?:%s[%s-%s])' % (unichr(ah), unichr(al), unichr(bl)) - else: - buf = [] - buf.append(u'%s[%s-%s]' % - (unichr(ah), unichr(al), - ah == bh and unichr(bl) or unichr(0xdfff))) - if ah - bh > 1: - buf.append(u'[%s-%s][%s-%s]' % - unichr(ah+1), unichr(bh-1), unichr(0xdc00), unichr(0xdfff)) - if ah != bh: - buf.append(u'%s[%s-%s]' % - (unichr(bh), unichr(0xdc00), unichr(bl))) - - return u'(?:' + u'|'.join(buf) + u')' - - def format_lines(var_name, seq, raw=False, indent_level=0): """Formats a sequence of strings for output.""" lines = [] @@ -289,7 +246,7 @@ def duplicates_removed(it, already_seen=()): return lst -class Future(object): +class Future: """Generic class to defer some work. Handled specially in RegexLexerMeta, to support regex string construction at @@ -345,44 +302,7 @@ def terminal_encoding(term): return locale.getpreferredencoding() -# Python 2/3 compatibility - -if sys.version_info < (3, 0): - unichr = unichr - xrange = xrange - string_types = (str, unicode) - text_type = unicode - u_prefix = 'u' - iteritems = dict.iteritems - itervalues = dict.itervalues - import StringIO - import cStringIO - # unfortunately, io.StringIO in Python 2 doesn't accept str at all - StringIO = StringIO.StringIO - BytesIO = cStringIO.StringIO -else: - unichr = chr - xrange = range - string_types = (str,) - text_type = str - u_prefix = '' - iteritems = dict.items - itervalues = dict.values - from io import StringIO, BytesIO, TextIOWrapper - - class UnclosingTextIOWrapper(TextIOWrapper): - # Don't close underlying buffer on destruction. - def close(self): - self.flush() - - -def add_metaclass(metaclass): - """Class decorator for creating a class with a metaclass.""" - def wrapper(cls): - orig_vars = cls.__dict__.copy() - orig_vars.pop('__dict__', None) - orig_vars.pop('__weakref__', None) - for slots_var in orig_vars.get('__slots__', ()): - orig_vars.pop(slots_var) - return metaclass(cls.__name__, cls.__bases__, orig_vars) - return wrapper +class UnclosingTextIOWrapper(TextIOWrapper): + # Don't close underlying buffer on destruction. + def close(self): + self.flush() diff --git a/src/syntax_highlighting/main.py b/src/syntax_highlighting/main.py index c4369ac..7ca4c21 100644 --- a/src/syntax_highlighting/main.py +++ b/src/syntax_highlighting/main.py @@ -53,7 +53,9 @@ # to show the user AND # The "language aliases": short, cryptic names for internal # use by HtmlFormatter -LANGUAGES_MAP = {lex[0]: lex[1][0] for lex in get_all_lexers()} +# filter out langauges without aliases +LEXERS = [item for item in get_all_lexers() if item[1]] +LANGUAGES_MAP = {lex[0]: lex[1][0] for lex in LEXERS} ERR_LEXER = ("Error: Selected language not found.
"