diff --git a/.github/workflows/test-docker.yml b/.github/workflows/test-docker.yml index df04d0a6cae..656df5e913c 100644 --- a/.github/workflows/test-docker.yml +++ b/.github/workflows/test-docker.yml @@ -19,7 +19,6 @@ jobs: amazon-2-amd64, arch, centos-7-amd64, - centos-8-amd64, centos-stream-8-amd64, debian-10-buster-x86, debian-11-bullseye-x86, diff --git a/.github/workflows/test-mingw.yml b/.github/workflows/test-mingw.yml index d94c7d53751..8a9c1725d9a 100644 --- a/.github/workflows/test-mingw.yml +++ b/.github/workflows/test-mingw.yml @@ -4,7 +4,7 @@ on: [push, pull_request, workflow_dispatch] jobs: build: - runs-on: windows-2019 + runs-on: windows-latest strategy: fail-fast: false matrix: diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml index c768838eb06..c78f9fd24ec 100644 --- a/.github/workflows/test-windows.yml +++ b/.github/workflows/test-windows.yml @@ -4,7 +4,7 @@ on: [push, pull_request, workflow_dispatch] jobs: build: - runs-on: windows-2019 + runs-on: windows-latest strategy: fail-fast: false matrix: diff --git a/CHANGES.rst b/CHANGES.rst index d6cd9d50f21..fc94556528a 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -5,6 +5,12 @@ Changelog (Pillow) 9.1.0 (unreleased) ------------------ +- Enable arm64 for MSVC on Windows #5811 + [gaborkertesz-linaro, gaborkertesz] + +- Keep IPython/Jupyter text/plain output stable #5891 + [shamrin, radarhere] + - Raise an error when performing a negative crop #5972 [radarhere, hugovk] @@ -26,6 +32,15 @@ Changelog (Pillow) - Remove readonly from Image.__eq__ #5930 [hugovk] +9.0.1 (2022-02-03) +------------------ + +- In show_file, use os.remove to remove temporary images. CVE-2022-24303 #6010 + [radarhere, hugovk] + +- Restrict builtins within lambdas for ImageMath.eval. CVE-2022-22817 #6009 + [radarhere] + 9.0.0 (2022-01-02) ------------------ diff --git a/Tests/helper.py b/Tests/helper.py index feccce6bcf9..08549aead58 100644 --- a/Tests/helper.py +++ b/Tests/helper.py @@ -27,8 +27,7 @@ class test_image_results: @staticmethod def upload(a, b): - a.show() - b.show() + return None elif "GITHUB_ACTIONS" in os.environ: HAS_UPLOADER = True diff --git a/Tests/test_file_eps.py b/Tests/test_file_eps.py index 4c0b96f7376..1790f4f7701 100644 --- a/Tests/test_file_eps.py +++ b/Tests/test_file_eps.py @@ -58,6 +58,15 @@ def test_sanity(): assert image2_scale2.format == "EPS" +@pytest.mark.skipif(not HAS_GHOSTSCRIPT, reason="Ghostscript not available") +def test_load(): + with Image.open(FILE1) as im: + assert im.load()[0, 0] == (255, 255, 255) + + # Test again now that it has already been loaded once + assert im.load()[0, 0] == (255, 255, 255) + + def test_invalid_file(): invalid_file = "Tests/images/flower.jpg" diff --git a/Tests/test_file_gbr.py b/Tests/test_file_gbr.py index 8d7fcf14779..1ea8af8ee34 100644 --- a/Tests/test_file_gbr.py +++ b/Tests/test_file_gbr.py @@ -5,20 +5,28 @@ from .helper import assert_image_equal_tofile -def test_invalid_file(): - invalid_file = "Tests/images/flower.jpg" - - with pytest.raises(SyntaxError): - GbrImagePlugin.GbrImageFile(invalid_file) - - def test_gbr_file(): with Image.open("Tests/images/gbr.gbr") as im: assert_image_equal_tofile(im, "Tests/images/gbr.png") +def test_load(): + with Image.open("Tests/images/gbr.gbr") as im: + assert im.load()[0, 0] == (0, 0, 0, 0) + + # Test again now that it has already been loaded once + assert im.load()[0, 0] == (0, 0, 0, 0) + + def test_multiple_load_operations(): with Image.open("Tests/images/gbr.gbr") as im: im.load() im.load() assert_image_equal_tofile(im, "Tests/images/gbr.png") + + +def test_invalid_file(): + invalid_file = "Tests/images/flower.jpg" + + with pytest.raises(SyntaxError): + GbrImagePlugin.GbrImageFile(invalid_file) diff --git a/Tests/test_file_icns.py b/Tests/test_file_icns.py index 3afbbeaac05..b492f6cb227 100644 --- a/Tests/test_file_icns.py +++ b/Tests/test_file_icns.py @@ -28,6 +28,14 @@ def test_sanity(): assert im.format == "ICNS" +def test_load(): + with Image.open(TEST_FILE) as im: + assert im.load()[0, 0] == (0, 0, 0, 0) + + # Test again now that it has already been loaded once + assert im.load()[0, 0] == (0, 0, 0, 0) + + def test_save(tmp_path): temp_file = str(tmp_path / "temp.icns") diff --git a/Tests/test_file_ico.py b/Tests/test_file_ico.py index 317264db646..73ac6f74286 100644 --- a/Tests/test_file_ico.py +++ b/Tests/test_file_ico.py @@ -18,6 +18,11 @@ def test_sanity(): assert im.get_format_mimetype() == "image/x-icon" +def test_load(): + with Image.open(TEST_ICO_FILE) as im: + assert im.load()[0, 0] == (1, 1, 9, 255) + + def test_mask(): with Image.open("Tests/images/hopper_mask.ico") as im: assert_image_equal_tofile(im, "Tests/images/hopper_mask.png") diff --git a/Tests/test_file_wal.py b/Tests/test_file_wal.py index f25b42fe0c4..4be46e9d673 100644 --- a/Tests/test_file_wal.py +++ b/Tests/test_file_wal.py @@ -2,15 +2,11 @@ from .helper import assert_image_equal_tofile +TEST_FILE = "Tests/images/hopper.wal" -def test_open(): - # Arrange - TEST_FILE = "Tests/images/hopper.wal" - # Act +def test_open(): with WalImageFile.open(TEST_FILE) as im: - - # Assert assert im.format == "WAL" assert im.format_description == "Quake2 Texture" assert im.mode == "P" @@ -19,3 +15,11 @@ def test_open(): assert isinstance(im, WalImageFile.WalImageFile) assert_image_equal_tofile(im, "Tests/images/hopper_wal.png") + + +def test_load(): + with WalImageFile.open(TEST_FILE) as im: + assert im.load()[0, 0] == 122 + + # Test again now that it has already been loaded once + assert im.load()[0, 0] == 122 diff --git a/Tests/test_file_wmf.py b/Tests/test_file_wmf.py index 3f8bc96ccdd..d6769a24b0b 100644 --- a/Tests/test_file_wmf.py +++ b/Tests/test_file_wmf.py @@ -24,6 +24,12 @@ def test_load_raw(): assert_image_similar_tofile(im, "Tests/images/drawing_wmf_ref.png", 2.0) +def test_load(): + with Image.open("Tests/images/drawing.emf") as im: + if hasattr(Image.core, "drawwmf"): + assert im.load()[0, 0] == (255, 255, 255) + + def test_register_handler(tmp_path): class TestHandler: methodCalled = False diff --git a/Tests/test_imagemath.py b/Tests/test_imagemath.py index 25811aa89d7..39d91eadea6 100644 --- a/Tests/test_imagemath.py +++ b/Tests/test_imagemath.py @@ -52,9 +52,17 @@ def test_ops(): assert pixel(ImageMath.eval("float(B)**33", images)) == "F 8589934592.0" -def test_prevent_exec(): +@pytest.mark.parametrize( + "expression", + ( + "exec('pass')", + "(lambda: exec('pass'))()", + "(lambda: (lambda: exec('pass'))())()", + ), +) +def test_prevent_exec(expression): with pytest.raises(ValueError): - ImageMath.eval("exec('pass')") + ImageMath.eval(expression) def test_logical(): diff --git a/Tests/test_imageshow.py b/Tests/test_imageshow.py index 02edfdfa1f7..03dc6a506c9 100644 --- a/Tests/test_imageshow.py +++ b/Tests/test_imageshow.py @@ -41,9 +41,11 @@ def show_image(self, image, **options): ImageShow._viewers.pop(0) -@pytest.mark.skipif( - not on_ci() or is_win32(), - reason="Only run on CIs; hangs on Windows CIs", +@pytest.mark.skip( + reason="""Due to implementation of Unix and Windows viewers", + a program or a test relying on the viewer will not terminate" + "till the image is explicitly closed" + """ ) def test_show(): for mode in ("1", "I;16", "LA", "RGB", "RGBA"): @@ -54,7 +56,7 @@ def test_show(): def test_viewer(): viewer = ImageShow.Viewer() - assert viewer.get_format(None) is None + assert viewer.get_format(None) == "PNG" with pytest.raises(NotImplementedError): viewer.get_command(None) @@ -85,11 +87,13 @@ def test_ipythonviewer(): not on_ci() or is_win32(), reason="Only run on CIs; hangs on Windows CIs", ) -def test_file_deprecated(): +def test_file_deprecated(tmp_path): + f = str(tmp_path / "temp.jpg") for viewer in ImageShow._viewers: + hopper().save(f) with pytest.warns(DeprecationWarning): try: - viewer.show_file(file="test.jpg") + viewer.show_file(file=f) except NotImplementedError: pass with pytest.raises(TypeError): diff --git a/Tests/test_numpy.py b/Tests/test_numpy.py index def7adf3f02..936474fe8d2 100644 --- a/Tests/test_numpy.py +++ b/Tests/test_numpy.py @@ -189,8 +189,9 @@ def test_putdata(): assert len(im.getdata()) == len(arr) -def test_roundtrip_eye(): - for dtype in ( +@pytest.mark.parametrize( + "dtype", + ( bool, numpy.bool8, numpy.int8, @@ -202,9 +203,11 @@ def test_roundtrip_eye(): float, numpy.float32, numpy.float64, - ): - arr = numpy.eye(10, dtype=dtype) - numpy.testing.assert_array_equal(arr, numpy.array(Image.fromarray(arr))) + ), +) +def test_roundtrip_eye(dtype): + arr = numpy.eye(10, dtype=dtype) + numpy.testing.assert_array_equal(arr, numpy.array(Image.fromarray(arr))) def test_zero_size(): diff --git a/depends/install_raqm.sh b/depends/install_raqm.sh index 39b998d0545..99250365065 100755 --- a/depends/install_raqm.sh +++ b/depends/install_raqm.sh @@ -2,7 +2,7 @@ # install raqm -archive=libraqm-0.8.0 +archive=libraqm-0.9.0 ./download-and-extract.sh $archive https://raw.githubusercontent.com/python-pillow/pillow-depends/main/$archive.tar.gz diff --git a/docs/installation.rst b/docs/installation.rst index ea0e7850010..984a689c2fb 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -169,7 +169,7 @@ Many of Pillow's features require external libraries: * **littlecms** provides color management * Pillow version 2.2.1 and below uses liblcms1, Pillow 2.3.0 and - above uses liblcms2. Tested with **1.19** and **2.7-2.12**. + above uses liblcms2. Tested with **1.19** and **2.7-2.13**. * **libwebp** provides the WebP format. @@ -453,8 +453,6 @@ These platforms are built and tested for every change. +----------------------------------+----------------------------+---------------------+ | CentOS 7 | 3.9 | x86-64 | +----------------------------------+----------------------------+---------------------+ -| CentOS 8 | 3.9 | x86-64 | -+----------------------------------+----------------------------+---------------------+ | CentOS Stream 8 | 3.9 | x86-64 | +----------------------------------+----------------------------+---------------------+ | Debian 10 Buster | 3.7 | x86 | @@ -530,6 +528,8 @@ These platforms have been reported to work at the versions mentioned. +----------------------------------+---------------------------+------------------+--------------+ | CentOS 6.3 | 2.7, 3.3 | |x86 | +----------------------------------+---------------------------+------------------+--------------+ +| CentOS 8 | 3.9 | 9.0.0 |x86-64 | ++----------------------------------+---------------------------+------------------+--------------+ | Fedora 23 | 2.7, 3.4 | 3.1.0 |x86-64 | +----------------------------------+---------------------------+------------------+--------------+ | Ubuntu Linux 12.04 LTS (Precise) | | 2.6, 3.2, 3.3, 3.4, 3.5 | 3.4.1 |x86,x86-64 | diff --git a/src/PIL/EpsImagePlugin.py b/src/PIL/EpsImagePlugin.py index 5d99202807e..b67363beb53 100644 --- a/src/PIL/EpsImagePlugin.py +++ b/src/PIL/EpsImagePlugin.py @@ -329,12 +329,12 @@ def _find_offset(self, fp): def load(self, scale=1, transparency=False): # Load EPS via Ghostscript - if not self.tile: - return - self.im = Ghostscript(self.tile, self.size, self.fp, scale, transparency) - self.mode = self.im.mode - self._size = self.im.size - self.tile = [] + if self.tile: + self.im = Ghostscript(self.tile, self.size, self.fp, scale, transparency) + self.mode = self.im.mode + self._size = self.im.size + self.tile = [] + return Image.Image.load(self) def load_seek(self, *args, **kwargs): # we can't incrementally load, so force ImageFile.parser to diff --git a/src/PIL/GbrImagePlugin.py b/src/PIL/GbrImagePlugin.py index 0f230602db8..3d8fc47b224 100644 --- a/src/PIL/GbrImagePlugin.py +++ b/src/PIL/GbrImagePlugin.py @@ -84,12 +84,10 @@ def _open(self): self._data_size = width * height * color_depth def load(self): - if self.im: - # Already loaded - return - - self.im = Image.core.new(self.mode, self.size) - self.frombytes(self.fp.read(self._data_size)) + if not self.im: + self.im = Image.core.new(self.mode, self.size) + self.frombytes(self.fp.read(self._data_size)) + return Image.Image.load(self) # diff --git a/src/PIL/IcnsImagePlugin.py b/src/PIL/IcnsImagePlugin.py index 6412d1cfb4b..069aff96b63 100644 --- a/src/PIL/IcnsImagePlugin.py +++ b/src/PIL/IcnsImagePlugin.py @@ -286,21 +286,22 @@ def load(self): self.best_size[1] * self.best_size[2], ) - Image.Image.load(self) + px = Image.Image.load(self) if self.im and self.im.size == self.size: # Already loaded - return + return px self.load_prepare() # This is likely NOT the best way to do it, but whatever. im = self.icns.getimage(self.best_size) # If this is a PNG or JPEG 2000, it won't be loaded yet - im.load() + px = im.load() self.im = im.im self.mode = im.mode self.size = im.size - self.load_end() + + return px def _save(im, fp, filename): diff --git a/src/PIL/IcoImagePlugin.py b/src/PIL/IcoImagePlugin.py index d9ff9b5e731..82837f3073f 100644 --- a/src/PIL/IcoImagePlugin.py +++ b/src/PIL/IcoImagePlugin.py @@ -306,7 +306,7 @@ def size(self, value): def load(self): if self.im and self.im.size == self.size: # Already loaded - return + return Image.Image.load(self) im = self.ico.getimage(self.size) # if tile is PNG, it won't really be loaded yet im.load() diff --git a/src/PIL/Image.py b/src/PIL/Image.py index 02b71e612a0..0736a58bd22 100644 --- a/src/PIL/Image.py +++ b/src/PIL/Image.py @@ -2269,14 +2269,16 @@ def show(self, title=None): The image is first saved to a temporary file. By default, it will be in PNG format. - On Unix, the image is then opened using the **display**, **eog** or - **xv** utility, depending on which one can be found. + On Unix, the image is then opened using the **display**, **gm**, **eog**, + or **xv** utility. Finally, if all the viewers failed, the attempt is made + to open the image file using **xdg-open** command, + this command uses the default system image viewer. On macOS, the image is opened with the native Preview application. On Windows, the image is opened with the standard PNG display utility. - :param title: Optional title to use for the image window, where possible. + :param title: has been deprecated """ _show(self, title=title) diff --git a/src/PIL/ImageFile.py b/src/PIL/ImageFile.py index 3374a5b1dae..331410f0e1a 100644 --- a/src/PIL/ImageFile.py +++ b/src/PIL/ImageFile.py @@ -328,6 +328,7 @@ def load(self): # become the other object (!) self.__class__ = image.__class__ self.__dict__ = image.__dict__ + return image.load() def _load(self): """(Hook) Find actual image loader.""" diff --git a/src/PIL/ImageMath.py b/src/PIL/ImageMath.py index 4b6e4ccda3a..09d9898d750 100644 --- a/src/PIL/ImageMath.py +++ b/src/PIL/ImageMath.py @@ -240,11 +240,18 @@ def eval(expression, _dict={}, **kw): if hasattr(v, "im"): args[k] = _Operand(v) - code = compile(expression, "", "eval") - for name in code.co_names: - if name not in args and name != "abs": - raise ValueError(f"'{name}' not allowed") + compiled_code = compile(expression, "", "eval") + def scan(code): + for const in code.co_consts: + if type(const) == type(compiled_code): + scan(const) + + for name in code.co_names: + if name not in args and name != "abs": + raise ValueError(f"'{name}' not allowed") + + scan(compiled_code) out = builtins.eval(expression, {"__builtins": {"abs": abs}}, args) try: return out.im diff --git a/src/PIL/ImageShow.py b/src/PIL/ImageShow.py index 2165da30732..62021f44fa8 100644 --- a/src/PIL/ImageShow.py +++ b/src/PIL/ImageShow.py @@ -15,7 +15,7 @@ import shutil import subprocess import sys -import tempfile +import threading import warnings from shlex import quote @@ -62,6 +62,9 @@ def show(image, title=None, **options): class Viewer: """Base class for viewers.""" + format = "PNG" + options = {"compress_level": 1} + # main api def show(self, image, **options): @@ -82,11 +85,6 @@ def show(self, image, **options): # hook methods - format = None - """The format to convert the image into.""" - options = {} - """Additional options used to convert the image.""" - def get_format(self, image): """Return format name, or ``None`` to save as PGM/PPM.""" return self.format @@ -128,15 +126,9 @@ def show_file(self, path=None, **options): return 1 -# -------------------------------------------------------------------- - - class WindowsViewer(Viewer): """The default viewer on Windows is the default system application for PNG files.""" - format = "PNG" - options = {"compress_level": 1} - def get_command(self, file, **options): return ( f'start "Pillow" /WAIT "{file}" ' @@ -152,9 +144,6 @@ def get_command(self, file, **options): class MacViewer(Viewer): """The default viewer on macOS using ``Preview.app``.""" - format = "PNG" - options = {"compress_level": 1} - def get_command(self, file, **options): # on darwin open returns immediately resulting in the temp # file removal while app is opening @@ -180,16 +169,8 @@ def show_file(self, path=None, **options): path = options.pop("file") else: raise TypeError("Missing required argument: 'path'") - fd, temp_path = tempfile.mkstemp() - with os.fdopen(fd, "w") as f: - f.write(path) - with open(temp_path) as f: - subprocess.Popen( - ["im=$(cat); open -a Preview.app $im; sleep 20; rm -f $im"], - shell=True, - stdin=f, - ) - os.remove(temp_path) + subprocess.call(["open", "-a", "Preview.app", path]) + self._remove_path_after_delay(path) return 1 @@ -198,12 +179,32 @@ def show_file(self, path=None, **options): class UnixViewer(Viewer): - format = "PNG" - options = {"compress_level": 1} + opened_images = [] def get_command(self, file, **options): - command = self.get_command_ex(file, **options)[0] - return f"({command} {quote(file)}; rm -f {quote(file)})&" + return self.get_command_ex(file, **options)[0] + + def get_executable(self, file, **options): + return self.get_command_ex(file, **options)[1] + + def thread_monitor(self, th): + """ + Monitors image viewing threads. + The last remaining monitoring thread is responsible + for removal of temporary images. + """ + th.join() + if threading.active_count() == 2: + for f in self.opened_images: + if os.path.isfile(f): + try: + os.remove(f) + except OSError as e: + print(f"failed to delete the file: {f}") + print(e) + return 1 + return 0 + def show_file(self, path=None, **options): """ @@ -223,15 +224,24 @@ def show_file(self, path=None, **options): path = options.pop("file") else: raise TypeError("Missing required argument: 'path'") - fd, temp_path = tempfile.mkstemp() - with os.fdopen(fd, "w") as f: - f.write(path) - with open(temp_path) as f: - command = self.get_command_ex(path, **options)[0] - subprocess.Popen( - ["im=$(cat);" + command + " $im; rm -f $im"], shell=True, stdin=f - ) - os.remove(temp_path) + + path = quote(path) + command = self.get_command(path, **options) + th = threading.Thread( + target=subprocess.run, + args=(command.split(),), + kwargs={ + "stdout": subprocess.PIPE, + "stderr": subprocess.PIPE, + }, + name=path, + ) + self.opened_images.append(th.name) + th.start() + + th_monitor = threading.Thread(target=self.thread_monitor, args=(th,)) + th_monitor.start() + return 1 @@ -241,20 +251,19 @@ class XDGViewer(UnixViewer): """ def get_command_ex(self, file, **options): - command = executable = "xdg-open" + executable = "xdg-open" + command = f"xdg-open {quote(file)}" return command, executable class DisplayViewer(UnixViewer): """ The ImageMagick ``display`` command. - This viewer supports the ``title`` parameter. """ - def get_command_ex(self, file, title=None, **options): - command = executable = "display" - if title: - command += f" -name {quote(title)}" + def get_command_ex(self, file, **options): + executable = "display" + command = f"display {quote(file)}" return command, executable @@ -263,7 +272,7 @@ class GmDisplayViewer(UnixViewer): def get_command_ex(self, file, **options): executable = "gm" - command = "gm display" + command = f"gm display {quote(file)}" return command, executable @@ -272,28 +281,24 @@ class EogViewer(UnixViewer): def get_command_ex(self, file, **options): executable = "eog" - command = "eog -n" + command = f"eog -n {quote(file)}" return command, executable class XVViewer(UnixViewer): """ The X Viewer ``xv`` command. - This viewer supports the ``title`` parameter. """ - def get_command_ex(self, file, title=None, **options): + def get_command_ex(self, file, **options): # note: xv is pretty outdated. most modern systems have # imagemagick's display command instead. - command = executable = "xv" - if title: - command += f" -name {quote(title)}" + executable = "xv" + command = f"xv {quote(file)}" return command, executable if sys.platform not in ("win32", "darwin"): # unixoids - if shutil.which("xdg-open"): - register(XDGViewer) if shutil.which("display"): register(DisplayViewer) if shutil.which("gm"): @@ -302,6 +307,8 @@ def get_command_ex(self, file, title=None, **options): register(EogViewer) if shutil.which("xv"): register(XVViewer) + if shutil.which("xdg-open"): + register(XDGViewer) class IPythonViewer(Viewer): diff --git a/src/PIL/WalImageFile.py b/src/PIL/WalImageFile.py index 1354ad32b52..0dc695a88d4 100644 --- a/src/PIL/WalImageFile.py +++ b/src/PIL/WalImageFile.py @@ -51,14 +51,11 @@ def _open(self): self.info["next_name"] = next_name def load(self): - if self.im: - # Already loaded - return - - self.im = Image.core.new(self.mode, self.size) - self.frombytes(self.fp.read(self.size[0] * self.size[1])) - self.putpalette(quake2palette) - Image.Image.load(self) + if not self.im: + self.im = Image.core.new(self.mode, self.size) + self.frombytes(self.fp.read(self.size[0] * self.size[1])) + self.putpalette(quake2palette) + return Image.Image.load(self) def open(filename): diff --git a/src/PIL/WmfImagePlugin.py b/src/PIL/WmfImagePlugin.py index 27f5d2f870c..c32cc52f8d0 100644 --- a/src/PIL/WmfImagePlugin.py +++ b/src/PIL/WmfImagePlugin.py @@ -158,7 +158,7 @@ def load(self, dpi=None): (x1 - x0) * self.info["dpi"] // self._inch, (y1 - y0) * self.info["dpi"] // self._inch, ) - super().load() + return super().load() def _save(im, fp, filename): diff --git a/src/thirdparty/raqm/COPYING b/src/thirdparty/raqm/COPYING index 196511ef688..c605a5dc67a 100644 --- a/src/thirdparty/raqm/COPYING +++ b/src/thirdparty/raqm/COPYING @@ -1,7 +1,7 @@ The MIT License (MIT) Copyright © 2015 Information Technology Authority (ITA) -Copyright © 2016 Khaled Hosny +Copyright © 2016-2022 Khaled Hosny Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/src/thirdparty/raqm/NEWS b/src/thirdparty/raqm/NEWS index c49176a95bf..ae1128485f2 100644 --- a/src/thirdparty/raqm/NEWS +++ b/src/thirdparty/raqm/NEWS @@ -1,4 +1,18 @@ -Overview of changes leading to 0.7.1 +Overview of changes leading to 0.8.0 +Monday, December 13, 2021 +==================================== + +Remove autotools build. + +Support using SheenBiDi instead of FriBiDi for Unicode BiDi support. + +Fix running tests with Python <= 3.6. + +New API: + * raqm_get_par_resolved_direction + * raqm_get_direction_at_index + +Overview of changes leading to 0.7.2 Monday, September 27, 2021 ==================================== diff --git a/src/thirdparty/raqm/README.md b/src/thirdparty/raqm/README.md index 64937343a6f..02e996e7a9c 100644 --- a/src/thirdparty/raqm/README.md +++ b/src/thirdparty/raqm/README.md @@ -6,26 +6,26 @@ Raqm Raqm is a small library that encapsulates the logic for complex text layout and provides a convenient API. -It currently provides bidirectional text support (using [FriBiDi][1]), shaping -(using [HarfBuzz][2]), and proper script itemization. As a result, -Raqm can support most writing systems covered by Unicode. +It currently provides bidirectional text support (using [FriBiDi][1] or +[SheenBidi][2]), shaping (using [HarfBuzz][3]), and proper script itemization. +As a result, Raqm can support most writing systems covered by Unicode. The documentation can be accessed on the web at: > http://host-oman.github.io/libraqm/ Raqm (Arabic: رَقْم) is writing, also number or digit and the Arabic word for -digital (رَقَمِيّ) shares the same root, so it is a play on “digital writing”. +digital (رَقَمِيّ) shares the same root, so it is a play on “digital writing”. Building -------- Raqm depends on the following libraries: -* [FreeType][3] -* [HarfBuzz][2] -* [FriBiDi][1] +* [FreeType][4] +* [HarfBuzz][3] +* [FriBiDi][1] or [SheenBidi][2] To build the documentation you will also need: -* [GTK-Doc][4] +* [GTK-Doc][5] To install dependencies on Fedora: @@ -48,11 +48,11 @@ directory: $ ninja -C build $ ninja -C build install -To build the documentation, pass `-Ddocs=enable` to the `meson`. +To build the documentation, pass `-Ddocs=true` to the `meson`. To run the tests: - $ ninja -C test + $ ninja -C build test Contributing ------------ @@ -68,6 +68,7 @@ Projects using Raqm 3. [FontView](https://github.com/googlei18n/fontview) 4. [Pillow](https://github.com/python-pillow) 5. [mplcairo](https://github.com/anntzer/mplcairo) +6. [CEGUI](https://github.com/cegui/cegui) The following projects have patches to support complex text layout using Raqm: @@ -77,7 +78,8 @@ The following projects have patches to support complex text layout using Raqm: -[1]: http://fribidi.org -[2]: http://harfbuzz.org -[3]: https://www.freetype.org -[4]: https://www.gtk.org/gtk-doc +[1]: https://github.com/fribidi/fribidi +[2]: https://github.com/Tehreer/SheenBidi +[3]: https://github.com/harfbuzz/harfbuzz +[4]: https://www.freetype.org +[5]: https://www.gtk.org/gtk-doc diff --git a/src/thirdparty/raqm/raqm-version.h b/src/thirdparty/raqm/raqm-version.h index 8b115f612c6..78b70a5615e 100644 --- a/src/thirdparty/raqm/raqm-version.h +++ b/src/thirdparty/raqm/raqm-version.h @@ -32,10 +32,10 @@ #define _RAQM_VERSION_H_ #define RAQM_VERSION_MAJOR 0 -#define RAQM_VERSION_MINOR 7 -#define RAQM_VERSION_MICRO 2 +#define RAQM_VERSION_MINOR 9 +#define RAQM_VERSION_MICRO 0 -#define RAQM_VERSION_STRING "0.7.2" +#define RAQM_VERSION_STRING "0.9.0" #define RAQM_VERSION_ATLEAST(major,minor,micro) \ ((major)*10000+(minor)*100+(micro) <= \ diff --git a/src/thirdparty/raqm/raqm.c b/src/thirdparty/raqm/raqm.c index 31161c9d91d..13f6e1f023c 100644 --- a/src/thirdparty/raqm/raqm.c +++ b/src/thirdparty/raqm/raqm.c @@ -1,6 +1,6 @@ /* * Copyright © 2015 Information Technology Authority (ITA) - * Copyright © 2016 Khaled Hosny + * Copyright © 2016-2022 Khaled Hosny * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to @@ -24,42 +24,26 @@ #ifdef HAVE_CONFIG_H #include "config.h" -#undef HAVE_CONFIG_H // Workaround for Fribidi 1.0.5 and earlier #endif #include #include +#ifdef RAQM_SHEENBIDI +#include +#else #ifdef HAVE_FRIBIDI_SYSTEM #include #else #include "../fribidi-shim/fribidi.h" #endif +#endif #include #include -#if FREETYPE_MAJOR > 2 || \ - FREETYPE_MAJOR == 2 && FREETYPE_MINOR >= 11 -#define HAVE_FT_GET_TRANSFORM -#endif - -#if HB_VERSION_ATLEAST(2, 0, 0) -#define HAVE_HB_BUFFER_SET_INVISIBLE_GLYPH -#endif - -#if HB_VERSION_ATLEAST(1, 8, 0) -#define HAVE_DECL_HB_BUFFER_FLAG_REMOVE_DEFAULT_IGNORABLES 1 -#else -#define HAVE_DECL_HB_BUFFER_FLAG_REMOVE_DEFAULT_IGNORABLES 0 -#endif - #include "raqm.h" -#if FRIBIDI_MAJOR_VERSION >= 1 -#define USE_FRIBIDI_EX_API -#endif - /** * SECTION:raqm * @title: Raqm @@ -178,13 +162,18 @@ # define RAQM_TEST(...) #endif -typedef enum { - RAQM_FLAG_NONE = 0, - RAQM_FLAG_UTF8 = 1 << 0 -} _raqm_flags_t; +#define RAQM_BIDI_LEVEL_IS_RTL(level) \ + ((level) & 1) + +#ifdef RAQM_SHEENBIDI + typedef SBLevel _raqm_bidi_level_t; +#else + typedef FriBidiLevel _raqm_bidi_level_t; +#endif typedef struct { FT_Face ftface; + int ftloadflags; hb_language_t lang; hb_script_t script; } _raqm_text_info; @@ -197,6 +186,7 @@ struct _raqm { uint32_t *text; char *text_utf8; size_t text_len; + size_t text_capacity_bytes; _raqm_text_info *text_info; @@ -207,17 +197,17 @@ struct _raqm { size_t features_len; raqm_run_t *runs; - raqm_glyph_t *glyphs; + raqm_run_t *runs_pool; - _raqm_flags_t flags; + raqm_glyph_t *glyphs; + size_t glyphs_capacity; - int ft_loadflags; int invisible_glyph; }; struct _raqm_run { - int pos; - int len; + uint32_t pos; + uint32_t len; hb_direction_t direction; hb_script_t script; @@ -231,31 +221,21 @@ static uint32_t _raqm_u8_to_u32_index (raqm_t *rq, uint32_t index); -static bool +static void _raqm_init_text_info (raqm_t *rq) { - hb_language_t default_lang; - - if (rq->text_info) - return true; - - rq->text_info = malloc (sizeof (_raqm_text_info) * rq->text_len); - if (!rq->text_info) - return false; - - default_lang = hb_language_get_default (); + hb_language_t default_lang = hb_language_get_default (); for (size_t i = 0; i < rq->text_len; i++) { rq->text_info[i].ftface = NULL; + rq->text_info[i].ftloadflags = -1; rq->text_info[i].lang = default_lang; rq->text_info[i].script = HB_SCRIPT_INVALID; } - - return true; } static void -_raqm_free_text_info (raqm_t *rq) +_raqm_release_text_info (raqm_t *rq) { if (!rq->text_info) return; @@ -265,9 +245,6 @@ _raqm_free_text_info (raqm_t *rq) if (rq->text_info[i].ftface) FT_Done_Face (rq->text_info[i].ftface); } - - free (rq->text_info); - rq->text_info = NULL; } static bool @@ -277,6 +254,9 @@ _raqm_compare_text_info (_raqm_text_info a, if (a.ftface != b.ftface) return false; + if (a.ftloadflags != b.ftloadflags) + return false; + if (a.lang != b.lang) return false; @@ -286,6 +266,88 @@ _raqm_compare_text_info (_raqm_text_info a, return true; } +static void +_raqm_free_text(raqm_t* rq) +{ + free (rq->text); + rq->text = NULL; + rq->text_info = NULL; + rq->text_utf8 = NULL; + rq->text_len = 0; + rq->text_capacity_bytes = 0; +} + +static bool +_raqm_alloc_text(raqm_t *rq, + size_t len, + bool need_utf8) +{ + /* Allocate contiguous memory block for texts and text_info */ + size_t mem_size = (sizeof (uint32_t) + sizeof (_raqm_text_info)) * len; + if (need_utf8) + mem_size += sizeof (char) * len; + + if (mem_size > rq->text_capacity_bytes) + { + void* new_mem = realloc (rq->text, mem_size); + if (!new_mem) + { + _raqm_free_text (rq); + return false; + } + + rq->text_capacity_bytes = mem_size; + rq->text = new_mem; + } + + rq->text_info = (_raqm_text_info*)(rq->text + len); + rq->text_utf8 = need_utf8 ? (char*)(rq->text_info + len) : NULL; + + return true; +} + +static raqm_run_t* +_raqm_alloc_run (raqm_t *rq) +{ + raqm_run_t *run = rq->runs_pool; + if (run) + { + rq->runs_pool = run->next; + } + else + { + run = malloc (sizeof (raqm_run_t)); + run->font = NULL; + run->buffer = NULL; + } + + run->pos = 0; + run->len = 0; + run->direction = HB_DIRECTION_INVALID; + run->script = HB_SCRIPT_INVALID; + run->next = NULL; + + return run; +} + +static void +_raqm_free_runs (raqm_run_t *runs) +{ + while (runs) + { + raqm_run_t *run = runs; + runs = runs->next; + + if (run->buffer) + hb_buffer_destroy (run->buffer); + + if (run->font) + hb_font_destroy (run->font); + + free (run); + } +} + /** * raqm_create: * @@ -310,25 +372,25 @@ raqm_create (void) rq->ref_count = 1; - rq->text = NULL; - rq->text_utf8 = NULL; - rq->text_len = 0; - - rq->text_info = NULL; - rq->base_dir = RAQM_DIRECTION_DEFAULT; rq->resolved_dir = RAQM_DIRECTION_DEFAULT; rq->features = NULL; rq->features_len = 0; - rq->runs = NULL; - rq->glyphs = NULL; + rq->invisible_glyph = 0; - rq->flags = RAQM_FLAG_NONE; + rq->text = NULL; + rq->text_utf8 = NULL; + rq->text_info = NULL; + rq->text_capacity_bytes = 0; + rq->text_len = 0; - rq->ft_loadflags = -1; - rq->invisible_glyph = 0; + rq->runs = NULL; + rq->runs_pool = NULL; + + rq->glyphs = NULL; + rq->glyphs_capacity = 0; return rq; } @@ -354,28 +416,13 @@ raqm_reference (raqm_t *rq) return rq; } -static void -_raqm_free_runs (raqm_t *rq) -{ - raqm_run_t *runs = rq->runs; - while (runs) - { - raqm_run_t *run = runs; - runs = runs->next; - - hb_buffer_destroy (run->buffer); - hb_font_destroy (run->font); - free (run); - } -} - /** * raqm_destroy: * @rq: a #raqm_t. * * Decreases the reference count on @rq by one. If the result is zero, then @rq * and all associated resources are freed. - * See cairo_reference(). + * See raqm_reference(). * * Since: 0.1 */ @@ -385,14 +432,60 @@ raqm_destroy (raqm_t *rq) if (!rq || --rq->ref_count != 0) return; - free (rq->text); - free (rq->text_utf8); - _raqm_free_text_info (rq); - _raqm_free_runs (rq); + _raqm_release_text_info (rq); + _raqm_free_text (rq); + _raqm_free_runs (rq->runs); + _raqm_free_runs (rq->runs_pool); free (rq->glyphs); + free (rq->features); free (rq); } +/** + * raqm_clear_contents: + * @rq: a #raqm_t. + * + * Clears internal state of previously used raqm_t object, making it ready + * for reuse and keeping some of allocated memory to increase performance. + * + * Since: 0.9 + */ +void +raqm_clear_contents (raqm_t *rq) +{ + if (!rq) + return; + + _raqm_release_text_info (rq); + + /* Return allocated runs to the pool, keep hb buffers for reuse */ + raqm_run_t *run = rq->runs; + while (run) + { + if (run->buffer) + hb_buffer_reset (run->buffer); + + if (run->font) + { + hb_font_destroy (run->font); + run->font = NULL; + } + + if (!run->next) + { + run->next = rq->runs_pool; + rq->runs_pool = rq->runs; + rq->runs = NULL; + break; + } + + run = run->next; + } + + rq->text_len = 0; + rq->resolved_dir = RAQM_DIRECTION_DEFAULT; +} + /** * raqm_set_text: * @rq: a #raqm_t. @@ -417,25 +510,69 @@ raqm_set_text (raqm_t *rq, if (!rq || !text) return false; - rq->text_len = len; + /* Call raqm_clear_contents to reuse this raqm_t */ + if (rq->text_len) + return false; /* Empty string, don’t fail but do nothing */ if (!len) return true; - free (rq->text); + if (!_raqm_alloc_text(rq, len, false)) + return false; - rq->text = malloc (sizeof (uint32_t) * rq->text_len); - if (!rq->text) - return false; + rq->text_len = len; + memcpy (rq->text, text, sizeof (uint32_t) * len); + _raqm_init_text_info (rq); - _raqm_free_text_info (rq); - if (!_raqm_init_text_info (rq)) - return false; + return true; +} + +static void * +_raqm_get_utf8_codepoint (const void *str, + uint32_t *out_codepoint) +{ + const char *s = (const char *)str; - memcpy (rq->text, text, sizeof (uint32_t) * rq->text_len); + if (0xf0 == (0xf8 & s[0])) + { + *out_codepoint = ((0x07 & s[0]) << 18) | ((0x3f & s[1]) << 12) | ((0x3f & s[2]) << 6) | (0x3f & s[3]); + s += 4; + } + else if (0xe0 == (0xf0 & s[0])) + { + *out_codepoint = ((0x0f & s[0]) << 12) | ((0x3f & s[1]) << 6) | (0x3f & s[2]); + s += 3; + } + else if (0xc0 == (0xe0 & s[0])) + { + *out_codepoint = ((0x1f & s[0]) << 6) | (0x3f & s[1]); + s += 2; + } + else + { + *out_codepoint = s[0]; + s += 1; + } - return true; + return (void *)s; +} + +static size_t +_raqm_u8_to_u32 (const char *text, size_t len, uint32_t *unicode) +{ + size_t in_len = 0; + uint32_t *out_utf32 = unicode; + const char *in_utf8 = text; + + while ((*in_utf8 != '\0') && (in_len < len)) + { + in_utf8 = _raqm_get_utf8_codepoint (in_utf8, out_utf32); + ++out_utf32; + ++in_len; + } + + return (out_utf32 - unicode); } /** @@ -452,43 +589,29 @@ raqm_set_text (raqm_t *rq, * Since: 0.1 */ bool -raqm_set_text_utf8 (raqm_t *rq, - const char *text, - size_t len) +raqm_set_text_utf8 (raqm_t *rq, + const char *text, + size_t len) { - uint32_t *unicode; - size_t ulen; - bool ok; - if (!rq || !text) return false; + /* Call raqm_clear_contents to reuse this raqm_t */ + if (rq->text_len) + return false; + /* Empty string, don’t fail but do nothing */ if (!len) - { - rq->text_len = len; return true; - } - - rq->flags |= RAQM_FLAG_UTF8; - rq->text_utf8 = malloc (sizeof (char) * len); - if (!rq->text_utf8) - return false; - - unicode = malloc (sizeof (uint32_t) * len); - if (!unicode) - return false; + if (!_raqm_alloc_text(rq, len, true)) + return false; + rq->text_len = _raqm_u8_to_u32 (text, len, rq->text); memcpy (rq->text_utf8, text, sizeof (char) * len); + _raqm_init_text_info (rq); - ulen = fribidi_charset_to_unicode (FRIBIDI_CHAR_SET_UTF8, - text, len, unicode); - - ok = raqm_set_text (rq, unicode, ulen); - - free (unicode); - return ok; + return true; } /** @@ -572,7 +695,7 @@ raqm_set_language (raqm_t *rq, if (!rq->text_len) return true; - if (rq->flags & RAQM_FLAG_UTF8) + if (rq->text_utf8) { start = _raqm_u8_to_u32_index (rq, start); end = _raqm_u8_to_u32_index (rq, end); @@ -629,13 +752,14 @@ raqm_add_font_feature (raqm_t *rq, ok = hb_feature_from_string (feature, len, &fea); if (ok) { - rq->features_len++; - rq->features = realloc (rq->features, - sizeof (hb_feature_t) * (rq->features_len)); - if (!rq->features) + void* new_features = realloc (rq->features, + sizeof (hb_feature_t) * (rq->features_len + 1)); + if (!new_features) return false; - rq->features[rq->features_len - 1] = fea; + rq->features = new_features; + rq->features[rq->features_len] = fea; + rq->features_len++; } return ok; @@ -643,12 +767,13 @@ raqm_add_font_feature (raqm_t *rq, static hb_font_t * _raqm_create_hb_font (raqm_t *rq, - FT_Face face) + FT_Face face, + int loadflags) { hb_font_t *font = hb_ft_font_create_referenced (face); - if (rq->ft_loadflags >= 0) - hb_ft_font_set_load_flags (font, rq->ft_loadflags); + if (loadflags >= 0) + hb_ft_font_set_load_flags (font, loadflags); return font; } @@ -739,7 +864,7 @@ raqm_set_freetype_face_range (raqm_t *rq, if (!rq->text_len) return true; - if (rq->flags & RAQM_FLAG_UTF8) + if (rq->text_utf8) { start = _raqm_u8_to_u32_index (rq, start); end = _raqm_u8_to_u32_index (rq, end); @@ -748,6 +873,30 @@ raqm_set_freetype_face_range (raqm_t *rq, return _raqm_set_freetype_face (rq, face, start, end); } +static bool +_raqm_set_freetype_load_flags (raqm_t *rq, + int flags, + size_t start, + size_t end) +{ + if (!rq) + return false; + + if (!rq->text_len) + return true; + + if (start >= rq->text_len || end > rq->text_len) + return false; + + if (!rq->text_info) + return false; + + for (size_t i = start; i < end; i++) + rq->text_info[i].ftloadflags = flags; + + return true; +} + /** * raqm_set_freetype_load_flags: * @rq: a #raqm_t. @@ -766,14 +915,59 @@ raqm_set_freetype_face_range (raqm_t *rq, */ bool raqm_set_freetype_load_flags (raqm_t *rq, - int flags) + int flags) { + return _raqm_set_freetype_load_flags(rq, flags, 0, rq->text_len); +} + +/** + * raqm_set_freetype_load_flags_range: + * @rq: a #raqm_t. + * @flags: FreeType load flags. + * @start: index of first character that should use @flags. + * @len: number of characters using @flags. + * + * Sets the load flags passed to FreeType when loading glyphs for @len-number + * of characters staring at @start. Flags should be the same as used by the + * client when rendering corresponding FreeType glyphs. The @start and @len + * are input string array indices (i.e. counting bytes in UTF-8 and scaler + * values in UTF-32). + * + * This method can be used repeatedly to set different flags for different + * parts of the text. It is the responsibility of the client to make sure that + * flag ranges cover the whole text. + * + * This requires version of HarfBuzz that has hb_ft_font_set_load_flags(), for + * older version the flags will be ignored. + * + * See also raqm_set_freetype_load_flags(). + * + * Return value: + * %true if no errors happened, %false otherwise. + * + * Since: 0.9 + */ +bool +raqm_set_freetype_load_flags_range (raqm_t *rq, + int flags, + size_t start, + size_t len) +{ + size_t end = start + len; + if (!rq) return false; - rq->ft_loadflags = flags; + if (!rq->text_len) + return true; - return true; + if (rq->text_utf8) + { + start = _raqm_u8_to_u32_index (rq, start); + end = _raqm_u8_to_u32_index (rq, end); + } + + return _raqm_set_freetype_load_flags (rq, flags, start, end); } /** @@ -784,17 +978,10 @@ raqm_set_freetype_load_flags (raqm_t *rq, * Sets the glyph id to be used for invisible glyhphs. * * If @gid is negative, invisible glyphs will be suppressed from the output. - * This requires HarfBuzz 1.8.0 or later. If raqm is used with an earlier - * HarfBuzz version, the return value will be %false and the shaping behavior - * does not change. * * If @gid is zero, invisible glyphs will be rendered as space. - * This works on all versions of HarfBuzz. * * If @gid is a positive number, it will be used for invisible glyphs. - * This requires a version of HarfBuzz that has - * hb_buffer_set_invisible_glyph(). For older versions, the return value - * will be %false and the shaping behavior does not change. * * Return value: * %true if no errors happened, %false otherwise. @@ -808,17 +995,6 @@ raqm_set_invisible_glyph (raqm_t *rq, if (!rq) return false; -#ifndef HAVE_HB_BUFFER_SET_INVISIBLE_GLYPH - if (gid > 0) - return false; -#endif - -#if !defined(HAVE_DECL_HB_BUFFER_FLAG_REMOVE_DEFAULT_IGNORABLES) || \ - !HAVE_DECL_HB_BUFFER_FLAG_REMOVE_DEFAULT_IGNORABLES - if (gid < 0) - return false; -#endif - rq->invisible_glyph = gid; return true; } @@ -904,18 +1080,21 @@ raqm_get_glyphs (raqm_t *rq, for (raqm_run_t *run = rq->runs; run != NULL; run = run->next) count += hb_buffer_get_length (run->buffer); - *length = count; - - if (rq->glyphs) - free (rq->glyphs); - - rq->glyphs = malloc (sizeof (raqm_glyph_t) * count); - if (!rq->glyphs) + if (count > rq->glyphs_capacity) { - *length = 0; - return NULL; + void* new_mem = realloc (rq->glyphs, sizeof (raqm_glyph_t) * count); + if (!new_mem) + { + *length = 0; + return NULL; + } + + rq->glyphs = new_mem; + rq->glyphs_capacity = count; } + *length = count; + RAQM_TEST ("Glyph information:\n"); count = 0; @@ -948,7 +1127,7 @@ raqm_get_glyphs (raqm_t *rq, count += len; } - if (rq->flags & RAQM_FLAG_UTF8) + if (rq->text_utf8) { #ifdef RAQM_TESTING RAQM_TEST ("\nUTF-32 clusters:"); @@ -971,17 +1150,78 @@ raqm_get_glyphs (raqm_t *rq, return rq->glyphs; } +/** + * raqm_get_par_resolved_direction: + * @rq: a #raqm_t. + * + * Gets the resolved direction of the paragraph; + * + * Return value: + * The #raqm_direction_t specifying the resolved direction of text, + * or #RAQM_DIRECTION_DEFAULT if raqm_layout() has not been called on @rq. + * + * Since: 0.8 + */ +RAQM_API raqm_direction_t +raqm_get_par_resolved_direction (raqm_t *rq) +{ + if (!rq) + return RAQM_DIRECTION_DEFAULT; + + return rq->resolved_dir; +} + +/** + * raqm_get_direction_at_index: + * @rq: a #raqm_t. + * @index: (in): character index. + * + * Gets the resolved direction of the character at specified index; + * + * Return value: + * The #raqm_direction_t specifying the resolved direction of text at the + * specified index, or #RAQM_DIRECTION_DEFAULT if raqm_layout() has not been + * called on @rq. + * + * Since: 0.8 + */ +RAQM_API raqm_direction_t +raqm_get_direction_at_index (raqm_t *rq, + size_t index) +{ + if (!rq) + return RAQM_DIRECTION_DEFAULT; + + for (raqm_run_t *run = rq->runs; run != NULL; run = run->next) + { + if (run->pos <= index && index < run->pos + run->len) { + switch (run->direction) { + case HB_DIRECTION_LTR: + return RAQM_DIRECTION_LTR; + case HB_DIRECTION_RTL: + return RAQM_DIRECTION_RTL; + case HB_DIRECTION_TTB: + return RAQM_DIRECTION_TTB; + default: + return RAQM_DIRECTION_DEFAULT; + } + } + } + + return RAQM_DIRECTION_DEFAULT; +} + static bool _raqm_resolve_scripts (raqm_t *rq); static hb_direction_t -_raqm_hb_dir (raqm_t *rq, FriBidiLevel level) +_raqm_hb_dir (raqm_t *rq, _raqm_bidi_level_t level) { hb_direction_t dir = HB_DIRECTION_LTR; if (rq->base_dir == RAQM_DIRECTION_TTB) dir = HB_DIRECTION_TTB; - else if (FRIBIDI_LEVEL_IS_RTL (level)) + else if (RAQM_BIDI_LEVEL_IS_RTL(level)) dir = HB_DIRECTION_RTL; return dir; @@ -990,9 +1230,65 @@ _raqm_hb_dir (raqm_t *rq, FriBidiLevel level) typedef struct { size_t pos; size_t len; - FriBidiLevel level; + _raqm_bidi_level_t level; } _raqm_bidi_run; +#ifdef RAQM_SHEENBIDI +static _raqm_bidi_run * +_raqm_bidi_itemize (raqm_t *rq, size_t *run_count) +{ + _raqm_bidi_run *runs; + SBAlgorithmRef bidi; + SBParagraphRef par; + SBUInteger par_len; + SBLineRef line; + + SBLevel base_level = SBLevelDefaultLTR; + SBCodepointSequence input = { + SBStringEncodingUTF32, + (void *) rq->text, + rq->text_len + }; + + if (rq->base_dir == RAQM_DIRECTION_RTL) + base_level = 1; + else if (rq->base_dir == RAQM_DIRECTION_LTR) + base_level = 0; + + /* paragraph */ + bidi = SBAlgorithmCreate (&input); + par = SBAlgorithmCreateParagraph (bidi, 0, INT32_MAX, base_level); + par_len = SBParagraphGetLength (par); + + /* lines */ + line = SBParagraphCreateLine (par, 0, par_len); + *run_count = SBLineGetRunCount (line); + + if (SBParagraphGetBaseLevel (par) == 0) + rq->resolved_dir = RAQM_DIRECTION_LTR; + else + rq->resolved_dir = RAQM_DIRECTION_RTL; + + runs = malloc (sizeof (_raqm_bidi_run) * (*run_count)); + if (runs) + { + const SBRun *sheenbidi_runs = SBLineGetRunsPtr(line); + + for (size_t i = 0; i < (*run_count); ++i) + { + runs[i].pos = sheenbidi_runs[i].offset; + runs[i].len = sheenbidi_runs[i].length; + runs[i].level = sheenbidi_runs[i].level; + } + } + + SBLineRelease (line); + SBParagraphRelease (par); + SBAlgorithmRelease (bidi); + + return runs; +} +#else static void _raqm_reverse_run (_raqm_bidi_run *run, const size_t len) { @@ -1093,19 +1389,61 @@ _raqm_reorder_runs (const FriBidiCharType *types, return runs; } -static bool -_raqm_itemize (raqm_t *rq) +static _raqm_bidi_run * +_raqm_bidi_itemize (raqm_t *rq, size_t *run_count) { FriBidiParType par_type = FRIBIDI_PAR_ON; + _raqm_bidi_run *runs = NULL; + FriBidiCharType *types; -#ifdef USE_FRIBIDI_EX_API + _raqm_bidi_level_t *levels; + int max_level = 0; FriBidiBracketType *btypes; + + types = calloc (rq->text_len, sizeof (FriBidiCharType)); + btypes = calloc (rq->text_len, sizeof (FriBidiBracketType)); + levels = calloc (rq->text_len, sizeof (_raqm_bidi_level_t)); + + if (!types || !levels || !btypes) + goto done; + + if (rq->base_dir == RAQM_DIRECTION_RTL) + par_type = FRIBIDI_PAR_RTL; + else if (rq->base_dir == RAQM_DIRECTION_LTR) + par_type = FRIBIDI_PAR_LTR; + + fribidi_get_bidi_types (rq->text, rq->text_len, types); + fribidi_get_bracket_types (rq->text, rq->text_len, types, btypes); + max_level = fribidi_get_par_embedding_levels_ex (types, btypes, + rq->text_len, &par_type, + levels); + + if (par_type == FRIBIDI_PAR_LTR) + rq->resolved_dir = RAQM_DIRECTION_LTR; + else + rq->resolved_dir = RAQM_DIRECTION_RTL; + + if (max_level == 0) + goto done; + + /* Get the number of bidi runs */ + runs = _raqm_reorder_runs (types, rq->text_len, par_type, levels, run_count); + +done: + free (types); + free (levels); + free (btypes); + + return runs; +} #endif - FriBidiLevel *levels; + +static bool +_raqm_itemize (raqm_t *rq) +{ _raqm_bidi_run *runs = NULL; raqm_run_t *last; - int max_level; - size_t run_count; + size_t run_count = 0; bool ok = true; #ifdef RAQM_TESTING @@ -1127,67 +1465,28 @@ _raqm_itemize (raqm_t *rq) } #endif - types = calloc (rq->text_len, sizeof (FriBidiCharType)); -#ifdef USE_FRIBIDI_EX_API - btypes = calloc (rq->text_len, sizeof (FriBidiBracketType)); -#endif - levels = calloc (rq->text_len, sizeof (FriBidiLevel)); - if (!types || !levels -#ifdef USE_FRIBIDI_EX_API - || !btypes -#endif - ) + if (!_raqm_resolve_scripts (rq)) { ok = false; goto done; } - if (rq->base_dir == RAQM_DIRECTION_RTL) - par_type = FRIBIDI_PAR_RTL; - else if (rq->base_dir == RAQM_DIRECTION_LTR) - par_type = FRIBIDI_PAR_LTR; - if (rq->base_dir == RAQM_DIRECTION_TTB) { /* Treat every thing as LTR in vertical text */ - max_level = 1; - memset (types, FRIBIDI_TYPE_LTR, rq->text_len); - memset (levels, 0, rq->text_len); - rq->resolved_dir = RAQM_DIRECTION_LTR; - } - else - { - fribidi_get_bidi_types (rq->text, rq->text_len, types); -#ifdef USE_FRIBIDI_EX_API - fribidi_get_bracket_types (rq->text, rq->text_len, types, btypes); - max_level = fribidi_get_par_embedding_levels_ex (types, btypes, - rq->text_len, &par_type, - levels); -#else - max_level = fribidi_get_par_embedding_levels (types, rq->text_len, - &par_type, levels); -#endif - - if (par_type == FRIBIDI_PAR_LTR) - rq->resolved_dir = RAQM_DIRECTION_LTR; - else - rq->resolved_dir = RAQM_DIRECTION_RTL; - } - - if (max_level == 0) - { - ok = false; - goto done; - } - - if (!_raqm_resolve_scripts (rq)) - { - ok = false; - goto done; + run_count = 1; + rq->resolved_dir = RAQM_DIRECTION_TTB; + runs = malloc (sizeof (_raqm_bidi_run)); + if (runs) + { + runs->pos = 0; + runs->len = rq->text_len; + runs->level = 0; + } + } else { + runs = _raqm_bidi_itemize (rq, &run_count); } - /* Get the number of bidi runs */ - runs = _raqm_reorder_runs (types, rq->text_len, par_type, levels, &run_count); if (!runs) { ok = false; @@ -1197,7 +1496,7 @@ _raqm_itemize (raqm_t *rq) #ifdef RAQM_TESTING RAQM_TEST ("Number of runs before script itemization: %zu\n\n", run_count); - RAQM_TEST ("Fribidi Runs:\n"); + RAQM_TEST ("BiDi Runs:\n"); for (size_t i = 0; i < run_count; i++) { RAQM_TEST ("run[%zu]:\t start: %zu\tlength: %zu\tlevel: %d\n", @@ -1209,7 +1508,7 @@ _raqm_itemize (raqm_t *rq) last = NULL; for (size_t i = 0; i < run_count; i++) { - raqm_run_t *run = calloc (1, sizeof (raqm_run_t)); + raqm_run_t *run = _raqm_alloc_run (rq); if (!run) { ok = false; @@ -1228,13 +1527,14 @@ _raqm_itemize (raqm_t *rq) { run->pos = runs[i].pos + runs[i].len - 1; run->script = rq->text_info[run->pos].script; - run->font = _raqm_create_hb_font (rq, rq->text_info[run->pos].ftface); + run->font = _raqm_create_hb_font (rq, rq->text_info[run->pos].ftface, + rq->text_info[run->pos].ftloadflags); for (int j = runs[i].len - 1; j >= 0; j--) { _raqm_text_info info = rq->text_info[runs[i].pos + j]; if (!_raqm_compare_text_info (rq->text_info[run->pos], info)) { - raqm_run_t *newrun = calloc (1, sizeof (raqm_run_t)); + raqm_run_t *newrun = _raqm_alloc_run (rq); if (!newrun) { ok = false; @@ -1244,7 +1544,8 @@ _raqm_itemize (raqm_t *rq) newrun->len = 1; newrun->direction = _raqm_hb_dir (rq, runs[i].level); newrun->script = info.script; - newrun->font = _raqm_create_hb_font (rq, info.ftface); + newrun->font = _raqm_create_hb_font (rq, info.ftface, + info.ftloadflags); run->next = newrun; run = newrun; } @@ -1259,13 +1560,14 @@ _raqm_itemize (raqm_t *rq) { run->pos = runs[i].pos; run->script = rq->text_info[run->pos].script; - run->font = _raqm_create_hb_font (rq, rq->text_info[run->pos].ftface); + run->font = _raqm_create_hb_font (rq, rq->text_info[run->pos].ftface, + rq->text_info[run->pos].ftloadflags); for (size_t j = 0; j < runs[i].len; j++) { _raqm_text_info info = rq->text_info[runs[i].pos + j]; if (!_raqm_compare_text_info (rq->text_info[run->pos], info)) { - raqm_run_t *newrun = calloc (1, sizeof (raqm_run_t)); + raqm_run_t *newrun = _raqm_alloc_run (rq); if (!newrun) { ok = false; @@ -1275,7 +1577,8 @@ _raqm_itemize (raqm_t *rq) newrun->len = 1; newrun->direction = _raqm_hb_dir (rq, runs[i].level); newrun->script = info.script; - newrun->font = _raqm_create_hb_font (rq, info.ftface); + newrun->font = _raqm_create_hb_font (rq, info.ftface, + info.ftloadflags); run->next = newrun; run = newrun; } @@ -1309,11 +1612,6 @@ _raqm_itemize (raqm_t *rq) done: free (runs); - free (types); -#ifdef USE_FRIBIDI_EX_API - free (btypes); -#endif - free (levels); return ok; } @@ -1328,7 +1626,7 @@ typedef struct { /* Special paired characters for script detection */ static size_t paired_len = 34; -static const FriBidiChar paired_chars[] = +static const uint32_t paired_chars[] = { 0x0028, 0x0029, /* ascii paired punctuation */ 0x003c, 0x003e, @@ -1431,7 +1729,7 @@ _raqm_stack_push (_raqm_stack_t *stack, } static int -_get_pair_index (const FriBidiChar ch) +_get_pair_index (const uint32_t ch) { int lower = 0; int upper = paired_len - 1; @@ -1589,15 +1887,13 @@ _raqm_shape (raqm_t *rq) { hb_buffer_flags_t hb_buffer_flags = HB_BUFFER_FLAG_BOT | HB_BUFFER_FLAG_EOT; -#if defined(HAVE_DECL_HB_BUFFER_FLAG_REMOVE_DEFAULT_IGNORABLES) && \ - HAVE_DECL_HB_BUFFER_FLAG_REMOVE_DEFAULT_IGNORABLES if (rq->invisible_glyph < 0) hb_buffer_flags |= HB_BUFFER_FLAG_REMOVE_DEFAULT_IGNORABLES; -#endif for (raqm_run_t *run = rq->runs; run != NULL; run = run->next) { - run->buffer = hb_buffer_create (); + if (!run->buffer) + run->buffer = hb_buffer_create (); hb_buffer_add_utf32 (run->buffer, rq->text, rq->text_len, run->pos, run->len); @@ -1606,15 +1902,12 @@ _raqm_shape (raqm_t *rq) hb_buffer_set_direction (run->buffer, run->direction); hb_buffer_set_flags (run->buffer, hb_buffer_flags); -#ifdef HAVE_HB_BUFFER_SET_INVISIBLE_GLYPH if (rq->invisible_glyph > 0) hb_buffer_set_invisible_glyph (run->buffer, rq->invisible_glyph); -#endif hb_shape_full (run->font, run->buffer, rq->features, rq->features_len, NULL); -#ifdef HAVE_FT_GET_TRANSFORM { FT_Matrix matrix; hb_glyph_position_t *pos; @@ -1628,26 +1921,35 @@ _raqm_shape (raqm_t *rq) _raqm_ft_transform (&pos[i].x_offset, &pos[i].y_offset, matrix); } } -#endif } return true; } +/* Count equivalent UTF-8 bytes in codepoint */ +static size_t +_raqm_count_codepoint_utf8_bytes (uint32_t chr) +{ + if (0 == ((uint32_t) 0xffffff80 & chr)) + return 1; + else if (0 == ((uint32_t) 0xfffff800 & chr)) + return 2; + else if (0 == ((uint32_t) 0xffff0000 & chr)) + return 3; + else + return 4; +} + /* Convert index from UTF-32 to UTF-8 */ static uint32_t _raqm_u32_to_u8_index (raqm_t *rq, uint32_t index) { - FriBidiStrIndex length; - char *output = malloc ((sizeof (char) * 4 * index) + 1); + size_t length = 0; - length = fribidi_unicode_to_charset (FRIBIDI_CHAR_SET_UTF8, - rq->text, - index, - output); + for (uint32_t i = 0; i < index; ++i) + length += _raqm_count_codepoint_utf8_bytes (rq->text[i]); - free (output); return length; } @@ -1656,15 +1958,27 @@ static uint32_t _raqm_u8_to_u32_index (raqm_t *rq, uint32_t index) { - FriBidiStrIndex length; - uint32_t *output = malloc (sizeof (uint32_t) * (index + 1)); + const unsigned char *s = (const unsigned char *) rq->text_utf8; + const unsigned char *t = s; + size_t length = 0; + + while (((size_t) (s - t) < index) && ('\0' != *s)) + { + if (0xf0 == (0xf8 & *s)) + s += 4; + else if (0xe0 == (0xf0 & *s)) + s += 3; + else if (0xc0 == (0xe0 & *s)) + s += 2; + else + s += 1; + + length++; + } - length = fribidi_charset_to_unicode (FRIBIDI_CHAR_SET_UTF8, - rq->text_utf8, - index, - output); + if ((size_t) (s-t) > index) + length--; - free (output); return length; } @@ -1704,7 +2018,7 @@ raqm_index_to_position (raqm_t *rq, if (rq == NULL) return false; - if (rq->flags & RAQM_FLAG_UTF8) + if (rq->text_utf8) *index = _raqm_u8_to_u32_index (rq, *index); if (*index >= rq->text_len) @@ -1761,7 +2075,7 @@ raqm_index_to_position (raqm_t *rq, } found: - if (rq->flags & RAQM_FLAG_UTF8) + if (rq->text_utf8) *index = _raqm_u32_to_u8_index (rq, *index); RAQM_TEST ("The position is %d at index %zu\n",*x ,*index); return true; diff --git a/src/thirdparty/raqm/raqm.h b/src/thirdparty/raqm/raqm.h index 342afc8b29b..bdb5a50d884 100644 --- a/src/thirdparty/raqm/raqm.h +++ b/src/thirdparty/raqm/raqm.h @@ -1,6 +1,6 @@ /* * Copyright © 2015 Information Technology Authority (ITA) - * Copyright © 2016 Khaled Hosny + * Copyright © 2016-2022 Khaled Hosny * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to @@ -106,6 +106,9 @@ raqm_reference (raqm_t *rq); RAQM_API void raqm_destroy (raqm_t *rq); +RAQM_API void +raqm_clear_contents (raqm_t *rq); + RAQM_API bool raqm_set_text (raqm_t *rq, const uint32_t *text, @@ -145,6 +148,12 @@ RAQM_API bool raqm_set_freetype_load_flags (raqm_t *rq, int flags); +RAQM_API bool +raqm_set_freetype_load_flags_range (raqm_t *rq, + int flags, + size_t start, + size_t len); + RAQM_API bool raqm_set_invisible_glyph (raqm_t *rq, int gid); @@ -156,6 +165,13 @@ RAQM_API raqm_glyph_t * raqm_get_glyphs (raqm_t *rq, size_t *length); +RAQM_API raqm_direction_t +raqm_get_par_resolved_direction (raqm_t *rq); + +RAQM_API raqm_direction_t +raqm_get_direction_at_index (raqm_t *rq, + size_t index); + RAQM_API bool raqm_index_to_position (raqm_t *rq, size_t *index, diff --git a/winbuild/build.rst b/winbuild/build.rst index b30a94226d7..661c5a5ecea 100644 --- a/winbuild/build.rst +++ b/winbuild/build.rst @@ -24,7 +24,7 @@ Download and install: * `CMake 3.12 or newer `_ (also available as Visual Studio component C++ CMake tools for Windows) -* `NASM `_ +* x86/x64: `NASM `_ Any version of Visual Studio 2017 or newer should be supported, including Visual Studio 2017 Community, or Build Tools for Visual Studio 2019. @@ -42,8 +42,8 @@ behaviour of ``build_prepare.py``: If ``PYTHON`` is unset, the version of Python used to run ``build_prepare.py`` will be used. If only ``PYTHON`` is set, ``EXECUTABLE`` defaults to ``python.exe``. -* ``ARCHITECTURE`` is used to select a ``x86`` or ``x64`` build. By default, - uses same architecture as the version of Python used to run ``build_prepare.py``. +* ``ARCHITECTURE`` is used to select a ``x86``, ``x64`` or ``ARM64``build. + By default, uses same architecture as the version of Python used to run ``build_prepare.py``. is used. * ``PILLOW_BUILD`` can be used to override the ``winbuild\build`` directory path, used to store generated build scripts and compiled libraries. diff --git a/winbuild/build_prepare.py b/winbuild/build_prepare.py index 3092c5a2ba8..01f1bac298b 100644 --- a/winbuild/build_prepare.py +++ b/winbuild/build_prepare.py @@ -1,4 +1,5 @@ import os +import platform import shutil import struct import subprocess @@ -93,6 +94,7 @@ def cmd_msbuild( architectures = { "x86": {"vcvars_arch": "x86", "msbuild_arch": "Win32"}, "x64": {"vcvars_arch": "x86_amd64", "msbuild_arch": "x64"}, + "ARM64": {"vcvars_arch": "x86_arm64", "msbuild_arch": "ARM64"}, } header = [ @@ -219,25 +221,25 @@ def cmd_msbuild( # "bins": [r"objs\{msbuild_arch}\Release\freetype.dll"], }, "lcms2": { - "url": SF_MIRROR + "/project/lcms/lcms/2.12/lcms2-2.12.tar.gz", - "filename": "lcms2-2.12.tar.gz", - "dir": "lcms2-2.12", + "url": SF_MIRROR + "/project/lcms/lcms/2.13/lcms2-2.13.tar.gz", + "filename": "lcms2-2.13.tar.gz", + "dir": "lcms2-2.13", "patch": { - r"Projects\VC2017\lcms2_static\lcms2_static.vcxproj": { + r"Projects\VC2019\lcms2_static\lcms2_static.vcxproj": { # default is /MD for x86 and /MT for x64, we need /MD always "MultiThreaded": "MultiThreadedDLL", # noqa: E501 # retarget to default toolset (selected by vcvarsall.bat) - "v141": "$(DefaultPlatformToolset)", # noqa: E501 + "v142": "$(DefaultPlatformToolset)", # noqa: E501 # retarget to latest (selected by vcvarsall.bat) - "10.0.17134.0": "$(WindowsSDKVersion)", # noqa: E501 + "10.0": "$(WindowsSDKVersion)", # noqa: E501 } }, "build": [ cmd_rmdir("Lib"), - cmd_rmdir(r"Projects\VC2017\Release"), - cmd_msbuild(r"Projects\VC2017\lcms2.sln", "Release", "Clean"), + cmd_rmdir(r"Projects\VC2019\Release"), + cmd_msbuild(r"Projects\VC2019\lcms2.sln", "Release", "Clean"), cmd_msbuild( - r"Projects\VC2017\lcms2.sln", "Release", "lcms2_static:Rebuild" + r"Projects\VC2019\lcms2.sln", "Release", "lcms2_static:Rebuild" ), cmd_xcopy("include", "{inc_dir}"), ], @@ -278,9 +280,9 @@ def cmd_msbuild( "libs": [r"imagequant.lib"], }, "harfbuzz": { - "url": "https://github.com/harfbuzz/harfbuzz/archive/3.2.0.zip", - "filename": "harfbuzz-3.2.0.zip", - "dir": "harfbuzz-3.2.0", + "url": "https://github.com/harfbuzz/harfbuzz/archive/3.3.1.zip", + "filename": "harfbuzz-3.3.1.zip", + "dir": "harfbuzz-3.3.1", "build": [ cmd_cmake("-DHB_HAVE_FREETYPE:BOOL=TRUE"), cmd_nmake(target="clean"), @@ -490,7 +492,10 @@ def build_pillow(): python_dir = os.environ.get("PYTHON") python_exe = os.environ.get("EXECUTABLE", "python.exe") architecture = os.environ.get( - "ARCHITECTURE", "x86" if struct.calcsize("P") == 4 else "x64" + "ARCHITECTURE", + "ARM64" + if platform.machine() == "ARM64" + else ("x86" if struct.calcsize("P") == 4 else "x64"), ) build_dir = os.environ.get("PILLOW_BUILD", os.path.join(winbuild_dir, "build")) sources_dir = ""