setup.py

# Available at setup time due to pyproject.toml
from pybind11.setup_helpers import Pybind11Extension, build_ext
from setuptools import setup
from pathlib import Path

__version__ = "0.0.1"

# The main interface is through Pybind11Extension.
# * You can add cxx_std=11/14/17, and then build_ext can be removed.
# * You can set include_pybind11=false to add the include directory yourself,
#   say from a submodule.
#
# Note:
#   Sort input source files if you glob sources to ensure bit-for-bit
#   reproducible builds (https://github.com/pybind/python_example/pull/53)

cpp_files = ["src/BPETokenizer.cpp", "src/binder.cpp"]

ext_modules = [
    Pybind11Extension(
        "tokenizers_cpp",
        # [str(fname) for fname in Path('src').glob('*.cpp')],
        cpp_files,
        # Example: passing in the version to the compiled code
        include_dirs=['include'],        
        extra_compile_args=["-O3"]
    ),
]

setup(
    name="tokenizers_cpp",
    version=__version__,
    author="Alexei Ivanov",
    author_email="alexivanov15@icloud.com",
    url="https://github.com/aivan6842/tokenizers",
    description="My implementation of common tokenizers!",
    long_description="",
    ext_modules=ext_modules,
    extras_require={"test": "pytest"},
    # Currently, build_ext only provides an optional "highest supported C++
    # level" feature, but in the future it may provide more features.
    cmdclass={"build_ext": build_ext},
    zip_safe=False,
    python_requires=">=3.10",
)