diff --git a/default.nix b/default.nix index ed9c1ae28..e688de58e 100644 --- a/default.nix +++ b/default.nix @@ -1,21 +1,38 @@ -{ pkgs ? import ./nix/default.nix {} +{ stdenv +, naersk +, libiconv +, pkgconfig }: - -with pkgs; - let - - self = { - inherit (tokenizersPackages) - tokenizers - tokenizers-haskell - ; - - shell = import ./shell.nix { - inherit pkgs; - }; + patchLibs = + if stdenv.isDarwin + then '' + install_name_tool -id $out/lib/libtokenizers_haskell.dylib $out/lib/libtokenizers_haskell.dylib + '' + else '' + patchelf --set-rpath "${stdenv.cc.cc.lib}/lib:$out/lib" $out/lib/libtokenizers_haskell.so + ''; +in { + tokenizers = naersk.buildPackage { + src = ./.; + buildInputs = [ libiconv pkgconfig ]; + release = true; + cargoBuildOptions = x: x ++ [ "-p" "tokenizers" ]; + cargoTestOptions = x: x ++ [ "-p" "tokenzers" ]; + copyBins = true; + copyLibs = true; + copyTarget = false; }; -in - - self + tokenizers-haskell = naersk.buildPackage { + src = ./.; + buildInputs = [ libiconv pkgconfig ]; + release = true; + cargoBuildOptions = x: x ++ [ "-p" "tokenizers-haskell" ]; + cargoTestOptions = x: x ++ [ "-p" "tokenizers-haskell" ]; + copyBins = false; + copyLibs = true; + copyTarget = false; + overrideMain = x: x // { postInstall = patchLibs; }; + }; +} diff --git a/flake.lock b/flake.lock new file mode 100644 index 000000000..1ccfd3e9f --- /dev/null +++ b/flake.lock @@ -0,0 +1,74 @@ +{ + "nodes": { + "naersk": { + "inputs": { + "nixpkgs": "nixpkgs" + }, + "locked": { + "lastModified": 1629707199, + "narHash": "sha256-sGxlmfp5eXL5sAMNqHSb04Zq6gPl+JeltIZ226OYN0w=", + "owner": "nix-community", + "repo": "naersk", + "rev": "df71f5e4babda41cd919a8684b72218e2e809fa9", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "naersk", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1630084536, + "narHash": "sha256-ZFO/c1n5NNzHdY848Lfrbh8QN8MdP6oajbj4Z55rVXs=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "40f337454d6c76bc12500564f4553d6853e83aeb", + "type": "github" + }, + "original": { + "id": "nixpkgs", + "type": "indirect" + } + }, + "nixpkgs_2": { + "locked": { + "lastModified": 1630084536, + "narHash": "sha256-ZFO/c1n5NNzHdY848Lfrbh8QN8MdP6oajbj4Z55rVXs=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "40f337454d6c76bc12500564f4553d6853e83aeb", + "type": "github" + }, + "original": { + "id": "nixpkgs", + "type": "indirect" + } + }, + "root": { + "inputs": { + "naersk": "naersk", + "nixpkgs": "nixpkgs_2", + "utils": "utils" + } + }, + "utils": { + "locked": { + "lastModified": 1629481132, + "narHash": "sha256-JHgasjPR0/J1J3DRm4KxM4zTyAj4IOJY8vIl75v/kPI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "997f7efcb746a9c140ce1f13c72263189225f482", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 000000000..334c42c06 --- /dev/null +++ b/flake.nix @@ -0,0 +1,27 @@ +{ + description = "huggingface tokenizers"; + + inputs = { + utils.url = "github:numtide/flake-utils"; + naersk.url = "github:nix-community/naersk"; + }; + + outputs = { self, nixpkgs, utils, naersk, ... }: { + + overlay = final: prev: rec { + tokenizersPackages = prev.callPackage ./. { naersk = naersk.lib."${final.system}"; }; + tokenizers-haskell = tokenizersPackages.tokenizers-haskell; + tokenizers_haskell = tokenizersPackages.tokenizers-haskell; # for backwards-compatibility + }; + + } // (utils.lib.eachSystem ["x86_64-darwin" "x86_64-linux"] (system: + let + pkgs = import nixpkgs { inherit system; overlays = [ naersk.overlay ]; }; + in { + packages = pkgs.callPackage ./. {}; + devShell = pkgs.mkShell { + nativeBuildInputs = with pkgs; [ cargo rustc rls libiconv pkgconfig ]; + }; + } + )); +} diff --git a/nix/default.nix b/nix/default.nix deleted file mode 100644 index 9dd0179f4..000000000 --- a/nix/default.nix +++ /dev/null @@ -1,19 +0,0 @@ -{}: - -let - - sources = import ./sources.nix { inherit pkgs; }; - nixpkgsSrc = sources.nixpkgs-unstable; - - overlays = - [ (pkgs: _: with pkgs; { - naersk = callPackage sources.naersk {}; - }) - (import ./pkgs.nix) - ]; - - pkgs = import nixpkgsSrc { - inherit overlays; - }; - -in pkgs diff --git a/nix/pkgs.nix b/nix/pkgs.nix deleted file mode 100644 index a88283c69..000000000 --- a/nix/pkgs.nix +++ /dev/null @@ -1,8 +0,0 @@ -pkgs: _: with pkgs; { - tokenizersPackages = import ./rust.nix { - inherit - stdenv - pkgs - ; - }; -} diff --git a/nix/rust.nix b/nix/rust.nix deleted file mode 100644 index bcd66b8b2..000000000 --- a/nix/rust.nix +++ /dev/null @@ -1,45 +0,0 @@ -{ stdenv -, pkgs -}: - -with pkgs; - -let - - patchLibs = - if stdenv.isDarwin - then '' - install_name_tool -id $out/lib/libtokenizers_haskell.dylib $out/lib/libtokenizers_haskell.dylib - '' - else '' - patchelf --set-rpath "${stdenv.cc.cc.lib}/lib:$out/lib" $out/lib/libtokenizers_haskell.so - ''; - - self = { - tokenizers = naersk.buildPackage { - src = ../.; - buildInputs = [ libiconv pkgconfig ]; - release = true; - cargoBuildOptions = x: x ++ [ "-p" "tokenizers" ]; - cargoTestOptions = x: x ++ [ "-p" "tokenzers" ]; - copyBins = true; - copyLibs = true; - copyTarget = false; - }; - - tokenizers-haskell = naersk.buildPackage { - src = ../.; - buildInputs = [ libiconv pkgconfig ]; - release = true; - cargoBuildOptions = x: x ++ [ "-p" "tokenizers-haskell" ]; - cargoTestOptions = x: x ++ [ "-p" "tokenizers-haskell" ]; - copyBins = false; - copyLibs = true; - copyTarget = false; - overrideMain = x: x // { postInstall = patchLibs; }; - }; - }; - -in - - self diff --git a/nix/sources.json b/nix/sources.json deleted file mode 100644 index a908cbc6d..000000000 --- a/nix/sources.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "naersk": { - "branch": "master", - "description": "Build rust crates in Nix. No configuration, no code generation, no IFD. Sandbox friendly.", - "homepage": "", - "owner": "nmattia", - "repo": "naersk", - "rev": "b3b099d669fc8b18d361c249091c9fe95d57ebbb", - "sha256": "156fbnr5s2n1xxbbk2z9xa7c5g2z5fdpqmjjs6n9ipbr038n0z3s", - "type": "tarball", - "url": "https://github.com/nmattia/naersk/archive/b3b099d669fc8b18d361c249091c9fe95d57ebbb.tar.gz", - "url_template": "https://github.com///archive/.tar.gz" - }, - "niv": { - "branch": "master", - "description": "Easy dependency management for Nix projects", - "homepage": "https://github.com/nmattia/niv", - "owner": "nmattia", - "repo": "niv", - "rev": "af958e8057f345ee1aca714c1247ef3ba1c15f5e", - "sha256": "1qjavxabbrsh73yck5dcq8jggvh3r2jkbr6b5nlz5d9yrqm9255n", - "type": "tarball", - "url": "https://github.com/nmattia/niv/archive/af958e8057f345ee1aca714c1247ef3ba1c15f5e.tar.gz", - "url_template": "https://github.com///archive/.tar.gz" - }, - "nixpkgs-unstable": { - "branch": "nixpkgs-unstable", - "description": "Nix Packages collection", - "homepage": "", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "410bbd828cdc6156aecd5bc91772ad3a6b1099c7", - "sha256": "0idvgvpgnzvk03yvd77lrca9qib936fq2x690jvk5gk3blsckz3r", - "type": "tarball", - "url": "https://github.com/NixOS/nixpkgs/archive/410bbd828cdc6156aecd5bc91772ad3a6b1099c7.tar.gz", - "url_template": "https://github.com///archive/.tar.gz" - } -} diff --git a/nix/sources.nix b/nix/sources.nix deleted file mode 100644 index 1938409dd..000000000 --- a/nix/sources.nix +++ /dev/null @@ -1,174 +0,0 @@ -# This file has been generated by Niv. - -let - - # - # The fetchers. fetch_ fetches specs of type . - # - - fetch_file = pkgs: name: spec: - let - name' = sanitizeName name + "-src"; - in - if spec.builtin or true then - builtins_fetchurl { inherit (spec) url sha256; name = name'; } - else - pkgs.fetchurl { inherit (spec) url sha256; name = name'; }; - - fetch_tarball = pkgs: name: spec: - let - name' = sanitizeName name + "-src"; - in - if spec.builtin or true then - builtins_fetchTarball { name = name'; inherit (spec) url sha256; } - else - pkgs.fetchzip { name = name'; inherit (spec) url sha256; }; - - fetch_git = name: spec: - let - ref = - if spec ? ref then spec.ref else - if spec ? branch then "refs/heads/${spec.branch}" else - if spec ? tag then "refs/tags/${spec.tag}" else - abort "In git source '${name}': Please specify `ref`, `tag` or `branch`!"; - in - builtins.fetchGit { url = spec.repo; inherit (spec) rev; inherit ref; }; - - fetch_local = spec: spec.path; - - fetch_builtin-tarball = name: throw - ''[${name}] The niv type "builtin-tarball" is deprecated. You should instead use `builtin = true`. - $ niv modify ${name} -a type=tarball -a builtin=true''; - - fetch_builtin-url = name: throw - ''[${name}] The niv type "builtin-url" will soon be deprecated. You should instead use `builtin = true`. - $ niv modify ${name} -a type=file -a builtin=true''; - - # - # Various helpers - # - - # https://github.com/NixOS/nixpkgs/pull/83241/files#diff-c6f540a4f3bfa4b0e8b6bafd4cd54e8bR695 - sanitizeName = name: - ( - concatMapStrings (s: if builtins.isList s then "-" else s) - ( - builtins.split "[^[:alnum:]+._?=-]+" - ((x: builtins.elemAt (builtins.match "\\.*(.*)" x) 0) name) - ) - ); - - # The set of packages used when specs are fetched using non-builtins. - mkPkgs = sources: system: - let - sourcesNixpkgs = - import (builtins_fetchTarball { inherit (sources.nixpkgs) url sha256; }) { inherit system; }; - hasNixpkgsPath = builtins.any (x: x.prefix == "nixpkgs") builtins.nixPath; - hasThisAsNixpkgsPath = == ./.; - in - if builtins.hasAttr "nixpkgs" sources - then sourcesNixpkgs - else if hasNixpkgsPath && ! hasThisAsNixpkgsPath then - import {} - else - abort - '' - Please specify either (through -I or NIX_PATH=nixpkgs=...) or - add a package called "nixpkgs" to your sources.json. - ''; - - # The actual fetching function. - fetch = pkgs: name: spec: - - if ! builtins.hasAttr "type" spec then - abort "ERROR: niv spec ${name} does not have a 'type' attribute" - else if spec.type == "file" then fetch_file pkgs name spec - else if spec.type == "tarball" then fetch_tarball pkgs name spec - else if spec.type == "git" then fetch_git name spec - else if spec.type == "local" then fetch_local spec - else if spec.type == "builtin-tarball" then fetch_builtin-tarball name - else if spec.type == "builtin-url" then fetch_builtin-url name - else - abort "ERROR: niv spec ${name} has unknown type ${builtins.toJSON spec.type}"; - - # If the environment variable NIV_OVERRIDE_${name} is set, then use - # the path directly as opposed to the fetched source. - replace = name: drv: - let - saneName = stringAsChars (c: if isNull (builtins.match "[a-zA-Z0-9]" c) then "_" else c) name; - ersatz = builtins.getEnv "NIV_OVERRIDE_${saneName}"; - in - if ersatz == "" then drv else - # this turns the string into an actual Nix path (for both absolute and - # relative paths) - if builtins.substring 0 1 ersatz == "/" then /. + ersatz else /. + builtins.getEnv "PWD" + "/${ersatz}"; - - # Ports of functions for older nix versions - - # a Nix version of mapAttrs if the built-in doesn't exist - mapAttrs = builtins.mapAttrs or ( - f: set: with builtins; - listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set)) - ); - - # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/lists.nix#L295 - range = first: last: if first > last then [] else builtins.genList (n: first + n) (last - first + 1); - - # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L257 - stringToCharacters = s: map (p: builtins.substring p 1 s) (range 0 (builtins.stringLength s - 1)); - - # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L269 - stringAsChars = f: s: concatStrings (map f (stringToCharacters s)); - concatMapStrings = f: list: concatStrings (map f list); - concatStrings = builtins.concatStringsSep ""; - - # https://github.com/NixOS/nixpkgs/blob/8a9f58a375c401b96da862d969f66429def1d118/lib/attrsets.nix#L331 - optionalAttrs = cond: as: if cond then as else {}; - - # fetchTarball version that is compatible between all the versions of Nix - builtins_fetchTarball = { url, name ? null, sha256 }@attrs: - let - inherit (builtins) lessThan nixVersion fetchTarball; - in - if lessThan nixVersion "1.12" then - fetchTarball ({ inherit url; } // (optionalAttrs (!isNull name) { inherit name; })) - else - fetchTarball attrs; - - # fetchurl version that is compatible between all the versions of Nix - builtins_fetchurl = { url, name ? null, sha256 }@attrs: - let - inherit (builtins) lessThan nixVersion fetchurl; - in - if lessThan nixVersion "1.12" then - fetchurl ({ inherit url; } // (optionalAttrs (!isNull name) { inherit name; })) - else - fetchurl attrs; - - # Create the final "sources" from the config - mkSources = config: - mapAttrs ( - name: spec: - if builtins.hasAttr "outPath" spec - then abort - "The values in sources.json should not have an 'outPath' attribute" - else - spec // { outPath = replace name (fetch config.pkgs name spec); } - ) config.sources; - - # The "config" used by the fetchers - mkConfig = - { sourcesFile ? if builtins.pathExists ./sources.json then ./sources.json else null - , sources ? if isNull sourcesFile then {} else builtins.fromJSON (builtins.readFile sourcesFile) - , system ? builtins.currentSystem - , pkgs ? mkPkgs sources system - }: rec { - # The sources, i.e. the attribute set of spec name to spec - inherit sources; - - # The "pkgs" (evaluated nixpkgs) to use for e.g. non-builtin fetchers - inherit pkgs; - }; - -in -mkSources (mkConfig {}) // { __functor = _: settings: mkSources (mkConfig settings); } diff --git a/shell.nix b/shell.nix deleted file mode 100644 index 6ac64a392..000000000 --- a/shell.nix +++ /dev/null @@ -1,14 +0,0 @@ -{ pkgs ? import ./nix/default.nix {} -}: - -with pkgs; - -let - - shell = mkShell { - nativeBuildInputs = [ cargo rustc rls libiconv pkgconfig ]; - }; - -in - - shell