Merge pull request #882 from input-output-hk/feature/ETCM-129-scala-point-13-ringer

Fix scala==2.13 nix build, cleanup

Author: Dmitry Voronov

.buildkite/pipeline.nix

@@ -14,24 +14,7 @@ in
     nixExpr = commonAttrs // {
       label = "ensure Nix expressions are up-to-date";
       command = ''
-        echo "Checking if Nix expressions are up-to-date..."
-
-        nix-shell --run 'sbtix-gen-all2'
-
-        set +e
-        git diff --exit-code > nix-expr.patch
-        if [ "$?" -eq "1" ]; then
-          set -e
-          echo "Nix expressions not up-to-date."
-          echo "Download and apply the patch available in the artifact paths of this step:"
-          echo "  patch -p1 < nix-expr.patch"
-          echo "Aborting."
-          exit 1
-        else
-          set -e
-          echo "Nix expressions up-to-date!"
-          exit 0
-        fi
+        ./update-nix.sh --check
       '';
       retry.automatic = false;
       artifactPaths = [

.buildkite/shell.nix

@@ -17,17 +17,12 @@ let
 
     exec ${pkgs.protobuf}/bin/protoc "$@"
   '';
-
-  sbtix = pkgs.callPackage sources.Sbtix { };
-
 in
 
   with pkgs;
-  
+
   mkShell {
-  
-    buildInputs = [ sbt solc jdk8 protoc-wrapper sbtix ];
+    nativeBuildInputs = [ sbt solc jdk8 protoc-wrapper ];
     # SBT = "sbt -v -mem 2048 -J-Xmx4g -Dsbt.ivy.home=/cache/ivy2 -Dsbt.boot.directory=/cache/sbt -Dmaven.repo.local=/cache/maven -Dnix=true";
     SBT = "sbt -v -mem 2048 -J-Xmx4g -Dnix=true";
-    SBTIX_GEN = "true";
   }

.gitignore

@@ -22,3 +22,4 @@ metals.sbt
 
 # Nix
 result
+.nix/

README.md

@@ -116,40 +116,20 @@ In the root of the project:
 nix-build
 ```
 
-##### Regenerate lock files
+##### Update sbt+nix dependencies
 
-```
-nix-shell
-sbtix-gen-all2
-```
-
-OR
-
-If the "ensure Nix expressions are up-to-date" step of your CI
-build has failed, check the artifacts of that step. There should be a
-patch provided, which you can apply locally with:
+When updating project dependencies, the nix fixed-output-derivation
+will need to be updated so that it includes the new dependency state.
 
+To do so, please run:
 ```
-patch -p1 < downloaded.patch
+./update-nix.sh
+git add ./nix/pkgs/mantis.nix
+git commit -m "Update nix-sbt sha"
 ```
 
-This patch will update the lock files for you.
-
-###### Why so many lock files?
-
-- `repo.nix`                 : generated by the `sbtix-gen` command and includes only the build dependencies for the project.
-- `project/repo.nix`         : generated by the `sbtix-gen-all` command and includes only the plugin dependencies. Also generates `repo.nix`.
-- `project/project/repo.nix` : generated by the `sbtix-gen-all2` command and includes only the plugin dependencies. Also generates `repo.nix` and `project/repo.nix`.
-
-##### error: unsupported argument 'submodules' to 'fetchGit'
-
-You get this error when you aren't using a new-enough version of Nix (fetchGit support for submodules is recent).
-
-To fix this, update the version of Nix you are using, or in a pinch:
-
-  - Remove the "submodules = true;" argument from fetchGit (in `./nix/pkgs/mantis/default.nix`).
-  - `git submodule update --recursive --init`
-  - `nix-build`
+*NOTE:* This should only be necessary when updating dependencies
+(For example, edits to build.sbt or project/plugins.sbt will likely need to be regenerated)
 
 ### Monitoring
 

build.sbt

@@ -14,7 +14,7 @@ def commonSettings(projectName: String): Seq[sbt.Def.Setting[_]] = Seq(
   name := projectName,
   organization := "io.iohk",
   version := "3.2.1",
-  scalaVersion := "2.13.3",
+  scalaVersion := "2.13.4",
   // Scalanet snapshots are published to Sonatype after each build.
   resolvers += "Sonatype OSS Snapshots" at "https://oss.sonatype.org/content/repositories/snapshots",
   testOptions in Test += Tests
@@ -203,7 +203,9 @@ lazy val node = {
   if (!nixBuild)
     node
   else
+    //node.settings(PB.protocExecutable := file("protoc"))
     node.settings(PB.runProtoc in Compile := (args => Process("protoc", args) !))
+
 }
 
 coverageExcludedPackages := "io\\.iohk\\.ethereum\\.extvm\\.msg.*"

manual-repo.nix

@@ -1,25 +1,24 @@
 { system ? builtins.currentSystem, sources ? import ./sources.nix, src ? ../. }:
 
 let
-  overlay = final: prev: {
-    inherit sources;
+  # we need to filter out the nix files in this repository as they will
+  # affect the fixed derivation calculation from sbt-derivation
+  overlay = final: prev: let
     inherit (import sources.gitignore { inherit (prev) lib; }) gitignoreSource;
+    cleanedSrc = prev.lib.cleanSource (gitignoreSource src);
+  in {
+    inherit sources;
 
-    sbtix = prev.callPackage ../sbtix.nix {
-      jdk = prev.openjdk8_headless;
-      jre = prev.openjdk8_headless.jre;
-    };
-
-    mantisPkgs = final.callPackage ./pkgs/mantis {
+    mantis = final.callPackage ./pkgs/mantis.nix {
       inherit (prev.openjdk8_headless) jre;
-      inherit src;
+      src = cleanedSrc;
     };
-
-    inherit (final.mantisPkgs) mantis;
-
-    mkSrc = import sources.nix-mksrc { inherit (final) lib; };
   };
+  sbt-derivation-overlay = import sources.sbt-derivation;
 in import sources.nixpkgs {
   inherit system;
-  overlays = [ overlay ];
+  overlays = [
+    sbt-derivation-overlay
+    overlay
+  ];
 }

nix/default.nix

@@ -1,9 +1,7 @@
-{ pkgs, lib, newScope, sources, src, impure ? false }:
+{ pkgs, lib, src, impure ? false }:
 
-lib.makeScope newScope (self:
-  with self;
-  let callPackages = lib.callPackagesWith (pkgs // self);
-  in {
-    inherit sources src impure;
-    jdk = prev.openjdk8_headless;
-  })
+lib.makeScope pkgs.newScope (self: with self; {
+  inherit src impure;
+  mantis = callPackage ./mantis.nix { };
+  jdk = openjdk8_headless;
+})

nix/pkgs/default.nix

@@ -0,0 +1,90 @@
+{ src
+, lib
+, stdenv
+, makeWrapper
+, writeShellScriptBin
+, bash
+, protobuf
+, coreutils
+, jdk
+, jre
+, gawk
+, sbt
+, impure ? false
+}:
+
+let
+
+  # sbt-protoc puts the scala plugin in /tmp/protobridge<some-random-number>.
+  # it is in fact a shell script with a standard `#!/usr/bin/env sh` shebang
+  # that makes the Nix sandbox ANGRY and breaks all the things in a cryptic,
+  # hairpull-inducing way. So we gotta sed it out. Not the prettiest thing
+  # but it works.
+  #
+  # This will be unnecessary in sbt-protoc>=1.0.0, as we can specify a file path
+  # Also, https://github.com/thesamet/sbt-protoc/issues/209 will need to be resolved
+  # for the plugins to not fail.
+  protoc-wrapper = writeShellScriptBin "protoc" ''
+    set -e
+
+    for f in "$@"; do
+      echo ''${f##*=}
+    done | grep protocbridge | xargs sed -i "1s|.*|#!${bash}/bin/bash|"
+
+    exec ${protobuf}/bin/protoc "$@"
+  '';
+  nativeBuildInputs = [ protoc-wrapper jdk makeWrapper ];
+
+  # read version from build.sbt
+  version = let
+    buildSbt = builtins.readFile ../../build.sbt;
+    captures = builtins.match ''.*version := "([^"]+)".*'' buildSbt;
+  in builtins.elemAt captures 0;
+
+  LD_LIBRARY_PATH = lib.makeLibraryPath [ stdenv.cc.cc.lib ];
+  PATH = lib.makeBinPath [ coreutils jre gawk ];
+
+in sbt.mkDerivation {
+  pname = "mantis";
+
+  inherit src nativeBuildInputs version;
+
+  # This sha represents the change dependencies of mantis.
+  # Update this sha whenever you change the dependencies
+  depsSha256 = "sha256-yAaJUJgg5Sdnhr3mX85BquC3X1ygF6If/FOngMUIkPU=";
+
+  # this is the command used to to create the fixed-output-derivation
+  depsWarmupCommand = "sbt compile --debug -Dnix=true";
+
+  # just inherit build artifacts from fixed-output-derivation
+  dontBuild = true;
+
+  overrideDepsAttrs = oldAttrs: oldAttrs // {
+    # $out is a directory, thus the archive will fail to write a file
+    preInstall = ''
+      rm -rf $out
+    '';
+  };
+
+  installPhase = ''
+    sbt stage -Dnix=true
+    mkdir -p $out/
+    cp -r target/universal/stage/* $out/
+    mkdir -p $out/share/mantis
+    mv $out/{LICENSE,RELEASE,mantis_config.txt} $_
+
+    # wrap executable so that java is available at runtime
+    for p in $(find $out/bin/* -executable); do
+    wrapProgram "$p" \
+      --prefix PATH : ${PATH} \
+      ${lib.optionalString (!stdenv.isDarwin)
+        "--prefix LD_LIBRARY_PATH : ${LD_LIBRARY_PATH}"
+      }
+    done
+  '';
+
+  dontPatchShebangs = impure;
+
+  # limit warnings from trying to strip jar files
+  dontStrip = true;
+}