nix: make derivation and update shell (#1003)

* nix: make derivation and update shell

Create a structure for nix files. Add the derivation file which is using
system Nim to compile Codex.

Referenced issue: #940

Signed-off-by: markoburcul <[email protected]>

* nim-circom-compat: update

Include commit which allows building circom-compat-ffi using Nix(doesn't
affect current usage of the submodule).

Referenced issue: #940

Signed-off-by: markoburcul <[email protected]>

* makefile: fix for detecting linux arch

Signed-off-by: markoburcul <[email protected]>

---------

Signed-off-by: markoburcul <[email protected]>

Makefile

@@ -48,7 +48,7 @@ ifeq ($(OS),Windows_NT)
         ARCH = arm64
     endif
 else
-    UNAME_P := $(shell uname -p)
+    UNAME_P := $(shell uname -m)
     ifneq ($(filter $(UNAME_P), i686 i386 x86_64),)
         ARCH = x86_64
     endif

flake.nix

@@ -3,32 +3,40 @@
 
   inputs = {
     nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05";
+    circom-compat.url = "github:codex-storage/circom-compat-ffi";
   };
 
-  outputs = { self, nixpkgs }:
+  outputs = { self, nixpkgs, circom-compat}:
     let
-      supportedSystems = [
+      stableSystems = [
         "x86_64-linux" "aarch64-linux"
         "x86_64-darwin" "aarch64-darwin"
       ];
-      forAllSystems = f: nixpkgs.lib.genAttrs supportedSystems (system: f system);
+      forAllSystems = f: nixpkgs.lib.genAttrs stableSystems (system: f system);
       pkgsFor = forAllSystems (system: import nixpkgs { inherit system; });
     in rec {
+      packages = forAllSystems (system: let
+        circomCompatPkg = circom-compat.packages.${system}.default;
+        buildTarget = pkgsFor.${system}.callPackage ./nix/default.nix {
+          inherit stableSystems circomCompatPkg;
+          src = self;
+        };
+        build = targets: buildTarget.override { inherit targets; };
+      in rec {
+        codex   = build ["all"];
+        default = codex;
+      });
+
       devShells = forAllSystems (system: let
         pkgs = pkgsFor.${system};
-        inherit (pkgs) lib stdenv mkShell;
       in {
-        default = mkShell.override { stdenv = pkgs.gcc11Stdenv; } {
-          buildInputs = with pkgs; [
-              # General
-              git pkg-config openssl lsb-release
-              # Build
-              rustc cargo nimble gcc11 cmake nim-unwrapped-1
-              # Libraries
-              gmp llvmPackages.openmp
-              # Tests
-              nodejs_18
+        default = pkgs.mkShell {
+          inputsFrom = [
+            packages.${system}.codex
+            circom-compat.packages.${system}.default
           ];
+          # Not using buildInputs to override fakeGit and fakeCargo.
+          nativeBuildInputs = with pkgs; [ git cargo nodejs_18 ];
         };
       });
     };

nix/README.md

@@ -0,0 +1,29 @@
+# Usage
+
+## Shell
+
+A development shell can be started using:
+```sh
+nix develop
+```
+
+## Building
+
+To build a Codex you can use:
+```sh
+nix build '.?submodules=1#codex'
+```
+The `?submodules=1` part should eventually not be necessary.
+For more details see:
+https://github.com/NixOS/nix/issues/4423
+
+It can be also done without even cloning the repo:
+```sh
+nix build 'github:codex-storage/nim-codex?submodules=1'
+```
+
+## Running
+
+```sh
+nix run 'github:codex-storage/nim-codex?submodules=1'
+```

nix/default.nix

@@ -0,0 +1,84 @@
+{
+  pkgs ? import <nixpkgs> { },
+  src ? ../.,
+  targets ? ["all"],
+  # Options: 0,1,2
+  verbosity ? 0,
+  # Use system Nim compiler instead of building it with nimbus-build-system
+  useSystemNim ? true,
+  commit ? builtins.substring 0 7 (src.rev or "dirty"),
+  # These are the only platforms tested in CI and considered stable.
+  stableSystems ? [
+    "x86_64-linux" "aarch64-linux"
+    "x86_64-darwin" "aarch64-darwin"
+  ],
+  circomCompatPkg ? (
+    builtins.getFlake "github:codex-storage/circom-compat-ffi"
+  ).packages.${builtins.currentSystem}.default
+}:
+
+let
+  inherit (pkgs) stdenv lib writeScriptBin callPackage;
+
+  revision = lib.substring 0 8 (src.rev or "dirty");
+
+  tools = callPackage ./tools.nix {};
+in pkgs.gcc11Stdenv.mkDerivation rec {
+
+  pname = "codex";
+
+  version = "${tools.findKeyValue "version = \"([0-9]+\.[0-9]+\.[0-9]+)\"" ../codex.nimble}-${revision}";
+
+  inherit src;
+
+  # Dependencies that should exist in the runtime environment.
+  buildInputs = with pkgs; [
+    openssl
+    gmp
+  ];
+
+  # Dependencies that should only exist in the build environment.
+  nativeBuildInputs = let
+    # Fix for Nim compiler calling 'git rev-parse' and 'lsb_release'.
+    fakeGit = writeScriptBin "git" "echo ${version}";
+    # Fix for the nim-circom-compat-ffi package that is built with cargo.
+    fakeCargo = writeScriptBin "cargo" "echo ${version}";
+  in
+    with pkgs; [
+      cmake
+      pkg-config
+      nimble
+      which
+      nim-unwrapped-1
+      lsb-release
+      circomCompatPkg
+      fakeGit
+      fakeCargo
+  ];
+
+  # Disable CPU optmizations that make binary not portable.
+  NIMFLAGS = "-d:disableMarchNative -d:git_revision_override=${revision}";
+  # Avoid Nim cache permission errors.
+  XDG_CACHE_HOME = "/tmp";
+
+  makeFlags = targets ++ [
+    "V=${toString verbosity}"
+    "USE_SYSTEM_NIM=${if useSystemNim then "1" else "0"}"
+  ];
+
+  configurePhase = ''
+    patchShebangs . > /dev/null
+  '';
+
+  installPhase = ''
+    mkdir -p $out/bin
+    cp build/codex $out/bin/
+  '';
+
+  meta = with pkgs.lib; {
+    description = "Codex storage system";
+    homepage = "https://github.com/codex-storage/nim-codex";
+    license = licenses.mit;
+    platforms = stableSystems;
+  };
+}

nix/tools.nix

@@ -0,0 +1,15 @@
+{ pkgs ? import <nixpkgs> { } }:
+
+let
+
+  inherit (pkgs.lib) fileContents last splitString flatten remove;
+  inherit (builtins) map match;
+in {
+  findKeyValue = regex: sourceFile:
+    let
+      linesFrom = file: splitString "\n" (fileContents file);
+      matching = regex: lines: map (line: match regex line) lines;
+      extractMatch = matches: last (flatten (remove null matches));
+    in
+      extractMatch (matching regex (linesFrom sourceFile));
+}