feat: add loom package (#3)

Adds loom, distributions, and some additional dependencies (parsable,
tcmalloc, goftests) which aren't currently available on Nixpkgs.
Currently some of the loom tests aren't Python3 compatible and so
running `make test` in the checkPhase will always fail, for the time
being pythonImportsCheck is used to check they can at least be imported.

.gitignore

@@ -1 +1,2 @@
 result
+result-*

flake.nix

@@ -15,6 +15,7 @@
         # 2. Add foo as a parameter to the outputs function
         # 3. Add here: foo.flakeModule
         ./lib/devtools
+        inputs.flake-parts.flakeModules.easyOverlay
       ];
       systems = [ "x86_64-linux" "aarch64-linux" "aarch64-darwin" "x86_64-darwin" ];
 
@@ -40,11 +41,33 @@
         };
 
         toolkit = self.lib.basicTools pkgs;
-      in {
-        # Per-system attributes can be defined here. The self' and inputs'
-        # module parameters provide easy access to attributes of the same
-        # system.
 
+        callPackage = pkgs.newScope (
+          pkgs
+          // { inherit callPackage; }
+          // packages
+        );
+        callPy3Package = pkgs.newScope (
+          pkgs
+          // pkgs.python3Packages
+          // { inherit callPackage; }
+          // packages
+        );
+
+        packages = {
+          inherit
+            sppl
+            ociImgBase
+            ociImgIqlQuery
+          ;
+
+          goftests = callPackage ./pkgs/goftests { };
+          parsable = callPackage ./pkgs/parsable { };
+          pymetis = callPackage ./pkgs/pymetis { };
+          distributions = callPackage ./pkgs/distributions { };
+          loom = callPy3Package ./pkgs/loom { };
+        };
+      in {
         devShells.default = pkgs.mkShell {
           packages = [] ++ toolkit;
         };
@@ -59,9 +82,7 @@
           '';
         };
 
-        packages = {
-          inherit sppl ociImgBase ociImgIqlQuery;
-        };
+        inherit packages;
       };
 
       # NOTE: this property is consumed by flake-parts.mkFlake to define fields

pkgs/distributions/default.nix

@@ -0,0 +1,107 @@
+{
+  fetchFromGitHub,
+  callPackage,
+  fetchPypi,
+  eigen,
+  parsable,
+  goftests,
+  protobuf3_20,
+  python3Packages,
+}:
+let
+  version = "2.2.1";
+
+  src = fetchFromGitHub {
+    owner = "posterior";
+    repo = "distributions";
+    rev = "43c11618b0f229682fb916612ba2437c5f22a753"; # there is no tag
+    sha256 = "sha256-DiJ6Ljwc5K1CrzzexAQ53g86sKqaroYRhmXuxAHAOq4=";
+  };
+
+  distributions-shared = callPackage ./distributions-shared.nix { inherit version src; };
+
+  imageio = python3Packages.buildPythonPackage rec {
+    pname = "imageio";
+    version = "2.6.1";
+
+    src = fetchPypi {
+      inherit pname version;
+      hash = "sha256-9E6yMbnfSFh08v/SLf0MPHEefeB2UWuTdO3qXGW8Z64=";
+    };
+
+    doCheck = false;
+
+    nativeBuildInputs = with python3Packages; [
+      pytest
+    ];
+
+    propagatedBuildInputs = with python3Packages; [
+      pillow
+    ];
+
+    buildInputs = with python3Packages; [
+      enum34
+      numpy
+    ];
+  };
+in
+python3Packages.buildPythonPackage {
+  pname = "distributions";
+
+  inherit version src;
+
+  nativeBuildInputs = [
+    protobuf3_20
+    python3Packages.pyflakes
+  ];
+
+  buildInputs = [
+    eigen
+    # TODO: we're not sure if this is even needed
+    distributions-shared
+    protobuf3_20
+  ];
+
+  propagatedBuildInputs = with python3Packages; [
+    protobuf3_20
+    protobuf
+    cython
+    numpy
+    parsable
+    scipy
+    simplejson
+  ];
+
+  # TODO: be more precise. Some tests seem to be still in Python 2.
+  doCheck = false;
+  nativeCheckInputs = with python3Packages; [
+    imageio
+    nose
+    goftests
+    pytest
+  ];
+
+  preBuild = ''
+    make protobuf
+  '';
+
+  patches = [
+    ./use-imread-instead-of-scipy.patch
+  ];
+
+  env.DISTRIBUTIONS_USE_PROTOBUF = 1;
+
+  # https://github.com/numba/numba/issues/8698#issuecomment-1584888063 
+  env.NUMPY_EXPERIMENTAL_DTYPE_API = 1;
+
+  pythonImportsCheck = [
+    "distributions"
+    "distributions.io"
+    "distributions.io.stream"
+  ];
+
+  passthru = {
+    # TODO: we're not sure if this is even needed
+    inherit distributions-shared;
+  };
+}

pkgs/distributions/distributions-shared.nix

@@ -0,0 +1,26 @@
+{
+  stdenv,
+  cmake,
+  eigen,
+  protobuf3_20,
+  src, version
+}:
+stdenv.mkDerivation {
+  pname = "distributions-shared";
+
+  inherit version src;
+
+  nativeBuildInputs = [ cmake ];
+  buildInputs = [eigen protobuf3_20 ];
+
+  env.DISTRIBUTIONS_USE_PROTOBUF = 1;
+
+  preConfigure = ''
+    make protobuf
+  '';
+
+  fixupPhase = ''
+    ln -sv $out/lib/libdistributions_shared_release.so $out/lib/libdistributions_shared.so
+    ln -sv $out/lib/libdistributions_shared_release.so $out/lib/libdistributions_shared_debug.so
+  '';
+}

pkgs/distributions/use-imread-instead-of-scipy.patch

@@ -0,0 +1,19 @@
+--- a/examples/mixture/main.py
++++ b/examples/mixture/main.py
+@@ -30,6 +30,7 @@
+ import numpy
+ import scipy
+ import scipy.misc
++import imageio
+ import scipy.ndimage
+ from distributions.dbg.random import sample_discrete, sample_discrete_log
+ from distributions.lp.models import nich
+@@ -45,7 +46,7 @@
+ DATA = os.path.join(ROOT, 'data')
+ RESULTS = os.path.join(ROOT, 'results')
+ SAMPLES = os.path.join(DATA, 'samples.json.gz')
+-IMAGE = scipy.misc.imread(os.path.join(ROOT, 'fox.png'))
++IMAGE = imageio.imread(os.path.join(ROOT, 'fox.png'))
+ SAMPLE_COUNT = 10000
+ PASSES = 10
+ EMPTY_GROUP_COUNT = 10

pkgs/goftests/default.nix

@@ -0,0 +1,25 @@
+{
+  fetchPypi,
+  python3Packages,
+}:
+python3Packages.buildPythonPackage rec {
+  pname = "goftests";
+  version = "0.2.7";
+  format = "setuptools";
+
+  src = fetchPypi {
+    inherit pname version;
+    hash = "sha256-5s0NugSus2TuZIInesCNJNAtxEHnZLQIjn0pxGgwL/o=";
+  };
+
+  buildInputs = with python3Packages; [ numpy scipy ];
+
+  doCheck = false;
+
+  # https://github.com/numba/numba/issues/8698#issuecomment-1584888063 
+  env.NUMPY_EXPERIMENTAL_DTYPE_API = 1;
+
+  patchPhase = ''
+    mkdir -p dist
+  '';
+}