add metadata to data files

bear/process_irs/Snakefile

@@ -38,6 +38,21 @@ rule find_global_delays:
     shell:
         "python -m bear.process_irs.find_global_delays {params.args} --tf-in {input[0]} {params.in_delays} --out {output}"
 
+def get_meta_args(wildcards):
+    metadata = config["params"][wildcards.params].get("metadata", {}).get(wildcards.sofa, {})
+    label = metadata.get("label", wildcards.sofa)
+    description = metadata.get("description")
+
+    args = ["--label", label]
+
+    if description is not None:
+        args.extend(["--description", description])
+
+    if metadata.get("released", False):
+        args.append("--released")
+
+    return args
+
 rule finalise:
     input:
         "irs/{params}/{sofa}.tf",
@@ -49,9 +64,10 @@ rule finalise:
         "bear_data/{params}/{sofa}.tf",
         "finalise_extra/{params}/{sofa}.tf",
     params:
-        args=lambda w: config["params"][w.params].get("finalise_args", "")
+        args=lambda w: config["params"][w.params].get("finalise_args", ""),
+        meta_args=get_meta_args,
     shell:
-        "python -m bear.process_irs.finalise {params.args} --tf-in {input[0]} --delays {input[1]} --tf-out {output[0]} --hoa-decoder {input[2]} --extra-out {output[1]}"
+        "python -m bear.process_irs.finalise {params.args} {params.meta_args:q} --tf-in {input[0]} --delays {input[1]} --tf-out {output[0]} --hoa-decoder {input[2]} --extra-out {output[1]}"
 
 rule report:
     input:
@@ -142,6 +158,10 @@ rule norm:
     shell:
         "python -m bear.test_files.normalize {input} - {output}"
 
+rule tfs:
+    input:
+        expand("bear_data/{tf}.tf", tf=NEW_TFS),
+
 rule all:
     input:
         expand(

bear/process_irs/finalise.py

@@ -46,6 +46,10 @@ def parse_args():
                         help="enable gain normalisation factors",
     )
 
+    parser.add_argument('--label', help="label for metadata", required=True)
+    parser.add_argument('--description', help="description for metadata")
+    parser.add_argument('--released', help=argparse.SUPPRESS, action="store_true")
+
     return parser.parse_args()
 
 
@@ -381,7 +385,16 @@ def main(args):
     # select front loudspeaker idx
     front_loudspeaker = np.argmin(np.linalg.norm(positions - [0, 1, 0], axis=1))
 
+    metadata = dict(
+        label=args.label,
+        released=args.released,
+    )
+    if args.description is not None:
+        metadata["description"] = args.description
+
     output = dict(
+        bear_data_version=0,
+        metadata=metadata,
         views=views.astype(np.float32),
         # shape (view, loudspeaker, ear, sample)
         brirs=irs.astype(np.float32),

bear/process_irs/snakemake_config.yaml

@@ -1,12 +1,18 @@
 params:
   brir:
     extract_args: "--ir-selection-algo closest"
+    metadata:
+      bbcrdlr_all_speakers:
+        label: "default room KU100 with multiple views v2.0"
   brir-fast:
     extract_args: "--ir-selection-algo closest --num-views 60 --layout 4+5+0"
     find_delay_over_view_args: "--default-schedule --opt-steps 10000"
   brir-one-view:
     extract_args: "--ir-selection-algo closest --num-views 1"
     one_view: true
+    metadata:
+      bbcrdlr_all_speakers:
+        label: "default room KU100 v2.0"
   brir-one-view-nowin:
     extract_args: "--ir-selection-algo closest --num-views 1"
     finalise_args: "--window none"
@@ -19,3 +25,10 @@ params:
     one_view: true
     hrir: true
     finalise_args: "--window none"
+    metadata:
+      HRIR_FULL2DEG:
+        label: "FHK KU100 HRIR v1.0"
+        description: "derived from the IRs described in \"Bernschütz, B. (2013). A Spherical Far Field HRIR/HRTF Compilation of the Neumann KU 100\""
+      sadie-d1-hrir:
+        label: "SADIE D1 KU100 HRIR v1.0"
+        description: "derived from subject D1 of the SADIE II Database"

doc/ir_processing.md

@@ -113,7 +113,7 @@ final data file.
 
 For BRIRs:
 
-    python -m bear.process_irs.finalise --tf-in irs.tf --delays global_delays.npy --tf-out bear_data.tf --hoa-decoder bear/process_irs/data/AmbiHOA3_magLS.sofa --extra-out finalise_extra.tf
+    python -m bear.process_irs.finalise --label "example v1" --tf-in irs.tf --delays global_delays.npy --tf-out bear_data.tf --hoa-decoder bear/process_irs/data/AmbiHOA3_magLS.sofa --extra-out finalise_extra.tf
 
 For HRIRs, add "--window none" to disable windowing.
 

nix/visr.nix

@@ -1,8 +1,8 @@
-{ lib, stdenv, cmake, python, boost, fetchzip, src, enable_python ? true }:
+{ lib, stdenv, cmake, ninja, python, boost, fetchzip, src, enable_python ? true }:
 stdenv.mkDerivation rec {
   name = "VISR";
   inherit src;
-  nativeBuildInputs = [ cmake boost ] ++ lib.optional enable_python python.buildEnv;
+  nativeBuildInputs = [ cmake ninja boost ] ++ lib.optional enable_python python.buildEnv;
   cmakeFlags = [
     "-DBUILD_AUDIOINTERFACES_PORTAUDIO=FALSE"
     "-DBUILD_USE_SNDFILE_LIBRARY=FALSE"

nix/visr_bear.nix

@@ -1,9 +1,10 @@
-{ lib, stdenv, src, cmake, boost, python, visr ? null, visr_python ? null, data_files, eigen, libear, rapidjson, enable_python ? true }:
+{ lib, stdenv, src, cmake, ninja, boost, python, visr ? null, visr_python ? null, data_files, eigen, libear, rapidjson, enable_python ? true }:
 stdenv.mkDerivation rec {
   name = "visr_bear";
   inherit src;
   nativeBuildInputs = [
     cmake
+    ninja
     boost
     eigen
     libear
@@ -20,5 +21,4 @@ stdenv.mkDerivation rec {
   preConfigure = ''cmakeFlags="$cmakeFlags -DBEAR_PYTHON_SITE_PACKAGES=$out/${python.sitePackages}"'';
 
   doCheck = true;
-  checkPhase = "make test";
 }

nix/xsimd.nix

@@ -1,6 +1,6 @@
 # adapted from https://github.com/NixOS/nixpkgs/blob/master/pkgs/development/libraries/xsimd/default.nix
 # MIT licensed
-{ lib, stdenv, fetchFromGitHub, cmake, doctest }:
+{ lib, stdenv, fetchFromGitHub, cmake, ninja, doctest }:
 stdenv.mkDerivation rec {
   pname = "xsimd";
   version = "10.0.0";
@@ -11,7 +11,7 @@ stdenv.mkDerivation rec {
     sha256 = "sha256-+ewKbce+rjNWQ0nQzm6O4xSwgzizSPpDPidkQYuoSTU=";
   };
 
-  nativeBuildInputs = [ cmake doctest ];
+  nativeBuildInputs = [ cmake ninja doctest ];
 
   cmakeFlags = [ "-DBUILD_TESTS=ON" ];
 

visr_bear/data/files.txt

@@ -1,9 +1,9 @@
-set(default_url https://github.com/ebu/bear/releases/download/v0.0.1-pre/default.tf)
+set(default_url https://github.com/ebu/bear/releases/download/v0.0.1-pre/default_v1.1.tf)
 set(default_fname default.tf)
-set(default_sha256 c23a36289f246c96779fdce75e108187185d3ec7aeedd6afa25f7c3dc5e42131)
+set(default_sha256 171acae2159e60ffe9d705abc16a79be129ecd06d37186fae413a265b6ed71e8)
 
 set(default_small_fname default_small.tf)
-set(default_small_url https://github.com/ebu/bear/releases/download/v0.0.1-pre/default_small.tf)
-set(default_small_sha256 25f4ed69153f6245e494fdffca27be9be632cb0013ddd1de6e1ec20f22b44ddf)
+set(default_small_url https://github.com/ebu/bear/releases/download/v0.0.1-pre/default_small_v1.1.tf)
+set(default_small_sha256 2afd36456dd45c1fc5f2abf94a19e945fbf40595d2a331669934e94c46cd1101)
 
 set(all_files default default_small)

visr_bear/include/bear/api.hpp

@@ -255,4 +255,35 @@ class Renderer {
   std::unique_ptr<RendererImpl> impl;
 };
 
+class DataFileMetadataImpl;
+
+/// metadata contained in a BEAR data file
+class DataFileMetadata {
+ public:
+  /// get the metadata for a file with a given path
+  static DataFileMetadata read_from_file(const std::string &path);
+
+  /// does the file include metadata, or was it a file from before when
+  /// metadata was added
+  bool has_metadata() const;
+
+  /// short label that describes this file
+  const std::string &get_label() const;
+
+  /// longer description of this file
+  const std::string &get_description() const;
+
+  /// is this a data file released by the BEAR project, or a preliminary or
+  /// custom one?
+  bool is_released() const;
+
+  DataFileMetadata() = delete;
+  DataFileMetadata(DataFileMetadata &&);
+  ~DataFileMetadata();
+
+ private:
+  DataFileMetadata(std::unique_ptr<DataFileMetadataImpl> impl);
+  std::unique_ptr<DataFileMetadataImpl> impl;
+};
+
 };  // namespace bear

visr_bear/pythonwrappers/api.cpp

@@ -171,6 +171,13 @@ namespace python {
         .def(py::init<int64_t, int64_t>())
         .def_property_readonly("numerator", &Time::numerator)
         .def_property_readonly("denominator", &Time::denominator);
+
+    py::class_<DataFileMetadata>(m, "DataFileMetadata")
+        .def_static("read_from_file", &DataFileMetadata::read_from_file)
+        .def_property_readonly("has_metadata", &DataFileMetadata::has_metadata)
+        .def_property_readonly("label", &DataFileMetadata::get_label)
+        .def_property_readonly("description", &DataFileMetadata::get_description)
+        .def_property_readonly("is_released", &DataFileMetadata::is_released);
   }
 
 }  // namespace python

visr_bear/src/CMakeLists.txt

@@ -15,6 +15,8 @@ add_library(
   constructor_thread.cpp
   control.cpp
   control.hpp
+  data_file.hpp
+  data_file.cpp
   direct_delay_calc.cpp
   direct_delay_calc.hpp
   direct_diffuse_split.cpp

visr_bear/src/api.cpp

@@ -8,6 +8,7 @@
 #include <libvisr/time.hpp>
 
 #include "config_impl.hpp"
+#include "data_file.hpp"
 #include "listener_impl.hpp"
 #include "parameters.hpp"
 #include "top.hpp"
@@ -278,4 +279,82 @@ void Renderer::set_listener(const Listener &l, const boost::optional<Time> &inte
 }
 
 Renderer::~Renderer() = default;
+
+struct DataFileMetadataImpl {
+  rapidjson::Value metadata;
+
+  bool has_metadata = false;
+  std::string label;
+  std::string description;
+  bool released = false;
+};
+
+namespace {
+  std::string parse_label(rapidjson::Value &metadata)
+  {
+    auto it = metadata.FindMember("label");
+    if (it == metadata.MemberEnd()) throw std::runtime_error("expected label");
+
+    if (!it->value.IsString()) throw std::runtime_error("label should be a string");
+
+    return it->value.GetString();
+  }
+
+  std::string parse_description(rapidjson::Value &metadata)
+  {
+    auto it = metadata.FindMember("description");
+    if (it == metadata.MemberEnd()) return "";
+
+    if (!it->value.IsString()) throw std::runtime_error("description should be a string");
+
+    return it->value.GetString();
+  }
+
+  bool parse_released(rapidjson::Value &metadata)
+  {
+    auto it = metadata.FindMember("released");
+    if (it == metadata.MemberEnd()) throw std::runtime_error("expected released");
+
+    if (!it->value.IsBool()) throw std::runtime_error("released should be a bool");
+
+    return it->value.GetBool();
+  }
+}  // namespace
+
+DataFileMetadata::DataFileMetadata(std::unique_ptr<DataFileMetadataImpl> impl) : impl(std::move(impl)) {}
+DataFileMetadata::DataFileMetadata(DataFileMetadata &&other) = default;
+DataFileMetadata::~DataFileMetadata() = default;
+
+DataFileMetadata DataFileMetadata::read_from_file(const std::string &path)
+{
+  auto tf = tensorfile::read(path);
+
+  check_data_file_version(tf);
+
+  auto impl = std::make_unique<DataFileMetadataImpl>();
+
+  auto it = tf.metadata.FindMember("metadata");
+  if (it != tf.metadata.MemberEnd()) {
+    if (!it->value.IsObject()) throw std::runtime_error("data file metadata should be object");
+
+    impl->has_metadata = true;
+
+    impl->metadata = it->value;
+
+    impl->label = parse_label(impl->metadata);
+    impl->description = parse_description(impl->metadata);
+    impl->released = parse_released(impl->metadata);
+  }
+
+  return {std::move(impl)};
+}
+
+bool DataFileMetadata::has_metadata() const { return impl->has_metadata; }
+
+const std::string &DataFileMetadata::get_label() const { return impl->label; }
+
+const std::string &DataFileMetadata::get_description() const { return impl->description; }
+
+bool DataFileMetadata::is_released() const { return impl->released; }
+
 }  // namespace bear

visr_bear/src/data_file.cpp

@@ -0,0 +1,22 @@
+#include "data_file.hpp"
+
+#include <string>
+
+namespace bear {
+int get_data_file_version(tensorfile::TensorFile &tf)
+{
+  auto it = tf.metadata.FindMember("bear_data_version");
+  if (it == tf.metadata.MemberEnd()) return 0;
+
+  if (!it->value.IsInt()) throw std::runtime_error("version must be an integer");
+
+  return it->value.GetInt();
+}
+
+void check_data_file_version(tensorfile::TensorFile &tf)
+{
+  int version = get_data_file_version(tf);
+  if (version > 0)
+    throw std::runtime_error("data file version mismatch: expected 0, got " + std::to_string(version));
+}
+}  // namespace bear

visr_bear/src/data_file.hpp

@@ -0,0 +1,8 @@
+#include "tensorfile.hpp"
+
+namespace bear {
+/// get the version number of a BEAR data file
+int get_data_file_version(tensorfile::TensorFile &tf);
+/// check that the data file version is supported by this library
+void check_data_file_version(tensorfile::TensorFile &tf);
+};  // namespace bear

visr_bear/src/panner.cpp

@@ -1,5 +1,6 @@
 #include "panner.hpp"
 
+#include "data_file.hpp"
 #include "utils.hpp"
 
 namespace {
@@ -52,6 +53,8 @@ Panner::Panner(const std::string &brir_file_name)
 {
   auto tf = tensorfile::read(brir_file_name);
 
+  check_data_file_version(tf);
+
   views = tf.unpack<float>(tf.metadata["views"]);
   brirs = tf.unpack<float>(tf.metadata["brirs"]);
   delays = tf.unpack<float>(tf.metadata["delays"]);