From 82c7c47b919c9625d8cc712a1a76eb0fdbdb9e1a Mon Sep 17 00:00:00 2001 From: Rafael Schouten Date: Sat, 11 Jan 2025 10:26:15 +0100 Subject: [PATCH 01/14] add MatrixLookup --- src/Dimensions/format.jl | 3 ++ src/Lookups/lookup_arrays.jl | 30 ++++++++++++++++-- src/Lookups/selector.jl | 61 ++++++++++++++++++++++++++++++++++++ src/Lookups/show.jl | 12 ++++++- src/array/array.jl | 14 ++++++++- 5 files changed, 116 insertions(+), 4 deletions(-) diff --git a/src/Dimensions/format.jl b/src/Dimensions/format.jl index 1616b4ce3..47657c8f7 100644 --- a/src/Dimensions/format.jl +++ b/src/Dimensions/format.jl @@ -54,6 +54,9 @@ format(m::Lookup, D::Type, axis::AbstractRange) = format(m, D, parent(m), axis) format(v::AutoVal, D::Type, axis::AbstractRange) = _valformaterror(val(v), D) format(v, D::Type, axis::AbstractRange) = _valformaterror(v, D) +format(m::Lookups.MatrixLookup, D::Type, ::AutoValues, axis::AbstractRange) = + rebuild(m; dim=D(), data=axis) + # Format Lookups # No more identification required for NoLookup format(m::Lookups.Length1NoLookup, D::Type, values, axis::AbstractRange) = m diff --git a/src/Lookups/lookup_arrays.jl b/src/Lookups/lookup_arrays.jl index 9f69dc1cb..62b7cace4 100644 --- a/src/Lookups/lookup_arrays.jl +++ b/src/Lookups/lookup_arrays.jl @@ -1,4 +1,3 @@ - """ Lookup @@ -613,6 +612,33 @@ Base.:(==)(l1::Transformed, l2::Transformed) = typeof(l1) == typeof(l2) && f(l1) # TODO Transformed bounds +struct MatrixLookup{T,A,D,Ma<:AbstractMatrix{T},Me} <: Unaligned{T,1} + data::A + dim::D + matrix::Ma + metadata::Me +end +MatrixLookup(matrix; metadata=NoMetadata()) = + MatrixLookup(AutoValues(), AutoDim(), matrix, metadata) +dim(lookup::MatrixLookup) = lookup.dim +matrix(l::MatrixLookup) = l.matrix + +# An array to lazily combine dimension matrices into points +struct ArrayOfPoints{T,N,M,A<:AbstractArray{<:Any,N}} <: AbstractArray{T,N} + arrays::NTuple{M,A} + function ArrayOfPoints(arrays::NTuple{M,A}) where {M,A<:AbstractArray{T1,N}} where {T1,N} + all(x -> size(x) == size(first(arrays)), arrays) || + throw(ArgumentError("Size of matrices must match")) + T = NTuple{M,T1} + new{T,N,M,A}(arrays) + end +end +Base.size(aop::ArrayOfPoints) = size(first(aop.arrays)) +@propagate_inbounds function Base.getindex(aop::ArrayOfPoints, I::Int...) + @boundscheck checkbounds(first(aop.arrays), I...) + map(A -> (@inbounds A[I...]), aop.arrays) +end + # Shared methods intervalbounds(l::Lookup, args...) = _intervalbounds_no_interval_error() @@ -971,4 +997,4 @@ function promote_first(a1::AbstractArray, as::AbstractArray...) end return convert(C, a1) -end +end \ No newline at end of file diff --git a/src/Lookups/selector.jl b/src/Lookups/selector.jl index 35e5fe1e8..53c47c5b3 100644 --- a/src/Lookups/selector.jl +++ b/src/Lookups/selector.jl @@ -1102,6 +1102,67 @@ end function select_unalligned_indices(lookups::LookupTuple, sel::Tuple{Selector,Vararg{Selector}}) throw(ArgumentError("only `Near`, `At` or `Contains` selectors currently work on `Unalligned` lookups")) end +function select_unalligned_indices( + lookups::Tuple{<:MatrixLookup,<:MatrixLookup}, + selectors::Tuple{<:At,<:At} +) + # TODO: this is completely unoptimised + # It may be better to use some kind of spatial index + sa, sb = map(val, selectors) + ata, atb = map(atol, selectors) + A = ArrayOfPoints(map(matrix, lookups)) + i = findfirst(A) do (a, b) + isnothing(ata) ? a == sa : isapprox(a, sa; atol=ata) && + isnothing(atb) ? b == sb : isapprox(b, sb; atol=atb) + end + isnothing(i) && throw(ArgumentError("($sa, $sb) not found in lookup")) + return Tuple(CartesianIndices(A)[i]) +end +function select_unalligned_indices( + lookups::Tuple{<:MatrixLookup,<:MatrixLookup}, + selectors::Tuple{<:Near,<:Near} +) + sa, sb = map(val, selectors) + A = ArrayOfPoints(map(matrix, lookups)) + _, i = findmin(A) do (a, b) + abs(sa - a) + abs(sb - b) + end + return Tuple(CartesianIndices(A)[i]) +end +function select_unalligned_indices( + lookups::Tuple{<:MatrixLookup,<:MatrixLookup}, + selectors::Tuple{<:Near,<:At} +) + near, at = selectors + A = ArrayOfPoints(map(matrix, lookups)) + _, i = findmin(A) do (a, b) + _at_near_dist(at, near, b, a) + end + return Tuple(CartesianIndices(A)[i]) +end +function select_unalligned_indices( + lookups::Tuple{<:MatrixLookup,<:MatrixLookup}, + selectors::Tuple{<:At,<:Near} +) + at, near = selectors + A = ArrayOfPoints(map(matrix, lookups)) + _, i = findmin(A) do (a, b) + _at_near_dist(at, near, a, b) + end + return Tuple(CartesianIndices(A)[i]) +end + +function _at_near_dist(at, near, at_x, near_x) + atdist = if isnothing(atol(at)) + at_x == val(at) ? zero(at_x) : typemax(at_x) + else + isapprox(at_x, val(at); atol=atol(at)) ? zero(at_x) : typemax(at_x) + end + neardist = abs(val(near) - near_x) + atdist + neardist +end + + _transform2int(lookup::AbstractArray, ::Near, x) = min(max(round(Int, x), firstindex(lookup)), lastindex(lookup)) _transform2int(lookup::AbstractArray, ::Contains, x) = round(Int, x) diff --git a/src/Lookups/show.jl b/src/Lookups/show.jl index 1e35bc876..cc35047bb 100644 --- a/src/Lookups/show.jl +++ b/src/Lookups/show.jl @@ -8,10 +8,20 @@ function Base.show(io::IO, mime::MIME"text/plain", lookup::Transformed) show_compact(io, mime, lookup) show(io, mime, lookup.f) print(io, " ") - ctx = IOContext(io, :compact=>true) + ctx = IOContext(io, :compact => true) show(ctx, mime, dim(lookup)) end +function Base.show(io::IO, mime::MIME"text/plain", lookup::MatrixLookup) + show_compact(io, mime, lookup) + if !get(io, :compact, false) + println(io) + ctx = IOContext(io, :compact => true) + show(ctx, mime, lookup.matrix) + show(ctx, mime, dim(lookup)) + end +end + function Base.show(io::IO, mime::MIME"text/plain", lookup::Lookup) show_compact(io, mime, lookup) get(io, :compact, false) && print_index(io, mime, parent(lookup)) diff --git a/src/array/array.jl b/src/array/array.jl index d3e054c28..469808520 100644 --- a/src/array/array.jl +++ b/src/array/array.jl @@ -719,7 +719,8 @@ for f in (:fill, :rand) end end # AbstractRNG rand DimArray creation methods -Base.rand(r::AbstractRNG, x, d1::Dimension, dims::Dimension...; kw...) = rand(r, x, (d1, dims...); kw...) +Base.rand(r::AbstractRNG, x, d1::Dimension, dims::Dimension...; kw...) = + rand(r, x, (d1, dims...); kw...) function Base.rand(r::AbstractRNG, x, dims::DimTuple; kw...) C = dimconstructor(dims) C(rand(r, x, _dimlength(dims)), _maybestripval(dims); kw...) @@ -739,6 +740,17 @@ function Base.rand(r::AbstractRNG, ::Type{T}, dims::DimTuple; kw...) where T C(rand(r, T, _dimlength(dims)), _maybestripval(dims); kw...) end +function _dimlength( + dims::Tuple{<:Dimension{<:Lookups.MatrixLookup},Vararg{Dimension{<:Lookups.MatrixLookup}}} +) + lookups = lookup(dims) + sz1 = size(first(lookups).matrix) + foreach(lookups) do l + sz = size(l.matrix) + sz1 == sz || throw(ArgumentError("MatrixLookup matrix sizes must match. Got $sz1 and $sz")) + end + return sz1 +end _dimlength(dims::Tuple) = map(_dimlength, dims) _dimlength(dim::Dimension{<:AbstractArray}) = length(dim) _dimlength(dim::Dimension{<:Val{Keys}}) where Keys = length(Keys) From 7b4fdc09f09bc428278e26fee4bc77f1c5b110d5 Mon Sep 17 00:00:00 2001 From: Rafael Schouten Date: Sat, 11 Jan 2025 12:42:25 +0100 Subject: [PATCH 02/14] matrix lookup with tests on at and near --- src/Lookups/selector.jl | 37 ++++++++++--------------------------- test/selector.jl | 24 ++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 27 deletions(-) diff --git a/src/Lookups/selector.jl b/src/Lookups/selector.jl index 53c47c5b3..afc0f43e3 100644 --- a/src/Lookups/selector.jl +++ b/src/Lookups/selector.jl @@ -215,6 +215,8 @@ function at(::Order, ::Span, lookup::Lookup, selval, atol, rtol::Nothing; err=_T end end + +_is_at(at::At, v) = _is_at(val(at), v, atol(at)) @inline _is_at(x, y, atol) = x == y @inline _is_at(x::Dates.AbstractTime, y::Dates.AbstractTime, atol::Dates.Period) = x >= y - atol && x <= y + atol @@ -1108,14 +1110,11 @@ function select_unalligned_indices( ) # TODO: this is completely unoptimised # It may be better to use some kind of spatial index - sa, sb = map(val, selectors) - ata, atb = map(atol, selectors) A = ArrayOfPoints(map(matrix, lookups)) i = findfirst(A) do (a, b) - isnothing(ata) ? a == sa : isapprox(a, sa; atol=ata) && - isnothing(atb) ? b == sb : isapprox(b, sb; atol=atb) + _is_at(selectors[1], a) && _is_at(selectors[2], b) end - isnothing(i) && throw(ArgumentError("($sa, $sb) not found in lookup")) + isnothing(i) && throw(ArgumentError("$(map(val, selectors)) not found in lookup")) return Tuple(CartesianIndices(A)[i]) end function select_unalligned_indices( @@ -1131,39 +1130,23 @@ function select_unalligned_indices( end function select_unalligned_indices( lookups::Tuple{<:MatrixLookup,<:MatrixLookup}, - selectors::Tuple{<:Near,<:At} -) - near, at = selectors - A = ArrayOfPoints(map(matrix, lookups)) - _, i = findmin(A) do (a, b) - _at_near_dist(at, near, b, a) - end - return Tuple(CartesianIndices(A)[i]) -end -function select_unalligned_indices( - lookups::Tuple{<:MatrixLookup,<:MatrixLookup}, - selectors::Tuple{<:At,<:Near} + selectors::Union{Tuple{<:At,<:Near},Tuple{<:Near,<:At}} ) - at, near = selectors A = ArrayOfPoints(map(matrix, lookups)) _, i = findmin(A) do (a, b) - _at_near_dist(at, near, a, b) + _at_near_dist(selectors..., a, b) end return Tuple(CartesianIndices(A)[i]) end -function _at_near_dist(at, near, at_x, near_x) - atdist = if isnothing(atol(at)) - at_x == val(at) ? zero(at_x) : typemax(at_x) - else - isapprox(at_x, val(at); atol=atol(at)) ? zero(at_x) : typemax(at_x) - end +_at_near_dist(near::Near, at::At, near_x, at_x) = + _at_near_dist(at, near, at_x, near_x) +function _at_near_dist(at::At, near::Near, at_x, near_x) + atdist = _is_at(at, at_x) ? zero(at_x) : typemax(at_x) neardist = abs(val(near) - near_x) atdist + neardist end - - _transform2int(lookup::AbstractArray, ::Near, x) = min(max(round(Int, x), firstindex(lookup)), lastindex(lookup)) _transform2int(lookup::AbstractArray, ::Contains, x) = round(Int, x) _transform2int(lookup::AbstractArray, sel::At, x) = _transform2int(sel, x, atol(sel)) diff --git a/test/selector.jl b/test/selector.jl index ffcebb4b2..42618fc6e 100644 --- a/test/selector.jl +++ b/test/selector.jl @@ -1449,3 +1449,27 @@ end @test all(map(d -> !hasselection(d, Contains(400.0)), cases)) @test all(map(d -> hasselection(d, Near(0.0)), cases)) end + +@testset "MatrixLookup selectors" begin + y = -100:100 + x = -200:200 + xs = [x + 0.01y^3 for x in x, y in y] + ys = [y + 10cos(x/40) for x in x, y in y] + xdim = X(DD.Dimensions.Lookups.MatrixLookup(xs)) + ydim = Y(DD.Dimensions.Lookups.MatrixLookup(ys)) + A = rand(xdim, ydim) + xval = xs[end-10] + yval = ys[end-10] + @test A[Y=At(yval; atol=0.001), X=At(xval; atol=0.001)] == + A[Y=Near(yval), X=Near(xval)] == + A[Y=At(yval; atol=0.001), X=Near(xval)] == + A[Y=Near(yval), X=At(xval; atol=0.001)] == + A[end-10] + xval = xs[end-10] + 0.0005 + yval = ys[end-10] + 0.0005 + @test A[Y=At(yval; atol=0.001), X=At(xval; atol=0.001)] == + A[Y=Near(yval), X=Near(xval)] == + A[Y=At(yval; atol=0.001), X=Near(xval)] == + A[Y=Near(yval), X=At(xval; atol=0.001)] == + A[end-10] +end \ No newline at end of file From 42e6b47644dc9b128893b8ff4ec2d9d15791c2c4 Mon Sep 17 00:00:00 2001 From: Rafael Schouten Date: Sun, 12 Jan 2025 14:47:36 +0100 Subject: [PATCH 03/14] fix exports --- src/Lookups/Lookups.jl | 2 +- test/selector.jl | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Lookups/Lookups.jl b/src/Lookups/Lookups.jl index e5f718208..062681162 100644 --- a/src/Lookups/Lookups.jl +++ b/src/Lookups/Lookups.jl @@ -54,7 +54,7 @@ export AutoStep, AutoBounds, AutoValues export Lookup export AutoLookup, AbstractNoLookup, NoLookup export Aligned, AbstractSampled, Sampled, AbstractCyclic, Cyclic, AbstractCategorical, Categorical -export Unaligned, Transformed +export Unaligned, Transformed, MatrixLookup # Deprecated export LookupArray diff --git a/test/selector.jl b/test/selector.jl index 42618fc6e..0bab23389 100644 --- a/test/selector.jl +++ b/test/selector.jl @@ -1451,12 +1451,14 @@ end end @testset "MatrixLookup selectors" begin + # Generate a warped matrix y = -100:100 x = -200:200 xs = [x + 0.01y^3 for x in x, y in y] ys = [y + 10cos(x/40) for x in x, y in y] - xdim = X(DD.Dimensions.Lookups.MatrixLookup(xs)) - ydim = Y(DD.Dimensions.Lookups.MatrixLookup(ys)) + # Define x and y lookup dimensions + xdim = X(MatrixLookup(xs)) + ydim = Y(MatrixLookup(ys)) A = rand(xdim, ydim) xval = xs[end-10] yval = ys[end-10] From 29b55aa755cb69cba8d0d88810b8067b97b0926c Mon Sep 17 00:00:00 2001 From: Rafael Schouten Date: Sun, 12 Jan 2025 20:57:21 +0100 Subject: [PATCH 04/14] bugfix constructor --- src/Lookups/lookup_arrays.jl | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/Lookups/lookup_arrays.jl b/src/Lookups/lookup_arrays.jl index 62b7cace4..210334bf2 100644 --- a/src/Lookups/lookup_arrays.jl +++ b/src/Lookups/lookup_arrays.jl @@ -624,12 +624,14 @@ dim(lookup::MatrixLookup) = lookup.dim matrix(l::MatrixLookup) = l.matrix # An array to lazily combine dimension matrices into points -struct ArrayOfPoints{T,N,M,A<:AbstractArray{<:Any,N}} <: AbstractArray{T,N} - arrays::NTuple{M,A} - function ArrayOfPoints(arrays::NTuple{M,A}) where {M,A<:AbstractArray{T1,N}} where {T1,N} +struct ArrayOfPoints{T,N,M,A<:Tuple{AbstractArray{<:Any,N},Vararg}} <: AbstractArray{T,N} + arrays::A + function ArrayOfPoints(arrays::A) where A<:Tuple all(x -> size(x) == size(first(arrays)), arrays) || throw(ArgumentError("Size of matrices must match")) - T = NTuple{M,T1} + T = Tuple{map(eltype, arrays)...} + N = ndims(first(arrays)) + M = length(arrays) new{T,N,M,A}(arrays) end end From 0262559e496890c2165ff37e6bb9e6b372bd0f6d Mon Sep 17 00:00:00 2001 From: Rafael Schouten Date: Fri, 31 Jan 2025 00:59:22 +0100 Subject: [PATCH 05/14] Add NearestNeighbors.jl extension --- .gitignore | 6 +-- ext/DimensionalDataNearestNeighborsExt.jl | 21 +++++++++ src/Lookups/lookup_arrays.jl | 13 +++--- src/Lookups/selector.jl | 53 +++++++---------------- 4 files changed, 46 insertions(+), 47 deletions(-) create mode 100644 ext/DimensionalDataNearestNeighborsExt.jl diff --git a/.gitignore b/.gitignore index d74ac92de..841105f41 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ -*.jl.*.cov -*.jl.cov -*.jl.mem +*.cov +*.cov +*.mem .DS_Store /Manifest.toml diff --git a/ext/DimensionalDataNearestNeighborsExt.jl b/ext/DimensionalDataNearestNeighborsExt.jl new file mode 100644 index 000000000..87781dc1b --- /dev/null +++ b/ext/DimensionalDataNearestNeighborsExt.jl @@ -0,0 +1,21 @@ +module DimensionalDataNearestNeighborsExt + +using DimensionalData +using NearestNeighbors +using NearestNeighbors.StaticArrays + +const DD = DimensionalData + +function select_array_lookups( + lookups::Tuple{<:ArrayLookup,<:ArrayLookup,Vararg{ArrayLookup}}, + selectors::Tuple{<:Union{At,Near},<:Union{Near,At},Vararg{Union{Near,At}}} +) + vals = map(val, selectors) + idx, dists = nn(tree(first(lookups)), SVector(vals)) + map(selectors, dists) do s, d + s isa At ? (d == zero(first(dists))) : true + end |> all || throw(ArgumentError("$(vals) not found in lookup")) + return idx +end + +end \ No newline at end of file diff --git a/src/Lookups/lookup_arrays.jl b/src/Lookups/lookup_arrays.jl index 210334bf2..71a9db714 100644 --- a/src/Lookups/lookup_arrays.jl +++ b/src/Lookups/lookup_arrays.jl @@ -611,17 +611,18 @@ transformfunc(lookup::Transformed) = lookup.f Base.:(==)(l1::Transformed, l2::Transformed) = typeof(l1) == typeof(l2) && f(l1) == f(l2) # TODO Transformed bounds - -struct MatrixLookup{T,A,D,Ma<:AbstractMatrix{T},Me} <: Unaligned{T,1} +struct ArrayLookup{T,A,D,Ma<:AbstractArray{T},Tr,Me} <: Unaligned{T,1} data::A dim::D matrix::Ma + tree::Tr metadata::Me end -MatrixLookup(matrix; metadata=NoMetadata()) = - MatrixLookup(AutoValues(), AutoDim(), matrix, metadata) -dim(lookup::MatrixLookup) = lookup.dim -matrix(l::MatrixLookup) = l.matrix +ArrayLookup(matrix; metadata=NoMetadata()) = + ArrayLookup(AutoValues(), AutoDim(), matrix, nothing, metadata) +dim(lookup::ArrayLookup) = lookup.dim +matrix(l::ArrayLookup) = l.matrix +tree(l::ArrayLookup) = l.matrix # An array to lazily combine dimension matrices into points struct ArrayOfPoints{T,N,M,A<:Tuple{AbstractArray{<:Any,N},Vararg}} <: AbstractArray{T,N} diff --git a/src/Lookups/selector.jl b/src/Lookups/selector.jl index afc0f43e3..d4861d345 100644 --- a/src/Lookups/selector.jl +++ b/src/Lookups/selector.jl @@ -1097,54 +1097,31 @@ end # We use the transformation from the first unaligned dim. # In practice the others could be empty. -function select_unalligned_indices(lookups::LookupTuple, sel::Tuple{IntSelector,Vararg{IntSelector}}) - transformed = transformfunc(lookups[1])(map(val, sel)) - map(_transform2int, lookups, sel, transformed) -end -function select_unalligned_indices(lookups::LookupTuple, sel::Tuple{Selector,Vararg{Selector}}) - throw(ArgumentError("only `Near`, `At` or `Contains` selectors currently work on `Unalligned` lookups")) -end function select_unalligned_indices( - lookups::Tuple{<:MatrixLookup,<:MatrixLookup}, - selectors::Tuple{<:At,<:At} + lookups::LookupTuple, sel::Tuple{IntSelector,Vararg{IntSelector}} ) - # TODO: this is completely unoptimised - # It may be better to use some kind of spatial index - A = ArrayOfPoints(map(matrix, lookups)) - i = findfirst(A) do (a, b) - _is_at(selectors[1], a) && _is_at(selectors[2], b) - end - isnothing(i) && throw(ArgumentError("$(map(val, selectors)) not found in lookup")) - return Tuple(CartesianIndices(A)[i]) + transformed = transformfunc(lookups[1])(map(val, sel)) + map(_transform2int, lookups, sel, transformed) end function select_unalligned_indices( - lookups::Tuple{<:MatrixLookup,<:MatrixLookup}, - selectors::Tuple{<:Near,<:Near} + lookups::LookupTuple, sel::Tuple{Selector,Vararg{Selector}} ) - sa, sb = map(val, selectors) - A = ArrayOfPoints(map(matrix, lookups)) - _, i = findmin(A) do (a, b) - abs(sa - a) + abs(sb - b) - end - return Tuple(CartesianIndices(A)[i]) + throw(ArgumentError("only `Near`, `At` or `Contains` selectors currently work on `Unalligned` lookups")) end function select_unalligned_indices( - lookups::Tuple{<:MatrixLookup,<:MatrixLookup}, - selectors::Union{Tuple{<:At,<:Near},Tuple{<:Near,<:At}} + lookups::Tuple{<:ArrayLookup,<:ArrayLookup,Vararg{ArrayLookup}}, + selectors::Tuple ) - A = ArrayOfPoints(map(matrix, lookups)) - _, i = findmin(A) do (a, b) - _at_near_dist(selectors..., a, b) - end - return Tuple(CartesianIndices(A)[i]) + select_array_lookups(lookups, selectors) end -_at_near_dist(near::Near, at::At, near_x, at_x) = - _at_near_dist(at, near, at_x, near_x) -function _at_near_dist(at::At, near::Near, at_x, near_x) - atdist = _is_at(at, at_x) ? zero(at_x) : typemax(at_x) - neardist = abs(val(near) - near_x) - atdist + neardist +# This implementation is extremely slow, +# it's expected user will use the NearestNeighbors.jl extension +function select_array_lookups( + lookups::Tuple{<:ArrayLookup,<:ArrayLookup,Vararg{ArrayLookup}}, + selectors::Tuple +) + throw(ArgumentError("Load NearestNeighbors.jl to use `At` on `ArrayLookup`s")) end _transform2int(lookup::AbstractArray, ::Near, x) = min(max(round(Int, x), firstindex(lookup)), lastindex(lookup)) From 57a26bfc18bcea37e45ac929609abb7c5962427b Mon Sep 17 00:00:00 2001 From: Rafael Schouten Date: Fri, 31 Jan 2025 10:44:27 +0100 Subject: [PATCH 06/14] abstract with_alignments --- ext/DimensionalDataNearestNeighborsExt.jl | 11 ++++++- src/Dimensions/format.jl | 9 ++++-- src/Dimensions/indexing.jl | 35 +++++++++++++---------- 3 files changed, 37 insertions(+), 18 deletions(-) diff --git a/ext/DimensionalDataNearestNeighborsExt.jl b/ext/DimensionalDataNearestNeighborsExt.jl index 87781dc1b..a3a9664c9 100644 --- a/ext/DimensionalDataNearestNeighborsExt.jl +++ b/ext/DimensionalDataNearestNeighborsExt.jl @@ -6,7 +6,7 @@ using NearestNeighbors.StaticArrays const DD = DimensionalData -function select_array_lookups( +function DD.Lookups.select_array_lookups( lookups::Tuple{<:ArrayLookup,<:ArrayLookup,Vararg{ArrayLookup}}, selectors::Tuple{<:Union{At,Near},<:Union{Near,At},Vararg{Union{Near,At}}} ) @@ -18,4 +18,13 @@ function select_array_lookups( return idx end +function DD.Dimensions.format_unaligned( + lookups::Tuple{<:ArrayLookup,<:ArrayLookup,Vararg{ArrayLookup}}, dims, +) + tree = knntree(ArrayOfPoints(map(matrix, lookups))) + return map(lookups, dims) do l, d + rebuild(d, rebuild(l; tree)) + end +end + end \ No newline at end of file diff --git a/src/Dimensions/format.jl b/src/Dimensions/format.jl index 47657c8f7..a6f403709 100644 --- a/src/Dimensions/format.jl +++ b/src/Dimensions/format.jl @@ -35,11 +35,16 @@ end format(dims::Tuple{Vararg{Any,N}}, A::AbstractArray{<:Any,N}) where N = format(dims, axes(A)) @noinline format(dims::Tuple{Vararg{Any,M}}, A::AbstractArray{<:Any,N}) where {N,M} = throw(DimensionMismatch("Array A has $N axes, while the number of dims is $M: $(map(basetypeof, dims))")) -format(dims::Tuple{Vararg{Any,N}}, axes::Tuple{Vararg{Any,N}}) where N = map(_format, dims, axes) +function format(dims::Tuple{Vararg{Any,N}}, axes::Tuple{Vararg{Any,N}}) where N + with_alignments(format, format_unaligned, newdims, axes) +end format(d::Dimension{<:AbstractArray}) = _format(d, axes(val(d), 1)) format(d::Dimension, axis::AbstractRange) = _format(d, axis) format(l::Lookup) = parent(format(AnonDim(l))) +# Fallback +format_unaligned(dims::DimTuple, axes) = dims + _format(dimname::Symbol, axis::AbstractRange) = Dim{dimname}(NoLookup(axes(axis, 1))) _format(::Type{D}, axis::AbstractRange) where D<:Dimension = D(NoLookup(axes(axis, 1))) _format(dim::Dimension{Colon}, axis::AbstractRange) = rebuild(dim, NoLookup(axes(axis, 1))) @@ -54,7 +59,7 @@ format(m::Lookup, D::Type, axis::AbstractRange) = format(m, D, parent(m), axis) format(v::AutoVal, D::Type, axis::AbstractRange) = _valformaterror(val(v), D) format(v, D::Type, axis::AbstractRange) = _valformaterror(v, D) -format(m::Lookups.MatrixLookup, D::Type, ::AutoValues, axis::AbstractRange) = +format(m::Lookups.ArrayLookup, D::Type, ::AutoValues, axis::AbstractRange) = rebuild(m; dim=D(), data=axis) # Format Lookups diff --git a/src/Dimensions/indexing.jl b/src/Dimensions/indexing.jl index 1dbca4179..168d9b9c8 100644 --- a/src/Dimensions/indexing.jl +++ b/src/Dimensions/indexing.jl @@ -47,12 +47,26 @@ Convert a `Dimension` or `Selector` `I` to indices of `Int`, `AbstractArray` or @inline function dims2indices(dims::DimTuple, I::DimTuple) extradims = otherdims(I, dims) length(extradims) > 0 && _extradimswarn(extradims) - _dims2indices(lookup(dims), dims, sortdims(I, dims)) + return with_alignements(dims2indices, unalligned_dims2indices, dims, I) end +@inline dims2indices(dims::Tuple{}, ::Tuple{}) = () -# Handle tuples with @generated -@inline _dims2indices(::Tuple{}, dims::Tuple{}, ::Tuple{}) = () -@generated function _dims2indices(lookups::Tuple, dims::Tuple, I::Tuple) +@inline function unalligned_dims2indices(dims::DimTuple, sel::Tuple) + map(sel) do s + s isa Union{Selector,Interval} && _unalligned_all_selector_error(dims) + isnothing(s) ? Colon() : s + end +end +@inline function unalligned_dims2indices(dims::DimTuple, sel::Tuple{Selector,Vararg{Selector}}) + Lookups.select_unalligned_indices(lookup(dims), sel) +end + +# Run fa on each aligned dimension d[n] and indices i[n], +# and fu on grouped unaligned dimensions and I. +# The result is the updated dimensions, but in the original order +@generated function with_alignments( + fa, fu, lookups::Tuple, dims::Tuple, I::Tuple +) # We separate out Aligned and Unaligned lookups as # Unaligned must be selected in groups e.g. X and Y together. unalligned = Expr(:tuple) @@ -68,7 +82,7 @@ end push!(dimmerge.args, :(uadims[$ua_count])) else a_count += 1 - push!(alligned.args, :(_dims2indices(dims[$i], I[$i]))) + push!(alligned.args, :(fa(dims[$i], I[$i]))) # Update the merged tuple push!(dimmerge.args, :(adims[$a_count])) end @@ -80,7 +94,7 @@ end quote adims = $alligned # Unaligned dims have to be run together as a set - uadims = unalligned_dims2indices($unalligned, map(_unwrapdim, $uaI)) + uadims = fu($unalligned, map(_unwrapdim, $uaI)) $dimmerge end else @@ -88,15 +102,6 @@ end end end -@inline function unalligned_dims2indices(dims::DimTuple, sel::Tuple) - map(sel) do s - s isa Union{Selector,Interval} && _unalligned_all_selector_error(dims) - isnothing(s) ? Colon() : s - end -end -@inline function unalligned_dims2indices(dims::DimTuple, sel::Tuple{Selector,Vararg{Selector}}) - Lookups.select_unalligned_indices(lookup(dims), sel) -end _unalligned_all_selector_error(dims) = throw(ArgumentError("Unalligned dims: use selectors for all $(join(map(name, dims), ", ")) dims, or none of them")) From 68339dfb55b2a6398a3f63a30fcb5d77588cc18d Mon Sep 17 00:00:00 2001 From: Rafael Schouten Date: Sat, 1 Feb 2025 12:22:18 +0100 Subject: [PATCH 07/14] finish knn --- Project.toml | 2 + ext/DimensionalDataNearestNeighborsExt.jl | 55 ++++++++++++++++++----- src/Dimensions/format.jl | 9 ++-- src/Dimensions/indexing.jl | 7 ++- src/Lookups/Lookups.jl | 2 +- src/Lookups/lookup_arrays.jl | 27 +++-------- src/Lookups/selector.jl | 7 ++- src/Lookups/show.jl | 2 +- src/array/array.jl | 4 +- test/selector.jl | 12 +++-- 10 files changed, 78 insertions(+), 49 deletions(-) diff --git a/Project.toml b/Project.toml index 40d25a21f..e9a57cf18 100644 --- a/Project.toml +++ b/Project.toml @@ -28,6 +28,7 @@ AlgebraOfGraphics = "cbdf2221-f076-402e-a563-3d30da359d67" CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597" DiskArrays = "3c3547ce-8d99-4f5e-a174-61eb10b00ae3" Makie = "ee78f7c6-11fb-53f2-987a-cfe4a2b5a57a" +NearestNeighbors = "b8a86587-4115-5ab1-83bc-aa920d37bbce" PythonCall = "6099a3de-0909-46bc-b1f4-468b9a2dfc0d" StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" @@ -36,6 +37,7 @@ DimensionalDataAlgebraOfGraphicsExt = "AlgebraOfGraphics" DimensionalDataCategoricalArraysExt = "CategoricalArrays" DimensionalDataDiskArraysExt = "DiskArrays" DimensionalDataMakie = "Makie" +DimensionalDataNearestNeighborsExt = "NearestNeighbors" DimensionalDataPythonCall = "PythonCall" DimensionalDataStatsBase = "StatsBase" diff --git a/ext/DimensionalDataNearestNeighborsExt.jl b/ext/DimensionalDataNearestNeighborsExt.jl index a3a9664c9..bebd978c6 100644 --- a/ext/DimensionalDataNearestNeighborsExt.jl +++ b/ext/DimensionalDataNearestNeighborsExt.jl @@ -3,28 +3,63 @@ module DimensionalDataNearestNeighborsExt using DimensionalData using NearestNeighbors using NearestNeighbors.StaticArrays +using DimensionalData.Lookups +using DimensionalData.Dimensions + +using DimensionalData.Lookups: ArrayLookup, VecOfPoints, matrix, atol const DD = DimensionalData +const NN = NearestNeighbors function DD.Lookups.select_array_lookups( lookups::Tuple{<:ArrayLookup,<:ArrayLookup,Vararg{ArrayLookup}}, selectors::Tuple{<:Union{At,Near},<:Union{Near,At},Vararg{Union{Near,At}}} ) - vals = map(val, selectors) - idx, dists = nn(tree(first(lookups)), SVector(vals)) - map(selectors, dists) do s, d - s isa At ? (d == zero(first(dists))) : true - end |> all || throw(ArgumentError("$(vals) not found in lookup")) - return idx + f1 = first(lookups) + vals = SVector(map(val, selectors)) + tree = Lookups.tree(f1) + knn!(f1.idxvec, f1.distvec, tree, vals, 1) + idx = f1.idxvec[1] + found_vals = tree.data[idx] + # @show idx vals found_vals + map(selectors, Tuple(found_vals)) do s, t + s isa At ? isapprox(val(s), t; atol=atol(s)) : true + end |> all || throw(ArgumentError("$(selectors) not found in lookup")) + return CartesianIndices(matrix(first(lookups)))[idx] |> Tuple end function DD.Dimensions.format_unaligned( - lookups::Tuple{<:ArrayLookup,<:ArrayLookup,Vararg{ArrayLookup}}, dims, + lookups::Tuple{<:ArrayLookup,<:ArrayLookup,Vararg{ArrayLookup}}, dims, axes, ) - tree = knntree(ArrayOfPoints(map(matrix, lookups))) - return map(lookups, dims) do l, d - rebuild(d, rebuild(l; tree)) + points = vec(SVector.(zip(map(matrix, lookups)...))) + idxvec = Vector{Int}(undef, 1) + distvec = Vector{NN.get_T(eltype(points))}(undef, 1) + tree = NN.KDTree(points, NN.Euclidean(); reorder=false) + return map(lookups, dims, axes) do l, d, a + newl = rebuild(l; + data=a, tree, dim=basedims(d), dims=basedims(dims), idxvec, distvec + ) + rebuild(d, newl) end end + +# An array to lazily combine dimension matrices into points +struct VecOfPoints{T,M,A<:Tuple{AbstractArray{<:Any},Vararg}} <: AbstractArray{T,1} + arrays::A +end +function VecOfPoints(arrays::A) where A<:Tuple + all(x -> size(x) == size(first(arrays)), arrays) || + throw(ArgumentError("Size of matrices must match")) + T = typeof(SVector(map(A -> zero(eltype(A)), arrays))) + M = length(arrays) + VecOfPoints{T,M,A}(arrays) +end + +Base.size(aop::VecOfPoints) = size(first(aop.arrays)) +Base.@propagate_inbounds function Base.getindex(aop::VecOfPoints, I::Int...) + @boundscheck checkbounds(first(aop.arrays), I...) + SVector(map(A -> (@inbounds A[I...]), aop.arrays)) +end + end \ No newline at end of file diff --git a/src/Dimensions/format.jl b/src/Dimensions/format.jl index a6f403709..735224034 100644 --- a/src/Dimensions/format.jl +++ b/src/Dimensions/format.jl @@ -35,15 +35,16 @@ end format(dims::Tuple{Vararg{Any,N}}, A::AbstractArray{<:Any,N}) where N = format(dims, axes(A)) @noinline format(dims::Tuple{Vararg{Any,M}}, A::AbstractArray{<:Any,N}) where {N,M} = throw(DimensionMismatch("Array A has $N axes, while the number of dims is $M: $(map(basetypeof, dims))")) -function format(dims::Tuple{Vararg{Any,N}}, axes::Tuple{Vararg{Any,N}}) where N - with_alignments(format, format_unaligned, newdims, axes) -end +format(dims::Tuple{Vararg{Any,N}}, axes::Tuple{Vararg{Any,N}}) where N = + split_alignments(format, format_unaligned, dims, axes) format(d::Dimension{<:AbstractArray}) = _format(d, axes(val(d), 1)) format(d::Dimension, axis::AbstractRange) = _format(d, axis) format(l::Lookup) = parent(format(AnonDim(l))) # Fallback -format_unaligned(dims::DimTuple, axes) = dims +function format_unaligned end +format_unaligned(dims::DimTuple, axes) = format_unaligned(val(dims), dims, axes) +format_unaligned(::Tuple, dims::DimTuple, axes) = map(format, dims, axes) _format(dimname::Symbol, axis::AbstractRange) = Dim{dimname}(NoLookup(axes(axis, 1))) _format(::Type{D}, axis::AbstractRange) where D<:Dimension = D(NoLookup(axes(axis, 1))) diff --git a/src/Dimensions/indexing.jl b/src/Dimensions/indexing.jl index 168d9b9c8..e4ed63538 100644 --- a/src/Dimensions/indexing.jl +++ b/src/Dimensions/indexing.jl @@ -47,7 +47,8 @@ Convert a `Dimension` or `Selector` `I` to indices of `Int`, `AbstractArray` or @inline function dims2indices(dims::DimTuple, I::DimTuple) extradims = otherdims(I, dims) length(extradims) > 0 && _extradimswarn(extradims) - return with_alignements(dims2indices, unalligned_dims2indices, dims, I) + Isorted = Dimensions.dims(I, dims) + return split_alignments(dims2indices, unalligned_dims2indices, dims, Isorted) end @inline dims2indices(dims::Tuple{}, ::Tuple{}) = () @@ -64,7 +65,9 @@ end # Run fa on each aligned dimension d[n] and indices i[n], # and fu on grouped unaligned dimensions and I. # The result is the updated dimensions, but in the original order -@generated function with_alignments( +split_alignments(fa, fu, dims::Tuple, I::Tuple) = + split_alignments(fa, fu, val(dims), dims, I) +@generated function split_alignments( fa, fu, lookups::Tuple, dims::Tuple, I::Tuple ) # We separate out Aligned and Unaligned lookups as diff --git a/src/Lookups/Lookups.jl b/src/Lookups/Lookups.jl index 062681162..fb7220d87 100644 --- a/src/Lookups/Lookups.jl +++ b/src/Lookups/Lookups.jl @@ -54,7 +54,7 @@ export AutoStep, AutoBounds, AutoValues export Lookup export AutoLookup, AbstractNoLookup, NoLookup export Aligned, AbstractSampled, Sampled, AbstractCyclic, Cyclic, AbstractCategorical, Categorical -export Unaligned, Transformed, MatrixLookup +export Unaligned, Transformed, ArrayLookup # Deprecated export LookupArray diff --git a/src/Lookups/lookup_arrays.jl b/src/Lookups/lookup_arrays.jl index 71a9db714..7043d841f 100644 --- a/src/Lookups/lookup_arrays.jl +++ b/src/Lookups/lookup_arrays.jl @@ -611,36 +611,21 @@ transformfunc(lookup::Transformed) = lookup.f Base.:(==)(l1::Transformed, l2::Transformed) = typeof(l1) == typeof(l2) && f(l1) == f(l2) # TODO Transformed bounds -struct ArrayLookup{T,A,D,Ma<:AbstractArray{T},Tr,Me} <: Unaligned{T,1} +struct ArrayLookup{T,A,D,Ds,Ma<:AbstractArray{T},Tr,IV,DV,Me} <: Unaligned{T,1} data::A dim::D + dims::Ds matrix::Ma tree::Tr + idxvec::IV + distvec::DV metadata::Me end ArrayLookup(matrix; metadata=NoMetadata()) = - ArrayLookup(AutoValues(), AutoDim(), matrix, nothing, metadata) + ArrayLookup(AutoValues(), AutoDim(), AutoDim(), matrix, nothing, nothing, nothing, metadata) dim(lookup::ArrayLookup) = lookup.dim matrix(l::ArrayLookup) = l.matrix -tree(l::ArrayLookup) = l.matrix - -# An array to lazily combine dimension matrices into points -struct ArrayOfPoints{T,N,M,A<:Tuple{AbstractArray{<:Any,N},Vararg}} <: AbstractArray{T,N} - arrays::A - function ArrayOfPoints(arrays::A) where A<:Tuple - all(x -> size(x) == size(first(arrays)), arrays) || - throw(ArgumentError("Size of matrices must match")) - T = Tuple{map(eltype, arrays)...} - N = ndims(first(arrays)) - M = length(arrays) - new{T,N,M,A}(arrays) - end -end -Base.size(aop::ArrayOfPoints) = size(first(aop.arrays)) -@propagate_inbounds function Base.getindex(aop::ArrayOfPoints, I::Int...) - @boundscheck checkbounds(first(aop.arrays), I...) - map(A -> (@inbounds A[I...]), aop.arrays) -end +tree(l::ArrayLookup) = l.tree # Shared methods diff --git a/src/Lookups/selector.jl b/src/Lookups/selector.jl index d4861d345..5ced44120 100644 --- a/src/Lookups/selector.jl +++ b/src/Lookups/selector.jl @@ -133,12 +133,11 @@ end function at(lookup::NoLookup, sel::At; err=_True(), kw...) v = val(sel) r = round(Int, v) - at = atol(sel) - if isnothing(at) + if isnothing(atol(sel)) v == r || _selnotfound_or_nothing(err, lookup, v) else at >= 0.5 && error("atol must be small than 0.5 for NoLookup") - isapprox(v, r; atol=at) || _selnotfound_or_nothing(err, lookup, v) + isapprox(v, r; atol=atol(at)) || _selnotfound_or_nothing(err, lookup, v) end if r in lookup return r @@ -1110,7 +1109,7 @@ function select_unalligned_indices( end function select_unalligned_indices( lookups::Tuple{<:ArrayLookup,<:ArrayLookup,Vararg{ArrayLookup}}, - selectors::Tuple + selectors::Tuple{<:IntSelector,<:IntSelector,Vararg{IntSelector}} ) select_array_lookups(lookups, selectors) end diff --git a/src/Lookups/show.jl b/src/Lookups/show.jl index cc35047bb..1432cde74 100644 --- a/src/Lookups/show.jl +++ b/src/Lookups/show.jl @@ -12,7 +12,7 @@ function Base.show(io::IO, mime::MIME"text/plain", lookup::Transformed) show(ctx, mime, dim(lookup)) end -function Base.show(io::IO, mime::MIME"text/plain", lookup::MatrixLookup) +function Base.show(io::IO, mime::MIME"text/plain", lookup::ArrayLookup) show_compact(io, mime, lookup) if !get(io, :compact, false) println(io) diff --git a/src/array/array.jl b/src/array/array.jl index 469808520..0b776fa67 100644 --- a/src/array/array.jl +++ b/src/array/array.jl @@ -741,13 +741,13 @@ function Base.rand(r::AbstractRNG, ::Type{T}, dims::DimTuple; kw...) where T end function _dimlength( - dims::Tuple{<:Dimension{<:Lookups.MatrixLookup},Vararg{Dimension{<:Lookups.MatrixLookup}}} + dims::Tuple{<:Dimension{<:Lookups.ArrayLookup},Vararg{Dimension{<:Lookups.ArrayLookup}}} ) lookups = lookup(dims) sz1 = size(first(lookups).matrix) foreach(lookups) do l sz = size(l.matrix) - sz1 == sz || throw(ArgumentError("MatrixLookup matrix sizes must match. Got $sz1 and $sz")) + sz1 == sz || throw(ArgumentError("ArrayLookup matrix sizes must match. Got $sz1 and $sz")) end return sz1 end diff --git a/test/selector.jl b/test/selector.jl index 0bab23389..f7c19856c 100644 --- a/test/selector.jl +++ b/test/selector.jl @@ -1310,7 +1310,7 @@ end @test @inferred da[Contains([1, 3]), Near([2, 3, 4])] == [2 3 4; 10 11 12] end -@testset "Selectors on TranformedIndex" begin +@testset "Selectors on Tranformed lookup" begin using CoordinateTransformations m = LinearMap([0.5 0.0; 0.0 0.5]) @@ -1450,22 +1450,26 @@ end @test all(map(d -> hasselection(d, Near(0.0)), cases)) end -@testset "MatrixLookup selectors" begin +@testset "ArrayLookup selectors" begin # Generate a warped matrix y = -100:100 x = -200:200 xs = [x + 0.01y^3 for x in x, y in y] ys = [y + 10cos(x/40) for x in x, y in y] # Define x and y lookup dimensions - xdim = X(MatrixLookup(xs)) - ydim = Y(MatrixLookup(ys)) + using NearestNeighbors + xdim = X(ArrayLookup(xs)) + ydim = Y(ArrayLookup(ys)) A = rand(xdim, ydim) + l = lookup(A, X) + l.dims xval = xs[end-10] yval = ys[end-10] @test A[Y=At(yval; atol=0.001), X=At(xval; atol=0.001)] == A[Y=Near(yval), X=Near(xval)] == A[Y=At(yval; atol=0.001), X=Near(xval)] == A[Y=Near(yval), X=At(xval; atol=0.001)] == + A[X=At(xval; atol=0.001), Y=Near(yval)] == A[end-10] xval = xs[end-10] + 0.0005 yval = ys[end-10] + 0.0005 From c1e9f81b90d0a109a845c6a6786a4d9e05a31ac8 Mon Sep 17 00:00:00 2001 From: Rafael Schouten Date: Sat, 1 Feb 2025 12:31:14 +0100 Subject: [PATCH 08/14] bugfix --- ext/DimensionalDataNearestNeighborsExt.jl | 26 +++-------------------- 1 file changed, 3 insertions(+), 23 deletions(-) diff --git a/ext/DimensionalDataNearestNeighborsExt.jl b/ext/DimensionalDataNearestNeighborsExt.jl index bebd978c6..fa09d8217 100644 --- a/ext/DimensionalDataNearestNeighborsExt.jl +++ b/ext/DimensionalDataNearestNeighborsExt.jl @@ -6,7 +6,7 @@ using NearestNeighbors.StaticArrays using DimensionalData.Lookups using DimensionalData.Dimensions -using DimensionalData.Lookups: ArrayLookup, VecOfPoints, matrix, atol +using DimensionalData.Lookups: ArrayLookup, matrix, atol const DD = DimensionalData const NN = NearestNeighbors @@ -21,15 +21,14 @@ function DD.Lookups.select_array_lookups( knn!(f1.idxvec, f1.distvec, tree, vals, 1) idx = f1.idxvec[1] found_vals = tree.data[idx] - # @show idx vals found_vals map(selectors, Tuple(found_vals)) do s, t - s isa At ? isapprox(val(s), t; atol=atol(s)) : true + s isa At ? Lookups._is_at(s, t) : true end |> all || throw(ArgumentError("$(selectors) not found in lookup")) return CartesianIndices(matrix(first(lookups)))[idx] |> Tuple end function DD.Dimensions.format_unaligned( - lookups::Tuple{<:ArrayLookup,<:ArrayLookup,Vararg{ArrayLookup}}, dims, axes, + lookups::Tuple{<:ArrayLookup,<:ArrayLookup,Vararg{ArrayLookup}}, dims::DD.DimTuple, axes, ) points = vec(SVector.(zip(map(matrix, lookups)...))) idxvec = Vector{Int}(undef, 1) @@ -43,23 +42,4 @@ function DD.Dimensions.format_unaligned( end end - -# An array to lazily combine dimension matrices into points -struct VecOfPoints{T,M,A<:Tuple{AbstractArray{<:Any},Vararg}} <: AbstractArray{T,1} - arrays::A -end -function VecOfPoints(arrays::A) where A<:Tuple - all(x -> size(x) == size(first(arrays)), arrays) || - throw(ArgumentError("Size of matrices must match")) - T = typeof(SVector(map(A -> zero(eltype(A)), arrays))) - M = length(arrays) - VecOfPoints{T,M,A}(arrays) -end - -Base.size(aop::VecOfPoints) = size(first(aop.arrays)) -Base.@propagate_inbounds function Base.getindex(aop::VecOfPoints, I::Int...) - @boundscheck checkbounds(first(aop.arrays), I...) - SVector(map(A -> (@inbounds A[I...]), aop.arrays)) -end - end \ No newline at end of file From 475958450c425919deac1008785180360cbf52b9 Mon Sep 17 00:00:00 2001 From: Rafael Schouten Date: Sat, 1 Feb 2025 17:18:06 +0100 Subject: [PATCH 09/14] bugfixes --- src/Dimensions/format.jl | 14 +++++++++++--- src/Dimensions/indexing.jl | 2 +- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/Dimensions/format.jl b/src/Dimensions/format.jl index 735224034..d46da79fd 100644 --- a/src/Dimensions/format.jl +++ b/src/Dimensions/format.jl @@ -32,13 +32,21 @@ function format(dims::DimTuple) A = CartesianIndices(ax) return format(dims, A) end -format(dims::Tuple{Vararg{Any,N}}, A::AbstractArray{<:Any,N}) where N = format(dims, axes(A)) +format(dims::Tuple{Vararg{Any,N}}, A::AbstractArray{<:Any,N}) where N = + format(dims, axes(A)) +format(dims::Tuple, A::AbstractArray) = + throw(ArgumentError("length of dims $(length(dims)) does not match array dimensionality $(ndims(A))")) @noinline format(dims::Tuple{Vararg{Any,M}}, A::AbstractArray{<:Any,N}) where {N,M} = throw(DimensionMismatch("Array A has $N axes, while the number of dims is $M: $(map(basetypeof, dims))")) -format(dims::Tuple{Vararg{Any,N}}, axes::Tuple{Vararg{Any,N}}) where N = - split_alignments(format, format_unaligned, dims, axes) +function format(dims::Tuple{Vararg{Any,N}}, axes::Tuple{Vararg{Any,N}}) where N + # We need to format first + fdims = map(format, dims, axes) + # Then format any unaligned dims as a group + split_alignments(first ∘ tuple, format_unaligned, fdims, axes) +end format(d::Dimension{<:AbstractArray}) = _format(d, axes(val(d), 1)) format(d::Dimension, axis::AbstractRange) = _format(d, axis) +format(d::Type{<:Dimension}, axis::AbstractRange) = _format(d, axis) format(l::Lookup) = parent(format(AnonDim(l))) # Fallback diff --git a/src/Dimensions/indexing.jl b/src/Dimensions/indexing.jl index e4ed63538..ce754e947 100644 --- a/src/Dimensions/indexing.jl +++ b/src/Dimensions/indexing.jl @@ -47,7 +47,7 @@ Convert a `Dimension` or `Selector` `I` to indices of `Int`, `AbstractArray` or @inline function dims2indices(dims::DimTuple, I::DimTuple) extradims = otherdims(I, dims) length(extradims) > 0 && _extradimswarn(extradims) - Isorted = Dimensions.dims(I, dims) + Isorted = Dimensions.sortdims(I, dims) return split_alignments(dims2indices, unalligned_dims2indices, dims, Isorted) end @inline dims2indices(dims::Tuple{}, ::Tuple{}) = () From 9321ab59b7e7bdc2ae1f712f92ec7981a8520378 Mon Sep 17 00:00:00 2001 From: Rafael Schouten Date: Sat, 1 Feb 2025 18:01:39 +0100 Subject: [PATCH 10/14] bugfix --- src/Dimensions/format.jl | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/Dimensions/format.jl b/src/Dimensions/format.jl index d46da79fd..ebeb6cb1a 100644 --- a/src/Dimensions/format.jl +++ b/src/Dimensions/format.jl @@ -13,7 +13,7 @@ and any fields holding `Auto-` objects are filled with guessed objects. If a [`Lookup`](@ref) hasn't been specified, a lookup is chosen based on the type and element type of the values. """ -format(dims, A::AbstractArray) = format((dims,), A) +format(dims::DimOrDimType, A::AbstractVector) = format((dims,), A) function format(dims::NamedTuple, A::AbstractArray) dims = map(keys(dims), values(dims)) do k, v rebuild(name2dim(k), v) @@ -29,18 +29,15 @@ end # Make a dummy array that assumes the dims are the correct length and don't hold `Colon`s function format(dims::DimTuple) ax = map(parent ∘ first ∘ axes, dims) - A = CartesianIndices(ax) - return format(dims, A) + return format(dims, ax) end format(dims::Tuple{Vararg{Any,N}}, A::AbstractArray{<:Any,N}) where N = format(dims, axes(A)) -format(dims::Tuple, A::AbstractArray) = - throw(ArgumentError("length of dims $(length(dims)) does not match array dimensionality $(ndims(A))")) @noinline format(dims::Tuple{Vararg{Any,M}}, A::AbstractArray{<:Any,N}) where {N,M} = throw(DimensionMismatch("Array A has $N axes, while the number of dims is $M: $(map(basetypeof, dims))")) function format(dims::Tuple{Vararg{Any,N}}, axes::Tuple{Vararg{Any,N}}) where N # We need to format first - fdims = map(format, dims, axes) + fdims = map(_format, dims, axes) # Then format any unaligned dims as a group split_alignments(first ∘ tuple, format_unaligned, fdims, axes) end From 19bcc0f3efe6cc97bc233ab99abb9d3e37ce7493 Mon Sep 17 00:00:00 2001 From: Rafael Schouten Date: Sun, 2 Feb 2025 12:21:29 +0100 Subject: [PATCH 11/14] add NearestNeighbors version --- Project.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/Project.toml b/Project.toml index e9a57cf18..56aa03bee 100644 --- a/Project.toml +++ b/Project.toml @@ -70,6 +70,7 @@ IteratorInterfaceExtensions = "1" JLArrays = "0.1" LinearAlgebra = "1" Makie = "0.20, 0.21" +NearestNeighbors = "0.4" OffsetArrays = "1" Plots = "1" PrecompileTools = "1" From 54671efe7ca426f494116a27be641f3a22bc9bd0 Mon Sep 17 00:00:00 2001 From: Rafael Schouten Date: Sun, 2 Feb 2025 14:50:22 +0100 Subject: [PATCH 12/14] add NN to test targets --- Project.toml | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 84b297f37..4cda60754 100644 --- a/Project.toml +++ b/Project.toml @@ -118,4 +118,33 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" Unitful = "1986cc42-f94f-5a68-af5c-568840ba703d" [targets] -test = ["AlgebraOfGraphics", "Aqua", "ArrayInterface", "BenchmarkTools", "CategoricalArrays", "ColorTypes", "Combinatorics", "CoordinateTransformations", "DataFrames", "DiskArrays", "Distributions", "Documenter", "GPUArrays", "ImageFiltering", "ImageTransformations", "JLArrays", "CairoMakie", "OffsetArrays", "Plots", "PythonCall", "Random", "SafeTestsets", "StatsBase", "StatsPlots", "Test", "Unitful"] +test = [ + "AlgebraOfGraphics", + "Aqua", + "ArrayInterface", + "BenchmarkTools", + "CairoMakie", + "CategoricalArrays", + "ColorTypes", + "Combinatorics", + "CoordinateTransformations", + "DataFrames", + "DiskArrays", + "Distributions", + "Documenter", + "GPUArrays", + "ImageFiltering", + "ImageTransformations", + "JLArrays", + "NearestNeighbors", + "OffsetArrays", + "Plots", + "PythonCall", + "Random", + "SafeTestsets", + "StatsBase", + "StatsPlots", + "Test", + "Unitful" +] + From 3e63bddb10e1133eabd0043c0ff3e0ecadfc3a1a Mon Sep 17 00:00:00 2001 From: Rafael Schouten Date: Mon, 3 Feb 2025 17:46:21 +0100 Subject: [PATCH 13/14] Update src/Lookups/selector.jl --- src/Lookups/selector.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Lookups/selector.jl b/src/Lookups/selector.jl index 1fa29c9de..7d5213124 100644 --- a/src/Lookups/selector.jl +++ b/src/Lookups/selector.jl @@ -142,7 +142,7 @@ function at(lookup::NoLookup, sel::At; err=_True(), kw...) v == r || _selnotfound_or_nothing(err, lookup, v) else at >= 0.5 && error("atol must be small than 0.5 for NoLookup") - isapprox(v, r; atol=atol(at)) || _selnotfound_or_nothing(err, lookup, v) + isapprox(v, r; atol=atol(sel)) || _selnotfound_or_nothing(err, lookup, v) end if r in lookup return r From b0aee7806cd8275f9cf7796875a103b208f67149 Mon Sep 17 00:00:00 2001 From: Rafael Schouten Date: Sun, 9 Feb 2025 03:50:26 +0100 Subject: [PATCH 14/14] Update src/Lookups/selector.jl --- src/Lookups/selector.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Lookups/selector.jl b/src/Lookups/selector.jl index 7d5213124..01c6f6d7e 100644 --- a/src/Lookups/selector.jl +++ b/src/Lookups/selector.jl @@ -141,7 +141,7 @@ function at(lookup::NoLookup, sel::At; err=_True(), kw...) if isnothing(atol(sel)) v == r || _selnotfound_or_nothing(err, lookup, v) else - at >= 0.5 && error("atol must be small than 0.5 for NoLookup") + atol(sel) >= 0.5 && error("atol must be small than 0.5 for NoLookup") isapprox(v, r; atol=atol(sel)) || _selnotfound_or_nothing(err, lookup, v) end if r in lookup