Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Rework LeptonicaFrameConverter#convert(Frame) #2178

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
Open

Rework LeptonicaFrameConverter#convert(Frame) #2178

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
84384d8
Fix Maven build (issue #2176)
msmuenchen Jan 27, 2024
aadc5fb
Clean up more Javadoc warnings (issue #2176)
msmuenchen Jan 27, 2024
dac0b8a
add JUnit support
msmuenchen Jan 27, 2024
2bf5cc9
rework LeptonicaFrameConverter#convert(Frame) to handle non-aligned s…
msmuenchen Jan 27, 2024
c0f53c5
add unit tests for LeptonicaFrameConverter#convert(Frame), see issue …
msmuenchen Jan 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 45 additions & 0 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,8 @@
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.build.timestamp.format>yyyyMMddhhmm</maven.build.timestamp.format>
<javacpp.version>${project.version}</javacpp.version>
<junit-jupiter-engine.version>5.9.1</junit-jupiter-engine.version>
<maven-surefire-plugin.version>3.2.3</maven-surefire-plugin.version>
</properties>

<dependencies>
Expand All @@ -72,11 +74,23 @@
<artifactId>openblas</artifactId>
<version>0.3.26-${javacpp.version}</version>
</dependency>
<dependency>
<groupId>org.bytedeco</groupId>
<artifactId>openblas-platform</artifactId>
<version>0.3.26-${javacpp.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.bytedeco</groupId>
<artifactId>opencv</artifactId>
<version>4.9.0-${javacpp.version}</version>
</dependency>
<dependency>
<groupId>org.bytedeco</groupId>
<artifactId>opencv-platform</artifactId>
<version>4.9.0-${javacpp.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.bytedeco</groupId>
<artifactId>ffmpeg</artifactId>
Expand Down Expand Up @@ -132,6 +146,12 @@
<artifactId>leptonica</artifactId>
<version>1.84.1-${javacpp.version}</version>
</dependency>
<dependency>
<groupId>org.bytedeco</groupId>
<artifactId>leptonica-platform</artifactId>
<version>1.84.1-${javacpp.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.bytedeco</groupId>
<artifactId>tesseract</artifactId>
Expand Down Expand Up @@ -169,6 +189,19 @@
<version>2.3.2</version>
<optional>true</optional>
</dependency>

<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-engine</artifactId>
<version>${junit-jupiter-engine.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-params</artifactId>
<version>${junit-jupiter-engine.version}</version>
<scope>test</scope>
</dependency>
</dependencies>

<repositories>
Expand Down Expand Up @@ -301,6 +334,10 @@
<link>https://jogamp.org/deployment/v2.3.2/javadoc/jogl/javadoc</link>
<link>http://junit.org/junit4/javadoc/4.13.2</link>
</links>
<sourceFileExcludes>
<exclude>org/bytedeco/javacv/FlyCaptureFrameGrabber.java</exclude>
<exclude>org/bytedeco/javacv/FFmpegLockCallback.java</exclude>
</sourceFileExcludes>
</configuration>
</execution>
</executions>
Expand All @@ -316,6 +353,14 @@
<skipStagingRepositoryClose>true</skipStagingRepositoryClose>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>${maven-surefire-plugin.version}</version>
<configuration>
<failIfNoTests>true</failIfNoTests>
</configuration>
</plugin>
</plugins>
</build>

Expand Down
2 changes: 1 addition & 1 deletion src/main/java/org/bytedeco/javacv/FFmpegLogCallback.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ public class FFmpegLogCallback extends LogCallback {

static final FFmpegLogCallback instance = new FFmpegLogCallback().retainReference();

/** Returns an instance that can be used with {@link #setLogCallback(LogCallback)}. */
/** Returns an instance that can be used with {@link org.bytedeco.ffmpeg.global.avutil#setLogCallback(LogCallback)}. */
public static FFmpegLogCallback getInstance() {
return instance;
}
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/org/bytedeco/javacv/Frame.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -213,7 +213,7 @@ public <I extends Indexer> I createIndexer(boolean direct, int i) {
* @return A deep copy of this frame.
* @see {@link #cloneBufferArray}
*
* @author Extension proposed by Dragos Dutu
* Extension proposed by Dragos Dutu
* */
@Override
public Frame clone() {
Expand Down
128 changes: 96 additions & 32 deletions src/main/java/org/bytedeco/javacv/LeptonicaFrameConverter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,14 +22,16 @@

package org.bytedeco.javacv;

import java.nio.Buffer;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import org.bytedeco.javacpp.BytePointer;
import org.bytedeco.javacpp.Loader;
import org.bytedeco.javacpp.Pointer;
import org.bytedeco.leptonica.PIX;

import java.nio.Buffer;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.IntBuffer;

import org.bytedeco.leptonica.*;
import static org.bytedeco.leptonica.global.leptonica.*;

/**
Expand All @@ -40,7 +42,9 @@
* @author Samuel Audet
*/
public class LeptonicaFrameConverter extends FrameConverter<PIX> {
static { Loader.load(org.bytedeco.leptonica.global.leptonica.class); }
static {
Loader.load(org.bytedeco.leptonica.global.leptonica.class);
}

PIX pix;
BytePointer frameData, pixData;
Expand All @@ -51,40 +55,99 @@ static boolean isEqual(Frame frame, PIX pix) {
&& frame.imageWidth == pix.w() && frame.imageHeight == pix.h()
&& frame.imageChannels == pix.d() / 8 && frame.imageDepth == Frame.DEPTH_UBYTE
&& (ByteOrder.nativeOrder().equals(ByteOrder.LITTLE_ENDIAN)
|| new Pointer(frame.image[0]).address() == pix.data().address())
|| new Pointer(frame.image[0]).address() == pix.data().address())
&& frame.imageStride * Math.abs(frame.imageDepth) / 8 == pix.wpl() * 4;
}

public PIX convert(Frame frame) {
if (frame == null || frame.image == null) {
return null;
} else if (frame.opaque instanceof PIX) {
return (PIX)frame.opaque;
return (PIX) frame.opaque;
} else if (!isEqual(frame, pix)) {
Pointer data;
if (ByteOrder.nativeOrder().equals(ByteOrder.LITTLE_ENDIAN)) {
if (pixData == null || pixData.capacity() < frame.imageHeight * frame.imageStride) {
if (pixData != null) {
pixData.releaseReference();
//I simply lack a machine to test this.
if (ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN)) {
System.err.println("This converter does not support running on big-endian machines");
return null;
}
//PIX data should be packed as tightly as possible, see https://github.com/DanBloomberg/leptonica/blob/0d4477653691a8cb4f63fa751d43574c757ccc9f/src/pix.h#L133
//For anything not greyscale or RGB @ 8 bit per pixel, this involves more bit-shift logic than I'm willing to write (and I lack test cases)
if (frame.imageChannels != 3 && frame.imageChannels != 1) {
System.out.println(String.format("Image has %d channels, converter only supports 3 (RGB) or 1 (grayscale) for input", frame.imageChannels));
return null;
}
if (frame.imageDepth != 8 || !(frame.image[0] instanceof ByteBuffer)) {
System.out.println(String.format("Image has bit depth %d, converter only supports 8 (1 byte/px) for input", frame.imageDepth));
return null;
}
// Leptonica frame scan lines must be padded to 32 bit / 4 bytes of stride (line) length, otherwise one gets nasty scan effects
// See http://www.leptonica.org/library-notes.html#PIX
int srcChannelDepthBytes = frame.imageDepth / 8;
int srcBytesPerPixel = srcChannelDepthBytes * frame.imageChannels;

// Leptonica counts RGB images as 24 bits per pixel, while the data actually is 32 bit per pixel
int destBytesPerPixel = srcBytesPerPixel;
if (frame.imageChannels == 3) {
// RGB pixels are stored as RGBA, so they take up 4 bytes!
// See https://github.com/DanBloomberg/leptonica/blob/master/src/pix.h#L157
destBytesPerPixel = 4;
}
int currentStrideLength = frame.imageWidth * destBytesPerPixel;
int targetStridePad = 4 - (currentStrideLength % 4);
if (targetStridePad == 4)
targetStridePad = 0;
int targetStrideLength = (currentStrideLength) + targetStridePad;
ByteBuffer src = ((ByteBuffer) frame.image[0]).order(ByteOrder.LITTLE_ENDIAN);
int newSize = targetStrideLength * frame.imageHeight;
ByteBuffer dst = ByteBuffer.allocate(newSize).order(ByteOrder.LITTLE_ENDIAN);
/*
System.out.println(String.format(
"src: %d bytes total, %d channels @ %d bytes per pixel, stride length %d",
frame.image[0].capacity(),
frame.imageChannels,
srcBytesPerPixel,
currentStrideLength
));
System.out.println(String.format(
"dst: %d bytes total, stride length %d, stride pad %d",
newSize,
targetStrideLength,
targetStridePad
));
*/
//The source bytes will be RGB, which means it will have to be copied byte-by-byte to match Leptonica RGBA
//todo: use qword copy ops?
byte[] rowData = new byte[targetStrideLength];
for (int row = 0; row < frame.imageHeight; row++) {
for (int col = 0; col < frame.imageWidth; col++) {
int srcIndex = (frame.imageStride * row) + (col * frame.imageChannels);
if (frame.imageChannels == 1) {
byte v = src.get(srcIndex);
rowData[col] = v;
//System.out.println(String.format("row %03d col %03d idx src %03d val %02x", row, col, srcIndex,v));
} else if (frame.imageChannels == 3) {
int dstIndex = col * destBytesPerPixel;
byte[] pixelData = new byte[3];
src.position(srcIndex);
src.get(pixelData, 0, pixelData.length);
// Convert BGR (OpenCV's standard ordering) to RGB (Leptonica)
// See https://learnopencv.com/why-does-opencv-use-bgr-color-format/ and https://github.com/DanBloomberg/leptonica/blob/master/src/pix.h#L157
rowData[dstIndex] = pixelData[2]; //dst: r
rowData[dstIndex + 1] = pixelData[1]; //dst: g
rowData[dstIndex + 2] = pixelData[0]; // dst: b
rowData[dstIndex + 3] = 0;
//System.out.println(String.format("row %03d col %03d idx src %03d dst %03d val r %02x g %02x b %02x", row, col, srcIndex,dstIndex, pixelData[2], pixelData[1], pixelData[1]));
}
pixData = new BytePointer(frame.imageHeight * frame.imageStride).retainReference();
}
data = pixData;
pixBuffer = data.asByteBuffer().order(ByteOrder.BIG_ENDIAN);
} else {
data = new Pointer(frame.image[0].position(0));
}
if (pix != null) {
pix.releaseReference();
//And since pixel data in source is little-endian, but Leptonica is big-endian on 32-bit level, now invert accordingly...
ByteBuffer rowBuffer = ByteBuffer.wrap(rowData);
IntBuffer inverted = rowBuffer.order(ByteOrder.BIG_ENDIAN).asIntBuffer();
//System.out.println(Arrays.toString(rowBuffer.array()));
dst.position(row * targetStrideLength).asIntBuffer().put(inverted);
}
pix = PIX.create(frame.imageWidth, frame.imageHeight, frame.imageChannels * 8, data)
.wpl(frame.imageStride / 4 * Math.abs(frame.imageDepth) / 8).retainReference();
pix = PIX.create(frame.imageWidth, frame.imageHeight, destBytesPerPixel * 8, new BytePointer(dst.position(0)));
}

if (ByteOrder.nativeOrder().equals(ByteOrder.LITTLE_ENDIAN)) {
((ByteBuffer)pixBuffer.position(0)).asIntBuffer()
.put(((ByteBuffer)frame.image[0].position(0)).order(ByteOrder.LITTLE_ENDIAN).asIntBuffer());
}
return pix;
}

Expand All @@ -99,10 +162,10 @@ public Frame convert(PIX pix) {
} else if (pix.d() < 8) {
switch (pix.d()) {
case 1:
tempPix = pix = pixConvert1To8(null, pix, (byte)0, (byte)255);
tempPix = pix = pixConvert1To8(null, pix, (byte) 0, (byte) 255);
break;
case 2:
tempPix = pix = pixConvert2To8(pix, (byte)0, (byte)85, (byte)170, (byte)255, 0);
tempPix = pix = pixConvert2To8(pix, (byte) 0, (byte) 85, (byte) 170, (byte) 255, 0);
break;
case 4:
tempPix = pix = pixConvert4To8(pix, 0);
Expand All @@ -128,7 +191,7 @@ public Frame convert(PIX pix) {
}
frameBuffer = frameData.asByteBuffer().order(ByteOrder.LITTLE_ENDIAN);
frame.opaque = frameData;
frame.image = new Buffer[] { frameBuffer };
frame.image = new Buffer[]{frameBuffer};
} else {
if (tempPix != null) {
if (this.pix != null) {
Expand All @@ -137,12 +200,12 @@ public Frame convert(PIX pix) {
this.pix = pix = pix.clone();
}
frame.opaque = pix;
frame.image = new Buffer[] { pix.createBuffer() };
frame.image = new Buffer[]{pix.createBuffer()};
}
}

if (ByteOrder.nativeOrder().equals(ByteOrder.LITTLE_ENDIAN)) {
((ByteBuffer)frameBuffer.position(0)).asIntBuffer()
((ByteBuffer) frameBuffer.position(0)).asIntBuffer()
.put(pix.createBuffer().order(ByteOrder.BIG_ENDIAN).asIntBuffer());
}

Expand All @@ -152,7 +215,8 @@ public Frame convert(PIX pix) {
return frame;
}

@Override public void close() {
@Override
public void close() {
super.close();
if (pix != null) {
pix.releaseReference();
Expand Down
12 changes: 5 additions & 7 deletions src/main/java/org/bytedeco/javacv/PS3EyeFrameGrabber.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ public PS3EyeFrameGrabber() throws Exception {
}

/** Color mode, VGA resolution, 60 FPS frame rate.
* @param system wide camera index
* @param cameraIndex system wide camera index
*/
public PS3EyeFrameGrabber(int cameraIndex) throws Exception {
this(cameraIndex, 640, 480, 60);
Expand Down Expand Up @@ -274,9 +274,8 @@ public Frame grab() throws Exception {
}


/** Start camera first (before grabbing).
*
* @return success/failure (true/false)
/**
* Start camera first (before grabbing).
*/
public void start() throws Exception {
boolean b;
Expand All @@ -301,9 +300,8 @@ public void start() throws Exception {
}


/** Stop camera. It can be re-started if needed.
*
* @return success/failure (true/false)
/**
* Stop camera. It can be re-started if needed.
*/
public void stop() throws Exception {
boolean b = camera.stopCamera();
Expand Down
Loading
Loading