Skip to content

Commit

Permalink
fix: Optimize iOS/Android codebases (type-safety and performance) (#52)
Browse files Browse the repository at this point in the history 
* fix: Optimize iOS codebase a bit

* Also improve Android codebase

* chore: Format

* fix: Upgrade VisionCamera

* fix: Fix iOS build

* fix: Fix Android build
  • Loading branch information
@mrousavy
mrousavy authored Apr 29, 2024
1 parent 0042bd9 commit c7e8703
Show file tree
Hide file tree
Showing 8 changed files with 177 additions and 114 deletions.
68 changes: 48 additions & 20 deletions android/src/main/cpp/ResizePlugin.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,12 +50,25 @@ int getBytesPerPixel(PixelFormat pixelFormat, DataType type) {
return getChannelCount(pixelFormat) * getBytesPerChannel(type);
}

int FrameBuffer::bytesPerRow() {
libyuv::RotationMode getRotationModeForRotation(Rotation rotation) {
switch (rotation) {
case Rotation0:
return libyuv::RotationMode::kRotate0;
case Rotation90:
return libyuv::RotationMode::kRotate90;
case Rotation180:
return libyuv::RotationMode::kRotate180;
case Rotation270:
return libyuv::RotationMode::kRotate270;
}
}

int FrameBuffer::bytesPerRow() const {
size_t bytesPerPixel = getBytesPerPixel(pixelFormat, dataType);
return width * bytesPerPixel;
}

uint8_t* FrameBuffer::data() {
uint8_t* FrameBuffer::data() const {
return buffer->getDirectBytes();
}

Expand All @@ -80,6 +93,7 @@ FrameBuffer ResizePlugin::imageToFrameBuffer(alias_ref<vision::JImage> image) {

size_t uvPixelStride = uPlane->getPixelStride();
if (uPlane->getPixelStride() != vPlane->getPixelStride()) {
[[unlikely]];
throw std::runtime_error("U and V planes do not have the same pixel stride! Are you sure this is a 4:2:0 YUV format?");
}

Expand All @@ -106,6 +120,7 @@ FrameBuffer ResizePlugin::imageToFrameBuffer(alias_ref<vision::JImage> image) {
destination.data(), width * channels * channelSize, width, height);

if (status != 0) {
[[unlikely]];
throw std::runtime_error("Failed to convert YUV 4:2:0 to ARGB! Error: " + std::to_string(status));
}

Expand All @@ -116,7 +131,7 @@ std::string rectToString(int x, int y, int width, int height) {
return std::to_string(x) + ", " + std::to_string(y) + " @ " + std::to_string(width) + "x" + std::to_string(height);
}

FrameBuffer ResizePlugin::cropARGBBuffer(vision::FrameBuffer frameBuffer, int x, int y, int width, int height) {
FrameBuffer ResizePlugin::cropARGBBuffer(const FrameBuffer& frameBuffer, int x, int y, int width, int height) {
if (width == frameBuffer.width && height == frameBuffer.height && x == 0 && y == 0) {
// already in correct size.
return frameBuffer;
Expand Down Expand Up @@ -144,13 +159,14 @@ FrameBuffer ResizePlugin::cropARGBBuffer(vision::FrameBuffer frameBuffer, int x,
destination.bytesPerRow(), x, y, frameBuffer.width, frameBuffer.height, width, height,
libyuv::kRotate0, libyuv::FOURCC_ARGB);
if (status != 0) {
[[unlikely]];
throw std::runtime_error("Failed to crop ARGB Buffer! Status: " + std::to_string(status));
}

return destination;
}

FrameBuffer ResizePlugin::mirrorARGBBuffer(FrameBuffer frameBuffer, bool mirror) {
FrameBuffer ResizePlugin::mirrorARGBBuffer(const FrameBuffer& frameBuffer, bool mirror) {
if (!mirror) {
return frameBuffer;
}
Expand All @@ -174,30 +190,36 @@ FrameBuffer ResizePlugin::mirrorARGBBuffer(FrameBuffer frameBuffer, bool mirror)
int status = libyuv::ARGBMirror(frameBuffer.data(), frameBuffer.bytesPerRow(), destination.data(), destination.bytesPerRow(),
frameBuffer.width, frameBuffer.height);
if (status != 0) {
[[unlikely]];
throw std::runtime_error("Failed to mirror ARGB Buffer! Status: " + std::to_string(status));
}

return destination;
}

FrameBuffer ResizePlugin::rotateARGBBuffer(FrameBuffer frameBuffer, int rotation) {
if (rotation == 0) {
FrameBuffer ResizePlugin::rotateARGBBuffer(const FrameBuffer& frameBuffer, Rotation rotation) {
if (rotation == Rotation::Rotation0) {
return frameBuffer;
}

int rotatedWidth = frameBuffer.width;
int rotatedHeight = frameBuffer.height;
if (rotation == 90 || rotation == 270) {
std::swap(rotatedWidth, rotatedHeight);
int rotatedWidth, rotatedHeight;
if (rotation == Rotation90 || rotation == Rotation270) {
// flipped to the side
rotatedWidth = frameBuffer.height;
rotatedHeight = frameBuffer.width;
} else {
// still uprighht, maybe upside down.
rotatedWidth = frameBuffer.width;
rotatedHeight = frameBuffer.height;
}

size_t channels = getChannelCount(PixelFormat::ARGB);
size_t channelSize = getBytesPerChannel(DataType::UINT8);
size_t destinationStride = rotation == 90 || rotation == 270 ? rotatedWidth * channels * channelSize : frameBuffer.bytesPerRow();
size_t argbSize = rotatedWidth * rotatedHeight * channels * channelSize;
size_t destinationStride = rotatedWidth * channels * channelSize;
size_t rotateSize = frameBuffer.buffer->getDirectSize();

if (_rotatedBuffer == nullptr || _rotatedBuffer->getDirectSize() != argbSize) {
_rotatedBuffer = allocateBuffer(argbSize, "_rotatedBuffer");
if (_rotatedBuffer == nullptr || _rotatedBuffer->getDirectSize() != rotateSize) {
_rotatedBuffer = allocateBuffer(rotateSize, "_rotatedBuffer");
}

FrameBuffer destination = {
Expand All @@ -208,16 +230,18 @@ FrameBuffer ResizePlugin::rotateARGBBuffer(FrameBuffer frameBuffer, int rotation
.buffer = _rotatedBuffer,
};

libyuv::RotationMode rotationMode = getRotationModeForRotation(rotation);
int status = libyuv::ARGBRotate(frameBuffer.data(), frameBuffer.bytesPerRow(), destination.data(), destinationStride, frameBuffer.width,
frameBuffer.height, static_cast<libyuv::RotationMode>(rotation));
frameBuffer.height, rotationMode);
if (status != 0) {
[[unlikely]];
throw std::runtime_error("Failed to rotate ARGB Buffer! Status: " + std::to_string(status));
}

return destination;
}

FrameBuffer ResizePlugin::scaleARGBBuffer(vision::FrameBuffer frameBuffer, int width, int height) {
FrameBuffer ResizePlugin::scaleARGBBuffer(const FrameBuffer& frameBuffer, int width, int height) {
if (width == frameBuffer.width && height == frameBuffer.height) {
// already in correct size.
return frameBuffer;
Expand All @@ -243,13 +267,14 @@ FrameBuffer ResizePlugin::scaleARGBBuffer(vision::FrameBuffer frameBuffer, int w
int status = libyuv::ARGBScale(frameBuffer.data(), frameBuffer.bytesPerRow(), frameBuffer.width, frameBuffer.height, destination.data(),
destination.bytesPerRow(), width, height, libyuv::FilterMode::kFilterBilinear);
if (status != 0) {
[[unlikely]];
throw std::runtime_error("Failed to scale ARGB Buffer! Status: " + std::to_string(status));
}

return destination;
}

FrameBuffer ResizePlugin::convertARGBBufferTo(FrameBuffer frameBuffer, PixelFormat pixelFormat) {
FrameBuffer ResizePlugin::convertARGBBufferTo(const FrameBuffer& frameBuffer, PixelFormat pixelFormat) {
if (frameBuffer.pixelFormat == pixelFormat) {
// Already in the correct format.
return frameBuffer;
Expand Down Expand Up @@ -300,13 +325,14 @@ FrameBuffer ResizePlugin::convertARGBBufferTo(FrameBuffer frameBuffer, PixelForm
}

if (error != 0) {
[[unlikely]];
throw std::runtime_error("Failed to convert ARGB Buffer to target Pixel Format! Error: " + std::to_string(error));
}

return destination;
}

FrameBuffer ResizePlugin::convertBufferToDataType(FrameBuffer frameBuffer, DataType dataType) {
FrameBuffer ResizePlugin::convertBufferToDataType(const FrameBuffer& frameBuffer, DataType dataType) {
if (frameBuffer.dataType == dataType) {
// Already in correct data-type
return frameBuffer;
Expand Down Expand Up @@ -340,17 +366,19 @@ FrameBuffer ResizePlugin::convertBufferToDataType(FrameBuffer frameBuffer, DataT
}

if (status != 0) {
[[unlikely]];
throw std::runtime_error("Failed to convert Buffer to target Data Type! Error: " + std::to_string(status));
}

return destination;
}

jni::global_ref<jni::JByteBuffer> ResizePlugin::resize(jni::alias_ref<JImage> image, int cropX, int cropY, int cropWidth, int cropHeight,
int scaleWidth, int scaleHeight, int rotationOrdinal, bool mirror,
int scaleWidth, int scaleHeight, int /* Rotation */ rotationOrdinal, bool mirror,
int /* PixelFormat */ pixelFormatOrdinal, int /* DataType */ dataTypeOrdinal) {
PixelFormat pixelFormat = static_cast<PixelFormat>(pixelFormatOrdinal);
DataType dataType = static_cast<DataType>(dataTypeOrdinal);
Rotation rotation = static_cast<Rotation>(rotationOrdinal);

// 1. Convert from YUV -> ARGB
FrameBuffer result = imageToFrameBuffer(image);
Expand All @@ -362,7 +390,7 @@ jni::global_ref<jni::JByteBuffer> ResizePlugin::resize(jni::alias_ref<JImage> im
result = scaleARGBBuffer(result, scaleWidth, scaleHeight);

// 4. Rotate ARGB
result = rotateARGBBuffer(result, rotationOrdinal);
result = rotateARGBBuffer(result, rotation);

// 5 Mirror ARGB if needed
result = mirrorARGBBuffer(result, mirror);
Expand Down
20 changes: 11 additions & 9 deletions android/src/main/cpp/ResizePlugin.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,17 @@ enum PixelFormat { RGB, BGR, ARGB, RGBA, BGRA, ABGR };

enum DataType { UINT8, FLOAT32 };

enum Rotation { Rotation0, Rotation90, Rotation180, Rotation270 };

struct FrameBuffer {
int width;
int height;
PixelFormat pixelFormat;
DataType dataType;
global_ref<JByteBuffer> buffer;

uint8_t* data();
int bytesPerRow();
uint8_t* data() const;
int bytesPerRow() const;
};

struct ResizePlugin : public HybridClass<ResizePlugin> {
Expand All @@ -41,16 +43,16 @@ struct ResizePlugin : public HybridClass<ResizePlugin> {
explicit ResizePlugin(const alias_ref<jhybridobject>& javaThis);

global_ref<JByteBuffer> resize(alias_ref<JImage> image, int cropX, int cropY, int cropWidth, int cropHeight, int scaleWidth,
int scaleHeight, int rotation, bool mirror, int /* PixelFormat */ pixelFormat,
int scaleHeight, int /* Rotation */ rotation, bool mirror, int /* PixelFormat */ pixelFormat,
int /* DataType */ dataType);

FrameBuffer imageToFrameBuffer(alias_ref<JImage> image);
FrameBuffer cropARGBBuffer(FrameBuffer frameBuffer, int x, int y, int width, int height);
FrameBuffer scaleARGBBuffer(FrameBuffer frameBuffer, int width, int height);
FrameBuffer convertARGBBufferTo(FrameBuffer frameBuffer, PixelFormat toFormat);
FrameBuffer convertBufferToDataType(FrameBuffer frameBuffer, DataType dataType);
FrameBuffer rotateARGBBuffer(FrameBuffer frameBuffer, int rotation);
FrameBuffer mirrorARGBBuffer(FrameBuffer frameBuffer, bool mirror);
FrameBuffer cropARGBBuffer(const FrameBuffer& frameBuffer, int x, int y, int width, int height);
FrameBuffer scaleARGBBuffer(const FrameBuffer& frameBuffer, int width, int height);
FrameBuffer convertARGBBufferTo(const FrameBuffer& frameBuffer, PixelFormat toFormat);
FrameBuffer convertBufferToDataType(const FrameBuffer& frameBuffer, DataType dataType);
FrameBuffer rotateARGBBuffer(const FrameBuffer& frameBuffer, Rotation rotation);
FrameBuffer mirrorARGBBuffer(const FrameBuffer& frameBuffer, bool mirror);
global_ref<JByteBuffer> allocateBuffer(size_t size, std::string debugName);

private:
Expand Down
23 changes: 15 additions & 8 deletions example/ios/Podfile.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -944,7 +944,7 @@ PODS:
- React-Mapbuffer (0.73.2):
- glog
- React-debug
- react-native-worklets-core (0.2.4):
- react-native-worklets-core (1.2.0):
- React
- React-callinvoker
- React-Core
Expand Down Expand Up @@ -1116,16 +1116,23 @@ PODS:
- React-logger (= 0.73.2)
- React-perflogger (= 0.73.2)
- SocketRocket (0.6.1)
- vision-camera-resize-plugin (2.0.0):
- vision-camera-resize-plugin (2.1.1):
- glog
- RCT-Folly (= 2022.05.16.00)
- React-Core
- VisionCamera
- VisionCamera (3.8.2):
- VisionCamera (4.0.1):
- VisionCamera/Core (= 4.0.1)
- VisionCamera/FrameProcessors (= 4.0.1)
- VisionCamera/React (= 4.0.1)
- VisionCamera/Core (4.0.1)
- VisionCamera/FrameProcessors (4.0.1):
- React
- React-callinvoker
- React-Core
- react-native-worklets-core
- VisionCamera/React (4.0.1):
- React-Core
- VisionCamera/FrameProcessors
- Yoga (1.14.0)

DEPENDENCIES:
Expand Down Expand Up @@ -1367,7 +1374,7 @@ SPEC CHECKSUMS:
React-jsinspector: 03644c063fc3621c9a4e8bf263a8150909129618
React-logger: 66b168e2b2bee57bd8ce9e69f739d805732a5570
React-Mapbuffer: 9ee041e1d7be96da6d76a251f92e72b711c651d6
react-native-worklets-core: b4094f51cb2bc649e297206425cb8956f4945e3e
react-native-worklets-core: a316975bba20b73d47aa4d41bffb0ca984ba592e
React-nativeconfig: d753fbbc8cecc8ae413d615599ac378bbf6999bb
React-NativeModulesApple: 964f4eeab1b4325e8b6a799cf4444c3fd4eb0a9c
React-perflogger: 29efe63b7ef5fbaaa50ef6eaa92482f98a24b97e
Expand All @@ -1389,9 +1396,9 @@ SPEC CHECKSUMS:
React-utils: f5bc61e7ea3325c0732ae2d755f4441940163b85
ReactCommon: 45b5d4f784e869c44a6f5a8fad5b114ca8f78c53
SocketRocket: f32cd54efbe0f095c4d7594881e52619cfe80b17
vision-camera-resize-plugin: 627fad747d8eb56033cd1a0ebd4581d26794e82d
VisionCamera: edbcd00e27a438b2228f67823e2b8d15a189065f
Yoga: 13c8ef87792450193e117976337b8527b49e8c03
vision-camera-resize-plugin: 5c457b56688c34d03352ea2ee66f975f0c3f3642
VisionCamera: 8e00df84a76cf26ca70ecd3b6ed05dc0d3b60beb
Yoga: e64aa65de36c0832d04e8c7bd614396c77a80047

PODFILE CHECKSUM: 9cf228fb4a0a05c32bc602c7f07154b180219b9e

Expand Down
4 changes: 2 additions & 2 deletions example/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@
"dependencies": {
"react": "^18.2.0",
"react-native": "^0.73.2",
"react-native-vision-camera": "^3.8.2",
"react-native-worklets-core": "^0.2.4"
"react-native-vision-camera": "^4.0.1",
"react-native-worklets-core": "^1.2.0"
},
"devDependencies": {
"@babel/core": "^7.23.9",
Expand Down
16 changes: 8 additions & 8 deletions example/yarn.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3711,15 +3711,15 @@ react-is@^17.0.1:
resolved "https://registry.yarnpkg.com/react-is/-/react-is-17.0.2.tgz#e691d4a8e9c789365655539ab372762b0efb54f0"
integrity sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==

react-native-vision-camera@^3.8.2:
version "3.8.2"
resolved "https://registry.yarnpkg.com/react-native-vision-camera/-/react-native-vision-camera-3.8.2.tgz#f4f75f84c6a19e1c3474ddc0f7f785b5a526739b"
integrity sha512-MY39l2e3hNRPUefn2JPShOFExcw0PblbAcUGvJrIfS9pMzdIyceo0umRAx8lOGXzDUAdb+xy/tFWb8zGxKimCQ==
react-native-vision-camera@^4.0.1:
version "4.0.1"
resolved "https://registry.yarnpkg.com/react-native-vision-camera/-/react-native-vision-camera-4.0.1.tgz#045c8ff9c236d3bf7e1dba24172a4c11b19bbd37"
integrity sha512-TuXLw/ak+LPaJOCuUaYG1g2EmTAEEeIVAhSbCsqEHlklMjkO0oRBGcPC6HpKw3++jkcKRBVVyY48HbB002UZfg==

react-native-worklets-core@^0.2.4:
version "0.2.4"
resolved "https://registry.yarnpkg.com/react-native-worklets-core/-/react-native-worklets-core-0.2.4.tgz#a4a79c04f2c6769ff03e96b7529c8638c88fa560"
integrity sha512-NKqxLRDOYfO7RJRZHHDA/1yztdiSadSTTILgHw2VwSrcQcmnWAZGXWr9EYpzKblpbk/vXfJa6QpWIKHjNrbe4w==
react-native-worklets-core@^1.2.0:
version "1.2.0"
resolved "https://registry.yarnpkg.com/react-native-worklets-core/-/react-native-worklets-core-1.2.0.tgz#155040a48d1e3f66107544867ec579c2f5044271"
integrity sha512-Uuf1Hg4V1w4I8eYNinwnnf+h+2YdQgwkUDQszY2yEEXWPWcZBWec0xjYn079RFRcKmGWuJ/zOtokjwy/IjJ9Fw==
dependencies:
string-hash-64 "^1.0.3"

Expand Down
Loading

0 comments on commit c7e8703

Please sign in to comment.