From ab2f2bc9a21c1a3d7be291e32e13d3f48db066aa Mon Sep 17 00:00:00 2001 From: rodgomesc Date: Mon, 11 Mar 2024 08:25:04 -0300 Subject: [PATCH] feat: implement flip and rotate for ios --- android/src/main/cpp/ResizePlugin.cpp | 55 ++++++---------- android/src/main/cpp/ResizePlugin.h | 9 +-- example/ios/.xcode.env.local | 2 +- example/src/App.tsx | 6 +- ios/ResizePlugin.mm | 95 ++++++++++++++++++++++++++- src/index.ts | 20 ++++-- 6 files changed, 134 insertions(+), 53 deletions(-) diff --git a/android/src/main/cpp/ResizePlugin.cpp b/android/src/main/cpp/ResizePlugin.cpp index eed1e4d..3a887eb 100644 --- a/android/src/main/cpp/ResizePlugin.cpp +++ b/android/src/main/cpp/ResizePlugin.cpp @@ -154,7 +154,7 @@ FrameBuffer ResizePlugin::flipARGBBuffer(FrameBuffer frameBuffer, bool flip) { if (!flip) { return frameBuffer; } - + __android_log_print(ANDROID_LOG_INFO, TAG, "Flipping ARGB buffer..."); size_t channels = getChannelCount(PixelFormat::ARGB); @@ -164,15 +164,14 @@ FrameBuffer ResizePlugin::flipARGBBuffer(FrameBuffer frameBuffer, bool flip) { _flipBuffer = allocateBuffer(argbSize, "_flipBuffer"); } FrameBuffer destination = { - .width = frameBuffer.width, - .height = frameBuffer.height, - .pixelFormat = PixelFormat::ARGB, - .dataType = DataType::UINT8, - .buffer = _flipBuffer, + .width = frameBuffer.width, + .height = frameBuffer.height, + .pixelFormat = PixelFormat::ARGB, + .dataType = DataType::UINT8, + .buffer = _flipBuffer, }; - int status = libyuv::ARGBMirror(frameBuffer.data(), frameBuffer.bytesPerRow(), - destination.data(), destination.bytesPerRow(), + int status = libyuv::ARGBMirror(frameBuffer.data(), frameBuffer.bytesPerRow(), destination.data(), destination.bytesPerRow(), frameBuffer.width, frameBuffer.height); if (status != 0) { throw std::runtime_error("Failed to flip ARGB Buffer! Status: " + std::to_string(status)); @@ -194,9 +193,7 @@ FrameBuffer ResizePlugin::rotateARGBBuffer(FrameBuffer frameBuffer, int rotation size_t channels = getChannelCount(PixelFormat::ARGB); size_t channelSize = getBytesPerChannel(DataType::UINT8); - size_t destinationStride = - rotation == 90 || rotation == 270 ? rotatedWidth * channels * channelSize - : frameBuffer.bytesPerRow(); + size_t destinationStride = rotation == 90 || rotation == 270 ? rotatedWidth * channels * channelSize : frameBuffer.bytesPerRow(); size_t argbSize = rotatedWidth * rotatedHeight * channels * channelSize; if (_rotatedBuffer == nullptr || _rotatedBuffer->getDirectSize() != argbSize) { @@ -204,20 +201,17 @@ FrameBuffer ResizePlugin::rotateARGBBuffer(FrameBuffer frameBuffer, int rotation } FrameBuffer destination = { - .width = rotatedWidth, - .height = rotatedHeight, - .pixelFormat = PixelFormat::ARGB, - .dataType = DataType::UINT8, - .buffer = _rotatedBuffer, + .width = rotatedWidth, + .height = rotatedHeight, + .pixelFormat = PixelFormat::ARGB, + .dataType = DataType::UINT8, + .buffer = _rotatedBuffer, }; - int status = libyuv::ARGBRotate(frameBuffer.data(), frameBuffer.bytesPerRow(), - destination.data(), destinationStride, - frameBuffer.width, frameBuffer.height, - static_cast(rotation)); + int status = libyuv::ARGBRotate(frameBuffer.data(), frameBuffer.bytesPerRow(), destination.data(), destinationStride, frameBuffer.width, + frameBuffer.height, static_cast(rotation)); if (status != 0) { - throw std::runtime_error( - "Failed to rotate ARGB Buffer! Status: " + std::to_string(status)); + throw std::runtime_error("Failed to rotate ARGB Buffer! Status: " + std::to_string(status)); } return destination; @@ -232,7 +226,6 @@ FrameBuffer ResizePlugin::scaleARGBBuffer(vision::FrameBuffer frameBuffer, int w auto targetString = rectToString(0, 0, width, height); __android_log_print(ANDROID_LOG_INFO, TAG, "Scaling [%s] ARGB buffer to [%s]...", rectString.c_str(), targetString.c_str()); - size_t channels = getChannelCount(PixelFormat::ARGB); size_t channelSize = getBytesPerChannel(DataType::UINT8); size_t argbSize = width * height * channels * channelSize; @@ -349,15 +342,9 @@ FrameBuffer ResizePlugin::convertBufferToDataType(FrameBuffer frameBuffer, DataT return destination; } -jni::global_ref ResizePlugin::resize(jni::alias_ref image, - int cropX, int cropY, - int cropWidth, int cropHeight, - int scaleWidth, int scaleHeight, - int rotation, - bool flip, - int /* PixelFormat */ pixelFormatOrdinal, - int /* DataType */ dataTypeOrdinal) -{ +jni::global_ref ResizePlugin::resize(jni::alias_ref image, int cropX, int cropY, int cropWidth, int cropHeight, + int scaleWidth, int scaleHeight, int rotation, bool flip, + int /* PixelFormat */ pixelFormatOrdinal, int /* DataType */ dataTypeOrdinal) { PixelFormat pixelFormat = static_cast(pixelFormatOrdinal); DataType dataType = static_cast(dataTypeOrdinal); @@ -376,10 +363,10 @@ jni::global_ref ResizePlugin::resize(jni::alias_ref im // 5 Flip ARGB if needed result = flipARGBBuffer(result, flip); - // 5. Convert from ARGB -> ???? + // 6. Convert from ARGB -> ???? result = convertARGBBufferTo(result, pixelFormat); - // 6. Convert from data type to other data type + // 7. Convert from data type to other data type result = convertBufferToDataType(result, dataType); return result.buffer; diff --git a/android/src/main/cpp/ResizePlugin.h b/android/src/main/cpp/ResizePlugin.h index dea237c..1f6f1d7 100644 --- a/android/src/main/cpp/ResizePlugin.h +++ b/android/src/main/cpp/ResizePlugin.h @@ -40,13 +40,8 @@ struct ResizePlugin : public HybridClass { private: explicit ResizePlugin(const alias_ref& javaThis); - global_ref resize(alias_ref image, - int cropX, int cropY, - int cropWidth, int cropHeight, - int scaleWidth, int scaleHeight, - int rotation, - bool flip, - int /* PixelFormat */ pixelFormat, int /* DataType */ dataType); + global_ref resize(alias_ref image, int cropX, int cropY, int cropWidth, int cropHeight, int scaleWidth, + int scaleHeight, int rotation, bool flip, int /* PixelFormat */ pixelFormat, int /* DataType */ dataType); FrameBuffer imageToFrameBuffer(alias_ref image); FrameBuffer cropARGBBuffer(FrameBuffer frameBuffer, int x, int y, int width, int height); diff --git a/example/ios/.xcode.env.local b/example/ios/.xcode.env.local index 8e465f4..c6686b0 100644 --- a/example/ios/.xcode.env.local +++ b/example/ios/.xcode.env.local @@ -1,2 +1,2 @@ -export NODE_BINARY=/Users/mrousavy/.nvm/versions/node/v18.17.0/bin/node +export NODE_BINARY=/Users/rodrigogomes/.nvm/versions/node/v18.2.0/bin/node diff --git a/example/src/App.tsx b/example/src/App.tsx index fc6d28d..0f9874c 100644 --- a/example/src/App.tsx +++ b/example/src/App.tsx @@ -7,11 +7,11 @@ import { useCameraPermission, useFrameProcessor, } from 'react-native-vision-camera'; -import { useResizePlugin } from 'vision-camera-resize-plugin'; +import { ROTATION, useResizePlugin } from 'vision-camera-resize-plugin'; export default function App() { const permission = useCameraPermission(); - const device = useCameraDevice('back'); + const device = useCameraDevice('front'); React.useEffect(() => { permission.requestPermission(); @@ -35,7 +35,7 @@ export default function App() { }, pixelFormat: 'rgba', dataType: 'uint8', - rotation: 180, + rotation: ROTATION.R90, flip: true, }); console.log( diff --git a/ios/ResizePlugin.mm b/ios/ResizePlugin.mm index f8d55d8..8be3dc0 100644 --- a/ios/ResizePlugin.mm +++ b/ios/ResizePlugin.mm @@ -27,6 +27,8 @@ @implementation ResizePlugin { FrameBuffer* _argbBuffer; // 2. ARGB (?x?) -> ARGB (!x!) FrameBuffer* _resizeBuffer; + FrameBuffer* _flipBuffer; + FrameBuffer* _rotateBuffer; // 3. ARGB (!x!) -> !!!! (!x!) FrameBuffer* _convertBuffer; // 3. uint8 -> other type (e.g. float32) if needed @@ -348,6 +350,85 @@ - (FrameBuffer*)convertInt8Buffer:(FrameBuffer*)buffer toDataType:(ConvertDataTy return _customTypeBuffer; } +- (FrameBuffer*)flipARGBBuffer:(FrameBuffer*)buffer flip:(BOOL)flip { + + if (!flip) { + return buffer; + } + + NSLog(@"Flipping ARGB buffer..."); + + if (_flipBuffer == nil || _flipBuffer.width != buffer.width || _flipBuffer.height != buffer.height) { + _flipBuffer = [[FrameBuffer alloc] initWithWidth:buffer.width + height:buffer.height + pixelFormat:buffer.pixelFormat + dataType:buffer.dataType + proxy:_proxy]; + } + + vImage_Buffer src = *buffer.imageBuffer; + vImage_Buffer dest = *_flipBuffer.imageBuffer; + + vImage_Error error = vImageHorizontalReflect_ARGB8888(&src, &dest, kvImageNoFlags); + if (error != kvImageNoError) { + [NSException raise:@"Flip Error" format:@"Failed to flip ARGB buffer! Error: %ld", error]; + } + + return _flipBuffer; +} + +- (FrameBuffer*)rotateARGBBuffer:(FrameBuffer*)buffer rotation:(int)rotation { + if (rotation == 0) { + return buffer; + } + + NSLog(@"Rotating ARGB buffer..."); + + int rotatedWidth = buffer.width; + int rotatedHeight = buffer.height; + if (rotation == 90 || rotation == 270) { + int temp = rotatedWidth; + rotatedWidth = rotatedHeight; + rotatedHeight = temp; + } + + size_t bytesPerPixel = [FrameBuffer getBytesPerPixel:buffer.pixelFormat withType:buffer.dataType]; + + if (_rotateBuffer == nil || _rotateBuffer.width != rotatedWidth || _rotateBuffer.height != rotatedHeight) { + _rotateBuffer = [[FrameBuffer alloc] initWithWidth:rotatedWidth + height:rotatedHeight + pixelFormat:buffer.pixelFormat + dataType:buffer.dataType + proxy:_proxy]; + } + + vImage_Buffer src = *buffer.imageBuffer; + vImage_Buffer dest = *_rotateBuffer.imageBuffer; + + vImage_Error error = kvImageNoError; + Pixel_8888 backgroundColor = {0, 0, 0, 0}; + if (rotation == 90) { + error = vImageRotate90_ARGB8888(&src, &dest, kRotate90DegreesClockwise, backgroundColor, kvImageNoFlags); + } else if (rotation == 180) { + error = vImageRotate90_ARGB8888(&src, &dest, kRotate180DegreesClockwise, backgroundColor, kvImageNoFlags); + } else if (rotation == 270) { + error = vImageRotate90_ARGB8888(&src, &dest, kRotate270DegreesClockwise, backgroundColor, kvImageNoFlags); + } else { + [NSException raise:@"Invalid Rotation" format:@"Rotation must be 0, 90, 180, or 270 degrees."]; + return nil; + } + + if (error != kvImageNoError) { + [NSException raise:@"Rotation Error" format:@"Failed to rotate ARGB buffer! Error: %ld", error]; + } + + if (error != kvImageNoError) { + [NSException raise:@"Rotation Error" format:@"Failed to rotate ARGB buffer! Error: %ld", error]; + } + + return _rotateBuffer; +} + // Used only for debugging/inspecting the Image. - (UIImage*)bufferToImage:(FrameBuffer*)buffer { CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB(); @@ -385,6 +466,8 @@ - (id)callback:(Frame*)frame withArguments:(NSDictionary*)arguments { double cropX = 0; double cropY = 0; NSDictionary* crop = arguments[@"crop"]; + BOOL flip = [[arguments objectForKey:@"flip"] boolValue]; + int rotation = [arguments[@"rotation"] intValue]; if (crop != nil) { cropWidth = ((NSNumber*)crop[@"width"]).doubleValue; cropHeight = ((NSNumber*)crop[@"height"]).doubleValue; @@ -451,13 +534,19 @@ - (id)callback:(Frame*)frame withArguments:(NSDictionary*)arguments { CGSize scaleSize = CGSizeMake(scaleWidth, scaleHeight); result = [self resizeARGB:result crop:cropRect scale:scaleSize]; - // 4. Convert ARGB -> ??? format + // 4. Rotate + result = [self rotateARGBBuffer:result rotation:rotation]; + + // 5. Flip + result = [self flipARGBBuffer:result flip:flip]; + + // 6. Convert ARGB -> ??? format result = [self convertARGB:result to:pixelFormat]; - // 5. Convert UINT8 -> ??? type + // 7. Convert UINT8 -> ??? type result = [self convertInt8Buffer:result toDataType:dataType]; - // 6. Return to JS + // 8. Return to JS return result.sharedArray; } diff --git a/src/index.ts b/src/index.ts index 1c81724..05b6323 100644 --- a/src/index.ts +++ b/src/index.ts @@ -8,6 +8,13 @@ type OutputArray = T extends 'uint8' ? Float32Array : never; +export enum ROTATION { + R0 = 0, + R90 = 90, + R180 = 180, + R270 = 270, +} + interface Size { /** * The width to resize the Frame to. @@ -31,6 +38,9 @@ interface Rect extends Size { } interface Options { + /** + * If set to `true`, the image will be flipped horizontally. + */ flip?: boolean; /** * Crops the image to the given target rect. This is applied first before scaling. @@ -42,6 +52,11 @@ interface Options { * Scale the image to the given target size. This is applied after cropping. */ scale?: Size; + /** + * rotation of the image in degrees. + * defaults to Rotation.R0 (0) + */ + rotation?: ROTATION; /** * Convert the Frame to the given target pixel format. * @@ -52,11 +67,6 @@ interface Options { * - `'bgr'`: [B, G, R] * - `'abgr'`: [A, B, G, R] */ - rotation?: 0 | 90 | 180 | 270; - /** - * rotation of the image in degrees. - * defaults to 0 - */ pixelFormat: 'rgb' | 'rgba' | 'argb' | 'bgra' | 'bgr' | 'abgr'; /** * The given type to use for the resulting buffer.