Skip to content

Commit

Permalink
feat: implement flip and rotate for ios
Browse files Browse the repository at this point in the history
  • Loading branch information
@rodgomesc
rodgomesc committed Mar 11, 2024
1 parent 211ed42 commit ab2f2bc
Show file tree
Hide file tree
Showing 6 changed files with 134 additions and 53 deletions.
55 changes: 21 additions & 34 deletions android/src/main/cpp/ResizePlugin.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ FrameBuffer ResizePlugin::flipARGBBuffer(FrameBuffer frameBuffer, bool flip) {
if (!flip) {
return frameBuffer;
}

__android_log_print(ANDROID_LOG_INFO, TAG, "Flipping ARGB buffer...");

size_t channels = getChannelCount(PixelFormat::ARGB);
Expand All @@ -164,15 +164,14 @@ FrameBuffer ResizePlugin::flipARGBBuffer(FrameBuffer frameBuffer, bool flip) {
_flipBuffer = allocateBuffer(argbSize, "_flipBuffer");
}
FrameBuffer destination = {
.width = frameBuffer.width,
.height = frameBuffer.height,
.pixelFormat = PixelFormat::ARGB,
.dataType = DataType::UINT8,
.buffer = _flipBuffer,
.width = frameBuffer.width,
.height = frameBuffer.height,
.pixelFormat = PixelFormat::ARGB,
.dataType = DataType::UINT8,
.buffer = _flipBuffer,
};

int status = libyuv::ARGBMirror(frameBuffer.data(), frameBuffer.bytesPerRow(),
destination.data(), destination.bytesPerRow(),
int status = libyuv::ARGBMirror(frameBuffer.data(), frameBuffer.bytesPerRow(), destination.data(), destination.bytesPerRow(),
frameBuffer.width, frameBuffer.height);
if (status != 0) {
throw std::runtime_error("Failed to flip ARGB Buffer! Status: " + std::to_string(status));
Expand All @@ -194,30 +193,25 @@ FrameBuffer ResizePlugin::rotateARGBBuffer(FrameBuffer frameBuffer, int rotation

size_t channels = getChannelCount(PixelFormat::ARGB);
size_t channelSize = getBytesPerChannel(DataType::UINT8);
size_t destinationStride =
rotation == 90 || rotation == 270 ? rotatedWidth * channels * channelSize
: frameBuffer.bytesPerRow();
size_t destinationStride = rotation == 90 || rotation == 270 ? rotatedWidth * channels * channelSize : frameBuffer.bytesPerRow();
size_t argbSize = rotatedWidth * rotatedHeight * channels * channelSize;

if (_rotatedBuffer == nullptr || _rotatedBuffer->getDirectSize() != argbSize) {
_rotatedBuffer = allocateBuffer(argbSize, "_rotatedBuffer");
}

FrameBuffer destination = {
.width = rotatedWidth,
.height = rotatedHeight,
.pixelFormat = PixelFormat::ARGB,
.dataType = DataType::UINT8,
.buffer = _rotatedBuffer,
.width = rotatedWidth,
.height = rotatedHeight,
.pixelFormat = PixelFormat::ARGB,
.dataType = DataType::UINT8,
.buffer = _rotatedBuffer,
};

int status = libyuv::ARGBRotate(frameBuffer.data(), frameBuffer.bytesPerRow(),
destination.data(), destinationStride,
frameBuffer.width, frameBuffer.height,
static_cast<libyuv::RotationMode>(rotation));
int status = libyuv::ARGBRotate(frameBuffer.data(), frameBuffer.bytesPerRow(), destination.data(), destinationStride, frameBuffer.width,
frameBuffer.height, static_cast<libyuv::RotationMode>(rotation));
if (status != 0) {
throw std::runtime_error(
"Failed to rotate ARGB Buffer! Status: " + std::to_string(status));
throw std::runtime_error("Failed to rotate ARGB Buffer! Status: " + std::to_string(status));
}

return destination;
Expand All @@ -232,7 +226,6 @@ FrameBuffer ResizePlugin::scaleARGBBuffer(vision::FrameBuffer frameBuffer, int w
auto targetString = rectToString(0, 0, width, height);
__android_log_print(ANDROID_LOG_INFO, TAG, "Scaling [%s] ARGB buffer to [%s]...", rectString.c_str(), targetString.c_str());


size_t channels = getChannelCount(PixelFormat::ARGB);
size_t channelSize = getBytesPerChannel(DataType::UINT8);
size_t argbSize = width * height * channels * channelSize;
Expand Down Expand Up @@ -349,15 +342,9 @@ FrameBuffer ResizePlugin::convertBufferToDataType(FrameBuffer frameBuffer, DataT
return destination;
}

jni::global_ref<jni::JByteBuffer> ResizePlugin::resize(jni::alias_ref<JImage> image,
int cropX, int cropY,
int cropWidth, int cropHeight,
int scaleWidth, int scaleHeight,
int rotation,
bool flip,
int /* PixelFormat */ pixelFormatOrdinal,
int /* DataType */ dataTypeOrdinal)
{
jni::global_ref<jni::JByteBuffer> ResizePlugin::resize(jni::alias_ref<JImage> image, int cropX, int cropY, int cropWidth, int cropHeight,
int scaleWidth, int scaleHeight, int rotation, bool flip,
int /* PixelFormat */ pixelFormatOrdinal, int /* DataType */ dataTypeOrdinal) {
PixelFormat pixelFormat = static_cast<PixelFormat>(pixelFormatOrdinal);
DataType dataType = static_cast<DataType>(dataTypeOrdinal);

Expand All @@ -376,10 +363,10 @@ jni::global_ref<jni::JByteBuffer> ResizePlugin::resize(jni::alias_ref<JImage> im
// 5 Flip ARGB if needed
result = flipARGBBuffer(result, flip);

// 5. Convert from ARGB -> ????
// 6. Convert from ARGB -> ????
result = convertARGBBufferTo(result, pixelFormat);

// 6. Convert from data type to other data type
// 7. Convert from data type to other data type
result = convertBufferToDataType(result, dataType);

return result.buffer;
Expand Down
9 changes: 2 additions & 7 deletions android/src/main/cpp/ResizePlugin.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,13 +40,8 @@ struct ResizePlugin : public HybridClass<ResizePlugin> {
private:
explicit ResizePlugin(const alias_ref<jhybridobject>& javaThis);

global_ref<JByteBuffer> resize(alias_ref<JImage> image,
int cropX, int cropY,
int cropWidth, int cropHeight,
int scaleWidth, int scaleHeight,
int rotation,
bool flip,
int /* PixelFormat */ pixelFormat, int /* DataType */ dataType);
global_ref<JByteBuffer> resize(alias_ref<JImage> image, int cropX, int cropY, int cropWidth, int cropHeight, int scaleWidth,
int scaleHeight, int rotation, bool flip, int /* PixelFormat */ pixelFormat, int /* DataType */ dataType);

FrameBuffer imageToFrameBuffer(alias_ref<JImage> image);
FrameBuffer cropARGBBuffer(FrameBuffer frameBuffer, int x, int y, int width, int height);
Expand Down
2 changes: 1 addition & 1 deletion example/ios/.xcode.env.local
View file
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
export NODE_BINARY=/Users/mrousavy/.nvm/versions/node/v18.17.0/bin/node
export NODE_BINARY=/Users/rodrigogomes/.nvm/versions/node/v18.2.0/bin/node

6 changes: 3 additions & 3 deletions example/src/App.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,11 @@ import {
useCameraPermission,
useFrameProcessor,
} from 'react-native-vision-camera';
import { useResizePlugin } from 'vision-camera-resize-plugin';
import { ROTATION, useResizePlugin } from 'vision-camera-resize-plugin';

export default function App() {
const permission = useCameraPermission();
const device = useCameraDevice('back');
const device = useCameraDevice('front');

React.useEffect(() => {
permission.requestPermission();
Expand All @@ -35,7 +35,7 @@ export default function App() {
},
pixelFormat: 'rgba',
dataType: 'uint8',
rotation: 180,
rotation: ROTATION.R90,
flip: true,
});
console.log(
Expand Down
95 changes: 92 additions & 3 deletions ios/ResizePlugin.mm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ @implementation ResizePlugin {
FrameBuffer* _argbBuffer;
// 2. ARGB (?x?) -> ARGB (!x!)
FrameBuffer* _resizeBuffer;
FrameBuffer* _flipBuffer;
FrameBuffer* _rotateBuffer;
// 3. ARGB (!x!) -> !!!! (!x!)
FrameBuffer* _convertBuffer;
// 3. uint8 -> other type (e.g. float32) if needed
Expand Down Expand Up @@ -348,6 +350,85 @@ - (FrameBuffer*)convertInt8Buffer:(FrameBuffer*)buffer toDataType:(ConvertDataTy
return _customTypeBuffer;
}

- (FrameBuffer*)flipARGBBuffer:(FrameBuffer*)buffer flip:(BOOL)flip {

if (!flip) {
return buffer;
}

NSLog(@"Flipping ARGB buffer...");

if (_flipBuffer == nil || _flipBuffer.width != buffer.width || _flipBuffer.height != buffer.height) {
_flipBuffer = [[FrameBuffer alloc] initWithWidth:buffer.width
height:buffer.height
pixelFormat:buffer.pixelFormat
dataType:buffer.dataType
proxy:_proxy];
}

vImage_Buffer src = *buffer.imageBuffer;
vImage_Buffer dest = *_flipBuffer.imageBuffer;

vImage_Error error = vImageHorizontalReflect_ARGB8888(&src, &dest, kvImageNoFlags);
if (error != kvImageNoError) {
[NSException raise:@"Flip Error" format:@"Failed to flip ARGB buffer! Error: %ld", error];
}

return _flipBuffer;
}

- (FrameBuffer*)rotateARGBBuffer:(FrameBuffer*)buffer rotation:(int)rotation {
if (rotation == 0) {
return buffer;
}

NSLog(@"Rotating ARGB buffer...");

int rotatedWidth = buffer.width;
int rotatedHeight = buffer.height;
if (rotation == 90 || rotation == 270) {
int temp = rotatedWidth;
rotatedWidth = rotatedHeight;
rotatedHeight = temp;
}

size_t bytesPerPixel = [FrameBuffer getBytesPerPixel:buffer.pixelFormat withType:buffer.dataType];

if (_rotateBuffer == nil || _rotateBuffer.width != rotatedWidth || _rotateBuffer.height != rotatedHeight) {
_rotateBuffer = [[FrameBuffer alloc] initWithWidth:rotatedWidth
height:rotatedHeight
pixelFormat:buffer.pixelFormat
dataType:buffer.dataType
proxy:_proxy];
}

vImage_Buffer src = *buffer.imageBuffer;
vImage_Buffer dest = *_rotateBuffer.imageBuffer;

vImage_Error error = kvImageNoError;
Pixel_8888 backgroundColor = {0, 0, 0, 0};
if (rotation == 90) {
error = vImageRotate90_ARGB8888(&src, &dest, kRotate90DegreesClockwise, backgroundColor, kvImageNoFlags);
} else if (rotation == 180) {
error = vImageRotate90_ARGB8888(&src, &dest, kRotate180DegreesClockwise, backgroundColor, kvImageNoFlags);
} else if (rotation == 270) {
error = vImageRotate90_ARGB8888(&src, &dest, kRotate270DegreesClockwise, backgroundColor, kvImageNoFlags);
} else {
[NSException raise:@"Invalid Rotation" format:@"Rotation must be 0, 90, 180, or 270 degrees."];
return nil;
}

if (error != kvImageNoError) {
[NSException raise:@"Rotation Error" format:@"Failed to rotate ARGB buffer! Error: %ld", error];
}

if (error != kvImageNoError) {
[NSException raise:@"Rotation Error" format:@"Failed to rotate ARGB buffer! Error: %ld", error];
}

return _rotateBuffer;
}

// Used only for debugging/inspecting the Image.
- (UIImage*)bufferToImage:(FrameBuffer*)buffer {
CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB();
Expand Down Expand Up @@ -385,6 +466,8 @@ - (id)callback:(Frame*)frame withArguments:(NSDictionary*)arguments {
double cropX = 0;
double cropY = 0;
NSDictionary* crop = arguments[@"crop"];
BOOL flip = [[arguments objectForKey:@"flip"] boolValue];
int rotation = [arguments[@"rotation"] intValue];
if (crop != nil) {
cropWidth = ((NSNumber*)crop[@"width"]).doubleValue;
cropHeight = ((NSNumber*)crop[@"height"]).doubleValue;
Expand Down Expand Up @@ -451,13 +534,19 @@ - (id)callback:(Frame*)frame withArguments:(NSDictionary*)arguments {
CGSize scaleSize = CGSizeMake(scaleWidth, scaleHeight);
result = [self resizeARGB:result crop:cropRect scale:scaleSize];

// 4. Convert ARGB -> ??? format
// 4. Rotate
result = [self rotateARGBBuffer:result rotation:rotation];

// 5. Flip
result = [self flipARGBBuffer:result flip:flip];

// 6. Convert ARGB -> ??? format
result = [self convertARGB:result to:pixelFormat];

// 5. Convert UINT8 -> ??? type
// 7. Convert UINT8 -> ??? type
result = [self convertInt8Buffer:result toDataType:dataType];

// 6. Return to JS
// 8. Return to JS
return result.sharedArray;
}

Expand Down
20 changes: 15 additions & 5 deletions src/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,13 @@ type OutputArray<T extends DataType> = T extends 'uint8'
? Float32Array
: never;

export enum ROTATION {
R0 = 0,
R90 = 90,
R180 = 180,
R270 = 270,
}

interface Size {
/**
* The width to resize the Frame to.
Expand All @@ -31,6 +38,9 @@ interface Rect extends Size {
}

interface Options<T extends DataType> {
/**
* If set to `true`, the image will be flipped horizontally.
*/
flip?: boolean;
/**
* Crops the image to the given target rect. This is applied first before scaling.
Expand All @@ -42,6 +52,11 @@ interface Options<T extends DataType> {
* Scale the image to the given target size. This is applied after cropping.
*/
scale?: Size;
/**
* rotation of the image in degrees.
* defaults to Rotation.R0 (0)
*/
rotation?: ROTATION;
/**
* Convert the Frame to the given target pixel format.
*
Expand All @@ -52,11 +67,6 @@ interface Options<T extends DataType> {
* - `'bgr'`: [B, G, R]
* - `'abgr'`: [A, B, G, R]
*/
rotation?: 0 | 90 | 180 | 270;
/**
* rotation of the image in degrees.
* defaults to 0
*/
pixelFormat: 'rgb' | 'rgba' | 'argb' | 'bgra' | 'bgr' | 'abgr';
/**
* The given type to use for the resulting buffer.
Expand Down

0 comments on commit ab2f2bc

Please sign in to comment.