From 99b177b3e5a50093f664eacd6b09576d17a7e64d Mon Sep 17 00:00:00 2001 From: Naushir Patuck Date: Tue, 23 May 2023 12:03:37 +0100 Subject: [PATCH] RASPBERRYPI ONLY: Handle mandatory stream flags Look for the RAW mandatory stream flag in the pipeline handler config file. If this flag is set, it guarantees that the application will provide buffers for Unicam Image, so override the minUnicamBuffers and minTotalUnicamBuffers config parameters in the following way: - If startup drop frames are required, allocate at least 1 internal buffer. - If no startup drop frames are required, do not allocate any internal buffers. Look for the Output 0 mandatory stream flag in in the pipeline handler config file. If this flag is set, it guarantees that the application will provide buffers for the ISP, do not allocate any internal buffers for the device. Add a new rpi_apps.yaml pipeline handler config file that enables both these flags. To use the file, set the following env variable for a custom build: export LIBCAMERA_RPI_CONFIG_FILE=/usr/local/share/libcamera/pipeline/rpi/vc4/rpi_apps.yaml or for a packaged install: export LIBCAMERA_RPI_CONFIG_FILE=/usr/share/libcamera/pipeline/rpi/vc4/rpi_apps.yaml Signed-off-by: Naushir Patuck --- .../pipeline/rpi/vc4/data/meson.build | 1 + .../pipeline/rpi/vc4/data/rpi_apps.yaml | 45 +++++++++++ src/libcamera/pipeline/rpi/vc4/vc4.cpp | 80 +++++++++++++++---- 3 files changed, 112 insertions(+), 14 deletions(-) create mode 100644 src/libcamera/pipeline/rpi/vc4/data/rpi_apps.yaml diff --git a/src/libcamera/pipeline/rpi/vc4/data/meson.build b/src/libcamera/pipeline/rpi/vc4/data/meson.build index cca5e3885..665ddb988 100644 --- a/src/libcamera/pipeline/rpi/vc4/data/meson.build +++ b/src/libcamera/pipeline/rpi/vc4/data/meson.build @@ -2,6 +2,7 @@ conf_files = files([ 'example.yaml', + 'rpi_apps.yaml', ]) install_data(conf_files, diff --git a/src/libcamera/pipeline/rpi/vc4/data/rpi_apps.yaml b/src/libcamera/pipeline/rpi/vc4/data/rpi_apps.yaml new file mode 100644 index 000000000..f2c849b7e --- /dev/null +++ b/src/libcamera/pipeline/rpi/vc4/data/rpi_apps.yaml @@ -0,0 +1,45 @@ +{ + "version": 1.0, + "target": "bcm2835", + + "pipeline_handler": + { + # The minimum number of internal buffers to be allocated for + # Unicam. This value must be greater than 0, but less than or + # equal to min_total_unicam_buffers. + # + # A larger number of internal buffers can reduce the occurrence + # of frame drops during high CPU loads, but might also cause + # additional latency in the system. + # + # Note that the pipeline handler might override this value and + # not allocate any internal buffers if it knows they will never + # be used. For example if the RAW stream is marked as mandatory + # and there are no dropped frames signalled for algorithm + # convergence. + # + "min_unicam_buffers": 2, + + # The minimum total (internal + external) buffer count used for + # Unicam. The number of internal buffers allocated for Unicam is + # given by: + # + # internal buffer count = max(min_unicam_buffers, + # min_total_unicam_buffers - external buffer count) + # + "min_total_unicam_buffers": 4, + + # Override any request from the IPA to drop a number of startup + # frames. + # + # "disable_startup_frame_drops": false, + + # The application will always provide a request buffer for the + # RAW stream, if it has been configured. + "raw_mandatory_stream": true, + + # The application will always provide a request buffer for the + # Output 0 stream, if it has been configured. + "output0_mandatory_stream": true, + } +} diff --git a/src/libcamera/pipeline/rpi/vc4/vc4.cpp b/src/libcamera/pipeline/rpi/vc4/vc4.cpp index a5cc4fc11..ea0be5942 100644 --- a/src/libcamera/pipeline/rpi/vc4/vc4.cpp +++ b/src/libcamera/pipeline/rpi/vc4/vc4.cpp @@ -105,6 +105,16 @@ class Vc4CameraData final : public RPi::CameraData * minTotalUnicamBuffers >= minUnicamBuffers */ unsigned int minTotalUnicamBuffers; + /* + * The application will always provide a request buffer for the + * RAW stream, if it has been configured. + */ + bool rawMandatoryStream; + /* + * The application will always provide a request buffer for the + * Output 0 stream, if it has been configured. + */ + bool output0MandatoryStream; }; Config config_; @@ -219,16 +229,47 @@ bool PipelineHandlerVc4::match(DeviceEnumerator *enumerator) int PipelineHandlerVc4::prepareBuffers(Camera *camera) { Vc4CameraData *data = cameraData(camera); - unsigned int numRawBuffers = 0; + unsigned int minUnicamBuffers = data->config_.minUnicamBuffers; + unsigned int minTotalUnicamBuffers = data->config_.minTotalUnicamBuffers; + unsigned int numRawBuffers = 0, minIspBuffers = 1; int ret; - for (Stream *s : camera->streams()) { - if (BayerFormat::fromPixelFormat(s->configuration().pixelFormat).isValid()) { - numRawBuffers = s->configuration().bufferCount; - break; + if (data->unicam_[Unicam::Image].getFlags() & StreamFlag::External) { + numRawBuffers = data->unicam_[Unicam::Image].getBuffers().size(); + /* + * If the application provides a guarantees that Unicam + * image buffers will always be provided for the RAW stream + * in a Request, we need: + * - at least 1 internal Unicam buffer to handle startup frame drops, + * - no internal Unicam buffers if there are no startup frame drops. + */ + if (data->config_.rawMandatoryStream) { + if (data->dropFrameCount_) { + minUnicamBuffers = 2; + minTotalUnicamBuffers = 2; + } else { + minUnicamBuffers = 0; + minTotalUnicamBuffers = 0; + } } } + if (data->isp_[Isp::Output0].getFlags() & StreamFlag::External) { + /* + * Since the ISP runs synchronous with the IPA and requests, + * we only ever need a maximum of one internal buffer. Any + * buffers the application wants to hold onto will already + * be exported through PipelineHandlerRPi::exportFrameBuffers(). + * + * However, as above, if the application provides a guarantee + * that the buffer will always be provided for the ISP Output0 + * stream in a Request, we don't need any internal buffers + * allocated. + */ + if (!data->dropFrameCount_ && data->config_.output0MandatoryStream) + minIspBuffers = 0; + } + /* Decide how many internal buffers to allocate. */ for (auto const stream : data->streams_) { unsigned int numBuffers; @@ -236,7 +277,6 @@ int PipelineHandlerVc4::prepareBuffers(Camera *camera) * For Unicam, allocate a minimum number of buffers for internal * use as we want to avoid any frame drops. */ - const unsigned int minBuffers = data->config_.minTotalUnicamBuffers; if (stream == &data->unicam_[Unicam::Image]) { /* * If an application has configured a RAW stream, allocate @@ -244,8 +284,9 @@ int PipelineHandlerVc4::prepareBuffers(Camera *camera) * we have at least minUnicamBuffers of internal buffers * to use to minimise frame drops. */ - numBuffers = std::max(data->config_.minUnicamBuffers, - minBuffers - numRawBuffers); + numBuffers = std::max(minUnicamBuffers, + minTotalUnicamBuffers - numRawBuffers); + LOG(RPI, Debug) << "Unicam::Image numBuffers " << numBuffers; } else if (stream == &data->isp_[Isp::Input]) { /* * ISP input buffers are imported from Unicam, so follow @@ -253,8 +294,9 @@ int PipelineHandlerVc4::prepareBuffers(Camera *camera) * available. */ numBuffers = numRawBuffers + - std::max(data->config_.minUnicamBuffers, - minBuffers - numRawBuffers); + std::max(minUnicamBuffers, + minTotalUnicamBuffers - numRawBuffers); + LOG(RPI, Debug) << "Isp::Input numBuffers " << numBuffers; } else if (stream == &data->unicam_[Unicam::Embedded]) { /* @@ -273,14 +315,18 @@ int PipelineHandlerVc4::prepareBuffers(Camera *camera) * buffers, as these will be recycled quicker. */ numBuffers = 12; + } else if (stream == &data->isp_[Isp::Output0]) { + /* Buffer count for this is handled in the earlier loop above. */ + numBuffers = minIspBuffers; + LOG(RPI, Debug) << "Isp::Output0 numBuffers " << numBuffers; } else { /* - * Since the ISP runs synchronous with the IPA and requests, - * we only ever need one set of internal buffers. Any buffers - * the application wants to hold onto will already be exported - * through PipelineHandlerRPi::exportFrameBuffers(). + * Same reasoning as for ISP Output 0, we only ever need + * a maximum of one internal buffer for Output1 (required + * for colour denoise) and ISP statistics. */ numBuffers = 1; + LOG(RPI, Debug) << "Other numBuffers " << numBuffers; } ret = stream->prepareBuffers(numBuffers); @@ -484,6 +530,8 @@ int Vc4CameraData::platformPipelineConfigure(const std::unique_ptr & config_ = { .minUnicamBuffers = 2, .minTotalUnicamBuffers = 4, + .rawMandatoryStream = false, + .output0MandatoryStream = false, }; if (!root) @@ -507,6 +555,10 @@ int Vc4CameraData::platformPipelineConfigure(const std::unique_ptr & phConfig["min_unicam_buffers"].get(config_.minUnicamBuffers); config_.minTotalUnicamBuffers = phConfig["min_total_unicam_buffers"].get(config_.minTotalUnicamBuffers); + config_.rawMandatoryStream = + phConfig["raw_mandatory_stream"].get(config_.rawMandatoryStream); + config_.output0MandatoryStream = + phConfig["output0_mandatory_stream"].get(config_.output0MandatoryStream); if (config_.minTotalUnicamBuffers < config_.minUnicamBuffers) { LOG(RPI, Error) << "Invalid configuration: min_total_unicam_buffers must be >= min_unicam_buffers";