diff --git a/README.md b/README.md index 020c81b..14d8453 100644 --- a/README.md +++ b/README.md @@ -40,15 +40,13 @@ The diagram below illustrates the RPS application architecture. During algorithm 1. [Create the input interface](./Documentation/README.md#input-interface-and-signal-conditioning), add signal conditioning, and start capturing data for ML model training. 2. [Select an ML model](./Documentation/README.md#create-ml-model), then use the captured data for training, analysis, and creation of the optimized ML model. 3. [Integrate the ML model](./Documentation/README.md#integrate-ml-model) into the SDS framework and analyze performance. -4. Configure `OUTPUT_PREDICTION_METADATA` based on your workflow: +4. Configure ML output prediction metadata at runtime using SDS user flag 5 (`1UL << 5`). - **Configuration file:** - `RockPaperScissors/AppKit-E8_USB/algorithm/AlgorithmTest.cproject.yml` - - Set `OUTPUT_PREDICTION_METADATA = 0` to view the generated `.sds` files using the Arm SDS VS Code extension. - - Set `OUTPUT_PREDICTION_METADATA = 1` to enable live inference streaming in Fusion Studio, which parses the prediction metadata (predicted class label, confidence score, and class index) to render overlayed frames. + - Set the flag to include prediction metadata (predicted class label, confidence score, and class index) for live inference streaming in Fusion Studio. + - Clear the flag to record `ML_Out.sds` with only class confidence scores. > [!Note] > - > By default, `OUTPUT_PREDICTION_METADATA` is set to `0` + > Prediction metadata is disabled by default. **Test Embedded Application:** diff --git a/RockPaperScissors/AppKit-E8_USB/algorithm/AlgorithmTest.cproject.yml b/RockPaperScissors/AppKit-E8_USB/algorithm/AlgorithmTest.cproject.yml index e1cad5d..ebc3409 100644 --- a/RockPaperScissors/AppKit-E8_USB/algorithm/AlgorithmTest.cproject.yml +++ b/RockPaperScissors/AppKit-E8_USB/algorithm/AlgorithmTest.cproject.yml @@ -27,12 +27,11 @@ project: - C10_USING_CUSTOM_GENERATED_MACROS - ET_NUM_INFERENCES: 1 - ET_LOG_DUMP_OUTPUT - - ET_DEBUG_BUFFER_SIZE: 0x8000 - - USE_PERFORMANCE_MONITOR - - USE_SEGGER_SYSVIEW - - OUTPUT_PREDICTION_METADATA: 0 - - setups: + - ET_DEBUG_BUFFER_SIZE: 0x8000 + - USE_PERFORMANCE_MONITOR + - USE_SEGGER_SYSVIEW + + setups: - setup: Scratch pool for non-simulator not-for-context: +SSE-320-U85 define: diff --git a/RockPaperScissors/AppKit-E8_USB/algorithm/algorithm.h b/RockPaperScissors/AppKit-E8_USB/algorithm/algorithm.h index f904420..4f653c9 100644 --- a/RockPaperScissors/AppKit-E8_USB/algorithm/algorithm.h +++ b/RockPaperScissors/AppKit-E8_USB/algorithm/algorithm.h @@ -17,9 +17,10 @@ */ #ifndef ALGORITHM_H_ -#define ALGORITHM_H_ - -#include +#define ALGORITHM_H_ + +#include +#include #ifdef __cplusplus extern "C" @@ -40,15 +41,18 @@ extern int32_t InitAlgorithm (void); extern void ResetAlgorithm (void); /** - \fn int32_t ExecuteAlgorithm (uint8_t *in_buf, uint32_t in_num, uint8_t *out_buf, uint32_t out_num) - \brief Execute algorithm under test. - \param[in] in_buf pointer to memory buffer containing input data for algorithm - \param[in] in_num number of data bytes in input data buffer (in bytes) - \param[out] out_buf pointer to memory buffer for returning algorithm output - \param[in] out_num maximum number of data bytes returned as algorithm output (in bytes) - \return 0 on success; -1 on error -*/ -extern int32_t ExecuteAlgorithm (uint8_t *in_buf, uint32_t in_num, uint8_t *out_buf, uint32_t out_num); + \fn int32_t ExecuteAlgorithm (uint8_t *in_buf, uint32_t in_num, uint8_t *out_buf, uint32_t out_num, bool output_prediction_metadata) + \brief Execute algorithm under test. + \param[in] in_buf pointer to memory buffer containing input data for algorithm + \param[in] in_num number of data bytes in input data buffer (in bytes) + \param[out] out_buf pointer to memory buffer for returning algorithm output + \param[in] out_num maximum number of data bytes returned as algorithm output (in bytes) + \param[in] output_prediction_metadata + true to emit predicted class metadata, false to emit class scores + \return 0 on success; -1 on error +*/ +extern int32_t ExecuteAlgorithm (uint8_t *in_buf, uint32_t in_num, uint8_t *out_buf, uint32_t out_num, + bool output_prediction_metadata); #ifdef __cplusplus } diff --git a/RockPaperScissors/AppKit-E8_USB/algorithm/algorithm_config.h b/RockPaperScissors/AppKit-E8_USB/algorithm/algorithm_config.h index 18a3f83..b06002e 100644 --- a/RockPaperScissors/AppKit-E8_USB/algorithm/algorithm_config.h +++ b/RockPaperScissors/AppKit-E8_USB/algorithm/algorithm_config.h @@ -25,14 +25,13 @@ // Input Data block size, in bytes #ifndef ALGO_DATA_IN_BLOCK_SIZE #define ALGO_DATA_IN_BLOCK_SIZE (ML_IMAGE_WIDTH * ML_IMAGE_HEIGHT * 3U) -#endif - -#ifndef ALGO_DATA_OUT_BLOCK_SIZE -#if defined(OUTPUT_PREDICTION_METADATA) && OUTPUT_PREDICTION_METADATA -#define ALGO_DATA_OUT_BLOCK_SIZE (120U) -#else -#define ALGO_DATA_OUT_BLOCK_SIZE (MODEL_NUM_CLASSES * sizeof(float)) -#endif -#endif - -#endif +#endif + +#define ALGO_DATA_OUT_CLASS_SCORES_BLOCK_SIZE (MODEL_NUM_CLASSES * sizeof(float)) +#define ALGO_DATA_OUT_METADATA_BLOCK_SIZE (120U) + +#ifndef ALGO_DATA_OUT_BLOCK_SIZE +#define ALGO_DATA_OUT_BLOCK_SIZE ALGO_DATA_OUT_CLASS_SCORES_BLOCK_SIZE +#endif + +#endif diff --git a/RockPaperScissors/AppKit-E8_USB/algorithm/algorithm_user.cpp b/RockPaperScissors/AppKit-E8_USB/algorithm/algorithm_user.cpp index d7c87a4..cc535bd 100644 --- a/RockPaperScissors/AppKit-E8_USB/algorithm/algorithm_user.cpp +++ b/RockPaperScissors/AppKit-E8_USB/algorithm/algorithm_user.cpp @@ -159,16 +159,19 @@ void ResetAlgorithm (void) { */ /** - \fn int32_t ExecuteAlgorithm (uint8_t *in_buf, uint32_t in_num, uint8_t *out_buf, uint32_t out_num) - \brief Execute algorithm under test. - \param[in] in_buf pointer to input frame buffer (RGB888, HWC, 224x224x3) - \param[in] in_num number of bytes in input buffer - \param[out] out_buf pointer to output buffer (receives runner_output_label_t) - \param[in] out_num maximum bytes available in output buffer - \return 0 on success; -1 on error -*/ -int32_t ExecuteAlgorithm(uint8_t *in_buf, uint32_t in_num, - uint8_t *out_buf, uint32_t out_num) { + \fn int32_t ExecuteAlgorithm (uint8_t *in_buf, uint32_t in_num, uint8_t *out_buf, uint32_t out_num, bool output_prediction_metadata) + \brief Execute algorithm under test. + \param[in] in_buf pointer to input frame buffer (RGB888, HWC, 224x224x3) + \param[in] in_num number of bytes in input buffer + \param[out] out_buf pointer to output buffer (receives runner_output_label_t) + \param[in] out_num maximum bytes available in output buffer + \param[in] output_prediction_metadata + true to emit predicted class metadata, false to emit class scores + \return 0 on success; -1 on error +*/ +int32_t ExecuteAlgorithm(uint8_t *in_buf, uint32_t in_num, + uint8_t *out_buf, uint32_t out_num, + bool output_prediction_metadata) { #ifndef SIMULATOR vStreamStatus_t v_status; @@ -223,7 +226,8 @@ int32_t ExecuteAlgorithm(uint8_t *in_buf, uint32_t in_num, uint32_t post_process_time = profiler_start(); #endif - postprocess(*ctx, in_buf, IMAGE_WIDTH, IMAGE_HEIGHT, out_buf, out_num); + postprocess(*ctx, in_buf, IMAGE_WIDTH, IMAGE_HEIGHT, out_buf, out_num, + output_prediction_metadata); #if !defined(SIMULATOR) && defined(USE_SEGGER_SYSVIEW) SEGGER_SYSVIEW_MarkStop(SYSVIEW_MARKER_POST_PROCESS); @@ -303,4 +307,4 @@ int32_t ExecuteAlgorithm(uint8_t *in_buf, uint32_t in_num, #endif return 0; -} \ No newline at end of file +} diff --git a/RockPaperScissors/AppKit-E8_USB/algorithm/arm_executor_runner.cc b/RockPaperScissors/AppKit-E8_USB/algorithm/arm_executor_runner.cc index 940882f..3603ee7 100644 --- a/RockPaperScissors/AppKit-E8_USB/algorithm/arm_executor_runner.cc +++ b/RockPaperScissors/AppKit-E8_USB/algorithm/arm_executor_runner.cc @@ -939,31 +939,35 @@ void print_outputs(RunnerContext& ctx) * \param[in] ctx RunnerContext after a successful run_inference(). * \param[in] img_buf RGB888 frame buffer to draw the label onto. * \param[in] img_width Frame width in pixels. - * \param[in] img_height Frame height in pixels - * \param[out] out_buf Caller buffer to receive detection_result_t result. - * \param[in] out_num Byte size of out_buf. - */ -void postprocess(RunnerContext& ctx, uint8_t* img_buf, - uint32_t img_width, uint32_t img_height, - uint8_t* out_buf, uint32_t out_num) { + * \param[in] img_height Frame height in pixels + * \param[out] out_buf Caller buffer to receive detection_result_t result. + * \param[in] out_num Byte size of out_buf. + * \param[in] output_prediction_metadata + * true to emit predicted class metadata, false to emit class scores. + */ +void postprocess(RunnerContext& ctx, uint8_t* img_buf, + uint32_t img_width, uint32_t img_height, + uint8_t* out_buf, uint32_t out_num, + bool output_prediction_metadata) { memset(&output_label, 0, sizeof(output_label)); /* Decode output tensor → output_label, conf_int, classify_object */ print_outputs(ctx); - /* Copy shortened label plus confidence into caller's output buffer */ -#if OUTPUT_PREDICTION_METADATA - if (out_num >= sizeof(output_label_t)) { - memcpy(out_buf, &output_label, sizeof(output_label)); - } -#else - if (out_num >= sizeof(class_probs)) { - memcpy(out_buf, class_probs, sizeof(class_probs)); - } -#endif - - /* Only draw if label is valid */ + if (output_prediction_metadata) { + /* Copy predicted label, confidence, and class index into caller's output buffer */ + if (out_num >= sizeof(output_label_t)) { + memcpy(out_buf, &output_label, sizeof(output_label)); + } + } else { + /* Copy class confidence scores into caller's output buffer */ + if (out_num >= sizeof(class_probs)) { + memcpy(out_buf, class_probs, sizeof(class_probs)); + } + } + + /* Only draw if label is valid */ if (output_label.label_name[0] != '\0') { /* Format label string and draw onto frame */ @@ -1046,4 +1050,4 @@ bool run_inference(RunnerContext& ctx) { ctx.method_name, status); return (status == Error::Ok); -} \ No newline at end of file +} diff --git a/RockPaperScissors/AppKit-E8_USB/algorithm/arm_executor_runner.h b/RockPaperScissors/AppKit-E8_USB/algorithm/arm_executor_runner.h index 3ee52df..2d6b861 100644 --- a/RockPaperScissors/AppKit-E8_USB/algorithm/arm_executor_runner.h +++ b/RockPaperScissors/AppKit-E8_USB/algorithm/arm_executor_runner.h @@ -110,11 +110,13 @@ bool run_inference(RunnerContext &ctx); * \param[in] ctx RunnerContext after a successful run_inference(). * \param[in] img_buf RGB888 frame buffer to draw the label onto. * \param[in] img_width Frame width in pixels. - * \param[in] img_height Frame height in pixels - * \param[out] out_buf Caller buffer to receive runner_output_label_t result. - * \param[in] out_num Byte size of out_buf. - */ -void postprocess(RunnerContext &ctx, uint8_t *img_buf, uint32_t img_width, uint32_t img_height, - uint8_t *out_buf, uint32_t out_num); - -#endif /* ARM_EXECUTOR_RUNNER_H */ \ No newline at end of file + * \param[in] img_height Frame height in pixels + * \param[out] out_buf Caller buffer to receive runner_output_label_t result. + * \param[in] out_num Byte size of out_buf. + * \param[in] output_prediction_metadata + * true to emit predicted class metadata, false to emit class scores. + */ +void postprocess(RunnerContext &ctx, uint8_t *img_buf, uint32_t img_width, uint32_t img_height, + uint8_t *out_buf, uint32_t out_num, bool output_prediction_metadata); + +#endif /* ARM_EXECUTOR_RUNNER_H */ diff --git a/RockPaperScissors/AppKit-E8_USB/algorithm/sds_control.c b/RockPaperScissors/AppKit-E8_USB/algorithm/sds_control.c index 2c8852d..0c236a8 100644 --- a/RockPaperScissors/AppKit-E8_USB/algorithm/sds_control.c +++ b/RockPaperScissors/AppKit-E8_USB/algorithm/sds_control.c @@ -45,6 +45,7 @@ osThreadAttr_t attr_sdsControlThread = { static volatile uint32_t idle_cnt = 0U; static volatile uint8_t rst_idle_cnt = 0U; static uint32_t no_load_cnt = 0U; +static volatile bool output_prediction_metadata_enabled = false; #ifdef RTE_CMSIS_RTOS2_RTX5 // Measure system idle time if OS is RTX5 @@ -152,6 +153,26 @@ void sdsStatusLED (void) { ticks--; } +// Output metadata control state driven by SDSIO user flag 5. +bool sdsIsPredictionMetadataEnabled (void) { + return output_prediction_metadata_enabled; +} + +static void sdsPredictionMetadataControlUpdate (void) { + bool requested; + + requested = ((sdsFlags & SDS_USER_FLAG_OUTPUT_PREDICTION_METADATA) != 0U); + + if (requested != output_prediction_metadata_enabled) { + output_prediction_metadata_enabled = requested; + if (requested) { + sdsFlagsModify(SDS_USER_FLAG_OUTPUT_PREDICTION_METADATA, 0U); + } else { + sdsFlagsModify(0U, SDS_USER_FLAG_OUTPUT_PREDICTION_METADATA); + } + } +} + // SDS event callback static void sds_event_callback (sdsId_t id, uint32_t event) { (void)id; @@ -193,6 +214,7 @@ __NO_RETURN void sdsControlThread (void *argument) { for (;;) { sdsExchange(); // Exchange control information with host + sdsPredictionMetadataControlUpdate(); // Update runtime ML_Out format control // Detect if user button was pressed btn_val = vioGetSignal(vioBUTTON0); diff --git a/RockPaperScissors/AppKit-E8_USB/algorithm/sds_control.h b/RockPaperScissors/AppKit-E8_USB/algorithm/sds_control.h index 16b4f22..82ab88c 100644 --- a/RockPaperScissors/AppKit-E8_USB/algorithm/sds_control.h +++ b/RockPaperScissors/AppKit-E8_USB/algorithm/sds_control.h @@ -17,18 +17,24 @@ */ #ifndef SDS_CONTROL_H_ -#define SDS_CONTROL_H_ - -#include -#include "cmsis_compiler.h" - -#ifdef __cplusplus -extern "C" -{ -#endif - -// SDS control thread function -extern __NO_RETURN void sdsControlThread (void *argument); +#define SDS_CONTROL_H_ + +#include +#include +#include "cmsis_compiler.h" + +#define SDS_USER_FLAG_OUTPUT_PREDICTION_METADATA (1UL << 5) + +#ifdef __cplusplus +extern "C" +{ +#endif + +// Returns true when ML_Out should include predicted class metadata. +extern bool sdsIsPredictionMetadataEnabled (void); + +// SDS control thread function +extern __NO_RETURN void sdsControlThread (void *argument); // Application main function extern int32_t app_main (void); diff --git a/RockPaperScissors/AppKit-E8_USB/algorithm/sds_main.c b/RockPaperScissors/AppKit-E8_USB/algorithm/sds_main.c index 8a9ee4d..16becfc 100644 --- a/RockPaperScissors/AppKit-E8_USB/algorithm/sds_main.c +++ b/RockPaperScissors/AppKit-E8_USB/algorithm/sds_main.c @@ -19,10 +19,11 @@ #include "cmsis_os2.h" #include "sds.h" #include "sds_main.h" -#include "algorithm_config.h" -#include "algorithm.h" -#include "data_in.h" -#include "app_setup.h" +#include "algorithm_config.h" +#include "algorithm.h" +#include "data_in.h" +#include "app_setup.h" +#include "sds_control.h" // SDS system state consistent with main AlgorithmThread loop @@ -34,26 +35,44 @@ uint32_t timeslot = 0U; // Flag indicating whether images captured by camera are being recorded uint8_t record_camera = 0U; -// Algorithm input/output data buffer -static uint8_t algo_data_in_buf [ALGO_DATA_IN_BLOCK_SIZE] __ALIGNED(4); -static uint8_t algo_data_out_buf[ALGO_DATA_OUT_BLOCK_SIZE] __ALIGNED(4); - -// SDS buffers -static uint8_t sds_camera_buf [CAMERA_FRAME_SIZE + 2048] __ALIGNED(4); -static uint8_t sds_data_in_buf [ALGO_DATA_IN_BLOCK_SIZE + 2048] __ALIGNED(4); -static uint8_t sds_data_out_buf[(ALGO_DATA_OUT_BLOCK_SIZE * 2) + 2048] __ALIGNED(4); +// Algorithm input/output data buffer +static uint8_t algo_data_in_buf [ALGO_DATA_IN_BLOCK_SIZE] __ALIGNED(4); +static uint8_t algo_data_out_buf[ALGO_DATA_OUT_BLOCK_SIZE] __ALIGNED(4); +static uint8_t algo_data_out_metadata_buf[ALGO_DATA_OUT_METADATA_BLOCK_SIZE] __ALIGNED(4); + +// SDS buffers +static uint8_t sds_camera_buf [CAMERA_FRAME_SIZE + 2048] __ALIGNED(4); +static uint8_t sds_data_in_buf [ALGO_DATA_IN_BLOCK_SIZE + 2048] __ALIGNED(4); +static uint8_t sds_data_out_buf[(ALGO_DATA_OUT_BLOCK_SIZE * 2) + 2048] __ALIGNED(4); +static uint8_t sds_data_out_metadata_buf[(ALGO_DATA_OUT_METADATA_BLOCK_SIZE * 2) + 2048] __ALIGNED(4); // SDS stream identifiers sdsId_t sds_camera_id = NULL; static sdsId_t sds_data_in_id = NULL; static sdsId_t sds_data_out_id = NULL; -// Recording/playback mode text -static const char *SDS_MODE[] = { "recording", "playback" }; - -// Public functions - -/** +// Recording/playback mode text +static const char *SDS_MODE[] = { "recording", "playback" }; + +// Public functions + +static uint8_t *active_algo_data_out_buf = algo_data_out_buf; +static uint32_t active_algo_data_out_size = ALGO_DATA_OUT_BLOCK_SIZE; +static bool active_output_prediction_metadata = false; + +static void ConfigureAlgorithmOutput (bool output_prediction_metadata) { + if (output_prediction_metadata) { + active_algo_data_out_buf = algo_data_out_metadata_buf; + active_algo_data_out_size = ALGO_DATA_OUT_METADATA_BLOCK_SIZE; + active_output_prediction_metadata = true; + } else { + active_algo_data_out_buf = algo_data_out_buf; + active_algo_data_out_size = ALGO_DATA_OUT_BLOCK_SIZE; + active_output_prediction_metadata = false; + } +} + +/** \fn int32_t OpenStreams (void) \brief Open streams used by the application. \return 0 on success; -1 on error @@ -87,11 +106,16 @@ int32_t OpenStreams (void) { } } sds_data_in_id = sdsOpen("ML_In", sdsModeWrite, sds_data_in_buf, sizeof(sds_data_in_buf)); - } - // Open stream for recording of output data - if (sds_data_in_id != NULL) { - sds_data_out_id = sdsOpen("ML_Out", sdsModeWrite, sds_data_out_buf, sizeof(sds_data_out_buf)); - } + } + // Open stream for recording of output data + if (sds_data_in_id != NULL) { + ConfigureAlgorithmOutput(sdsIsPredictionMetadataEnabled()); + if (active_output_prediction_metadata) { + sds_data_out_id = sdsOpen("ML_Out", sdsModeWrite, sds_data_out_metadata_buf, sizeof(sds_data_out_metadata_buf)); + } else { + sds_data_out_id = sdsOpen("ML_Out", sdsModeWrite, sds_data_out_buf, sizeof(sds_data_out_buf)); + } + } SDS_ASSERT(sds_data_in_id != NULL); SDS_ASSERT(sds_data_out_id != NULL); @@ -160,10 +184,10 @@ int32_t CloseStreams (void) { // Algorithm Thread function -__NO_RETURN void AlgorithmThread (void *argument) { - uint32_t sds_flags; - int32_t ret; - (void)argument; +__NO_RETURN void AlgorithmThread (void *argument) { + uint32_t sds_flags; + int32_t ret; + (void)argument; // Initialize data acquisition InitInputData(); @@ -230,23 +254,24 @@ __NO_RETURN void AlgorithmThread (void *argument) { } while (ret == SDS_NO_SPACE); SDS_ASSERT(ret == sizeof(algo_data_in_buf)); } - } - - // Execute algorithm under test - if (ExecuteAlgorithm(algo_data_in_buf, sizeof(algo_data_in_buf), algo_data_out_buf, sizeof(algo_data_out_buf)) != 0) { - // If there was an error executing algorithm skip recording - continue; - } + } + + // Execute algorithm under test + if (ExecuteAlgorithm(algo_data_in_buf, sizeof(algo_data_in_buf), active_algo_data_out_buf, + active_algo_data_out_size, active_output_prediction_metadata) != 0) { + // If there was an error executing algorithm skip recording + continue; + } if (sds_state == SDS_STATE_ACTIVE) { - // Record algorithm output data - do { - ret = sdsWrite(sds_data_out_id, timeslot, algo_data_out_buf, sizeof(algo_data_out_buf)); - if (ret == SDS_NO_SPACE) { - osDelay(10U); - } - } while (ret == SDS_NO_SPACE); - SDS_ASSERT(ret == sizeof(algo_data_out_buf)); - } - } -} + // Record algorithm output data + do { + ret = sdsWrite(sds_data_out_id, timeslot, active_algo_data_out_buf, active_algo_data_out_size); + if (ret == SDS_NO_SPACE) { + osDelay(10U); + } + } while (ret == SDS_NO_SPACE); + SDS_ASSERT(ret == active_algo_data_out_size); + } + } +}