Tolerate unknown SEI and expose SEI payloads on decoded pictures

README.md

@@ -48,6 +48,11 @@ The EVC defines two profiles, including "**Baseline Profile**" and "**Main Profi
     You can change '-G' option with proper version of Visual Studio.
 
 ### ARM (64-bit)
+On an aarch64 host no special option is needed: the architecture is detected
+automatically, so the Linux instructions above apply as-is. Architectures
+without SIMD support build automatically with a plain C fallback.
+The instructions below are for cross-compiling on an x86 host.
+
 - Build Requirements
   - CMake 3.5 or later (download from [https://cmake.org/](https://cmake.org/))
   - gcc-aarch64-linux-gnu 
@@ -146,14 +151,59 @@ XEVD supports main and baseline profiles of EVC.
 |-----------------------|-----------|------------------------------------------------|
 | -i, --input           | -         | file name of input bitstream                   |
 | -o, --output          | -         | file name of output video                      |
-| -m, --threads         | 1         | mumber of threads to be created                |
+| -m, --threads         | 1         | number of threads to be created                |
 
 
 >More options can be found when type **xevd_app** only.
 
 ### Example
 	xevd_app -i input_bitstream.evc -o output_video.yuv
 
+## Programming Guide
+The following code is a pseudo code for understanding how to use the library
+```c
+#include <xevd.h>
+
+XEVD_CDSC cdsc;
+memset(&cdsc, 0, sizeof(XEVD_CDSC));
+cdsc.threads = 1;
+
+XEVD id = xevd_create(&cdsc, NULL);
+
+XEVD_BITB bitb; /* one nal unit per call */
+XEVD_STAT stat;
+XEVD_IMGB *imgb;
+
+while (read_nal_unit(&bitb))
+{
+    xevd_decode(id, &bitb, &stat);
+    if (stat.fnum >= 0 && xevd_pull(id, &imgb) == XEVD_OK)
+    {
+        write_image(imgb);
+        imgb->release(imgb);
+    }
+}
+/* flush remaining (reordered) pictures with xevd_pull(), then clean up */
+xevd_delete(id);
+```
+
+### Reading SEI payloads
+SEI payloads found in the access unit of a picture (e.g. HDR metadata) are
+exposed on the picture returned by `xevd_pull()`. The memory belongs to the
+library and is valid until the picture is released, so copy the payloads
+before calling `imgb->release()`.
+```c
+if (imgb->ndata[XEVD_IMGB_SEI_SLOT] == XEVD_SEI_MAGIC)
+{
+    XEVD_SEI *sei = (XEVD_SEI *)imgb->pdata[XEVD_IMGB_SEI_SLOT];
+    for (int i = 0; i < sei->num_payloads; i++)
+    {
+        XEVD_SEI_PAYLOAD *p = &sei->payloads[i];
+        /* p->payload_type, p->payload_size, p->payload */
+    }
+}
+```
+
 ## How to contribute
 Contributions are welcome through GitHub pull requests.
 

inc/xevd.h

@@ -139,6 +139,41 @@ extern "C"
 #define XEVD_NUT_FD                      (27)
 #define XEVD_NUT_SEI                     (28)
 
+/*****************************************************************************
+ * SEI payloads (ISO/IEC 23094-1 Annex D)
+ *****************************************************************************/
+typedef enum _XEVD_SEI_PAYLOAD_TYPE {
+    XEVD_SEI_BUFFERING_PERIOD                = 0,
+    XEVD_SEI_PICTURE_TIMING                  = 1,
+    XEVD_SEI_USER_DATA_REGISTERED_ITU_T_T35  = 4,
+    XEVD_SEI_USER_DATA_UNREGISTERED          = 5,
+    XEVD_SEI_RECOVERY_POINT                  = 6,
+    XEVD_SEI_MASTERING_DISPLAY_INFO          = 137,
+    XEVD_SEI_CONTENT_LIGHT_LEVEL_INFO        = 144,
+    XEVD_SEI_AMBIENT_VIEWING_ENVIRONMENT     = 148,
+} XEVD_SEI_PAYLOAD_TYPE;
+
+typedef struct _XEVD_SEI_PAYLOAD {
+    int                     payload_size;
+    XEVD_SEI_PAYLOAD_TYPE   payload_type;
+    unsigned char         * payload;
+} XEVD_SEI_PAYLOAD;
+
+typedef struct _XEVD_SEI {
+    int                num_payloads;
+    XEVD_SEI_PAYLOAD * payloads;
+} XEVD_SEI;
+
+/* SEI payloads parsed from the access unit of an output picture are exposed
+   on the picture returned by xevd_pull(): when imgb->ndata[XEVD_IMGB_SEI_SLOT]
+   equals XEVD_SEI_MAGIC, imgb->pdata[XEVD_IMGB_SEI_SLOT] points to an XEVD_SEI.
+   The memory belongs to the library and stays valid until the imgb is
+   released, so copy the payloads before releasing the picture. Note that
+   ndata[XEVD_IMGB_SEI_SLOT]/pdata[XEVD_IMGB_SEI_SLOT] of the input bitstream
+   buffer are not propagated to output pictures. */
+#define XEVD_IMGB_SEI_SLOT               (3)
+#define XEVD_SEI_MAGIC                   (0x58534549) /* 'XSEI' */
+
 /*****************************************************************************
  * slice type
  *****************************************************************************/

src_base/xevd.c

@@ -63,6 +63,7 @@ static XEVD_CTX * ctx_alloc(void)
 
 static void ctx_free(XEVD_CTX * ctx)
 {
+    xevd_mfree(ctx->sei_pend);
     xevd_mfree_fast(ctx);
 }
 
@@ -1995,12 +1996,16 @@ int xevd_dec_nalu(XEVD_CTX * ctx, XEVD_BITB * bitb, XEVD_STAT * stat)
         ctx->pic->imgb->ts[XEVD_TS_PTS] = bitb->ts[XEVD_TS_DTS] + coding_delay * ctx->ts.frame_duration_time;
 
         for (int i=0; i<XEVD_NDATA_NUM; i++) {
+            if (i == XEVD_IMGB_SEI_SLOT) continue; /* reserved for SEI exposure */
             ctx->pic->imgb->ndata[i] = bitb->ndata[i];
         }
 
         for (int i=0; i<XEVD_PDATA_NUM; i++) {
+            if (i == XEVD_IMGB_SEI_SLOT) continue; /* reserved for SEI exposure */
             ctx->pic->imgb->pdata[i] = bitb->pdata[i];
         }
+
+        xevd_sei_attach(ctx, ctx->pic->imgb);
     }
     else if (nalu->nal_unit_type_plus1 - 1 == XEVD_NUT_SEI)
     {

src_base/xevd_def.h

@@ -1470,6 +1470,13 @@ struct _XEVD_CTX
     void                  * pf;
 
     XEVD_SCAN_TABLES      * scan_tables;
+
+    /* SEI payloads parsed for the current access unit, pending until the
+       picture is decoded. layout: [s32 type][s32 size][size bytes] repeated */
+    u8                    * sei_pend;
+    int                     sei_pend_size;
+    int                     sei_pend_cap;
+    int                     sei_pend_num;
 };
 
 

src_base/xevd_eco.c

@@ -1579,6 +1579,41 @@ int xevd_eco_sh(XEVD_BSR * bs, XEVD_SPS * sps, XEVD_PPS * pps, XEVD_SH * sh, int
     return XEVD_OK;
 }
 
+/* read a payload from the bitstream and append it to the pending SEI list */
+static int sei_pend_add(XEVD_CTX * ctx, u32 payload_type, u32 payload_size, XEVD_BSR * bs)
+{
+    int need = ctx->sei_pend_size + 8 + (int)payload_size;
+    u32 val;
+
+    if(need > ctx->sei_pend_cap)
+    {
+        int cap = ctx->sei_pend_cap == 0 ? 4096 : ctx->sei_pend_cap;
+        while(cap < need) cap *= 2;
+        u8 *buf = (u8 *)xevd_malloc(cap);
+        xevd_assert_rv(buf, XEVD_ERR_OUT_OF_MEMORY);
+        if(ctx->sei_pend)
+        {
+            xevd_mcpy(buf, ctx->sei_pend, ctx->sei_pend_size);
+            xevd_mfree(ctx->sei_pend);
+        }
+        ctx->sei_pend = buf;
+        ctx->sei_pend_cap = cap;
+    }
+
+    u8 *dst = ctx->sei_pend + ctx->sei_pend_size;
+    *((s32 *)dst) = (s32)payload_type;
+    *((s32 *)(dst + 4)) = (s32)payload_size;
+    dst += 8;
+    for(u32 i = 0; i < payload_size; i++)
+    {
+        xevd_bsr_read(bs, &val, 8);
+        dst[i] = (u8)val;
+    }
+    ctx->sei_pend_size = need;
+    ctx->sei_pend_num++;
+    return XEVD_OK;
+}
+
 int xevd_eco_sei(XEVD_CTX * ctx, XEVD_BSR * bs)
 {
 #if TRACE_HLS
@@ -1587,12 +1622,18 @@ int xevd_eco_sei(XEVD_CTX * ctx, XEVD_BSR * bs)
 #endif
     u32 payload_type, payload_size;
     u32 pic_sign[N_C][16];
+    u32 val;
+    int ret;
 
     /* should be aligned before adding user data */
     xevd_assert_rv(XEVD_BSR_IS_BYTE_ALIGN(bs), XEVD_ERR_UNKNOWN);
 
+    /* an SEI NAL unit may carry multiple sei_message()s; the last byte of the
+       rbsp is the trailing bits byte */
+    do
+    {
     payload_type = 0;
-    u32 val = 0;
+    val = 0;
 
     do
     {
@@ -1610,23 +1651,6 @@ int xevd_eco_sei(XEVD_CTX * ctx, XEVD_BSR * bs)
 
     switch (payload_type)
     {
-    case XEVD_USER_DATA_UNREGISTERED:
-        xevd_assert(payload_size >= ISO_IEC_11578_LEN);
-        u32 val;
-
-        for (u32 i = 0; i < ISO_IEC_11578_LEN; i++)
-        {
-            u8 uuid_iso_iec_11578_out[16];
-            xevd_bsr_read(bs, &val, 8);
-            uuid_iso_iec_11578_out[i] = val;
-        }
-
-        u32 sei_resize = payload_size - ISO_IEC_11578_LEN;
-        for (u32 i = 0; i < sei_resize; i++)
-        {
-            xevd_bsr_read(bs, &val, 8);
-        }
-        break;
     case XEVD_UD_PIC_SIGNATURE:
         /* read signature (HASH) from bitstream */
         for (int i = 0; i < ctx->pic[0].imgb->np; ++i)
@@ -1641,8 +1665,12 @@ int xevd_eco_sei(XEVD_CTX * ctx, XEVD_BSR * bs)
         break;
 
     default:
-        xevd_assert_rv(0, XEVD_ERR_UNEXPECTED);
+        /* keep the payload for the caller; unsupported SEI must not fail the decode */
+        ret = sei_pend_add(ctx, payload_type, payload_size, bs);
+        xevd_assert_rv(ret == XEVD_OK, ret);
+        break;
     }
+    } while (bs->size - XEVD_BSR_GET_READ_BYTE(bs) > 1);
 #if TRACE_HLS
     XEVD_TRACE_STR("************ SEI End   ************\n");
     XEVD_TRACE_STR("***********************************\n");

src_base/xevd_util.c

@@ -104,6 +104,13 @@ static void imgb_delete(XEVD_IMGB * imgb)
     int i;
     xevd_assert_r(imgb);
 
+    if(imgb->ndata[XEVD_IMGB_SEI_SLOT] == XEVD_SEI_MAGIC && imgb->pdata[XEVD_IMGB_SEI_SLOT])
+    {
+        xevd_mfree(imgb->pdata[XEVD_IMGB_SEI_SLOT]);
+        imgb->pdata[XEVD_IMGB_SEI_SLOT] = NULL;
+        imgb->ndata[XEVD_IMGB_SEI_SLOT] = 0;
+    }
+
     for(i=0; i<XEVD_IMGB_MAX_PLANE; i++)
     {
         if (imgb->baddr[i]) xevd_mfree(imgb->baddr[i]);
@@ -1492,6 +1499,61 @@ void xevd_picbuf_free(PICBUF_ALLOCATOR * pa, XEVD_PIC * pic)
     xevd_picbuf_lc_free(pic);
 }
 
+/* move the pending SEI payloads of the current access unit onto the decoded
+   picture, replacing whatever a previous use of the (recycled) buffer left */
+void xevd_sei_attach(XEVD_CTX * ctx, XEVD_IMGB * imgb)
+{
+    if(imgb->ndata[XEVD_IMGB_SEI_SLOT] == XEVD_SEI_MAGIC && imgb->pdata[XEVD_IMGB_SEI_SLOT])
+    {
+        xevd_mfree(imgb->pdata[XEVD_IMGB_SEI_SLOT]);
+    }
+    imgb->pdata[XEVD_IMGB_SEI_SLOT] = NULL;
+    imgb->ndata[XEVD_IMGB_SEI_SLOT] = 0;
+
+    if(ctx->sei_pend_num <= 0)
+    {
+        return;
+    }
+
+    int bytes = ctx->sei_pend_size - ctx->sei_pend_num * 8;
+    int total = (int)sizeof(XEVD_SEI) + ctx->sei_pend_num * (int)sizeof(XEVD_SEI_PAYLOAD) + bytes;
+
+    XEVD_SEI *sei = (XEVD_SEI *)xevd_malloc(total);
+    if(sei == NULL) /* dropping SEI is not fatal for the decode */
+    {
+        ctx->sei_pend_size = 0;
+        ctx->sei_pend_num  = 0;
+        return;
+    }
+
+    XEVD_SEI_PAYLOAD *pls = (XEVD_SEI_PAYLOAD *)(sei + 1);
+    u8               *dst = (u8 *)(pls + ctx->sei_pend_num);
+    u8               *src = ctx->sei_pend;
+
+    sei->num_payloads = ctx->sei_pend_num;
+    sei->payloads     = pls;
+
+    for(int i = 0; i < ctx->sei_pend_num; i++)
+    {
+        s32 type = *((s32 *)src);
+        s32 size = *((s32 *)(src + 4));
+        src += 8;
+
+        pls[i].payload_type = (XEVD_SEI_PAYLOAD_TYPE)type;
+        pls[i].payload_size = size;
+        pls[i].payload      = dst;
+        xevd_mcpy(dst, src, size);
+        src += size;
+        dst += size;
+    }
+
+    imgb->pdata[XEVD_IMGB_SEI_SLOT] = sei;
+    imgb->ndata[XEVD_IMGB_SEI_SLOT] = XEVD_SEI_MAGIC;
+
+    ctx->sei_pend_size = 0;
+    ctx->sei_pend_num  = 0;
+}
+
 int xevd_picbuf_check_signature(XEVD_PIC * pic, u8 signature[N_C][16]
 ,int bit_depth)
 {

src_base/xevd_util.h

@@ -95,6 +95,7 @@ u16 xevd_get_avail_inter(int x_scu, int y_scu, int w_scu, int h_scu, int scup, i
 u16 xevd_get_avail_intra(int x_scu, int y_scu, int w_scu, int h_scu, int scup, int log2_cuw, int log2_cuh, u32 *map_scu, u8* map_tidx);
 XEVD_PIC* xevd_picbuf_lc_alloc(int w, int h, int pad_l, int pad_c, int *err, int idc, int bit_depth);
 void xevd_picbuf_lc_free(XEVD_PIC *pic);
+void xevd_sei_attach(XEVD_CTX * ctx, XEVD_IMGB * imgb);
 void xevd_picbuf_lc_expand(XEVD_PIC *pic, int exp_l, int exp_c);
 void xevd_poc_derivation(XEVD_SPS * sps, int tid, XEVD_POC *poc);
 void xevd_get_motion(int scup, int lidx, s8(*map_refi)[REFP_NUM], s16(*map_mv)[REFP_NUM][MV_D], XEVD_REFP(*refp)[REFP_NUM], int cuw, int cuh, int w_scu, u16 avail, s8 refi[MAX_NUM_MVP], s16 mvp[MAX_NUM_MVP][MV_D]);

src_main/xevdm.c

@@ -91,6 +91,7 @@ static void ctx_free(XEVD_CTX * ctx)
     XEVDM_CTX *mctx = (XEVDM_CTX *)ctx;
     xevd_mfree(mctx->aps_gen_array);
     xevd_mfree(mctx->dra_array);
+    xevd_mfree(ctx->sei_pend);
     xevd_mfree_fast(ctx);
 }
 
@@ -3231,13 +3232,17 @@ int xevd_dec_nalu(XEVD_CTX * ctx, XEVD_BITB * bitb, XEVD_STAT * stat)
         ctx->pic->imgb->ts[XEVD_TS_PTS] = bitb->ts[XEVD_TS_DTS] + coding_delay * ctx->ts.frame_duration_time;
 
         for (int i=0; i<XEVD_NDATA_NUM; i++) {
+            if (i == XEVD_IMGB_SEI_SLOT) continue; /* reserved for SEI exposure */
             ctx->pic->imgb->ndata[i] = bitb->ndata[i];
         }
 
         for (int i=0; i<XEVD_PDATA_NUM; i++) {
+            if (i == XEVD_IMGB_SEI_SLOT) continue; /* reserved for SEI exposure */
             ctx->pic->imgb->pdata[i] = bitb->pdata[i];
         }
 
+        xevd_sei_attach(ctx, ctx->pic->imgb);
+
         slice_deinit(ctx);
     }
     else if (nalu->nal_unit_type_plus1 - 1 == XEVD_NUT_SEI)