exynos_omx: do not override format in set portDefinition for ANB

If using AndroidNativeBuffer, do not override the color format
during the SetParameter(OMX_IndexParamPortDefinition) call as
the GetParameter call would have faked a HAL format.

This seems to be a general problem with abusing the eColorFormat field
for nativewindow specific information. We should fix this.

Change-Id: I9e2bd74f53fabaef23782e034fce7c11b098ea1e
Signed-off-by: Dima Zavin <dima@android.com>

diff --git a/exynos_omx/openmax/exynos_omx/component/video/dec/h264/Exynos_OMX_H264dec.c b/exynos_omx/openmax/exynos_omx/component/video/dec/h264/Exynos_OMX_H264dec.c
index febe580631faee2aeca0b29eb49c7b285ea9facf..5f341a8d281fe8fce51bccb7b31dec6b7403bc46 100644 (file)
--- a/exynos_omx/openmax/exynos_omx/component/video/dec/h264/Exynos_OMX_H264dec.c
+++ b/exynos_omx/openmax/exynos_omx/component/video/dec/h264/Exynos_OMX_H264dec.c
@@ -436,6 +436,7 @@ OMX_ERRORTYPE Exynos_MFC_H264Dec_SetParameter(
         EXYNOS_OMX_BASEPORT          *pExynosPort;
         OMX_U32 width, height, size;
         OMX_U32 realWidth, realHeight;
+        OMX_COLOR_FORMATTYPE old_format;
 
         if (portIndex >= pExynosComponent->portParam.nPorts) {
             ret = OMX_ErrorBadPortIndex;
@@ -459,8 +460,17 @@ OMX_ERRORTYPE Exynos_MFC_H264Dec_SetParameter(
             goto EXIT;
         }
 
+        old_format = pExynosPort->portDefinition.format.video.eColorFormat;
         Exynos_OSAL_Memcpy(&pExynosPort->portDefinition, pPortDefinition, pPortDefinition->nSize);
 
+#ifdef USE_ANB
+        /* should not affect the format since in ANB case, the caller
+         * is providing us a HAL format */
+        if (pExynosPort->bIsANBEnabled == OMX_TRUE) {
+            pExynosPort->portDefinition.format.video.eColorFormat = old_format;
+        }
+#endif
+
         realWidth = pExynosPort->portDefinition.format.video.nFrameWidth;
         realHeight = pExynosPort->portDefinition.format.video.nFrameHeight;
         width = ((realWidth + 15) & (~15));

diff --git a/exynos_omx/openmax/exynos_omx/component/video/dec/mpeg4/Exynos_OMX_Mpeg4dec.c b/exynos_omx/openmax/exynos_omx/component/video/dec/mpeg4/Exynos_OMX_Mpeg4dec.c
index a4e953130c6540599725212ffab4cc9dd778aa12..a50cb82811f4a0e24dd6b34ab733f7ea8a497922 100644 (file)
--- a/exynos_omx/openmax/exynos_omx/component/video/dec/mpeg4/Exynos_OMX_Mpeg4dec.c
+++ b/exynos_omx/openmax/exynos_omx/component/video/dec/mpeg4/Exynos_OMX_Mpeg4dec.c
@@ -588,6 +588,7 @@ OMX_ERRORTYPE Exynos_MFC_Mpeg4Dec_SetParameter(
         EXYNOS_OMX_BASEPORT          *pExynosPort;
         OMX_U32                       width, height, size;
         OMX_U32                       realWidth, realHeight;
+        OMX_COLOR_FORMATTYPE          old_format;
 
         if (portIndex >= pExynosComponent->portParam.nPorts) {
             ret = OMX_ErrorBadPortIndex;
@@ -611,8 +612,17 @@ OMX_ERRORTYPE Exynos_MFC_Mpeg4Dec_SetParameter(
             goto EXIT;
         }
 
+        old_format = pExynosPort->portDefinition.format.video.eColorFormat;
         Exynos_OSAL_Memcpy(&pExynosPort->portDefinition, pPortDefinition, pPortDefinition->nSize);
 
+#ifdef USE_ANB
+        /* should not affect the format since in ANB case, the caller
+         * is providing us a HAL format */
+        if (pExynosPort->bIsANBEnabled == OMX_TRUE) {
+            pExynosPort->portDefinition.format.video.eColorFormat = old_format;
+        }
+#endif
+
         realWidth = pExynosPort->portDefinition.format.video.nFrameWidth;
         realHeight = pExynosPort->portDefinition.format.video.nFrameHeight;
         width = ((realWidth + 15) & (~15));

diff --git a/exynos_omx/openmax/exynos_omx/component/video/dec/vp8/Exynos_OMX_Vp8dec.c b/exynos_omx/openmax/exynos_omx/component/video/dec/vp8/Exynos_OMX_Vp8dec.c
index e6c4aab07f4c087ad0fd1217c04c52d473618b33..1a438399ad3ef753b13b720228c2c2e688267e06 100644 (file)
--- a/exynos_omx/openmax/exynos_omx/component/video/dec/vp8/Exynos_OMX_Vp8dec.c
+++ b/exynos_omx/openmax/exynos_omx/component/video/dec/vp8/Exynos_OMX_Vp8dec.c
@@ -267,6 +267,7 @@ OMX_ERRORTYPE Exynos_MFC_VP8Dec_SetParameter(
         EXYNOS_OMX_BASEPORT          *pExynosPort;
         OMX_U32                       width, height, size;
         OMX_U32                       realWidth, realHeight;
+        OMX_COLOR_FORMATTYPE          old_format;
 
         if (portIndex >= pExynosComponent->portParam.nPorts) {
             ret = OMX_ErrorBadPortIndex;
@@ -290,8 +291,17 @@ OMX_ERRORTYPE Exynos_MFC_VP8Dec_SetParameter(
             goto EXIT;
         }
 
+        old_format = pExynosPort->portDefinition.format.video.eColorFormat;
         Exynos_OSAL_Memcpy(&pExynosPort->portDefinition, pPortDefinition, pPortDefinition->nSize);
 
+#ifdef USE_ANB
+        /* should not affect the format since in ANB case, the caller
+         * is providing us a HAL format */
+        if (pExynosPort->bIsANBEnabled == OMX_TRUE) {
+            pExynosPort->portDefinition.format.video.eColorFormat = old_format;
+        }
+#endif
+
         realWidth = pExynosPort->portDefinition.format.video.nFrameWidth;
         realHeight = pExynosPort->portDefinition.format.video.nFrameHeight;
         width = ((realWidth + 15) & (~15));