gralloc: use BGRA byte order for video encoder targets

Encoder does not recognize RGBA byte order, so refuse to allocate if SW
usage flags are set

Bug: 10429036

Change-Id: I3b9d0e71566c3565068b7c38c4f9189f4bb93fd0
Signed-off-by: Greg Hackmann <ghackmann@google.com>

diff --git a/gralloc/gralloc.cpp b/gralloc/gralloc.cpp
index 58e17cb369879366f4d040f3743030129c143e6e..ece99641d9859e0f1c2b97ea733bdfbb054947dc 100644 (file)
--- a/gralloc/gralloc.cpp
+++ b/gralloc/gralloc.cpp
@@ -138,11 +138,19 @@ static int gralloc_alloc_rgb(int ionfd, int w, int h, int format, int usage,
     int bpp = 0, vstride, fd, err;
     unsigned int heap_mask = _select_heap(usage);
 
-    if (format == HAL_PIXEL_FORMAT_RGBA_8888 && (usage & GRALLOC_USAGE_HW_FB)) {
-        ALOGW_IF(usage & (GRALLOC_USAGE_SW_READ_MASK |
-                GRALLOC_USAGE_SW_WRITE_MASK),
-                "framebuffer target should not have SW usage bits; ignoring");
-        format = HAL_PIXEL_FORMAT_BGRA_8888;
+    if (format == HAL_PIXEL_FORMAT_RGBA_8888) {
+        bool sw_usage = !!(usage & (GRALLOC_USAGE_SW_READ_MASK |
+                GRALLOC_USAGE_SW_WRITE_MASK));
+
+        if (usage & GRALLOC_USAGE_HW_FB) {
+            ALOGW_IF(sw_usage,
+                    "framebuffer target should not have SW usage bits; ignoring");
+            format = HAL_PIXEL_FORMAT_BGRA_8888;
+        } else if (usage & GRALLOC_USAGE_HW_VIDEO_ENCODER) {
+            if (sw_usage)
+                return -EINVAL;
+            format = HAL_PIXEL_FORMAT_BGRA_8888;
+        }
     }
 
     switch (format) {