drm-backend: Enable plane rotations

Search for planes that support the rotation required to properly display
a paint node, and properly set coordinates and rotation properties.

Signed-off-by: Derek Foreman <derek.foreman@collabora.com>

libweston/backend-drm/drm-internal.h

@@ -695,14 +695,14 @@ drm_rotation_from_output_transform(struct drm_plane *plane,
 				   enum wl_output_transform ot);
 
 static inline bool
-drm_paint_node_transform_supported(struct weston_paint_node *node, struct weston_output *output)
+drm_paint_node_transform_supported(struct weston_paint_node *node, struct drm_plane *plane)
 {
 	/* if false, the transform doesn't map to any of the standard
 	 * (ie: 90 degree) output transformations. */
 	if (!node->valid_transform)
 		return false;
 
-	if (node->transform != WL_OUTPUT_TRANSFORM_NORMAL)
+	if (drm_rotation_from_output_transform(plane, node->transform) == 0)
 		return false;
 
 	return true;

libweston/backend-drm/fb.c

@@ -546,12 +546,6 @@ drm_fb_get_from_paint_node(struct drm_output_state *state,
 		return NULL;
 	}
 
-	if (!drm_paint_node_transform_supported(pnode, &output->base)) {
-		pnode->try_view_on_plane_failure_reasons |=
-			FAILURE_REASONS_INCOMPATIBLE_TRANSFORM;
-		return NULL;
-	}
-
 	if (ev->surface->protection_mode == WESTON_SURFACE_PROTECTION_MODE_ENFORCED &&
 	    ev->surface->desired_protection > output->base.current_protection) {
 		pnode->try_view_on_plane_failure_reasons |=
@@ -612,8 +606,9 @@ drm_fb_get_from_paint_node(struct drm_output_state *state,
 		/* only SHM buffers can go into cursor planes */
 		if (plane->type == WDRM_PLANE_TYPE_CURSOR)
 			continue;
+
 		if (drm_fb_compatible_with_plane(fb, plane))
-			fb->plane_mask |= (1 << plane->plane_idx);
+			fb->plane_mask |= 1 << (plane->plane_idx);
 	}
 	if (fb->plane_mask == 0) {
 		drm_fb_unref(fb);

libweston/backend-drm/state-helpers.c

@@ -219,9 +219,12 @@ drm_plane_state_coords_for_paint_node(struct drm_plane_state *state,
 	struct weston_coord corners[2];
 	float sxf1, syf1, sxf2, syf2;
 
-	if (!drm_paint_node_transform_supported(node, &output->base))
+	if (!drm_paint_node_transform_supported(node, state->plane))
 		return false;
 
+	assert(node->valid_transform);
+	state->rotation = drm_rotation_from_output_transform(state->plane, node->transform);
+
 	/* Update the base weston_plane co-ordinates. */
 	box = pixman_region32_extents(&ev->transform.boundingbox);
 	state->plane->base.x = box->x1;
@@ -256,11 +259,21 @@ drm_plane_state_coords_for_paint_node(struct drm_plane_state *state,
 	syf2 = corners[1].y;
 	pixman_region32_fini(&dest_rect);
 
-	/* We currently only support WL_OUTPUT_TRANSFORM_NORMAL, so it's
-	 * not possible for x2 to be left of x1 or y2 above y1.
+	/* Make sure that our post-transform coordinates are in the
+	 * right order.
 	 */
-	assert(sxf1 < sxf2);
-	assert(syf1 < syf2);
+	if (sxf1 > sxf2) {
+		float temp = sxf1;
+
+		sxf1 = sxf2;
+		sxf2 = temp;
+	}
+	if (syf1 > syf2) {
+		float temp = syf1;
+
+		syf1 = syf2;
+		syf2 = temp;
+	}
 
 	/* Shift from S23.8 wl_fixed to U16.16 KMS fixed-point encoding. */
 	state->src_x = wl_fixed_from_double(sxf1) << 8;

libweston/backend-drm/state-propose.c

@@ -499,6 +499,20 @@ drm_output_find_plane_for_view(struct drm_output_state *state,
 			return NULL;
 		}
 
+		wl_list_for_each(plane, &device->plane_list, link) {
+			if (plane->type == WDRM_PLANE_TYPE_CURSOR)
+				continue;
+
+			if (drm_paint_node_transform_supported(pnode, plane))
+				possible_plane_mask |= 1 << plane->plane_idx;
+		}
+
+		if (!possible_plane_mask) {
+			pnode->try_view_on_plane_failure_reasons |=
+				FAILURE_REASONS_INCOMPATIBLE_TRANSFORM;
+			return NULL;
+		}
+
 		fb = drm_fb_get_from_paint_node(state, pnode);
 		if (!fb) {
 			drm_debug(b, "\t\t\t[view] couldn't get FB for view: 0x%lx\n",
@@ -506,7 +520,7 @@ drm_output_find_plane_for_view(struct drm_output_state *state,
 			return NULL;
 		}
 
-		possible_plane_mask = fb->plane_mask;
+		possible_plane_mask &= fb->plane_mask;
 	}
 
 	view_matches_entire_output =