aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--main/xf86-video-intel/APKBUILD48
-rw-r--r--main/xf86-video-intel/O_CLOEXEC.patch10
-rw-r--r--main/xf86-video-intel/gcc5-workaround.patch22
-rw-r--r--main/xf86-video-intel/git.patch43164
-rw-r--r--main/xf86-video-intel/xf86-video-intel-2.99.917-libdrm-kernel-4_0-crash.patch65
-rw-r--r--main/xf86-video-intel/xorg-1.18.patch24
6 files changed, 43180 insertions, 153 deletions
diff --git a/main/xf86-video-intel/APKBUILD b/main/xf86-video-intel/APKBUILD
index 183bc3a433..154a763c6f 100644
--- a/main/xf86-video-intel/APKBUILD
+++ b/main/xf86-video-intel/APKBUILD
@@ -1,36 +1,32 @@
# Maintainer: Natanael Copa <ncopa@alpinelinux.org>
pkgname=xf86-video-intel
-pkgver=2.99.917
-pkgrel=4
+pkgver=2.99.917_git20170325
+pkgrel=0
pkgdesc="X.Org driver for Intel cards"
url="http://xorg.freedesktop.org/"
arch="x86 x86_64"
-license="custom"
+license="MIT"
subpackages="$pkgname-doc"
depends="mesa-dri-intel"
makedepends="xorg-server-dev libxi-dev fontsproto randrproto
videoproto renderproto glproto xineramaproto libdrm-dev xf86driproto
- mesa-dev libxvmc-dev xcb-util-dev eudev-dev"
+ mesa-dev libxvmc-dev xcb-util-dev eudev-dev
+ util-macros autoconf automake libtool"
-source="http://xorg.freedesktop.org/releases/individual/driver/$pkgname-$pkgver.tar.bz2
- gcc5-workaround.patch
- xf86-video-intel-2.99.917-libdrm-kernel-4_0-crash.patch
- O_CLOEXEC.patch
- xorg-1.18.patch
+_ver=${pkgver%_git*}
+source="http://xorg.freedesktop.org/releases/individual/driver/$pkgname-$_ver.tar.bz2
+ git.patch
"
-_builddir="$srcdir"/$pkgname-$pkgver
+builddir="$srcdir"/$pkgname-$_ver
prepare() {
- cd "$_builddir"
- for i in $source; do
- case $i in
- *.patch) msg $i; patch -p1 -i "$srcdir"/$i || return 1;;
- esac
- done
+ cd "$builddir"
+ default_prepare
+ autoreconf -vif
}
build() {
- cd "$srcdir"/$pkgname-$pkgver
+ cd "$builddir"
export LDFLAGS="$LDFLAGS -Wl,-z,lazy"
./configure \
--build=$CBUILD \
@@ -38,30 +34,18 @@ build() {
--prefix=/usr \
--enable-xvmc \
--disable-selective-werror \
+ --with-default-dri=3 \
|| return 1
make || return 1
}
package() {
- cd "$srcdir"/$pkgname-$pkgver
+ cd "$builddir"
make DESTDIR="$pkgdir" install || return 1
install -Dm644 COPYING "$pkgdir"/usr/share/licenses/$pkgname/COPYING
# http://bugs.alpinelinux.org/issues/3312
chmod o-x "$pkgdir"/usr/libexec/xf86-video-intel-backlight-helper
}
-md5sums="fa196a66e52c0c624fe5d350af7a5e7b xf86-video-intel-2.99.917.tar.bz2
-2e9c5ee749f0a255d2b10ce18b3512fa gcc5-workaround.patch
-2fa815b66eb6896b3962074731b0b4bb xf86-video-intel-2.99.917-libdrm-kernel-4_0-crash.patch
-d5c410d504c58aa641658a19e4950ea5 O_CLOEXEC.patch
-d64095af23cf26c3559bba6f739e371e xorg-1.18.patch"
-sha256sums="00b781eea055582820a123c47b62411bdf6aabf4f03dc0568faec55faf9667c9 xf86-video-intel-2.99.917.tar.bz2
-55367cd8dbe58d1097e2cf6cee11895acadc1a5ef527b8d39361e3975a6943e5 gcc5-workaround.patch
-54298cb4a59016be0451e3ea72b2c2c6b2a97cb9ec2c8f45d62c12447d14b361 xf86-video-intel-2.99.917-libdrm-kernel-4_0-crash.patch
-79f6c0bf8eb56d631857a0064e6c4ba1582acfb12c467f29c211e4fc4e628b98 O_CLOEXEC.patch
-f45ad7f6d8a78f282faf283057cc7fa82d2b1c99979dba563c1a734a25e4fb7a xorg-1.18.patch"
sha512sums="cbf4d46ad1ad5e5587c0f1f620ff534ef0645270517b60056b9f03e83d8216e2f456de46352a06c37c0c46963cc4ed20b71b815b20ec1bf680ff046e535f580f xf86-video-intel-2.99.917.tar.bz2
-b208508d229f53f18cf3aa8de2c3637964d8b22f8a615fc4759a2bb58cbe9db4dca7a79129a7b59fd138980c90bdcaf1aec142e1f13954c4cf25a817a2125998 gcc5-workaround.patch
-003fc22a9446cdfcb8d51cbface096187f93a0c54b024ee34b160ca41a491c35e8b387caabc3c3f6411b93663c5119f48dc2adae0d76878723c02483306972ac xf86-video-intel-2.99.917-libdrm-kernel-4_0-crash.patch
-1054d8e4f314b061209d74d05037abefec64ab0c4a1efcf82e512ea8db9022c56cf7891ca4ed08af88f560e986ea0e726144f5bde11212e938cc741c40da5348 O_CLOEXEC.patch
-f9c22684d50e4bd567efcb38f93b40b3bad2d56d8c7ae96bf768064dffbff1f9147e40ae415a7bbef61d8f3140ef6746a372059cea11861bc61b4b94acfa91a4 xorg-1.18.patch"
+0fe4e455dcbc4ae6622dca483ef3ddc765c43009fdb0fef82bdaa835a737796a6caf8afa9c6630919f43c977a6f736770c3779f04d8c823da4fc9cee17d16f19 git.patch"
diff --git a/main/xf86-video-intel/O_CLOEXEC.patch b/main/xf86-video-intel/O_CLOEXEC.patch
deleted file mode 100644
index 9dbe933588..0000000000
--- a/main/xf86-video-intel/O_CLOEXEC.patch
+++ /dev/null
@@ -1,10 +0,0 @@
---- ./src/sna/kgem.c.orig
-+++ ./src/sna/kgem.c
-@@ -37,6 +37,7 @@
- #include <sys/mman.h>
- #include <time.h>
- #include <errno.h>
-+#define __USE_GNU
- #include <fcntl.h>
-
- #include <xf86drm.h>
diff --git a/main/xf86-video-intel/gcc5-workaround.patch b/main/xf86-video-intel/gcc5-workaround.patch
deleted file mode 100644
index dd832546e4..0000000000
--- a/main/xf86-video-intel/gcc5-workaround.patch
+++ /dev/null
@@ -1,22 +0,0 @@
---- ./src/sna/compiler.h.orig
-+++ ./src/sna/compiler.h
-@@ -65,16 +65,14 @@
- #define avx2 __attribute__((target("avx2,sse4.2,sse2,fpmath=sse")))
- #endif
-
--#if HAS_GCC(4, 6) && defined(__OPTIMIZE__)
-+#if HAS_GCC(4, 6) && !HAS_GCC(5,0) && defined(__OPTIMIZE__)
- #define fast __attribute__((optimize("Ofast")))
- #else
- #define fast
- #endif
-
--#if HAS_GCC(4, 6) && defined(__OPTIMIZE__)
--#define fast_memcpy __attribute__((optimize("Ofast"))) __attribute__((target("inline-all-stringops")))
--#elif HAS_GCC(4, 5) && defined(__OPTIMIZE__)
--#define fast_memcpy __attribute__((target("inline-all-stringops")))
-+#if HAS_GCC(4, 5) && defined(__OPTIMIZE__)
-+#define fast_memcpy fast __attribute__((target("inline-all-stringops")))
- #else
- #define fast_memcpy
- #endif
diff --git a/main/xf86-video-intel/git.patch b/main/xf86-video-intel/git.patch
new file mode 100644
index 0000000000..20084425a4
--- /dev/null
+++ b/main/xf86-video-intel/git.patch
@@ -0,0 +1,43164 @@
+diff --git a/Makefile.am b/Makefile.am
+index 418fdc92..de5fbe12 100644
+--- a/Makefile.am
++++ b/Makefile.am
+@@ -18,14 +18,16 @@
+ # IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+-ACLOCAL_AMFLAGS = ${ACLOCAL_FLAGS} -I m4
++#Having problems passing through user flags as libtool complains
++#ACLOCAL_AMFLAGS = ${ACLOCAL_FLAGS} -I m4
++ACLOCAL_AMFLAGS = -I m4
+
+ SUBDIRS = man libobj xvmc src tools
+
+ MAINTAINERCLEANFILES = ChangeLog INSTALL
+
+ if HAVE_X11
+-SUBDIRS += test
++SUBDIRS += test benchmarks
+ endif
+
+ .PHONY: ChangeLog INSTALL
+diff --git a/NEWS b/NEWS
+index 604b9cce..0e200332 100644
+--- a/NEWS
++++ b/NEWS
+@@ -21,7 +21,7 @@ should make one more snapshot before an imminent release.
+ Before kernel 3.19, O_NONBLOCK support is broken and so we must avoid
+ reading if we are not expecting an event.
+
+- * Backwards compatibilty fix for fake triple buffering with PRIME and
++ * Backwards compatibility fix for fake triple buffering with PRIME and
+ Xorg-1.15
+ https://bugs.freedesktop.org/show_bug.cgi?id=85144#c12
+
+@@ -51,7 +51,7 @@ should make one more snapshot before an imminent release.
+ Snapshot 2.99.916 (2014-09-08)
+ ==============================
+ Quick update for MST in UXA - we need to hook up the RandR outputs for
+-dynamicaly added connectors.
++dynamically added connectors.
+
+
+ Snapshot 2.99.915 (2014-09-08)
+@@ -503,7 +503,7 @@ release.
+ backlight property is queried whilst the connector is disabled
+ https://bugs.freedesktop.org/show_bug.cgi?id=70406
+
+- * Pad GETCONNECTOR ioctl for compatability between 32/64-bit userspace
++ * Pad GETCONNECTOR ioctl for compatibility between 32/64-bit userspace
+ and kernel
+
+ * Handle long glyph runs correctly
+@@ -523,7 +523,7 @@ snapshot beforehand to push out the bug fixes from the last week.
+
+ * Fix video output using sprites when changing the image size
+
+- * Apply more restrictive tile constaints for 915g class devices
++ * Apply more restrictive tile constraints for 915g class devices
+ https://bugs.launchpad.net/ubuntu/+source/xserver-xorg-video-intel/+bug/1232546
+
+ * Ensure all overlapping rectangles are drawn for XRenderFillRectangles
+@@ -1132,7 +1132,7 @@ operation.
+ * Explicitly prevent ring-switching for synchronized rendering to
+ scanouts (for vsync).
+
+- * Clip dirty region to slave pixmaps (otherwise UDL is nigh unusuable)
++ * Clip dirty region to slave pixmaps (otherwise UDL is nigh unusable)
+ https://bugs.freedesktop.org/show_bug.cgi?id=59539
+
+
+@@ -1226,7 +1226,7 @@ Release 2.20.15 (2012-12-03)
+ ============================
+ And lo, enabling more of the common acceleration paths for gen4 revealed
+ another lurking bug - something is wrong with how we prepare Y-tiling
+-surfaces for rendering. For the time being, we can surreptiously disable
++surfaces for rendering. For the time being, we can surreptitiously disable
+ them for gen4 and avoid hitting GPU hangs.
+
+ * Avoid clobbering the render state after failing to convert the
+@@ -1515,7 +1515,7 @@ Release 2.20.5 (2012-08-26)
+ Another silly bug found, another small bugfix release. The goal was for
+ the driver to bind to all Intel devices supported by the kernel.
+ Unfortunately we were too successful and started claiming Pouslbo,
+-Medfield and Cedarview devices which are still encumbered by propietary
++Medfield and Cedarview devices which are still encumbered by proprietary
+ IP and not supported by this driver.
+
+ Bugs fixed since 2.20.4:
+diff --git a/README b/README
+index cf4d88d8..348983b4 100644
+--- a/README
++++ b/README
+@@ -15,9 +15,9 @@ Intel graphics chipsets including:
+ G/Q33,G/Q35,G41,G/Q43,G/GM/Q45
+ PineView-M (Atom N400 series)
+ PineView-D (Atom D400/D500 series)
+- Intel(R) HD Graphics: 2000-6000,
+- Intel(R) Iris(TM) Graphics: 5100/6100, and
+- Intel(R) Iris(TM) Pro Graphics: 5200/6200/P6300.
++ Intel(R) HD Graphics,
++ Intel(R) Iris(TM) Graphics,
++ Intel(R) Iris(TM) Pro Graphics.
+
+ Where to get more information about the driver
+ ----------------------------------------------
+diff --git a/benchmarks/.gitignore b/benchmarks/.gitignore
+new file mode 100644
+index 00000000..301c0129
+--- /dev/null
++++ b/benchmarks/.gitignore
+@@ -0,0 +1,2 @@
++dri2-swap
++dri3-swap
+diff --git a/benchmarks/Makefile.am b/benchmarks/Makefile.am
+new file mode 100644
+index 00000000..4976e8a3
+--- /dev/null
++++ b/benchmarks/Makefile.am
+@@ -0,0 +1,14 @@
++AM_CFLAGS = @CWARNFLAGS@ $(X11_CFLAGS) $(DRM_CFLAGS)
++LDADD = $(X11_LIBS) $(DRM_LIBS) $(CLOCK_GETTIME_LIBS)
++
++check_PROGRAMS =
++
++if DRI2
++check_PROGRAMS += dri2-swap
++endif
++
++if DRI3
++check_PROGRAMS += dri3-swap
++AM_CFLAGS += $(X11_DRI3_CFLAGS)
++LDADD += $(X11_DRI3_LIBS)
++endif
+diff --git a/benchmarks/dri2-swap.c b/benchmarks/dri2-swap.c
+new file mode 100644
+index 00000000..3d9d30aa
+--- /dev/null
++++ b/benchmarks/dri2-swap.c
+@@ -0,0 +1,588 @@
++/*
++ * Copyright (c) 2015 Intel Corporation
++ *
++ * Permission is hereby granted, free of charge, to any person obtaining a
++ * copy of this software and associated documentation files (the "Software"),
++ * to deal in the Software without restriction, including without limitation
++ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
++ * and/or sell copies of the Software, and to permit persons to whom the
++ * Software is furnished to do so, subject to the following conditions:
++ *
++ * The above copyright notice and this permission notice (including the next
++ * paragraph) shall be included in all copies or substantial portions of the
++ * Software.
++ *
++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
++ * SOFTWARE.
++ *
++ */
++
++#ifdef HAVE_CONFIG_H
++#include "config.h"
++#endif
++
++#include <X11/Xlib.h>
++#include <X11/Xatom.h>
++#include <X11/Xlib-xcb.h>
++#include <X11/Xutil.h>
++#include <X11/Xlibint.h>
++#include <X11/extensions/dpms.h>
++#include <X11/extensions/randr.h>
++#include <X11/extensions/Xcomposite.h>
++#include <X11/extensions/Xdamage.h>
++#include <X11/extensions/Xrandr.h>
++#include <xcb/xcb.h>
++#include <xcb/dri2.h>
++#include <xf86drm.h>
++
++#include <stdio.h>
++#include <string.h>
++#include <fcntl.h>
++#include <unistd.h>
++#include <assert.h>
++#include <errno.h>
++#include <setjmp.h>
++#include <signal.h>
++
++#include <X11/Xlibint.h>
++#include <X11/extensions/Xext.h>
++#include <X11/extensions/extutil.h>
++#include <X11/extensions/dri2proto.h>
++#include <X11/extensions/dri2tokens.h>
++#include <X11/extensions/Xfixes.h>
++
++static char dri2ExtensionName[] = DRI2_NAME;
++static XExtensionInfo *dri2Info;
++static XEXT_GENERATE_CLOSE_DISPLAY (DRI2CloseDisplay, dri2Info)
++
++static Bool
++DRI2WireToEvent(Display *dpy, XEvent *event, xEvent *wire);
++static Status
++DRI2EventToWire(Display *dpy, XEvent *event, xEvent *wire);
++static int
++DRI2Error(Display *display, xError *err, XExtCodes *codes, int *ret_code);
++
++static /* const */ XExtensionHooks dri2ExtensionHooks = {
++ NULL, /* create_gc */
++ NULL, /* copy_gc */
++ NULL, /* flush_gc */
++ NULL, /* free_gc */
++ NULL, /* create_font */
++ NULL, /* free_font */
++ DRI2CloseDisplay, /* close_display */
++ DRI2WireToEvent, /* wire_to_event */
++ DRI2EventToWire, /* event_to_wire */
++ DRI2Error, /* error */
++ NULL, /* error_string */
++};
++
++static XEXT_GENERATE_FIND_DISPLAY (DRI2FindDisplay,
++ dri2Info,
++ dri2ExtensionName,
++ &dri2ExtensionHooks,
++ 0, NULL)
++
++static Bool
++DRI2WireToEvent(Display *dpy, XEvent *event, xEvent *wire)
++{
++ XExtDisplayInfo *info = DRI2FindDisplay(dpy);
++
++ XextCheckExtension(dpy, info, dri2ExtensionName, False);
++
++ switch ((wire->u.u.type & 0x7f) - info->codes->first_event) {
++#ifdef X_DRI2SwapBuffers
++ case DRI2_BufferSwapComplete:
++ return False;
++#endif
++#ifdef DRI2_InvalidateBuffers
++ case DRI2_InvalidateBuffers:
++ return False;
++#endif
++ default:
++ /* client doesn't support server event */
++ break;
++ }
++
++ return False;
++}
++
++/* We don't actually support this. It doesn't make sense for clients to
++ * send each other DRI2 events.
++ */
++static Status
++DRI2EventToWire(Display *dpy, XEvent *event, xEvent *wire)
++{
++ XExtDisplayInfo *info = DRI2FindDisplay(dpy);
++
++ XextCheckExtension(dpy, info, dri2ExtensionName, False);
++
++ switch (event->type) {
++ default:
++ /* client doesn't support server event */
++ break;
++ }
++
++ return Success;
++}
++
++static int
++DRI2Error(Display *display, xError *err, XExtCodes *codes, int *ret_code)
++{
++ if (err->majorCode == codes->major_opcode &&
++ err->errorCode == BadDrawable &&
++ err->minorCode == X_DRI2CopyRegion)
++ return True;
++
++ /* If the X drawable was destroyed before the GLX drawable, the
++ * DRI2 drawble will be gone by the time we call
++ * DRI2DestroyDrawable. So just ignore BadDrawable here. */
++ if (err->majorCode == codes->major_opcode &&
++ err->errorCode == BadDrawable &&
++ err->minorCode == X_DRI2DestroyDrawable)
++ return True;
++
++ /* If the server is non-local DRI2Connect will raise BadRequest.
++ * Swallow this so that DRI2Connect can signal this in its return code */
++ if (err->majorCode == codes->major_opcode &&
++ err->minorCode == X_DRI2Connect &&
++ err->errorCode == BadRequest) {
++ *ret_code = False;
++ return True;
++ }
++
++ return False;
++}
++
++static Bool
++DRI2QueryExtension(Display * dpy, int *eventBase, int *errorBase)
++{
++ XExtDisplayInfo *info = DRI2FindDisplay(dpy);
++
++ if (XextHasExtension(info)) {
++ *eventBase = info->codes->first_event;
++ *errorBase = info->codes->first_error;
++ return True;
++ }
++
++ return False;
++}
++
++static Bool
++DRI2Connect(Display * dpy, XID window, char **driverName, char **deviceName)
++{
++ XExtDisplayInfo *info = DRI2FindDisplay(dpy);
++ xDRI2ConnectReply rep;
++ xDRI2ConnectReq *req;
++
++ XextCheckExtension(dpy, info, dri2ExtensionName, False);
++
++ LockDisplay(dpy);
++ GetReq(DRI2Connect, req);
++ req->reqType = info->codes->major_opcode;
++ req->dri2ReqType = X_DRI2Connect;
++ req->window = window;
++ req->driverType = DRI2DriverDRI;
++ if (!_XReply(dpy, (xReply *) & rep, 0, xFalse)) {
++ UnlockDisplay(dpy);
++ SyncHandle();
++ return False;
++ }
++
++ if (rep.driverNameLength == 0 && rep.deviceNameLength == 0) {
++ UnlockDisplay(dpy);
++ SyncHandle();
++ return False;
++ }
++
++ *driverName = Xmalloc(rep.driverNameLength + 1);
++ if (*driverName == NULL) {
++ _XEatData(dpy,
++ ((rep.driverNameLength + 3) & ~3) +
++ ((rep.deviceNameLength + 3) & ~3));
++ UnlockDisplay(dpy);
++ SyncHandle();
++ return False;
++ }
++ _XReadPad(dpy, *driverName, rep.driverNameLength);
++ (*driverName)[rep.driverNameLength] = '\0';
++
++ *deviceName = Xmalloc(rep.deviceNameLength + 1);
++ if (*deviceName == NULL) {
++ Xfree(*driverName);
++ _XEatData(dpy, ((rep.deviceNameLength + 3) & ~3));
++ UnlockDisplay(dpy);
++ SyncHandle();
++ return False;
++ }
++ _XReadPad(dpy, *deviceName, rep.deviceNameLength);
++ (*deviceName)[rep.deviceNameLength] = '\0';
++
++ UnlockDisplay(dpy);
++ SyncHandle();
++
++ return True;
++}
++
++static Bool
++DRI2Authenticate(Display * dpy, XID window, unsigned int magic)
++{
++ XExtDisplayInfo *info = DRI2FindDisplay(dpy);
++ xDRI2AuthenticateReq *req;
++ xDRI2AuthenticateReply rep;
++
++ XextCheckExtension(dpy, info, dri2ExtensionName, False);
++
++ LockDisplay(dpy);
++ GetReq(DRI2Authenticate, req);
++ req->reqType = info->codes->major_opcode;
++ req->dri2ReqType = X_DRI2Authenticate;
++ req->window = window;
++ req->magic = magic;
++
++ if (!_XReply(dpy, (xReply *) & rep, 0, xFalse)) {
++ UnlockDisplay(dpy);
++ SyncHandle();
++ return False;
++ }
++
++ UnlockDisplay(dpy);
++ SyncHandle();
++
++ return rep.authenticated;
++}
++
++static void
++DRI2CreateDrawable(Display * dpy, XID drawable)
++{
++ XExtDisplayInfo *info = DRI2FindDisplay(dpy);
++ xDRI2CreateDrawableReq *req;
++
++ XextSimpleCheckExtension(dpy, info, dri2ExtensionName);
++
++ LockDisplay(dpy);
++ GetReq(DRI2CreateDrawable, req);
++ req->reqType = info->codes->major_opcode;
++ req->dri2ReqType = X_DRI2CreateDrawable;
++ req->drawable = drawable;
++ UnlockDisplay(dpy);
++ SyncHandle();
++}
++
++static void DRI2SwapInterval(Display *dpy, XID drawable, int interval)
++{
++ XExtDisplayInfo *info = DRI2FindDisplay(dpy);
++ xDRI2SwapIntervalReq *req;
++
++ XextSimpleCheckExtension (dpy, info, dri2ExtensionName);
++
++ LockDisplay(dpy);
++ GetReq(DRI2SwapInterval, req);
++ req->reqType = info->codes->major_opcode;
++ req->dri2ReqType = X_DRI2SwapInterval;
++ req->drawable = drawable;
++ req->interval = interval;
++ UnlockDisplay(dpy);
++ SyncHandle();
++}
++
++static int _x_error_occurred;
++
++static int
++_check_error_handler(Display *display,
++ XErrorEvent *event)
++{
++ fprintf(stderr,
++ "X11 error from display %s, serial=%ld, error=%d, req=%d.%d\n",
++ DisplayString(display),
++ event->serial,
++ event->error_code,
++ event->request_code,
++ event->minor_code);
++ _x_error_occurred++;
++ return False; /* ignored */
++}
++
++static double elapsed(const struct timespec *start,
++ const struct timespec *end)
++{
++ return 1e6*(end->tv_sec - start->tv_sec) + (end->tv_nsec - start->tv_nsec)/1000;
++}
++
++static void run(Display *dpy, Window win)
++{
++ xcb_connection_t *c = XGetXCBConnection(dpy);
++ struct timespec start, end;
++ int n, completed = 0;
++
++ clock_gettime(CLOCK_MONOTONIC, &start);
++ do {
++ for (n = 0; n < 1000; n++) {
++ unsigned int attachments[] = { DRI2BufferBackLeft };
++ unsigned int seq[2];
++
++ seq[0] = xcb_dri2_swap_buffers_unchecked(c, win,
++ 0, 0, 0, 0, 0, 0).sequence;
++
++
++ seq[1] = xcb_dri2_get_buffers_unchecked(c, win,
++ 1, 1, attachments).sequence;
++
++ xcb_flush(c);
++ xcb_discard_reply(c, seq[0]);
++ xcb_discard_reply(c, seq[1]);
++ completed++;
++ }
++ clock_gettime(CLOCK_MONOTONIC, &end);
++ } while (end.tv_sec < start.tv_sec + 10);
++
++ printf("%f\n", completed / (elapsed(&start, &end) / 1000000));
++}
++
++static inline XRRScreenResources *_XRRGetScreenResourcesCurrent(Display *dpy, Window window)
++{
++ XRRScreenResources *res;
++
++ res = XRRGetScreenResourcesCurrent(dpy, window);
++ if (res == NULL)
++ res = XRRGetScreenResources(dpy, window);
++
++ return res;
++}
++
++static XRRModeInfo *lookup_mode(XRRScreenResources *res, int id)
++{
++ int i;
++
++ for (i = 0; i < res->nmode; i++) {
++ if (res->modes[i].id == id)
++ return &res->modes[i];
++ }
++
++ return NULL;
++}
++
++static int dri2_open(Display *dpy)
++{
++ drm_auth_t auth;
++ char *driver, *device;
++ int fd;
++
++ if (!DRI2QueryExtension(dpy, &fd, &fd))
++ return -1;
++
++ if (!DRI2Connect(dpy, DefaultRootWindow(dpy), &driver, &device))
++ return -1;
++
++ fd = open(device, O_RDWR);
++ if (fd < 0)
++ return -1;
++
++ if (drmIoctl(fd, DRM_IOCTL_GET_MAGIC, &auth))
++ return -1;
++
++ if (!DRI2Authenticate(dpy, DefaultRootWindow(dpy), auth.magic))
++ return -1;
++
++ return fd;
++}
++
++static void fullscreen(Display *dpy, Window win)
++{
++ Atom atom = XInternAtom(dpy, "_NET_WM_STATE_FULLSCREEN", False);
++ XChangeProperty(dpy, win,
++ XInternAtom(dpy, "_NET_WM_STATE", False),
++ XA_ATOM, 32, PropModeReplace,
++ (unsigned char *)&atom, 1);
++}
++
++static int has_composite(Display *dpy)
++{
++ int event, error;
++ int major, minor;
++
++ if (!XDamageQueryExtension (dpy, &event, &error))
++ return 0;
++
++ if (!XCompositeQueryExtension(dpy, &event, &error))
++ return 0;
++
++ XCompositeQueryVersion(dpy, &major, &minor);
++
++ return major > 0 || minor >= 4;
++}
++
++int main(int argc, char **argv)
++{
++ Display *dpy;
++ Window root, win;
++ XRRScreenResources *res;
++ XRRCrtcInfo **original_crtc;
++ XSetWindowAttributes attr;
++ enum window { ROOT, FULLSCREEN, WINDOW } w = FULLSCREEN;
++ enum visible {REDIRECTED, NORMAL } v = NORMAL;
++ enum display { OFF, ON } d = OFF;
++ int width, height;
++ int i, fd;
++ int c;
++
++ while ((c = getopt(argc, argv, "d:v:w:")) != -1) {
++ switch (c) {
++ case 'd':
++ if (strcmp(optarg, "off") == 0)
++ d = OFF;
++ else if (strcmp(optarg, "on") == 0)
++ d = ON;
++ else
++ abort();
++ break;
++
++ case 'v':
++ if (strcmp(optarg, "redirected") == 0)
++ v = REDIRECTED;
++ else if (strcmp(optarg, "normal") == 0)
++ v = NORMAL;
++ else
++ abort();
++ break;
++
++ case 'w':
++ if (strcmp(optarg, "fullscreen") == 0)
++ w = FULLSCREEN;
++ else if (strcmp(optarg, "window") == 0)
++ w = WINDOW;
++ else if (strcmp(optarg, "root") == 0)
++ w = ROOT;
++ else
++ abort();
++ break;
++ }
++ }
++
++ attr.override_redirect = 1;
++
++ dpy = XOpenDisplay(NULL);
++ if (dpy == NULL)
++ return 77;
++
++ width = DisplayWidth(dpy, DefaultScreen(dpy));
++ height = DisplayHeight(dpy, DefaultScreen(dpy));
++
++ fd = dri2_open(dpy);
++ if (fd < 0)
++ return 77;
++
++ if (DPMSQueryExtension(dpy, &i, &i))
++ DPMSDisable(dpy);
++
++ root = DefaultRootWindow(dpy);
++
++ signal(SIGALRM, SIG_IGN);
++ XSetErrorHandler(_check_error_handler);
++
++ res = NULL;
++ if (XRRQueryVersion(dpy, &i, &i))
++ res = _XRRGetScreenResourcesCurrent(dpy, root);
++ if (res == NULL)
++ return 77;
++
++ if (v == REDIRECTED && !has_composite(dpy))
++ return 77;
++
++ original_crtc = malloc(sizeof(XRRCrtcInfo *)*res->ncrtc);
++ for (i = 0; i < res->ncrtc; i++)
++ original_crtc[i] = XRRGetCrtcInfo(dpy, res, res->crtcs[i]);
++
++ for (i = 0; i < res->ncrtc; i++)
++ XRRSetCrtcConfig(dpy, res, res->crtcs[i], CurrentTime,
++ 0, 0, None, RR_Rotate_0, NULL, 0);
++
++ DRI2CreateDrawable(dpy, root);
++ DRI2SwapInterval(dpy, root, 0);
++
++ if (d != OFF) {
++ for (i = 0; i < res->noutput; i++) {
++ XRROutputInfo *output;
++ XRRModeInfo *mode;
++
++ output = XRRGetOutputInfo(dpy, res, res->outputs[i]);
++ if (output == NULL)
++ continue;
++
++ mode = NULL;
++ if (res->nmode)
++ mode = lookup_mode(res, output->modes[0]);
++ if (mode == NULL)
++ continue;
++
++ XRRSetCrtcConfig(dpy, res, output->crtcs[0], CurrentTime,
++ 0, 0, output->modes[0], RR_Rotate_0, &res->outputs[i], 1);
++ width = mode->width;
++ height = mode->height;
++ break;
++ }
++ if (i == res->noutput) {
++ _x_error_occurred = 77;
++ goto restore;
++ }
++ }
++
++ if (w == ROOT) {
++ run(dpy, root);
++ } else if (w == FULLSCREEN) {
++ win = XCreateWindow(dpy, root,
++ 0, 0, width, height, 0,
++ DefaultDepth(dpy, DefaultScreen(dpy)),
++ InputOutput,
++ DefaultVisual(dpy, DefaultScreen(dpy)),
++ CWOverrideRedirect, &attr);
++ DRI2CreateDrawable(dpy, win);
++ DRI2SwapInterval(dpy, win, 0);
++ if (v == REDIRECTED) {
++ XCompositeRedirectWindow(dpy, win, CompositeRedirectManual);
++ XDamageCreate(dpy, win, XDamageReportRawRectangles);
++ } else
++ fullscreen(dpy, win);
++ XMapWindow(dpy, win);
++ run(dpy, win);
++ } else if (w == WINDOW) {
++ win = XCreateWindow(dpy, root,
++ 0, 0, width/2, height/2, 0,
++ DefaultDepth(dpy, DefaultScreen(dpy)),
++ InputOutput,
++ DefaultVisual(dpy, DefaultScreen(dpy)),
++ CWOverrideRedirect, &attr);
++ DRI2CreateDrawable(dpy, win);
++ DRI2SwapInterval(dpy, win, 0);
++ if (v == REDIRECTED) {
++ XCompositeRedirectWindow(dpy, win, CompositeRedirectManual);
++ XDamageCreate(dpy, win, XDamageReportRawRectangles);
++ }
++ XMapWindow(dpy, win);
++ run(dpy, win);
++ }
++
++restore:
++ for (i = 0; i < res->ncrtc; i++)
++ XRRSetCrtcConfig(dpy, res, res->crtcs[i], CurrentTime,
++ 0, 0, None, RR_Rotate_0, NULL, 0);
++
++ for (i = 0; i < res->ncrtc; i++)
++ XRRSetCrtcConfig(dpy, res, res->crtcs[i], CurrentTime,
++ original_crtc[i]->x,
++ original_crtc[i]->y,
++ original_crtc[i]->mode,
++ original_crtc[i]->rotation,
++ original_crtc[i]->outputs,
++ original_crtc[i]->noutput);
++
++ if (DPMSQueryExtension(dpy, &i, &i))
++ DPMSEnable(dpy);
++
++ XSync(dpy, True);
++ return _x_error_occurred;
++}
+diff --git a/benchmarks/dri3-swap.c b/benchmarks/dri3-swap.c
+new file mode 100644
+index 00000000..4dd423b3
+--- /dev/null
++++ b/benchmarks/dri3-swap.c
+@@ -0,0 +1,595 @@
++/*
++ * Copyright (c) 2015 Intel Corporation
++ *
++ * Permission is hereby granted, free of charge, to any person obtaining a
++ * copy of this software and associated documentation files (the "Software"),
++ * to deal in the Software without restriction, including without limitation
++ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
++ * and/or sell copies of the Software, and to permit persons to whom the
++ * Software is furnished to do so, subject to the following conditions:
++ *
++ * The above copyright notice and this permission notice (including the next
++ * paragraph) shall be included in all copies or substantial portions of the
++ * Software.
++ *
++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
++ * SOFTWARE.
++ *
++ */
++
++#ifdef HAVE_CONFIG_H
++#include "config.h"
++#endif
++
++#include <X11/Xlib.h>
++#include <X11/Xatom.h>
++#include <X11/Xlib-xcb.h>
++#include <X11/xshmfence.h>
++#include <X11/Xutil.h>
++#include <X11/Xlibint.h>
++#include <X11/extensions/Xcomposite.h>
++#include <X11/extensions/Xdamage.h>
++#include <X11/extensions/dpms.h>
++#include <X11/extensions/randr.h>
++#include <X11/extensions/Xrandr.h>
++#include <xcb/xcb.h>
++#include <xcb/present.h>
++#include <xcb/dri3.h>
++#include <xcb/xfixes.h>
++#include <xf86drm.h>
++#include <i915_drm.h>
++
++#include <stdio.h>
++#include <string.h>
++#include <fcntl.h>
++#include <unistd.h>
++#include <assert.h>
++#include <errno.h>
++#include <setjmp.h>
++#include <signal.h>
++
++struct dri3_fence {
++ XID xid;
++ void *addr;
++};
++
++static int _x_error_occurred;
++static uint32_t stamp;
++
++struct list {
++ struct list *next, *prev;
++};
++
++static void
++list_init(struct list *list)
++{
++ list->next = list->prev = list;
++}
++
++static inline void
++__list_add(struct list *entry,
++ struct list *prev,
++ struct list *next)
++{
++ next->prev = entry;
++ entry->next = next;
++ entry->prev = prev;
++ prev->next = entry;
++}
++
++static inline void
++list_add(struct list *entry, struct list *head)
++{
++ __list_add(entry, head, head->next);
++}
++
++static inline void
++__list_del(struct list *prev, struct list *next)
++{
++ next->prev = prev;
++ prev->next = next;
++}
++
++static inline void
++_list_del(struct list *entry)
++{
++ __list_del(entry->prev, entry->next);
++}
++
++static inline void
++list_move(struct list *list, struct list *head)
++{
++ if (list->prev != head) {
++ _list_del(list);
++ list_add(list, head);
++ }
++}
++
++#define __container_of(ptr, sample, member) \
++ (void *)((char *)(ptr) - ((char *)&(sample)->member - (char *)(sample)))
++
++#define list_for_each_entry(pos, head, member) \
++ for (pos = __container_of((head)->next, pos, member); \
++ &pos->member != (head); \
++ pos = __container_of(pos->member.next, pos, member))
++
++static int
++_check_error_handler(Display *display,
++ XErrorEvent *event)
++{
++ printf("X11 error from display %s, serial=%ld, error=%d, req=%d.%d\n",
++ DisplayString(display),
++ event->serial,
++ event->error_code,
++ event->request_code,
++ event->minor_code);
++ _x_error_occurred++;
++ return False; /* ignored */
++}
++
++static int dri3_create_fence(Display *dpy,
++ Pixmap pixmap,
++ struct dri3_fence *fence)
++{
++ xcb_connection_t *c = XGetXCBConnection(dpy);
++ struct dri3_fence f;
++ int fd;
++
++ fd = xshmfence_alloc_shm();
++ if (fd < 0)
++ return -1;
++
++ f.addr = xshmfence_map_shm(fd);
++ if (f.addr == NULL) {
++ close(fd);
++ return -1;
++ }
++
++ f.xid = xcb_generate_id(c);
++ xcb_dri3_fence_from_fd(c, pixmap, f.xid, 0, fd);
++
++ *fence = f;
++ return 0;
++}
++
++static double elapsed(const struct timespec *start,
++ const struct timespec *end)
++{
++ return 1e6*(end->tv_sec - start->tv_sec) + (end->tv_nsec - start->tv_nsec)/1000;
++}
++
++struct buffer {
++ struct list link;
++ Pixmap pixmap;
++ struct dri3_fence fence;
++ int fd;
++ int busy;
++};
++
++static void run(Display *dpy, Window win)
++{
++ xcb_connection_t *c = XGetXCBConnection(dpy);
++ struct timespec start, end;
++#define N_BACK 8
++ struct buffer buffer[N_BACK];
++ struct list mru;
++ Window root;
++ unsigned int width, height;
++ unsigned border, depth;
++ unsigned present_flags = XCB_PRESENT_OPTION_ASYNC;
++ xcb_xfixes_region_t update = 0;
++ int completed = 0;
++ int queued = 0;
++ uint32_t eid;
++ void *Q;
++ int i, n;
++
++ list_init(&mru);
++
++ XGetGeometry(dpy, win,
++ &root, &i, &n, &width, &height, &border, &depth);
++
++ _x_error_occurred = 0;
++
++ for (n = 0; n < N_BACK; n++) {
++ xcb_dri3_buffer_from_pixmap_reply_t *reply;
++ int *fds;
++
++ buffer[n].pixmap =
++ XCreatePixmap(dpy, win, width, height, depth);
++ buffer[n].fence.xid = 0;
++ buffer[n].fd = -1;
++
++ if (dri3_create_fence(dpy, win, &buffer[n].fence))
++ return;
++
++ reply = xcb_dri3_buffer_from_pixmap_reply (c,
++ xcb_dri3_buffer_from_pixmap(c, buffer[n].pixmap),
++ NULL);
++ if (reply == NULL)
++ return;
++
++ fds = xcb_dri3_buffer_from_pixmap_reply_fds (c, reply);
++ buffer[n].fd = fds[0];
++ free(reply);
++
++ /* start idle */
++ xshmfence_trigger(buffer[n].fence.addr);
++ buffer[n].busy = 0;
++ list_add(&buffer[n].link, &mru);
++ }
++
++ eid = xcb_generate_id(c);
++ xcb_present_select_input(c, eid, win,
++ XCB_PRESENT_EVENT_MASK_IDLE_NOTIFY |
++ XCB_PRESENT_EVENT_MASK_COMPLETE_NOTIFY);
++ Q = xcb_register_for_special_xge(c, &xcb_present_id, eid, &stamp);
++
++ clock_gettime(CLOCK_MONOTONIC, &start);
++ do {
++ for (n = 0; n < 1000; n++) {
++ struct buffer *tmp, *b = NULL;
++ list_for_each_entry(tmp, &mru, link) {
++ if (!tmp->busy) {
++ b = tmp;
++ break;
++ }
++ }
++ while (b == NULL) {
++ xcb_present_generic_event_t *ev;
++
++ ev = (xcb_present_generic_event_t *)
++ xcb_wait_for_special_event(c, Q);
++ if (ev == NULL)
++ abort();
++
++ do {
++ switch (ev->evtype) {
++ case XCB_PRESENT_COMPLETE_NOTIFY:
++ completed++;
++ queued--;
++ break;
++
++ case XCB_PRESENT_EVENT_IDLE_NOTIFY:
++ {
++ xcb_present_idle_notify_event_t *ie = (xcb_present_idle_notify_event_t *)ev;
++ assert(ie->serial < N_BACK);
++ buffer[ie->serial].busy = 0;
++ if (b == NULL)
++ b = &buffer[ie->serial];
++ break;
++ }
++ }
++ free(ev);
++ } while ((ev = (xcb_present_generic_event_t *)xcb_poll_for_special_event(c, Q)));
++ }
++
++ b->busy = 1;
++ if (b->fence.xid) {
++ xshmfence_await(b->fence.addr);
++ xshmfence_reset(b->fence.addr);
++ }
++ xcb_present_pixmap(c, win, b->pixmap, b - buffer,
++ 0, /* valid */
++ update, /* update */
++ 0, /* x_off */
++ 0, /* y_off */
++ None,
++ None, /* wait fence */
++ b->fence.xid,
++ present_flags,
++ 0, /* target msc */
++ 0, /* divisor */
++ 0, /* remainder */
++ 0, NULL);
++ list_move(&b->link, &mru);
++ queued++;
++ xcb_flush(c);
++ }
++ clock_gettime(CLOCK_MONOTONIC, &end);
++ } while (end.tv_sec < start.tv_sec + 10);
++
++ while (queued) {
++ xcb_present_generic_event_t *ev;
++
++ ev = (xcb_present_generic_event_t *)
++ xcb_wait_for_special_event(c, Q);
++ if (ev == NULL)
++ abort();
++
++ do {
++ switch (ev->evtype) {
++ case XCB_PRESENT_COMPLETE_NOTIFY:
++ completed++;
++ queued--;
++ break;
++
++ case XCB_PRESENT_EVENT_IDLE_NOTIFY:
++ break;
++ }
++ free(ev);
++ } while ((ev = (xcb_present_generic_event_t *)xcb_poll_for_special_event(c, Q)));
++ }
++ clock_gettime(CLOCK_MONOTONIC, &end);
++
++ printf("%f\n", completed / (elapsed(&start, &end) / 1000000));
++}
++
++static int has_present(Display *dpy)
++{
++ xcb_connection_t *c = XGetXCBConnection(dpy);
++ xcb_generic_error_t *error = NULL;
++ void *reply;
++
++ reply = xcb_present_query_version_reply(c,
++ xcb_present_query_version(c,
++ XCB_PRESENT_MAJOR_VERSION,
++ XCB_PRESENT_MINOR_VERSION),
++ &error);
++
++ free(reply);
++ free(error);
++ if (reply == NULL) {
++ fprintf(stderr, "Present not supported on %s\n", DisplayString(dpy));
++ return 0;
++ }
++
++ return 1;
++}
++
++static int has_composite(Display *dpy)
++{
++ int event, error;
++ int major, minor;
++
++ if (!XDamageQueryExtension (dpy, &event, &error))
++ return 0;
++
++ if (!XCompositeQueryExtension(dpy, &event, &error))
++ return 0;
++
++ XCompositeQueryVersion(dpy, &major, &minor);
++
++ return major > 0 || minor >= 4;
++}
++
++static inline XRRScreenResources *_XRRGetScreenResourcesCurrent(Display *dpy, Window window)
++{
++ XRRScreenResources *res;
++
++ res = XRRGetScreenResourcesCurrent(dpy, window);
++ if (res == NULL)
++ res = XRRGetScreenResources(dpy, window);
++
++ return res;
++}
++
++static XRRModeInfo *lookup_mode(XRRScreenResources *res, int id)
++{
++ int i;
++
++ for (i = 0; i < res->nmode; i++) {
++ if (res->modes[i].id == id)
++ return &res->modes[i];
++ }
++
++ return NULL;
++}
++
++static void fullscreen(Display *dpy, Window win)
++{
++ Atom atom = XInternAtom(dpy, "_NET_WM_STATE_FULLSCREEN", False);
++ XChangeProperty(dpy, win,
++ XInternAtom(dpy, "_NET_WM_STATE", False),
++ XA_ATOM, 32, PropModeReplace,
++ (unsigned char *)&atom, 1);
++}
++
++static int dri3_query_version(Display *dpy, int *major, int *minor)
++{
++ xcb_connection_t *c = XGetXCBConnection(dpy);
++ xcb_dri3_query_version_reply_t *reply;
++ xcb_generic_error_t *error;
++
++ *major = *minor = -1;
++
++ reply = xcb_dri3_query_version_reply(c,
++ xcb_dri3_query_version(c,
++ XCB_DRI3_MAJOR_VERSION,
++ XCB_DRI3_MINOR_VERSION),
++ &error);
++ free(error);
++ if (reply == NULL)
++ return -1;
++
++ *major = reply->major_version;
++ *minor = reply->minor_version;
++ free(reply);
++
++ return 0;
++}
++
++static int has_dri3(Display *dpy)
++{
++ const xcb_query_extension_reply_t *ext;
++ int major, minor;
++
++ ext = xcb_get_extension_data(XGetXCBConnection(dpy), &xcb_dri3_id);
++ if (ext == NULL || !ext->present)
++ return 0;
++
++ if (dri3_query_version(dpy, &major, &minor) < 0)
++ return 0;
++
++ return major >= 0;
++}
++
++int main(int argc, char **argv)
++{
++ Display *dpy;
++ Window root, win;
++ XRRScreenResources *res;
++ XRRCrtcInfo **original_crtc;
++ XSetWindowAttributes attr;
++ enum window { ROOT, FULLSCREEN, WINDOW } w = FULLSCREEN;
++ enum visible {REDIRECTED, NORMAL } v = NORMAL;
++ enum display { OFF, ON } d = OFF;
++ int width, height;
++ int i;
++
++ while ((i = getopt(argc, argv, "d:v:w:")) != -1) {
++ switch (i) {
++ case 'd':
++ if (strcmp(optarg, "off") == 0)
++ d = OFF;
++ else if (strcmp(optarg, "on") == 0)
++ d = ON;
++ else
++ abort();
++ break;
++
++ case 'v':
++ if (strcmp(optarg, "redirected") == 0)
++ v = REDIRECTED;
++ else if (strcmp(optarg, "normal") == 0)
++ v = NORMAL;
++ else
++ abort();
++ break;
++
++ case 'w':
++ if (strcmp(optarg, "fullscreen") == 0)
++ w = FULLSCREEN;
++ else if (strcmp(optarg, "window") == 0)
++ w = WINDOW;
++ else if (strcmp(optarg, "root") == 0)
++ w = ROOT;
++ else
++ abort();
++ break;
++ }
++ }
++
++ attr.override_redirect = 1;
++
++ dpy = XOpenDisplay(NULL);
++ if (dpy == NULL)
++ return 77;
++
++ width = DisplayWidth(dpy, DefaultScreen(dpy));
++ height = DisplayHeight(dpy, DefaultScreen(dpy));
++
++ if (!has_present(dpy))
++ return 77;
++
++ if (!has_dri3(dpy))
++ return 77;
++
++ if (DPMSQueryExtension(dpy, &i, &i))
++ DPMSDisable(dpy);
++
++ root = DefaultRootWindow(dpy);
++
++ signal(SIGALRM, SIG_IGN);
++ XSetErrorHandler(_check_error_handler);
++
++ res = NULL;
++ if (XRRQueryVersion(dpy, &i, &i))
++ res = _XRRGetScreenResourcesCurrent(dpy, root);
++ if (res == NULL)
++ return 77;
++
++ if (v == REDIRECTED && !has_composite(dpy))
++ return 77;
++
++ original_crtc = malloc(sizeof(XRRCrtcInfo *)*res->ncrtc);
++ for (i = 0; i < res->ncrtc; i++)
++ original_crtc[i] = XRRGetCrtcInfo(dpy, res, res->crtcs[i]);
++
++ for (i = 0; i < res->ncrtc; i++)
++ XRRSetCrtcConfig(dpy, res, res->crtcs[i], CurrentTime,
++ 0, 0, None, RR_Rotate_0, NULL, 0);
++
++ if (d != OFF) {
++ for (i = 0; i < res->noutput; i++) {
++ XRROutputInfo *output;
++ XRRModeInfo *mode;
++
++ output = XRRGetOutputInfo(dpy, res, res->outputs[i]);
++ if (output == NULL)
++ continue;
++
++ mode = NULL;
++ if (res->nmode)
++ mode = lookup_mode(res, output->modes[0]);
++ if (mode == NULL)
++ continue;
++
++ XRRSetCrtcConfig(dpy, res, output->crtcs[0], CurrentTime,
++ 0, 0, output->modes[0], RR_Rotate_0, &res->outputs[i], 1);
++ width = mode->width;
++ height = mode->height;
++ break;
++ }
++ if (i == res->noutput) {
++ _x_error_occurred = 77;
++ goto restore;
++ }
++ }
++
++ if (w == ROOT) {
++ run(dpy, root);
++ } else if (w == FULLSCREEN) {
++ win = XCreateWindow(dpy, root,
++ 0, 0, width, height, 0,
++ DefaultDepth(dpy, DefaultScreen(dpy)),
++ InputOutput,
++ DefaultVisual(dpy, DefaultScreen(dpy)),
++ CWOverrideRedirect, &attr);
++ if (v == REDIRECTED) {
++ XCompositeRedirectWindow(dpy, win, CompositeRedirectManual);
++ XDamageCreate(dpy, win, XDamageReportRawRectangles);
++ } else
++ fullscreen(dpy, win);
++ XMapWindow(dpy, win);
++ run(dpy, win);
++ } else if (w == WINDOW) {
++ win = XCreateWindow(dpy, root,
++ 0, 0, width/2, height/2, 0,
++ DefaultDepth(dpy, DefaultScreen(dpy)),
++ InputOutput,
++ DefaultVisual(dpy, DefaultScreen(dpy)),
++ CWOverrideRedirect, &attr);
++ if (v == REDIRECTED) {
++ XCompositeRedirectWindow(dpy, win, CompositeRedirectManual);
++ XDamageCreate(dpy, win, XDamageReportRawRectangles);
++ }
++ XMapWindow(dpy, win);
++ run(dpy, win);
++ }
++
++restore:
++ for (i = 0; i < res->ncrtc; i++)
++ XRRSetCrtcConfig(dpy, res, res->crtcs[i], CurrentTime,
++ 0, 0, None, RR_Rotate_0, NULL, 0);
++
++ for (i = 0; i < res->ncrtc; i++)
++ XRRSetCrtcConfig(dpy, res, res->crtcs[i], CurrentTime,
++ original_crtc[i]->x,
++ original_crtc[i]->y,
++ original_crtc[i]->mode,
++ original_crtc[i]->rotation,
++ original_crtc[i]->outputs,
++ original_crtc[i]->noutput);
++
++ if (DPMSQueryExtension(dpy, &i, &i))
++ DPMSEnable(dpy);
++
++ XSync(dpy, True);
++ return _x_error_occurred;
++}
+diff --git a/configure.ac b/configure.ac
+index 61bea435..d13917ec 100644
+--- a/configure.ac
++++ b/configure.ac
+@@ -195,18 +195,24 @@ AC_ARG_ENABLE(udev,
+ [UDEV="$enableval"],
+ [UDEV=auto])
+
++udev_msg=" disabled"
+ if test "x$UDEV" != "xno"; then
+ PKG_CHECK_MODULES(UDEV, [libudev], [udev="yes"], [udev="no"])
++ AC_CHECK_HEADERS([sys/stat.h], [], [udev="no"])
+ if test "x$UDEV" = "xyes" -a "x$udev" != "xyes"; then
+ AC_MSG_ERROR([udev support requested but not found (libudev)])
+ fi
+ if test "x$udev" = "xyes"; then
+ AC_DEFINE(HAVE_UDEV,1,[Enable udev-based monitor hotplug detection])
++ udev_msg=" yes"
++ else
++ udev_msg=" no"
+ fi
+ fi
+
+-PKG_CHECK_MODULES(X11, [x11 xrender xrandr xext xfixes cairo cairo-xlib-xrender pixman-1 libpng], [x11="yes"], [x11="no"])
++PKG_CHECK_MODULES(X11, [x11 x11-xcb xcb-dri2 xcomposite xdamage xrender xrandr xext xfixes cairo cairo-xlib-xrender pixman-1 libpng], [x11="yes"], [x11="no"])
+ AM_CONDITIONAL(HAVE_X11, test "x$x11" = "xyes")
++echo X11_CLFAGS="$X11_CLFAGS" X11_LIBS="$X11_LIBS"
+
+ cpuid="yes"
+ AC_TRY_LINK([
+@@ -270,10 +276,13 @@ if test "x$shm" = "xyes"; then
+ AC_DEFINE([HAVE_MIT_SHM], 1, [Define to 1 if MIT-SHM is available])
+ fi
+
+-PKG_CHECK_MODULES(X11_DRI3, [xcb-dri3 xcb-sync xcb-present x11-xcb xshmfence x11 xrender xext libdrm], [x11_dri3="yes"], [x11_dri3="no"])
++PKG_CHECK_MODULES(X11_DRI3, [xcb-dri3 xcb-sync xcb-xfixes xcb-present x11-xcb xshmfence x11 xcomposite xdamage xrender xrandr xxf86vm xext libdrm], [x11_dri3="yes"], [x11_dri3="no"])
+ AM_CONDITIONAL(X11_DRI3, test "x$x11_dri3" = "xyes" -a "x$shm" = "xyes")
+ AM_CONDITIONAL(X11_SHM, test "x$shm" = "xyes")
+
++PKG_CHECK_MODULES(X11_VM, [xxf86vm], [x11_vm="yes"], [x11_vm="no"])
++AM_CONDITIONAL(X11_VM, test "x$x11_vm" = "xyes")
++
+ AC_ARG_ENABLE(tools,
+ AS_HELP_STRING([--disable-tools],
+ [Enable building and installing the miscellaneous tools [default=auto]]),
+@@ -285,7 +294,7 @@ if test "x$shm" != "xyes"; then
+ tools="no"
+ fi
+ if test "x$tools" != "xno"; then
+- ivo_requires="xrandr xdamage xfixes xcursor xtst xrender xext x11 pixman-1"
++ ivo_requires="xrandr xdamage xfixes xcursor xtst xrender xscrnsaver xext x11 pixman-1"
+ extra_cflags=""
+
+ ignore="xinerama"
+@@ -307,6 +316,8 @@ if test "x$tools" != "xno"; then
+ tools="no"
+ fi
+
++ PKG_CHECK_MODULES(TOOL_CURSOR, [xfixes x11 libpng], [cursor="yes"], [ivo="no"])
++
+ IVO_CFLAGS="$IVO_CFLAGS $extra_cflags"
+ fi
+ if test "x$tools" != "xno"; then
+@@ -315,6 +326,7 @@ fi
+ AC_MSG_CHECKING([whether to build additional tools])
+ AC_MSG_RESULT([$tools])
+ AM_CONDITIONAL(BUILD_TOOLS, test "x$tools" != "xno")
++AM_CONDITIONAL(BUILD_TOOL_CURSOR, test "x$cursor" = "xyes")
+
+ # Define a configure option for an alternate module directory
+ AC_ARG_WITH(xorg-module-dir,
+@@ -339,10 +351,20 @@ AC_ARG_ENABLE(dri2,
+ [DRI2=$enableval],
+ [DRI2=yes])
+ AC_ARG_ENABLE(dri3,
+- AS_HELP_STRING([--enable-dri3],
+- [Enable DRI3 support [[default=no]]]),
++ AS_HELP_STRING([--disable-dri3],
++ [Disable DRI3 support [[default=yes]]]),
+ [DRI3=$enableval],
+- [DRI3=no])
++ [DRI3=yes])
++AC_ARG_WITH(default-dri,
++ AS_HELP_STRING([--with-default-dri],
++ [Select the default maximum DRI level [default 2]]),
++ [DRI_DEFAULT=$withval],
++ [DRI_DEFAULT=2])
++if test "x$DRI_DEFAULT" = "x0"; then
++ AC_DEFINE(DEFAULT_DRI_LEVEL, 0,[Default DRI level])
++else
++ AC_DEFINE(DEFAULT_DRI_LEVEL, ~0, [Default DRI level])
++fi
+
+ AC_ARG_ENABLE(xvmc, AS_HELP_STRING([--disable-xvmc],
+ [Disable XvMC support [[default=yes]]]),
+@@ -375,14 +397,12 @@ AC_ARG_ENABLE(ums-only,
+ required_xorg_server_version=1.6
+ required_pixman_version=0.16
+
+-if pkg-config --exists 'pixman-1 >= 0.27.1'; then
+- AC_DEFINE([HAS_PIXMAN_GLYPHS], 1, [Enable pixman glyph cache])
+-fi
+-
+-if pkg-config --exists 'pixman-1 >= 0.24.0'; then
+- AC_DEFINE([HAS_PIXMAN_TRIANGLES], 1, [Enable pixman triangle rasterisation])
+-fi
+-
++PKG_CHECK_EXISTS([pixman-1 >= 0.24.0],
++ AC_DEFINE([HAS_PIXMAN_TRIANGLES], 1, [Enable pixman triangle rasterisation])
++ [])
++PKG_CHECK_EXISTS([pixman-1 >= 0.27.1],
++ [AC_DEFINE([HAS_PIXMAN_GLYPHS], 1, [Enable pixman glyph cache])],
++ [])
+ # Store the list of server defined optional extensions in REQUIRED_MODULES
+ XORG_DRIVER_CHECK_EXT(RANDR, randrproto)
+ XORG_DRIVER_CHECK_EXT(RENDER, renderproto)
+@@ -398,24 +418,25 @@ AC_ARG_ENABLE(sna,
+ [SNA="$enableval"],
+ [SNA=auto])
+
++AC_CHECK_HEADERS([dev/wscons/wsconsio.h])
++AC_FUNC_ALLOCA
++AC_HEADER_MAJOR
++
+ if test "x$SNA" != "xno"; then
+ AC_DEFINE(USE_SNA, 1, [Enable SNA support])
+ AC_CHECK_HEADERS([sys/sysinfo.h], AC_CHECK_MEMBERS([struct sysinfo.totalram], [], [], [[#include <sys/sysinfo.h>]]))
+ fi
+
+ uxa_requires_libdrm=2.4.52
++uxa_requires_pixman=0.24.0
++
+ AC_ARG_ENABLE(uxa,
+ AS_HELP_STRING([--enable-uxa],
+ [Enable Unified Acceleration Architecture (UXA) [default=auto]]),
+ [UXA="$enableval"],
+ [UXA=auto])
+ if test "x$UXA" = "xauto"; then
+- if ! pkg-config --exists "libdrm_intel >= $uxa_requires_libdrm"; then
+- UXA=no
+- fi
+- if ! pkg-config --exists 'pixman-1 >= 0.24.0'; then
+- UXA=no
+- fi
++ PKG_CHECK_EXISTS([libdrm_intel >= $uxa_requires_libdrm pixman-1 >= $uxa_requires_pixman], [], [UXA=no])
+ fi
+ if test "x$UXA" != "xno"; then
+ AC_DEFINE(USE_UXA, 1, [Enable UXA support])
+@@ -424,8 +445,10 @@ if test "x$UXA" != "xno"; then
+ UXA=yes
+ fi
+
+-PKG_CHECK_MODULES(XORG, [xorg-server >= $required_xorg_server_version xproto fontsproto pixman-1 >= $required_pixman_version $REQUIRED_MODULES])
++PKG_CHECK_MODULES(XORG, [xorg-server >= $required_xorg_server_version xproto fontsproto damageproto pixman-1 >= $required_pixman_version $REQUIRED_MODULES])
+ ABI_VERSION=`$PKG_CONFIG --variable=abi_videodrv xorg-server`
++XSERVER_VERSION=`$PKG_CONFIG --modversion xorg-server`
++PIXMAN_VERSION=`$PKG_CONFIG --modversion pixman-1`
+
+ if test "x$ONLY_UMS" = "xyes"; then
+ UMS="yes"
+@@ -519,7 +542,12 @@ AC_MSG_RESULT([$have_dri1])
+ AM_CONDITIONAL(DRI1, test "x$have_dri1" != "xno")
+ if test "x$have_dri1" != "xno"; then
+ AC_DEFINE(HAVE_DRI1,1,[Enable DRI1 driver support])
+- dri_msg="$dri_msg DRI1"
++ str="DRI1"
++ if test "x$DRI_DEFAULT" = "x1"; then
++ AC_DEFINE(DEFAULT_DRI_LEVEL,1,[Default DRI level])
++ str="*$str"
++ fi
++ dri_msg="$dri_msg $str"
+ else
+ DRI1_CFLAGS=""
+ DRI1_LIBS=""
+@@ -576,7 +604,12 @@ AM_CONDITIONAL(DRI2, test "x$have_dri2" != "xno")
+ AC_MSG_RESULT([$have_dri2])
+ if test "x$have_dri2" != "xno"; then
+ AC_DEFINE(HAVE_DRI2,1,[Enable DRI2 driver support])
+- dri_msg="$dri_msg DRI2"
++ str="DRI2"
++ if test "x$DRI_DEFAULT" = "x2"; then
++ AC_DEFINE(DEFAULT_DRI_LEVEL,2,[Default DRI level])
++ str="*$str"
++ fi
++ dri_msg="$dri_msg $str"
+ else
+ if test "x$DRI" = "xyes" -a "x$DRI2" != "xno" -a "x$KMS" = "xyes"; then
+ AC_MSG_ERROR([DRI2 requested but prerequisites not found])
+@@ -591,13 +624,21 @@ AM_CONDITIONAL(DRI3, test "x$have_dri3" != "xno")
+ AC_MSG_RESULT([$have_dri3])
+ if test "x$have_dri3" != "xno"; then
+ AC_DEFINE(HAVE_DRI3,1,[Enable DRI3 driver support])
+- dri_msg="$dri_msg DRI3"
++ str="DRI3"
++ if test "x$DRI_DEFAULT" = "x3"; then
++ AC_DEFINE(DEFAULT_DRI_LEVEL,3,[Default DRI level])
++ str="*$str"
++ fi
++ dri_msg="$dri_msg $str"
+ else
+ if test "x$DRI" = "xyes" -a "x$DRI3" != "xno" -a "x$KMS" = "xyes"; then
+ AC_MSG_ERROR([DRI3 requested but prerequisites not found])
+ fi
+ fi
+
++AC_MSG_CHECKING([default DRI support])
++AC_MSG_RESULT([$DEFAULT_DRI_DEFAULT])
++
+ AC_CHECK_HEADERS([X11/extensions/dpmsconst.h])
+
+ PRESENT="no"
+@@ -711,27 +752,6 @@ if test "x$TEARFREE" = "xyes"; then
+ xp_msg="$xp_msg TearFree"
+ fi
+
+-AC_ARG_ENABLE(rendernode,
+- AS_HELP_STRING([--enable-rendernode],
+- [Enable use of render nodes (experimental) [default=no]]),
+- [RENDERNODE="$enableval"],
+- [RENDERNODE="no"])
+-AM_CONDITIONAL(USE_RENDERNODE, test "x$RENDERNODE" = "xyes")
+-if test "x$RENDERNODE" = "xyes"; then
+- AC_DEFINE(USE_RENDERNODE,1,[Assume "rendernode" support])
+- xp_msg="$xp_msg rendernode"
+-fi
+-
+-AC_ARG_ENABLE(wc-mmap,
+- AS_HELP_STRING([--enable-wc-mmap],
+- [Enable use of WriteCombining mmaps [default=no]]),
+- [WC_MMAP="$enableval"],
+- [WC_MMAP="no"])
+-if test "x$WC_MMAP" = "xyes"; then
+- AC_DEFINE(USE_WC_MMAP,1,[Enable use of WriteCombining mmaps])
+- xp_msg="$xp_msg mmap(wc)"
+-fi
+-
+ AC_ARG_ENABLE(create2,
+ AS_HELP_STRING([--enable-create2],
+ [Enable use of create2 ioctl (experimental) [default=no]]),
+@@ -848,6 +868,7 @@ AC_CONFIG_FILES([
+ xvmc/shader/mc/Makefile
+ xvmc/shader/vld/Makefile
+ test/Makefile
++ benchmarks/Makefile
+ tools/Makefile
+ tools/org.x.xf86-video-intel.backlight-helper.policy
+ ])
+@@ -855,7 +876,7 @@ AC_OUTPUT
+
+ echo ""
+ echo ""
+-test -e `pwd $0`/README && cat `pwd $0`/README
++cat $srcdir/README
+
+ accel_msg=""
+ if test "x$SNA" != "xno"; then
+@@ -895,13 +916,15 @@ fi
+
+ echo ""
+ echo "AC_PACKAGE_STRING will be compiled with:"
+-echo " Xorg Video ABI version: $ABI_VERSION"
++echo " Xorg Video ABI version: $ABI_VERSION (xorg-server-$XSERVER_VERSION)"
++echo " pixman version: pixman-1-$PIXMAN_VERSION"
+ echo " Acceleration backends:$accel_msg"
+ echo " Additional debugging support?$debug_msg"
+ echo " Support for Kernel Mode Setting? $KMS"
+ echo " Support for legacy User Mode Setting (for i810)? $UMS"
+ echo " Support for Direct Rendering Infrastructure:$dri_msg"
+ echo " Support for Xv motion compensation (XvMC and libXvMC):$xvmc_msg"
++echo " Support for display hotplug notifications (udev):$udev_msg"
+ echo " Build additional tools and utilities?$tools_msg"
+ if test -n "$xp_msg"; then
+ echo " Experimental support:$xp_msg"
+diff --git a/libobj/alloca.c b/libobj/alloca.c
+new file mode 100644
+index 00000000..883e1e9f
+--- /dev/null
++++ b/libobj/alloca.c
+@@ -0,0 +1,4 @@
++void *alloca(size_t sz)
++{
++ return NULL;
++}
+diff --git a/man/intel.man b/man/intel.man
+index 17515206..be398fbe 100644
+--- a/man/intel.man
++++ b/man/intel.man
+@@ -27,9 +27,9 @@ supports the i810, i810-DC100, i810e, i815, i830M, 845G, 852GM, 855GM,
+ 865G, 915G, 915GM, 945G, 945GM, 965G, 965Q, 946GZ, 965GM, 945GME,
+ G33, Q33, Q35, G35, GM45, G45, Q45, G43, G41 chipsets, Pineview-M in
+ Atom N400 series, Pineview-D in Atom D400/D500 series,
+-Intel(R) HD Graphics: 2000-6000,
+-Intel(R) Iris(TM) Graphics: 5100/6100, and
+-Intel(R) Iris(TM) Pro Graphics: 5200/6200/P6300.
++Intel(R) HD Graphics,
++Intel(R) Iris(TM) Graphics,
++Intel(R) Iris(TM) Pro Graphics.
+
+ .SH CONFIGURATION DETAILS
+ Please refer to __xconfigfile__(__filemansuffix__) for general configuration
+@@ -112,8 +112,8 @@ The default is 8192 if AGP allocable memory is < 128 MB, 16384 if < 192 MB,
+ 24576 if higher. DRI require at least a value of 16384. Higher values may give
+ better 3D performance, at expense of available system memory.
+ .TP
+-.BI "Option \*qNoAccel\*q \*q" boolean \*q
+-Disable or enable acceleration.
++.BI "Option \*qAccel\*q \*q" boolean \*q
++Enable or disable acceleration.
+ .IP
+ Default: acceleration is enabled.
+
+@@ -122,8 +122,8 @@ The following driver
+ .B Options
+ are supported for the 830M and later chipsets:
+ .TP
+-.BI "Option \*qNoAccel\*q \*q" boolean \*q
+-Disable or enable acceleration.
++.BI "Option \*qAccel\*q \*q" boolean \*q
++Enable or disable acceleration.
+ .IP
+ Default: acceleration is enabled.
+ .TP
+@@ -201,6 +201,16 @@ that choice by specifying the entry under /sys/class/backlight to use.
+ .IP
+ Default: Automatic selection.
+ .TP
++.BI "Option \*qCustomEDID\*q \*q" string \*q
++Override the probed EDID on particular outputs. Sometimes the manufacturer
++supplied EDID is corrupt or lacking a few usable modes and supplying a
++corrected EDID may be easier than specifying every modeline. This option
++allows to pass the path to load an EDID from per output. The format is a
++comma separated string of output:path pairs, e.g.
++DP1:/path/to/dp1.edid,DP2:/path/to/dp2.edid
++.IP
++Default: No override, use manufacturer supplied EDIDs.
++.TP
+ .BI "Option \*qFallbackDebug\*q \*q" boolean \*q
+ Enable printing of debugging information on acceleration fallbacks to the
+ server log.
+@@ -225,6 +235,15 @@ i.e. perform synchronous rendering.
+ .IP
+ Default: Disabled
+ .TP
++.BI "Option \*qHWRotation\*q \*q" boolean \*q
++Override the use of native hardware rotation and force the use of software,
++but GPU accelerated where possible, rotation. On some platforms the hardware
++can scanout directly into a rotated output bypassing the intermediate rendering
++and extra allocations required for software implemented rotation (i.e. native
++rotation uses less resources, is quicker and uses less power). This allows you
++to disable the native rotation in case of errors.
++.IP
++Default: Enabled (use hardware rotation)
+ .TP
+ .BI "Option \*qVSync\*q \*q" boolean \*q
+ This option controls the use of commands to synchronise rendering with the
+@@ -324,13 +343,29 @@ Default: 0
+ .BI "Option \*qZaphodHeads\*q \*q" string \*q
+ .IP
+ Specify the randr output(s) to use with zaphod mode for a particular driver
+-instance. If you this option you must use it with all instances of the
+-driver
++instance. If you set this option you must use it with all instances of the
++driver. By default, each head is assigned only one CRTC (which limits
++using multiple outputs with that head to cloned mode). CRTC can be manually
++assigned to individual heads by preceding the output names with a comma
++delimited list of pipe numbers followed by a colon. Note that different pipes
++may be limited in their functionality and some outputs may only work with
++different pipes.
+ .br
+ For example:
++
++.RS
+ .B
+ Option \*qZaphodHeads\*q \*qLVDS1,VGA1\*q
+-will assign xrandr outputs LVDS1 and VGA0 to this instance of the driver.
++
++will assign xrandr outputs LVDS1 and VGA1 to this instance of the driver.
++.RE
++
++.RS
++.B
++Option \*qZaphodHeads\*q \*q0,2:HDMI1,DP2\*q
++
++will assign xrandr outputs HDMI1 and DP2 and CRTCs 0 and 2 to this instance of the driver.
++.RE
+
+ .SH OUTPUT CONFIGURATION
+ On 830M and better chipsets, the driver supports runtime configuration of
+@@ -431,11 +466,11 @@ First DVI SDVO output
+ Second DVI SDVO output
+
+ .SS "TMDS-1", "TMDS-2", "HDMI-1", "HDMI-2"
+-DVI/HDMI outputs. Avaliable common properties include:
++DVI/HDMI outputs. Available common properties include:
+ .TP
+ \fBBROADCAST_RGB\fP - method used to set RGB color range
+ Adjusting this property allows you to set RGB color range on each
+-channel in order to match HDTV requirment(default 0 for full
++channel in order to match HDTV requirement(default 0 for full
+ range). Setting 1 means RGB color range is 16-235, 0 means RGB color
+ range is 0-255 on each channel. (Full range is 0-255, not 16-235)
+
+diff --git a/src/backlight.c b/src/backlight.c
+index 9f239867..fcbb279f 100644
+--- a/src/backlight.c
++++ b/src/backlight.c
+@@ -34,6 +34,12 @@
+ #include <sys/stat.h>
+ #include <sys/ioctl.h>
+
++#if MAJOR_IN_MKDEV
++#include <sys/mkdev.h>
++#elif MAJOR_IN_SYSMACROS
++#include <sys/sysmacros.h>
++#endif
++
+ #include <stdio.h>
+ #include <stdlib.h>
+ #include <string.h>
+@@ -42,6 +48,7 @@
+ #include <fcntl.h>
+ #include <unistd.h>
+ #include <dirent.h>
++#include <errno.h>
+
+ #include <xorg-server.h>
+ #include <xf86.h>
+@@ -84,7 +91,7 @@ void backlight_init(struct backlight *b)
+ b->has_power = 0;
+ }
+
+-#ifdef __OpenBSD__
++#ifdef HAVE_DEV_WSCONS_WSCONSIO_H
+
+ #include <dev/wscons/wsconsio.h>
+ #include <xf86Priv.h>
+@@ -122,6 +129,11 @@ int backlight_get(struct backlight *b)
+ return param.curval;
+ }
+
++char *backlight_find_for_device(struct pci_device *pci)
++{
++ return NULL;
++}
++
+ int backlight_open(struct backlight *b, char *iface)
+ {
+ struct wsdisplay_param param;
+@@ -146,12 +158,9 @@ int backlight_open(struct backlight *b, char *iface)
+ return param.curval;
+ }
+
+-enum backlight_type backlight_exists(const char *iface)
++int backlight_exists(const char *iface)
+ {
+- if (iface != NULL)
+- return BL_NONE;
+-
+- return BL_PLATFORM;
++ return iface == NULL;
+ }
+
+ int backlight_on(struct backlight *b)
+@@ -163,6 +172,7 @@ int backlight_off(struct backlight *b)
+ {
+ return 0;
+ }
++
+ #else
+
+ static int
+@@ -213,6 +223,24 @@ __backlight_read(const char *iface, const char *file)
+ }
+
+ static int
++writen(int fd, const char *value, int len)
++{
++ int ret;
++
++ do {
++ ret = write(fd, value, len);
++ if (ret < 0) {
++ if (errno == EAGAIN || errno == EINTR)
++ continue;
++
++ return ret;
++ }
++ } while (value += ret, len -= ret);
++
++ return 0;
++}
++
++static int
+ __backlight_write(const char *iface, const char *file, const char *value)
+ {
+ int fd, ret;
+@@ -221,7 +249,7 @@ __backlight_write(const char *iface, const char *file, const char *value)
+ if (fd < 0)
+ return -1;
+
+- ret = write(fd, value, strlen(value)+1);
++ ret = writen(fd, value, strlen(value)+1);
+ close(fd);
+
+ return ret;
+@@ -244,10 +272,10 @@ static const char *known_interfaces[] = {
+ "intel_backlight",
+ };
+
+-static enum backlight_type __backlight_type(const char *iface)
++static int __backlight_type(const char *iface)
+ {
+ char buf[1024];
+- int fd, v;
++ int fd, v, i;
+
+ v = -1;
+ fd = __backlight_open(iface, "type", O_RDONLY);
+@@ -261,39 +289,41 @@ static enum backlight_type __backlight_type(const char *iface)
+ buf[v] = '\0';
+
+ if (strcmp(buf, "raw") == 0)
+- v = BL_RAW;
++ v = BL_RAW << 8;
+ else if (strcmp(buf, "platform") == 0)
+- v = BL_PLATFORM;
++ v = BL_PLATFORM << 8;
+ else if (strcmp(buf, "firmware") == 0)
+- v = BL_FIRMWARE;
++ v = BL_FIRMWARE << 8;
+ else
+- v = BL_NAMED;
++ v = BL_NAMED << 8;
+ } else
+- v = BL_NAMED;
++ v = BL_NAMED << 8;
+
+- if (v == BL_NAMED) {
+- int i;
+- for (i = 0; i < ARRAY_SIZE(known_interfaces); i++) {
+- if (strcmp(iface, known_interfaces[i]) == 0)
+- break;
+- }
+- v += i;
++ for (i = 0; i < ARRAY_SIZE(known_interfaces); i++) {
++ if (strcmp(iface, known_interfaces[i]) == 0)
++ break;
+ }
++ v += i;
+
+ return v;
+ }
+
+-enum backlight_type backlight_exists(const char *iface)
++static int __backlight_exists(const char *iface)
+ {
+ if (__backlight_read(iface, "brightness") < 0)
+- return BL_NONE;
++ return -1;
+
+ if (__backlight_read(iface, "max_brightness") <= 0)
+- return BL_NONE;
++ return -1;
+
+ return __backlight_type(iface);
+ }
+
++int backlight_exists(const char *iface)
++{
++ return __backlight_exists(iface) != -1;
++}
++
+ static int __backlight_init(struct backlight *b, char *iface, int fd)
+ {
+ b->fd = fd_move_cloexec(fd_set_nonblock(fd));
+@@ -399,7 +429,50 @@ __backlight_find(void)
+ continue;
+
+ /* Fallback to priority list of known iface for old kernels */
+- v = backlight_exists(de->d_name);
++ v = __backlight_exists(de->d_name);
++ if (v < 0)
++ continue;
++
++ if (v < best_type) {
++ char *copy = strdup(de->d_name);
++ if (copy) {
++ free(best_iface);
++ best_iface = copy;
++ best_type = v;
++ }
++ }
++ }
++ closedir(dir);
++
++ return best_iface;
++}
++
++char *backlight_find_for_device(struct pci_device *pci)
++{
++ char path[200];
++ unsigned best_type = INT_MAX;
++ char *best_iface = NULL;
++ DIR *dir;
++ struct dirent *de;
++
++ snprintf(path, sizeof(path),
++ "/sys/bus/pci/devices/%04x:%02x:%02x.%d/backlight",
++ pci->domain, pci->bus, pci->dev, pci->func);
++
++ dir = opendir(path);
++ if (dir == NULL)
++ return NULL;
++
++ while ((de = readdir(dir))) {
++ int v;
++
++ if (*de->d_name == '.')
++ continue;
++
++ v = __backlight_exists(de->d_name);
++ if (v < 0)
++ continue;
++
+ if (v < best_type) {
+ char *copy = strdup(de->d_name);
+ if (copy) {
+@@ -416,14 +489,17 @@ __backlight_find(void)
+
+ int backlight_open(struct backlight *b, char *iface)
+ {
+- int level;
++ int level, type;
+
+ if (iface == NULL)
+ iface = __backlight_find();
+ if (iface == NULL)
+ goto err;
+
+- b->type = __backlight_type(iface);
++ type = __backlight_type(iface);
++ if (type < 0)
++ goto err;
++ b->type = type >> 8;
+
+ b->max = __backlight_read(iface, "max_brightness");
+ if (b->max <= 0)
+@@ -447,7 +523,7 @@ err:
+ int backlight_set(struct backlight *b, int level)
+ {
+ char val[BACKLIGHT_VALUE_LEN];
+- int len, ret = 0;
++ int len;
+
+ if (b->iface == NULL)
+ return 0;
+@@ -456,10 +532,7 @@ int backlight_set(struct backlight *b, int level)
+ level = b->max;
+
+ len = snprintf(val, BACKLIGHT_VALUE_LEN, "%d\n", level);
+- if (write(b->fd, val, len) != len)
+- ret = -1;
+-
+- return ret;
++ return writen(b->fd, val, len);
+ }
+
+ int backlight_get(struct backlight *b)
+@@ -517,43 +590,6 @@ void backlight_disable(struct backlight *b)
+ void backlight_close(struct backlight *b)
+ {
+ backlight_disable(b);
+- if (b->pid)
++ if (b->pid > 0)
+ waitpid(b->pid, NULL, 0);
+ }
+-
+-char *backlight_find_for_device(struct pci_device *pci)
+-{
+- char path[200];
+- unsigned best_type = INT_MAX;
+- char *best_iface = NULL;
+- DIR *dir;
+- struct dirent *de;
+-
+- snprintf(path, sizeof(path),
+- "/sys/bus/pci/devices/%04x:%02x:%02x.%d/backlight",
+- pci->domain, pci->bus, pci->dev, pci->func);
+-
+- dir = opendir(path);
+- if (dir == NULL)
+- return NULL;
+-
+- while ((de = readdir(dir))) {
+- int v;
+-
+- if (*de->d_name == '.')
+- continue;
+-
+- v = backlight_exists(de->d_name);
+- if (v < best_type) {
+- char *copy = strdup(de->d_name);
+- if (copy) {
+- free(best_iface);
+- best_iface = copy;
+- best_type = v;
+- }
+- }
+- }
+- closedir(dir);
+-
+- return best_iface;
+-}
+diff --git a/src/backlight.h b/src/backlight.h
+index bb0e28bc..ba17755b 100644
+--- a/src/backlight.h
++++ b/src/backlight.h
+@@ -43,7 +43,7 @@ struct backlight {
+ int pid, fd;
+ };
+
+-enum backlight_type backlight_exists(const char *iface);
++int backlight_exists(const char *iface);
+
+ void backlight_init(struct backlight *backlight);
+ int backlight_open(struct backlight *backlight, char *iface);
+diff --git a/src/compat-api.h b/src/compat-api.h
+index d09e1fb3..05797a08 100644
+--- a/src/compat-api.h
++++ b/src/compat-api.h
+@@ -30,6 +30,7 @@
+
+ #include <xorg-server.h>
+ #include <xorgVersion.h>
++#include <xf86Module.h>
+
+ #include <picturestr.h>
+ #ifndef GLYPH_HAS_GLYPH_PICTURE_ACCESSOR
+@@ -39,7 +40,17 @@
+
+ #ifndef XF86_HAS_SCRN_CONV
+ #define xf86ScreenToScrn(s) xf86Screens[(s)->myNum]
++#if XORG_VERSION_CURRENT < XORG_VERSION_NUMERIC(1,1,0,0,0)
+ #define xf86ScrnToScreen(s) screenInfo.screens[(s)->scrnIndex]
++#else
++#define xf86ScrnToScreen(s) ((s)->pScreen)
++#endif
++#else
++#define xf86ScrnToScreen(s) ((s)->pScreen)
++#endif
++
++#if GET_ABI_MAJOR(ABI_VIDEODRV_VERSION) >= 22
++#define HAVE_NOTIFY_FD 1
+ #endif
+
+ #ifndef XF86_SCRN_INTERFACE
+@@ -131,6 +142,17 @@ region_rects(const RegionRec *r)
+ return r->data ? (const BoxRec *)(r->data + 1) : &r->extents;
+ }
+
++inline static void
++region_get_boxes(const RegionRec *r, const BoxRec **s, const BoxRec **e)
++{
++ int n;
++ if (r->data)
++ *s = region_boxptr(r), n = r->data->numRects;
++ else
++ *s = &r->extents, n = 1;
++ *e = *s + n;
++}
++
+ #ifndef INCLUDE_LEGACY_REGION_DEFINES
+ #define RegionCreate(r, s) REGION_CREATE(NULL, r, s)
+ #define RegionBreak(r) REGION_BREAK(NULL, r)
+@@ -223,4 +245,19 @@ static inline void FreePixmap(PixmapPtr pixmap)
+ dstx, dsty)
+ #endif
+
++#if XORG_VERSION_CURRENT >= XORG_VERSION_NUMERIC(1,12,99,901,0)
++#define isGPU(S) (S)->is_gpu
++#else
++#define isGPU(S) 0
++#endif
++
++#if HAS_DIRTYTRACKING_ROTATION
++#define PixmapSyncDirtyHelper(d, dd) PixmapSyncDirtyHelper(d)
++#endif
++
++#if !HAVE_NOTIFY_FD
++#define SetNotifyFd(fd, cb, mode, data) AddGeneralSocket(fd);
++#define RemoveNotifyFd(fd) RemoveGeneralSocket(fd)
++#endif
++
+ #endif
+diff --git a/src/i915_pciids.h b/src/i915_pciids.h
+index 180ad0e6..466c7159 100644
+--- a/src/i915_pciids.h
++++ b/src/i915_pciids.h
+@@ -134,7 +134,7 @@
+ #define INTEL_IVB_Q_IDS(info) \
+ INTEL_QUANTA_VGA_DEVICE(info) /* Quanta transcode */
+
+-#define INTEL_HSW_D_IDS(info) \
++#define INTEL_HSW_IDS(info) \
+ INTEL_VGA_DEVICE(0x0402, info), /* GT1 desktop */ \
+ INTEL_VGA_DEVICE(0x0412, info), /* GT2 desktop */ \
+ INTEL_VGA_DEVICE(0x0422, info), /* GT3 desktop */ \
+@@ -179,9 +179,7 @@
+ INTEL_VGA_DEVICE(0x0D2B, info), /* CRW GT3 reserved */ \
+ INTEL_VGA_DEVICE(0x0D0E, info), /* CRW GT1 reserved */ \
+ INTEL_VGA_DEVICE(0x0D1E, info), /* CRW GT2 reserved */ \
+- INTEL_VGA_DEVICE(0x0D2E, info) /* CRW GT3 reserved */ \
+-
+-#define INTEL_HSW_M_IDS(info) \
++ INTEL_VGA_DEVICE(0x0D2E, info), /* CRW GT3 reserved */ \
+ INTEL_VGA_DEVICE(0x0406, info), /* GT1 mobile */ \
+ INTEL_VGA_DEVICE(0x0416, info), /* GT2 mobile */ \
+ INTEL_VGA_DEVICE(0x0426, info), /* GT2 mobile */ \
+@@ -198,60 +196,48 @@
+ INTEL_VGA_DEVICE(0x0D16, info), /* CRW GT2 mobile */ \
+ INTEL_VGA_DEVICE(0x0D26, info) /* CRW GT3 mobile */
+
+-#define INTEL_VLV_M_IDS(info) \
++#define INTEL_VLV_IDS(info) \
+ INTEL_VGA_DEVICE(0x0f30, info), \
+ INTEL_VGA_DEVICE(0x0f31, info), \
+ INTEL_VGA_DEVICE(0x0f32, info), \
+ INTEL_VGA_DEVICE(0x0f33, info), \
+- INTEL_VGA_DEVICE(0x0157, info)
+-
+-#define INTEL_VLV_D_IDS(info) \
++ INTEL_VGA_DEVICE(0x0157, info), \
+ INTEL_VGA_DEVICE(0x0155, info)
+
+-#define _INTEL_BDW_M(gt, id, info) \
+- INTEL_VGA_DEVICE((((gt) - 1) << 4) | (id), info)
+-#define _INTEL_BDW_D(gt, id, info) \
+- INTEL_VGA_DEVICE((((gt) - 1) << 4) | (id), info)
+-
+-#define _INTEL_BDW_M_IDS(gt, info) \
+- _INTEL_BDW_M(gt, 0x1602, info), /* ULT */ \
+- _INTEL_BDW_M(gt, 0x1606, info), /* ULT */ \
+- _INTEL_BDW_M(gt, 0x160B, info), /* Iris */ \
+- _INTEL_BDW_M(gt, 0x160E, info) /* ULX */
+-
+-#define _INTEL_BDW_D_IDS(gt, info) \
+- _INTEL_BDW_D(gt, 0x160A, info), /* Server */ \
+- _INTEL_BDW_D(gt, 0x160D, info) /* Workstation */
+-
+-#define INTEL_BDW_GT12M_IDS(info) \
+- _INTEL_BDW_M_IDS(1, info), \
+- _INTEL_BDW_M_IDS(2, info)
+-
+-#define INTEL_BDW_GT12D_IDS(info) \
+- _INTEL_BDW_D_IDS(1, info), \
+- _INTEL_BDW_D_IDS(2, info)
+-
+-#define INTEL_BDW_GT3M_IDS(info) \
+- _INTEL_BDW_M_IDS(3, info)
+-
+-#define INTEL_BDW_GT3D_IDS(info) \
+- _INTEL_BDW_D_IDS(3, info)
+-
+-#define INTEL_BDW_RSVDM_IDS(info) \
+- _INTEL_BDW_M_IDS(4, info)
+-
+-#define INTEL_BDW_RSVDD_IDS(info) \
+- _INTEL_BDW_D_IDS(4, info)
+-
+-#define INTEL_BDW_M_IDS(info) \
+- INTEL_BDW_GT12M_IDS(info), \
+- INTEL_BDW_GT3M_IDS(info), \
+- INTEL_BDW_RSVDM_IDS(info)
+-
+-#define INTEL_BDW_D_IDS(info) \
+- INTEL_BDW_GT12D_IDS(info), \
+- INTEL_BDW_GT3D_IDS(info), \
+- INTEL_BDW_RSVDD_IDS(info)
++#define INTEL_BDW_GT12_IDS(info) \
++ INTEL_VGA_DEVICE(0x1602, info), /* GT1 ULT */ \
++ INTEL_VGA_DEVICE(0x1606, info), /* GT1 ULT */ \
++ INTEL_VGA_DEVICE(0x160B, info), /* GT1 Iris */ \
++ INTEL_VGA_DEVICE(0x160E, info), /* GT1 ULX */ \
++ INTEL_VGA_DEVICE(0x1612, info), /* GT2 Halo */ \
++ INTEL_VGA_DEVICE(0x1616, info), /* GT2 ULT */ \
++ INTEL_VGA_DEVICE(0x161B, info), /* GT2 ULT */ \
++ INTEL_VGA_DEVICE(0x161E, info), /* GT2 ULX */ \
++ INTEL_VGA_DEVICE(0x160A, info), /* GT1 Server */ \
++ INTEL_VGA_DEVICE(0x160D, info), /* GT1 Workstation */ \
++ INTEL_VGA_DEVICE(0x161A, info), /* GT2 Server */ \
++ INTEL_VGA_DEVICE(0x161D, info) /* GT2 Workstation */
++
++#define INTEL_BDW_GT3_IDS(info) \
++ INTEL_VGA_DEVICE(0x1622, info), /* ULT */ \
++ INTEL_VGA_DEVICE(0x1626, info), /* ULT */ \
++ INTEL_VGA_DEVICE(0x162B, info), /* Iris */ \
++ INTEL_VGA_DEVICE(0x162E, info), /* ULX */\
++ INTEL_VGA_DEVICE(0x162A, info), /* Server */ \
++ INTEL_VGA_DEVICE(0x162D, info) /* Workstation */
++
++#define INTEL_BDW_RSVD_IDS(info) \
++ INTEL_VGA_DEVICE(0x1632, info), /* ULT */ \
++ INTEL_VGA_DEVICE(0x1636, info), /* ULT */ \
++ INTEL_VGA_DEVICE(0x163B, info), /* Iris */ \
++ INTEL_VGA_DEVICE(0x163E, info), /* ULX */ \
++ INTEL_VGA_DEVICE(0x163A, info), /* Server */ \
++ INTEL_VGA_DEVICE(0x163D, info) /* Workstation */
++
++#define INTEL_BDW_IDS(info) \
++ INTEL_BDW_GT12_IDS(info), \
++ INTEL_BDW_GT3_IDS(info), \
++ INTEL_BDW_RSVD_IDS(info)
+
+ #define INTEL_CHV_IDS(info) \
+ INTEL_VGA_DEVICE(0x22b0, info), \
+@@ -259,21 +245,85 @@
+ INTEL_VGA_DEVICE(0x22b2, info), \
+ INTEL_VGA_DEVICE(0x22b3, info)
+
+-#define INTEL_SKL_IDS(info) \
+- INTEL_VGA_DEVICE(0x1916, info), /* ULT GT2 */ \
++#define INTEL_SKL_GT1_IDS(info) \
+ INTEL_VGA_DEVICE(0x1906, info), /* ULT GT1 */ \
+- INTEL_VGA_DEVICE(0x1926, info), /* ULT GT3 */ \
+- INTEL_VGA_DEVICE(0x1921, info), /* ULT GT2F */ \
+ INTEL_VGA_DEVICE(0x190E, info), /* ULX GT1 */ \
++ INTEL_VGA_DEVICE(0x1902, info), /* DT GT1 */ \
++ INTEL_VGA_DEVICE(0x190B, info), /* Halo GT1 */ \
++ INTEL_VGA_DEVICE(0x190A, info) /* SRV GT1 */
++
++#define INTEL_SKL_GT2_IDS(info) \
++ INTEL_VGA_DEVICE(0x1916, info), /* ULT GT2 */ \
++ INTEL_VGA_DEVICE(0x1921, info), /* ULT GT2F */ \
+ INTEL_VGA_DEVICE(0x191E, info), /* ULX GT2 */ \
+ INTEL_VGA_DEVICE(0x1912, info), /* DT GT2 */ \
+- INTEL_VGA_DEVICE(0x1902, info), /* DT GT1 */ \
+ INTEL_VGA_DEVICE(0x191B, info), /* Halo GT2 */ \
+- INTEL_VGA_DEVICE(0x192B, info), /* Halo GT3 */ \
+- INTEL_VGA_DEVICE(0x190B, info), /* Halo GT1 */ \
+ INTEL_VGA_DEVICE(0x191A, info), /* SRV GT2 */ \
+- INTEL_VGA_DEVICE(0x192A, info), /* SRV GT3 */ \
+- INTEL_VGA_DEVICE(0x190A, info), /* SRV GT1 */ \
+ INTEL_VGA_DEVICE(0x191D, info) /* WKS GT2 */
+
++#define INTEL_SKL_GT3_IDS(info) \
++ INTEL_VGA_DEVICE(0x1923, info), /* ULT GT3 */ \
++ INTEL_VGA_DEVICE(0x1926, info), /* ULT GT3 */ \
++ INTEL_VGA_DEVICE(0x1927, info), /* ULT GT3 */ \
++ INTEL_VGA_DEVICE(0x192B, info), /* Halo GT3 */ \
++ INTEL_VGA_DEVICE(0x192D, info) /* SRV GT3 */
++
++#define INTEL_SKL_GT4_IDS(info) \
++ INTEL_VGA_DEVICE(0x1932, info), /* DT GT4 */ \
++ INTEL_VGA_DEVICE(0x193B, info), /* Halo GT4 */ \
++ INTEL_VGA_DEVICE(0x193D, info), /* WKS GT4 */ \
++ INTEL_VGA_DEVICE(0x192A, info), /* SRV GT4 */ \
++ INTEL_VGA_DEVICE(0x193A, info) /* SRV GT4e */
++
++#define INTEL_SKL_IDS(info) \
++ INTEL_SKL_GT1_IDS(info), \
++ INTEL_SKL_GT2_IDS(info), \
++ INTEL_SKL_GT3_IDS(info), \
++ INTEL_SKL_GT4_IDS(info)
++
++#define INTEL_BXT_IDS(info) \
++ INTEL_VGA_DEVICE(0x0A84, info), \
++ INTEL_VGA_DEVICE(0x1A84, info), \
++ INTEL_VGA_DEVICE(0x1A85, info), \
++ INTEL_VGA_DEVICE(0x5A84, info), /* APL HD Graphics 505 */ \
++ INTEL_VGA_DEVICE(0x5A85, info) /* APL HD Graphics 500 */
++
++#define INTEL_GLK_IDS(info) \
++ INTEL_VGA_DEVICE(0x3184, info), \
++ INTEL_VGA_DEVICE(0x3185, info)
++
++#define INTEL_KBL_GT1_IDS(info) \
++ INTEL_VGA_DEVICE(0x5913, info), /* ULT GT1.5 */ \
++ INTEL_VGA_DEVICE(0x5915, info), /* ULX GT1.5 */ \
++ INTEL_VGA_DEVICE(0x5917, info), /* DT GT1.5 */ \
++ INTEL_VGA_DEVICE(0x5906, info), /* ULT GT1 */ \
++ INTEL_VGA_DEVICE(0x590E, info), /* ULX GT1 */ \
++ INTEL_VGA_DEVICE(0x5902, info), /* DT GT1 */ \
++ INTEL_VGA_DEVICE(0x5908, info), /* Halo GT1 */ \
++ INTEL_VGA_DEVICE(0x590B, info), /* Halo GT1 */ \
++ INTEL_VGA_DEVICE(0x590A, info) /* SRV GT1 */
++
++#define INTEL_KBL_GT2_IDS(info) \
++ INTEL_VGA_DEVICE(0x5916, info), /* ULT GT2 */ \
++ INTEL_VGA_DEVICE(0x5921, info), /* ULT GT2F */ \
++ INTEL_VGA_DEVICE(0x591E, info), /* ULX GT2 */ \
++ INTEL_VGA_DEVICE(0x5912, info), /* DT GT2 */ \
++ INTEL_VGA_DEVICE(0x591B, info), /* Halo GT2 */ \
++ INTEL_VGA_DEVICE(0x591A, info), /* SRV GT2 */ \
++ INTEL_VGA_DEVICE(0x591D, info) /* WKS GT2 */
++
++#define INTEL_KBL_GT3_IDS(info) \
++ INTEL_VGA_DEVICE(0x5923, info), /* ULT GT3 */ \
++ INTEL_VGA_DEVICE(0x5926, info), /* ULT GT3 */ \
++ INTEL_VGA_DEVICE(0x5927, info) /* ULT GT3 */
++
++#define INTEL_KBL_GT4_IDS(info) \
++ INTEL_VGA_DEVICE(0x593B, info) /* Halo GT4 */
++
++#define INTEL_KBL_IDS(info) \
++ INTEL_KBL_GT1_IDS(info), \
++ INTEL_KBL_GT2_IDS(info), \
++ INTEL_KBL_GT3_IDS(info), \
++ INTEL_KBL_GT4_IDS(info)
++
+ #endif /* _I915_PCIIDS_H */
+diff --git a/src/intel_device.c b/src/intel_device.c
+index 140e1536..c4910cd8 100644
+--- a/src/intel_device.c
++++ b/src/intel_device.c
+@@ -38,6 +38,12 @@
+ #include <dirent.h>
+ #include <errno.h>
+
++#if MAJOR_IN_MKDEV
++#include <sys/mkdev.h>
++#elif MAJOR_IN_SYSMACROS
++#include <sys/sysmacros.h>
++#endif
++
+ #include <pciaccess.h>
+
+ #include <xorg-server.h>
+@@ -197,9 +203,15 @@ static inline struct intel_device *intel_device(ScrnInfoPtr scrn)
+ return xf86GetEntityPrivate(scrn->entityList[0], intel_device_key)->ptr;
+ }
+
++static const char *kernel_module_names[] ={
++ "i915",
++ NULL,
++};
++
+ static int is_i915_device(int fd)
+ {
+ drm_version_t version;
++ const char **kn;
+ char name[5] = "";
+
+ memset(&version, 0, sizeof(version));
+@@ -209,7 +221,22 @@ static int is_i915_device(int fd)
+ if (drmIoctl(fd, DRM_IOCTL_VERSION, &version))
+ return 0;
+
+- return strcmp("i915", name) == 0;
++ for (kn = kernel_module_names; *kn; kn++)
++ if (strcmp(*kn, name) == 0)
++ return 1;
++
++ return 0;
++}
++
++static int load_i915_kernel_module(void)
++{
++ const char **kn;
++
++ for (kn = kernel_module_names; *kn; kn++)
++ if (xf86LoadKernelModule(*kn))
++ return 0;
++
++ return -1;
+ }
+
+ static int is_i915_gem(int fd)
+@@ -336,7 +363,7 @@ static int __intel_open_device__pci(const struct pci_device *pci)
+
+ sprintf(path + base, "driver");
+ if (stat(path, &st)) {
+- if (xf86LoadKernelModule("i915"))
++ if (load_i915_kernel_module())
+ return -1;
+ (void)xf86LoadKernelModule("fbcon");
+ }
+@@ -399,7 +426,7 @@ static int __intel_open_device__legacy(const struct pci_device *pci)
+
+ ret = drmCheckModesettingSupported(id);
+ if (ret) {
+- if (xf86LoadKernelModule("i915"))
++ if (load_i915_kernel_module() == 0)
+ ret = drmCheckModesettingSupported(id);
+ if (ret)
+ return -1;
+@@ -461,9 +488,9 @@ static int is_render_node(int fd, struct stat *st)
+
+ static char *find_render_node(int fd)
+ {
+-#if defined(USE_RENDERNODE)
+ struct stat master, render;
+ char buf[128];
++ int i;
+
+ /* Are we a render-node ourselves? */
+ if (is_render_node(fd, &master))
+@@ -472,9 +499,17 @@ static char *find_render_node(int fd)
+ sprintf(buf, "/dev/dri/renderD%d", (int)((master.st_rdev | 0x80) & 0xbf));
+ if (stat(buf, &render) == 0 &&
+ master.st_mode == render.st_mode &&
+- render.st_rdev == ((master.st_rdev | 0x80) & 0xbf))
++ render.st_rdev == (master.st_rdev | 0x80))
+ return strdup(buf);
+-#endif
++
++ /* Misaligned card <-> renderD, do a full search */
++ for (i = 0; i < 16; i++) {
++ sprintf(buf, "/dev/dri/renderD%d", i + 128);
++ if (stat(buf, &render) == 0 &&
++ master.st_mode == render.st_mode &&
++ render.st_rdev == (master.st_rdev | 0x80))
++ return strdup(buf);
++ }
+
+ return NULL;
+ }
+@@ -608,6 +643,27 @@ err_path:
+ return -1;
+ }
+
++void intel_close_device(int entity_num)
++{
++ struct intel_device *dev;
++
++ if (intel_device_key == -1)
++ return;
++
++ dev = xf86GetEntityPrivate(entity_num, intel_device_key)->ptr;
++ xf86GetEntityPrivate(entity_num, intel_device_key)->ptr = NULL;
++ if (!dev)
++ return;
++
++ if (dev->master_count == 0) /* Don't close server-fds */
++ close(dev->fd);
++
++ if (dev->render_node != dev->master_node)
++ free(dev->render_node);
++ free(dev->master_node);
++ free(dev);
++}
++
+ int __intel_peek_fd(ScrnInfoPtr scrn)
+ {
+ struct intel_device *dev;
+@@ -672,6 +728,12 @@ struct intel_device *intel_get_device(ScrnInfoPtr scrn, int *fd)
+ return dev;
+ }
+
++const char *intel_get_master_name(struct intel_device *dev)
++{
++ assert(dev && dev->master_node);
++ return dev->master_node;
++}
++
+ const char *intel_get_client_name(struct intel_device *dev)
+ {
+ assert(dev && dev->render_node);
+diff --git a/src/intel_driver.h b/src/intel_driver.h
+index 28ed1a0e..bece88a0 100644
+--- a/src/intel_driver.h
++++ b/src/intel_driver.h
+@@ -124,9 +124,11 @@ int intel_entity_get_devid(int index);
+ int intel_open_device(int entity_num,
+ const struct pci_device *pci,
+ struct xf86_platform_device *dev);
++void intel_close_device(int entity_num);
+ int __intel_peek_fd(ScrnInfoPtr scrn);
+ struct intel_device *intel_get_device(ScrnInfoPtr scrn, int *fd);
+ int intel_has_render_node(struct intel_device *dev);
++const char *intel_get_master_name(struct intel_device *dev);
+ const char *intel_get_client_name(struct intel_device *dev);
+ int intel_get_client_fd(struct intel_device *dev);
+ int intel_get_device_id(struct intel_device *dev);
+diff --git a/src/intel_list.h b/src/intel_list.h
+index 51af825d..c8a3187a 100644
+--- a/src/intel_list.h
++++ b/src/intel_list.h
+@@ -306,8 +306,7 @@ list_is_empty(const struct list *head)
+ list_entry((ptr)->prev, type, member)
+
+ #define __container_of(ptr, sample, member) \
+- (void *)((char *)(ptr) \
+- - ((char *)&(sample)->member - (char *)(sample)))
++ (void *)((char *)(ptr) - ((char *)&(sample)->member - (char *)(sample)))
+ /**
+ * Loop through the list given by head and set pos to struct in the list.
+ *
+@@ -392,17 +391,50 @@ static inline void list_move_tail(struct list *list, struct list *head)
+ #define list_last_entry(ptr, type, member) \
+ list_entry((ptr)->prev, type, member)
+
+-#define list_for_each_entry_reverse(pos, head, member) \
++#define list_for_each_entry_reverse(pos, head, member) \
+ for (pos = __container_of((head)->prev, pos, member); \
+ &pos->member != (head); \
+ pos = __container_of(pos->member.prev, pos, member))
+
+ #endif
+
++#define list_for_each_entry_safe_from(pos, tmp, head, member) \
++ for (tmp = __container_of(pos->member.next, pos, member); \
++ &pos->member != (head); \
++ pos = tmp, tmp = __container_of(tmp->member.next, tmp, member))
++
+ #undef container_of
+ #define container_of(ptr, type, member) \
+ ((type *)((char *)(ptr) - (char *) &((type *)0)->member))
+
++static inline void __list_splice(const struct list *list,
++ struct list *prev,
++ struct list *next)
++{
++ struct list *first = list->next;
++ struct list *last = list->prev;
++
++ first->prev = prev;
++ prev->next = first;
++
++ last->next = next;
++ next->prev = last;
++}
++
++static inline void list_splice(const struct list *list,
++ struct list *head)
++{
++ if (!list_is_empty(list))
++ __list_splice(list, head, head->next);
++}
++
++static inline void list_splice_tail(const struct list *list,
++ struct list *head)
++{
++ if (!list_is_empty(list))
++ __list_splice(list, head->prev, head);
++}
++
+ static inline int list_is_singular(const struct list *list)
+ {
+ return list->next == list->prev;
+diff --git a/src/intel_module.c b/src/intel_module.c
+index 102d52aa..2e97b5ea 100644
+--- a/src/intel_module.c
++++ b/src/intel_module.c
+@@ -126,6 +126,17 @@ static const struct intel_device_info intel_skylake_info = {
+ .gen = 0110,
+ };
+
++static const struct intel_device_info intel_broxton_info = {
++ .gen = 0111,
++};
++
++static const struct intel_device_info intel_kabylake_info = {
++ .gen = 0112,
++};
++
++static const struct intel_device_info intel_geminilake_info = {
++ .gen = 0113,
++};
+
+ static const SymTabRec intel_chipsets[] = {
+ {PCI_CHIP_I810, "i810"},
+@@ -234,30 +245,63 @@ static const SymTabRec intel_chipsets[] = {
+ {0x0157, "HD Graphics"},
+
+ /* Broadwell Marketing names */
+- {0x1602, "HD graphics"},
+- {0x1606, "HD graphics"},
+- {0x160B, "HD graphics"},
+- {0x160A, "HD graphics"},
+- {0x160D, "HD graphics"},
+- {0x160E, "HD graphics"},
+- {0x1612, "HD graphics 5600"},
+- {0x1616, "HD graphics 5500"},
+- {0x161B, "HD graphics"},
+- {0x161A, "HD graphics"},
+- {0x161D, "HD graphics"},
+- {0x161E, "HD graphics 5300"},
+- {0x1622, "Iris Pro graphics 6200"},
+- {0x1626, "HD graphics 6000"},
+- {0x162B, "Iris graphics 6100"},
+- {0x162A, "Iris Pro graphics P6300"},
+- {0x162D, "HD graphics"},
+- {0x162E, "HD graphics"},
+- {0x1632, "HD graphics"},
+- {0x1636, "HD graphics"},
+- {0x163B, "HD graphics"},
+- {0x163A, "HD graphics"},
+- {0x163D, "HD graphics"},
+- {0x163E, "HD graphics"},
++ {0x1602, "HD Graphics"},
++ {0x1606, "HD Graphics"},
++ {0x160B, "HD Graphics"},
++ {0x160A, "HD Graphics"},
++ {0x160D, "HD Graphics"},
++ {0x160E, "HD Graphics"},
++ {0x1612, "HD Graphics 5600"},
++ {0x1616, "HD Graphics 5500"},
++ {0x161B, "HD Graphics"},
++ {0x161A, "HD Graphics"},
++ {0x161D, "HD Graphics"},
++ {0x161E, "HD Graphics 5300"},
++ {0x1622, "Iris Pro Graphics 6200"},
++ {0x1626, "HD Graphics 6000"},
++ {0x162B, "Iris Graphics 6100"},
++ {0x162A, "Iris Pro Graphics P6300"},
++ {0x162D, "HD Graphics"},
++ {0x162E, "HD Graphics"},
++ {0x1632, "HD Graphics"},
++ {0x1636, "HD Graphics"},
++ {0x163B, "HD Graphics"},
++ {0x163A, "HD Graphics"},
++ {0x163D, "HD Graphics"},
++ {0x163E, "HD Graphics"},
++
++ /* Cherryview (Cherrytrail/Braswell) */
++ {0x22b0, "HD Graphics"},
++ {0x22b1, "HD Graphics"},
++ {0x22b2, "HD Graphics"},
++ {0x22b3, "HD Graphics"},
++
++ /* Skylake */
++ {0x1902, "HD Graphics 510"},
++ {0x1906, "HD Graphics 510"},
++ {0x190B, "HD Graphics 510"},
++ {0x1912, "HD Graphics 530"},
++ {0x1916, "HD Graphics 520"},
++ {0x191B, "HD Graphics 530"},
++ {0x191D, "HD Graphics P530"},
++ {0x191E, "HD Graphics 515"},
++ {0x1921, "HD Graphics 520"},
++ {0x1926, "Iris Graphics 540"},
++ {0x1927, "Iris Graphics 550"},
++ {0x192B, "Iris Graphics 555"},
++ {0x192D, "Iris Graphics P555"},
++ {0x1932, "Iris Pro Graphics 580"},
++ {0x193A, "Iris Pro Graphics P580"},
++ {0x193B, "Iris Pro Graphics 580"},
++ {0x193D, "Iris Pro Graphics P580"},
++
++ /* Broxton (Apollolake) */
++ {0x5A84, "HD Graphics 505"},
++ {0x5A85, "HD Graphics 500"},
++
++ /* Kabylake */
++ {0x5916, "HD Graphics 620"},
++ {0x591E, "HD Graphics 615"},
+
+ /* When adding new identifiers, also update:
+ * 1. intel_identify()
+@@ -305,18 +349,14 @@ static const struct pci_id_match intel_device_match[] = {
+ INTEL_IVB_D_IDS(&intel_ivybridge_info),
+ INTEL_IVB_M_IDS(&intel_ivybridge_info),
+
+- INTEL_HSW_D_IDS(&intel_haswell_info),
+- INTEL_HSW_M_IDS(&intel_haswell_info),
+-
+- INTEL_VLV_D_IDS(&intel_valleyview_info),
+- INTEL_VLV_M_IDS(&intel_valleyview_info),
+-
+- INTEL_BDW_D_IDS(&intel_broadwell_info),
+- INTEL_BDW_M_IDS(&intel_broadwell_info),
+-
++ INTEL_HSW_IDS(&intel_haswell_info),
++ INTEL_VLV_IDS(&intel_valleyview_info),
++ INTEL_BDW_IDS(&intel_broadwell_info),
+ INTEL_CHV_IDS(&intel_cherryview_info),
+-
+ INTEL_SKL_IDS(&intel_skylake_info),
++ INTEL_BXT_IDS(&intel_broxton_info),
++ INTEL_KBL_IDS(&intel_kabylake_info),
++ INTEL_GLK_IDS(&intel_geminilake_info),
+
+ INTEL_VGA_DEVICE(PCI_MATCH_ANY, &intel_generic_info),
+ #endif
+@@ -448,9 +488,9 @@ static void intel_identify(int flags)
+ if (unique != stack)
+ free(unique);
+
+- xf86Msg(X_INFO, INTEL_NAME ": Driver for Intel(R) HD Graphics: 2000-6000\n");
+- xf86Msg(X_INFO, INTEL_NAME ": Driver for Intel(R) Iris(TM) Graphics: 5100, 6100\n");
+- xf86Msg(X_INFO, INTEL_NAME ": Driver for Intel(R) Iris(TM) Pro Graphics: 5200, 6200, P6300\n");
++ xf86Msg(X_INFO, INTEL_NAME ": Driver for Intel(R) HD Graphics\n");
++ xf86Msg(X_INFO, INTEL_NAME ": Driver for Intel(R) Iris(TM) Graphics\n");
++ xf86Msg(X_INFO, INTEL_NAME ": Driver for Intel(R) Iris(TM) Pro Graphics\n");
+ }
+
+ static Bool intel_driver_func(ScrnInfoPtr pScrn,
+@@ -508,6 +548,9 @@ static enum accel_method { NOACCEL, SNA, UXA } get_accel_method(void)
+ if (hosted())
+ return SNA;
+
++ if (xf86configptr == NULL) /* X -configure */
++ return SNA;
++
+ dev = _xf86findDriver("intel", xf86configptr->conf_device_lst);
+ if (dev && dev->dev_option_lst) {
+ const char *s;
+@@ -582,10 +625,17 @@ intel_scrn_create(DriverPtr driver,
+ case NOACCEL:
+ #endif
+ case UXA:
+- return intel_init_scrn(scrn);
++ return intel_init_scrn(scrn);
+ #endif
+
+- default: break;
++ default:
++#if USE_SNA
++ return sna_init_scrn(scrn, entity_num);
++#elif USE_UXA
++ return intel_init_scrn(scrn);
++#else
++ break;
++#endif
+ }
+ #endif
+
+@@ -604,6 +654,8 @@ static Bool intel_pci_probe(DriverPtr driver,
+ struct pci_device *pci,
+ intptr_t match_data)
+ {
++ Bool ret;
++
+ if (intel_open_device(entity_num, pci, NULL) == -1) {
+ #if UMS
+ switch (pci->device_id) {
+@@ -621,7 +673,11 @@ static Bool intel_pci_probe(DriverPtr driver,
+ #endif
+ }
+
+- return intel_scrn_create(driver, entity_num, match_data, 0);
++ ret = intel_scrn_create(driver, entity_num, match_data, 0);
++ if (!ret)
++ intel_close_device(entity_num);
++
++ return ret;
+ }
+
+ #ifdef XSERVER_PLATFORM_BUS
+@@ -644,9 +700,16 @@ intel_platform_probe(DriverPtr driver,
+
+ /* if we get any flags we don't understand fail to probe for now */
+ if (flags)
+- return FALSE;
++ goto err;
++
++ if (!intel_scrn_create(driver, entity_num, match_data, scrn_flags))
++ goto err;
+
+- return intel_scrn_create(driver, entity_num, match_data, scrn_flags);
++ return TRUE;
++
++err:
++ intel_close_device(entity_num);
++ return FALSE;
+ }
+ #endif
+
+diff --git a/src/intel_options.c b/src/intel_options.c
+index ff8541a4..7f253ac1 100644
+--- a/src/intel_options.c
++++ b/src/intel_options.c
+@@ -2,18 +2,24 @@
+ #include "config.h"
+ #endif
+
++#include <xorg-server.h>
++#include <xorgVersion.h>
++#include <xf86Parser.h>
++
+ #include "intel_options.h"
+
+ const OptionInfoRec intel_options[] = {
+- {OPTION_ACCEL_DISABLE, "NoAccel", OPTV_BOOLEAN, {0}, 0},
++ {OPTION_ACCEL_ENABLE, "Accel", OPTV_BOOLEAN, {0}, 0},
+ {OPTION_ACCEL_METHOD, "AccelMethod", OPTV_STRING, {0}, 0},
+ {OPTION_BACKLIGHT, "Backlight", OPTV_STRING, {0}, 0},
++ {OPTION_EDID, "CustomEDID", OPTV_STRING, {0}, 0},
+ {OPTION_DRI, "DRI", OPTV_STRING, {0}, 0},
+ {OPTION_PRESENT, "Present", OPTV_BOOLEAN, {0}, 1},
+ {OPTION_COLOR_KEY, "ColorKey", OPTV_INTEGER, {0}, 0},
+ {OPTION_VIDEO_KEY, "VideoKey", OPTV_INTEGER, {0}, 0},
+ {OPTION_TILING_2D, "Tiling", OPTV_BOOLEAN, {0}, 1},
+ {OPTION_TILING_FB, "LinearFramebuffer", OPTV_BOOLEAN, {0}, 0},
++ {OPTION_ROTATION, "HWRotation", OPTV_BOOLEAN, {0}, 1},
+ {OPTION_VSYNC, "VSync", OPTV_BOOLEAN, {0}, 1},
+ {OPTION_PAGEFLIP, "PageFlip", OPTV_BOOLEAN, {0}, 1},
+ {OPTION_SWAPBUFFERS_WAIT, "SwapbuffersWait", OPTV_BOOLEAN, {0}, 1},
+@@ -21,7 +27,6 @@ const OptionInfoRec intel_options[] = {
+ {OPTION_PREFER_OVERLAY, "XvPreferOverlay", OPTV_BOOLEAN, {0}, 0},
+ {OPTION_HOTPLUG, "HotPlug", OPTV_BOOLEAN, {0}, 1},
+ {OPTION_REPROBE, "ReprobeOutputs", OPTV_BOOLEAN, {0}, 0},
+- {OPTION_DELETE_DP12, "DeleteUnusedDP12Displays", OPTV_BOOLEAN, {0}, 0},
+ #ifdef INTEL_XVMC
+ {OPTION_XVMC, "XvMC", OPTV_BOOLEAN, {0}, 1},
+ #endif
+@@ -54,3 +59,85 @@ OptionInfoPtr intel_options_get(ScrnInfoPtr scrn)
+
+ return options;
+ }
++
++Bool intel_option_cast_to_bool(OptionInfoPtr options, int id, Bool val)
++{
++#if XORG_VERSION_CURRENT >= XORG_VERSION_NUMERIC(1,7,99,901,0)
++ xf86getBoolValue(&val, xf86GetOptValString(options, id));
++#endif
++ return val;
++}
++
++static int
++namecmp(const char *s1, const char *s2)
++{
++ char c1, c2;
++
++ if (!s1 || *s1 == 0) {
++ if (!s2 || *s2 == 0)
++ return 0;
++ else
++ return 1;
++ }
++
++ while (*s1 == '_' || *s1 == ' ' || *s1 == '\t')
++ s1++;
++
++ while (*s2 == '_' || *s2 == ' ' || *s2 == '\t')
++ s2++;
++
++ c1 = isupper(*s1) ? tolower(*s1) : *s1;
++ c2 = isupper(*s2) ? tolower(*s2) : *s2;
++ while (c1 == c2) {
++ if (c1 == '\0')
++ return 0;
++
++ s1++;
++ while (*s1 == '_' || *s1 == ' ' || *s1 == '\t')
++ s1++;
++
++ s2++;
++ while (*s2 == '_' || *s2 == ' ' || *s2 == '\t')
++ s2++;
++
++ c1 = isupper(*s1) ? tolower(*s1) : *s1;
++ c2 = isupper(*s2) ? tolower(*s2) : *s2;
++ }
++
++ return c1 - c2;
++}
++
++unsigned intel_option_cast_to_unsigned(OptionInfoPtr options, int id, unsigned val)
++{
++#if XORG_VERSION_CURRENT >= XORG_VERSION_NUMERIC(1,7,99,901,0)
++ const char *str = xf86GetOptValString(options, id);
++#else
++ const char *str = NULL;
++#endif
++ unsigned v;
++
++ if (str == NULL || *str == '\0')
++ return val;
++
++ if (namecmp(str, "on") == 0)
++ return val;
++ if (namecmp(str, "true") == 0)
++ return val;
++ if (namecmp(str, "yes") == 0)
++ return val;
++
++ if (namecmp(str, "0") == 0)
++ return 0;
++ if (namecmp(str, "off") == 0)
++ return 0;
++ if (namecmp(str, "false") == 0)
++ return 0;
++ if (namecmp(str, "no") == 0)
++ return 0;
++
++ v = atoi(str);
++ if (v)
++ return v;
++
++ return val;
++}
+diff --git a/src/intel_options.h b/src/intel_options.h
+index 7e2cbd9b..43635f1f 100644
+--- a/src/intel_options.h
++++ b/src/intel_options.h
+@@ -12,15 +12,17 @@
+ */
+
+ enum intel_options {
+- OPTION_ACCEL_DISABLE,
++ OPTION_ACCEL_ENABLE,
+ OPTION_ACCEL_METHOD,
+ OPTION_BACKLIGHT,
++ OPTION_EDID,
+ OPTION_DRI,
+ OPTION_PRESENT,
+ OPTION_VIDEO_KEY,
+ OPTION_COLOR_KEY,
+ OPTION_TILING_2D,
+ OPTION_TILING_FB,
++ OPTION_ROTATION,
+ OPTION_VSYNC,
+ OPTION_PAGEFLIP,
+ OPTION_SWAPBUFFERS_WAIT,
+@@ -28,7 +30,6 @@ enum intel_options {
+ OPTION_PREFER_OVERLAY,
+ OPTION_HOTPLUG,
+ OPTION_REPROBE,
+- OPTION_DELETE_DP12,
+ #if defined(XvMCExtension) && defined(ENABLE_XVMC)
+ OPTION_XVMC,
+ #define INTEL_XVMC 1
+@@ -51,5 +52,7 @@ enum intel_options {
+
+ extern const OptionInfoRec intel_options[];
+ OptionInfoPtr intel_options_get(ScrnInfoPtr scrn);
++unsigned intel_option_cast_to_unsigned(OptionInfoPtr, int id, unsigned val);
++Bool intel_option_cast_to_bool(OptionInfoPtr, int id, Bool val);
+
+ #endif /* INTEL_OPTIONS_H */
+diff --git a/src/legacy/i810/i810_common.h b/src/legacy/i810/i810_common.h
+index 4cc10e8b..8355708c 100644
+--- a/src/legacy/i810/i810_common.h
++++ b/src/legacy/i810/i810_common.h
+@@ -52,7 +52,7 @@
+
+ #define ALIGN(i,m) (((i) + (m) - 1) & ~((m) - 1))
+
+-/* Using usleep() makes things noticably slow. */
++/* Using usleep() makes things noticeably slow. */
+ #if 0
+ #define DELAY(x) usleep(x)
+ #else
+@@ -185,7 +185,7 @@ enum {
+ * - zbuffer linear offset and pitch -- also invarient
+ * - drawing origin in back and depth buffers.
+ *
+- * Keep the depth/back buffer state here to acommodate private buffers
++ * Keep the depth/back buffer state here to accommodate private buffers
+ * in the future.
+ */
+ #define I810_DESTREG_DI0 0 /* CMD_OP_DESTBUFFER_INFO (2 dwords) */
+diff --git a/src/legacy/i810/i810_hwmc.c b/src/legacy/i810/i810_hwmc.c
+index 7cb9c1ab..58661b0a 100644
+--- a/src/legacy/i810/i810_hwmc.c
++++ b/src/legacy/i810/i810_hwmc.c
+@@ -171,7 +171,7 @@ static XF86MCAdaptorPtr ppAdapt[1] =
+ *
+ * I810InitMC
+ *
+- * Initialize the hardware motion compenstation extention for this
++ * Initialize the hardware motion compensation extension for this
+ * hardware. The initialization routines want the address of the pointers
+ * to the structures, not the address of the structures. This means we
+ * allocate (or create static?) the pointer memory and pass that
+diff --git a/src/legacy/i810/i810_memory.c b/src/legacy/i810/i810_memory.c
+index c3de2777..6f274836 100644
+--- a/src/legacy/i810/i810_memory.c
++++ b/src/legacy/i810/i810_memory.c
+@@ -76,7 +76,7 @@ I810AllocateGARTMemory(ScrnInfoPtr pScrn)
+ unsigned long size = pScrn->videoRam * 1024UL;
+ I810Ptr pI810 = I810PTR(pScrn);
+ int key;
+- long tom = 0;
++ unsigned long tom = 0;
+ unsigned long physical;
+
+ if (!xf86AgpGARTSupported() || !xf86AcquireGART(pScrn->scrnIndex)) {
+@@ -132,8 +132,8 @@ I810AllocateGARTMemory(ScrnInfoPtr pScrn)
+ * Keep it 512K aligned for the sake of tiled regions.
+ */
+
+- tom += 0x7ffff;
+- tom &= ~0x7ffff;
++ tom += 0x7ffffUL;
++ tom &= ~0x7ffffUL;
+
+ if ((key = xf86AllocateGARTMemory(pScrn->scrnIndex, size, 1, NULL)) != -1) {
+ pI810->DcacheOffset = tom;
+diff --git a/src/legacy/i810/i810_reg.h b/src/legacy/i810/i810_reg.h
+index 54faeb3d..fa091c5b 100644
+--- a/src/legacy/i810/i810_reg.h
++++ b/src/legacy/i810/i810_reg.h
+@@ -245,7 +245,7 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ * not sure they refer to local (graphics) memory.
+ *
+ * These details are for the local memory control registers,
+- * (pp301-310). The test machines are not equiped with local memory,
++ * (pp301-310). The test machines are not equipped with local memory,
+ * so nothing is tested. Only a single row seems to be supported.
+ */
+ #define DRAM_ROW_TYPE 0x3000
+diff --git a/src/legacy/i810/i810_video.c b/src/legacy/i810/i810_video.c
+index be49b91d..af683c81 100644
+--- a/src/legacy/i810/i810_video.c
++++ b/src/legacy/i810/i810_video.c
+@@ -77,7 +77,11 @@ static int I810PutImage( ScrnInfoPtr,
+ static int I810QueryImageAttributes(ScrnInfoPtr,
+ int, unsigned short *, unsigned short *, int *, int *);
+
++#if !HAVE_NOTIFY_FD
+ static void I810BlockHandler(BLOCKHANDLER_ARGS_DECL);
++#else
++static void I810BlockHandler(void *data, void *_timeout);
++#endif
+
+ #define MAKE_ATOM(a) MakeAtom(a, sizeof(a) - 1, TRUE)
+
+@@ -418,8 +422,14 @@ I810SetupImageVideo(ScreenPtr screen)
+
+ pI810->adaptor = adapt;
+
++#if !HAVE_NOTIFY_FD
+ pI810->BlockHandler = screen->BlockHandler;
+ screen->BlockHandler = I810BlockHandler;
++#else
++ RegisterBlockAndWakeupHandlers(I810BlockHandler,
++ (ServerWakeupHandlerProcPtr)NoopDDA,
++ pScrn);
++#endif
+
+ xvBrightness = MAKE_ATOM("XV_BRIGHTNESS");
+ xvContrast = MAKE_ATOM("XV_CONTRAST");
+@@ -1135,6 +1145,7 @@ I810QueryImageAttributes(
+ return size;
+ }
+
++#if !HAVE_NOTIFY_FD
+ static void
+ I810BlockHandler (BLOCKHANDLER_ARGS_DECL)
+ {
+@@ -1172,6 +1183,38 @@ I810BlockHandler (BLOCKHANDLER_ARGS_DECL)
+ }
+ }
+ }
++#else
++static void
++I810BlockHandler(void *data, void *_timeout)
++{
++ ScrnInfoPtr pScrn = data;
++ I810Ptr pI810 = I810PTR(pScrn);
++ I810PortPrivPtr pPriv = GET_PORT_PRIVATE(pScrn);
++ I810OverlayRegPtr overlay = (I810OverlayRegPtr) (pI810->FbBase + pI810->OverlayStart);
++
++ if(pPriv->videoStatus & TIMER_MASK) {
++ UpdateCurrentTime();
++ if(pPriv->videoStatus & OFF_TIMER) {
++ if(pPriv->offTime < currentTime.milliseconds) {
++ /* Turn off the overlay */
++ overlay->OV0CMD &= 0xFFFFFFFE;
++ OVERLAY_UPDATE(pI810->OverlayPhysical);
++
++ pPriv->videoStatus = FREE_TIMER;
++ pPriv->freeTime = currentTime.milliseconds + FREE_DELAY;
++ }
++ } else { /* FREE_TIMER */
++ if(pPriv->freeTime < currentTime.milliseconds) {
++ if(pPriv->linear) {
++ xf86FreeOffscreenLinear(pPriv->linear);
++ pPriv->linear = NULL;
++ }
++ pPriv->videoStatus = 0;
++ }
++ }
++ }
++}
++#endif
+
+
+ /***************************************************************************
+@@ -1373,7 +1416,6 @@ I810DisplaySurface(
+ UpdateCurrentTime();
+ pI810Priv->videoStatus = FREE_TIMER;
+ pI810Priv->freeTime = currentTime.milliseconds + FREE_DELAY;
+- pScrn->pScreen->BlockHandler = I810BlockHandler;
+ }
+
+ return Success;
+diff --git a/src/legacy/i810/xvmc/I810XvMC.c b/src/legacy/i810/xvmc/I810XvMC.c
+index e6b63d30..a538e999 100644
+--- a/src/legacy/i810/xvmc/I810XvMC.c
++++ b/src/legacy/i810/xvmc/I810XvMC.c
+@@ -61,7 +61,7 @@ static int event_base;
+ // Arguments: pI810XvMC private data structure from the current context.
+ // Notes: We faked the drmMapBufs for the i810's security so now we have
+ // to insert an allocated page into the correct spot in the faked
+-// list to keep up appearences.
++// list to keep up appearances.
+ // Concept for this function was taken from Mesa sources.
+ // Returns: drmBufPtr containing the information about the allocated page.
+ ***************************************************************************/
+@@ -188,7 +188,7 @@ _X_EXPORT Status XvMCCreateContext(Display *display, XvPortID port,
+
+ /* Check for drm */
+ if(! drmAvailable()) {
+- printf("Direct Rendering is not avilable on this system!\n");
++ printf("Direct Rendering is not available on this system!\n");
+ return BadAlloc;
+ }
+
+@@ -3279,7 +3279,7 @@ _X_EXPORT Status XvMCSyncSurface(Display *display,XvMCSurface *surface) {
+ // display - Connection to X server
+ // surface - Surface to flush
+ // Info:
+-// This command is a noop for i810 becuase we always dispatch buffers in
++// This command is a noop for i810 because we always dispatch buffers in
+ // render. There is little gain to be had with 4k buffers.
+ // Returns: Status
+ ***************************************************************************/
+diff --git a/src/render_program/exa_wm.g4i b/src/render_program/exa_wm.g4i
+index 5d3d45b1..587b581c 100644
+--- a/src/render_program/exa_wm.g4i
++++ b/src/render_program/exa_wm.g4i
+@@ -57,7 +57,7 @@ define(`mask_dw_dy', `g6.4<0,1,0>F')
+ define(`mask_wo', `g6.12<0,1,0>F')
+
+ /*
+- * Local variables. Pairs must be aligned on even reg boundry
++ * Local variables. Pairs must be aligned on even reg boundary
+ */
+
+ /* this holds the X dest coordinates */
+diff --git a/src/render_program/exa_wm_yuv_rgb.g8a b/src/render_program/exa_wm_yuv_rgb.g8a
+index 7def0930..34973ba8 100644
+--- a/src/render_program/exa_wm_yuv_rgb.g8a
++++ b/src/render_program/exa_wm_yuv_rgb.g8a
+@@ -76,7 +76,7 @@ add (16) Cbn<1>F Cb<8,8,1>F -0.501961F { compr align1 };
+ /*
+ * R = Y + Cr * 1.596
+ */
+-mov (8) acc0<1>F Yn<8,8,1>F { compr align1 };
++mov (8) acc0<1>F Yn_01<8,8,1>F { compr align1 };
+ mac.sat(8) src_sample_r_01<1>F Crn_01<8,8,1>F 1.596F { compr align1 };
+
+ mov (8) acc0<1>F Yn_23<8,8,1>F { compr align1 };
+@@ -84,7 +84,7 @@ mac.sat(8) src_sample_r_23<1>F Crn_23<8,8,1>F 1.596F { compr align1 };
+ /*
+ * G = Crn * -0.813 + Cbn * -0.392 + Y
+ */
+-mov (8) acc0<1>F Yn_23<8,8,1>F { compr align1 };
++mov (8) acc0<1>F Yn_01<8,8,1>F { compr align1 };
+ mac (8) acc0<1>F Crn_01<8,8,1>F -0.813F { compr align1 };
+ mac.sat(8) src_sample_g_01<1>F Cbn_01<8,8,1>F -0.392F { compr align1 };
+
+diff --git a/src/render_program/exa_wm_yuv_rgb.g8b b/src/render_program/exa_wm_yuv_rgb.g8b
+index 44949538..2cd6fc44 100644
+--- a/src/render_program/exa_wm_yuv_rgb.g8b
++++ b/src/render_program/exa_wm_yuv_rgb.g8b
+@@ -6,7 +6,7 @@
+ { 0x80600048, 0x21c03ae8, 0x3e8d02c0, 0x3fcc49ba },
+ { 0x00600001, 0x24003ae0, 0x008d0320, 0x00000000 },
+ { 0x80600048, 0x21e03ae8, 0x3e8d02e0, 0x3fcc49ba },
+- { 0x00600001, 0x24003ae0, 0x008d0320, 0x00000000 },
++ { 0x00600001, 0x24003ae0, 0x008d0300, 0x00000000 },
+ { 0x00600048, 0x24003ae0, 0x3e8d02c0, 0xbf5020c5 },
+ { 0x80600048, 0x22003ae8, 0x3e8d0340, 0xbec8b439 },
+ { 0x00600001, 0x24003ae0, 0x008d0320, 0x00000000 },
+diff --git a/src/sna/Makefile.am b/src/sna/Makefile.am
+index e09a8d49..adf13963 100644
+--- a/src/sna/Makefile.am
++++ b/src/sna/Makefile.am
+@@ -107,6 +107,8 @@ libsna_la_SOURCES = \
+ gen8_render.h \
+ gen8_vertex.c \
+ gen8_vertex.h \
++ gen9_render.c \
++ gen9_render.h \
+ xassert.h \
+ $(NULL)
+
+diff --git a/src/sna/blt.c b/src/sna/blt.c
+index b5bfee69..cb90437a 100644
+--- a/src/sna/blt.c
++++ b/src/sna/blt.c
+@@ -30,112 +30,608 @@
+ #endif
+
+ #include "sna.h"
++#include <pixman.h>
+
+-#if __x86_64__
+-#define USE_SSE2 1
+-#endif
+-
+-#if USE_SSE2
++#if defined(sse2)
++#pragma GCC push_options
++#pragma GCC target("sse2,inline-all-stringops,fpmath=sse")
++#pragma GCC optimize("Ofast")
+ #include <xmmintrin.h>
+
+ #if __x86_64__
+ #define have_sse2() 1
+ #else
+-enum {
+- MMX = 0x1,
+- MMX_EXTENSIONS = 0x2,
+- SSE = 0x6,
+- SSE2 = 0x8,
+- CMOV = 0x10
+-};
+-
+-#ifdef __GNUC__
+-static unsigned int
+-detect_cpu_features(void)
+-{
+- unsigned int features;
+- unsigned int result = 0;
+-
+- char vendor[13];
+- vendor[0] = 0;
+- vendor[12] = 0;
+-
+- asm (
+- "pushf\n"
+- "pop %%eax\n"
+- "mov %%eax, %%ecx\n"
+- "xor $0x00200000, %%eax\n"
+- "push %%eax\n"
+- "popf\n"
+- "pushf\n"
+- "pop %%eax\n"
+- "mov $0x0, %%edx\n"
+- "xor %%ecx, %%eax\n"
+- "jz 1f\n"
+-
+- "mov $0x00000000, %%eax\n"
+- "push %%ebx\n"
+- "cpuid\n"
+- "mov %%ebx, %%eax\n"
+- "pop %%ebx\n"
+- "mov %%eax, %1\n"
+- "mov %%edx, %2\n"
+- "mov %%ecx, %3\n"
+- "mov $0x00000001, %%eax\n"
+- "push %%ebx\n"
+- "cpuid\n"
+- "pop %%ebx\n"
+- "1:\n"
+- "mov %%edx, %0\n"
+- : "=r" (result), "=m" (vendor[0]), "=m" (vendor[4]), "=m" (vendor[8])
+- :: "%eax", "%ecx", "%edx");
+-
+- features = 0;
+- if (result) {
+- /* result now contains the standard feature bits */
+- if (result & (1 << 15))
+- features |= CMOV;
+- if (result & (1 << 23))
+- features |= MMX;
+- if (result & (1 << 25))
+- features |= SSE;
+- if (result & (1 << 26))
+- features |= SSE2;
+- }
+- return features;
+-}
+-#else
+-static unsigned int detect_cpu_features(void) { return 0; }
+-#endif
+-
+ static bool have_sse2(void)
+ {
+ static int sse2_present = -1;
+
+ if (sse2_present == -1)
+- sse2_present = detect_cpu_features() & SSE2;
++ sse2_present = sna_cpu_detect() & SSE2;
+
+ return sse2_present;
+ }
+ #endif
+
+-static inline __m128i
++static force_inline __m128i
+ xmm_create_mask_32(uint32_t mask)
+ {
+ return _mm_set_epi32(mask, mask, mask, mask);
+ }
+
+-static inline __m128i
++static force_inline __m128i
++xmm_load_128(const __m128i *src)
++{
++ return _mm_load_si128(src);
++}
++
++static force_inline __m128i
+ xmm_load_128u(const __m128i *src)
+ {
+ return _mm_loadu_si128(src);
+ }
+
+-static inline void
++static force_inline void
+ xmm_save_128(__m128i *dst, __m128i data)
+ {
+ _mm_store_si128(dst, data);
+ }
++
++static force_inline void
++xmm_save_128u(__m128i *dst, __m128i data)
++{
++ _mm_storeu_si128(dst, data);
++}
++
++static force_inline void
++to_sse128xN(uint8_t *dst, const uint8_t *src, int bytes)
++{
++ int i;
++
++ for (i = 0; i < bytes / 128; i++) {
++ __m128i xmm0, xmm1, xmm2, xmm3;
++ __m128i xmm4, xmm5, xmm6, xmm7;
++
++ xmm0 = xmm_load_128u((const __m128i*)src + 0);
++ xmm1 = xmm_load_128u((const __m128i*)src + 1);
++ xmm2 = xmm_load_128u((const __m128i*)src + 2);
++ xmm3 = xmm_load_128u((const __m128i*)src + 3);
++ xmm4 = xmm_load_128u((const __m128i*)src + 4);
++ xmm5 = xmm_load_128u((const __m128i*)src + 5);
++ xmm6 = xmm_load_128u((const __m128i*)src + 6);
++ xmm7 = xmm_load_128u((const __m128i*)src + 7);
++
++ xmm_save_128((__m128i*)dst + 0, xmm0);
++ xmm_save_128((__m128i*)dst + 1, xmm1);
++ xmm_save_128((__m128i*)dst + 2, xmm2);
++ xmm_save_128((__m128i*)dst + 3, xmm3);
++ xmm_save_128((__m128i*)dst + 4, xmm4);
++ xmm_save_128((__m128i*)dst + 5, xmm5);
++ xmm_save_128((__m128i*)dst + 6, xmm6);
++ xmm_save_128((__m128i*)dst + 7, xmm7);
++
++ dst += 128;
++ src += 128;
++ }
++}
++
++static force_inline void
++to_sse64(uint8_t *dst, const uint8_t *src)
++{
++ __m128i xmm1, xmm2, xmm3, xmm4;
++
++ xmm1 = xmm_load_128u((const __m128i*)src + 0);
++ xmm2 = xmm_load_128u((const __m128i*)src + 1);
++ xmm3 = xmm_load_128u((const __m128i*)src + 2);
++ xmm4 = xmm_load_128u((const __m128i*)src + 3);
++
++ xmm_save_128((__m128i*)dst + 0, xmm1);
++ xmm_save_128((__m128i*)dst + 1, xmm2);
++ xmm_save_128((__m128i*)dst + 2, xmm3);
++ xmm_save_128((__m128i*)dst + 3, xmm4);
++}
++
++static force_inline void
++to_sse32(uint8_t *dst, const uint8_t *src)
++{
++ __m128i xmm1, xmm2;
++
++ xmm1 = xmm_load_128u((const __m128i*)src + 0);
++ xmm2 = xmm_load_128u((const __m128i*)src + 1);
++
++ xmm_save_128((__m128i*)dst + 0, xmm1);
++ xmm_save_128((__m128i*)dst + 1, xmm2);
++}
++
++static force_inline void
++to_sse16(uint8_t *dst, const uint8_t *src)
++{
++ xmm_save_128((__m128i*)dst, xmm_load_128u((const __m128i*)src));
++}
++
++static void to_memcpy(uint8_t *dst, const uint8_t *src, unsigned len)
++{
++ assert(len);
++ if ((uintptr_t)dst & 15) {
++ if (len <= 16 - ((uintptr_t)dst & 15)) {
++ memcpy(dst, src, len);
++ return;
++ }
++
++ if ((uintptr_t)dst & 1) {
++ assert(len >= 1);
++ *dst++ = *src++;
++ len--;
++ }
++ if ((uintptr_t)dst & 2) {
++ assert(((uintptr_t)dst & 1) == 0);
++ assert(len >= 2);
++ *(uint16_t *)dst = *(const uint16_t *)src;
++ dst += 2;
++ src += 2;
++ len -= 2;
++ }
++ if ((uintptr_t)dst & 4) {
++ assert(((uintptr_t)dst & 3) == 0);
++ assert(len >= 4);
++ *(uint32_t *)dst = *(const uint32_t *)src;
++ dst += 4;
++ src += 4;
++ len -= 4;
++ }
++ if ((uintptr_t)dst & 8) {
++ assert(((uintptr_t)dst & 7) == 0);
++ assert(len >= 8);
++ *(uint64_t *)dst = *(const uint64_t *)src;
++ dst += 8;
++ src += 8;
++ len -= 8;
++ }
++ }
++
++ assert(((uintptr_t)dst & 15) == 0);
++ while (len >= 64) {
++ to_sse64(dst, src);
++ dst += 64;
++ src += 64;
++ len -= 64;
++ }
++ if (len == 0)
++ return;
++
++ if (len & 32) {
++ to_sse32(dst, src);
++ dst += 32;
++ src += 32;
++ }
++ if (len & 16) {
++ to_sse16(dst, src);
++ dst += 16;
++ src += 16;
++ }
++ if (len & 8) {
++ *(uint64_t *)dst = *(uint64_t *)src;
++ dst += 8;
++ src += 8;
++ }
++ if (len & 4) {
++ *(uint32_t *)dst = *(uint32_t *)src;
++ dst += 4;
++ src += 4;
++ }
++ memcpy(dst, src, len & 3);
++}
++
++static void
++memcpy_to_tiled_x__swizzle_0__sse2(const void *src, void *dst, int bpp,
++ int32_t src_stride, int32_t dst_stride,
++ int16_t src_x, int16_t src_y,
++ int16_t dst_x, int16_t dst_y,
++ uint16_t width, uint16_t height)
++{
++ const unsigned tile_width = 512;
++ const unsigned tile_height = 8;
++ const unsigned tile_size = 4096;
++
++ const unsigned cpp = bpp / 8;
++ const unsigned tile_pixels = tile_width / cpp;
++ const unsigned tile_shift = ffs(tile_pixels) - 1;
++ const unsigned tile_mask = tile_pixels - 1;
++
++ unsigned offset_x, length_x;
++
++ DBG(("%s(bpp=%d): src=(%d, %d), dst=(%d, %d), size=%dx%d, pitch=%d/%d\n",
++ __FUNCTION__, bpp, src_x, src_y, dst_x, dst_y, width, height, src_stride, dst_stride));
++ assert(src != dst);
++
++ if (src_x | src_y)
++ src = (const uint8_t *)src + src_y * src_stride + src_x * cpp;
++ width *= cpp;
++ assert(src_stride >= width);
++
++ if (dst_x & tile_mask) {
++ offset_x = (dst_x & tile_mask) * cpp;
++ length_x = min(tile_width - offset_x, width);
++ } else
++ length_x = 0;
++ dst = (uint8_t *)dst + (dst_x >> tile_shift) * tile_size;
++
++ while (height--) {
++ unsigned w = width;
++ const uint8_t *src_row = src;
++ uint8_t *tile_row = dst;
++
++ src = (const uint8_t *)src + src_stride;
++
++ tile_row += dst_y / tile_height * dst_stride * tile_height;
++ tile_row += (dst_y & (tile_height-1)) * tile_width;
++ dst_y++;
++
++ if (length_x) {
++ to_memcpy(tile_row + offset_x, src_row, length_x);
++
++ tile_row += tile_size;
++ src_row = (const uint8_t *)src_row + length_x;
++ w -= length_x;
++ }
++ while (w >= tile_width) {
++ assert(((uintptr_t)tile_row & (tile_width - 1)) == 0);
++ to_sse128xN(assume_aligned(tile_row, tile_width),
++ src_row, tile_width);
++ tile_row += tile_size;
++ src_row = (const uint8_t *)src_row + tile_width;
++ w -= tile_width;
++ }
++ if (w) {
++ assert(((uintptr_t)tile_row & (tile_width - 1)) == 0);
++ to_memcpy(assume_aligned(tile_row, tile_width),
++ src_row, w);
++ }
++ }
++}
++
++static force_inline void
++from_sse128xNu(uint8_t *dst, const uint8_t *src, int bytes)
++{
++ int i;
++
++ assert(((uintptr_t)src & 15) == 0);
++
++ for (i = 0; i < bytes / 128; i++) {
++ __m128i xmm0, xmm1, xmm2, xmm3;
++ __m128i xmm4, xmm5, xmm6, xmm7;
++
++ xmm0 = xmm_load_128((const __m128i*)src + 0);
++ xmm1 = xmm_load_128((const __m128i*)src + 1);
++ xmm2 = xmm_load_128((const __m128i*)src + 2);
++ xmm3 = xmm_load_128((const __m128i*)src + 3);
++ xmm4 = xmm_load_128((const __m128i*)src + 4);
++ xmm5 = xmm_load_128((const __m128i*)src + 5);
++ xmm6 = xmm_load_128((const __m128i*)src + 6);
++ xmm7 = xmm_load_128((const __m128i*)src + 7);
++
++ xmm_save_128u((__m128i*)dst + 0, xmm0);
++ xmm_save_128u((__m128i*)dst + 1, xmm1);
++ xmm_save_128u((__m128i*)dst + 2, xmm2);
++ xmm_save_128u((__m128i*)dst + 3, xmm3);
++ xmm_save_128u((__m128i*)dst + 4, xmm4);
++ xmm_save_128u((__m128i*)dst + 5, xmm5);
++ xmm_save_128u((__m128i*)dst + 6, xmm6);
++ xmm_save_128u((__m128i*)dst + 7, xmm7);
++
++ dst += 128;
++ src += 128;
++ }
++}
++
++static force_inline void
++from_sse128xNa(uint8_t *dst, const uint8_t *src, int bytes)
++{
++ int i;
++
++ assert(((uintptr_t)dst & 15) == 0);
++ assert(((uintptr_t)src & 15) == 0);
++
++ for (i = 0; i < bytes / 128; i++) {
++ __m128i xmm0, xmm1, xmm2, xmm3;
++ __m128i xmm4, xmm5, xmm6, xmm7;
++
++ xmm0 = xmm_load_128((const __m128i*)src + 0);
++ xmm1 = xmm_load_128((const __m128i*)src + 1);
++ xmm2 = xmm_load_128((const __m128i*)src + 2);
++ xmm3 = xmm_load_128((const __m128i*)src + 3);
++ xmm4 = xmm_load_128((const __m128i*)src + 4);
++ xmm5 = xmm_load_128((const __m128i*)src + 5);
++ xmm6 = xmm_load_128((const __m128i*)src + 6);
++ xmm7 = xmm_load_128((const __m128i*)src + 7);
++
++ xmm_save_128((__m128i*)dst + 0, xmm0);
++ xmm_save_128((__m128i*)dst + 1, xmm1);
++ xmm_save_128((__m128i*)dst + 2, xmm2);
++ xmm_save_128((__m128i*)dst + 3, xmm3);
++ xmm_save_128((__m128i*)dst + 4, xmm4);
++ xmm_save_128((__m128i*)dst + 5, xmm5);
++ xmm_save_128((__m128i*)dst + 6, xmm6);
++ xmm_save_128((__m128i*)dst + 7, xmm7);
++
++ dst += 128;
++ src += 128;
++ }
++}
++
++static force_inline void
++from_sse64u(uint8_t *dst, const uint8_t *src)
++{
++ __m128i xmm1, xmm2, xmm3, xmm4;
++
++ assert(((uintptr_t)src & 15) == 0);
++
++ xmm1 = xmm_load_128((const __m128i*)src + 0);
++ xmm2 = xmm_load_128((const __m128i*)src + 1);
++ xmm3 = xmm_load_128((const __m128i*)src + 2);
++ xmm4 = xmm_load_128((const __m128i*)src + 3);
++
++ xmm_save_128u((__m128i*)dst + 0, xmm1);
++ xmm_save_128u((__m128i*)dst + 1, xmm2);
++ xmm_save_128u((__m128i*)dst + 2, xmm3);
++ xmm_save_128u((__m128i*)dst + 3, xmm4);
++}
++
++static force_inline void
++from_sse64a(uint8_t *dst, const uint8_t *src)
++{
++ __m128i xmm1, xmm2, xmm3, xmm4;
++
++ assert(((uintptr_t)dst & 15) == 0);
++ assert(((uintptr_t)src & 15) == 0);
++
++ xmm1 = xmm_load_128((const __m128i*)src + 0);
++ xmm2 = xmm_load_128((const __m128i*)src + 1);
++ xmm3 = xmm_load_128((const __m128i*)src + 2);
++ xmm4 = xmm_load_128((const __m128i*)src + 3);
++
++ xmm_save_128((__m128i*)dst + 0, xmm1);
++ xmm_save_128((__m128i*)dst + 1, xmm2);
++ xmm_save_128((__m128i*)dst + 2, xmm3);
++ xmm_save_128((__m128i*)dst + 3, xmm4);
++}
++
++static force_inline void
++from_sse32u(uint8_t *dst, const uint8_t *src)
++{
++ __m128i xmm1, xmm2;
++
++ xmm1 = xmm_load_128((const __m128i*)src + 0);
++ xmm2 = xmm_load_128((const __m128i*)src + 1);
++
++ xmm_save_128u((__m128i*)dst + 0, xmm1);
++ xmm_save_128u((__m128i*)dst + 1, xmm2);
++}
++
++static force_inline void
++from_sse32a(uint8_t *dst, const uint8_t *src)
++{
++ __m128i xmm1, xmm2;
++
++ assert(((uintptr_t)dst & 15) == 0);
++ assert(((uintptr_t)src & 15) == 0);
++
++ xmm1 = xmm_load_128((const __m128i*)src + 0);
++ xmm2 = xmm_load_128((const __m128i*)src + 1);
++
++ xmm_save_128((__m128i*)dst + 0, xmm1);
++ xmm_save_128((__m128i*)dst + 1, xmm2);
++}
++
++static force_inline void
++from_sse16u(uint8_t *dst, const uint8_t *src)
++{
++ assert(((uintptr_t)src & 15) == 0);
++
++ xmm_save_128u((__m128i*)dst, xmm_load_128((const __m128i*)src));
++}
++
++static force_inline void
++from_sse16a(uint8_t *dst, const uint8_t *src)
++{
++ assert(((uintptr_t)dst & 15) == 0);
++ assert(((uintptr_t)src & 15) == 0);
++
++ xmm_save_128((__m128i*)dst, xmm_load_128((const __m128i*)src));
++}
++
++static void
++memcpy_from_tiled_x__swizzle_0__sse2(const void *src, void *dst, int bpp,
++ int32_t src_stride, int32_t dst_stride,
++ int16_t src_x, int16_t src_y,
++ int16_t dst_x, int16_t dst_y,
++ uint16_t width, uint16_t height)
++{
++ const unsigned tile_width = 512;
++ const unsigned tile_height = 8;
++ const unsigned tile_size = 4096;
++
++ const unsigned cpp = bpp / 8;
++ const unsigned tile_pixels = tile_width / cpp;
++ const unsigned tile_shift = ffs(tile_pixels) - 1;
++ const unsigned tile_mask = tile_pixels - 1;
++
++ unsigned length_x, offset_x;
++
++ DBG(("%s(bpp=%d): src=(%d, %d), dst=(%d, %d), size=%dx%d, pitch=%d/%d\n",
++ __FUNCTION__, bpp, src_x, src_y, dst_x, dst_y, width, height, src_stride, dst_stride));
++ assert(src != dst);
++
++ if (dst_x | dst_y)
++ dst = (uint8_t *)dst + dst_y * dst_stride + dst_x * cpp;
++ width *= cpp;
++ assert(dst_stride >= width);
++ if (src_x & tile_mask) {
++ offset_x = (src_x & tile_mask) * cpp;
++ length_x = min(tile_width - offset_x, width);
++ dst_stride -= width;
++ dst_stride += (width - length_x) & 15;
++ } else {
++ offset_x = 0;
++ dst_stride -= width & ~15;
++ }
++ assert(dst_stride >= 0);
++ src = (const uint8_t *)src + (src_x >> tile_shift) * tile_size;
++
++ while (height--) {
++ unsigned w = width;
++ const uint8_t *tile_row = src;
++
++ tile_row += src_y / tile_height * src_stride * tile_height;
++ tile_row += (src_y & (tile_height-1)) * tile_width;
++ src_y++;
++
++ if (offset_x) {
++ memcpy(dst, tile_row + offset_x, length_x);
++ tile_row += tile_size;
++ dst = (uint8_t *)dst + length_x;
++ w -= length_x;
++ }
++
++ if ((uintptr_t)dst & 15) {
++ while (w >= tile_width) {
++ from_sse128xNu(dst,
++ assume_aligned(tile_row, tile_width),
++ tile_width);
++ tile_row += tile_size;
++ dst = (uint8_t *)dst + tile_width;
++ w -= tile_width;
++ }
++ while (w >= 64) {
++ from_sse64u(dst, tile_row);
++ tile_row += 64;
++ dst = (uint8_t *)dst + 64;
++ w -= 64;
++ }
++ if (w & 32) {
++ from_sse32u(dst, tile_row);
++ tile_row += 32;
++ dst = (uint8_t *)dst + 32;
++ }
++ if (w & 16) {
++ from_sse16u(dst, tile_row);
++ tile_row += 16;
++ dst = (uint8_t *)dst + 16;
++ }
++ memcpy(dst, assume_aligned(tile_row, 16), w & 15);
++ } else {
++ while (w >= tile_width) {
++ from_sse128xNa(assume_aligned(dst, 16),
++ assume_aligned(tile_row, tile_width),
++ tile_width);
++ tile_row += tile_size;
++ dst = (uint8_t *)dst + tile_width;
++ w -= tile_width;
++ }
++ while (w >= 64) {
++ from_sse64a(dst, tile_row);
++ tile_row += 64;
++ dst = (uint8_t *)dst + 64;
++ w -= 64;
++ }
++ if (w & 32) {
++ from_sse32a(dst, tile_row);
++ tile_row += 32;
++ dst = (uint8_t *)dst + 32;
++ }
++ if (w & 16) {
++ from_sse16a(dst, tile_row);
++ tile_row += 16;
++ dst = (uint8_t *)dst + 16;
++ }
++ memcpy(assume_aligned(dst, 16),
++ assume_aligned(tile_row, 16),
++ w & 15);
++ }
++ dst = (uint8_t *)dst + dst_stride;
++ }
++}
++
++static void
++memcpy_between_tiled_x__swizzle_0__sse2(const void *src, void *dst, int bpp,
++ int32_t src_stride, int32_t dst_stride,
++ int16_t src_x, int16_t src_y,
++ int16_t dst_x, int16_t dst_y,
++ uint16_t width, uint16_t height)
++{
++ const unsigned tile_width = 512;
++ const unsigned tile_height = 8;
++ const unsigned tile_size = 4096;
++
++ const unsigned cpp = bpp / 8;
++ const unsigned tile_pixels = tile_width / cpp;
++ const unsigned tile_shift = ffs(tile_pixels) - 1;
++ const unsigned tile_mask = tile_pixels - 1;
++
++ unsigned ox, lx;
++
++ DBG(("%s(bpp=%d): src=(%d, %d), dst=(%d, %d), size=%dx%d, pitch=%d/%d\n",
++ __FUNCTION__, bpp, src_x, src_y, dst_x, dst_y, width, height, src_stride, dst_stride));
++ assert(src != dst);
++
++ width *= cpp;
++ dst_stride *= tile_height;
++ src_stride *= tile_height;
++
++ assert((dst_x & tile_mask) == (src_x & tile_mask));
++ if (dst_x & tile_mask) {
++ ox = (dst_x & tile_mask) * cpp;
++ lx = min(tile_width - ox, width);
++ assert(lx != 0);
++ } else
++ lx = 0;
++
++ if (dst_x)
++ dst = (uint8_t *)dst + (dst_x >> tile_shift) * tile_size;
++ if (src_x)
++ src = (const uint8_t *)src + (src_x >> tile_shift) * tile_size;
++
++ while (height--) {
++ const uint8_t *src_row;
++ uint8_t *dst_row;
++ unsigned w = width;
++
++ dst_row = dst;
++ dst_row += dst_y / tile_height * dst_stride;
++ dst_row += (dst_y & (tile_height-1)) * tile_width;
++ dst_y++;
++
++ src_row = src;
++ src_row += src_y / tile_height * src_stride;
++ src_row += (src_y & (tile_height-1)) * tile_width;
++ src_y++;
++
++ if (lx) {
++ to_memcpy(dst_row + ox, src_row + ox, lx);
++ dst_row += tile_size;
++ src_row += tile_size;
++ w -= lx;
++ }
++ while (w >= tile_width) {
++ assert(((uintptr_t)dst_row & (tile_width - 1)) == 0);
++ assert(((uintptr_t)src_row & (tile_width - 1)) == 0);
++ to_sse128xN(assume_aligned(dst_row, tile_width),
++ assume_aligned(src_row, tile_width),
++ tile_width);
++ dst_row += tile_size;
++ src_row += tile_size;
++ w -= tile_width;
++ }
++ if (w) {
++ assert(((uintptr_t)dst_row & (tile_width - 1)) == 0);
++ assert(((uintptr_t)src_row & (tile_width - 1)) == 0);
++ to_memcpy(assume_aligned(dst_row, tile_width),
++ assume_aligned(src_row, tile_width),
++ w);
++ }
++ }
++}
++
++#pragma GCC push_options
+ #endif
+
+ fast void
+@@ -257,7 +753,8 @@ memcpy_to_tiled_x__swizzle_0(const void *src, void *dst, int bpp,
+ if (dst_x & tile_mask) {
+ const unsigned x = (dst_x & tile_mask) * cpp;
+ const unsigned len = min(tile_width - x, w);
+- memcpy(tile_row + x, src, len);
++ memcpy(assume_misaligned(tile_row + x, tile_width, x),
++ src, len);
+
+ tile_row += tile_size;
+ src = (const uint8_t *)src + len;
+@@ -265,13 +762,13 @@ memcpy_to_tiled_x__swizzle_0(const void *src, void *dst, int bpp,
+ }
+ }
+ while (w >= tile_width) {
+- memcpy(tile_row, src, tile_width);
+-
++ memcpy(assume_aligned(tile_row, tile_width),
++ src, tile_width);
+ tile_row += tile_size;
+ src = (const uint8_t *)src + tile_width;
+ w -= tile_width;
+ }
+- memcpy(tile_row, src, w);
++ memcpy(assume_aligned(tile_row, tile_width), src, w);
+ src = (const uint8_t *)src + src_stride + w;
+ dst_y++;
+ }
+@@ -313,7 +810,7 @@ memcpy_from_tiled_x__swizzle_0(const void *src, void *dst, int bpp,
+ if (src_x & tile_mask) {
+ const unsigned x = (src_x & tile_mask) * cpp;
+ const unsigned len = min(tile_width - x, w);
+- memcpy(dst, tile_row + x, len);
++ memcpy(dst, assume_misaligned(tile_row + x, tile_width, x), len);
+
+ tile_row += tile_size;
+ dst = (uint8_t *)dst + len;
+@@ -321,440 +818,371 @@ memcpy_from_tiled_x__swizzle_0(const void *src, void *dst, int bpp,
+ }
+ }
+ while (w >= tile_width) {
+- memcpy(dst, tile_row, tile_width);
++ memcpy(dst,
++ assume_aligned(tile_row, tile_width),
++ tile_width);
+
+ tile_row += tile_size;
+ dst = (uint8_t *)dst + tile_width;
+ w -= tile_width;
+ }
+- memcpy(dst, tile_row, w);
++ memcpy(dst, assume_aligned(tile_row, tile_width), w);
+ dst = (uint8_t *)dst + dst_stride + w;
+ src_y++;
+ }
+ }
+
+-fast_memcpy static void
+-memcpy_to_tiled_x__swizzle_9(const void *src, void *dst, int bpp,
+- int32_t src_stride, int32_t dst_stride,
+- int16_t src_x, int16_t src_y,
+- int16_t dst_x, int16_t dst_y,
+- uint16_t width, uint16_t height)
++static fast_memcpy void
++memcpy_between_tiled_x__swizzle_0(const void *src, void *dst, int bpp,
++ int32_t src_stride, int32_t dst_stride,
++ int16_t src_x, int16_t src_y,
++ int16_t dst_x, int16_t dst_y,
++ uint16_t width, uint16_t height)
+ {
+ const unsigned tile_width = 512;
+ const unsigned tile_height = 8;
+ const unsigned tile_size = 4096;
+
+ const unsigned cpp = bpp / 8;
+- const unsigned stride_tiles = dst_stride / tile_width;
+- const unsigned swizzle_pixels = 64 / cpp;
+- const unsigned tile_pixels = ffs(tile_width / cpp) - 1;
+- const unsigned tile_mask = (1 << tile_pixels) - 1;
+-
+- unsigned x, y;
++ const unsigned tile_pixels = tile_width / cpp;
++ const unsigned tile_shift = ffs(tile_pixels) - 1;
++ const unsigned tile_mask = tile_pixels - 1;
+
+ DBG(("%s(bpp=%d): src=(%d, %d), dst=(%d, %d), size=%dx%d, pitch=%d/%d\n",
+ __FUNCTION__, bpp, src_x, src_y, dst_x, dst_y, width, height, src_stride, dst_stride));
++ assert(src != dst);
++ assert((dst_x & tile_mask) == (src_x & tile_mask));
+
+- src = (const uint8_t *)src + src_y * src_stride + src_x * cpp;
+-
+- for (y = 0; y < height; ++y) {
+- const uint32_t dy = y + dst_y;
+- const uint32_t tile_row =
+- (dy / tile_height * stride_tiles * tile_size +
+- (dy & (tile_height-1)) * tile_width);
+- const uint8_t *src_row = (const uint8_t *)src + src_stride * y;
+- uint32_t dx = dst_x, offset;
+-
+- x = width * cpp;
+- if (dx & (swizzle_pixels - 1)) {
+- const uint32_t swizzle_bound_pixels = ALIGN(dx + 1, swizzle_pixels);
+- const uint32_t length = min(dst_x + width, swizzle_bound_pixels) - dx;
+- offset = tile_row +
+- (dx >> tile_pixels) * tile_size +
+- (dx & tile_mask) * cpp;
+- offset ^= (offset >> 3) & 64;
+-
+- memcpy((char *)dst + offset, src_row, length * cpp);
+-
+- src_row += length * cpp;
+- x -= length * cpp;
+- dx += length;
+- }
+- while (x >= 64) {
+- offset = tile_row +
+- (dx >> tile_pixels) * tile_size +
+- (dx & tile_mask) * cpp;
+- offset ^= (offset >> 3) & 64;
+-
+- memcpy((char *)dst + offset, src_row, 64);
+-
+- src_row += 64;
+- x -= 64;
+- dx += swizzle_pixels;
+- }
+- if (x) {
+- offset = tile_row +
+- (dx >> tile_pixels) * tile_size +
+- (dx & tile_mask) * cpp;
+- offset ^= (offset >> 3) & 64;
+- memcpy((char *)dst + offset, src_row, x);
+- }
+- }
+-}
++ while (height--) {
++ unsigned w = width * cpp;
++ uint8_t *dst_row = dst;
++ const uint8_t *src_row = src;
+
+-fast_memcpy static void
+-memcpy_from_tiled_x__swizzle_9(const void *src, void *dst, int bpp,
+- int32_t src_stride, int32_t dst_stride,
+- int16_t src_x, int16_t src_y,
+- int16_t dst_x, int16_t dst_y,
+- uint16_t width, uint16_t height)
+-{
+- const unsigned tile_width = 512;
+- const unsigned tile_height = 8;
+- const unsigned tile_size = 4096;
++ dst_row += dst_y / tile_height * dst_stride * tile_height;
++ dst_row += (dst_y & (tile_height-1)) * tile_width;
++ if (dst_x)
++ dst_row += (dst_x >> tile_shift) * tile_size;
++ dst_y++;
+
+- const unsigned cpp = bpp / 8;
+- const unsigned stride_tiles = src_stride / tile_width;
+- const unsigned swizzle_pixels = 64 / cpp;
+- const unsigned tile_pixels = ffs(tile_width / cpp) - 1;
+- const unsigned tile_mask = (1 << tile_pixels) - 1;
++ src_row += src_y / tile_height * src_stride * tile_height;
++ src_row += (src_y & (tile_height-1)) * tile_width;
++ if (src_x)
++ src_row += (src_x >> tile_shift) * tile_size;
++ src_y++;
+
+- unsigned x, y;
++ if (dst_x & tile_mask) {
++ const unsigned x = (dst_x & tile_mask) * cpp;
++ const unsigned len = min(tile_width - x, w);
+
+- DBG(("%s(bpp=%d): src=(%d, %d), dst=(%d, %d), size=%dx%d, pitch=%d/%d\n",
+- __FUNCTION__, bpp, src_x, src_y, dst_x, dst_y, width, height, src_stride, dst_stride));
++ memcpy(assume_misaligned(dst_row + x, tile_width, x),
++ assume_misaligned(src_row + x, tile_width, x),
++ len);
+
+- dst = (uint8_t *)dst + dst_y * dst_stride + dst_x * cpp;
+-
+- for (y = 0; y < height; ++y) {
+- const uint32_t sy = y + src_y;
+- const uint32_t tile_row =
+- (sy / tile_height * stride_tiles * tile_size +
+- (sy & (tile_height-1)) * tile_width);
+- uint8_t *dst_row = (uint8_t *)dst + dst_stride * y;
+- uint32_t sx = src_x, offset;
+-
+- x = width * cpp;
+- if (sx & (swizzle_pixels - 1)) {
+- const uint32_t swizzle_bound_pixels = ALIGN(sx + 1, swizzle_pixels);
+- const uint32_t length = min(src_x + width, swizzle_bound_pixels) - sx;
+- offset = tile_row +
+- (sx >> tile_pixels) * tile_size +
+- (sx & tile_mask) * cpp;
+- offset ^= (offset >> 3) & 64;
+-
+- memcpy(dst_row, (const char *)src + offset, length * cpp);
+-
+- dst_row += length * cpp;
+- x -= length * cpp;
+- sx += length;
++ dst_row += tile_size;
++ src_row += tile_size;
++ w -= len;
+ }
+- while (x >= 64) {
+- offset = tile_row +
+- (sx >> tile_pixels) * tile_size +
+- (sx & tile_mask) * cpp;
+- offset ^= (offset >> 3) & 64;
+
+- memcpy(dst_row, (const char *)src + offset, 64);
+-
+- dst_row += 64;
+- x -= 64;
+- sx += swizzle_pixels;
+- }
+- if (x) {
+- offset = tile_row +
+- (sx >> tile_pixels) * tile_size +
+- (sx & tile_mask) * cpp;
+- offset ^= (offset >> 3) & 64;
+- memcpy(dst_row, (const char *)src + offset, x);
++ while (w >= tile_width) {
++ memcpy(assume_aligned(dst_row, tile_width),
++ assume_aligned(src_row, tile_width),
++ tile_width);
++ dst_row += tile_size;
++ src_row += tile_size;
++ w -= tile_width;
+ }
++ memcpy(assume_aligned(dst_row, tile_width),
++ assume_aligned(src_row, tile_width),
++ w);
+ }
+ }
+
+-fast_memcpy static void
+-memcpy_to_tiled_x__swizzle_9_10(const void *src, void *dst, int bpp,
+- int32_t src_stride, int32_t dst_stride,
+- int16_t src_x, int16_t src_y,
+- int16_t dst_x, int16_t dst_y,
+- uint16_t width, uint16_t height)
+-{
+- const unsigned tile_width = 512;
+- const unsigned tile_height = 8;
+- const unsigned tile_size = 4096;
+-
+- const unsigned cpp = bpp / 8;
+- const unsigned stride_tiles = dst_stride / tile_width;
+- const unsigned swizzle_pixels = 64 / cpp;
+- const unsigned tile_pixels = ffs(tile_width / cpp) - 1;
+- const unsigned tile_mask = (1 << tile_pixels) - 1;
++#define memcpy_to_tiled_x(swizzle) \
++fast_memcpy static void \
++memcpy_to_tiled_x__##swizzle (const void *src, void *dst, int bpp, \
++ int32_t src_stride, int32_t dst_stride, \
++ int16_t src_x, int16_t src_y, \
++ int16_t dst_x, int16_t dst_y, \
++ uint16_t width, uint16_t height) \
++{ \
++ const unsigned tile_width = 512; \
++ const unsigned tile_height = 8; \
++ const unsigned tile_size = 4096; \
++ const unsigned cpp = bpp / 8; \
++ const unsigned stride_tiles = dst_stride / tile_width; \
++ const unsigned swizzle_pixels = 64 / cpp; \
++ const unsigned tile_pixels = ffs(tile_width / cpp) - 1; \
++ const unsigned tile_mask = (1 << tile_pixels) - 1; \
++ unsigned x, y; \
++ DBG(("%s(bpp=%d): src=(%d, %d), dst=(%d, %d), size=%dx%d, pitch=%d/%d\n", \
++ __FUNCTION__, bpp, src_x, src_y, dst_x, dst_y, width, height, src_stride, dst_stride)); \
++ src = (const uint8_t *)src + src_y * src_stride + src_x * cpp; \
++ for (y = 0; y < height; ++y) { \
++ const uint32_t dy = y + dst_y; \
++ const uint32_t tile_row = \
++ (dy / tile_height * stride_tiles * tile_size + \
++ (dy & (tile_height-1)) * tile_width); \
++ const uint8_t *src_row = (const uint8_t *)src + src_stride * y; \
++ uint32_t dx = dst_x; \
++ x = width * cpp; \
++ if (dx & (swizzle_pixels - 1)) { \
++ const uint32_t swizzle_bound_pixels = ALIGN(dx + 1, swizzle_pixels); \
++ const uint32_t length = min(dst_x + width, swizzle_bound_pixels) - dx; \
++ uint32_t offset = \
++ tile_row + \
++ (dx >> tile_pixels) * tile_size + \
++ (dx & tile_mask) * cpp; \
++ memcpy((char *)dst + swizzle(offset), src_row, length * cpp); \
++ src_row += length * cpp; \
++ x -= length * cpp; \
++ dx += length; \
++ } \
++ while (x >= 64) { \
++ uint32_t offset = \
++ tile_row + \
++ (dx >> tile_pixels) * tile_size + \
++ (dx & tile_mask) * cpp; \
++ memcpy(assume_aligned((char *)dst+swizzle(offset),64), \
++ src_row, 64); \
++ src_row += 64; \
++ x -= 64; \
++ dx += swizzle_pixels; \
++ } \
++ if (x) { \
++ uint32_t offset = \
++ tile_row + \
++ (dx >> tile_pixels) * tile_size + \
++ (dx & tile_mask) * cpp; \
++ memcpy(assume_aligned((char *)dst + swizzle(offset), 64), src_row, x); \
++ } \
++ } \
++}
+
+- unsigned x, y;
++#define memcpy_from_tiled_x(swizzle) \
++fast_memcpy static void \
++memcpy_from_tiled_x__##swizzle (const void *src, void *dst, int bpp, \
++ int32_t src_stride, int32_t dst_stride, \
++ int16_t src_x, int16_t src_y, \
++ int16_t dst_x, int16_t dst_y, \
++ uint16_t width, uint16_t height) \
++{ \
++ const unsigned tile_width = 512; \
++ const unsigned tile_height = 8; \
++ const unsigned tile_size = 4096; \
++ const unsigned cpp = bpp / 8; \
++ const unsigned stride_tiles = src_stride / tile_width; \
++ const unsigned swizzle_pixels = 64 / cpp; \
++ const unsigned tile_pixels = ffs(tile_width / cpp) - 1; \
++ const unsigned tile_mask = (1 << tile_pixels) - 1; \
++ unsigned x, y; \
++ DBG(("%s(bpp=%d): src=(%d, %d), dst=(%d, %d), size=%dx%d, pitch=%d/%d\n", \
++ __FUNCTION__, bpp, src_x, src_y, dst_x, dst_y, width, height, src_stride, dst_stride)); \
++ dst = (uint8_t *)dst + dst_y * dst_stride + dst_x * cpp; \
++ for (y = 0; y < height; ++y) { \
++ const uint32_t sy = y + src_y; \
++ const uint32_t tile_row = \
++ (sy / tile_height * stride_tiles * tile_size + \
++ (sy & (tile_height-1)) * tile_width); \
++ uint8_t *dst_row = (uint8_t *)dst + dst_stride * y; \
++ uint32_t sx = src_x; \
++ x = width * cpp; \
++ if (sx & (swizzle_pixels - 1)) { \
++ const uint32_t swizzle_bound_pixels = ALIGN(sx + 1, swizzle_pixels); \
++ const uint32_t length = min(src_x + width, swizzle_bound_pixels) - sx; \
++ uint32_t offset = \
++ tile_row + \
++ (sx >> tile_pixels) * tile_size + \
++ (sx & tile_mask) * cpp; \
++ memcpy(dst_row, (const char *)src + swizzle(offset), length * cpp); \
++ dst_row += length * cpp; \
++ x -= length * cpp; \
++ sx += length; \
++ } \
++ while (x >= 64) { \
++ uint32_t offset = \
++ tile_row + \
++ (sx >> tile_pixels) * tile_size + \
++ (sx & tile_mask) * cpp; \
++ memcpy(dst_row, assume_aligned((const char *)src + swizzle(offset), 64), 64); \
++ dst_row += 64; \
++ x -= 64; \
++ sx += swizzle_pixels; \
++ } \
++ if (x) { \
++ uint32_t offset = \
++ tile_row + \
++ (sx >> tile_pixels) * tile_size + \
++ (sx & tile_mask) * cpp; \
++ memcpy(dst_row, assume_aligned((const char *)src + swizzle(offset), 64), x); \
++ } \
++ } \
++}
+
+- DBG(("%s(bpp=%d): src=(%d, %d), dst=(%d, %d), size=%dx%d, pitch=%d/%d\n",
+- __FUNCTION__, bpp, src_x, src_y, dst_x, dst_y, width, height, src_stride, dst_stride));
++#define swizzle_9(X) ((X) ^ (((X) >> 3) & 64))
++memcpy_to_tiled_x(swizzle_9)
++memcpy_from_tiled_x(swizzle_9)
++#undef swizzle_9
+
+- src = (const uint8_t *)src + src_y * src_stride + src_x * cpp;
+-
+- for (y = 0; y < height; ++y) {
+- const uint32_t dy = y + dst_y;
+- const uint32_t tile_row =
+- (dy / tile_height * stride_tiles * tile_size +
+- (dy & (tile_height-1)) * tile_width);
+- const uint8_t *src_row = (const uint8_t *)src + src_stride * y;
+- uint32_t dx = dst_x, offset;
+-
+- x = width * cpp;
+- if (dx & (swizzle_pixels - 1)) {
+- const uint32_t swizzle_bound_pixels = ALIGN(dx + 1, swizzle_pixels);
+- const uint32_t length = min(dst_x + width, swizzle_bound_pixels) - dx;
+- offset = tile_row +
+- (dx >> tile_pixels) * tile_size +
+- (dx & tile_mask) * cpp;
+- offset ^= ((offset ^ (offset >> 1)) >> 3) & 64;
+-
+- memcpy((char *)dst + offset, src_row, length * cpp);
+-
+- src_row += length * cpp;
+- x -= length * cpp;
+- dx += length;
+- }
+- while (x >= 64) {
+- offset = tile_row +
+- (dx >> tile_pixels) * tile_size +
+- (dx & tile_mask) * cpp;
+- offset ^= ((offset ^ (offset >> 1)) >> 3) & 64;
++#define swizzle_9_10(X) ((X) ^ ((((X) ^ ((X) >> 1)) >> 3) & 64))
++memcpy_to_tiled_x(swizzle_9_10)
++memcpy_from_tiled_x(swizzle_9_10)
++#undef swizzle_9_10
+
+- memcpy((char *)dst + offset, src_row, 64);
++#define swizzle_9_11(X) ((X) ^ ((((X) ^ ((X) >> 2)) >> 3) & 64))
++memcpy_to_tiled_x(swizzle_9_11)
++memcpy_from_tiled_x(swizzle_9_11)
++#undef swizzle_9_11
+
+- src_row += 64;
+- x -= 64;
+- dx += swizzle_pixels;
+- }
+- if (x) {
+- offset = tile_row +
+- (dx >> tile_pixels) * tile_size +
+- (dx & tile_mask) * cpp;
+- offset ^= ((offset ^ (offset >> 1)) >> 3) & 64;
+- memcpy((char *)dst + offset, src_row, x);
+- }
+- }
+-}
++#define swizzle_9_10_11(X) ((X) ^ ((((X) ^ ((X) >> 1) ^ ((X) >> 2)) >> 3) & 64))
++memcpy_to_tiled_x(swizzle_9_10_11)
++memcpy_from_tiled_x(swizzle_9_10_11)
++#undef swizzle_9_10_11
+
+-fast_memcpy static void
+-memcpy_from_tiled_x__swizzle_9_10(const void *src, void *dst, int bpp,
+- int32_t src_stride, int32_t dst_stride,
+- int16_t src_x, int16_t src_y,
+- int16_t dst_x, int16_t dst_y,
+- uint16_t width, uint16_t height)
++static fast_memcpy void
++memcpy_to_tiled_x__gen2(const void *src, void *dst, int bpp,
++ int32_t src_stride, int32_t dst_stride,
++ int16_t src_x, int16_t src_y,
++ int16_t dst_x, int16_t dst_y,
++ uint16_t width, uint16_t height)
+ {
+- const unsigned tile_width = 512;
+- const unsigned tile_height = 8;
+- const unsigned tile_size = 4096;
++ const unsigned tile_width = 128;
++ const unsigned tile_height = 16;
++ const unsigned tile_size = 2048;
+
+ const unsigned cpp = bpp / 8;
+- const unsigned stride_tiles = src_stride / tile_width;
+- const unsigned swizzle_pixels = 64 / cpp;
+- const unsigned tile_pixels = ffs(tile_width / cpp) - 1;
+- const unsigned tile_mask = (1 << tile_pixels) - 1;
+-
+- unsigned x, y;
++ const unsigned tile_pixels = tile_width / cpp;
++ const unsigned tile_shift = ffs(tile_pixels) - 1;
++ const unsigned tile_mask = tile_pixels - 1;
+
+ DBG(("%s(bpp=%d): src=(%d, %d), dst=(%d, %d), size=%dx%d, pitch=%d/%d\n",
+ __FUNCTION__, bpp, src_x, src_y, dst_x, dst_y, width, height, src_stride, dst_stride));
++ assert(src != dst);
+
+- dst = (uint8_t *)dst + dst_y * dst_stride + dst_x * cpp;
+-
+- for (y = 0; y < height; ++y) {
+- const uint32_t sy = y + src_y;
+- const uint32_t tile_row =
+- (sy / tile_height * stride_tiles * tile_size +
+- (sy & (tile_height-1)) * tile_width);
+- uint8_t *dst_row = (uint8_t *)dst + dst_stride * y;
+- uint32_t sx = src_x, offset;
+-
+- x = width * cpp;
+- if (sx & (swizzle_pixels - 1)) {
+- const uint32_t swizzle_bound_pixels = ALIGN(sx + 1, swizzle_pixels);
+- const uint32_t length = min(src_x + width, swizzle_bound_pixels) - sx;
+- offset = tile_row +
+- (sx >> tile_pixels) * tile_size +
+- (sx & tile_mask) * cpp;
+- offset ^= ((offset ^ (offset >> 1)) >> 3) & 64;
+-
+- memcpy(dst_row, (const char *)src + offset, length * cpp);
+-
+- dst_row += length * cpp;
+- x -= length * cpp;
+- sx += length;
+- }
+- while (x >= 64) {
+- offset = tile_row +
+- (sx >> tile_pixels) * tile_size +
+- (sx & tile_mask) * cpp;
+- offset ^= ((offset ^ (offset >> 1)) >> 3) & 64;
+-
+- memcpy(dst_row, (const char *)src + offset, 64);
+-
+- dst_row += 64;
+- x -= 64;
+- sx += swizzle_pixels;
+- }
+- if (x) {
+- offset = tile_row +
+- (sx >> tile_pixels) * tile_size +
+- (sx & tile_mask) * cpp;
+- offset ^= ((offset ^ (offset >> 1)) >> 3) & 64;
+- memcpy(dst_row, (const char *)src + offset, x);
+- }
+- }
+-}
+-
+-fast_memcpy static void
+-memcpy_to_tiled_x__swizzle_9_11(const void *src, void *dst, int bpp,
+- int32_t src_stride, int32_t dst_stride,
+- int16_t src_x, int16_t src_y,
+- int16_t dst_x, int16_t dst_y,
+- uint16_t width, uint16_t height)
+-{
+- const unsigned tile_width = 512;
+- const unsigned tile_height = 8;
+- const unsigned tile_size = 4096;
+-
+- const unsigned cpp = bpp / 8;
+- const unsigned stride_tiles = dst_stride / tile_width;
+- const unsigned swizzle_pixels = 64 / cpp;
+- const unsigned tile_pixels = ffs(tile_width / cpp) - 1;
+- const unsigned tile_mask = (1 << tile_pixels) - 1;
++ if (src_x | src_y)
++ src = (const uint8_t *)src + src_y * src_stride + src_x * cpp;
++ assert(src_stride >= width * cpp);
++ src_stride -= width * cpp;
+
+- unsigned x, y;
++ while (height--) {
++ unsigned w = width * cpp;
++ uint8_t *tile_row = dst;
+
+- DBG(("%s(bpp=%d): src=(%d, %d), dst=(%d, %d), size=%dx%d, pitch=%d/%d\n",
+- __FUNCTION__, bpp, src_x, src_y, dst_x, dst_y, width, height, src_stride, dst_stride));
++ tile_row += dst_y / tile_height * dst_stride * tile_height;
++ tile_row += (dst_y & (tile_height-1)) * tile_width;
++ if (dst_x) {
++ tile_row += (dst_x >> tile_shift) * tile_size;
++ if (dst_x & tile_mask) {
++ const unsigned x = (dst_x & tile_mask) * cpp;
++ const unsigned len = min(tile_width - x, w);
++ memcpy(assume_misaligned(tile_row + x, tile_width, x), src, len);
+
+- src = (const uint8_t *)src + src_y * src_stride + src_x * cpp;
+-
+- for (y = 0; y < height; ++y) {
+- const uint32_t dy = y + dst_y;
+- const uint32_t tile_row =
+- (dy / tile_height * stride_tiles * tile_size +
+- (dy & (tile_height-1)) * tile_width);
+- const uint8_t *src_row = (const uint8_t *)src + src_stride * y;
+- uint32_t dx = dst_x, offset;
+-
+- x = width * cpp;
+- if (dx & (swizzle_pixels - 1)) {
+- const uint32_t swizzle_bound_pixels = ALIGN(dx + 1, swizzle_pixels);
+- const uint32_t length = min(dst_x + width, swizzle_bound_pixels) - dx;
+- offset = tile_row +
+- (dx >> tile_pixels) * tile_size +
+- (dx & tile_mask) * cpp;
+- offset ^= ((offset ^ (offset >> 2)) >> 3) & 64;
+- memcpy((char *)dst + offset, src_row, length * cpp);
+-
+- src_row += length * cpp;
+- x -= length * cpp;
+- dx += length;
++ tile_row += tile_size;
++ src = (const uint8_t *)src + len;
++ w -= len;
++ }
+ }
+- while (x >= 64) {
+- offset = tile_row +
+- (dx >> tile_pixels) * tile_size +
+- (dx & tile_mask) * cpp;
+- offset ^= ((offset ^ (offset >> 2)) >> 3) & 64;
+-
+- memcpy((char *)dst + offset, src_row, 64);
++ while (w >= tile_width) {
++ memcpy(assume_aligned(tile_row, tile_width),
++ src, tile_width);
+
+- src_row += 64;
+- x -= 64;
+- dx += swizzle_pixels;
+- }
+- if (x) {
+- offset = tile_row +
+- (dx >> tile_pixels) * tile_size +
+- (dx & tile_mask) * cpp;
+- offset ^= ((offset ^ (offset >> 2)) >> 3) & 64;
+- memcpy((char *)dst + offset, src_row, x);
++ tile_row += tile_size;
++ src = (const uint8_t *)src + tile_width;
++ w -= tile_width;
+ }
++ memcpy(assume_aligned(tile_row, tile_width), src, w);
++ src = (const uint8_t *)src + src_stride + w;
++ dst_y++;
+ }
+ }
+
+-fast_memcpy static void
+-memcpy_from_tiled_x__swizzle_9_11(const void *src, void *dst, int bpp,
+- int32_t src_stride, int32_t dst_stride,
+- int16_t src_x, int16_t src_y,
+- int16_t dst_x, int16_t dst_y,
+- uint16_t width, uint16_t height)
++static fast_memcpy void
++memcpy_from_tiled_x__gen2(const void *src, void *dst, int bpp,
++ int32_t src_stride, int32_t dst_stride,
++ int16_t src_x, int16_t src_y,
++ int16_t dst_x, int16_t dst_y,
++ uint16_t width, uint16_t height)
+ {
+- const unsigned tile_width = 512;
+- const unsigned tile_height = 8;
+- const unsigned tile_size = 4096;
++ const unsigned tile_width = 128;
++ const unsigned tile_height = 16;
++ const unsigned tile_size = 2048;
+
+ const unsigned cpp = bpp / 8;
+- const unsigned stride_tiles = src_stride / tile_width;
+- const unsigned swizzle_pixels = 64 / cpp;
+- const unsigned tile_pixels = ffs(tile_width / cpp) - 1;
+- const unsigned tile_mask = (1 << tile_pixels) - 1;
+-
+- unsigned x, y;
++ const unsigned tile_pixels = tile_width / cpp;
++ const unsigned tile_shift = ffs(tile_pixels) - 1;
++ const unsigned tile_mask = tile_pixels - 1;
+
+ DBG(("%s(bpp=%d): src=(%d, %d), dst=(%d, %d), size=%dx%d, pitch=%d/%d\n",
+ __FUNCTION__, bpp, src_x, src_y, dst_x, dst_y, width, height, src_stride, dst_stride));
++ assert(src != dst);
+
+- dst = (uint8_t *)dst + dst_y * dst_stride + dst_x * cpp;
+-
+- for (y = 0; y < height; ++y) {
+- const uint32_t sy = y + src_y;
+- const uint32_t tile_row =
+- (sy / tile_height * stride_tiles * tile_size +
+- (sy & (tile_height-1)) * tile_width);
+- uint8_t *dst_row = (uint8_t *)dst + dst_stride * y;
+- uint32_t sx = src_x, offset;
+-
+- x = width * cpp;
+- if (sx & (swizzle_pixels - 1)) {
+- const uint32_t swizzle_bound_pixels = ALIGN(sx + 1, swizzle_pixels);
+- const uint32_t length = min(src_x + width, swizzle_bound_pixels) - sx;
+- offset = tile_row +
+- (sx >> tile_pixels) * tile_size +
+- (sx & tile_mask) * cpp;
+- offset ^= ((offset ^ (offset >> 2)) >> 3) & 64;
+- memcpy(dst_row, (const char *)src + offset, length * cpp);
+-
+- dst_row += length * cpp;
+- x -= length * cpp;
+- sx += length;
+- }
+- while (x >= 64) {
+- offset = tile_row +
+- (sx >> tile_pixels) * tile_size +
+- (sx & tile_mask) * cpp;
+- offset ^= ((offset ^ (offset >> 2)) >> 3) & 64;
++ if (dst_x | dst_y)
++ dst = (uint8_t *)dst + dst_y * dst_stride + dst_x * cpp;
++ assert(dst_stride >= width * cpp);
++ dst_stride -= width * cpp;
++
++ while (height--) {
++ unsigned w = width * cpp;
++ const uint8_t *tile_row = src;
+
+- memcpy(dst_row, (const char *)src + offset, 64);
++ tile_row += src_y / tile_height * src_stride * tile_height;
++ tile_row += (src_y & (tile_height-1)) * tile_width;
++ if (src_x) {
++ tile_row += (src_x >> tile_shift) * tile_size;
++ if (src_x & tile_mask) {
++ const unsigned x = (src_x & tile_mask) * cpp;
++ const unsigned len = min(tile_width - x, w);
++ memcpy(dst, assume_misaligned(tile_row + x, tile_width, x), len);
+
+- dst_row += 64;
+- x -= 64;
+- sx += swizzle_pixels;
++ tile_row += tile_size;
++ dst = (uint8_t *)dst + len;
++ w -= len;
++ }
+ }
+- if (x) {
+- offset = tile_row +
+- (sx >> tile_pixels) * tile_size +
+- (sx & tile_mask) * cpp;
+- offset ^= ((offset ^ (offset >> 2)) >> 3) & 64;
+- memcpy(dst_row, (const char *)src + offset, x);
++ while (w >= tile_width) {
++ memcpy(dst,
++ assume_aligned(tile_row, tile_width),
++ tile_width);
++
++ tile_row += tile_size;
++ dst = (uint8_t *)dst + tile_width;
++ w -= tile_width;
+ }
++ memcpy(dst, assume_aligned(tile_row, tile_width), w);
++ dst = (uint8_t *)dst + dst_stride + w;
++ src_y++;
+ }
+ }
+
+-void choose_memcpy_tiled_x(struct kgem *kgem, int swizzling)
++void choose_memcpy_tiled_x(struct kgem *kgem, int swizzling, unsigned cpu)
+ {
++ if (kgem->gen < 030) {
++ if (swizzling == I915_BIT_6_SWIZZLE_NONE) {
++ DBG(("%s: gen2, no swizzling\n", __FUNCTION__));
++ kgem->memcpy_to_tiled_x = memcpy_to_tiled_x__gen2;
++ kgem->memcpy_from_tiled_x = memcpy_from_tiled_x__gen2;
++ } else
++ DBG(("%s: no detiling with swizzle functions for gen2\n", __FUNCTION__));
++ return;
++ }
++
+ switch (swizzling) {
+ default:
+ DBG(("%s: unknown swizzling, %d\n", __FUNCTION__, swizzling));
+ break;
+ case I915_BIT_6_SWIZZLE_NONE:
+ DBG(("%s: no swizzling\n", __FUNCTION__));
+- kgem->memcpy_to_tiled_x = memcpy_to_tiled_x__swizzle_0;
+- kgem->memcpy_from_tiled_x = memcpy_from_tiled_x__swizzle_0;
++#if defined(sse2)
++ if (cpu & SSE2) {
++ kgem->memcpy_to_tiled_x = memcpy_to_tiled_x__swizzle_0__sse2;
++ kgem->memcpy_from_tiled_x = memcpy_from_tiled_x__swizzle_0__sse2;
++ kgem->memcpy_between_tiled_x = memcpy_between_tiled_x__swizzle_0__sse2;
++ } else
++#endif
++ {
++ kgem->memcpy_to_tiled_x = memcpy_to_tiled_x__swizzle_0;
++ kgem->memcpy_from_tiled_x = memcpy_from_tiled_x__swizzle_0;
++ kgem->memcpy_between_tiled_x = memcpy_between_tiled_x__swizzle_0;
++ }
+ break;
+ case I915_BIT_6_SWIZZLE_9:
+ DBG(("%s: 6^9 swizzling\n", __FUNCTION__));
+@@ -771,6 +1199,11 @@ void choose_memcpy_tiled_x(struct kgem *kgem, int swizzling)
+ kgem->memcpy_to_tiled_x = memcpy_to_tiled_x__swizzle_9_11;
+ kgem->memcpy_from_tiled_x = memcpy_from_tiled_x__swizzle_9_11;
+ break;
++ case I915_BIT_6_SWIZZLE_9_10_11:
++ DBG(("%s: 6^9^10^11 swizzling\n", __FUNCTION__));
++ kgem->memcpy_to_tiled_x = memcpy_to_tiled_x__swizzle_9_10_11;
++ kgem->memcpy_from_tiled_x = memcpy_from_tiled_x__swizzle_9_10_11;
++ break;
+ }
+ }
+
+@@ -995,7 +1428,7 @@ memcpy_xor(const void *src, void *dst, int bpp,
+ height = 1;
+ }
+
+-#if USE_SSE2
++#if defined(sse2) && __x86_64__
+ if (have_sse2()) {
+ do {
+ uint32_t *d = (uint32_t *)dst_bytes;
+@@ -1118,3 +1551,241 @@ memcpy_xor(const void *src, void *dst, int bpp,
+ }
+ }
+ }
++
++#define BILINEAR_INTERPOLATION_BITS 4
++static inline int
++bilinear_weight(pixman_fixed_t x)
++{
++ return (x >> (16 - BILINEAR_INTERPOLATION_BITS)) &
++ ((1 << BILINEAR_INTERPOLATION_BITS) - 1);
++}
++
++#if BILINEAR_INTERPOLATION_BITS <= 4
++/* Inspired by Filter_32_opaque from Skia */
++static inline uint32_t
++bilinear_interpolation(uint32_t tl, uint32_t tr,
++ uint32_t bl, uint32_t br,
++ int distx, int disty)
++{
++ int distxy, distxiy, distixy, distixiy;
++ uint32_t lo, hi;
++
++ distx <<= (4 - BILINEAR_INTERPOLATION_BITS);
++ disty <<= (4 - BILINEAR_INTERPOLATION_BITS);
++
++ distxy = distx * disty;
++ distxiy = (distx << 4) - distxy; /* distx * (16 - disty) */
++ distixy = (disty << 4) - distxy; /* disty * (16 - distx) */
++ distixiy =
++ 16 * 16 - (disty << 4) -
++ (distx << 4) + distxy; /* (16 - distx) * (16 - disty) */
++
++ lo = (tl & 0xff00ff) * distixiy;
++ hi = ((tl >> 8) & 0xff00ff) * distixiy;
++
++ lo += (tr & 0xff00ff) * distxiy;
++ hi += ((tr >> 8) & 0xff00ff) * distxiy;
++
++ lo += (bl & 0xff00ff) * distixy;
++ hi += ((bl >> 8) & 0xff00ff) * distixy;
++
++ lo += (br & 0xff00ff) * distxy;
++ hi += ((br >> 8) & 0xff00ff) * distxy;
++
++ return ((lo >> 8) & 0xff00ff) | (hi & ~0xff00ff);
++}
++#elif SIZEOF_LONG > 4
++static inline uint32_t
++bilinear_interpolation(uint32_t tl, uint32_t tr,
++ uint32_t bl, uint32_t br,
++ int distx, int disty)
++{
++ uint64_t distxy, distxiy, distixy, distixiy;
++ uint64_t tl64, tr64, bl64, br64;
++ uint64_t f, r;
++
++ distx <<= (8 - BILINEAR_INTERPOLATION_BITS);
++ disty <<= (8 - BILINEAR_INTERPOLATION_BITS);
++
++ distxy = distx * disty;
++ distxiy = distx * (256 - disty);
++ distixy = (256 - distx) * disty;
++ distixiy = (256 - distx) * (256 - disty);
++
++ /* Alpha and Blue */
++ tl64 = tl & 0xff0000ff;
++ tr64 = tr & 0xff0000ff;
++ bl64 = bl & 0xff0000ff;
++ br64 = br & 0xff0000ff;
++
++ f = tl64 * distixiy + tr64 * distxiy + bl64 * distixy + br64 * distxy;
++ r = f & 0x0000ff0000ff0000ull;
++
++ /* Red and Green */
++ tl64 = tl;
++ tl64 = ((tl64 << 16) & 0x000000ff00000000ull) | (tl64 & 0x0000ff00ull);
++
++ tr64 = tr;
++ tr64 = ((tr64 << 16) & 0x000000ff00000000ull) | (tr64 & 0x0000ff00ull);
++
++ bl64 = bl;
++ bl64 = ((bl64 << 16) & 0x000000ff00000000ull) | (bl64 & 0x0000ff00ull);
++
++ br64 = br;
++ br64 = ((br64 << 16) & 0x000000ff00000000ull) | (br64 & 0x0000ff00ull);
++
++ f = tl64 * distixiy + tr64 * distxiy + bl64 * distixy + br64 * distxy;
++ r |= ((f >> 16) & 0x000000ff00000000ull) | (f & 0xff000000ull);
++
++ return (uint32_t)(r >> 16);
++}
++#else
++static inline uint32_t
++bilinear_interpolation(uint32_t tl, uint32_t tr,
++ uint32_t bl, uint32_t br,
++ int distx, int disty)
++{
++ int distxy, distxiy, distixy, distixiy;
++ uint32_t f, r;
++
++ distx <<= (8 - BILINEAR_INTERPOLATION_BITS);
++ disty <<= (8 - BILINEAR_INTERPOLATION_BITS);
++
++ distxy = distx * disty;
++ distxiy = (distx << 8) - distxy; /* distx * (256 - disty) */
++ distixy = (disty << 8) - distxy; /* disty * (256 - distx) */
++ distixiy =
++ 256 * 256 - (disty << 8) -
++ (distx << 8) + distxy; /* (256 - distx) * (256 - disty) */
++
++ /* Blue */
++ r = ((tl & 0x000000ff) * distixiy + (tr & 0x000000ff) * distxiy +
++ (bl & 0x000000ff) * distixy + (br & 0x000000ff) * distxy);
++
++ /* Green */
++ f = ((tl & 0x0000ff00) * distixiy + (tr & 0x0000ff00) * distxiy +
++ (bl & 0x0000ff00) * distixy + (br & 0x0000ff00) * distxy);
++ r |= f & 0xff000000;
++
++ tl >>= 16;
++ tr >>= 16;
++ bl >>= 16;
++ br >>= 16;
++ r >>= 16;
++
++ /* Red */
++ f = ((tl & 0x000000ff) * distixiy + (tr & 0x000000ff) * distxiy +
++ (bl & 0x000000ff) * distixy + (br & 0x000000ff) * distxy);
++ r |= f & 0x00ff0000;
++
++ /* Alpha */
++ f = ((tl & 0x0000ff00) * distixiy + (tr & 0x0000ff00) * distxiy +
++ (bl & 0x0000ff00) * distixy + (br & 0x0000ff00) * distxy);
++ r |= f & 0xff000000;
++
++ return r;
++}
++#endif
++
++static inline uint32_t convert_pixel(const uint8_t *p, int x)
++{
++ return ((uint32_t *)p)[x];
++}
++
++fast void
++affine_blt(const void *src, void *dst, int bpp,
++ int16_t src_x, int16_t src_y,
++ int16_t src_width, int16_t src_height,
++ int32_t src_stride,
++ int16_t dst_x, int16_t dst_y,
++ uint16_t dst_width, uint16_t dst_height,
++ int32_t dst_stride,
++ const struct pixman_f_transform *t)
++{
++ static const uint8_t zero[8] = { 0, 0, 0, 0, 0, 0, 0, 0 };
++ const pixman_fixed_t ux = pixman_double_to_fixed(t->m[0][0]);
++ const pixman_fixed_t uy = pixman_double_to_fixed(t->m[1][0]);
++ int i, j;
++
++ assert(bpp == 32);
++
++ for (j = 0; j < dst_height; j++) {
++ pixman_fixed_t x, y;
++ struct pixman_f_vector v;
++ uint32_t *b;
++
++ /* reference point is the center of the pixel */
++ v.v[0] = dst_x + 0.5;
++ v.v[1] = dst_y + j + 0.5;
++ v.v[2] = 1.0;
++
++ pixman_f_transform_point_3d(t, &v);
++
++ x = pixman_double_to_fixed(v.v[0]);
++ x += pixman_int_to_fixed(src_x - dst_x);
++ y = pixman_double_to_fixed(v.v[1]);
++ y += pixman_int_to_fixed(src_y - dst_y);
++
++ b = (uint32_t*)((uint8_t *)dst + (dst_y + j) * dst_stride + dst_x * bpp / 8);
++ for (i = 0; i < dst_width; i++) {
++ const uint8_t *row1;
++ const uint8_t *row2;
++ int x1, y1, x2, y2;
++ uint32_t tl, tr, bl, br;
++ int32_t fx, fy;
++
++ x1 = x - pixman_fixed_1/2;
++ y1 = y - pixman_fixed_1/2;
++
++ fx = bilinear_weight(x1);
++ fy = bilinear_weight(y1);
++
++ x1 = pixman_fixed_to_int(x1);
++ x2 = x1 + 1;
++ y1 = pixman_fixed_to_int(y1);
++ y2 = y1 + 1;
++
++ if (x1 >= src_width || x2 < 0 ||
++ y1 >= src_height || y2 < 0) {
++ b[i] = 0;
++ goto next;
++ }
++
++ if (y2 == 0) {
++ row1 = zero;
++ } else {
++ row1 = (uint8_t *)src + src_stride * y1;
++ row1 += bpp / 8 * x1;
++ }
++
++ if (y1 == src_height - 1) {
++ row2 = zero;
++ } else {
++ row2 = (uint8_t *)src + src_stride * y2;
++ row2 += bpp / 8 * x1;
++ }
++
++ if (x2 == 0) {
++ tl = 0;
++ bl = 0;
++ } else {
++ tl = convert_pixel(row1, 0);
++ bl = convert_pixel(row2, 0);
++ }
++
++ if (x1 == src_width - 1) {
++ tr = 0;
++ br = 0;
++ } else {
++ tr = convert_pixel(row1, 1);
++ br = convert_pixel(row2, 1);
++ }
++
++ b[i] = bilinear_interpolation(tl, tr, bl, br, fx, fy);
++
++next:
++ x += ux;
++ y += uy;
++ }
++ }
++}
+diff --git a/src/sna/brw/brw_eu_emit.c b/src/sna/brw/brw_eu_emit.c
+index 00c984d9..154f939a 100644
+--- a/src/sna/brw/brw_eu_emit.c
++++ b/src/sna/brw/brw_eu_emit.c
+@@ -178,7 +178,7 @@ validate_reg(struct brw_instruction *insn, struct brw_reg reg)
+ }
+
+ if (reg.file == BRW_ARCHITECTURE_REGISTER_FILE &&
+- reg.file == BRW_ARF_NULL)
++ reg.nr == BRW_ARF_NULL)
+ return;
+
+ assert(reg.hstride >= 0 && reg.hstride < ARRAY_SIZE(hstride_for_reg));
+@@ -700,7 +700,7 @@ push_if_stack(struct brw_compile *p, struct brw_instruction *inst)
+ *
+ * When the matching 'else' instruction is reached (presumably by
+ * countdown of the instruction count patched in by our ELSE/ENDIF
+- * functions), the relevent flags are inverted.
++ * functions), the relevant flags are inverted.
+ *
+ * When the matching 'endif' instruction is reached, the flags are
+ * popped off. If the stack is now empty, normal execution resumes.
+diff --git a/src/sna/compiler.h b/src/sna/compiler.h
+index ff412179..0f3775ec 100644
+--- a/src/sna/compiler.h
++++ b/src/sna/compiler.h
+@@ -39,6 +39,7 @@
+ #define pure __attribute__((pure))
+ #define tightly_packed __attribute__((__packed__))
+ #define flatten __attribute__((flatten))
++#define nonnull __attribute__((nonnull))
+ #define page_aligned __attribute__((aligned(4096)))
+ #else
+ #define likely(expr) (expr)
+@@ -51,18 +52,15 @@
+ #define pure
+ #define tighly_packed
+ #define flatten
++#define nonnull
+ #define page_aligned
+ #endif
+
+ #define HAS_GCC(major, minor) defined(__GNUC__) && (__GNUC__ > (major) || __GNUC__ == (major) && __GNUC_MINOR__ >= (minor))
+
+ #if HAS_GCC(4, 5)
+-#define sse2 __attribute__((target("sse2,fpmath=sse")))
+-#define sse4_2 __attribute__((target("sse4.2,sse2,fpmath=sse")))
+-#endif
+-
+-#if HAS_GCC(4, 7)
+-#define avx2 __attribute__((target("avx2,sse4.2,sse2,fpmath=sse")))
++#define sse2 fast __attribute__((target("sse2,fpmath=sse")))
++#define sse4_2 fast __attribute__((target("sse4.2,sse2,fpmath=sse")))
+ #endif
+
+ #if HAS_GCC(4, 6) && defined(__OPTIMIZE__)
+@@ -71,10 +69,17 @@
+ #define fast
+ #endif
+
+-#if HAS_GCC(4, 6) && defined(__OPTIMIZE__)
+-#define fast_memcpy __attribute__((optimize("Ofast"))) __attribute__((target("inline-all-stringops")))
+-#elif HAS_GCC(4, 5) && defined(__OPTIMIZE__)
+-#define fast_memcpy __attribute__((target("inline-all-stringops")))
++#if HAS_GCC(4, 7)
++#define avx2 fast __attribute__((target("avx2,avx,sse4.2,sse2,fpmath=sse")))
++#define assume_aligned(ptr, align) __builtin_assume_aligned((ptr), (align))
++#define assume_misaligned(ptr, align, offset) __builtin_assume_aligned((ptr), (align), (offset))
++#else
++#define assume_aligned(ptr, align) (ptr)
++#define assume_misaligned(ptr, align, offset) (ptr)
++#endif
++
++#if HAS_GCC(4, 5) && defined(__OPTIMIZE__)
++#define fast_memcpy fast __attribute__((target("inline-all-stringops")))
+ #else
+ #define fast_memcpy
+ #endif
+diff --git a/src/sna/fb/fb.h b/src/sna/fb/fb.h
+index 8bf9008a..90431747 100644
+--- a/src/sna/fb/fb.h
++++ b/src/sna/fb/fb.h
+@@ -24,10 +24,6 @@
+ #ifndef FB_H
+ #define FB_H
+
+-#ifdef HAVE_CONFIG_H
+-#include "config.h"
+-#endif
+-
+ #include <xorg-server.h>
+ #include <servermd.h>
+ #include <gcstruct.h>
+diff --git a/src/sna/fb/fbimage.c b/src/sna/fb/fbimage.c
+index 5af23890..cc81c85b 100644
+--- a/src/sna/fb/fbimage.c
++++ b/src/sna/fb/fbimage.c
+@@ -229,13 +229,19 @@ fbGetImage(DrawablePtr drawable,
+ FbBits pm;
+
+ pm = fbReplicatePixel(planeMask, srcBpp);
++
+ dstStride = PixmapBytePad(w, drawable->depth);
+- if (pm != FB_ALLONES)
+- memset(d, 0, dstStride * h);
+ dstStride /= sizeof(FbStip);
++
+ fbBltStip((FbStip *)(src + (y + srcYoff) * srcStride), srcStride,
+ (x + srcXoff) * srcBpp,
+- dst, dstStride, 0, w * srcBpp, h, GXcopy, pm, srcBpp);
++ dst, dstStride, 0, w * srcBpp, h, GXcopy, FB_ALLONES, srcBpp);
++
++ if (pm != FB_ALLONES) {
++ int i = dstStride * h;
++ while (i--)
++ *dst++ &= pm;
++ }
+ } else {
+ dstStride = BitmapBytePad(w) / sizeof(FbStip);
+ fbBltPlane(src + (y + srcYoff) * srcStride,
+diff --git a/src/sna/fb/fbpict.h b/src/sna/fb/fbpict.h
+index 932032f9..20877777 100644
+--- a/src/sna/fb/fbpict.h
++++ b/src/sna/fb/fbpict.h
+@@ -24,10 +24,6 @@
+ #ifndef FBPICT_H
+ #define FBPICT_H
+
+-#ifdef HAVE_CONFIG_H
+-#include "config.h"
+-#endif
+-
+ #include <xorg-server.h>
+ #include <picturestr.h>
+
+diff --git a/src/sna/gen2_render.c b/src/sna/gen2_render.c
+index 1104f462..49ad16a3 100644
+--- a/src/sna/gen2_render.c
++++ b/src/sna/gen2_render.c
+@@ -35,6 +35,7 @@
+ #include "sna_reg.h"
+ #include "sna_render.h"
+ #include "sna_render_inline.h"
++#include "sna_video.h"
+
+ #include "gen2_render.h"
+
+@@ -48,6 +49,7 @@
+
+ #define MAX_3D_SIZE 2048
+ #define MAX_3D_PITCH 8192
++#define MAX_INLINE (1 << 18)
+
+ #define BATCH(v) batch_emit(sna, v)
+ #define BATCH_F(v) batch_emit_float(sna, v)
+@@ -596,39 +598,43 @@ gen2_get_batch(struct sna *sna, const struct sna_composite_op *op)
+ gen2_emit_invariant(sna);
+ }
+
+-static void gen2_emit_target(struct sna *sna, const struct sna_composite_op *op)
++static void gen2_emit_target(struct sna *sna,
++ struct kgem_bo *bo,
++ int width,
++ int height,
++ int format)
+ {
+- assert(!too_large(op->dst.width, op->dst.height));
+- assert(op->dst.bo->pitch >= 8 && op->dst.bo->pitch <= MAX_3D_PITCH);
++ assert(!too_large(width, height));
++ assert(bo->pitch >= 8 && bo->pitch <= MAX_3D_PITCH);
+ assert(sna->render.vertex_offset == 0);
+
+- assert(op->dst.bo->unique_id);
+- if (sna->render_state.gen2.target == op->dst.bo->unique_id) {
+- kgem_bo_mark_dirty(op->dst.bo);
++ assert(bo->unique_id);
++ if (sna->render_state.gen2.target == bo->unique_id) {
++ kgem_bo_mark_dirty(bo);
+ return;
+ }
+
+ BATCH(_3DSTATE_BUF_INFO_CMD);
+ BATCH(BUF_3D_ID_COLOR_BACK |
+- gen2_buf_tiling(op->dst.bo->tiling) |
+- BUF_3D_PITCH(op->dst.bo->pitch));
++ gen2_buf_tiling(bo->tiling) |
++ BUF_3D_PITCH(bo->pitch));
+ BATCH(kgem_add_reloc(&sna->kgem, sna->kgem.nbatch,
+- op->dst.bo,
++ bo,
+ I915_GEM_DOMAIN_RENDER << 16 |
+ I915_GEM_DOMAIN_RENDER,
+ 0));
+
+ BATCH(_3DSTATE_DST_BUF_VARS_CMD);
+- BATCH(gen2_get_dst_format(op->dst.format));
++ BATCH(gen2_get_dst_format(format));
+
+ BATCH(_3DSTATE_DRAW_RECT_CMD);
+ BATCH(0);
+ BATCH(0); /* ymin, xmin */
+- BATCH(DRAW_YMAX(op->dst.height - 1) |
+- DRAW_XMAX(op->dst.width - 1));
++ BATCH(DRAW_YMAX(height - 1) |
++ DRAW_XMAX(width - 1));
+ BATCH(0); /* yorig, xorig */
+
+- sna->render_state.gen2.target = op->dst.bo->unique_id;
++ sna->render_state.gen2.target = bo->unique_id;
+ }
+
+ static void gen2_disable_logic_op(struct sna *sna)
+@@ -701,7 +707,11 @@ static void gen2_emit_composite_state(struct sna *sna,
+ kgem_clear_dirty(&sna->kgem);
+ }
+
+- gen2_emit_target(sna, op);
++ gen2_emit_target(sna,
++ op->dst.bo,
++ op->dst.width,
++ op->dst.height,
++ op->dst.format);
+
+ unwind = sna->kgem.nbatch;
+ BATCH(_3DSTATE_LOAD_STATE_IMMEDIATE_1 |
+@@ -1190,7 +1200,13 @@ inline static int gen2_get_rectangles(struct sna *sna,
+ sna->render.vertex_offset = sna->kgem.nbatch;
+ BATCH(PRIM3D_INLINE | PRIM3D_RECTLIST);
+ }
+- }
++
++ need = 0;
++ } else
++ need = sna->kgem.nbatch - sna->render.vertex_offset;
++
++ if (rem > MAX_INLINE - need)
++ rem = MAX_INLINE -need;
+
+ if (want > 1 && want * size > rem)
+ want = rem / size;
+@@ -1572,12 +1588,12 @@ gen2_composite_picture(struct sna *sna,
+ if (channel->repeat &&
+ (x >= 0 &&
+ y >= 0 &&
+- x + w < pixmap->drawable.width &&
+- y + h < pixmap->drawable.height)) {
++ x + w <= pixmap->drawable.width &&
++ y + h <= pixmap->drawable.height)) {
+ struct sna_pixmap *priv = sna_pixmap(pixmap);
+ if (priv && priv->clear) {
+ DBG(("%s: converting large pixmap source into solid [%08x]\n", __FUNCTION__, priv->clear_color));
+- return gen2_composite_solid_init(sna, channel, priv->clear_color);
++ return gen2_composite_solid_init(sna, channel, solid_color(picture->format, priv->clear_color));
+ }
+ }
+ } else
+@@ -1619,7 +1635,9 @@ gen2_composite_set_target(struct sna *sna,
+ } else
+ sna_render_picture_extents(dst, &box);
+
+- hint = PREFER_GPU | FORCE_GPU | RENDER_GPU;
++ hint = PREFER_GPU | RENDER_GPU;
++ if (!need_tiling(sna, op->dst.width, op->dst.height))
++ hint |= FORCE_GPU;
+ if (!partial) {
+ hint |= IGNORE_DAMAGE;
+ if (w == op->dst.width && h == op->dst.height)
+@@ -2423,7 +2441,11 @@ static void gen2_emit_composite_spans_state(struct sna *sna,
+ uint32_t unwind;
+
+ gen2_get_batch(sna, &op->base);
+- gen2_emit_target(sna, &op->base);
++ gen2_emit_target(sna,
++ op->base.dst.bo,
++ op->base.dst.width,
++ op->base.dst.height,
++ op->base.dst.format);
+
+ unwind = sna->kgem.nbatch;
+ BATCH(_3DSTATE_LOAD_STATE_IMMEDIATE_1 |
+@@ -2706,7 +2728,11 @@ static void gen2_emit_fill_composite_state(struct sna *sna,
+ uint32_t ls1;
+
+ gen2_get_batch(sna, op);
+- gen2_emit_target(sna, op);
++ gen2_emit_target(sna,
++ op->dst.bo,
++ op->dst.width,
++ op->dst.height,
++ op->dst.format);
+
+ ls1 = sna->kgem.nbatch;
+ BATCH(_3DSTATE_LOAD_STATE_IMMEDIATE_1 |
+@@ -2868,7 +2894,11 @@ static void gen2_emit_fill_state(struct sna *sna,
+ uint32_t ls1;
+
+ gen2_get_batch(sna, op);
+- gen2_emit_target(sna, op);
++ gen2_emit_target(sna,
++ op->dst.bo,
++ op->dst.width,
++ op->dst.height,
++ op->dst.format);
+
+ ls1 = sna->kgem.nbatch;
+ BATCH(_3DSTATE_LOAD_STATE_IMMEDIATE_1 |
+@@ -3102,6 +3132,276 @@ gen2_render_fill_one(struct sna *sna, PixmapPtr dst, struct kgem_bo *bo,
+ }
+
+ static void
++gen2_emit_video_state(struct sna *sna,
++ struct sna_video *video,
++ struct sna_video_frame *frame,
++ PixmapPtr pixmap,
++ struct kgem_bo *dst_bo,
++ int width, int height,
++ bool bilinear)
++{
++ uint32_t ms1, v, unwind;
++
++ gen2_emit_target(sna, dst_bo, width, height,
++ sna_format_for_depth(pixmap->drawable.depth));
++
++ unwind = sna->kgem.nbatch;
++ BATCH(_3DSTATE_LOAD_STATE_IMMEDIATE_1 |
++ I1_LOAD_S(2) | I1_LOAD_S(3) | I1_LOAD_S(8) | 2);
++ BATCH(1 << 12);
++ BATCH(S3_CULLMODE_NONE | S3_VERTEXHAS_XY);
++ BATCH(S8_ENABLE_COLOR_BUFFER_WRITE);
++ if (memcmp(sna->kgem.batch + sna->render_state.gen2.ls1 + 1,
++ sna->kgem.batch + unwind + 1,
++ 3 * sizeof(uint32_t)) == 0)
++ sna->kgem.nbatch = unwind;
++ else
++ sna->render_state.gen2.ls1 = unwind;
++
++ gen2_disable_logic_op(sna);
++
++ unwind = sna->kgem.nbatch;
++ BATCH(_3DSTATE_LOAD_STATE_IMMEDIATE_2 |
++ LOAD_TEXTURE_BLEND_STAGE(0) | 1);
++ BATCH(TB0C_LAST_STAGE | TB0C_RESULT_SCALE_1X | TB0C_OUTPUT_WRITE_CURRENT |
++ TB0C_OP_ARG1 | TB0C_ARG1_SEL_TEXEL0);
++ BATCH(TB0A_RESULT_SCALE_1X | TB0A_OUTPUT_WRITE_CURRENT |
++ TB0A_OP_ARG1 | TB0A_ARG1_SEL_ONE);
++ if (memcmp(sna->kgem.batch + sna->render_state.gen2.ls2 + 1,
++ sna->kgem.batch + unwind + 1,
++ 2 * sizeof(uint32_t)) == 0)
++ sna->kgem.nbatch = unwind;
++ else
++ sna->render_state.gen2.ls2 = unwind;
++
++ BATCH(_3DSTATE_LOAD_STATE_IMMEDIATE_2 | LOAD_TEXTURE_MAP(0) | 4);
++ BATCH(kgem_add_reloc(&sna->kgem, sna->kgem.nbatch,
++ frame->bo,
++ I915_GEM_DOMAIN_SAMPLER << 16,
++ 0));
++ ms1 = MAPSURF_422 | TM0S1_COLORSPACE_CONVERSION;
++ switch (frame->id) {
++ case FOURCC_YUY2:
++ ms1 |= MT_422_YCRCB_NORMAL;
++ break;
++ case FOURCC_UYVY:
++ ms1 |= MT_422_YCRCB_SWAPY;
++ break;
++ }
++ BATCH(((frame->height - 1) << TM0S1_HEIGHT_SHIFT) |
++ ((frame->width - 1) << TM0S1_WIDTH_SHIFT) |
++ ms1 |
++ gen2_sampler_tiling_bits(frame->bo->tiling));
++ BATCH((frame->pitch[0] / 4 - 1) << TM0S2_PITCH_SHIFT | TM0S2_MAP_2D);
++ if (bilinear)
++ BATCH(FILTER_LINEAR << TM0S3_MAG_FILTER_SHIFT |
++ FILTER_LINEAR << TM0S3_MIN_FILTER_SHIFT |
++ MIPFILTER_NONE << TM0S3_MIP_FILTER_SHIFT);
++ else
++ BATCH(FILTER_NEAREST << TM0S3_MAG_FILTER_SHIFT |
++ FILTER_NEAREST << TM0S3_MIN_FILTER_SHIFT |
++ MIPFILTER_NONE << TM0S3_MIP_FILTER_SHIFT);
++ BATCH(0); /* default color */
++
++ BATCH(_3DSTATE_MAP_COORD_SET_CMD | TEXCOORD_SET(0) |
++ ENABLE_TEXCOORD_PARAMS | TEXCOORDS_ARE_NORMAL | TEXCOORDTYPE_CARTESIAN |
++ ENABLE_ADDR_V_CNTL | TEXCOORD_ADDR_V_MODE(TEXCOORDMODE_CLAMP) |
++ ENABLE_ADDR_U_CNTL | TEXCOORD_ADDR_U_MODE(TEXCOORDMODE_CLAMP));
++
++ v = _3DSTATE_VERTEX_FORMAT_2_CMD | TEXCOORDFMT_2D;
++ if (sna->render_state.gen2.vft != v) {
++ BATCH(v);
++ sna->render_state.gen2.vft = v;
++ }
++}
++
++static void
++gen2_video_get_batch(struct sna *sna, struct kgem_bo *bo)
++{
++ kgem_set_mode(&sna->kgem, KGEM_RENDER, bo);
++
++ if (!kgem_check_batch(&sna->kgem, 120) ||
++ !kgem_check_reloc(&sna->kgem, 4) ||
++ !kgem_check_exec(&sna->kgem, 2)) {
++ _kgem_submit(&sna->kgem);
++ _kgem_set_mode(&sna->kgem, KGEM_RENDER);
++ }
++
++ if (sna->render_state.gen2.need_invariant)
++ gen2_emit_invariant(sna);
++}
++
++static int
++gen2_get_inline_rectangles(struct sna *sna, int want, int floats_per_vertex)
++{
++ int size = floats_per_vertex * 3;
++ int rem = batch_space(sna) - 1;
++
++ if (rem > MAX_INLINE)
++ rem = MAX_INLINE;
++
++ if (size * want > rem)
++ want = rem / size;
++
++ return want;
++}
++
++static bool
++gen2_render_video(struct sna *sna,
++ struct sna_video *video,
++ struct sna_video_frame *frame,
++ RegionPtr dstRegion,
++ PixmapPtr pixmap)
++{
++ struct sna_pixmap *priv = sna_pixmap(pixmap);
++ const BoxRec *pbox = region_rects(dstRegion);
++ int nbox = region_num_rects(dstRegion);
++ int dst_width = dstRegion->extents.x2 - dstRegion->extents.x1;
++ int dst_height = dstRegion->extents.y2 - dstRegion->extents.y1;
++ int src_width = frame->src.x2 - frame->src.x1;
++ int src_height = frame->src.y2 - frame->src.y1;
++ float src_offset_x, src_offset_y;
++ float src_scale_x, src_scale_y;
++ int pix_xoff, pix_yoff;
++ struct kgem_bo *dst_bo;
++ bool bilinear;
++ int copy = 0;
++
++ DBG(("%s: src:%dx%d (frame:%dx%d) -> dst:%dx%d\n", __FUNCTION__,
++ src_width, src_height, frame->width, frame->height, dst_width, dst_height));
++
++ assert(priv->gpu_bo);
++ dst_bo = priv->gpu_bo;
++
++ bilinear = src_width != dst_width || src_height != dst_height;
++
++ src_scale_x = (float)src_width / dst_width / frame->width;
++ src_offset_x = (float)frame->src.x1 / frame->width - dstRegion->extents.x1 * src_scale_x;
++
++ src_scale_y = (float)src_height / dst_height / frame->height;
++ src_offset_y = (float)frame->src.y1 / frame->height - dstRegion->extents.y1 * src_scale_y;
++ DBG(("%s: src offset (%f, %f), scale (%f, %f)\n",
++ __FUNCTION__, src_offset_x, src_offset_y, src_scale_x, src_scale_y));
++
++ if (too_large(pixmap->drawable.width, pixmap->drawable.height) ||
++ dst_bo->pitch > MAX_3D_PITCH) {
++ int bpp = pixmap->drawable.bitsPerPixel;
++
++ if (too_large(dst_width, dst_height))
++ return false;
++
++ dst_bo = kgem_create_2d(&sna->kgem,
++ dst_width, dst_height, bpp,
++ kgem_choose_tiling(&sna->kgem,
++ I915_TILING_X,
++ dst_width, dst_height, bpp),
++ 0);
++ if (!dst_bo)
++ return false;
++
++ pix_xoff = -dstRegion->extents.x1;
++ pix_yoff = -dstRegion->extents.y1;
++ copy = 1;
++ } else {
++ /* Set up the offset for translating from the given region
++ * (in screen coordinates) to the backing pixmap.
++ */
++#ifdef COMPOSITE
++ pix_xoff = -pixmap->screen_x + pixmap->drawable.x;
++ pix_yoff = -pixmap->screen_y + pixmap->drawable.y;
++#else
++ pix_xoff = 0;
++ pix_yoff = 0;
++#endif
++
++ dst_width = pixmap->drawable.width;
++ dst_height = pixmap->drawable.height;
++ }
++
++ gen2_video_get_batch(sna, dst_bo);
++ gen2_emit_video_state(sna, video, frame, pixmap,
++ dst_bo, dst_width, dst_height, bilinear);
++ do {
++ int nbox_this_time = gen2_get_inline_rectangles(sna, nbox, 4);
++ if (nbox_this_time == 0) {
++ gen2_video_get_batch(sna, dst_bo);
++ gen2_emit_video_state(sna, video, frame, pixmap,
++ dst_bo, dst_width, dst_height, bilinear);
++ nbox_this_time = gen2_get_inline_rectangles(sna, nbox, 4);
++ assert(nbox_this_time);
++ }
++ nbox -= nbox_this_time;
++
++ BATCH(PRIM3D_INLINE | PRIM3D_RECTLIST |
++ ((12 * nbox_this_time) - 1));
++ do {
++ int box_x1 = pbox->x1;
++ int box_y1 = pbox->y1;
++ int box_x2 = pbox->x2;
++ int box_y2 = pbox->y2;
++
++ pbox++;
++
++ DBG(("%s: dst (%d, %d), (%d, %d) + (%d, %d); src (%f, %f), (%f, %f)\n",
++ __FUNCTION__, box_x1, box_y1, box_x2, box_y2, pix_xoff, pix_yoff,
++ box_x1 * src_scale_x + src_offset_x,
++ box_y1 * src_scale_y + src_offset_y,
++ box_x2 * src_scale_x + src_offset_x,
++ box_y2 * src_scale_y + src_offset_y));
++
++ /* bottom right */
++ BATCH_F(box_x2 + pix_xoff);
++ BATCH_F(box_y2 + pix_yoff);
++ BATCH_F(box_x2 * src_scale_x + src_offset_x);
++ BATCH_F(box_y2 * src_scale_y + src_offset_y);
++
++ /* bottom left */
++ BATCH_F(box_x1 + pix_xoff);
++ BATCH_F(box_y2 + pix_yoff);
++ BATCH_F(box_x1 * src_scale_x + src_offset_x);
++ BATCH_F(box_y2 * src_scale_y + src_offset_y);
++
++ /* top left */
++ BATCH_F(box_x1 + pix_xoff);
++ BATCH_F(box_y1 + pix_yoff);
++ BATCH_F(box_x1 * src_scale_x + src_offset_x);
++ BATCH_F(box_y1 * src_scale_y + src_offset_y);
++ } while (--nbox_this_time);
++ } while (nbox);
++
++ if (copy) {
++#ifdef COMPOSITE
++ pix_xoff = -pixmap->screen_x + pixmap->drawable.x;
++ pix_yoff = -pixmap->screen_y + pixmap->drawable.y;
++#else
++ pix_xoff = 0;
++ pix_yoff = 0;
++#endif
++ sna_blt_copy_boxes(sna, GXcopy,
++ dst_bo, -dstRegion->extents.x1, -dstRegion->extents.y1,
++ priv->gpu_bo, pix_xoff, pix_yoff,
++ pixmap->drawable.bitsPerPixel,
++ region_rects(dstRegion),
++ region_num_rects(dstRegion));
++
++ kgem_bo_destroy(&sna->kgem, dst_bo);
++ }
++
++ if (!DAMAGE_IS_ALL(priv->gpu_damage)) {
++ if ((pix_xoff | pix_yoff) == 0) {
++ sna_damage_add(&priv->gpu_damage, dstRegion);
++ } else {
++ sna_damage_add_boxes(&priv->gpu_damage,
++ region_rects(dstRegion),
++ region_num_rects(dstRegion),
++ pix_xoff, pix_yoff);
++ }
++ }
++
++ return true;
++}
++
++static void
+ gen2_render_copy_setup_source(struct sna_composite_channel *channel,
+ const DrawableRec *draw,
+ struct kgem_bo *bo)
+@@ -3176,7 +3476,11 @@ static void gen2_emit_copy_state(struct sna *sna, const struct sna_composite_op
+ PIPELINE_FLUSH_TEXTURE_CACHE);
+ kgem_clear_dirty(&sna->kgem);
+ }
+- gen2_emit_target(sna, op);
++ gen2_emit_target(sna,
++ op->dst.bo,
++ op->dst.width,
++ op->dst.height,
++ op->dst.format);
+
+ ls1 = sna->kgem.nbatch;
+ BATCH(_3DSTATE_LOAD_STATE_IMMEDIATE_1 |
+@@ -3511,7 +3815,7 @@ const char *gen2_render_init(struct sna *sna, const char *backend)
+ render->copy = gen2_render_copy;
+ render->copy_boxes = gen2_render_copy_boxes;
+
+- /* XXX YUV color space conversion for video? */
++ render->video = gen2_render_video;
+
+ render->reset = gen2_render_reset;
+ render->flush = gen2_render_flush;
+diff --git a/src/sna/gen3_render.c b/src/sna/gen3_render.c
+index 78289f00..4459a562 100644
+--- a/src/sna/gen3_render.c
++++ b/src/sna/gen3_render.c
+@@ -448,14 +448,14 @@ gen3_emit_composite_boxes_constant(const struct sna_composite_op *op,
+ float *v)
+ {
+ do {
+- v[0] = box->x2;
+- v[1] = box->y2;
++ v[0] = box->x2 + op->dst.x;
++ v[1] = box->y2 + op->dst.y;
+
+- v[2] = box->x1;
+- v[3] = box->y2;
++ v[2] = box->x1 + op->dst.x;
++ v[3] = box->y2 + op->dst.y;
+
+- v[4] = box->x1;
+- v[5] = box->y1;
++ v[4] = box->x1 + op->dst.x;
++ v[5] = box->y1 + op->dst.y;
+
+ box++;
+ v += 6;
+@@ -494,18 +494,18 @@ gen3_emit_composite_boxes_identity_gradient(const struct sna_composite_op *op,
+ float *v)
+ {
+ do {
+- v[0] = box->x2;
+- v[1] = box->y2;
++ v[0] = box->x2 + op->dst.x;
++ v[1] = box->y2 + op->dst.y;
+ v[2] = box->x2 + op->src.offset[0];
+ v[3] = box->y2 + op->src.offset[1];
+
+- v[4] = box->x1;
+- v[5] = box->y2;
++ v[4] = box->x1 + op->dst.x;
++ v[5] = box->y2 + op->dst.y;
+ v[6] = box->x1 + op->src.offset[0];
+ v[7] = box->y2 + op->src.offset[1];
+
+- v[8] = box->x1;
+- v[9] = box->y1;
++ v[8] = box->x1 + op->dst.x;
++ v[9] = box->y1 + op->dst.y;
+ v[10] = box->x1 + op->src.offset[0];
+ v[11] = box->y1 + op->src.offset[1];
+
+@@ -531,6 +531,7 @@ gen3_emit_composite_primitive_affine_gradient(struct sna *sna,
+
+ v = sna->render.vertices + sna->render.vertex_used;
+ sna->render.vertex_used += 12;
++ assert(sna->render.vertex_used <= sna->render.vertex_size);
+
+ v[0] = dst_x + r->width;
+ v[1] = dst_y + r->height;
+@@ -559,22 +560,22 @@ gen3_emit_composite_boxes_affine_gradient(const struct sna_composite_op *op,
+ const PictTransform *transform = op->src.transform;
+
+ do {
+- v[0] = box->x2;
+- v[1] = box->y2;
++ v[0] = box->x2 + op->dst.x;
++ v[1] = box->y2 + op->dst.y;
+ _sna_get_transformed_scaled(box->x2 + op->src.offset[0],
+ box->y2 + op->src.offset[1],
+ transform, op->src.scale,
+ &v[2], &v[3]);
+
+- v[4] = box->x1;
+- v[5] = box->y2;
++ v[4] = box->x1 + op->dst.x;
++ v[5] = box->y2 + op->dst.y;
+ _sna_get_transformed_scaled(box->x1 + op->src.offset[0],
+ box->y2 + op->src.offset[1],
+ transform, op->src.scale,
+ &v[6], &v[7]);
+
+- v[8] = box->x1;
+- v[9] = box->y1;
++ v[8] = box->x1 + op->dst.x;
++ v[9] = box->y1 + op->dst.y;
+ _sna_get_transformed_scaled(box->x1 + op->src.offset[0],
+ box->y1 + op->src.offset[1],
+ transform, op->src.scale,
+@@ -596,6 +597,7 @@ gen3_emit_composite_primitive_identity_source(struct sna *sna,
+
+ v = sna->render.vertices + sna->render.vertex_used;
+ sna->render.vertex_used += 12;
++ assert(sna->render.vertex_used <= sna->render.vertex_size);
+
+ v[8] = v[4] = r->dst.x + op->dst.x;
+ v[0] = v[4] + w;
+@@ -643,6 +645,7 @@ gen3_emit_composite_primitive_identity_source_no_offset(struct sna *sna,
+
+ v = sna->render.vertices + sna->render.vertex_used;
+ sna->render.vertex_used += 12;
++ assert(sna->render.vertex_used <= sna->render.vertex_size);
+
+ v[8] = v[4] = r->dst.x;
+ v[9] = r->dst.y;
+@@ -693,6 +696,7 @@ gen3_emit_composite_primitive_affine_source(struct sna *sna,
+
+ v = sna->render.vertices + sna->render.vertex_used;
+ sna->render.vertex_used += 12;
++ assert(sna->render.vertex_used <= sna->render.vertex_size);
+
+ v[0] = dst_x + r->width;
+ v[5] = v[1] = dst_y + r->height;
+@@ -720,10 +724,10 @@ gen3_emit_composite_boxes_affine_source(const struct sna_composite_op *op,
+ const PictTransform *transform = op->src.transform;
+
+ do {
+- v[0] = box->x2;
+- v[5] = v[1] = box->y2;
+- v[8] = v[4] = box->x1;
+- v[9] = box->y1;
++ v[0] = box->x2 + op->dst.x;
++ v[5] = v[1] = box->y2 + op->dst.y;
++ v[8] = v[4] = box->x1 + op->dst.x;
++ v[9] = box->y1 + op->dst.y;
+
+ _sna_get_transformed_scaled(box->x2 + op->src.offset[0],
+ box->y2 + op->src.offset[1],
+@@ -756,6 +760,7 @@ gen3_emit_composite_primitive_constant_identity_mask(struct sna *sna,
+
+ v = sna->render.vertices + sna->render.vertex_used;
+ sna->render.vertex_used += 12;
++ assert(sna->render.vertex_used <= sna->render.vertex_size);
+
+ v[8] = v[4] = r->dst.x + op->dst.x;
+ v[0] = v[4] + w;
+@@ -781,6 +786,7 @@ gen3_emit_composite_primitive_constant_identity_mask_no_offset(struct sna *sna,
+
+ v = sna->render.vertices + sna->render.vertex_used;
+ sna->render.vertex_used += 12;
++ assert(sna->render.vertex_used <= sna->render.vertex_size);
+
+ v[8] = v[4] = r->dst.x;
+ v[9] = r->dst.y;
+@@ -817,6 +823,7 @@ gen3_emit_composite_primitive_identity_source_mask(struct sna *sna,
+
+ v = sna->render.vertices + sna->render.vertex_used;
+ sna->render.vertex_used += 18;
++ assert(sna->render.vertex_used <= sna->render.vertex_size);
+
+ v[0] = dst_x + w;
+ v[1] = dst_y + h;
+@@ -862,6 +869,7 @@ gen3_emit_composite_primitive_affine_source_mask(struct sna *sna,
+
+ v = sna->render.vertices + sna->render.vertex_used;
+ sna->render.vertex_used += 18;
++ assert(sna->render.vertex_used <= sna->render.vertex_size);
+
+ v[0] = dst_x + w;
+ v[1] = dst_y + h;
+@@ -978,6 +986,7 @@ gen3_emit_composite_primitive_constant__sse2(struct sna *sna,
+
+ v = sna->render.vertices + sna->render.vertex_used;
+ sna->render.vertex_used += 6;
++ assert(sna->render.vertex_used <= sna->render.vertex_size);
+
+ v[4] = v[2] = r->dst.x + op->dst.x;
+ v[5] = r->dst.y + op->dst.y;
+@@ -993,10 +1002,10 @@ gen3_emit_composite_boxes_constant__sse2(const struct sna_composite_op *op,
+ float *v)
+ {
+ do {
+- v[0] = box->x2;
+- v[3] = v[1] = box->y2;
+- v[4] = v[2] = box->x1;
+- v[5] = box->y1;
++ v[0] = box->x2 + op->dst.x;
++ v[3] = v[1] = box->y2 + op->dst.y;
++ v[4] = v[2] = box->x1 + op->dst.x;
++ v[5] = box->y1 + op->dst.y;
+
+ box++;
+ v += 6;
+@@ -1013,6 +1022,7 @@ gen3_emit_composite_primitive_identity_gradient__sse2(struct sna *sna,
+
+ v = sna->render.vertices + sna->render.vertex_used;
+ sna->render.vertex_used += 12;
++ assert(sna->render.vertex_used <= sna->render.vertex_size);
+
+ x = r->dst.x + op->dst.x;
+ y = r->dst.y + op->dst.y;
+@@ -1035,10 +1045,10 @@ gen3_emit_composite_boxes_identity_gradient__sse2(const struct sna_composite_op
+ float *v)
+ {
+ do {
+- v[0] = box->x2;
+- v[5] = v[1] = box->y2;
+- v[8] = v[4] = box->x1;
+- v[9] = box->y1;
++ v[0] = box->x2 + op->dst.x;
++ v[5] = v[1] = box->y2 + op->dst.y;
++ v[8] = v[4] = box->x1 + op->dst.x;
++ v[9] = box->y1 + op->dst.y;
+
+ v[2] = box->x2 + op->src.offset[0];
+ v[7] = v[3] = box->y2 + op->src.offset[1];
+@@ -1067,6 +1077,7 @@ gen3_emit_composite_primitive_affine_gradient__sse2(struct sna *sna,
+
+ v = sna->render.vertices + sna->render.vertex_used;
+ sna->render.vertex_used += 12;
++ assert(sna->render.vertex_used <= sna->render.vertex_size);
+
+ v[0] = dst_x + r->width;
+ v[1] = dst_y + r->height;
+@@ -1095,22 +1106,22 @@ gen3_emit_composite_boxes_affine_gradient__sse2(const struct sna_composite_op *o
+ const PictTransform *transform = op->src.transform;
+
+ do {
+- v[0] = box->x2;
+- v[1] = box->y2;
++ v[0] = box->x2 + op->dst.x;
++ v[1] = box->y2 + op->dst.y;
+ _sna_get_transformed_scaled(box->x2 + op->src.offset[0],
+ box->y2 + op->src.offset[1],
+ transform, op->src.scale,
+ &v[2], &v[3]);
+
+- v[4] = box->x1;
+- v[5] = box->y2;
++ v[4] = box->x1 + op->dst.x;
++ v[5] = box->y2 + op->dst.y;
+ _sna_get_transformed_scaled(box->x1 + op->src.offset[0],
+ box->y2 + op->src.offset[1],
+ transform, op->src.scale,
+ &v[6], &v[7]);
+
+- v[8] = box->x1;
+- v[9] = box->y1;
++ v[8] = box->x1 + op->dst.x;
++ v[9] = box->y1 + op->dst.y;
+ _sna_get_transformed_scaled(box->x1 + op->src.offset[0],
+ box->y1 + op->src.offset[1],
+ transform, op->src.scale,
+@@ -1132,6 +1143,7 @@ gen3_emit_composite_primitive_identity_source__sse2(struct sna *sna,
+
+ v = sna->render.vertices + sna->render.vertex_used;
+ sna->render.vertex_used += 12;
++ assert(sna->render.vertex_used <= sna->render.vertex_size);
+
+ v[8] = v[4] = r->dst.x + op->dst.x;
+ v[0] = v[4] + w;
+@@ -1179,6 +1191,7 @@ gen3_emit_composite_primitive_identity_source_no_offset__sse2(struct sna *sna,
+
+ v = sna->render.vertices + sna->render.vertex_used;
+ sna->render.vertex_used += 12;
++ assert(sna->render.vertex_used <= sna->render.vertex_size);
+
+ v[8] = v[4] = r->dst.x;
+ v[9] = r->dst.y;
+@@ -1227,8 +1240,12 @@ gen3_emit_composite_primitive_affine_source__sse2(struct sna *sna,
+ int src_y = r->src.y + (int)op->src.offset[1];
+ float *v;
+
++ DBG(("%s: src=(%d, %d), dst=(%d, %d), size=%dx%d\n",
++ __FUNCTION__, src_x, src_y, dst_x, dst_y, r->width, r->height));
++
+ v = sna->render.vertices + sna->render.vertex_used;
+ sna->render.vertex_used += 12;
++ assert(sna->render.vertex_used <= sna->render.vertex_size);
+
+ v[0] = dst_x + r->width;
+ v[5] = v[1] = dst_y + r->height;
+@@ -1256,10 +1273,13 @@ gen3_emit_composite_boxes_affine_source__sse2(const struct sna_composite_op *op,
+ const PictTransform *transform = op->src.transform;
+
+ do {
+- v[0] = box->x2;
+- v[5] = v[1] = box->y2;
+- v[8] = v[4] = box->x1;
+- v[9] = box->y1;
++ DBG(("%s: box=(%d, %d), (%d, %d), src.offset=(%d, %d)\n",
++ __FUNCTION__, box->x1, box->y1, box->x2, box->y2, op->src.offset[0], op->src.offset[1]));
++
++ v[0] = box->x2 + op->dst.x;
++ v[5] = v[1] = box->y2 + op->dst.y;
++ v[8] = v[4] = box->x1 + op->dst.x;
++ v[9] = box->y1 + op->dst.y;
+
+ _sna_get_transformed_scaled(box->x2 + op->src.offset[0],
+ box->y2 + op->src.offset[1],
+@@ -1292,6 +1312,7 @@ gen3_emit_composite_primitive_constant_identity_mask__sse2(struct sna *sna,
+
+ v = sna->render.vertices + sna->render.vertex_used;
+ sna->render.vertex_used += 12;
++ assert(sna->render.vertex_used <= sna->render.vertex_size);
+
+ v[8] = v[4] = r->dst.x + op->dst.x;
+ v[0] = v[4] + w;
+@@ -1317,6 +1338,7 @@ gen3_emit_composite_primitive_constant_identity_mask_no_offset__sse2(struct sna
+
+ v = sna->render.vertices + sna->render.vertex_used;
+ sna->render.vertex_used += 12;
++ assert(sna->render.vertex_used <= sna->render.vertex_size);
+
+ v[8] = v[4] = r->dst.x;
+ v[9] = r->dst.y;
+@@ -1353,6 +1375,7 @@ gen3_emit_composite_primitive_identity_source_mask__sse2(struct sna *sna,
+
+ v = sna->render.vertices + sna->render.vertex_used;
+ sna->render.vertex_used += 18;
++ assert(sna->render.vertex_used <= sna->render.vertex_size);
+
+ v[0] = dst_x + w;
+ v[1] = dst_y + h;
+@@ -1398,6 +1421,7 @@ gen3_emit_composite_primitive_affine_source_mask__sse2(struct sna *sna,
+
+ v = sna->render.vertices + sna->render.vertex_used;
+ sna->render.vertex_used += 18;
++ assert(sna->render.vertex_used <= sna->render.vertex_size);
+
+ v[0] = dst_x + w;
+ v[1] = dst_y + h;
+@@ -2233,6 +2257,7 @@ static void gen3_vertex_flush(struct sna *sna)
+ static int gen3_vertex_finish(struct sna *sna)
+ {
+ struct kgem_bo *bo;
++ unsigned hint, size;
+
+ DBG(("%s: used=%d/%d, vbo active? %d\n",
+ __FUNCTION__, sna->render.vertex_used, sna->render.vertex_size,
+@@ -2243,6 +2268,7 @@ static int gen3_vertex_finish(struct sna *sna)
+
+ sna_vertex_wait__locked(&sna->render);
+
++ hint = CREATE_GTT_MAP;
+ bo = sna->render.vbo;
+ if (bo) {
+ DBG(("%s: reloc = %d\n", __FUNCTION__,
+@@ -2251,7 +2277,7 @@ static int gen3_vertex_finish(struct sna *sna)
+ if (sna->render.vertex_reloc[0]) {
+ sna->kgem.batch[sna->render.vertex_reloc[0]] =
+ kgem_add_reloc(&sna->kgem, sna->render.vertex_reloc[0],
+- bo, I915_GEM_DOMAIN_VERTEX << 16, 0);
++ bo, I915_GEM_DOMAIN_VERTEX << 16 | KGEM_RELOC_FENCED, 0);
+
+ sna->render.vertex_reloc[0] = 0;
+ }
+@@ -2260,17 +2286,29 @@ static int gen3_vertex_finish(struct sna *sna)
+ sna->render.vbo = NULL;
+
+ kgem_bo_destroy(&sna->kgem, bo);
++ hint |= CREATE_CACHED | CREATE_NO_THROTTLE;
+ }
+
++ size = 256*1024;
+ sna->render.vertices = NULL;
+- sna->render.vbo = kgem_create_linear(&sna->kgem,
+- 256*1024, CREATE_GTT_MAP);
+- if (sna->render.vbo)
++ sna->render.vbo = kgem_create_linear(&sna->kgem, size, hint);
++ while (sna->render.vbo == NULL && size > sizeof(sna->render.vertex_data)) {
++ size /= 2;
++ sna->render.vbo = kgem_create_linear(&sna->kgem, size, hint);
++ }
++ if (sna->render.vbo == NULL)
++ sna->render.vbo = kgem_create_linear(&sna->kgem,
++ 256*1024, CREATE_GTT_MAP);
++ if (sna->render.vbo &&
++ kgem_check_bo(&sna->kgem, sna->render.vbo, NULL))
+ sna->render.vertices = kgem_bo_map(&sna->kgem, sna->render.vbo);
+ if (sna->render.vertices == NULL) {
+- if (sna->render.vbo)
++ if (sna->render.vbo) {
+ kgem_bo_destroy(&sna->kgem, sna->render.vbo);
+- sna->render.vbo = NULL;
++ sna->render.vbo = NULL;
++ }
++ sna->render.vertices = sna->render.vertex_data;
++ sna->render.vertex_size = ARRAY_SIZE(sna->render.vertex_data);
+ return 0;
+ }
+ assert(sna->render.vbo->snoop == false);
+@@ -2280,8 +2318,14 @@ static int gen3_vertex_finish(struct sna *sna)
+ sna->render.vertex_data,
+ sizeof(float)*sna->render.vertex_used);
+ }
+- sna->render.vertex_size = 64 * 1024 - 1;
+- return sna->render.vertex_size - sna->render.vertex_used;
++
++ size = __kgem_bo_size(sna->render.vbo)/4;
++ if (size >= UINT16_MAX)
++ size = UINT16_MAX - 1;
++ assert(size > sna->render.vertex_used);
++
++ sna->render.vertex_size = size;
++ return size - sna->render.vertex_used;
+ }
+
+ static void gen3_vertex_close(struct sna *sna)
+@@ -2345,7 +2389,7 @@ static void gen3_vertex_close(struct sna *sna)
+ DBG(("%s: reloc = %d\n", __FUNCTION__, sna->render.vertex_reloc[0]));
+ sna->kgem.batch[sna->render.vertex_reloc[0]] =
+ kgem_add_reloc(&sna->kgem, sna->render.vertex_reloc[0],
+- bo, I915_GEM_DOMAIN_VERTEX << 16, delta);
++ bo, I915_GEM_DOMAIN_VERTEX << 16 | KGEM_RELOC_FENCED, delta);
+ sna->render.vertex_reloc[0] = 0;
+
+ if (sna->render.vbo == NULL) {
+@@ -2580,6 +2624,7 @@ gen3_render_composite_boxes(struct sna *sna,
+
+ v = sna->render.vertices + sna->render.vertex_used;
+ sna->render.vertex_used += nbox_this_time * op->floats_per_rect;
++ assert(sna->render.vertex_used <= sna->render.vertex_size);
+
+ op->emit_boxes(op, box, nbox_this_time, v);
+ box += nbox_this_time;
+@@ -2604,6 +2649,7 @@ gen3_render_composite_boxes__thread(struct sna *sna,
+
+ v = sna->render.vertices + sna->render.vertex_used;
+ sna->render.vertex_used += nbox_this_time * op->floats_per_rect;
++ assert(sna->render.vertex_used <= sna->render.vertex_size);
+
+ sna_vertex_acquire__locked(&sna->render);
+ sna_vertex_unlock(&sna->render);
+@@ -3065,7 +3111,7 @@ gen3_composite_picture(struct sna *sna,
+
+ if (sna_picture_is_clear(picture, x, y, w, h, &color)) {
+ DBG(("%s: clear drawable [%08x]\n", __FUNCTION__, color));
+- return gen3_init_solid(channel, color_convert(color, picture->format, PICT_a8r8g8b8));
++ return gen3_init_solid(channel, solid_color(picture->format, color));
+ }
+
+ if (!gen3_check_repeat(picture))
+@@ -3097,12 +3143,12 @@ gen3_composite_picture(struct sna *sna,
+ if (channel->repeat ||
+ (x >= 0 &&
+ y >= 0 &&
+- x + w < pixmap->drawable.width &&
+- y + h < pixmap->drawable.height)) {
++ x + w <= pixmap->drawable.width &&
++ y + h <= pixmap->drawable.height)) {
+ struct sna_pixmap *priv = sna_pixmap(pixmap);
+ if (priv && priv->clear) {
+ DBG(("%s: converting large pixmap source into solid [%08x]\n", __FUNCTION__, priv->clear_color));
+- return gen3_init_solid(channel, priv->clear_color);
++ return gen3_init_solid(channel, solid_color(picture->format, priv->clear_color));
+ }
+ }
+ } else {
+@@ -3182,7 +3228,9 @@ gen3_composite_set_target(struct sna *sna,
+ } else
+ sna_render_picture_extents(dst, &box);
+
+- hint = PREFER_GPU | FORCE_GPU | RENDER_GPU;
++ hint = PREFER_GPU | RENDER_GPU;
++ if (!need_tiling(sna, op->dst.width, op->dst.height))
++ hint |= FORCE_GPU;
+ if (!partial) {
+ hint |= IGNORE_DAMAGE;
+ if (w == op->dst.width && h == op->dst.height)
+@@ -3645,8 +3693,11 @@ gen3_render_composite(struct sna *sna,
+ }
+ }
+ }
+- DBG(("%s: final src/mask type=%d/%d, affine=%d/%d\n", __FUNCTION__,
++ DBG(("%s: final src/mask type=%d/%d [constant? %d/%d], transform? %d/%d, affine=%d/%d\n", __FUNCTION__,
+ tmp->src.u.gen3.type, tmp->mask.u.gen3.type,
++ is_constant_ps(tmp->src.u.gen3.type),
++ is_constant_ps(tmp->mask.u.gen3.type),
++ !!tmp->src.transform, !!tmp->mask.transform,
+ tmp->src.is_affine, tmp->mask.is_affine));
+
+ tmp->prim_emit = gen3_emit_composite_primitive;
+@@ -3862,6 +3913,7 @@ gen3_emit_composite_spans_primitive_zero(struct sna *sna,
+ {
+ float *v = sna->render.vertices + sna->render.vertex_used;
+ sna->render.vertex_used += 6;
++ assert(sna->render.vertex_used <= sna->render.vertex_size);
+
+ v[0] = op->base.dst.x + box->x2;
+ v[1] = op->base.dst.y + box->y2;
+@@ -3901,6 +3953,7 @@ gen3_emit_composite_spans_primitive_zero_no_offset(struct sna *sna,
+ {
+ float *v = sna->render.vertices + sna->render.vertex_used;
+ sna->render.vertex_used += 6;
++ assert(sna->render.vertex_used <= sna->render.vertex_size);
+
+ v[0] = box->x2;
+ v[3] = v[1] = box->y2;
+@@ -3932,6 +3985,7 @@ gen3_emit_composite_spans_primitive_constant(struct sna *sna,
+ {
+ float *v = sna->render.vertices + sna->render.vertex_used;
+ sna->render.vertex_used += 9;
++ assert(sna->render.vertex_used <= sna->render.vertex_size);
+
+ v[0] = op->base.dst.x + box->x2;
+ v[6] = v[3] = op->base.dst.x + box->x1;
+@@ -3966,6 +4020,7 @@ gen3_emit_composite_spans_primitive_constant_no_offset(struct sna *sna,
+ {
+ float *v = sna->render.vertices + sna->render.vertex_used;
+ sna->render.vertex_used += 9;
++ assert(sna->render.vertex_used <= sna->render.vertex_size);
+
+ v[0] = box->x2;
+ v[6] = v[3] = box->x1;
+@@ -3999,6 +4054,7 @@ gen3_emit_composite_spans_primitive_identity_source(struct sna *sna,
+ {
+ float *v = sna->render.vertices + sna->render.vertex_used;
+ sna->render.vertex_used += 15;
++ assert(sna->render.vertex_used <= sna->render.vertex_size);
+
+ v[0] = op->base.dst.x + box->x2;
+ v[1] = op->base.dst.y + box->y2;
+@@ -4060,6 +4116,7 @@ gen3_emit_composite_spans_primitive_affine_source(struct sna *sna,
+
+ v = sna->render.vertices + sna->render.vertex_used;
+ sna->render.vertex_used += 15;
++ assert(sna->render.vertex_used <= sna->render.vertex_size);
+
+ v[0] = op->base.dst.x + box->x2;
+ v[6] = v[1] = op->base.dst.y + box->y2;
+@@ -4125,6 +4182,7 @@ gen3_emit_composite_spans_primitive_identity_gradient(struct sna *sna,
+ {
+ float *v = sna->render.vertices + sna->render.vertex_used;
+ sna->render.vertex_used += 15;
++ assert(sna->render.vertex_used <= sna->render.vertex_size);
+
+ v[0] = op->base.dst.x + box->x2;
+ v[1] = op->base.dst.y + box->y2;
+@@ -4184,6 +4242,7 @@ gen3_emit_composite_spans_primitive_constant__sse2(struct sna *sna,
+ {
+ float *v = sna->render.vertices + sna->render.vertex_used;
+ sna->render.vertex_used += 9;
++ assert(sna->render.vertex_used <= sna->render.vertex_size);
+
+ v[0] = op->base.dst.x + box->x2;
+ v[6] = v[3] = op->base.dst.x + box->x1;
+@@ -4229,6 +4288,7 @@ gen3_render_composite_spans_constant_box__sse2(struct sna *sna,
+
+ v = sna->render.vertices + sna->render.vertex_used;
+ sna->render.vertex_used += 9;
++ assert(sna->render.vertex_used <= sna->render.vertex_size);
+
+ v[0] = box->x2;
+ v[6] = v[3] = box->x1;
+@@ -4259,6 +4319,7 @@ gen3_render_composite_spans_constant_thread__sse2__boxes(struct sna *sna,
+
+ v = sna->render.vertices + sna->render.vertex_used;
+ sna->render.vertex_used += nbox_this_time * 9;
++ assert(sna->render.vertex_used <= sna->render.vertex_size);
+
+ sna_vertex_acquire__locked(&sna->render);
+ sna_vertex_unlock(&sna->render);
+@@ -4287,6 +4348,7 @@ gen3_emit_composite_spans_primitive_constant__sse2__no_offset(struct sna *sna,
+ {
+ float *v = sna->render.vertices + sna->render.vertex_used;
+ sna->render.vertex_used += 9;
++ assert(sna->render.vertex_used <= sna->render.vertex_size);
+
+ v[0] = box->x2;
+ v[6] = v[3] = box->x1;
+@@ -4320,6 +4382,7 @@ gen3_emit_composite_spans_primitive_identity_source__sse2(struct sna *sna,
+ {
+ float *v = sna->render.vertices + sna->render.vertex_used;
+ sna->render.vertex_used += 15;
++ assert(sna->render.vertex_used <= sna->render.vertex_size);
+
+ v[0] = op->base.dst.x + box->x2;
+ v[1] = op->base.dst.y + box->y2;
+@@ -4380,6 +4443,7 @@ gen3_emit_composite_spans_primitive_affine_source__sse2(struct sna *sna,
+
+ v = sna->render.vertices + sna->render.vertex_used;
+ sna->render.vertex_used += 15;
++ assert(sna->render.vertex_used <= sna->render.vertex_size);
+
+ v[0] = op->base.dst.x + box->x2;
+ v[6] = v[1] = op->base.dst.y + box->y2;
+@@ -4445,6 +4509,7 @@ gen3_emit_composite_spans_primitive_identity_gradient__sse2(struct sna *sna,
+ {
+ float *v = sna->render.vertices + sna->render.vertex_used;
+ sna->render.vertex_used += 15;
++ assert(sna->render.vertex_used <= sna->render.vertex_size);
+
+ v[0] = op->base.dst.x + box->x2;
+ v[1] = op->base.dst.y + box->y2;
+@@ -4504,6 +4569,7 @@ gen3_emit_composite_spans_primitive_affine_gradient__sse2(struct sna *sna,
+ PictTransform *transform = op->base.src.transform;
+ float *v = sna->render.vertices + sna->render.vertex_used;
+ sna->render.vertex_used += 15;
++ assert(sna->render.vertex_used <= sna->render.vertex_size);
+
+ v[0] = op->base.dst.x + box->x2;
+ v[1] = op->base.dst.y + box->y2;
+@@ -4577,6 +4643,7 @@ gen3_emit_composite_spans_primitive_affine_gradient(struct sna *sna,
+ PictTransform *transform = op->base.src.transform;
+ float *v = sna->render.vertices + sna->render.vertex_used;
+ sna->render.vertex_used += 15;
++ assert(sna->render.vertex_used <= sna->render.vertex_size);
+
+ v[0] = op->base.dst.x + box->x2;
+ v[1] = op->base.dst.y + box->y2;
+@@ -4676,6 +4743,7 @@ gen3_render_composite_spans_constant_box(struct sna *sna,
+
+ v = sna->render.vertices + sna->render.vertex_used;
+ sna->render.vertex_used += 9;
++ assert(sna->render.vertex_used <= sna->render.vertex_size);
+
+ v[0] = box->x2;
+ v[6] = v[3] = box->x1;
+@@ -4706,6 +4774,7 @@ gen3_render_composite_spans_constant_thread_boxes(struct sna *sna,
+
+ v = sna->render.vertices + sna->render.vertex_used;
+ sna->render.vertex_used += nbox_this_time * 9;
++ assert(sna->render.vertex_used <= sna->render.vertex_size);
+
+ sna_vertex_acquire__locked(&sna->render);
+ sna_vertex_unlock(&sna->render);
+@@ -4795,6 +4864,7 @@ gen3_render_composite_spans_boxes__thread(struct sna *sna,
+
+ v = sna->render.vertices + sna->render.vertex_used;
+ sna->render.vertex_used += nbox_this_time * op->base.floats_per_rect;
++ assert(sna->render.vertex_used <= sna->render.vertex_size);
+
+ sna_vertex_acquire__locked(&sna->render);
+ sna_vertex_unlock(&sna->render);
+@@ -5436,17 +5506,7 @@ gen3_render_video(struct sna *sna,
+ pix_yoff = -dstRegion->extents.y1;
+ copy = 1;
+ } else {
+- /* Set up the offset for translating from the given region
+- * (in screen coordinates) to the backing pixmap.
+- */
+-#ifdef COMPOSITE
+- pix_xoff = -pixmap->screen_x + pixmap->drawable.x;
+- pix_yoff = -pixmap->screen_y + pixmap->drawable.y;
+-#else
+- pix_xoff = 0;
+- pix_yoff = 0;
+-#endif
+-
++ pix_xoff = pix_yoff = 0;
+ dst_width = pixmap->drawable.width;
+ dst_height = pixmap->drawable.height;
+ }
+@@ -5502,16 +5562,9 @@ gen3_render_video(struct sna *sna,
+ } while (nbox);
+
+ if (copy) {
+-#ifdef COMPOSITE
+- pix_xoff = -pixmap->screen_x + pixmap->drawable.x;
+- pix_yoff = -pixmap->screen_y + pixmap->drawable.y;
+-#else
+- pix_xoff = 0;
+- pix_yoff = 0;
+-#endif
+ sna_blt_copy_boxes(sna, GXcopy,
+ dst_bo, -dstRegion->extents.x1, -dstRegion->extents.y1,
+- priv->gpu_bo, pix_xoff, pix_yoff,
++ priv->gpu_bo, 0, 0,
+ pixmap->drawable.bitsPerPixel,
+ region_rects(dstRegion),
+ region_num_rects(dstRegion));
+@@ -5519,21 +5572,8 @@ gen3_render_video(struct sna *sna,
+ kgem_bo_destroy(&sna->kgem, dst_bo);
+ }
+
+- if (!DAMAGE_IS_ALL(priv->gpu_damage)) {
+- if ((pix_xoff | pix_yoff) == 0) {
+- sna_damage_add(&priv->gpu_damage, dstRegion);
+- sna_damage_subtract(&priv->cpu_damage, dstRegion);
+- } else {
+- sna_damage_add_boxes(&priv->gpu_damage,
+- region_rects(dstRegion),
+- region_num_rects(dstRegion),
+- pix_xoff, pix_yoff);
+- sna_damage_subtract_boxes(&priv->cpu_damage,
+- region_rects(dstRegion),
+- region_num_rects(dstRegion),
+- pix_xoff, pix_yoff);
+- }
+- }
++ if (!DAMAGE_IS_ALL(priv->gpu_damage))
++ sna_damage_add(&priv->gpu_damage, dstRegion);
+
+ return true;
+ }
+diff --git a/src/sna/gen4_render.c b/src/sna/gen4_render.c
+index 6c2d3808..72a98aee 100644
+--- a/src/sna/gen4_render.c
++++ b/src/sna/gen4_render.c
+@@ -1405,8 +1405,8 @@ gen4_render_video(struct sna *sna,
+ int src_height = frame->src.y2 - frame->src.y1;
+ float src_offset_x, src_offset_y;
+ float src_scale_x, src_scale_y;
+- int nbox, pix_xoff, pix_yoff;
+ const BoxRec *box;
++ int nbox;
+
+ DBG(("%s: %dx%d -> %dx%d\n", __FUNCTION__,
+ src_width, src_height, dst_width, dst_height));
+@@ -1445,17 +1445,6 @@ gen4_render_video(struct sna *sna,
+ gen4_align_vertex(sna, &tmp);
+ gen4_video_bind_surfaces(sna, &tmp);
+
+- /* Set up the offset for translating from the given region (in screen
+- * coordinates) to the backing pixmap.
+- */
+-#ifdef COMPOSITE
+- pix_xoff = -pixmap->screen_x + pixmap->drawable.x;
+- pix_yoff = -pixmap->screen_y + pixmap->drawable.y;
+-#else
+- pix_xoff = 0;
+- pix_yoff = 0;
+-#endif
+-
+ src_scale_x = (float)src_width / dst_width / frame->width;
+ src_offset_x = (float)frame->src.x1 / frame->width - dstRegion->extents.x1 * src_scale_x;
+
+@@ -1473,34 +1462,26 @@ gen4_render_video(struct sna *sna,
+ nbox -= n;
+
+ do {
+- BoxRec r;
+-
+- r.x1 = box->x1 + pix_xoff;
+- r.x2 = box->x2 + pix_xoff;
+- r.y1 = box->y1 + pix_yoff;
+- r.y2 = box->y2 + pix_yoff;
+-
+- OUT_VERTEX(r.x2, r.y2);
++ OUT_VERTEX(box->x2, box->y2);
+ OUT_VERTEX_F(box->x2 * src_scale_x + src_offset_x);
+ OUT_VERTEX_F(box->y2 * src_scale_y + src_offset_y);
+
+- OUT_VERTEX(r.x1, r.y2);
++ OUT_VERTEX(box->x1, box->y2);
+ OUT_VERTEX_F(box->x1 * src_scale_x + src_offset_x);
+ OUT_VERTEX_F(box->y2 * src_scale_y + src_offset_y);
+
+- OUT_VERTEX(r.x1, r.y1);
++ OUT_VERTEX(box->x1, box->y1);
+ OUT_VERTEX_F(box->x1 * src_scale_x + src_offset_x);
+ OUT_VERTEX_F(box->y1 * src_scale_y + src_offset_y);
+
+- if (!DAMAGE_IS_ALL(priv->gpu_damage)) {
+- sna_damage_add_box(&priv->gpu_damage, &r);
+- sna_damage_subtract_box(&priv->cpu_damage, &r);
+- }
+ box++;
+ } while (--n);
+ } while (nbox);
+ gen4_vertex_flush(sna);
+
++ if (!DAMAGE_IS_ALL(priv->gpu_damage))
++ sna_damage_add(&priv->gpu_damage, dstRegion);
++
+ return true;
+ }
+
+@@ -1585,12 +1566,14 @@ gen4_composite_picture(struct sna *sna,
+ if (channel->repeat &&
+ (x >= 0 &&
+ y >= 0 &&
+- x + w < pixmap->drawable.width &&
+- y + h < pixmap->drawable.height)) {
++ x + w <= pixmap->drawable.width &&
++ y + h <= pixmap->drawable.height)) {
+ struct sna_pixmap *priv = sna_pixmap(pixmap);
+ if (priv && priv->clear) {
+ DBG(("%s: converting large pixmap source into solid [%08x]\n", __FUNCTION__, priv->clear_color));
+- return gen4_channel_init_solid(sna, channel, priv->clear_color);
++ return gen4_channel_init_solid(sna, channel,
++ solid_color(picture->format,
++ priv->clear_color));
+ }
+ }
+ } else
+@@ -1664,7 +1647,9 @@ gen4_composite_set_target(struct sna *sna,
+ } else
+ sna_render_picture_extents(dst, &box);
+
+- hint = PREFER_GPU | FORCE_GPU | RENDER_GPU;
++ hint = PREFER_GPU | RENDER_GPU;
++ if (!need_tiling(sna, op->dst.width, op->dst.height))
++ hint |= FORCE_GPU;
+ if (!partial) {
+ hint |= IGNORE_DAMAGE;
+ if (w == op->dst.width && h == op->dst.height)
+@@ -2738,6 +2723,20 @@ gen4_render_fill_boxes(struct sna *sna,
+ tmp.dst.format = format;
+ tmp.dst.bo = dst_bo;
+
++ sna_render_composite_redirect_init(&tmp);
++ if (too_large(dst->width, dst->height)) {
++ BoxRec extents;
++
++ boxes_extents(box, n, &extents);
++ if (!sna_render_composite_redirect(sna, &tmp,
++ extents.x1, extents.y1,
++ extents.x2 - extents.x1,
++ extents.y2 - extents.y1,
++ n > 1))
++ return sna_tiling_fill_boxes(sna, op, format, color,
++ dst, dst_bo, box, n);
++ }
++
+ gen4_channel_init_solid(sna, &tmp.src, pixel);
+
+ tmp.is_affine = true;
+@@ -2748,8 +2747,10 @@ gen4_render_fill_boxes(struct sna *sna,
+
+ if (!kgem_check_bo(&sna->kgem, dst_bo, NULL)) {
+ kgem_submit(&sna->kgem);
+- if (!kgem_check_bo(&sna->kgem, dst_bo, NULL))
++ if (!kgem_check_bo(&sna->kgem, dst_bo, NULL)) {
++ kgem_bo_destroy(&sna->kgem, tmp.src.bo);
+ return false;
++ }
+ }
+
+ gen4_align_vertex(sna, &tmp);
+@@ -2765,6 +2766,7 @@ gen4_render_fill_boxes(struct sna *sna,
+
+ gen4_vertex_flush(sna);
+ kgem_bo_destroy(&sna->kgem, tmp.src.bo);
++ sna_render_composite_redirect_done(sna, &tmp);
+ return true;
+ }
+
+diff --git a/src/sna/gen5_render.c b/src/sna/gen5_render.c
+index 37cf1ff9..fb3e79bf 100644
+--- a/src/sna/gen5_render.c
++++ b/src/sna/gen5_render.c
+@@ -1355,8 +1355,8 @@ gen5_render_video(struct sna *sna,
+ int src_height = frame->src.y2 - frame->src.y1;
+ float src_offset_x, src_offset_y;
+ float src_scale_x, src_scale_y;
+- int nbox, pix_xoff, pix_yoff;
+ const BoxRec *box;
++ int nbox;
+
+ DBG(("%s: %dx%d -> %dx%d\n", __FUNCTION__,
+ src_width, src_height, dst_width, dst_height));
+@@ -1395,17 +1395,6 @@ gen5_render_video(struct sna *sna,
+ gen5_align_vertex(sna, &tmp);
+ gen5_video_bind_surfaces(sna, &tmp);
+
+- /* Set up the offset for translating from the given region (in screen
+- * coordinates) to the backing pixmap.
+- */
+-#ifdef COMPOSITE
+- pix_xoff = -pixmap->screen_x + pixmap->drawable.x;
+- pix_yoff = -pixmap->screen_y + pixmap->drawable.y;
+-#else
+- pix_xoff = 0;
+- pix_yoff = 0;
+-#endif
+-
+ src_scale_x = (float)src_width / dst_width / frame->width;
+ src_offset_x = (float)frame->src.x1 / frame->width - dstRegion->extents.x1 * src_scale_x;
+
+@@ -1415,35 +1404,27 @@ gen5_render_video(struct sna *sna,
+ box = region_rects(dstRegion);
+ nbox = region_num_rects(dstRegion);
+ while (nbox--) {
+- BoxRec r;
+-
+- r.x1 = box->x1 + pix_xoff;
+- r.x2 = box->x2 + pix_xoff;
+- r.y1 = box->y1 + pix_yoff;
+- r.y2 = box->y2 + pix_yoff;
+-
+ gen5_get_rectangles(sna, &tmp, 1, gen5_video_bind_surfaces);
+
+- OUT_VERTEX(r.x2, r.y2);
++ OUT_VERTEX(box->x2, box->y2);
+ OUT_VERTEX_F(box->x2 * src_scale_x + src_offset_x);
+ OUT_VERTEX_F(box->y2 * src_scale_y + src_offset_y);
+
+- OUT_VERTEX(r.x1, r.y2);
++ OUT_VERTEX(box->x1, box->y2);
+ OUT_VERTEX_F(box->x1 * src_scale_x + src_offset_x);
+ OUT_VERTEX_F(box->y2 * src_scale_y + src_offset_y);
+
+- OUT_VERTEX(r.x1, r.y1);
++ OUT_VERTEX(box->x1, box->y1);
+ OUT_VERTEX_F(box->x1 * src_scale_x + src_offset_x);
+ OUT_VERTEX_F(box->y1 * src_scale_y + src_offset_y);
+
+- if (!DAMAGE_IS_ALL(priv->gpu_damage)) {
+- sna_damage_add_box(&priv->gpu_damage, &r);
+- sna_damage_subtract_box(&priv->cpu_damage, &r);
+- }
+ box++;
+ }
+-
+ gen4_vertex_flush(sna);
++
++ if (!DAMAGE_IS_ALL(priv->gpu_damage))
++ sna_damage_add(&priv->gpu_damage, dstRegion);
++
+ return true;
+ }
+
+@@ -1524,12 +1505,12 @@ gen5_composite_picture(struct sna *sna,
+ if (channel->repeat ||
+ (x >= 0 &&
+ y >= 0 &&
+- x + w < pixmap->drawable.width &&
+- y + h < pixmap->drawable.height)) {
++ x + w <= pixmap->drawable.width &&
++ y + h <= pixmap->drawable.height)) {
+ struct sna_pixmap *priv = sna_pixmap(pixmap);
+ if (priv && priv->clear) {
+ DBG(("%s: converting large pixmap source into solid [%08x]\n", __FUNCTION__, priv->clear_color));
+- return gen4_channel_init_solid(sna, channel, priv->clear_color);
++ return gen4_channel_init_solid(sna, channel, solid_color(picture->format, priv->clear_color));
+ }
+ }
+ } else
+@@ -1618,7 +1599,9 @@ gen5_composite_set_target(struct sna *sna,
+ } else
+ sna_render_picture_extents(dst, &box);
+
+- hint = PREFER_GPU | FORCE_GPU | RENDER_GPU;
++ hint = PREFER_GPU | RENDER_GPU;
++ if (!need_tiling(sna, op->dst.width, op->dst.height))
++ hint |= FORCE_GPU;
+ if (!partial) {
+ hint |= IGNORE_DAMAGE;
+ if (w == op->dst.width && h == op->dst.height)
+@@ -2734,6 +2717,19 @@ gen5_render_fill_boxes(struct sna *sna,
+ tmp.dst.format = format;
+ tmp.dst.bo = dst_bo;
+
++ if (too_large(dst->width, dst->height)) {
++ BoxRec extents;
++
++ boxes_extents(box, n, &extents);
++ if (!sna_render_composite_redirect(sna, &tmp,
++ extents.x1, extents.y1,
++ extents.x2 - extents.x1,
++ extents.y2 - extents.y1,
++ n > 1))
++ return sna_tiling_fill_boxes(sna, op, format, color,
++ dst, dst_bo, box, n);
++ }
++
+ tmp.src.bo = sna_render_get_solid(sna, pixel);
+ tmp.src.filter = SAMPLER_FILTER_NEAREST;
+ tmp.src.repeat = SAMPLER_EXTEND_REPEAT;
+@@ -2780,6 +2776,7 @@ gen5_render_fill_boxes(struct sna *sna,
+
+ gen4_vertex_flush(sna);
+ kgem_bo_destroy(&sna->kgem, tmp.src.bo);
++ sna_render_composite_redirect_done(sna, &tmp);
+ return true;
+ }
+
+diff --git a/src/sna/gen6_common.h b/src/sna/gen6_common.h
+index 6668620b..b53ec0c9 100644
+--- a/src/sna/gen6_common.h
++++ b/src/sna/gen6_common.h
+@@ -30,8 +30,8 @@
+
+ #include "sna.h"
+
+-#define NO_RING_SWITCH 0
+-#define PREFER_RENDER 0
++#define NO_RING_SWITCH(sna) (!(sna)->kgem.has_semaphores)
++#define PREFER_RENDER 0 /* -1 -> BLT, 1 -> RENDER */
+
+ static inline bool is_uncached(struct sna *sna,
+ struct kgem_bo *bo)
+@@ -46,40 +46,28 @@ inline static bool can_switch_to_blt(struct sna *sna,
+ if (sna->kgem.ring != KGEM_RENDER)
+ return true;
+
+- if (NO_RING_SWITCH)
+- return false;
+-
+- if (!sna->kgem.has_semaphores)
+- return false;
+-
+- if (flags & COPY_LAST)
+- return true;
+-
+ if (bo && RQ_IS_BLT(bo->rq))
+ return true;
+
+- if (sna->render_state.gt < 2)
+- return true;
++ if (bo && bo->tiling == I915_TILING_Y)
++ return false;
+
+- return kgem_ring_is_idle(&sna->kgem, KGEM_BLT);
+-}
++ if (bo && !kgem_bo_can_blt(&sna->kgem, bo))
++ return false;
+
+-inline static bool can_switch_to_render(struct sna *sna,
+- struct kgem_bo *bo)
+-{
+- if (sna->kgem.ring == KGEM_RENDER)
++ if (sna->render_state.gt < 2)
+ return true;
+
+- if (NO_RING_SWITCH)
++ if (bo && RQ_IS_RENDER(bo->rq))
+ return false;
+
+- if (!sna->kgem.has_semaphores)
++ if (NO_RING_SWITCH(sna))
+ return false;
+
+- if (bo && !RQ_IS_BLT(bo->rq) && !is_uncached(sna, bo))
++ if (flags & COPY_LAST)
+ return true;
+
+- return !kgem_ring_is_idle(&sna->kgem, KGEM_RENDER);
++ return kgem_ring_is_idle(&sna->kgem, KGEM_BLT);
+ }
+
+ static inline bool untiled_tlb_miss(struct kgem_bo *bo)
+@@ -90,57 +78,95 @@ static inline bool untiled_tlb_miss(struct kgem_bo *bo)
+ return bo->tiling == I915_TILING_NONE && bo->pitch >= 4096;
+ }
+
+-static int prefer_blt_bo(struct sna *sna, struct kgem_bo *bo)
++static int prefer_blt_bo(struct sna *sna,
++ struct kgem_bo *src,
++ struct kgem_bo *dst)
+ {
++ assert(dst != NULL);
++
+ if (PREFER_RENDER)
+ return PREFER_RENDER < 0;
+
+- if (bo->rq)
+- return RQ_IS_BLT(bo->rq);
++ if (dst->rq)
++ return RQ_IS_BLT(dst->rq);
+
+ if (sna->flags & SNA_POWERSAVE)
+ return true;
+
+- return bo->tiling == I915_TILING_NONE || is_uncached(sna, bo);
+-}
++ if (src) {
++ if (sna->render_state.gt > 1)
++ return false;
+
+-inline static bool force_blt_ring(struct sna *sna)
+-{
+- if (sna->flags & SNA_POWERSAVE)
++ if (src->rq)
++ return RQ_IS_BLT(src->rq);
++
++ if (src->tiling == I915_TILING_Y)
++ return false;
++ } else {
++ if (sna->render_state.gt > 2)
++ return false;
++ }
++
++ if (sna->render_state.gt < 2)
+ return true;
+
++ return dst->tiling == I915_TILING_NONE || is_uncached(sna, dst);
++}
++
++inline static bool force_blt_ring(struct sna *sna, struct kgem_bo *bo)
++{
+ if (sna->kgem.mode == KGEM_RENDER)
+ return false;
+
++ if (NO_RING_SWITCH(sna))
++ return sna->kgem.ring == KGEM_BLT;
++
++ if (bo->tiling == I915_TILING_Y)
++ return false;
++
++ if (sna->flags & SNA_POWERSAVE)
++ return true;
++
+ if (sna->render_state.gt < 2)
+ return true;
+
+ return false;
+ }
+
+-inline static bool prefer_blt_ring(struct sna *sna,
+- struct kgem_bo *bo,
+- unsigned flags)
++nonnull inline static bool
++prefer_blt_ring(struct sna *sna, struct kgem_bo *bo, unsigned flags)
+ {
+ if (PREFER_RENDER)
+ return PREFER_RENDER < 0;
+
+- assert(!force_blt_ring(sna));
+- assert(!kgem_bo_is_render(bo));
++ assert(!force_blt_ring(sna, bo));
++ assert(!kgem_bo_is_render(bo) || NO_RING_SWITCH(sna));
++
++ if (kgem_bo_is_blt(bo))
++ return true;
+
+ return can_switch_to_blt(sna, bo, flags);
+ }
+
+-inline static bool prefer_render_ring(struct sna *sna,
+- struct kgem_bo *bo)
++nonnull inline static bool
++prefer_render_ring(struct sna *sna, struct kgem_bo *bo)
+ {
++ if (sna->kgem.ring == KGEM_RENDER)
++ return true;
++
++ if (sna->kgem.ring != KGEM_NONE && NO_RING_SWITCH(sna))
++ return false;
++
++ if (kgem_bo_is_render(bo))
++ return true;
++
+ if (sna->flags & SNA_POWERSAVE)
+ return false;
+
+- if (sna->render_state.gt < 2)
+- return false;
++ if (!prefer_blt_bo(sna, NULL, bo))
++ return true;
+
+- return can_switch_to_render(sna, bo);
++ return !kgem_ring_is_idle(&sna->kgem, KGEM_RENDER);
+ }
+
+ inline static bool
+@@ -153,25 +179,20 @@ prefer_blt_composite(struct sna *sna, struct sna_composite_op *tmp)
+ untiled_tlb_miss(tmp->src.bo))
+ return true;
+
+- if (force_blt_ring(sna))
++ if (force_blt_ring(sna, tmp->dst.bo))
+ return true;
+
+- if (kgem_bo_is_render(tmp->dst.bo) ||
+- kgem_bo_is_render(tmp->src.bo))
+- return false;
+-
+ if (prefer_render_ring(sna, tmp->dst.bo))
+ return false;
+
+ if (!prefer_blt_ring(sna, tmp->dst.bo, 0))
+ return false;
+
+- return prefer_blt_bo(sna, tmp->dst.bo) || prefer_blt_bo(sna, tmp->src.bo);
++ return prefer_blt_bo(sna, tmp->src.bo, tmp->dst.bo);
+ }
+
+-static inline bool prefer_blt_fill(struct sna *sna,
+- struct kgem_bo *bo,
+- unsigned flags)
++nonnull static inline bool
++prefer_blt_fill(struct sna *sna, struct kgem_bo *bo, unsigned flags)
+ {
+ if (PREFER_RENDER)
+ return PREFER_RENDER < 0;
+@@ -179,24 +200,21 @@ static inline bool prefer_blt_fill(struct sna *sna,
+ if (untiled_tlb_miss(bo))
+ return true;
+
+- if (force_blt_ring(sna))
++ if (force_blt_ring(sna, bo))
+ return true;
+
+ if ((flags & (FILL_POINTS | FILL_SPANS)) == 0) {
+- if (kgem_bo_is_render(bo))
+- return false;
+-
+ if (prefer_render_ring(sna, bo))
+ return false;
+
+ if (!prefer_blt_ring(sna, bo, 0))
+ return false;
+ } else {
+- if (can_switch_to_blt(sna, bo, 0))
++ if (can_switch_to_blt(sna, bo, COPY_LAST))
+ return true;
+ }
+
+- return prefer_blt_bo(sna, bo);
++ return prefer_blt_bo(sna, NULL, bo);
+ }
+
+ void gen6_render_context_switch(struct kgem *kgem, int new_mode);
+diff --git a/src/sna/gen6_render.c b/src/sna/gen6_render.c
+index 25044685..6b69f216 100644
+--- a/src/sna/gen6_render.c
++++ b/src/sna/gen6_render.c
+@@ -1633,9 +1633,9 @@ gen6_render_video(struct sna *sna,
+ int src_height = frame->src.y2 - frame->src.y1;
+ float src_offset_x, src_offset_y;
+ float src_scale_x, src_scale_y;
+- int nbox, pix_xoff, pix_yoff;
+ unsigned filter;
+ const BoxRec *box;
++ int nbox;
+
+ DBG(("%s: src=(%d, %d), dst=(%d, %d), %dx[(%d, %d), (%d, %d)...]\n",
+ __FUNCTION__,
+@@ -1686,17 +1686,6 @@ gen6_render_video(struct sna *sna,
+ gen6_align_vertex(sna, &tmp);
+ gen6_emit_video_state(sna, &tmp);
+
+- /* Set up the offset for translating from the given region (in screen
+- * coordinates) to the backing pixmap.
+- */
+-#ifdef COMPOSITE
+- pix_xoff = -pixmap->screen_x + pixmap->drawable.x;
+- pix_yoff = -pixmap->screen_y + pixmap->drawable.y;
+-#else
+- pix_xoff = 0;
+- pix_yoff = 0;
+-#endif
+-
+ src_scale_x = (float)src_width / dst_width / frame->width;
+ src_offset_x = (float)frame->src.x1 / frame->width - dstRegion->extents.x1 * src_scale_x;
+
+@@ -1706,35 +1695,27 @@ gen6_render_video(struct sna *sna,
+ box = region_rects(dstRegion);
+ nbox = region_num_rects(dstRegion);
+ while (nbox--) {
+- BoxRec r;
+-
+- r.x1 = box->x1 + pix_xoff;
+- r.x2 = box->x2 + pix_xoff;
+- r.y1 = box->y1 + pix_yoff;
+- r.y2 = box->y2 + pix_yoff;
+-
+ gen6_get_rectangles(sna, &tmp, 1, gen6_emit_video_state);
+
+- OUT_VERTEX(r.x2, r.y2);
++ OUT_VERTEX(box->x2, box->y2);
+ OUT_VERTEX_F(box->x2 * src_scale_x + src_offset_x);
+ OUT_VERTEX_F(box->y2 * src_scale_y + src_offset_y);
+
+- OUT_VERTEX(r.x1, r.y2);
++ OUT_VERTEX(box->x1, box->y2);
+ OUT_VERTEX_F(box->x1 * src_scale_x + src_offset_x);
+ OUT_VERTEX_F(box->y2 * src_scale_y + src_offset_y);
+
+- OUT_VERTEX(r.x1, r.y1);
++ OUT_VERTEX(box->x1, box->y1);
+ OUT_VERTEX_F(box->x1 * src_scale_x + src_offset_x);
+ OUT_VERTEX_F(box->y1 * src_scale_y + src_offset_y);
+
+- if (!DAMAGE_IS_ALL(priv->gpu_damage)) {
+- sna_damage_add_box(&priv->gpu_damage, &r);
+- sna_damage_subtract_box(&priv->cpu_damage, &r);
+- }
+ box++;
+ }
+-
+ gen4_vertex_flush(sna);
++
++ if (!DAMAGE_IS_ALL(priv->gpu_damage))
++ sna_damage_add(&priv->gpu_damage, dstRegion);
++
+ return true;
+ }
+
+@@ -1815,12 +1796,12 @@ gen6_composite_picture(struct sna *sna,
+ if (channel->repeat &&
+ (x >= 0 &&
+ y >= 0 &&
+- x + w < pixmap->drawable.width &&
+- y + h < pixmap->drawable.height)) {
++ x + w <= pixmap->drawable.width &&
++ y + h <= pixmap->drawable.height)) {
+ struct sna_pixmap *priv = sna_pixmap(pixmap);
+ if (priv && priv->clear) {
+ DBG(("%s: converting large pixmap source into solid [%08x]\n", __FUNCTION__, priv->clear_color));
+- return gen4_channel_init_solid(sna, channel, priv->clear_color);
++ return gen4_channel_init_solid(sna, channel, solid_color(picture->format, priv->clear_color));
+ }
+ }
+ } else
+@@ -1927,7 +1908,9 @@ gen6_composite_set_target(struct sna *sna,
+ } else
+ sna_render_picture_extents(dst, &box);
+
+- hint = PREFER_GPU | FORCE_GPU | RENDER_GPU;
++ hint = PREFER_GPU | RENDER_GPU;
++ if (!need_tiling(sna, op->dst.width, op->dst.height))
++ hint |= FORCE_GPU;
+ if (!partial) {
+ hint |= IGNORE_DAMAGE;
+ if (w == op->dst.width && h == op->dst.height)
+@@ -1965,46 +1948,77 @@ gen6_composite_set_target(struct sna *sna,
+
+ static bool
+ try_blt(struct sna *sna,
+- PicturePtr dst, PicturePtr src,
+- int width, int height)
++ uint8_t op,
++ PicturePtr src,
++ PicturePtr mask,
++ PicturePtr dst,
++ int16_t src_x, int16_t src_y,
++ int16_t msk_x, int16_t msk_y,
++ int16_t dst_x, int16_t dst_y,
++ int16_t width, int16_t height,
++ unsigned flags,
++ struct sna_composite_op *tmp)
+ {
+ struct kgem_bo *bo;
+
+ if (sna->kgem.mode == KGEM_BLT) {
+ DBG(("%s: already performing BLT\n", __FUNCTION__));
+- return true;
++ goto execute;
+ }
+
+ if (too_large(width, height)) {
+ DBG(("%s: operation too large for 3D pipe (%d, %d)\n",
+ __FUNCTION__, width, height));
+- return true;
++ goto execute;
+ }
+
+ bo = __sna_drawable_peek_bo(dst->pDrawable);
+ if (bo == NULL)
+- return true;
+- if (bo->rq)
+- return RQ_IS_BLT(bo->rq);
++ goto execute;
++
++ if (untiled_tlb_miss(bo))
++ goto execute;
++
++ if (bo->rq) {
++ if (RQ_IS_BLT(bo->rq))
++ goto execute;
++
++ return false;
++ }
++
++ if (bo->tiling == I915_TILING_Y)
++ goto upload;
++
++ if (src->pDrawable == dst->pDrawable &&
++ can_switch_to_blt(sna, bo, 0))
++ goto execute;
+
+ if (sna_picture_is_solid(src, NULL) && can_switch_to_blt(sna, bo, 0))
+- return true;
++ goto execute;
+
+ if (src->pDrawable) {
+- bo = __sna_drawable_peek_bo(src->pDrawable);
+- if (bo == NULL)
+- return true;
++ struct kgem_bo *s = __sna_drawable_peek_bo(src->pDrawable);
++ if (s == NULL)
++ goto execute;
+
+- if (prefer_blt_bo(sna, bo))
+- return true;
++ if (prefer_blt_bo(sna, s, bo))
++ goto execute;
+ }
+
+ if (sna->kgem.ring == KGEM_BLT) {
+ DBG(("%s: already performing BLT\n", __FUNCTION__));
+- return true;
++ goto execute;
+ }
+
+- return false;
++upload:
++ flags |= COMPOSITE_UPLOAD;
++execute:
++ return sna_blt_composite(sna, op,
++ src, dst,
++ src_x, src_y,
++ dst_x, dst_y,
++ width, height,
++ flags, tmp);
+ }
+
+ static bool
+@@ -2234,13 +2248,13 @@ gen6_render_composite(struct sna *sna,
+ width, height, sna->kgem.ring));
+
+ if (mask == NULL &&
+- try_blt(sna, dst, src, width, height) &&
+- sna_blt_composite(sna, op,
+- src, dst,
+- src_x, src_y,
+- dst_x, dst_y,
+- width, height,
+- flags, tmp))
++ try_blt(sna, op,
++ src, mask, dst,
++ src_x, src_y,
++ msk_x, msk_y,
++ dst_x, dst_y,
++ width, height,
++ flags, tmp))
+ return true;
+
+ if (gen6_composite_fallback(sna, src, mask, dst))
+@@ -2676,27 +2690,35 @@ static inline bool prefer_blt_copy(struct sna *sna,
+ if (sna->kgem.ring == KGEM_BLT)
+ return true;
+
+- if (src_bo == dst_bo && can_switch_to_blt(sna, dst_bo, flags))
++ if (flags & COPY_DRI && !sna->kgem.has_semaphores)
++ return false;
++
++ if ((flags & COPY_SMALL || src_bo == dst_bo) &&
++ can_switch_to_blt(sna, dst_bo, flags))
+ return true;
+
+ if (untiled_tlb_miss(src_bo) ||
+ untiled_tlb_miss(dst_bo))
+ return true;
+
+- if (force_blt_ring(sna))
++ if (force_blt_ring(sna, dst_bo))
+ return true;
+
+ if (kgem_bo_is_render(dst_bo) ||
+ kgem_bo_is_render(src_bo))
+ return false;
+
++ if (flags & COPY_LAST &&
++ can_switch_to_blt(sna, dst_bo, flags))
++ return true;
++
+ if (prefer_render_ring(sna, dst_bo))
+ return false;
+
+ if (!prefer_blt_ring(sna, dst_bo, flags))
+ return false;
+
+- return prefer_blt_bo(sna, src_bo) || prefer_blt_bo(sna, dst_bo);
++ return prefer_blt_bo(sna, src_bo, dst_bo);
+ }
+
+ static bool
+@@ -2758,8 +2780,7 @@ fallback_blt:
+ assert(src->depth == dst->depth);
+ assert(src->width == dst->width);
+ assert(src->height == dst->height);
+- return sna_render_copy_boxes__overlap(sna, alu,
+- src, src_bo,
++ return sna_render_copy_boxes__overlap(sna, alu, dst, dst_bo,
+ src_dx, src_dy,
+ dst_dx, dst_dy,
+ box, n, &extents);
+diff --git a/src/sna/gen7_render.c b/src/sna/gen7_render.c
+index 2ecfd641..aabb8693 100644
+--- a/src/sna/gen7_render.c
++++ b/src/sna/gen7_render.c
+@@ -60,8 +60,6 @@
+ #define NO_FILL_ONE 0
+ #define NO_FILL_CLEAR 0
+
+-#define NO_RING_SWITCH 0
+-
+ #define USE_8_PIXEL_DISPATCH 1
+ #define USE_16_PIXEL_DISPATCH 1
+ #define USE_32_PIXEL_DISPATCH 0
+@@ -149,7 +147,7 @@ static const struct gt_info hsw_gt1_info = {
+ .max_vs_threads = 70,
+ .max_gs_threads = 70,
+ .max_wm_threads =
+- (102 - 1) << HSW_PS_MAX_THREADS_SHIFT |
++ (70 - 1) << HSW_PS_MAX_THREADS_SHIFT |
+ 1 << HSW_PS_SAMPLE_MASK_SHIFT,
+ .urb = { 128, 640, 256, 8 },
+ .gt = 1,
+@@ -209,6 +207,12 @@ static const uint32_t ps_kernel_planar[][4] = {
+ #include "exa_wm_write.g7b"
+ };
+
++static const uint32_t ps_kernel_rgb[][4] = {
++#include "exa_wm_src_affine.g7b"
++#include "exa_wm_src_sample_argb.g7b"
++#include "exa_wm_write.g7b"
++};
++
+ #define KERNEL(kernel_enum, kernel, num_surfaces) \
+ [GEN7_WM_KERNEL_##kernel_enum] = {#kernel_enum, kernel, sizeof(kernel), num_surfaces}
+ #define NOKERNEL(kernel_enum, func, num_surfaces) \
+@@ -218,7 +222,7 @@ static const struct wm_kernel_info {
+ const void *data;
+ unsigned int size;
+ int num_surfaces;
+-} wm_kernels[] = {
++} wm_kernels[GEN7_WM_KERNEL_COUNT] = {
+ NOKERNEL(NOMASK, brw_wm_kernel__affine, 2),
+ NOKERNEL(NOMASK_P, brw_wm_kernel__projective, 2),
+
+@@ -236,6 +240,7 @@ static const struct wm_kernel_info {
+
+ KERNEL(VIDEO_PLANAR, ps_kernel_planar, 7),
+ KERNEL(VIDEO_PACKED, ps_kernel_packed, 2),
++ KERNEL(VIDEO_RGB, ps_kernel_rgb, 2),
+ };
+ #undef KERNEL
+
+@@ -810,7 +815,7 @@ gen7_emit_cc(struct sna *sna, uint32_t blend_offset)
+
+ DBG(("%s: blend = %x\n", __FUNCTION__, blend_offset));
+
+- /* XXX can have upto 8 blend states preload, selectable via
++ /* XXX can have up to 8 blend states preload, selectable via
+ * Render Target Index. What other side-effects of Render Target Index?
+ */
+
+@@ -1792,7 +1797,9 @@ static void gen7_emit_video_state(struct sna *sna,
+ frame->pitch[0];
+ n_src = 6;
+ } else {
+- if (frame->id == FOURCC_UYVY)
++ if (frame->id == FOURCC_RGB888)
++ src_surf_format = GEN7_SURFACEFORMAT_B8G8R8X8_UNORM;
++ else if (frame->id == FOURCC_UYVY)
+ src_surf_format = GEN7_SURFACEFORMAT_YCRCB_SWAPY;
+ else
+ src_surf_format = GEN7_SURFACEFORMAT_YCRCB_NORMAL;
+@@ -1826,6 +1833,23 @@ static void gen7_emit_video_state(struct sna *sna,
+ gen7_emit_state(sna, op, offset | dirty);
+ }
+
++static unsigned select_video_kernel(const struct sna_video_frame *frame)
++{
++ switch (frame->id) {
++ case FOURCC_YV12:
++ case FOURCC_I420:
++ case FOURCC_XVMC:
++ return GEN7_WM_KERNEL_VIDEO_PLANAR;
++
++ case FOURCC_RGB888:
++ case FOURCC_RGB565:
++ return GEN7_WM_KERNEL_VIDEO_RGB;
++
++ default:
++ return GEN7_WM_KERNEL_VIDEO_PACKED;
++ }
++}
++
+ static bool
+ gen7_render_video(struct sna *sna,
+ struct sna_video *video,
+@@ -1841,9 +1865,9 @@ gen7_render_video(struct sna *sna,
+ int src_height = frame->src.y2 - frame->src.y1;
+ float src_offset_x, src_offset_y;
+ float src_scale_x, src_scale_y;
+- int nbox, pix_xoff, pix_yoff;
+ unsigned filter;
+ const BoxRec *box;
++ int nbox;
+
+ DBG(("%s: src=(%d, %d), dst=(%d, %d), %dx[(%d, %d), (%d, %d)...]\n",
+ __FUNCTION__,
+@@ -1878,9 +1902,7 @@ gen7_render_video(struct sna *sna,
+ GEN7_SET_FLAGS(SAMPLER_OFFSET(filter, SAMPLER_EXTEND_PAD,
+ SAMPLER_FILTER_NEAREST, SAMPLER_EXTEND_NONE),
+ NO_BLEND,
+- is_planar_fourcc(frame->id) ?
+- GEN7_WM_KERNEL_VIDEO_PLANAR :
+- GEN7_WM_KERNEL_VIDEO_PACKED,
++ select_video_kernel(frame),
+ 2);
+ tmp.priv = frame;
+
+@@ -1896,17 +1918,6 @@ gen7_render_video(struct sna *sna,
+ gen7_align_vertex(sna, &tmp);
+ gen7_emit_video_state(sna, &tmp);
+
+- /* Set up the offset for translating from the given region (in screen
+- * coordinates) to the backing pixmap.
+- */
+-#ifdef COMPOSITE
+- pix_xoff = -pixmap->screen_x + pixmap->drawable.x;
+- pix_yoff = -pixmap->screen_y + pixmap->drawable.y;
+-#else
+- pix_xoff = 0;
+- pix_yoff = 0;
+-#endif
+-
+ DBG(("%s: src=(%d, %d)x(%d, %d); frame=(%dx%d), dst=(%dx%d)\n",
+ __FUNCTION__,
+ frame->src.x1, frame->src.y1,
+@@ -1928,45 +1939,36 @@ gen7_render_video(struct sna *sna,
+ box = region_rects(dstRegion);
+ nbox = region_num_rects(dstRegion);
+ while (nbox--) {
+- BoxRec r;
+-
+- DBG(("%s: dst=(%d, %d), (%d, %d) + (%d, %d); src=(%f, %f), (%f, %f)\n",
++ DBG(("%s: dst=(%d, %d), (%d, %d); src=(%f, %f), (%f, %f)\n",
+ __FUNCTION__,
+ box->x1, box->y1,
+ box->x2, box->y2,
+- pix_xoff, pix_yoff,
+ box->x1 * src_scale_x + src_offset_x,
+ box->y1 * src_scale_y + src_offset_y,
+ box->x2 * src_scale_x + src_offset_x,
+ box->y2 * src_scale_y + src_offset_y));
+
+- r.x1 = box->x1 + pix_xoff;
+- r.x2 = box->x2 + pix_xoff;
+- r.y1 = box->y1 + pix_yoff;
+- r.y2 = box->y2 + pix_yoff;
+-
+ gen7_get_rectangles(sna, &tmp, 1, gen7_emit_video_state);
+
+- OUT_VERTEX(r.x2, r.y2);
++ OUT_VERTEX(box->x2, box->y2);
+ OUT_VERTEX_F(box->x2 * src_scale_x + src_offset_x);
+ OUT_VERTEX_F(box->y2 * src_scale_y + src_offset_y);
+
+- OUT_VERTEX(r.x1, r.y2);
++ OUT_VERTEX(box->x1, box->y2);
+ OUT_VERTEX_F(box->x1 * src_scale_x + src_offset_x);
+ OUT_VERTEX_F(box->y2 * src_scale_y + src_offset_y);
+
+- OUT_VERTEX(r.x1, r.y1);
++ OUT_VERTEX(box->x1, box->y1);
+ OUT_VERTEX_F(box->x1 * src_scale_x + src_offset_x);
+ OUT_VERTEX_F(box->y1 * src_scale_y + src_offset_y);
+
+- if (!DAMAGE_IS_ALL(priv->gpu_damage)) {
+- sna_damage_add_box(&priv->gpu_damage, &r);
+- sna_damage_subtract_box(&priv->cpu_damage, &r);
+- }
+ box++;
+ }
+-
+ gen4_vertex_flush(sna);
++
++ if (!DAMAGE_IS_ALL(priv->gpu_damage))
++ sna_damage_add(&priv->gpu_damage, dstRegion);
++
+ return true;
+ }
+
+@@ -2048,12 +2050,13 @@ gen7_composite_picture(struct sna *sna,
+ if (channel->repeat ||
+ (x >= 0 &&
+ y >= 0 &&
+- x + w < pixmap->drawable.width &&
+- y + h < pixmap->drawable.height)) {
++ x + w <= pixmap->drawable.width &&
++ y + h <= pixmap->drawable.height)) {
+ struct sna_pixmap *priv = sna_pixmap(pixmap);
+ if (priv && priv->clear) {
+ DBG(("%s: converting large pixmap source into solid [%08x]\n", __FUNCTION__, priv->clear_color));
+- return gen4_channel_init_solid(sna, channel, priv->clear_color);
++ return gen4_channel_init_solid(sna, channel,
++ solid_color(picture->format, priv->clear_color));
+ }
+ }
+ } else
+@@ -2147,7 +2150,9 @@ gen7_composite_set_target(struct sna *sna,
+ } else
+ sna_render_picture_extents(dst, &box);
+
+- hint = PREFER_GPU | FORCE_GPU | RENDER_GPU;
++ hint = PREFER_GPU | RENDER_GPU;
++ if (!need_tiling(sna, op->dst.width, op->dst.height))
++ hint |= FORCE_GPU;
+ if (!partial) {
+ hint |= IGNORE_DAMAGE;
+ if (w == op->dst.width && h == op->dst.height)
+@@ -2185,46 +2190,78 @@ gen7_composite_set_target(struct sna *sna,
+
+ static bool
+ try_blt(struct sna *sna,
+- PicturePtr dst, PicturePtr src,
+- int width, int height)
++ uint8_t op,
++ PicturePtr src,
++ PicturePtr mask,
++ PicturePtr dst,
++ int16_t src_x, int16_t src_y,
++ int16_t msk_x, int16_t msk_y,
++ int16_t dst_x, int16_t dst_y,
++ int16_t width, int16_t height,
++ unsigned flags,
++ struct sna_composite_op *tmp)
+ {
+ struct kgem_bo *bo;
+
+ if (sna->kgem.mode == KGEM_BLT) {
+ DBG(("%s: already performing BLT\n", __FUNCTION__));
+- return true;
++ goto execute;
+ }
+
+ if (too_large(width, height)) {
+ DBG(("%s: operation too large for 3D pipe (%d, %d)\n",
+ __FUNCTION__, width, height));
+- return true;
++ goto execute;
+ }
+
+ bo = __sna_drawable_peek_bo(dst->pDrawable);
+ if (bo == NULL)
+- return true;
+- if (bo->rq)
+- return RQ_IS_BLT(bo->rq);
++ goto execute;
++
++ if (untiled_tlb_miss(bo))
++ goto execute;
++
++ if (bo->rq) {
++ if (RQ_IS_BLT(bo->rq))
++ goto execute;
++
++ return false;
++ }
++
++ if (bo->tiling == I915_TILING_Y)
++ goto upload;
++
++ if (src->pDrawable == dst->pDrawable &&
++ (sna->render_state.gt < 3 || width*height < 1024) &&
++ can_switch_to_blt(sna, bo, 0))
++ goto execute;
+
+ if (sna_picture_is_solid(src, NULL) && can_switch_to_blt(sna, bo, 0))
+- return true;
++ goto execute;
+
+ if (src->pDrawable) {
+- bo = __sna_drawable_peek_bo(src->pDrawable);
+- if (bo == NULL)
+- return true;
++ struct kgem_bo *s = __sna_drawable_peek_bo(src->pDrawable);
++ if (s == NULL)
++ goto upload;
+
+- if (prefer_blt_bo(sna, bo))
+- return true;
++ if (prefer_blt_bo(sna, s, bo))
++ goto execute;
+ }
+
+ if (sna->kgem.ring == KGEM_BLT) {
+ DBG(("%s: already performing BLT\n", __FUNCTION__));
+- return true;
++ goto execute;
+ }
+
+- return false;
++upload:
++ flags |= COMPOSITE_UPLOAD;
++execute:
++ return sna_blt_composite(sna, op,
++ src, dst,
++ src_x, src_y,
++ dst_x, dst_y,
++ width, height,
++ flags, tmp);
+ }
+
+ static bool
+@@ -2454,13 +2491,13 @@ gen7_render_composite(struct sna *sna,
+ width, height, sna->kgem.mode, sna->kgem.ring));
+
+ if (mask == NULL &&
+- try_blt(sna, dst, src, width, height) &&
+- sna_blt_composite(sna, op,
+- src, dst,
+- src_x, src_y,
+- dst_x, dst_y,
+- width, height,
+- flags, tmp))
++ try_blt(sna, op,
++ src, mask, dst,
++ src_x, src_y,
++ msk_x, msk_y,
++ dst_x, dst_y,
++ width, height,
++ flags, tmp))
+ return true;
+
+ if (gen7_composite_fallback(sna, src, mask, dst))
+@@ -2878,27 +2915,37 @@ prefer_blt_copy(struct sna *sna,
+
+ assert((flags & COPY_SYNC) == 0);
+
+- if (src_bo == dst_bo && can_switch_to_blt(sna, dst_bo, flags))
+- return true;
+-
+ if (untiled_tlb_miss(src_bo) ||
+ untiled_tlb_miss(dst_bo))
+ return true;
+
+- if (force_blt_ring(sna))
++ if (flags & COPY_DRI && !sna->kgem.has_semaphores)
++ return false;
++
++ if (force_blt_ring(sna, dst_bo))
++ return true;
++
++ if ((flags & COPY_SMALL ||
++ (sna->render_state.gt < 3 && src_bo == dst_bo)) &&
++ can_switch_to_blt(sna, dst_bo, flags))
+ return true;
+
+ if (kgem_bo_is_render(dst_bo) ||
+ kgem_bo_is_render(src_bo))
+ return false;
+
++ if (flags & COPY_LAST &&
++ sna->render_state.gt < 3 &&
++ can_switch_to_blt(sna, dst_bo, flags))
++ return true;
++
+ if (prefer_render_ring(sna, dst_bo))
+ return false;
+
+ if (!prefer_blt_ring(sna, dst_bo, flags))
+ return false;
+
+- return prefer_blt_bo(sna, src_bo) || prefer_blt_bo(sna, dst_bo);
++ return prefer_blt_bo(sna, src_bo, dst_bo);
+ }
+
+ static bool
+@@ -2946,7 +2993,7 @@ fallback_blt:
+ &extents)) {
+ bool big = too_large(extents.x2-extents.x1, extents.y2-extents.y1);
+
+- if ((big || can_switch_to_blt(sna, dst_bo, flags)) &&
++ if ((big || !prefer_render_ring(sna, dst_bo)) &&
+ sna_blt_copy_boxes(sna, alu,
+ src_bo, src_dx, src_dy,
+ dst_bo, dst_dx, dst_dy,
+@@ -2961,8 +3008,7 @@ fallback_blt:
+ assert(src->depth == dst->depth);
+ assert(src->width == dst->width);
+ assert(src->height == dst->height);
+- return sna_render_copy_boxes__overlap(sna, alu,
+- src, src_bo,
++ return sna_render_copy_boxes__overlap(sna, alu, dst, dst_bo,
+ src_dx, src_dy,
+ dst_dx, dst_dy,
+ box, n, &extents);
+diff --git a/src/sna/gen8_render.c b/src/sna/gen8_render.c
+index 6eb11452..445983b1 100644
+--- a/src/sna/gen8_render.c
++++ b/src/sna/gen8_render.c
+@@ -106,6 +106,12 @@ static const uint32_t ps_kernel_planar[][4] = {
+ #include "exa_wm_yuv_rgb.g8b"
+ #include "exa_wm_write.g8b"
+ };
++
++static const uint32_t ps_kernel_rgb[][4] = {
++#include "exa_wm_src_affine.g8b"
++#include "exa_wm_src_sample_argb.g8b"
++#include "exa_wm_write.g8b"
++};
+ #endif
+
+ #define SURFACE_DW (64 / sizeof(uint32_t));
+@@ -119,7 +125,7 @@ static const struct wm_kernel_info {
+ const void *data;
+ unsigned int size;
+ int num_surfaces;
+-} wm_kernels[] = {
++} wm_kernels[GEN8_WM_KERNEL_COUNT] = {
+ NOKERNEL(NOMASK, gen8_wm_kernel__affine, 2),
+ NOKERNEL(NOMASK_P, gen8_wm_kernel__projective, 2),
+
+@@ -138,6 +144,7 @@ static const struct wm_kernel_info {
+ #if !NO_VIDEO
+ KERNEL(VIDEO_PLANAR, ps_kernel_planar, 7),
+ KERNEL(VIDEO_PACKED, ps_kernel_packed, 2),
++ KERNEL(VIDEO_RGB, ps_kernel_rgb, 2),
+ #endif
+ };
+ #undef KERNEL
+@@ -205,6 +212,33 @@ static const struct blendinfo {
+ #define OUT_VERTEX(x,y) vertex_emit_2s(sna, x,y)
+ #define OUT_VERTEX_F(v) vertex_emit(sna, v)
+
++struct gt_info {
++ const char *name;
++ struct {
++ int max_vs_entries;
++ } urb;
++};
++
++static const struct gt_info bdw_gt_info = {
++ .name = "Broadwell (gen8)",
++ .urb = { .max_vs_entries = 960 },
++};
++
++static bool is_bdw(struct sna *sna)
++{
++ return sna->kgem.gen == 0100;
++}
++
++static const struct gt_info chv_gt_info = {
++ .name = "Cherryview (gen8)",
++ .urb = { .max_vs_entries = 640 },
++};
++
++static bool is_chv(struct sna *sna)
++{
++ return sna->kgem.gen == 0101;
++}
++
+ static inline bool too_large(int width, int height)
+ {
+ return width > GEN8_MAX_SIZE || height > GEN8_MAX_SIZE;
+@@ -462,7 +496,7 @@ gen8_emit_urb(struct sna *sna)
+ {
+ /* num of VS entries must be divisible by 8 if size < 9 */
+ OUT_BATCH(GEN8_3DSTATE_URB_VS | (2 - 2));
+- OUT_BATCH(960 << URB_ENTRY_NUMBER_SHIFT |
++ OUT_BATCH(sna->render_state.gen8.info->urb.max_vs_entries << URB_ENTRY_NUMBER_SHIFT |
+ (2 - 1) << URB_ENTRY_SIZE_SHIFT |
+ 4 << URB_STARTING_ADDRESS_SHIFT);
+
+@@ -873,7 +907,7 @@ gen8_emit_cc(struct sna *sna, uint32_t blend)
+ assert(blend / GEN8_BLENDFACTOR_COUNT > 0);
+ assert(blend % GEN8_BLENDFACTOR_COUNT > 0);
+
+- /* XXX can have upto 8 blend states preload, selectable via
++ /* XXX can have up to 8 blend states preload, selectable via
+ * Render Target Index. What other side-effects of Render Target Index?
+ */
+
+@@ -1167,6 +1201,7 @@ gen8_emit_pipe_stall(struct sna *sna)
+ {
+ OUT_BATCH(GEN8_PIPE_CONTROL | (6 - 2));
+ OUT_BATCH(PIPE_CONTROL_CS_STALL |
++ PIPE_CONTROL_FLUSH |
+ PIPE_CONTROL_STALL_AT_SCOREBOARD);
+ OUT_BATCH64(0);
+ OUT_BATCH64(0);
+@@ -1876,12 +1911,12 @@ gen8_composite_picture(struct sna *sna,
+ if (channel->repeat ||
+ (x >= 0 &&
+ y >= 0 &&
+- x + w < pixmap->drawable.width &&
+- y + h < pixmap->drawable.height)) {
++ x + w <= pixmap->drawable.width &&
++ y + h <= pixmap->drawable.height)) {
+ struct sna_pixmap *priv = sna_pixmap(pixmap);
+ if (priv && priv->clear) {
+ DBG(("%s: converting large pixmap source into solid [%08x]\n", __FUNCTION__, priv->clear_color));
+- return gen4_channel_init_solid(sna, channel, priv->clear_color);
++ return gen4_channel_init_solid(sna, channel, solid_color(picture->format, priv->clear_color));
+ }
+ }
+ } else
+@@ -1961,7 +1996,9 @@ gen8_composite_set_target(struct sna *sna,
+ } else
+ sna_render_picture_extents(dst, &box);
+
+- hint = PREFER_GPU | FORCE_GPU | RENDER_GPU;
++ hint = PREFER_GPU | RENDER_GPU;
++ if (!need_tiling(sna, op->dst.width, op->dst.height))
++ hint |= FORCE_GPU;
+ if (!partial) {
+ hint |= IGNORE_DAMAGE;
+ if (w == op->dst.width && h == op->dst.height)
+@@ -2002,46 +2039,78 @@ gen8_composite_set_target(struct sna *sna,
+
+ static bool
+ try_blt(struct sna *sna,
+- PicturePtr dst, PicturePtr src,
+- int width, int height)
++ uint8_t op,
++ PicturePtr src,
++ PicturePtr mask,
++ PicturePtr dst,
++ int16_t src_x, int16_t src_y,
++ int16_t msk_x, int16_t msk_y,
++ int16_t dst_x, int16_t dst_y,
++ int16_t width, int16_t height,
++ unsigned flags,
++ struct sna_composite_op *tmp)
+ {
+ struct kgem_bo *bo;
+
+ if (sna->kgem.mode == KGEM_BLT) {
+ DBG(("%s: already performing BLT\n", __FUNCTION__));
+- return true;
++ goto execute;
+ }
+
+ if (too_large(width, height)) {
+ DBG(("%s: operation too large for 3D pipe (%d, %d)\n",
+ __FUNCTION__, width, height));
+- return true;
++ goto execute;
+ }
+
+ bo = __sna_drawable_peek_bo(dst->pDrawable);
+ if (bo == NULL)
+- return true;
+- if (bo->rq)
+- return RQ_IS_BLT(bo->rq);
++ goto execute;
++
++ if (untiled_tlb_miss(bo))
++ goto execute;
++
++ if (bo->rq) {
++ if (RQ_IS_BLT(bo->rq))
++ goto execute;
++
++ return false;
++ }
++
++ if (bo->tiling == I915_TILING_Y)
++ goto upload;
+
+ if (sna_picture_is_solid(src, NULL) && can_switch_to_blt(sna, bo, 0))
+- return true;
++ goto execute;
++
++ if (src->pDrawable == dst->pDrawable &&
++ (sna->render_state.gt < 3 || width*height < 1024) &&
++ can_switch_to_blt(sna, bo, 0))
++ goto execute;
+
+ if (src->pDrawable) {
+- bo = __sna_drawable_peek_bo(src->pDrawable);
+- if (bo == NULL)
+- return true;
++ struct kgem_bo *s = __sna_drawable_peek_bo(src->pDrawable);
++ if (s == NULL)
++ goto upload;
+
+- if (prefer_blt_bo(sna, bo))
+- return RQ_IS_BLT(bo->rq);
++ if (prefer_blt_bo(sna, s, bo))
++ goto execute;
+ }
+
+ if (sna->kgem.ring == KGEM_BLT) {
+ DBG(("%s: already performing BLT\n", __FUNCTION__));
+- return true;
++ goto execute;
+ }
+
+- return false;
++upload:
++ flags |= COMPOSITE_UPLOAD;
++execute:
++ return sna_blt_composite(sna, op,
++ src, dst,
++ src_x, src_y,
++ dst_x, dst_y,
++ width, height,
++ flags, tmp);
+ }
+
+ static bool
+@@ -2271,13 +2340,13 @@ gen8_render_composite(struct sna *sna,
+ width, height, sna->kgem.mode, sna->kgem.ring));
+
+ if (mask == NULL &&
+- try_blt(sna, dst, src, width, height) &&
+- sna_blt_composite(sna, op,
+- src, dst,
+- src_x, src_y,
+- dst_x, dst_y,
+- width, height,
+- flags, tmp))
++ try_blt(sna, op,
++ src, mask, dst,
++ src_x, src_y,
++ msk_x, msk_y,
++ dst_x, dst_y,
++ width, height,
++ flags, tmp))
+ return true;
+
+ if (gen8_composite_fallback(sna, src, mask, dst))
+@@ -2700,27 +2769,37 @@ prefer_blt_copy(struct sna *sna,
+
+ assert((flags & COPY_SYNC) == 0);
+
+- if (src_bo == dst_bo && can_switch_to_blt(sna, dst_bo, flags))
+- return true;
+-
+ if (untiled_tlb_miss(src_bo) ||
+ untiled_tlb_miss(dst_bo))
+ return true;
+
+- if (force_blt_ring(sna))
++ if (flags & COPY_DRI && !sna->kgem.has_semaphores)
++ return false;
++
++ if (force_blt_ring(sna, dst_bo))
++ return true;
++
++ if ((flags & COPY_SMALL ||
++ (sna->render_state.gt < 3 && src_bo == dst_bo)) &&
++ can_switch_to_blt(sna, dst_bo, flags))
+ return true;
+
+ if (kgem_bo_is_render(dst_bo) ||
+ kgem_bo_is_render(src_bo))
+ return false;
+
++ if (flags & COPY_LAST &&
++ sna->render_state.gt < 3 &&
++ can_switch_to_blt(sna, dst_bo, flags))
++ return true;
++
+ if (prefer_render_ring(sna, dst_bo))
+ return false;
+
+ if (!prefer_blt_ring(sna, dst_bo, flags))
+ return false;
+
+- return prefer_blt_bo(sna, src_bo) || prefer_blt_bo(sna, dst_bo);
++ return prefer_blt_bo(sna, src_bo, dst_bo);
+ }
+
+ static bool
+@@ -2770,7 +2849,7 @@ fallback_blt:
+ &extents)) {
+ bool big = too_large(extents.x2-extents.x1, extents.y2-extents.y1);
+
+- if ((big || can_switch_to_blt(sna, dst_bo, flags)) &&
++ if ((big || !prefer_render_ring(sna, dst_bo)) &&
+ sna_blt_copy_boxes(sna, alu,
+ src_bo, src_dx, src_dy,
+ dst_bo, dst_dx, dst_dy,
+@@ -2785,8 +2864,7 @@ fallback_blt:
+ assert(src->depth == dst->depth);
+ assert(src->width == dst->width);
+ assert(src->height == dst->height);
+- return sna_render_copy_boxes__overlap(sna, alu,
+- src, src_bo,
++ return sna_render_copy_boxes__overlap(sna, alu, dst, dst_bo,
+ src_dx, src_dy,
+ dst_dx, dst_dy,
+ box, n, &extents);
+@@ -3665,7 +3743,9 @@ static void gen8_emit_video_state(struct sna *sna,
+ frame->pitch[0];
+ n_src = 6;
+ } else {
+- if (frame->id == FOURCC_UYVY)
++ if (frame->id == FOURCC_RGB888)
++ src_surf_format = SURFACEFORMAT_B8G8R8X8_UNORM;
++ else if (frame->id == FOURCC_UYVY)
+ src_surf_format = SURFACEFORMAT_YCRCB_SWAPY;
+ else
+ src_surf_format = SURFACEFORMAT_YCRCB_NORMAL;
+@@ -3697,6 +3777,23 @@ static void gen8_emit_video_state(struct sna *sna,
+ gen8_emit_state(sna, op, offset);
+ }
+
++static unsigned select_video_kernel(const struct sna_video_frame *frame)
++{
++ switch (frame->id) {
++ case FOURCC_YV12:
++ case FOURCC_I420:
++ case FOURCC_XVMC:
++ return GEN8_WM_KERNEL_VIDEO_PLANAR;
++
++ case FOURCC_RGB888:
++ case FOURCC_RGB565:
++ return GEN8_WM_KERNEL_VIDEO_RGB;
++
++ default:
++ return GEN8_WM_KERNEL_VIDEO_PACKED;
++ }
++}
++
+ static bool
+ gen8_render_video(struct sna *sna,
+ struct sna_video *video,
+@@ -3712,9 +3809,9 @@ gen8_render_video(struct sna *sna,
+ int src_height = frame->src.y2 - frame->src.y1;
+ float src_offset_x, src_offset_y;
+ float src_scale_x, src_scale_y;
+- int nbox, pix_xoff, pix_yoff;
+ unsigned filter;
+ const BoxRec *box;
++ int nbox;
+
+ DBG(("%s: src=(%d, %d), dst=(%d, %d), %dx[(%d, %d), (%d, %d)...]\n",
+ __FUNCTION__,
+@@ -3743,6 +3840,11 @@ gen8_render_video(struct sna *sna,
+ tmp.floats_per_vertex = 3;
+ tmp.floats_per_rect = 9;
+
++ DBG(("%s: scaling?=%d, planar?=%d [%x]\n",
++ __FUNCTION__,
++ src_width != dst_width || src_height != dst_height,
++ is_planar_fourcc(frame->id), frame->id));
++
+ if (src_width == dst_width && src_height == dst_height)
+ filter = SAMPLER_FILTER_NEAREST;
+ else
+@@ -3752,9 +3854,7 @@ gen8_render_video(struct sna *sna,
+ GEN8_SET_FLAGS(SAMPLER_OFFSET(filter, SAMPLER_EXTEND_PAD,
+ SAMPLER_FILTER_NEAREST, SAMPLER_EXTEND_NONE),
+ NO_BLEND,
+- is_planar_fourcc(frame->id) ?
+- GEN8_WM_KERNEL_VIDEO_PLANAR :
+- GEN8_WM_KERNEL_VIDEO_PACKED,
++ select_video_kernel(frame),
+ 2);
+ tmp.priv = frame;
+
+@@ -3770,17 +3870,6 @@ gen8_render_video(struct sna *sna,
+ gen8_align_vertex(sna, &tmp);
+ gen8_emit_video_state(sna, &tmp);
+
+- /* Set up the offset for translating from the given region (in screen
+- * coordinates) to the backing pixmap.
+- */
+-#ifdef COMPOSITE
+- pix_xoff = -pixmap->screen_x + pixmap->drawable.x;
+- pix_yoff = -pixmap->screen_y + pixmap->drawable.y;
+-#else
+- pix_xoff = 0;
+- pix_yoff = 0;
+-#endif
+-
+ DBG(("%s: src=(%d, %d)x(%d, %d); frame=(%dx%d), dst=(%dx%d)\n",
+ __FUNCTION__,
+ frame->src.x1, frame->src.y1,
+@@ -3802,45 +3891,36 @@ gen8_render_video(struct sna *sna,
+ box = region_rects(dstRegion);
+ nbox = region_num_rects(dstRegion);
+ while (nbox--) {
+- BoxRec r;
+-
+ DBG(("%s: dst=(%d, %d), (%d, %d) + (%d, %d); src=(%f, %f), (%f, %f)\n",
+ __FUNCTION__,
+ box->x1, box->y1,
+ box->x2, box->y2,
+- pix_xoff, pix_yoff,
+ box->x1 * src_scale_x + src_offset_x,
+ box->y1 * src_scale_y + src_offset_y,
+ box->x2 * src_scale_x + src_offset_x,
+ box->y2 * src_scale_y + src_offset_y));
+
+- r.x1 = box->x1 + pix_xoff;
+- r.x2 = box->x2 + pix_xoff;
+- r.y1 = box->y1 + pix_yoff;
+- r.y2 = box->y2 + pix_yoff;
+-
+ gen8_get_rectangles(sna, &tmp, 1, gen8_emit_video_state);
+
+- OUT_VERTEX(r.x2, r.y2);
++ OUT_VERTEX(box->x2, box->y2);
+ OUT_VERTEX_F(box->x2 * src_scale_x + src_offset_x);
+ OUT_VERTEX_F(box->y2 * src_scale_y + src_offset_y);
+
+- OUT_VERTEX(r.x1, r.y2);
++ OUT_VERTEX(box->x1, box->y2);
+ OUT_VERTEX_F(box->x1 * src_scale_x + src_offset_x);
+ OUT_VERTEX_F(box->y2 * src_scale_y + src_offset_y);
+
+- OUT_VERTEX(r.x1, r.y1);
++ OUT_VERTEX(box->x1, box->y1);
+ OUT_VERTEX_F(box->x1 * src_scale_x + src_offset_x);
+ OUT_VERTEX_F(box->y1 * src_scale_y + src_offset_y);
+
+- if (!DAMAGE_IS_ALL(priv->gpu_damage)) {
+- sna_damage_add_box(&priv->gpu_damage, &r);
+- sna_damage_subtract_box(&priv->cpu_damage, &r);
+- }
+ box++;
+ }
+-
+ gen8_vertex_flush(sna);
++
++ if (!DAMAGE_IS_ALL(priv->gpu_damage))
++ sna_damage_add(&priv->gpu_damage, dstRegion);
++
+ return true;
+ }
+ #endif
+@@ -3896,6 +3976,13 @@ static bool gen8_render_setup(struct sna *sna)
+ state->gt = ((devid >> 4) & 0xf) + 1;
+ DBG(("%s: gt=%d\n", __FUNCTION__, state->gt));
+
++ if (is_bdw(sna))
++ state->info = &bdw_gt_info;
++ else if (is_chv(sna))
++ state->info = &chv_gt_info;
++ else
++ return false;
++
+ sna_static_stream_init(&general);
+
+ /* Zero pad the start. If you see an offset of 0x0 in the batchbuffer
+@@ -4007,5 +4094,5 @@ const char *gen8_render_init(struct sna *sna, const char *backend)
+
+ sna->render.max_3d_size = GEN8_MAX_SIZE;
+ sna->render.max_3d_pitch = 1 << 18;
+- return "Broadwell";
++ return sna->render_state.gen8.info->name;
+ }
+diff --git a/src/sna/gen8_render.h b/src/sna/gen8_render.h
+index eb4928e7..e6a8dc55 100644
+--- a/src/sna/gen8_render.h
++++ b/src/sna/gen8_render.h
+@@ -335,6 +335,7 @@
+ #define PIPE_CONTROL_IS_FLUSH (1 << 11)
+ #define PIPE_CONTROL_TC_FLUSH (1 << 10)
+ #define PIPE_CONTROL_NOTIFY_ENABLE (1 << 8)
++#define PIPE_CONTROL_FLUSH (1 << 7)
+ #define PIPE_CONTROL_GLOBAL_GTT (1 << 2)
+ #define PIPE_CONTROL_LOCAL_PGTT (0 << 2)
+ #define PIPE_CONTROL_STALL_AT_SCOREBOARD (1 << 1)
+diff --git a/src/sna/gen9_render.c b/src/sna/gen9_render.c
+new file mode 100644
+index 00000000..e5f12c72
+--- /dev/null
++++ b/src/sna/gen9_render.c
+@@ -0,0 +1,4156 @@
++/*
++ * Copyright © 2012,2013 Intel Corporation
++ *
++ * Permission is hereby granted, free of charge, to any person obtaining a
++ * copy of this software and associated documentation files (the "Software"),
++ * to deal in the Software without restriction, including without limitation
++ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
++ * and/or sell copies of the Software, and to permit persons to whom the
++ * Software is furnished to do so, subject to the following conditions:
++ *
++ * The above copyright notice and this permission notice (including the next
++ * paragraph) shall be included in all copies or substantial portions of the
++ * Software.
++ *
++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
++ * SOFTWARE.
++ *
++ * Authors:
++ * Chris Wilson <chris@chris-wilson.co.uk>
++ *
++ */
++
++#ifdef HAVE_CONFIG_H
++#include "config.h"
++#endif
++
++#include "sna.h"
++#include "sna_reg.h"
++#include "sna_render.h"
++#include "sna_render_inline.h"
++#include "sna_video.h"
++
++#include "gen9_render.h"
++#include "gen8_eu.h"
++#include "gen4_common.h"
++#include "gen4_source.h"
++#include "gen4_vertex.h"
++#include "gen6_common.h"
++#include "gen8_vertex.h"
++
++#define SIM 1
++
++#define ALWAYS_INVALIDATE 0
++#define ALWAYS_FLUSH 0
++#define ALWAYS_STALL 0
++
++#define NO_COMPOSITE 0
++#define NO_COMPOSITE_SPANS 0
++#define NO_COPY 0
++#define NO_COPY_BOXES 0
++#define NO_FILL 0
++#define NO_FILL_BOXES 0
++#define NO_FILL_ONE 0
++#define NO_FILL_CLEAR 0
++#define NO_VIDEO 0
++
++#define USE_8_PIXEL_DISPATCH 1
++#define USE_16_PIXEL_DISPATCH 1
++#define USE_32_PIXEL_DISPATCH 0
++
++#if !USE_8_PIXEL_DISPATCH && !USE_16_PIXEL_DISPATCH && !USE_32_PIXEL_DISPATCH
++#error "Must select at least 8, 16 or 32 pixel dispatch"
++#endif
++
++#define GEN9_MAX_SIZE 16384
++#define GEN9_GT_BIAS 1 /* Each GT is bigger than previous gen */
++
++/* XXX Todo
++ *
++ * STR (software tiled rendering) mode. No, really.
++ * 64x32 pixel blocks align with the rendering cache. Worth considering.
++ */
++
++#define is_aligned(x, y) (((x) & ((y) - 1)) == 0)
++
++/* Pipeline stages:
++ * 1. Command Streamer (CS)
++ * 2. Vertex Fetch (VF)
++ * 3. Vertex Shader (VS)
++ * 4. Hull Shader (HS)
++ * 5. Tesselation Engine (TE)
++ * 6. Domain Shader (DS)
++ * 7. Geometry Shader (GS)
++ * 8. Stream Output Logic (SOL)
++ * 9. Clipper (CLIP)
++ * 10. Strip/Fan (SF)
++ * 11. Windower/Masker (WM)
++ * 12. Color Calculator (CC)
++ */
++
++#if !NO_VIDEO
++static const uint32_t ps_kernel_packed[][4] = {
++#include "exa_wm_src_affine.g8b"
++#include "exa_wm_src_sample_argb.g8b"
++#include "exa_wm_yuv_rgb.g8b"
++#include "exa_wm_write.g8b"
++};
++
++static const uint32_t ps_kernel_planar[][4] = {
++#include "exa_wm_src_affine.g8b"
++#include "exa_wm_src_sample_planar.g8b"
++#include "exa_wm_yuv_rgb.g8b"
++#include "exa_wm_write.g8b"
++};
++
++static const uint32_t ps_kernel_rgb[][4] = {
++#include "exa_wm_src_affine.g8b"
++#include "exa_wm_src_sample_argb.g8b"
++#include "exa_wm_write.g8b"
++};
++#endif
++
++#define SURFACE_DW (64 / sizeof(uint32_t));
++
++#define KERNEL(kernel_enum, kernel, num_surfaces) \
++ [GEN9_WM_KERNEL_##kernel_enum] = {#kernel_enum, kernel, sizeof(kernel), num_surfaces}
++#define NOKERNEL(kernel_enum, func, num_surfaces) \
++ [GEN9_WM_KERNEL_##kernel_enum] = {#kernel_enum, (void *)func, 0, num_surfaces}
++static const struct wm_kernel_info {
++ const char *name;
++ const void *data;
++ unsigned int size;
++ int num_surfaces;
++} wm_kernels[] = {
++ NOKERNEL(NOMASK, gen8_wm_kernel__affine, 2),
++ NOKERNEL(NOMASK_P, gen8_wm_kernel__projective, 2),
++
++ NOKERNEL(MASK, gen8_wm_kernel__affine_mask, 3),
++ NOKERNEL(MASK_P, gen8_wm_kernel__projective_mask, 3),
++
++ NOKERNEL(MASKCA, gen8_wm_kernel__affine_mask_ca, 3),
++ NOKERNEL(MASKCA_P, gen8_wm_kernel__projective_mask_ca, 3),
++
++ NOKERNEL(MASKSA, gen8_wm_kernel__affine_mask_sa, 3),
++ NOKERNEL(MASKSA_P, gen8_wm_kernel__projective_mask_sa, 3),
++
++ NOKERNEL(OPACITY, gen8_wm_kernel__affine_opacity, 2),
++ NOKERNEL(OPACITY_P, gen8_wm_kernel__projective_opacity, 2),
++
++#if !NO_VIDEO
++ KERNEL(VIDEO_PLANAR, ps_kernel_planar, 7),
++ KERNEL(VIDEO_PACKED, ps_kernel_packed, 2),
++ KERNEL(VIDEO_RGB, ps_kernel_rgb, 2),
++#endif
++};
++#undef KERNEL
++
++static const struct blendinfo {
++ uint8_t src_alpha;
++ uint8_t src_blend;
++ uint8_t dst_blend;
++} gen9_blend_op[] = {
++ /* Clear */ {0, BLENDFACTOR_ZERO, BLENDFACTOR_ZERO},
++ /* Src */ {0, BLENDFACTOR_ONE, BLENDFACTOR_ZERO},
++ /* Dst */ {0, BLENDFACTOR_ZERO, BLENDFACTOR_ONE},
++ /* Over */ {1, BLENDFACTOR_ONE, BLENDFACTOR_INV_SRC_ALPHA},
++ /* OverReverse */ {0, BLENDFACTOR_INV_DST_ALPHA, BLENDFACTOR_ONE},
++ /* In */ {0, BLENDFACTOR_DST_ALPHA, BLENDFACTOR_ZERO},
++ /* InReverse */ {1, BLENDFACTOR_ZERO, BLENDFACTOR_SRC_ALPHA},
++ /* Out */ {0, BLENDFACTOR_INV_DST_ALPHA, BLENDFACTOR_ZERO},
++ /* OutReverse */ {1, BLENDFACTOR_ZERO, BLENDFACTOR_INV_SRC_ALPHA},
++ /* Atop */ {1, BLENDFACTOR_DST_ALPHA, BLENDFACTOR_INV_SRC_ALPHA},
++ /* AtopReverse */ {1, BLENDFACTOR_INV_DST_ALPHA, BLENDFACTOR_SRC_ALPHA},
++ /* Xor */ {1, BLENDFACTOR_INV_DST_ALPHA, BLENDFACTOR_INV_SRC_ALPHA},
++ /* Add */ {0, BLENDFACTOR_ONE, BLENDFACTOR_ONE},
++};
++
++/**
++ * Highest-valued BLENDFACTOR used in gen9_blend_op.
++ *
++ * This leaves out GEN9_BLENDFACTOR_INV_DST_COLOR,
++ * GEN9_BLENDFACTOR_INV_CONST_{COLOR,ALPHA},
++ * GEN9_BLENDFACTOR_INV_SRC1_{COLOR,ALPHA}
++ */
++#define GEN9_BLENDFACTOR_COUNT (BLENDFACTOR_INV_DST_ALPHA + 1)
++
++#define GEN9_BLEND_STATE_PADDED_SIZE ALIGN(sizeof(struct gen9_blend_state), 64)
++
++#define BLEND_OFFSET(s, d) \
++ ((d != BLENDFACTOR_ZERO) << 15 | ((s) * GEN9_BLENDFACTOR_COUNT + (d)) << 4)
++
++#define NO_BLEND BLEND_OFFSET(BLENDFACTOR_ONE, BLENDFACTOR_ZERO)
++#define CLEAR BLEND_OFFSET(BLENDFACTOR_ZERO, BLENDFACTOR_ZERO)
++
++#define SAMPLER_OFFSET(sf, se, mf, me) \
++ (((((sf) * EXTEND_COUNT + (se)) * FILTER_COUNT + (mf)) * EXTEND_COUNT + (me)) + 2)
++
++#define VERTEX_2s2s 0
++
++#define COPY_SAMPLER 0
++#define COPY_VERTEX VERTEX_2s2s
++#define COPY_FLAGS(a) GEN9_SET_FLAGS(COPY_SAMPLER, (a) == GXcopy ? NO_BLEND : CLEAR, GEN9_WM_KERNEL_NOMASK, COPY_VERTEX)
++
++#define FILL_SAMPLER 1
++#define FILL_VERTEX VERTEX_2s2s
++#define FILL_FLAGS(op, format) GEN9_SET_FLAGS(FILL_SAMPLER, gen9_get_blend((op), false, (format)), GEN9_WM_KERNEL_NOMASK, FILL_VERTEX)
++#define FILL_FLAGS_NOBLEND GEN9_SET_FLAGS(FILL_SAMPLER, NO_BLEND, GEN9_WM_KERNEL_NOMASK, FILL_VERTEX)
++
++#define GEN9_SAMPLER(f) (((f) >> 20) & 0xfff)
++#define GEN9_BLEND(f) (((f) >> 4) & 0x7ff)
++#define GEN9_READS_DST(f) (((f) >> 15) & 1)
++#define GEN9_KERNEL(f) (((f) >> 16) & 0xf)
++#define GEN9_VERTEX(f) (((f) >> 0) & 0xf)
++#define GEN9_SET_FLAGS(S, B, K, V) ((S) << 20 | (K) << 16 | (B) | (V))
++
++#define OUT_BATCH(v) batch_emit(sna, v)
++#define OUT_BATCH64(v) batch_emit64(sna, v)
++#define OUT_VERTEX(x,y) vertex_emit_2s(sna, x,y)
++#define OUT_VERTEX_F(v) vertex_emit(sna, v)
++
++struct gt_info {
++ const char *name;
++ struct {
++ int max_vs_entries;
++ } urb;
++};
++
++static const struct gt_info min_gt_info = {
++ .name = "Skylake (gen9)",
++ .urb = { .max_vs_entries = 240 },
++};
++
++static const struct gt_info skl_gt_info = {
++ .name = "Skylake (gen9)",
++ .urb = { .max_vs_entries = 960 },
++};
++
++static const struct gt_info bxt_gt_info = {
++ .name = "Broxton (gen9)",
++ .urb = { .max_vs_entries = 320 },
++};
++
++static const struct gt_info kbl_gt_info = {
++ .name = "Kabylake (gen9)",
++ .urb = { .max_vs_entries = 960 },
++};
++
++static const struct gt_info glk_gt_info = {
++ .name = "Geminilake (gen9)",
++ .urb = { .max_vs_entries = 320 },
++};
++
++static bool is_skl(struct sna *sna)
++{
++ return sna->kgem.gen == 0110;
++}
++
++static bool is_bxt(struct sna *sna)
++{
++ return sna->kgem.gen == 0111;
++}
++
++static bool is_kbl(struct sna *sna)
++{
++ return sna->kgem.gen == 0112;
++}
++
++static bool is_glk(struct sna *sna)
++{
++ return sna->kgem.gen == 0113;
++}
++
++
++static inline bool too_large(int width, int height)
++{
++ return width > GEN9_MAX_SIZE || height > GEN9_MAX_SIZE;
++}
++
++static inline bool unaligned(struct kgem_bo *bo, int bpp)
++{
++ /* XXX What exactly do we need to meet H_ALIGN and V_ALIGN? */
++#if 0
++ int x, y;
++
++ if (bo->proxy == NULL)
++ return false;
++
++ /* Assume that all tiled proxies are constructed correctly. */
++ if (bo->tiling)
++ return false;
++
++ DBG(("%s: checking alignment of a linear proxy, offset=%d, pitch=%d, bpp=%d: => (%d, %d)\n",
++ __FUNCTION__, bo->delta, bo->pitch, bpp,
++ 8 * (bo->delta % bo->pitch) / bpp, bo->delta / bo->pitch));
++
++ /* This may be a random userptr map, check that it meets the
++ * render alignment of SURFACE_VALIGN_4 | SURFACE_HALIGN_4.
++ */
++ y = bo->delta / bo->pitch;
++ if (y & 3)
++ return true;
++
++ x = 8 * (bo->delta - y * bo->pitch);
++ if (x & (4*bpp - 1))
++ return true;
++
++ return false;
++#else
++ return false;
++#endif
++}
++
++static uint32_t gen9_get_blend(int op,
++ bool has_component_alpha,
++ uint32_t dst_format)
++{
++ uint32_t src, dst;
++
++ COMPILE_TIME_ASSERT(BLENDFACTOR_INV_DST_ALPHA*GEN9_BLENDFACTOR_COUNT + BLENDFACTOR_INV_DST_ALPHA <= 0x7ff);
++
++ src = gen9_blend_op[op].src_blend;
++ dst = gen9_blend_op[op].dst_blend;
++
++ /* If there's no dst alpha channel, adjust the blend op so that
++ * we'll treat it always as 1.
++ */
++ if (PICT_FORMAT_A(dst_format) == 0) {
++ if (src == BLENDFACTOR_DST_ALPHA)
++ src = BLENDFACTOR_ONE;
++ else if (src == BLENDFACTOR_INV_DST_ALPHA)
++ src = BLENDFACTOR_ZERO;
++ }
++
++ /* If the source alpha is being used, then we should only be in a
++ * case where the source blend factor is 0, and the source blend
++ * value is the mask channels multiplied by the source picture's alpha.
++ */
++ if (has_component_alpha && gen9_blend_op[op].src_alpha) {
++ if (dst == BLENDFACTOR_SRC_ALPHA)
++ dst = BLENDFACTOR_SRC_COLOR;
++ else if (dst == BLENDFACTOR_INV_SRC_ALPHA)
++ dst = BLENDFACTOR_INV_SRC_COLOR;
++ }
++
++ DBG(("blend op=%d, dst=%x [A=%d] => src=%d, dst=%d => offset=%x\n",
++ op, dst_format, PICT_FORMAT_A(dst_format),
++ src, dst, (int)(BLEND_OFFSET(src, dst)>>4)));
++ assert(BLEND_OFFSET(src, dst) >> 4 <= 0xfff);
++ return BLEND_OFFSET(src, dst);
++}
++
++static uint32_t gen9_get_card_format(PictFormat format)
++{
++ switch (format) {
++ default:
++ return -1;
++ case PICT_a8r8g8b8:
++ return SURFACEFORMAT_B8G8R8A8_UNORM;
++ case PICT_x8r8g8b8:
++ return SURFACEFORMAT_B8G8R8X8_UNORM;
++ case PICT_a8b8g8r8:
++ return SURFACEFORMAT_R8G8B8A8_UNORM;
++ case PICT_x8b8g8r8:
++ return SURFACEFORMAT_R8G8B8X8_UNORM;
++#ifdef PICT_a2r10g10b10
++ case PICT_a2r10g10b10:
++ return SURFACEFORMAT_B10G10R10A2_UNORM;
++ case PICT_x2r10g10b10:
++ return SURFACEFORMAT_B10G10R10X2_UNORM;
++#endif
++ case PICT_r8g8b8:
++ return SURFACEFORMAT_R8G8B8_UNORM;
++ case PICT_r5g6b5:
++ return SURFACEFORMAT_B5G6R5_UNORM;
++ case PICT_a1r5g5b5:
++ return SURFACEFORMAT_B5G5R5A1_UNORM;
++ case PICT_a8:
++ return SURFACEFORMAT_A8_UNORM;
++ case PICT_a4r4g4b4:
++ return SURFACEFORMAT_B4G4R4A4_UNORM;
++ }
++}
++
++static uint32_t gen9_get_dest_format(PictFormat format)
++{
++ switch (format) {
++ default:
++ return -1;
++ case PICT_a8r8g8b8:
++ case PICT_x8r8g8b8:
++ return SURFACEFORMAT_B8G8R8A8_UNORM;
++ case PICT_a8b8g8r8:
++ case PICT_x8b8g8r8:
++ return SURFACEFORMAT_R8G8B8A8_UNORM;
++#ifdef PICT_a2r10g10b10
++ case PICT_a2r10g10b10:
++ case PICT_x2r10g10b10:
++ return SURFACEFORMAT_B10G10R10A2_UNORM;
++#endif
++ case PICT_r5g6b5:
++ return SURFACEFORMAT_B5G6R5_UNORM;
++ case PICT_x1r5g5b5:
++ case PICT_a1r5g5b5:
++ return SURFACEFORMAT_B5G5R5A1_UNORM;
++ case PICT_a8:
++ return SURFACEFORMAT_A8_UNORM;
++ case PICT_a4r4g4b4:
++ case PICT_x4r4g4b4:
++ return SURFACEFORMAT_B4G4R4A4_UNORM;
++ }
++}
++
++static bool gen9_check_dst_format(PictFormat format)
++{
++ if (gen9_get_dest_format(format) != -1)
++ return true;
++
++ DBG(("%s: unhandled format: %x\n", __FUNCTION__, (int)format));
++ return false;
++}
++
++static bool gen9_check_format(uint32_t format)
++{
++ if (gen9_get_card_format(format) != -1)
++ return true;
++
++ DBG(("%s: unhandled format: %x\n", __FUNCTION__, (int)format));
++ return false;
++}
++
++static uint32_t gen9_filter(uint32_t filter)
++{
++ switch (filter) {
++ default:
++ assert(0);
++ case PictFilterNearest:
++ return SAMPLER_FILTER_NEAREST;
++ case PictFilterBilinear:
++ return SAMPLER_FILTER_BILINEAR;
++ }
++}
++
++static uint32_t gen9_check_filter(PicturePtr picture)
++{
++ switch (picture->filter) {
++ case PictFilterNearest:
++ case PictFilterBilinear:
++ return true;
++ default:
++ return false;
++ }
++}
++
++static uint32_t gen9_repeat(uint32_t repeat)
++{
++ switch (repeat) {
++ default:
++ assert(0);
++ case RepeatNone:
++ return SAMPLER_EXTEND_NONE;
++ case RepeatNormal:
++ return SAMPLER_EXTEND_REPEAT;
++ case RepeatPad:
++ return SAMPLER_EXTEND_PAD;
++ case RepeatReflect:
++ return SAMPLER_EXTEND_REFLECT;
++ }
++}
++
++static bool gen9_check_repeat(PicturePtr picture)
++{
++ if (!picture->repeat)
++ return true;
++
++ switch (picture->repeatType) {
++ case RepeatNone:
++ case RepeatNormal:
++ case RepeatPad:
++ case RepeatReflect:
++ return true;
++ default:
++ return false;
++ }
++}
++
++static int
++gen9_choose_composite_kernel(int op, bool has_mask, bool is_ca, bool is_affine)
++{
++ int base;
++
++ if (has_mask) {
++ if (is_ca) {
++ if (gen9_blend_op[op].src_alpha)
++ base = GEN9_WM_KERNEL_MASKSA;
++ else
++ base = GEN9_WM_KERNEL_MASKCA;
++ } else
++ base = GEN9_WM_KERNEL_MASK;
++ } else
++ base = GEN9_WM_KERNEL_NOMASK;
++
++ return base + !is_affine;
++}
++
++static void
++gen9_emit_push_constants(struct sna *sna)
++{
++#if SIM
++ OUT_BATCH(GEN9_3DSTATE_PUSH_CONSTANT_ALLOC_VS | (2 - 2));
++ OUT_BATCH(0);
++
++ OUT_BATCH(GEN9_3DSTATE_PUSH_CONSTANT_ALLOC_HS | (2 - 2));
++ OUT_BATCH(0);
++
++ OUT_BATCH(GEN9_3DSTATE_PUSH_CONSTANT_ALLOC_DS | (2 - 2));
++ OUT_BATCH(0);
++
++ OUT_BATCH(GEN9_3DSTATE_PUSH_CONSTANT_ALLOC_GS | (2 - 2));
++ OUT_BATCH(0);
++
++ OUT_BATCH(GEN9_3DSTATE_PUSH_CONSTANT_ALLOC_PS | (2 - 2));
++ OUT_BATCH(0);
++#endif
++}
++
++static void
++gen9_emit_urb(struct sna *sna)
++{
++ /* num of VS entries must be divisible by 8 if size < 9 */
++ OUT_BATCH(GEN9_3DSTATE_URB_VS | (2 - 2));
++ OUT_BATCH(sna->render_state.gen9.info->urb.max_vs_entries << URB_ENTRY_NUMBER_SHIFT |
++ (2 - 1) << URB_ENTRY_SIZE_SHIFT |
++ 4 << URB_STARTING_ADDRESS_SHIFT);
++
++ OUT_BATCH(GEN9_3DSTATE_URB_HS | (2 - 2));
++ OUT_BATCH(0 << URB_ENTRY_SIZE_SHIFT |
++ 4 << URB_STARTING_ADDRESS_SHIFT);
++
++ OUT_BATCH(GEN9_3DSTATE_URB_DS | (2 - 2));
++ OUT_BATCH(0 << URB_ENTRY_SIZE_SHIFT |
++ 4 << URB_STARTING_ADDRESS_SHIFT);
++
++ OUT_BATCH(GEN9_3DSTATE_URB_GS | (2 - 2));
++ OUT_BATCH(0 << URB_ENTRY_SIZE_SHIFT |
++ 4 << URB_STARTING_ADDRESS_SHIFT);
++}
++
++static void
++gen9_emit_state_base_address(struct sna *sna)
++{
++ uint32_t num_pages;
++
++ assert(sna->kgem.surface - sna->kgem.nbatch <= 16384);
++
++ /* WaBindlessSurfaceStateModifyEnable:skl,bxt */
++ OUT_BATCH(GEN9_STATE_BASE_ADDRESS | (19 - 1 - 2));
++ OUT_BATCH64(0); /* general */
++ OUT_BATCH(0); /* stateless dataport */
++ OUT_BATCH64(kgem_add_reloc64(&sna->kgem, /* surface */
++ sna->kgem.nbatch,
++ NULL,
++ I915_GEM_DOMAIN_INSTRUCTION << 16,
++ BASE_ADDRESS_MODIFY));
++ OUT_BATCH64(kgem_add_reloc64(&sna->kgem, /* dynamic */
++ sna->kgem.nbatch,
++ sna->render_state.gen9.general_bo,
++ I915_GEM_DOMAIN_INSTRUCTION << 16,
++ BASE_ADDRESS_MODIFY));
++ OUT_BATCH64(0); /* indirect */
++ OUT_BATCH64(kgem_add_reloc64(&sna->kgem, /* instruction */
++ sna->kgem.nbatch,
++ sna->render_state.gen9.general_bo,
++ I915_GEM_DOMAIN_INSTRUCTION << 16,
++ BASE_ADDRESS_MODIFY));
++ /* upper bounds */
++ num_pages = sna->render_state.gen9.general_bo->size.pages.count;
++ OUT_BATCH(0); /* general */
++ OUT_BATCH(num_pages << 12 | 1); /* dynamic */
++ OUT_BATCH(0); /* indirect */
++ OUT_BATCH(num_pages << 12 | 1); /* instruction */
++
++ /* Bindless */
++ OUT_BATCH(0);
++ OUT_BATCH(0);
++ OUT_BATCH(0);
++}
++
++static void
++gen9_emit_vs_invariant(struct sna *sna)
++{
++ OUT_BATCH(GEN9_3DSTATE_VS | (9 - 2));
++ OUT_BATCH64(0); /* no VS kernel */
++ OUT_BATCH(0);
++ OUT_BATCH64(0); /* scratch */
++ OUT_BATCH(0);
++ OUT_BATCH(1 << 1); /* pass-through */
++ OUT_BATCH(1 << 16 | 1 << 21); /* urb write to SBE */
++
++#if SIM
++ OUT_BATCH(GEN9_3DSTATE_CONSTANT_VS | (11 - 2));
++ OUT_BATCH(0);
++ OUT_BATCH(0);
++ OUT_BATCH64(0);
++ OUT_BATCH64(0);
++ OUT_BATCH64(0);
++ OUT_BATCH64(0);
++
++ OUT_BATCH(GEN9_3DSTATE_BINDING_TABLE_POINTERS_VS | (2 - 2));
++ OUT_BATCH(0);
++
++ OUT_BATCH(GEN9_3DSTATE_SAMPLER_STATE_POINTERS_VS | (2 - 2));
++ OUT_BATCH(0);
++#endif
++}
++
++static void
++gen9_emit_hs_invariant(struct sna *sna)
++{
++ OUT_BATCH(GEN9_3DSTATE_HS | (9 - 2));
++ OUT_BATCH(0);
++ OUT_BATCH(0);
++ OUT_BATCH64(0); /* no HS kernel */
++ OUT_BATCH64(0); /* scratch */
++ OUT_BATCH(0);
++ OUT_BATCH(0); /* pass-through */
++
++#if SIM
++ OUT_BATCH(GEN9_3DSTATE_CONSTANT_HS | (11 - 2));
++ OUT_BATCH(0);
++ OUT_BATCH(0);
++ OUT_BATCH64(0);
++ OUT_BATCH64(0);
++ OUT_BATCH64(0);
++ OUT_BATCH64(0);
++
++#if 1
++ OUT_BATCH(GEN9_3DSTATE_BINDING_TABLE_POINTERS_HS | (2 - 2));
++ OUT_BATCH(0);
++
++ OUT_BATCH(GEN9_3DSTATE_SAMPLER_STATE_POINTERS_HS | (2 - 2));
++ OUT_BATCH(0);
++#endif
++#endif
++}
++
++static void
++gen9_emit_te_invariant(struct sna *sna)
++{
++ OUT_BATCH(GEN9_3DSTATE_TE | (4 - 2));
++ OUT_BATCH(0);
++ OUT_BATCH(0);
++ OUT_BATCH(0);
++}
++
++static void
++gen9_emit_ds_invariant(struct sna *sna)
++{
++ OUT_BATCH(GEN9_3DSTATE_DS | (11 - 2));
++ OUT_BATCH64(0); /* no kernel */
++ OUT_BATCH(0);
++ OUT_BATCH64(0); /* scratch */
++ OUT_BATCH(0);
++ OUT_BATCH(0);
++ OUT_BATCH(0);
++ OUT_BATCH(0);
++ OUT_BATCH(0);
++
++#if SIM
++ OUT_BATCH(GEN9_3DSTATE_CONSTANT_DS | (11 - 2));
++ OUT_BATCH(0);
++ OUT_BATCH(0);
++ OUT_BATCH64(0);
++ OUT_BATCH64(0);
++ OUT_BATCH64(0);
++ OUT_BATCH64(0);
++
++#if 1
++ OUT_BATCH(GEN9_3DSTATE_BINDING_TABLE_POINTERS_DS | (2 - 2));
++ OUT_BATCH(0);
++
++ OUT_BATCH(GEN9_3DSTATE_SAMPLER_STATE_POINTERS_DS | (2 - 2));
++ OUT_BATCH(0);
++#endif
++#endif
++}
++
++static void
++gen9_emit_gs_invariant(struct sna *sna)
++{
++ OUT_BATCH(GEN9_3DSTATE_GS | (10 - 2));
++ OUT_BATCH64(0); /* no GS kernel */
++ OUT_BATCH(0);
++ OUT_BATCH64(0); /* scratch */
++ OUT_BATCH(0);
++ OUT_BATCH(0); /* pass-through */
++ OUT_BATCH(0);
++ OUT_BATCH(0);
++
++#if SIM
++ OUT_BATCH(GEN9_3DSTATE_CONSTANT_GS | (11 - 2));
++ OUT_BATCH(0);
++ OUT_BATCH(0);
++ OUT_BATCH64(0);
++ OUT_BATCH64(0);
++ OUT_BATCH64(0);
++ OUT_BATCH64(0);
++
++#if 1
++ OUT_BATCH(GEN9_3DSTATE_BINDING_TABLE_POINTERS_GS | (2 - 2));
++ OUT_BATCH(0);
++
++ OUT_BATCH(GEN9_3DSTATE_SAMPLER_STATE_POINTERS_GS | (2 - 2));
++ OUT_BATCH(0);
++#endif
++#endif
++}
++
++static void
++gen9_emit_sol_invariant(struct sna *sna)
++{
++ OUT_BATCH(GEN9_3DSTATE_STREAMOUT | (5 - 2));
++ OUT_BATCH(0);
++ OUT_BATCH(0);
++ OUT_BATCH(0);
++ OUT_BATCH(0);
++}
++
++static void
++gen9_emit_sf_invariant(struct sna *sna)
++{
++ OUT_BATCH(GEN9_3DSTATE_SF | (4 - 2));
++ OUT_BATCH(0);
++ OUT_BATCH(0);
++ OUT_BATCH(0);
++}
++
++static void
++gen9_emit_clip_invariant(struct sna *sna)
++{
++ OUT_BATCH(GEN9_3DSTATE_CLIP | (4 - 2));
++ OUT_BATCH(0);
++ OUT_BATCH(0); /* pass-through */
++ OUT_BATCH(0);
++
++ OUT_BATCH(GEN9_3DSTATE_VIEWPORT_STATE_POINTERS_SF_CLIP | (2 - 2));
++ OUT_BATCH(0);
++
++ OUT_BATCH(GEN9_3DSTATE_VIEWPORT_STATE_POINTERS_CC | (2 - 2));
++ OUT_BATCH(0);
++}
++
++static void
++gen9_emit_null_depth_buffer(struct sna *sna)
++{
++ OUT_BATCH(GEN9_3DSTATE_DEPTH_BUFFER | (8 - 2));
++#if 1
++ OUT_BATCH(SURFACE_NULL << DEPTH_BUFFER_TYPE_SHIFT |
++ DEPTHFORMAT_D32_FLOAT << DEPTH_BUFFER_FORMAT_SHIFT);
++#else
++ OUT_BATCH(SURFACE_2D << DEPTH_BUFFER_TYPE_SHIFT |
++ DEPTHFORMAT_D16_UNORM << DEPTH_BUFFER_FORMAT_SHIFT);
++#endif
++ OUT_BATCH64(0);
++ OUT_BATCH(0);
++ OUT_BATCH(0);
++ OUT_BATCH(0);
++ OUT_BATCH(0);
++
++#if SIM
++ OUT_BATCH(GEN9_3DSTATE_HIER_DEPTH_BUFFER | (5 - 2));
++ OUT_BATCH(0);
++ OUT_BATCH64(0);
++ OUT_BATCH(0);
++#endif
++
++#if SIM
++ OUT_BATCH(GEN9_3DSTATE_STENCIL_BUFFER | (5 - 2));
++ OUT_BATCH(0);
++ OUT_BATCH64(0);
++ OUT_BATCH(0);
++#endif
++
++#if SIM
++ OUT_BATCH(GEN9_3DSTATE_WM_DEPTH_STENCIL | (4 - 2));
++ OUT_BATCH(0);
++ OUT_BATCH(0);
++ OUT_BATCH(0);
++#endif
++
++#if SIM
++ OUT_BATCH(GEN9_3DSTATE_CLEAR_PARAMS | (3 - 2));
++ OUT_BATCH(0);
++ OUT_BATCH(0);
++#endif
++}
++
++static void
++gen9_emit_wm_invariant(struct sna *sna)
++{
++ gen9_emit_null_depth_buffer(sna);
++
++#if SIM
++ OUT_BATCH(GEN9_3DSTATE_SCISSOR_STATE_POINTERS | (2 - 2));
++ OUT_BATCH(0);
++#endif
++
++ OUT_BATCH(GEN9_3DSTATE_WM | (2 - 2));
++ //OUT_BATCH(WM_NONPERSPECTIVE_PIXEL_BARYCENTRIC); /* XXX */
++ OUT_BATCH(WM_PERSPECTIVE_PIXEL_BARYCENTRIC);
++
++#if SIM
++ OUT_BATCH(GEN9_3DSTATE_WM_CHROMAKEY | (2 - 2));
++ OUT_BATCH(0);
++#endif
++
++#if 0
++ OUT_BATCH(GEN9_3DSTATE_WM_HZ_OP | (5 - 2));
++ OUT_BATCH(0);
++ OUT_BATCH(0);
++ OUT_BATCH(0);
++ OUT_BATCH(0);
++#endif
++
++ OUT_BATCH(GEN9_3DSTATE_PS_EXTRA | (2 - 2));
++ OUT_BATCH(PSX_PIXEL_SHADER_VALID |
++ PSX_ATTRIBUTE_ENABLE);
++
++ OUT_BATCH(GEN9_3DSTATE_RASTER | (5 - 2));
++ OUT_BATCH(RASTER_FRONT_WINDING_CCW |
++ RASTER_CULL_NONE);
++ OUT_BATCH(0);
++ OUT_BATCH(0);
++ OUT_BATCH(0);
++
++ OUT_BATCH(GEN9_3DSTATE_SBE_SWIZ | (11 - 2));
++ OUT_BATCH(0);
++ OUT_BATCH(0);
++ OUT_BATCH(0);
++ OUT_BATCH(0);
++ OUT_BATCH(0);
++ OUT_BATCH(0);
++ OUT_BATCH(0);
++ OUT_BATCH(0);
++ OUT_BATCH(0);
++ OUT_BATCH(0);
++
++#if SIM
++ OUT_BATCH(GEN9_3DSTATE_CONSTANT_PS | (11 - 2));
++ OUT_BATCH(0);
++ OUT_BATCH(0);
++ OUT_BATCH64(0);
++ OUT_BATCH64(0);
++ OUT_BATCH64(0);
++ OUT_BATCH64(0);
++#endif
++}
++
++static void
++gen9_emit_cc_invariant(struct sna *sna)
++{
++}
++
++static void
++gen9_emit_vf_invariant(struct sna *sna)
++{
++ int n;
++
++#if 1
++ OUT_BATCH(GEN9_3DSTATE_VF | (2 - 2));
++ OUT_BATCH(0);
++#endif
++
++ OUT_BATCH(GEN9_3DSTATE_VF_SGVS | (2 - 2));
++ OUT_BATCH(0);
++
++ OUT_BATCH(GEN9_3DSTATE_VF_TOPOLOGY | (2 - 2));
++ OUT_BATCH(RECTLIST);
++
++ OUT_BATCH(GEN9_3DSTATE_VF_STATISTICS | 0);
++
++ for (n = 1; n <= 3; n++) {
++ OUT_BATCH(GEN9_3DSTATE_VF_INSTANCING | (3 - 2));
++ OUT_BATCH(n);
++ OUT_BATCH(0);
++ }
++}
++
++static void
++gen9_emit_invariant(struct sna *sna)
++{
++ OUT_BATCH(GEN9_PIPELINE_SELECT |
++ PIPELINE_SELECTION_MASK |
++ PIPELINE_SELECT_3D);
++
++#if SIM
++ OUT_BATCH(GEN9_STATE_SIP | (3 - 2));
++ OUT_BATCH64(0);
++#endif
++
++ OUT_BATCH(GEN9_3DSTATE_MULTISAMPLE | (2 - 2));
++ OUT_BATCH(MULTISAMPLE_PIXEL_LOCATION_CENTER |
++ MULTISAMPLE_NUMSAMPLES_1); /* 1 sample/pixel */
++
++ OUT_BATCH(GEN9_3DSTATE_SAMPLE_MASK | (2 - 2));
++ OUT_BATCH(1);
++
++#if SIM
++ OUT_BATCH(GEN9_3DSTATE_SAMPLE_PATTERN | (5 - 2));
++ OUT_BATCH(0);
++ OUT_BATCH(0);
++ OUT_BATCH(0);
++ //OUT_BATCH(8<<20 | 8<<16);
++ OUT_BATCH(0);
++#endif
++
++ gen9_emit_push_constants(sna);
++ gen9_emit_urb(sna);
++
++ gen9_emit_state_base_address(sna);
++
++ gen9_emit_vf_invariant(sna);
++ gen9_emit_vs_invariant(sna);
++ gen9_emit_hs_invariant(sna);
++ gen9_emit_te_invariant(sna);
++ gen9_emit_ds_invariant(sna);
++ gen9_emit_gs_invariant(sna);
++ gen9_emit_sol_invariant(sna);
++ gen9_emit_clip_invariant(sna);
++ gen9_emit_sf_invariant(sna);
++ gen9_emit_wm_invariant(sna);
++ gen9_emit_cc_invariant(sna);
++
++ sna->render_state.gen9.needs_invariant = false;
++}
++
++static void
++gen9_emit_cc(struct sna *sna, uint32_t blend)
++{
++ struct gen9_render_state *render = &sna->render_state.gen9;
++
++ if (render->blend == blend)
++ return;
++
++ DBG(("%s: blend=%x (current=%x), src=%d, dst=%d\n",
++ __FUNCTION__, blend, render->blend,
++ blend / GEN9_BLENDFACTOR_COUNT,
++ blend % GEN9_BLENDFACTOR_COUNT));
++
++ assert(blend < GEN9_BLENDFACTOR_COUNT * GEN9_BLENDFACTOR_COUNT);
++ assert(blend / GEN9_BLENDFACTOR_COUNT > 0);
++ assert(blend % GEN9_BLENDFACTOR_COUNT > 0);
++
++ /* XXX can have up to 8 blend states preload, selectable via
++ * Render Target Index. What other side-effects of Render Target Index?
++ */
++
++ OUT_BATCH(GEN9_3DSTATE_PS_BLEND | (2 - 2));
++ if (blend != GEN9_BLEND(NO_BLEND)) {
++ uint32_t src = blend / GEN9_BLENDFACTOR_COUNT;
++ uint32_t dst = blend % GEN9_BLENDFACTOR_COUNT;
++ OUT_BATCH(PS_BLEND_HAS_WRITEABLE_RT |
++ PS_BLEND_COLOR_BLEND_ENABLE |
++ src << PS_BLEND_SRC_ALPHA_SHIFT |
++ dst << PS_BLEND_DST_ALPHA_SHIFT |
++ src << PS_BLEND_SRC_SHIFT |
++ dst << PS_BLEND_DST_SHIFT);
++ } else
++ OUT_BATCH(PS_BLEND_HAS_WRITEABLE_RT);
++
++ assert(is_aligned(render->cc_blend + blend * GEN9_BLEND_STATE_PADDED_SIZE, 64));
++ OUT_BATCH(GEN9_3DSTATE_BLEND_STATE_POINTERS | (2 - 2));
++ OUT_BATCH((render->cc_blend + blend * GEN9_BLEND_STATE_PADDED_SIZE) | 1);
++
++ /* Force a CC_STATE pointer change to improve blend performance */
++ OUT_BATCH(GEN9_3DSTATE_CC_STATE_POINTERS | (2 - 2));
++ OUT_BATCH(0);
++
++ render->blend = blend;
++}
++
++static void
++gen9_emit_sampler(struct sna *sna, uint32_t state)
++{
++ if (sna->render_state.gen9.samplers == state)
++ return;
++
++ sna->render_state.gen9.samplers = state;
++
++ DBG(("%s: sampler = %x\n", __FUNCTION__, state));
++
++ assert(2 * sizeof(struct gen9_sampler_state) == 32);
++ OUT_BATCH(GEN9_3DSTATE_SAMPLER_STATE_POINTERS_PS | (2 - 2));
++ OUT_BATCH(sna->render_state.gen9.wm_state + state * 2 * sizeof(struct gen9_sampler_state));
++}
++
++static void
++gen9_emit_sf(struct sna *sna, bool has_mask)
++{
++ int num_sf_outputs = has_mask ? 2 : 1;
++
++ if (sna->render_state.gen9.num_sf_outputs == num_sf_outputs)
++ return;
++
++ DBG(("%s: num_sf_outputs=%d\n", __FUNCTION__, num_sf_outputs));
++
++ sna->render_state.gen9.num_sf_outputs = num_sf_outputs;
++
++ OUT_BATCH(GEN9_3DSTATE_SBE | (6 - 2));
++ OUT_BATCH(num_sf_outputs << SBE_NUM_OUTPUTS_SHIFT |
++ SBE_FORCE_VERTEX_URB_READ_LENGTH | /* forced is faster */
++ SBE_FORCE_VERTEX_URB_READ_OFFSET |
++ 1 << SBE_URB_ENTRY_READ_LENGTH_SHIFT |
++ 1 << SBE_URB_ENTRY_READ_OFFSET_SHIFT);
++ OUT_BATCH(0);
++ OUT_BATCH(0);
++ OUT_BATCH(SBE_ACTIVE_COMPONENT_XYZW << 0 |
++ SBE_ACTIVE_COMPONENT_XYZW << 1);
++ OUT_BATCH(0);
++}
++
++static void
++gen9_emit_wm(struct sna *sna, int kernel)
++{
++ const uint32_t *kernels;
++
++ assert(kernel < ARRAY_SIZE(wm_kernels));
++ if (sna->render_state.gen9.kernel == kernel)
++ return;
++
++ sna->render_state.gen9.kernel = kernel;
++ kernels = sna->render_state.gen9.wm_kernel[kernel];
++
++ DBG(("%s: switching to %s, num_surfaces=%d (8-wide? %d, 16-wide? %d, 32-wide? %d)\n",
++ __FUNCTION__,
++ wm_kernels[kernel].name,
++ wm_kernels[kernel].num_surfaces,
++ kernels[0], kernels[1], kernels[2]));
++ assert(is_aligned(kernels[0], 64));
++ assert(is_aligned(kernels[1], 64));
++ assert(is_aligned(kernels[2], 64));
++
++ OUT_BATCH(GEN9_3DSTATE_PS | (12 - 2));
++ OUT_BATCH64(kernels[0] ?: kernels[1] ?: kernels[2]);
++ OUT_BATCH(1 << PS_SAMPLER_COUNT_SHIFT |
++ PS_VECTOR_MASK_ENABLE |
++ wm_kernels[kernel].num_surfaces << PS_BINDING_TABLE_ENTRY_COUNT_SHIFT);
++ OUT_BATCH64(0); /* scratch address */
++ OUT_BATCH(PS_MAX_THREADS |
++ (kernels[0] ? PS_8_DISPATCH_ENABLE : 0) |
++ (kernels[1] ? PS_16_DISPATCH_ENABLE : 0) |
++ (kernels[2] ? PS_32_DISPATCH_ENABLE : 0));
++ OUT_BATCH((kernels[0] ? 4 : kernels[1] ? 6 : 8) << PS_DISPATCH_START_GRF_SHIFT_0 |
++ 8 << PS_DISPATCH_START_GRF_SHIFT_1 |
++ 6 << PS_DISPATCH_START_GRF_SHIFT_2);
++ OUT_BATCH64(kernels[2]);
++ OUT_BATCH64(kernels[1]);
++}
++
++static bool
++gen9_emit_binding_table(struct sna *sna, uint16_t offset)
++{
++ if (sna->render_state.gen9.surface_table == offset)
++ return false;
++
++ /* Binding table pointers */
++ assert(is_aligned(4*offset, 32));
++ OUT_BATCH(GEN9_3DSTATE_BINDING_TABLE_POINTERS_PS | (2 - 2));
++ OUT_BATCH(offset*4);
++
++ sna->render_state.gen9.surface_table = offset;
++ return true;
++}
++
++static bool
++gen9_emit_drawing_rectangle(struct sna *sna,
++ const struct sna_composite_op *op)
++{
++ uint32_t limit = (op->dst.height - 1) << 16 | (op->dst.width - 1);
++ uint32_t offset = (uint16_t)op->dst.y << 16 | (uint16_t)op->dst.x;
++
++ assert(!too_large(abs(op->dst.x), abs(op->dst.y)));
++ assert(!too_large(op->dst.width, op->dst.height));
++
++ if (sna->render_state.gen9.drawrect_limit == limit &&
++ sna->render_state.gen9.drawrect_offset == offset)
++ return true;
++
++ sna->render_state.gen9.drawrect_offset = offset;
++ sna->render_state.gen9.drawrect_limit = limit;
++
++ OUT_BATCH(GEN9_3DSTATE_DRAWING_RECTANGLE | (4 - 2));
++ OUT_BATCH(0);
++ OUT_BATCH(limit);
++ OUT_BATCH(offset);
++ return false;
++}
++
++static void
++gen9_emit_vertex_elements(struct sna *sna,
++ const struct sna_composite_op *op)
++{
++ /*
++ * vertex data in vertex buffer
++ * position: (x, y)
++ * texture coordinate 0: (u0, v0) if (is_affine is true) else (u0, v0, w0)
++ * texture coordinate 1 if (has_mask is true): same as above
++ */
++ struct gen9_render_state *render = &sna->render_state.gen9;
++ uint32_t src_format, dw;
++ int id = GEN9_VERTEX(op->u.gen9.flags);
++ bool has_mask;
++
++ DBG(("%s: setup id=%d\n", __FUNCTION__, id));
++
++ if (render->ve_id == id)
++ return;
++ render->ve_id = id;
++
++ if (render->ve_dirty) {
++ /* dummy primitive to flush vertex before change? */
++ OUT_BATCH(GEN9_3DPRIMITIVE | (7 - 2));
++ OUT_BATCH(0); /* ignored, see VF_TOPOLOGY */
++ OUT_BATCH(0);
++ OUT_BATCH(0);
++ OUT_BATCH(1); /* single instance */
++ OUT_BATCH(0); /* start instance location */
++ OUT_BATCH(0); /* index buffer offset, ignored */
++ }
++
++ /* The VUE layout
++ * dword 0-3: pad (0.0, 0.0, 0.0. 0.0)
++ * dword 4-7: position (x, y, 1.0, 1.0),
++ * dword 8-11: texture coordinate 0 (u0, v0, w0, 1.0)
++ * dword 12-15: texture coordinate 1 (u1, v1, w1, 1.0)
++ *
++ * dword 4-15 are fetched from vertex buffer
++ */
++ has_mask = (id >> 2) != 0;
++ OUT_BATCH(GEN9_3DSTATE_VERTEX_ELEMENTS |
++ ((2 * (3 + has_mask)) + 1 - 2));
++
++ OUT_BATCH(id << VE_INDEX_SHIFT | VE_VALID |
++ SURFACEFORMAT_R32G32B32A32_FLOAT << VE_FORMAT_SHIFT |
++ 0 << VE_OFFSET_SHIFT);
++ OUT_BATCH(COMPONENT_STORE_0 << VE_COMPONENT_0_SHIFT |
++ COMPONENT_STORE_0 << VE_COMPONENT_1_SHIFT |
++ COMPONENT_STORE_0 << VE_COMPONENT_2_SHIFT |
++ COMPONENT_STORE_0 << VE_COMPONENT_3_SHIFT);
++
++ /* x,y */
++ OUT_BATCH(id << VE_INDEX_SHIFT | VE_VALID |
++ SURFACEFORMAT_R16G16_SSCALED << VE_FORMAT_SHIFT |
++ 0 << VE_OFFSET_SHIFT);
++ OUT_BATCH(COMPONENT_STORE_SRC << VE_COMPONENT_0_SHIFT |
++ COMPONENT_STORE_SRC << VE_COMPONENT_1_SHIFT |
++ COMPONENT_STORE_0 << VE_COMPONENT_2_SHIFT |
++ COMPONENT_STORE_1_FLT << VE_COMPONENT_3_SHIFT);
++
++ /* u0, v0, w0 */
++ DBG(("%s: first channel %d floats, offset=4\n", __FUNCTION__, id & 3));
++ dw = COMPONENT_STORE_1_FLT << VE_COMPONENT_3_SHIFT;
++ switch (id & 3) {
++ default:
++ assert(0);
++ case 0:
++ src_format = SURFACEFORMAT_R16G16_SSCALED;
++ dw |= COMPONENT_STORE_SRC << VE_COMPONENT_0_SHIFT;
++ dw |= COMPONENT_STORE_SRC << VE_COMPONENT_1_SHIFT;
++ dw |= COMPONENT_STORE_0 << VE_COMPONENT_2_SHIFT;
++ break;
++ case 1:
++ src_format = SURFACEFORMAT_R32_FLOAT;
++ dw |= COMPONENT_STORE_SRC << VE_COMPONENT_0_SHIFT;
++ dw |= COMPONENT_STORE_0 << VE_COMPONENT_1_SHIFT;
++ dw |= COMPONENT_STORE_0 << VE_COMPONENT_2_SHIFT;
++ break;
++ case 2:
++ src_format = SURFACEFORMAT_R32G32_FLOAT;
++ dw |= COMPONENT_STORE_SRC << VE_COMPONENT_0_SHIFT;
++ dw |= COMPONENT_STORE_SRC << VE_COMPONENT_1_SHIFT;
++ dw |= COMPONENT_STORE_0 << VE_COMPONENT_2_SHIFT;
++ break;
++ case 3:
++ src_format = SURFACEFORMAT_R32G32B32_FLOAT;
++ dw |= COMPONENT_STORE_SRC << VE_COMPONENT_0_SHIFT;
++ dw |= COMPONENT_STORE_SRC << VE_COMPONENT_1_SHIFT;
++ dw |= COMPONENT_STORE_SRC << VE_COMPONENT_2_SHIFT;
++ break;
++ }
++ OUT_BATCH(id << VE_INDEX_SHIFT | VE_VALID |
++ src_format << VE_FORMAT_SHIFT |
++ 4 << VE_OFFSET_SHIFT);
++ OUT_BATCH(dw);
++
++ /* u1, v1, w1 */
++ if (has_mask) {
++ unsigned offset = 4 + ((id & 3) ?: 1) * sizeof(float);
++ DBG(("%s: second channel %d floats, offset=%d\n", __FUNCTION__, (id >> 2) & 3, offset));
++ dw = COMPONENT_STORE_1_FLT << VE_COMPONENT_3_SHIFT;
++ switch (id >> 2) {
++ case 1:
++ src_format = SURFACEFORMAT_R32_FLOAT;
++ dw |= COMPONENT_STORE_SRC << VE_COMPONENT_0_SHIFT;
++ dw |= COMPONENT_STORE_0 << VE_COMPONENT_1_SHIFT;
++ dw |= COMPONENT_STORE_0 << VE_COMPONENT_2_SHIFT;
++ break;
++ default:
++ assert(0);
++ case 2:
++ src_format = SURFACEFORMAT_R32G32_FLOAT;
++ dw |= COMPONENT_STORE_SRC << VE_COMPONENT_0_SHIFT;
++ dw |= COMPONENT_STORE_SRC << VE_COMPONENT_1_SHIFT;
++ dw |= COMPONENT_STORE_0 << VE_COMPONENT_2_SHIFT;
++ break;
++ case 3:
++ src_format = SURFACEFORMAT_R32G32B32_FLOAT;
++ dw |= COMPONENT_STORE_SRC << VE_COMPONENT_0_SHIFT;
++ dw |= COMPONENT_STORE_SRC << VE_COMPONENT_1_SHIFT;
++ dw |= COMPONENT_STORE_SRC << VE_COMPONENT_2_SHIFT;
++ break;
++ }
++ OUT_BATCH(id << VE_INDEX_SHIFT | VE_VALID |
++ src_format << VE_FORMAT_SHIFT |
++ offset << VE_OFFSET_SHIFT);
++ OUT_BATCH(dw);
++ }
++
++ render->ve_dirty = true;
++}
++
++inline static void
++gen9_emit_pipe_invalidate(struct sna *sna)
++{
++ OUT_BATCH(GEN9_PIPE_CONTROL | (6 - 2));
++ OUT_BATCH(PIPE_CONTROL_WC_FLUSH |
++ PIPE_CONTROL_TC_FLUSH |
++ PIPE_CONTROL_CS_STALL);
++ OUT_BATCH64(0);
++ OUT_BATCH64(0);
++}
++
++inline static void
++gen9_emit_pipe_flush(struct sna *sna, bool need_stall)
++{
++ unsigned stall;
++
++ stall = 0;
++ if (need_stall)
++ stall = (PIPE_CONTROL_CS_STALL |
++ PIPE_CONTROL_STALL_AT_SCOREBOARD);
++
++ OUT_BATCH(GEN9_PIPE_CONTROL | (6 - 2));
++ OUT_BATCH(PIPE_CONTROL_WC_FLUSH | stall);
++ OUT_BATCH64(0);
++ OUT_BATCH64(0);
++}
++
++inline static void
++gen9_emit_pipe_stall(struct sna *sna)
++{
++ OUT_BATCH(GEN9_PIPE_CONTROL | (6 - 2));
++ OUT_BATCH(PIPE_CONTROL_CS_STALL |
++ PIPE_CONTROL_FLUSH |
++ PIPE_CONTROL_STALL_AT_SCOREBOARD);
++ OUT_BATCH64(0);
++ OUT_BATCH64(0);
++}
++
++static void
++gen9_emit_state(struct sna *sna,
++ const struct sna_composite_op *op,
++ uint16_t wm_binding_table)
++{
++ bool need_invalidate;
++ bool need_flush;
++ bool need_stall;
++
++ assert(op->dst.bo->exec);
++
++ need_flush = wm_binding_table & 1 ||
++ (sna->render_state.gen9.emit_flush && GEN9_READS_DST(op->u.gen9.flags));
++ if (ALWAYS_FLUSH)
++ need_flush = true;
++
++ wm_binding_table &= ~1;
++
++ need_stall = sna->render_state.gen9.surface_table != wm_binding_table;
++
++ need_invalidate = kgem_bo_is_dirty(op->src.bo) || kgem_bo_is_dirty(op->mask.bo);
++ if (ALWAYS_INVALIDATE)
++ need_invalidate = true;
++
++ need_stall &= gen9_emit_drawing_rectangle(sna, op);
++ if (ALWAYS_STALL)
++ need_stall = true;
++
++ if (need_invalidate) {
++ gen9_emit_pipe_invalidate(sna);
++ kgem_clear_dirty(&sna->kgem);
++ assert(op->dst.bo->exec);
++ kgem_bo_mark_dirty(op->dst.bo);
++
++ need_flush = false;
++ need_stall = false;
++ }
++ if (need_flush) {
++ gen9_emit_pipe_flush(sna, need_stall);
++ need_stall = false;
++ }
++ if (need_stall)
++ gen9_emit_pipe_stall(sna);
++
++ gen9_emit_cc(sna, GEN9_BLEND(op->u.gen9.flags));
++ gen9_emit_sampler(sna, GEN9_SAMPLER(op->u.gen9.flags));
++ gen9_emit_sf(sna, GEN9_VERTEX(op->u.gen9.flags) >> 2);
++ gen9_emit_wm(sna, GEN9_KERNEL(op->u.gen9.flags));
++ gen9_emit_vertex_elements(sna, op);
++ gen9_emit_binding_table(sna, wm_binding_table);
++
++ sna->render_state.gen9.emit_flush = GEN9_READS_DST(op->u.gen9.flags);
++}
++
++static bool gen9_magic_ca_pass(struct sna *sna,
++ const struct sna_composite_op *op)
++{
++ struct gen9_render_state *state = &sna->render_state.gen9;
++
++ if (!op->need_magic_ca_pass)
++ return false;
++
++ DBG(("%s: CA fixup (%d -> %d)\n", __FUNCTION__,
++ sna->render.vertex_start, sna->render.vertex_index));
++
++ gen9_emit_pipe_stall(sna);
++
++ gen9_emit_cc(sna,
++ GEN9_BLEND(gen9_get_blend(PictOpAdd, true,
++ op->dst.format)));
++ gen9_emit_wm(sna,
++ gen9_choose_composite_kernel(PictOpAdd,
++ true, true,
++ op->is_affine));
++
++ OUT_BATCH(GEN9_3DPRIMITIVE | (7 - 2));
++ OUT_BATCH(0); /* ignored, see VF_TOPOLOGY */
++ OUT_BATCH(sna->render.vertex_index - sna->render.vertex_start);
++ OUT_BATCH(sna->render.vertex_start);
++ OUT_BATCH(1); /* single instance */
++ OUT_BATCH(0); /* start instance location */
++ OUT_BATCH(0); /* index buffer offset, ignored */
++
++ state->last_primitive = sna->kgem.nbatch;
++ state->ve_dirty = false;
++ return true;
++}
++
++static void null_create(struct sna_static_stream *stream)
++{
++ /* A bunch of zeros useful for legacy border color and depth-stencil */
++ sna_static_stream_map(stream, 64, 64);
++}
++
++static void
++sampler_state_init(struct gen9_sampler_state *sampler_state,
++ sampler_filter_t filter,
++ sampler_extend_t extend)
++{
++ COMPILE_TIME_ASSERT(sizeof(*sampler_state) == 4*sizeof(uint32_t));
++
++ sampler_state->ss0.lod_preclamp = 2; /* GL mode */
++ sampler_state->ss0.default_color_mode = 1;
++
++ switch (filter) {
++ default:
++ case SAMPLER_FILTER_NEAREST:
++ sampler_state->ss0.min_filter = MAPFILTER_NEAREST;
++ sampler_state->ss0.mag_filter = MAPFILTER_NEAREST;
++ break;
++ case SAMPLER_FILTER_BILINEAR:
++ sampler_state->ss0.min_filter = MAPFILTER_LINEAR;
++ sampler_state->ss0.mag_filter = MAPFILTER_LINEAR;
++ break;
++ }
++
++ /* XXX bicubic filter using MAPFILTER_FLEXIBLE */
++
++ switch (extend) {
++ default:
++ case SAMPLER_EXTEND_NONE:
++ sampler_state->ss3.r_wrap_mode = TEXCOORDMODE_CLAMP_BORDER;
++ sampler_state->ss3.s_wrap_mode = TEXCOORDMODE_CLAMP_BORDER;
++ sampler_state->ss3.t_wrap_mode = TEXCOORDMODE_CLAMP_BORDER;
++ break;
++ case SAMPLER_EXTEND_REPEAT:
++ sampler_state->ss3.r_wrap_mode = TEXCOORDMODE_WRAP;
++ sampler_state->ss3.s_wrap_mode = TEXCOORDMODE_WRAP;
++ sampler_state->ss3.t_wrap_mode = TEXCOORDMODE_WRAP;
++ break;
++ case SAMPLER_EXTEND_PAD:
++ sampler_state->ss3.r_wrap_mode = TEXCOORDMODE_CLAMP;
++ sampler_state->ss3.s_wrap_mode = TEXCOORDMODE_CLAMP;
++ sampler_state->ss3.t_wrap_mode = TEXCOORDMODE_CLAMP;
++ break;
++ case SAMPLER_EXTEND_REFLECT:
++ sampler_state->ss3.r_wrap_mode = TEXCOORDMODE_MIRROR;
++ sampler_state->ss3.s_wrap_mode = TEXCOORDMODE_MIRROR;
++ sampler_state->ss3.t_wrap_mode = TEXCOORDMODE_MIRROR;
++ break;
++ }
++}
++
++static void
++sampler_copy_init(struct gen9_sampler_state *ss)
++{
++ sampler_state_init(ss, SAMPLER_FILTER_NEAREST, SAMPLER_EXTEND_NONE);
++ ss->ss3.non_normalized_coord = 1;
++
++ sampler_state_init(ss+1, SAMPLER_FILTER_NEAREST, SAMPLER_EXTEND_NONE);
++}
++
++static void
++sampler_fill_init(struct gen9_sampler_state *ss)
++{
++ sampler_state_init(ss, SAMPLER_FILTER_NEAREST, SAMPLER_EXTEND_REPEAT);
++ ss->ss3.non_normalized_coord = 1;
++
++ sampler_state_init(ss+1, SAMPLER_FILTER_NEAREST, SAMPLER_EXTEND_NONE);
++}
++
++static uint32_t
++gen9_tiling_bits(uint32_t tiling)
++{
++ switch (tiling) {
++ default: assert(0);
++ case I915_TILING_NONE: return 0;
++ case I915_TILING_X: return SURFACE_TILED;
++ case I915_TILING_Y: return SURFACE_TILED | SURFACE_TILED_Y;
++ }
++}
++
++#define MOCS_PTE (1 << 1)
++#define MOCS_WB (2 << 1)
++
++/**
++ * Sets up the common fields for a surface state buffer for the given
++ * picture in the given surface state buffer.
++ */
++static uint32_t
++gen9_bind_bo(struct sna *sna,
++ struct kgem_bo *bo,
++ uint32_t width,
++ uint32_t height,
++ uint32_t format,
++ bool is_dst)
++{
++ uint32_t *ss;
++ uint32_t domains;
++ int offset;
++ uint32_t is_scanout = is_dst && bo->scanout;
++
++ /* After the first bind, we manage the cache domains within the batch */
++ offset = kgem_bo_get_binding(bo, format | is_dst << 30 | is_scanout << 31);
++ if (offset) {
++ if (is_dst)
++ kgem_bo_mark_dirty(bo);
++ assert(offset >= sna->kgem.surface);
++ return offset * sizeof(uint32_t);
++ }
++
++ offset = sna->kgem.surface -= SURFACE_DW;
++ ss = sna->kgem.batch + offset;
++ ss[0] = (SURFACE_2D << SURFACE_TYPE_SHIFT |
++ gen9_tiling_bits(bo->tiling) |
++ format << SURFACE_FORMAT_SHIFT |
++ SURFACE_VALIGN_4 | SURFACE_HALIGN_4);
++ if (is_dst) {
++ ss[0] |= SURFACE_RC_READ_WRITE;
++ domains = I915_GEM_DOMAIN_RENDER << 16 |I915_GEM_DOMAIN_RENDER;
++ } else
++ domains = I915_GEM_DOMAIN_SAMPLER << 16;
++ ss[1] = (is_scanout || (is_dst && is_uncached(sna, bo))) ? MOCS_PTE << 24 : MOCS_WB << 24;
++ ss[2] = ((width - 1) << SURFACE_WIDTH_SHIFT |
++ (height - 1) << SURFACE_HEIGHT_SHIFT);
++ ss[3] = (bo->pitch - 1) << SURFACE_PITCH_SHIFT;
++ ss[4] = 0;
++ ss[5] = 0;
++ ss[6] = 0;
++ ss[7] = SURFACE_SWIZZLE(RED, GREEN, BLUE, ALPHA);
++ *(uint64_t *)(ss+8) = kgem_add_reloc64(&sna->kgem, offset + 8, bo, domains, 0);
++ ss[10] = 0;
++ ss[11] = 0;
++ ss[12] = 0;
++ ss[13] = 0;
++ ss[14] = 0;
++ ss[15] = 0;
++
++ kgem_bo_set_binding(bo, format | is_dst << 30 | is_scanout << 31, offset);
++
++ DBG(("[%x] bind bo(handle=%d, addr=%lx), format=%d, width=%d, height=%d, pitch=%d, tiling=%d -> %s\n",
++ offset, bo->handle, *(uint64_t *)(ss+8),
++ format, width, height, bo->pitch, bo->tiling,
++ domains & 0xffff ? "render" : "sampler"));
++
++ return offset * sizeof(uint32_t);
++}
++
++static void gen9_emit_vertex_buffer(struct sna *sna,
++ const struct sna_composite_op *op)
++{
++ int id = GEN9_VERTEX(op->u.gen9.flags);
++
++ OUT_BATCH(GEN9_3DSTATE_VERTEX_BUFFERS | (5 - 2));
++ OUT_BATCH(id << VB_INDEX_SHIFT | VB_MODIFY_ENABLE |
++ 4*op->floats_per_vertex);
++ sna->render.vertex_reloc[sna->render.nvertex_reloc++] = sna->kgem.nbatch;
++ OUT_BATCH64(0);
++ OUT_BATCH(~0); /* buffer size: disabled */
++
++ sna->render.vb_id |= 1 << id;
++}
++
++static void gen9_emit_primitive(struct sna *sna)
++{
++ if (sna->kgem.nbatch == sna->render_state.gen9.last_primitive) {
++ sna->render.vertex_offset = sna->kgem.nbatch - 5;
++ return;
++ }
++
++ OUT_BATCH(GEN9_3DPRIMITIVE | (7 - 2));
++ OUT_BATCH(0); /* ignored, see VF_TOPOLOGY */
++ sna->render.vertex_offset = sna->kgem.nbatch;
++ OUT_BATCH(0); /* vertex count, to be filled in later */
++ OUT_BATCH(sna->render.vertex_index);
++ OUT_BATCH(1); /* single instance */
++ OUT_BATCH(0); /* start instance location */
++ OUT_BATCH(0); /* index buffer offset, ignored */
++ sna->render.vertex_start = sna->render.vertex_index;
++
++ sna->render_state.gen9.last_primitive = sna->kgem.nbatch;
++ sna->render_state.gen9.ve_dirty = false;
++}
++
++static bool gen9_rectangle_begin(struct sna *sna,
++ const struct sna_composite_op *op)
++{
++ int id = 1 << GEN9_VERTEX(op->u.gen9.flags);
++ int ndwords;
++
++ if (sna_vertex_wait__locked(&sna->render) && sna->render.vertex_offset)
++ return true;
++
++ ndwords = op->need_magic_ca_pass ? 60 : 6;
++ if ((sna->render.vb_id & id) == 0)
++ ndwords += 5;
++ if (!kgem_check_batch(&sna->kgem, ndwords))
++ return false;
++
++ if ((sna->render.vb_id & id) == 0)
++ gen9_emit_vertex_buffer(sna, op);
++
++ gen9_emit_primitive(sna);
++ return true;
++}
++
++static int gen9_get_rectangles__flush(struct sna *sna,
++ const struct sna_composite_op *op)
++{
++ /* Preventing discarding new vbo after lock contention */
++ if (sna_vertex_wait__locked(&sna->render)) {
++ int rem = vertex_space(sna);
++ if (rem > op->floats_per_rect)
++ return rem;
++ }
++
++ if (!kgem_check_batch(&sna->kgem, op->need_magic_ca_pass ? 65 : 6))
++ return 0;
++ if (!kgem_check_reloc_and_exec(&sna->kgem, 2))
++ return 0;
++
++ if (sna->render.vertex_offset) {
++ gen8_vertex_flush(sna);
++ if (gen9_magic_ca_pass(sna, op)) {
++ gen9_emit_pipe_invalidate(sna);
++ gen9_emit_cc(sna, GEN9_BLEND(op->u.gen9.flags));
++ gen9_emit_wm(sna, GEN9_KERNEL(op->u.gen9.flags));
++ }
++ }
++
++ return gen8_vertex_finish(sna);
++}
++
++inline static int gen9_get_rectangles(struct sna *sna,
++ const struct sna_composite_op *op,
++ int want,
++ void (*emit_state)(struct sna *sna, const struct sna_composite_op *op))
++{
++ int rem;
++
++ assert(want);
++
++start:
++ rem = vertex_space(sna);
++ if (unlikely(rem < op->floats_per_rect)) {
++ DBG(("flushing vbo for %s: %d < %d\n",
++ __FUNCTION__, rem, op->floats_per_rect));
++ rem = gen9_get_rectangles__flush(sna, op);
++ if (unlikely(rem == 0))
++ goto flush;
++ }
++
++ if (unlikely(sna->render.vertex_offset == 0)) {
++ if (!gen9_rectangle_begin(sna, op))
++ goto flush;
++ else
++ goto start;
++ }
++
++ assert(rem <= vertex_space(sna));
++ assert(op->floats_per_rect <= rem);
++ if (want > 1 && want * op->floats_per_rect > rem)
++ want = rem / op->floats_per_rect;
++
++ assert(want > 0);
++ sna->render.vertex_index += 3*want;
++ return want;
++
++flush:
++ if (sna->render.vertex_offset) {
++ gen8_vertex_flush(sna);
++ gen9_magic_ca_pass(sna, op);
++ }
++ sna_vertex_wait__locked(&sna->render);
++ _kgem_submit(&sna->kgem);
++ emit_state(sna, op);
++ goto start;
++}
++
++inline static uint32_t *gen9_composite_get_binding_table(struct sna *sna,
++ uint16_t *offset)
++{
++ uint32_t *table;
++
++ assert(sna->kgem.surface <= 16384);
++ sna->kgem.surface -= SURFACE_DW;
++ /* Clear all surplus entries to zero in case of prefetch */
++ table = memset(sna->kgem.batch + sna->kgem.surface, 0, 64);
++
++ DBG(("%s(%x)\n", __FUNCTION__, 4*sna->kgem.surface));
++
++ *offset = sna->kgem.surface;
++ return table;
++}
++
++static void
++gen9_get_batch(struct sna *sna, const struct sna_composite_op *op)
++{
++ kgem_set_mode(&sna->kgem, KGEM_RENDER, op->dst.bo);
++
++ if (!kgem_check_batch_with_surfaces(&sna->kgem, 150, 2*(1+3))) {
++ DBG(("%s: flushing batch: %d < %d+%d\n",
++ __FUNCTION__, sna->kgem.surface - sna->kgem.nbatch,
++ 150, 4*8*2));
++ _kgem_submit(&sna->kgem);
++ _kgem_set_mode(&sna->kgem, KGEM_RENDER);
++ }
++
++ assert(sna->kgem.mode == KGEM_RENDER);
++ assert(sna->kgem.ring == KGEM_RENDER);
++
++ if (sna->render_state.gen9.needs_invariant)
++ gen9_emit_invariant(sna);
++}
++
++static void gen9_emit_composite_state(struct sna *sna,
++ const struct sna_composite_op *op)
++{
++ uint32_t *binding_table;
++ uint16_t offset, dirty;
++
++ gen9_get_batch(sna, op);
++
++ binding_table = gen9_composite_get_binding_table(sna, &offset);
++
++ dirty = kgem_bo_is_dirty(op->dst.bo);
++
++ binding_table[0] =
++ gen9_bind_bo(sna,
++ op->dst.bo, op->dst.width, op->dst.height,
++ gen9_get_dest_format(op->dst.format),
++ true);
++ binding_table[1] =
++ gen9_bind_bo(sna,
++ op->src.bo, op->src.width, op->src.height,
++ op->src.card_format,
++ false);
++ if (op->mask.bo) {
++ binding_table[2] =
++ gen9_bind_bo(sna,
++ op->mask.bo,
++ op->mask.width,
++ op->mask.height,
++ op->mask.card_format,
++ false);
++ }
++
++ if (sna->kgem.surface == offset &&
++ *(uint64_t *)(sna->kgem.batch + sna->render_state.gen9.surface_table) == *(uint64_t*)binding_table &&
++ (op->mask.bo == NULL ||
++ sna->kgem.batch[sna->render_state.gen9.surface_table+2] == binding_table[2])) {
++ sna->kgem.surface += SURFACE_DW;
++ offset = sna->render_state.gen9.surface_table;
++ }
++
++ if (sna->kgem.batch[sna->render_state.gen9.surface_table] == binding_table[0])
++ dirty = 0;
++
++ gen9_emit_state(sna, op, offset | dirty);
++}
++
++static void
++gen9_align_vertex(struct sna *sna, const struct sna_composite_op *op)
++{
++ if (op->floats_per_vertex != sna->render_state.gen9.floats_per_vertex) {
++ DBG(("aligning vertex: was %d, now %d floats per vertex\n",
++ sna->render_state.gen9.floats_per_vertex, op->floats_per_vertex));
++ gen8_vertex_align(sna, op);
++ sna->render_state.gen9.floats_per_vertex = op->floats_per_vertex;
++ }
++}
++
++fastcall static void
++gen9_render_composite_blt(struct sna *sna,
++ const struct sna_composite_op *op,
++ const struct sna_composite_rectangles *r)
++{
++ gen9_get_rectangles(sna, op, 1, gen9_emit_composite_state);
++ op->prim_emit(sna, op, r);
++}
++
++fastcall static void
++gen9_render_composite_box(struct sna *sna,
++ const struct sna_composite_op *op,
++ const BoxRec *box)
++{
++ struct sna_composite_rectangles r;
++
++ gen9_get_rectangles(sna, op, 1, gen9_emit_composite_state);
++
++ DBG((" %s: (%d, %d), (%d, %d)\n",
++ __FUNCTION__,
++ box->x1, box->y1, box->x2, box->y2));
++
++ r.dst.x = box->x1;
++ r.dst.y = box->y1;
++ r.width = box->x2 - box->x1;
++ r.height = box->y2 - box->y1;
++ r.src = r.mask = r.dst;
++
++ op->prim_emit(sna, op, &r);
++}
++
++static void
++gen9_render_composite_boxes__blt(struct sna *sna,
++ const struct sna_composite_op *op,
++ const BoxRec *box, int nbox)
++{
++ DBG(("composite_boxes(%d)\n", nbox));
++
++ do {
++ int nbox_this_time;
++
++ nbox_this_time = gen9_get_rectangles(sna, op, nbox,
++ gen9_emit_composite_state);
++ nbox -= nbox_this_time;
++
++ do {
++ struct sna_composite_rectangles r;
++
++ DBG((" %s: (%d, %d), (%d, %d)\n",
++ __FUNCTION__,
++ box->x1, box->y1, box->x2, box->y2));
++
++ r.dst.x = box->x1;
++ r.dst.y = box->y1;
++ r.width = box->x2 - box->x1;
++ r.height = box->y2 - box->y1;
++ r.src = r.mask = r.dst;
++
++ op->prim_emit(sna, op, &r);
++ box++;
++ } while (--nbox_this_time);
++ } while (nbox);
++}
++
++static void
++gen9_render_composite_boxes(struct sna *sna,
++ const struct sna_composite_op *op,
++ const BoxRec *box, int nbox)
++{
++ DBG(("%s: nbox=%d\n", __FUNCTION__, nbox));
++
++ do {
++ int nbox_this_time;
++ float *v;
++
++ nbox_this_time = gen9_get_rectangles(sna, op, nbox,
++ gen9_emit_composite_state);
++ assert(nbox_this_time);
++ nbox -= nbox_this_time;
++
++ v = sna->render.vertices + sna->render.vertex_used;
++ sna->render.vertex_used += nbox_this_time * op->floats_per_rect;
++
++ op->emit_boxes(op, box, nbox_this_time, v);
++ box += nbox_this_time;
++ } while (nbox);
++}
++
++static void
++gen9_render_composite_boxes__thread(struct sna *sna,
++ const struct sna_composite_op *op,
++ const BoxRec *box, int nbox)
++{
++ DBG(("%s: nbox=%d\n", __FUNCTION__, nbox));
++
++ sna_vertex_lock(&sna->render);
++ do {
++ int nbox_this_time;
++ float *v;
++
++ nbox_this_time = gen9_get_rectangles(sna, op, nbox,
++ gen9_emit_composite_state);
++ assert(nbox_this_time);
++ nbox -= nbox_this_time;
++
++ v = sna->render.vertices + sna->render.vertex_used;
++ sna->render.vertex_used += nbox_this_time * op->floats_per_rect;
++
++ sna_vertex_acquire__locked(&sna->render);
++ sna_vertex_unlock(&sna->render);
++
++ op->emit_boxes(op, box, nbox_this_time, v);
++ box += nbox_this_time;
++
++ sna_vertex_lock(&sna->render);
++ sna_vertex_release__locked(&sna->render);
++ } while (nbox);
++ sna_vertex_unlock(&sna->render);
++}
++
++static uint32_t
++gen9_create_blend_state(struct sna_static_stream *stream)
++{
++ char *base, *ptr;
++ int src, dst;
++
++ COMPILE_TIME_ASSERT(((GEN9_BLENDFACTOR_COUNT * GEN9_BLENDFACTOR_COUNT << 4) & (1 << 15)) == 0);
++
++ base = sna_static_stream_map(stream,
++ GEN9_BLENDFACTOR_COUNT * GEN9_BLENDFACTOR_COUNT * GEN9_BLEND_STATE_PADDED_SIZE,
++ 64);
++
++ ptr = base;
++ for (src = 0; src < GEN9_BLENDFACTOR_COUNT; src++) {
++ for (dst = 0; dst < GEN9_BLENDFACTOR_COUNT; dst++) {
++ struct gen9_blend_state *blend =
++ (struct gen9_blend_state *)ptr;
++
++ assert(((ptr - base) & 63) == 0);
++ COMPILE_TIME_ASSERT(sizeof(blend->common) == 4);
++ COMPILE_TIME_ASSERT(sizeof(blend->rt) == 8);
++ COMPILE_TIME_ASSERT((char *)&blend->rt - (char *)blend == 4);
++
++ blend->rt.post_blend_clamp = 1;
++ blend->rt.pre_blend_clamp = 1;
++
++ blend->rt.color_blend =
++ !(dst == BLENDFACTOR_ZERO && src == BLENDFACTOR_ONE);
++ blend->rt.dest_blend_factor = dst;
++ blend->rt.source_blend_factor = src;
++ blend->rt.color_blend_function = BLENDFUNCTION_ADD;
++
++ blend->rt.dest_alpha_blend_factor = dst;
++ blend->rt.source_alpha_blend_factor = src;
++ blend->rt.alpha_blend_function = BLENDFUNCTION_ADD;
++
++ ptr += GEN9_BLEND_STATE_PADDED_SIZE;
++ }
++ }
++
++ return sna_static_stream_offsetof(stream, base);
++}
++
++static int
++gen9_composite_picture(struct sna *sna,
++ PicturePtr picture,
++ struct sna_composite_channel *channel,
++ int x, int y,
++ int w, int h,
++ int dst_x, int dst_y,
++ bool precise)
++{
++ PixmapPtr pixmap;
++ uint32_t color;
++ int16_t dx, dy;
++
++ DBG(("%s: (%d, %d)x(%d, %d), dst=(%d, %d)\n",
++ __FUNCTION__, x, y, w, h, dst_x, dst_y));
++
++ channel->is_solid = false;
++ channel->card_format = -1;
++
++ if (sna_picture_is_solid(picture, &color))
++ return gen4_channel_init_solid(sna, channel, color);
++
++ if (picture->pDrawable == NULL) {
++ int ret;
++
++ if (picture->pSourcePict->type == SourcePictTypeLinear)
++ return gen4_channel_init_linear(sna, picture, channel,
++ x, y,
++ w, h,
++ dst_x, dst_y);
++
++ DBG(("%s -- fixup, gradient\n", __FUNCTION__));
++ ret = -1;
++ if (!precise)
++ ret = sna_render_picture_approximate_gradient(sna, picture, channel,
++ x, y, w, h, dst_x, dst_y);
++ if (ret == -1)
++ ret = sna_render_picture_fixup(sna, picture, channel,
++ x, y, w, h, dst_x, dst_y);
++ return ret;
++ }
++
++ if (picture->alphaMap) {
++ DBG(("%s -- fallback, alphamap\n", __FUNCTION__));
++ return sna_render_picture_fixup(sna, picture, channel,
++ x, y, w, h, dst_x, dst_y);
++ }
++
++ if (!gen9_check_repeat(picture))
++ return sna_render_picture_fixup(sna, picture, channel,
++ x, y, w, h, dst_x, dst_y);
++
++ if (!gen9_check_filter(picture))
++ return sna_render_picture_fixup(sna, picture, channel,
++ x, y, w, h, dst_x, dst_y);
++
++ channel->repeat = picture->repeat ? picture->repeatType : RepeatNone;
++ channel->filter = picture->filter;
++
++ pixmap = get_drawable_pixmap(picture->pDrawable);
++ get_drawable_deltas(picture->pDrawable, pixmap, &dx, &dy);
++
++ x += dx + picture->pDrawable->x;
++ y += dy + picture->pDrawable->y;
++
++ channel->is_affine = sna_transform_is_affine(picture->transform);
++ if (sna_transform_is_imprecise_integer_translation(picture->transform, picture->filter, precise, &dx, &dy)) {
++ DBG(("%s: integer translation (%d, %d), removing\n",
++ __FUNCTION__, dx, dy));
++ x += dx;
++ y += dy;
++ channel->transform = NULL;
++ channel->filter = PictFilterNearest;
++
++ if (channel->repeat ||
++ (x >= 0 &&
++ y >= 0 &&
++ x + w <= pixmap->drawable.width &&
++ y + h <= pixmap->drawable.height)) {
++ struct sna_pixmap *priv = sna_pixmap(pixmap);
++ if (priv && priv->clear) {
++ DBG(("%s: converting large pixmap source into solid [%08x]\n", __FUNCTION__, priv->clear_color));
++ return gen4_channel_init_solid(sna, channel, solid_color(picture->format, priv->clear_color));
++ }
++ }
++ } else
++ channel->transform = picture->transform;
++
++ channel->pict_format = picture->format;
++ channel->card_format = gen9_get_card_format(picture->format);
++ if (channel->card_format == (unsigned)-1)
++ return sna_render_picture_convert(sna, picture, channel, pixmap,
++ x, y, w, h, dst_x, dst_y,
++ false);
++
++ if (too_large(pixmap->drawable.width, pixmap->drawable.height)) {
++ DBG(("%s: extracting from pixmap %dx%d\n", __FUNCTION__,
++ pixmap->drawable.width, pixmap->drawable.height));
++ return sna_render_picture_extract(sna, picture, channel,
++ x, y, w, h, dst_x, dst_y);
++ }
++
++ return sna_render_pixmap_bo(sna, channel, pixmap,
++ x, y, w, h, dst_x, dst_y);
++}
++
++inline static bool gen9_composite_channel_convert(struct sna_composite_channel *channel)
++{
++ if (unaligned(channel->bo, PICT_FORMAT_BPP(channel->pict_format)))
++ return false;
++
++ channel->repeat = gen9_repeat(channel->repeat);
++ channel->filter = gen9_filter(channel->filter);
++ if (channel->card_format == (unsigned)-1)
++ channel->card_format = gen9_get_card_format(channel->pict_format);
++ assert(channel->card_format != (unsigned)-1);
++
++ return true;
++}
++
++static void gen9_render_composite_done(struct sna *sna,
++ const struct sna_composite_op *op)
++{
++ if (sna->render.vertex_offset) {
++ gen8_vertex_flush(sna);
++ gen9_magic_ca_pass(sna, op);
++ }
++
++ if (op->mask.bo)
++ kgem_bo_destroy(&sna->kgem, op->mask.bo);
++ if (op->src.bo)
++ kgem_bo_destroy(&sna->kgem, op->src.bo);
++
++ sna_render_composite_redirect_done(sna, op);
++}
++
++inline static bool
++gen9_composite_set_target(struct sna *sna,
++ struct sna_composite_op *op,
++ PicturePtr dst,
++ int x, int y, int w, int h,
++ bool partial)
++{
++ BoxRec box;
++ unsigned int hint;
++
++ DBG(("%s: (%d, %d)x(%d, %d), partial?=%d\n", __FUNCTION__, x, y, w, h, partial));
++
++ op->dst.pixmap = get_drawable_pixmap(dst->pDrawable);
++ op->dst.format = dst->format;
++ op->dst.width = op->dst.pixmap->drawable.width;
++ op->dst.height = op->dst.pixmap->drawable.height;
++
++ if (w | h) {
++ assert(w && h);
++ box.x1 = x;
++ box.y1 = y;
++ box.x2 = x + w;
++ box.y2 = y + h;
++ } else
++ sna_render_picture_extents(dst, &box);
++
++ hint = PREFER_GPU | RENDER_GPU;
++ if (!need_tiling(sna, op->dst.width, op->dst.height))
++ hint |= FORCE_GPU;
++ if (!partial) {
++ hint |= IGNORE_DAMAGE;
++ if (w == op->dst.width && h == op->dst.height)
++ hint |= REPLACES;
++ }
++
++ op->dst.bo = sna_drawable_use_bo(dst->pDrawable, hint, &box, &op->damage);
++ if (op->dst.bo == NULL)
++ return false;
++
++ assert(!op->damage || !DAMAGE_IS_ALL(*op->damage));
++
++ if (unaligned(op->dst.bo, dst->pDrawable->bitsPerPixel))
++ return false;
++
++ if (hint & REPLACES) {
++ struct sna_pixmap *priv = sna_pixmap(op->dst.pixmap);
++ kgem_bo_pair_undo(&sna->kgem, priv->gpu_bo, priv->cpu_bo);
++ }
++
++ get_drawable_deltas(dst->pDrawable, op->dst.pixmap,
++ &op->dst.x, &op->dst.y);
++
++ DBG(("%s: pixmap=%ld, format=%08x, size=%dx%d, pitch=%d, delta=(%d,%d),damage=%p\n",
++ __FUNCTION__,
++ op->dst.pixmap->drawable.serialNumber, (int)op->dst.format,
++ op->dst.width, op->dst.height,
++ op->dst.bo->pitch,
++ op->dst.x, op->dst.y,
++ op->damage ? *op->damage : (void *)-1));
++
++ assert(op->dst.bo->proxy == NULL);
++
++ if (too_large(op->dst.width, op->dst.height) &&
++ !sna_render_composite_redirect(sna, op, x, y, w, h, partial))
++ return false;
++
++ return true;
++}
++
++static bool
++try_blt(struct sna *sna,
++ uint8_t op,
++ PicturePtr src,
++ PicturePtr mask,
++ PicturePtr dst,
++ int16_t src_x, int16_t src_y,
++ int16_t msk_x, int16_t msk_y,
++ int16_t dst_x, int16_t dst_y,
++ int16_t width, int16_t height,
++ unsigned flags,
++ struct sna_composite_op *tmp)
++{
++ struct kgem_bo *bo;
++
++ if (sna->kgem.mode == KGEM_BLT) {
++ DBG(("%s: already performing BLT\n", __FUNCTION__));
++ goto execute;
++ }
++
++ if (too_large(width, height)) {
++ DBG(("%s: operation too large for 3D pipe (%d, %d)\n",
++ __FUNCTION__, width, height));
++ goto execute;
++ }
++
++ bo = __sna_drawable_peek_bo(dst->pDrawable);
++ if (bo == NULL)
++ goto execute;
++
++ if (untiled_tlb_miss(bo))
++ goto execute;
++
++ if (bo->rq) {
++ if (RQ_IS_BLT(bo->rq))
++ goto execute;
++
++ return false;
++ }
++
++ if (bo->tiling == I915_TILING_Y)
++ goto upload;
++
++ if (sna_picture_is_solid(src, NULL) && can_switch_to_blt(sna, bo, 0))
++ goto execute;
++
++ if (src->pDrawable == dst->pDrawable &&
++ (sna->render_state.gt < 3 || width*height < 1024) &&
++ can_switch_to_blt(sna, bo, 0))
++ goto execute;
++
++ if (src->pDrawable) {
++ struct kgem_bo *s = __sna_drawable_peek_bo(src->pDrawable);
++ if (s == NULL)
++ goto upload;
++
++ if (prefer_blt_bo(sna, s, bo))
++ goto execute;
++ }
++
++ if (sna->kgem.ring == KGEM_BLT) {
++ DBG(("%s: already performing BLT\n", __FUNCTION__));
++ goto execute;
++ }
++
++upload:
++ flags |= COMPOSITE_UPLOAD;
++execute:
++ return sna_blt_composite(sna, op,
++ src, dst,
++ src_x, src_y,
++ dst_x, dst_y,
++ width, height,
++ flags, tmp);
++}
++
++static bool
++check_gradient(PicturePtr picture, bool precise)
++{
++ if (picture->pDrawable)
++ return false;
++
++ switch (picture->pSourcePict->type) {
++ case SourcePictTypeSolidFill:
++ case SourcePictTypeLinear:
++ return false;
++ default:
++ return precise;
++ }
++}
++
++static bool
++has_alphamap(PicturePtr p)
++{
++ return p->alphaMap != NULL;
++}
++
++static bool
++need_upload(PicturePtr p)
++{
++ return p->pDrawable && unattached(p->pDrawable) && untransformed(p);
++}
++
++static bool
++source_is_busy(PixmapPtr pixmap)
++{
++ struct sna_pixmap *priv = sna_pixmap(pixmap);
++ if (priv == NULL || priv->clear)
++ return false;
++
++ if (priv->gpu_bo && kgem_bo_is_busy(priv->gpu_bo))
++ return true;
++
++ if (priv->cpu_bo && kgem_bo_is_busy(priv->cpu_bo))
++ return true;
++
++ return priv->gpu_damage && !priv->cpu_damage;
++}
++
++static bool
++source_fallback(PicturePtr p, PixmapPtr pixmap, bool precise)
++{
++ if (sna_picture_is_solid(p, NULL))
++ return false;
++
++ if (p->pSourcePict)
++ return check_gradient(p, precise);
++
++ if (!gen9_check_repeat(p) || !gen9_check_format(p->format))
++ return true;
++
++ if (pixmap && source_is_busy(pixmap))
++ return false;
++
++ return has_alphamap(p) || !gen9_check_filter(p) || need_upload(p);
++}
++
++static bool
++gen9_composite_fallback(struct sna *sna,
++ PicturePtr src,
++ PicturePtr mask,
++ PicturePtr dst)
++{
++ PixmapPtr src_pixmap;
++ PixmapPtr mask_pixmap;
++ PixmapPtr dst_pixmap;
++ bool src_fallback, mask_fallback;
++
++ if (!gen9_check_dst_format(dst->format)) {
++ DBG(("%s: unknown destination format: %d\n",
++ __FUNCTION__, dst->format));
++ return true;
++ }
++
++ dst_pixmap = get_drawable_pixmap(dst->pDrawable);
++
++ src_pixmap = src->pDrawable ? get_drawable_pixmap(src->pDrawable) : NULL;
++ src_fallback = source_fallback(src, src_pixmap,
++ dst->polyMode == PolyModePrecise);
++
++ if (mask) {
++ mask_pixmap = mask->pDrawable ? get_drawable_pixmap(mask->pDrawable) : NULL;
++ mask_fallback = source_fallback(mask, mask_pixmap,
++ dst->polyMode == PolyModePrecise);
++ } else {
++ mask_pixmap = NULL;
++ mask_fallback = false;
++ }
++
++ /* If we are using the destination as a source and need to
++ * readback in order to upload the source, do it all
++ * on the cpu.
++ */
++ if (src_pixmap == dst_pixmap && src_fallback) {
++ DBG(("%s: src is dst and will fallback\n",__FUNCTION__));
++ return true;
++ }
++ if (mask_pixmap == dst_pixmap && mask_fallback) {
++ DBG(("%s: mask is dst and will fallback\n",__FUNCTION__));
++ return true;
++ }
++
++ /* If anything is on the GPU, push everything out to the GPU */
++ if (dst_use_gpu(dst_pixmap)) {
++ DBG(("%s: dst is already on the GPU, try to use GPU\n",
++ __FUNCTION__));
++ return false;
++ }
++
++ if (src_pixmap && !src_fallback) {
++ DBG(("%s: src is already on the GPU, try to use GPU\n",
++ __FUNCTION__));
++ return false;
++ }
++ if (mask_pixmap && !mask_fallback) {
++ DBG(("%s: mask is already on the GPU, try to use GPU\n",
++ __FUNCTION__));
++ return false;
++ }
++
++ /* However if the dst is not on the GPU and we need to
++ * render one of the sources using the CPU, we may
++ * as well do the entire operation in place onthe CPU.
++ */
++ if (src_fallback) {
++ DBG(("%s: dst is on the CPU and src will fallback\n",
++ __FUNCTION__));
++ return true;
++ }
++
++ if (mask && mask_fallback) {
++ DBG(("%s: dst is on the CPU and mask will fallback\n",
++ __FUNCTION__));
++ return true;
++ }
++
++ if (too_large(dst_pixmap->drawable.width,
++ dst_pixmap->drawable.height) &&
++ dst_is_cpu(dst_pixmap)) {
++ DBG(("%s: dst is on the CPU and too large\n", __FUNCTION__));
++ return true;
++ }
++
++ DBG(("%s: dst is not on the GPU and the operation should not fallback\n",
++ __FUNCTION__));
++ return dst_use_cpu(dst_pixmap);
++}
++
++static int
++reuse_source(struct sna *sna,
++ PicturePtr src, struct sna_composite_channel *sc, int src_x, int src_y,
++ PicturePtr mask, struct sna_composite_channel *mc, int msk_x, int msk_y)
++{
++ uint32_t color;
++
++ if (src_x != msk_x || src_y != msk_y)
++ return false;
++
++ if (src == mask) {
++ DBG(("%s: mask is source\n", __FUNCTION__));
++ *mc = *sc;
++ mc->bo = kgem_bo_reference(mc->bo);
++ return true;
++ }
++
++ if (sna_picture_is_solid(mask, &color))
++ return gen4_channel_init_solid(sna, mc, color);
++
++ if (sc->is_solid)
++ return false;
++
++ if (src->pDrawable == NULL || mask->pDrawable != src->pDrawable)
++ return false;
++
++ DBG(("%s: mask reuses source drawable\n", __FUNCTION__));
++
++ if (!sna_transform_equal(src->transform, mask->transform))
++ return false;
++
++ if (!sna_picture_alphamap_equal(src, mask))
++ return false;
++
++ if (!gen9_check_repeat(mask))
++ return false;
++
++ if (!gen9_check_filter(mask))
++ return false;
++
++ if (!gen9_check_format(mask->format))
++ return false;
++
++ DBG(("%s: reusing source channel for mask with a twist\n",
++ __FUNCTION__));
++
++ *mc = *sc;
++ mc->repeat = gen9_repeat(mask->repeat ? mask->repeatType : RepeatNone);
++ mc->filter = gen9_filter(mask->filter);
++ mc->pict_format = mask->format;
++ mc->card_format = gen9_get_card_format(mask->format);
++ mc->bo = kgem_bo_reference(mc->bo);
++ return true;
++}
++
++static bool
++gen9_render_composite(struct sna *sna,
++ uint8_t op,
++ PicturePtr src,
++ PicturePtr mask,
++ PicturePtr dst,
++ int16_t src_x, int16_t src_y,
++ int16_t msk_x, int16_t msk_y,
++ int16_t dst_x, int16_t dst_y,
++ int16_t width, int16_t height,
++ unsigned flags,
++ struct sna_composite_op *tmp)
++{
++ if (op >= ARRAY_SIZE(gen9_blend_op))
++ return false;
++
++ DBG(("%s: %dx%d, current mode=%d/%d\n", __FUNCTION__,
++ width, height, sna->kgem.mode, sna->kgem.ring));
++
++ if (mask == NULL &&
++ try_blt(sna, op,
++ src, mask, dst,
++ src_x, src_y,
++ msk_x, msk_y,
++ dst_x, dst_y,
++ width, height,
++ flags, tmp))
++ return true;
++
++ if (gen9_composite_fallback(sna, src, mask, dst))
++ goto fallback;
++
++ if (need_tiling(sna, width, height))
++ return sna_tiling_composite(op, src, mask, dst,
++ src_x, src_y,
++ msk_x, msk_y,
++ dst_x, dst_y,
++ width, height,
++ tmp);
++
++ if (op == PictOpClear && src == sna->clear)
++ op = PictOpSrc;
++ tmp->op = op;
++ if (!gen9_composite_set_target(sna, tmp, dst,
++ dst_x, dst_y, width, height,
++ flags & COMPOSITE_PARTIAL || op > PictOpSrc))
++ goto fallback;
++
++ switch (gen9_composite_picture(sna, src, &tmp->src,
++ src_x, src_y,
++ width, height,
++ dst_x, dst_y,
++ dst->polyMode == PolyModePrecise)) {
++ case -1:
++ goto cleanup_dst;
++ case 0:
++ if (!gen4_channel_init_solid(sna, &tmp->src, 0))
++ goto cleanup_dst;
++ /* fall through to fixup */
++ case 1:
++ /* Did we just switch rings to prepare the source? */
++ if (mask == NULL &&
++ (prefer_blt_composite(sna, tmp) ||
++ unaligned(tmp->src.bo, PICT_FORMAT_BPP(tmp->src.pict_format))) &&
++ sna_blt_composite__convert(sna,
++ dst_x, dst_y, width, height,
++ tmp))
++ return true;
++
++ if (!gen9_composite_channel_convert(&tmp->src))
++ goto cleanup_src;
++
++ break;
++ }
++
++ tmp->is_affine = tmp->src.is_affine;
++ tmp->has_component_alpha = false;
++ tmp->need_magic_ca_pass = false;
++
++ tmp->mask.bo = NULL;
++ tmp->mask.filter = SAMPLER_FILTER_NEAREST;
++ tmp->mask.repeat = SAMPLER_EXTEND_NONE;
++
++ if (mask) {
++ if (mask->componentAlpha && PICT_FORMAT_RGB(mask->format)) {
++ tmp->has_component_alpha = true;
++
++ /* Check if it's component alpha that relies on a source alpha and on
++ * the source value. We can only get one of those into the single
++ * source value that we get to blend with.
++ */
++ if (gen9_blend_op[op].src_alpha &&
++ (gen9_blend_op[op].src_blend != BLENDFACTOR_ZERO)) {
++ if (op != PictOpOver)
++ goto cleanup_src;
++
++ tmp->need_magic_ca_pass = true;
++ tmp->op = PictOpOutReverse;
++ }
++ }
++
++ if (!reuse_source(sna,
++ src, &tmp->src, src_x, src_y,
++ mask, &tmp->mask, msk_x, msk_y)) {
++ switch (gen9_composite_picture(sna, mask, &tmp->mask,
++ msk_x, msk_y,
++ width, height,
++ dst_x, dst_y,
++ dst->polyMode == PolyModePrecise)) {
++ case -1:
++ goto cleanup_src;
++ case 0:
++ if (!gen4_channel_init_solid(sna, &tmp->mask, 0))
++ goto cleanup_src;
++ /* fall through to fixup */
++ case 1:
++ if (!gen9_composite_channel_convert(&tmp->mask))
++ goto cleanup_mask;
++ break;
++ }
++ }
++
++ tmp->is_affine &= tmp->mask.is_affine;
++ }
++
++ tmp->u.gen9.flags =
++ GEN9_SET_FLAGS(SAMPLER_OFFSET(tmp->src.filter,
++ tmp->src.repeat,
++ tmp->mask.filter,
++ tmp->mask.repeat),
++ gen9_get_blend(tmp->op,
++ tmp->has_component_alpha,
++ tmp->dst.format),
++ gen9_choose_composite_kernel(tmp->op,
++ tmp->mask.bo != NULL,
++ tmp->has_component_alpha,
++ tmp->is_affine),
++ gen4_choose_composite_emitter(sna, tmp));
++
++ tmp->blt = gen9_render_composite_blt;
++ tmp->box = gen9_render_composite_box;
++ tmp->boxes = gen9_render_composite_boxes__blt;
++ if (tmp->emit_boxes){
++ tmp->boxes = gen9_render_composite_boxes;
++ tmp->thread_boxes = gen9_render_composite_boxes__thread;
++ }
++ tmp->done = gen9_render_composite_done;
++
++ kgem_set_mode(&sna->kgem, KGEM_RENDER, tmp->dst.bo);
++ if (!kgem_check_bo(&sna->kgem,
++ tmp->dst.bo, tmp->src.bo, tmp->mask.bo,
++ NULL)) {
++ kgem_submit(&sna->kgem);
++ if (!kgem_check_bo(&sna->kgem,
++ tmp->dst.bo, tmp->src.bo, tmp->mask.bo,
++ NULL))
++ goto cleanup_mask;
++ _kgem_set_mode(&sna->kgem, KGEM_RENDER);
++ }
++
++ gen9_align_vertex(sna, tmp);
++ gen9_emit_composite_state(sna, tmp);
++ return true;
++
++cleanup_mask:
++ if (tmp->mask.bo) {
++ kgem_bo_destroy(&sna->kgem, tmp->mask.bo);
++ tmp->mask.bo = NULL;
++ }
++cleanup_src:
++ if (tmp->src.bo) {
++ kgem_bo_destroy(&sna->kgem, tmp->src.bo);
++ tmp->src.bo = NULL;
++ }
++cleanup_dst:
++ if (tmp->redirect.real_bo) {
++ kgem_bo_destroy(&sna->kgem, tmp->dst.bo);
++ tmp->redirect.real_bo = NULL;
++ }
++fallback:
++ return (mask == NULL &&
++ sna_blt_composite(sna, op,
++ src, dst,
++ src_x, src_y,
++ dst_x, dst_y,
++ width, height,
++ flags | COMPOSITE_FALLBACK, tmp));
++}
++
++#if !NO_COMPOSITE_SPANS
++fastcall static void
++gen9_render_composite_spans_box(struct sna *sna,
++ const struct sna_composite_spans_op *op,
++ const BoxRec *box, float opacity)
++{
++ DBG(("%s: src=+(%d, %d), opacity=%f, dst=+(%d, %d), box=(%d, %d) x (%d, %d)\n",
++ __FUNCTION__,
++ op->base.src.offset[0], op->base.src.offset[1],
++ opacity,
++ op->base.dst.x, op->base.dst.y,
++ box->x1, box->y1,
++ box->x2 - box->x1,
++ box->y2 - box->y1));
++
++ gen9_get_rectangles(sna, &op->base, 1, gen9_emit_composite_state);
++ op->prim_emit(sna, op, box, opacity);
++}
++
++static void
++gen9_render_composite_spans_boxes(struct sna *sna,
++ const struct sna_composite_spans_op *op,
++ const BoxRec *box, int nbox,
++ float opacity)
++{
++ DBG(("%s: nbox=%d, src=+(%d, %d), opacity=%f, dst=+(%d, %d)\n",
++ __FUNCTION__, nbox,
++ op->base.src.offset[0], op->base.src.offset[1],
++ opacity,
++ op->base.dst.x, op->base.dst.y));
++
++ do {
++ int nbox_this_time;
++
++ nbox_this_time = gen9_get_rectangles(sna, &op->base, nbox,
++ gen9_emit_composite_state);
++ nbox -= nbox_this_time;
++
++ do {
++ DBG((" %s: (%d, %d) x (%d, %d)\n", __FUNCTION__,
++ box->x1, box->y1,
++ box->x2 - box->x1,
++ box->y2 - box->y1));
++
++ op->prim_emit(sna, op, box++, opacity);
++ } while (--nbox_this_time);
++ } while (nbox);
++}
++
++fastcall static void
++gen9_render_composite_spans_boxes__thread(struct sna *sna,
++ const struct sna_composite_spans_op *op,
++ const struct sna_opacity_box *box,
++ int nbox)
++{
++ DBG(("%s: nbox=%d, src=+(%d, %d), dst=+(%d, %d)\n",
++ __FUNCTION__, nbox,
++ op->base.src.offset[0], op->base.src.offset[1],
++ op->base.dst.x, op->base.dst.y));
++
++ sna_vertex_lock(&sna->render);
++ do {
++ int nbox_this_time;
++ float *v;
++
++ nbox_this_time = gen9_get_rectangles(sna, &op->base, nbox,
++ gen9_emit_composite_state);
++ assert(nbox_this_time);
++ nbox -= nbox_this_time;
++
++ v = sna->render.vertices + sna->render.vertex_used;
++ sna->render.vertex_used += nbox_this_time * op->base.floats_per_rect;
++
++ sna_vertex_acquire__locked(&sna->render);
++ sna_vertex_unlock(&sna->render);
++
++ op->emit_boxes(op, box, nbox_this_time, v);
++ box += nbox_this_time;
++
++ sna_vertex_lock(&sna->render);
++ sna_vertex_release__locked(&sna->render);
++ } while (nbox);
++ sna_vertex_unlock(&sna->render);
++}
++
++fastcall static void
++gen9_render_composite_spans_done(struct sna *sna,
++ const struct sna_composite_spans_op *op)
++{
++ if (sna->render.vertex_offset)
++ gen8_vertex_flush(sna);
++
++ DBG(("%s()\n", __FUNCTION__));
++
++ if (op->base.src.bo)
++ kgem_bo_destroy(&sna->kgem, op->base.src.bo);
++
++ sna_render_composite_redirect_done(sna, &op->base);
++}
++
++static bool
++gen9_check_composite_spans(struct sna *sna,
++ uint8_t op, PicturePtr src, PicturePtr dst,
++ int16_t width, int16_t height, unsigned flags)
++{
++ if (op >= ARRAY_SIZE(gen9_blend_op))
++ return false;
++
++ if (gen9_composite_fallback(sna, src, NULL, dst))
++ return false;
++
++ if (need_tiling(sna, width, height) &&
++ !is_gpu(sna, dst->pDrawable, PREFER_GPU_SPANS)) {
++ DBG(("%s: fallback, tiled operation not on GPU\n",
++ __FUNCTION__));
++ return false;
++ }
++
++ return true;
++}
++
++static bool
++gen9_render_composite_spans(struct sna *sna,
++ uint8_t op,
++ PicturePtr src,
++ PicturePtr dst,
++ int16_t src_x, int16_t src_y,
++ int16_t dst_x, int16_t dst_y,
++ int16_t width, int16_t height,
++ unsigned flags,
++ struct sna_composite_spans_op *tmp)
++{
++ DBG(("%s: %dx%d with flags=%x, current mode=%d\n", __FUNCTION__,
++ width, height, flags, sna->kgem.ring));
++
++ assert(gen9_check_composite_spans(sna, op, src, dst, width, height, flags));
++
++ if (need_tiling(sna, width, height)) {
++ DBG(("%s: tiling, operation (%dx%d) too wide for pipeline\n",
++ __FUNCTION__, width, height));
++ return sna_tiling_composite_spans(op, src, dst,
++ src_x, src_y, dst_x, dst_y,
++ width, height, flags, tmp);
++ }
++
++ tmp->base.op = op;
++ if (!gen9_composite_set_target(sna, &tmp->base, dst,
++ dst_x, dst_y, width, height, true))
++ return false;
++
++ switch (gen9_composite_picture(sna, src, &tmp->base.src,
++ src_x, src_y,
++ width, height,
++ dst_x, dst_y,
++ dst->polyMode == PolyModePrecise)) {
++ case -1:
++ goto cleanup_dst;
++ case 0:
++ if (!gen4_channel_init_solid(sna, &tmp->base.src, 0))
++ goto cleanup_dst;
++ /* fall through to fixup */
++ case 1:
++ if (!gen9_composite_channel_convert(&tmp->base.src))
++ goto cleanup_src;
++ break;
++ }
++ tmp->base.mask.bo = NULL;
++
++ tmp->base.is_affine = tmp->base.src.is_affine;
++ tmp->base.need_magic_ca_pass = false;
++
++ tmp->base.u.gen9.flags =
++ GEN9_SET_FLAGS(SAMPLER_OFFSET(tmp->base.src.filter,
++ tmp->base.src.repeat,
++ SAMPLER_FILTER_NEAREST,
++ SAMPLER_EXTEND_PAD),
++ gen9_get_blend(tmp->base.op, false, tmp->base.dst.format),
++ GEN9_WM_KERNEL_OPACITY | !tmp->base.is_affine,
++ gen4_choose_spans_emitter(sna, tmp));
++
++ tmp->box = gen9_render_composite_spans_box;
++ tmp->boxes = gen9_render_composite_spans_boxes;
++ if (tmp->emit_boxes)
++ tmp->thread_boxes = gen9_render_composite_spans_boxes__thread;
++ tmp->done = gen9_render_composite_spans_done;
++
++ kgem_set_mode(&sna->kgem, KGEM_RENDER, tmp->base.dst.bo);
++ if (!kgem_check_bo(&sna->kgem,
++ tmp->base.dst.bo, tmp->base.src.bo,
++ NULL)) {
++ kgem_submit(&sna->kgem);
++ if (!kgem_check_bo(&sna->kgem,
++ tmp->base.dst.bo, tmp->base.src.bo,
++ NULL))
++ goto cleanup_src;
++ _kgem_set_mode(&sna->kgem, KGEM_RENDER);
++ }
++
++ gen9_align_vertex(sna, &tmp->base);
++ gen9_emit_composite_state(sna, &tmp->base);
++ return true;
++
++cleanup_src:
++ if (tmp->base.src.bo)
++ kgem_bo_destroy(&sna->kgem, tmp->base.src.bo);
++cleanup_dst:
++ if (tmp->base.redirect.real_bo)
++ kgem_bo_destroy(&sna->kgem, tmp->base.dst.bo);
++ return false;
++}
++#endif
++
++static void
++gen9_emit_copy_state(struct sna *sna,
++ const struct sna_composite_op *op)
++{
++ uint32_t *binding_table;
++ uint16_t offset, dirty;
++
++ gen9_get_batch(sna, op);
++
++ binding_table = gen9_composite_get_binding_table(sna, &offset);
++
++ dirty = kgem_bo_is_dirty(op->dst.bo);
++
++ binding_table[0] =
++ gen9_bind_bo(sna,
++ op->dst.bo, op->dst.width, op->dst.height,
++ gen9_get_dest_format(op->dst.format),
++ true);
++ binding_table[1] =
++ gen9_bind_bo(sna,
++ op->src.bo, op->src.width, op->src.height,
++ op->src.card_format,
++ false);
++
++ if (sna->kgem.surface == offset &&
++ *(uint64_t *)(sna->kgem.batch + sna->render_state.gen9.surface_table) == *(uint64_t*)binding_table) {
++ sna->kgem.surface += SURFACE_DW;
++ offset = sna->render_state.gen9.surface_table;
++ }
++
++ if (sna->kgem.batch[sna->render_state.gen9.surface_table] == binding_table[0])
++ dirty = 0;
++
++ assert(!GEN9_READS_DST(op->u.gen9.flags));
++ gen9_emit_state(sna, op, offset | dirty);
++}
++
++static inline bool
++prefer_blt_copy(struct sna *sna,
++ struct kgem_bo *src_bo,
++ struct kgem_bo *dst_bo,
++ unsigned flags)
++{
++ if (sna->kgem.mode == KGEM_BLT)
++ return true;
++
++ assert((flags & COPY_SYNC) == 0);
++
++ if (untiled_tlb_miss(src_bo) ||
++ untiled_tlb_miss(dst_bo))
++ return true;
++
++ if (flags & COPY_DRI && !sna->kgem.has_semaphores)
++ return false;
++
++ if (force_blt_ring(sna, dst_bo))
++ return true;
++
++ if ((flags & COPY_SMALL ||
++ (sna->render_state.gt < 3 && src_bo == dst_bo)) &&
++ can_switch_to_blt(sna, dst_bo, flags))
++ return true;
++
++ if (kgem_bo_is_render(dst_bo) ||
++ kgem_bo_is_render(src_bo))
++ return false;
++
++ if (flags & COPY_LAST &&
++ sna->render_state.gt < 3 &&
++ can_switch_to_blt(sna, dst_bo, flags))
++ return true;
++
++ if (prefer_render_ring(sna, dst_bo))
++ return false;
++
++ if (!prefer_blt_ring(sna, dst_bo, flags))
++ return false;
++
++ return prefer_blt_bo(sna, src_bo, dst_bo);
++}
++
++static bool
++gen9_render_copy_boxes(struct sna *sna, uint8_t alu,
++ const DrawableRec *src, struct kgem_bo *src_bo, int16_t src_dx, int16_t src_dy,
++ const DrawableRec *dst, struct kgem_bo *dst_bo, int16_t dst_dx, int16_t dst_dy,
++ const BoxRec *box, int n, unsigned flags)
++{
++ struct sna_composite_op tmp;
++ BoxRec extents;
++
++ DBG(("%s (%d, %d)->(%d, %d) x %d, alu=%x, flags=%x, self-copy=%d, overlaps? %d\n",
++ __FUNCTION__, src_dx, src_dy, dst_dx, dst_dy, n, alu, flags,
++ src_bo == dst_bo,
++ overlaps(sna,
++ src_bo, src_dx, src_dy,
++ dst_bo, dst_dx, dst_dy,
++ box, n, flags, &extents)));
++
++ if (prefer_blt_copy(sna, src_bo, dst_bo, flags) &&
++ sna_blt_compare_depth(src, dst) &&
++ sna_blt_copy_boxes(sna, alu,
++ src_bo, src_dx, src_dy,
++ dst_bo, dst_dx, dst_dy,
++ dst->bitsPerPixel,
++ box, n))
++ return true;
++
++ if (!(alu == GXcopy || alu == GXclear) ||
++ unaligned(src_bo, src->bitsPerPixel) ||
++ unaligned(dst_bo, dst->bitsPerPixel)) {
++fallback_blt:
++ DBG(("%s: fallback blt\n", __FUNCTION__));
++ if (!sna_blt_compare_depth(src, dst))
++ return false;
++
++ return sna_blt_copy_boxes_fallback(sna, alu,
++ src, src_bo, src_dx, src_dy,
++ dst, dst_bo, dst_dx, dst_dy,
++ box, n);
++ }
++
++ if (overlaps(sna,
++ src_bo, src_dx, src_dy,
++ dst_bo, dst_dx, dst_dy,
++ box, n, flags,
++ &extents)) {
++ bool big = too_large(extents.x2-extents.x1, extents.y2-extents.y1);
++
++ if ((big || !prefer_render_ring(sna, dst_bo)) &&
++ sna_blt_copy_boxes(sna, alu,
++ src_bo, src_dx, src_dy,
++ dst_bo, dst_dx, dst_dy,
++ dst->bitsPerPixel,
++ box, n))
++ return true;
++
++ if (big)
++ goto fallback_blt;
++
++ assert(src_bo == dst_bo);
++ assert(src->depth == dst->depth);
++ assert(src->width == dst->width);
++ assert(src->height == dst->height);
++ return sna_render_copy_boxes__overlap(sna, alu, dst, dst_bo,
++ src_dx, src_dy,
++ dst_dx, dst_dy,
++ box, n, &extents);
++ }
++
++ if (dst->depth == src->depth) {
++ tmp.dst.format = sna_render_format_for_depth(dst->depth);
++ tmp.src.pict_format = tmp.dst.format;
++ } else {
++ tmp.dst.format = sna_format_for_depth(dst->depth);
++ tmp.src.pict_format = sna_format_for_depth(src->depth);
++ }
++ if (!gen9_check_format(tmp.src.pict_format))
++ goto fallback_blt;
++
++ tmp.dst.pixmap = (PixmapPtr)dst;
++ tmp.dst.width = dst->width;
++ tmp.dst.height = dst->height;
++ tmp.dst.bo = dst_bo;
++ tmp.dst.x = tmp.dst.y = 0;
++ tmp.damage = NULL;
++
++ sna_render_composite_redirect_init(&tmp);
++ if (too_large(tmp.dst.width, tmp.dst.height)) {
++ int i;
++
++ extents = box[0];
++ for (i = 1; i < n; i++) {
++ if (box[i].x1 < extents.x1)
++ extents.x1 = box[i].x1;
++ if (box[i].y1 < extents.y1)
++ extents.y1 = box[i].y1;
++
++ if (box[i].x2 > extents.x2)
++ extents.x2 = box[i].x2;
++ if (box[i].y2 > extents.y2)
++ extents.y2 = box[i].y2;
++ }
++
++ if (!sna_render_composite_redirect(sna, &tmp,
++ extents.x1 + dst_dx,
++ extents.y1 + dst_dy,
++ extents.x2 - extents.x1,
++ extents.y2 - extents.y1,
++ n > 1))
++ goto fallback_tiled;
++ }
++
++ tmp.src.card_format = gen9_get_card_format(tmp.src.pict_format);
++ if (too_large(src->width, src->height)) {
++ int i;
++
++ extents = box[0];
++ for (i = 1; i < n; i++) {
++ if (box[i].x1 < extents.x1)
++ extents.x1 = box[i].x1;
++ if (box[i].y1 < extents.y1)
++ extents.y1 = box[i].y1;
++
++ if (box[i].x2 > extents.x2)
++ extents.x2 = box[i].x2;
++ if (box[i].y2 > extents.y2)
++ extents.y2 = box[i].y2;
++ }
++
++ if (!sna_render_pixmap_partial(sna, src, src_bo, &tmp.src,
++ extents.x1 + src_dx,
++ extents.y1 + src_dy,
++ extents.x2 - extents.x1,
++ extents.y2 - extents.y1))
++ goto fallback_tiled_dst;
++ } else {
++ tmp.src.bo = src_bo;
++ tmp.src.width = src->width;
++ tmp.src.height = src->height;
++ tmp.src.offset[0] = tmp.src.offset[1] = 0;
++ }
++
++ tmp.mask.bo = NULL;
++
++ tmp.floats_per_vertex = 2;
++ tmp.floats_per_rect = 6;
++ tmp.need_magic_ca_pass = 0;
++
++ tmp.u.gen9.flags = COPY_FLAGS(alu);
++
++ kgem_set_mode(&sna->kgem, KGEM_RENDER, tmp.dst.bo);
++ if (!kgem_check_bo(&sna->kgem, tmp.dst.bo, tmp.src.bo, NULL)) {
++ kgem_submit(&sna->kgem);
++ if (!kgem_check_bo(&sna->kgem, tmp.dst.bo, tmp.src.bo, NULL)) {
++ if (tmp.src.bo != src_bo)
++ kgem_bo_destroy(&sna->kgem, tmp.src.bo);
++ if (tmp.redirect.real_bo)
++ kgem_bo_destroy(&sna->kgem, tmp.dst.bo);
++ goto fallback_blt;
++ }
++ _kgem_set_mode(&sna->kgem, KGEM_RENDER);
++ }
++
++ src_dx += tmp.src.offset[0];
++ src_dy += tmp.src.offset[1];
++
++ dst_dx += tmp.dst.x;
++ dst_dy += tmp.dst.y;
++
++ tmp.dst.x = tmp.dst.y = 0;
++
++ gen9_align_vertex(sna, &tmp);
++ gen9_emit_copy_state(sna, &tmp);
++
++ do {
++ int16_t *v;
++ int n_this_time;
++
++ n_this_time = gen9_get_rectangles(sna, &tmp, n,
++ gen9_emit_copy_state);
++ n -= n_this_time;
++
++ v = (int16_t *)(sna->render.vertices + sna->render.vertex_used);
++ sna->render.vertex_used += 6 * n_this_time;
++ assert(sna->render.vertex_used <= sna->render.vertex_size);
++ do {
++
++ DBG((" (%d, %d) -> (%d, %d) + (%d, %d)\n",
++ box->x1 + src_dx, box->y1 + src_dy,
++ box->x1 + dst_dx, box->y1 + dst_dy,
++ box->x2 - box->x1, box->y2 - box->y1));
++ v[0] = box->x2 + dst_dx;
++ v[2] = box->x2 + src_dx;
++ v[1] = v[5] = box->y2 + dst_dy;
++ v[3] = v[7] = box->y2 + src_dy;
++ v[8] = v[4] = box->x1 + dst_dx;
++ v[10] = v[6] = box->x1 + src_dx;
++ v[9] = box->y1 + dst_dy;
++ v[11] = box->y1 + src_dy;
++ v += 12; box++;
++ } while (--n_this_time);
++ } while (n);
++
++ gen8_vertex_flush(sna);
++ sna_render_composite_redirect_done(sna, &tmp);
++ if (tmp.src.bo != src_bo)
++ kgem_bo_destroy(&sna->kgem, tmp.src.bo);
++ return true;
++
++fallback_tiled_dst:
++ if (tmp.redirect.real_bo)
++ kgem_bo_destroy(&sna->kgem, tmp.dst.bo);
++fallback_tiled:
++ DBG(("%s: fallback tiled\n", __FUNCTION__));
++ if (sna_blt_compare_depth(src, dst) &&
++ sna_blt_copy_boxes(sna, alu,
++ src_bo, src_dx, src_dy,
++ dst_bo, dst_dx, dst_dy,
++ dst->bitsPerPixel,
++ box, n))
++ return true;
++
++ return sna_tiling_copy_boxes(sna, alu,
++ src, src_bo, src_dx, src_dy,
++ dst, dst_bo, dst_dx, dst_dy,
++ box, n);
++}
++
++static void
++gen9_render_copy_blt(struct sna *sna,
++ const struct sna_copy_op *op,
++ int16_t sx, int16_t sy,
++ int16_t w, int16_t h,
++ int16_t dx, int16_t dy)
++{
++ int16_t *v;
++
++ gen9_get_rectangles(sna, &op->base, 1, gen9_emit_copy_state);
++
++ v = (int16_t *)&sna->render.vertices[sna->render.vertex_used];
++ sna->render.vertex_used += 6;
++ assert(sna->render.vertex_used <= sna->render.vertex_size);
++
++ v[0] = dx+w; v[1] = dy+h;
++ v[2] = sx+w; v[3] = sy+h;
++ v[4] = dx; v[5] = dy+h;
++ v[6] = sx; v[7] = sy+h;
++ v[8] = dx; v[9] = dy;
++ v[10] = sx; v[11] = sy;
++}
++
++static void
++gen9_render_copy_done(struct sna *sna, const struct sna_copy_op *op)
++{
++ if (sna->render.vertex_offset)
++ gen8_vertex_flush(sna);
++}
++
++static bool
++gen9_render_copy(struct sna *sna, uint8_t alu,
++ PixmapPtr src, struct kgem_bo *src_bo,
++ PixmapPtr dst, struct kgem_bo *dst_bo,
++ struct sna_copy_op *op)
++{
++ DBG(("%s (alu=%d, src=(%dx%d), dst=(%dx%d))\n",
++ __FUNCTION__, alu,
++ src->drawable.width, src->drawable.height,
++ dst->drawable.width, dst->drawable.height));
++
++ if (prefer_blt_copy(sna, src_bo, dst_bo, 0) &&
++ sna_blt_compare_depth(&src->drawable, &dst->drawable) &&
++ sna_blt_copy(sna, alu,
++ src_bo, dst_bo,
++ dst->drawable.bitsPerPixel,
++ op))
++ return true;
++
++ if (!(alu == GXcopy || alu == GXclear) || src_bo == dst_bo ||
++ too_large(src->drawable.width, src->drawable.height) ||
++ too_large(dst->drawable.width, dst->drawable.height) ||
++ unaligned(src_bo, src->drawable.bitsPerPixel) ||
++ unaligned(dst_bo, dst->drawable.bitsPerPixel)) {
++fallback:
++ if (!sna_blt_compare_depth(&src->drawable, &dst->drawable))
++ return false;
++
++ return sna_blt_copy(sna, alu, src_bo, dst_bo,
++ dst->drawable.bitsPerPixel,
++ op);
++ }
++
++ if (dst->drawable.depth == src->drawable.depth) {
++ op->base.dst.format = sna_render_format_for_depth(dst->drawable.depth);
++ op->base.src.pict_format = op->base.dst.format;
++ } else {
++ op->base.dst.format = sna_format_for_depth(dst->drawable.depth);
++ op->base.src.pict_format = sna_format_for_depth(src->drawable.depth);
++ }
++ if (!gen9_check_format(op->base.src.pict_format))
++ goto fallback;
++
++ op->base.dst.pixmap = dst;
++ op->base.dst.width = dst->drawable.width;
++ op->base.dst.height = dst->drawable.height;
++ op->base.dst.bo = dst_bo;
++
++ op->base.src.bo = src_bo;
++ op->base.src.card_format =
++ gen9_get_card_format(op->base.src.pict_format);
++ op->base.src.width = src->drawable.width;
++ op->base.src.height = src->drawable.height;
++
++ op->base.mask.bo = NULL;
++
++ op->base.floats_per_vertex = 2;
++ op->base.floats_per_rect = 6;
++
++ op->base.u.gen9.flags = COPY_FLAGS(alu);
++
++ kgem_set_mode(&sna->kgem, KGEM_RENDER, dst_bo);
++ if (!kgem_check_bo(&sna->kgem, dst_bo, src_bo, NULL)) {
++ kgem_submit(&sna->kgem);
++ if (!kgem_check_bo(&sna->kgem, dst_bo, src_bo, NULL))
++ goto fallback;
++ _kgem_set_mode(&sna->kgem, KGEM_RENDER);
++ }
++
++ gen9_align_vertex(sna, &op->base);
++ gen9_emit_copy_state(sna, &op->base);
++
++ op->blt = gen9_render_copy_blt;
++ op->done = gen9_render_copy_done;
++ return true;
++}
++
++static void
++gen9_emit_fill_state(struct sna *sna, const struct sna_composite_op *op)
++{
++ uint32_t *binding_table;
++ uint16_t offset, dirty;
++
++ /* XXX Render Target Fast Clear
++ * Set RTFC Enable in PS and render a rectangle.
++ * Limited to a clearing the full MSC surface only with a
++ * specific kernel.
++ */
++
++ gen9_get_batch(sna, op);
++
++ binding_table = gen9_composite_get_binding_table(sna, &offset);
++
++ dirty = kgem_bo_is_dirty(op->dst.bo);
++
++ binding_table[0] =
++ gen9_bind_bo(sna,
++ op->dst.bo, op->dst.width, op->dst.height,
++ gen9_get_dest_format(op->dst.format),
++ true);
++ binding_table[1] =
++ gen9_bind_bo(sna,
++ op->src.bo, 1, 1,
++ SURFACEFORMAT_B8G8R8A8_UNORM,
++ false);
++
++ if (sna->kgem.surface == offset &&
++ *(uint64_t *)(sna->kgem.batch + sna->render_state.gen9.surface_table) == *(uint64_t*)binding_table) {
++ sna->kgem.surface += SURFACE_DW;
++ offset = sna->render_state.gen9.surface_table;
++ }
++
++ if (sna->kgem.batch[sna->render_state.gen9.surface_table] == binding_table[0])
++ dirty = 0;
++
++ gen9_emit_state(sna, op, offset | dirty);
++}
++
++static bool
++gen9_render_fill_boxes(struct sna *sna,
++ CARD8 op,
++ PictFormat format,
++ const xRenderColor *color,
++ const DrawableRec *dst, struct kgem_bo *dst_bo,
++ const BoxRec *box, int n)
++{
++ struct sna_composite_op tmp;
++ uint32_t pixel;
++
++ DBG(("%s (op=%d, color=(%04x, %04x, %04x, %04x) [%08x])\n",
++ __FUNCTION__, op,
++ color->red, color->green, color->blue, color->alpha, (int)format));
++
++ if (op >= ARRAY_SIZE(gen9_blend_op)) {
++ DBG(("%s: fallback due to unhandled blend op: %d\n",
++ __FUNCTION__, op));
++ return false;
++ }
++
++ if (prefer_blt_fill(sna, dst_bo, FILL_BOXES) ||
++ !gen9_check_dst_format(format) ||
++ unaligned(dst_bo, PICT_FORMAT_BPP(format))) {
++ uint8_t alu = GXinvalid;
++
++ if (op <= PictOpSrc) {
++ pixel = 0;
++ if (op == PictOpClear)
++ alu = GXclear;
++ else if (sna_get_pixel_from_rgba(&pixel,
++ color->red,
++ color->green,
++ color->blue,
++ color->alpha,
++ format))
++ alu = GXcopy;
++ }
++
++ if (alu != GXinvalid &&
++ sna_blt_fill_boxes(sna, alu,
++ dst_bo, dst->bitsPerPixel,
++ pixel, box, n))
++ return true;
++
++ if (!gen9_check_dst_format(format))
++ return false;
++ }
++
++ if (op == PictOpClear) {
++ pixel = 0;
++ op = PictOpSrc;
++ } else if (!sna_get_pixel_from_rgba(&pixel,
++ color->red,
++ color->green,
++ color->blue,
++ color->alpha,
++ PICT_a8r8g8b8))
++ return false;
++
++ DBG(("%s(%08x x %d [(%d, %d), (%d, %d) ...])\n",
++ __FUNCTION__, pixel, n,
++ box[0].x1, box[0].y1, box[0].x2, box[0].y2));
++
++ tmp.dst.pixmap = (PixmapPtr)dst;
++ tmp.dst.width = dst->width;
++ tmp.dst.height = dst->height;
++ tmp.dst.format = format;
++ tmp.dst.bo = dst_bo;
++ tmp.dst.x = tmp.dst.y = 0;
++ tmp.damage = NULL;
++
++ sna_render_composite_redirect_init(&tmp);
++ if (too_large(dst->width, dst->height)) {
++ BoxRec extents;
++
++ boxes_extents(box, n, &extents);
++ if (!sna_render_composite_redirect(sna, &tmp,
++ extents.x1, extents.y1,
++ extents.x2 - extents.x1,
++ extents.y2 - extents.y1,
++ n > 1))
++ return sna_tiling_fill_boxes(sna, op, format, color,
++ dst, dst_bo, box, n);
++ }
++
++ tmp.src.bo = sna_render_get_solid(sna, pixel);
++ tmp.mask.bo = NULL;
++
++ tmp.floats_per_vertex = 2;
++ tmp.floats_per_rect = 6;
++ tmp.need_magic_ca_pass = false;
++
++ tmp.u.gen9.flags = FILL_FLAGS(op, format);
++
++ kgem_set_mode(&sna->kgem, KGEM_RENDER, dst_bo);
++ if (!kgem_check_bo(&sna->kgem, dst_bo, NULL)) {
++ kgem_submit(&sna->kgem);
++ if (!kgem_check_bo(&sna->kgem, dst_bo, NULL)) {
++ kgem_bo_destroy(&sna->kgem, tmp.src.bo);
++ tmp.src.bo = NULL;
++
++ if (tmp.redirect.real_bo) {
++ kgem_bo_destroy(&sna->kgem, tmp.dst.bo);
++ tmp.redirect.real_bo = NULL;
++ }
++
++ return false;
++ }
++ _kgem_set_mode(&sna->kgem, KGEM_RENDER);
++ }
++
++ gen9_align_vertex(sna, &tmp);
++ gen9_emit_fill_state(sna, &tmp);
++
++ do {
++ int n_this_time;
++ int16_t *v;
++
++ n_this_time = gen9_get_rectangles(sna, &tmp, n,
++ gen9_emit_fill_state);
++ n -= n_this_time;
++
++ v = (int16_t *)(sna->render.vertices + sna->render.vertex_used);
++ sna->render.vertex_used += 6 * n_this_time;
++ assert(sna->render.vertex_used <= sna->render.vertex_size);
++ do {
++ DBG((" (%d, %d), (%d, %d)\n",
++ box->x1, box->y1, box->x2, box->y2));
++
++ v[0] = box->x2;
++ v[5] = v[1] = box->y2;
++ v[8] = v[4] = box->x1;
++ v[9] = box->y1;
++ v[2] = v[3] = v[7] = 1;
++ v[6] = v[10] = v[11] = 0;
++ v += 12; box++;
++ } while (--n_this_time);
++ } while (n);
++
++ gen8_vertex_flush(sna);
++ kgem_bo_destroy(&sna->kgem, tmp.src.bo);
++ sna_render_composite_redirect_done(sna, &tmp);
++ return true;
++}
++
++static void
++gen9_render_fill_op_blt(struct sna *sna,
++ const struct sna_fill_op *op,
++ int16_t x, int16_t y, int16_t w, int16_t h)
++{
++ int16_t *v;
++
++ DBG(("%s: (%d, %d)x(%d, %d)\n", __FUNCTION__, x, y, w, h));
++
++ gen9_get_rectangles(sna, &op->base, 1, gen9_emit_fill_state);
++
++ v = (int16_t *)&sna->render.vertices[sna->render.vertex_used];
++ sna->render.vertex_used += 6;
++ assert(sna->render.vertex_used <= sna->render.vertex_size);
++
++ v[0] = x+w;
++ v[4] = v[8] = x;
++ v[1] = v[5] = y+h;
++ v[9] = y;
++
++ v[2] = v[3] = v[7] = 1;
++ v[6] = v[10] = v[11] = 0;
++}
++
++fastcall static void
++gen9_render_fill_op_box(struct sna *sna,
++ const struct sna_fill_op *op,
++ const BoxRec *box)
++{
++ int16_t *v;
++
++ DBG(("%s: (%d, %d),(%d, %d)\n", __FUNCTION__,
++ box->x1, box->y1, box->x2, box->y2));
++
++ gen9_get_rectangles(sna, &op->base, 1, gen9_emit_fill_state);
++
++ v = (int16_t *)&sna->render.vertices[sna->render.vertex_used];
++ sna->render.vertex_used += 6;
++ assert(sna->render.vertex_used <= sna->render.vertex_size);
++
++ v[0] = box->x2;
++ v[8] = v[4] = box->x1;
++ v[5] = v[1] = box->y2;
++ v[9] = box->y1;
++
++ v[7] = v[2] = v[3] = 1;
++ v[6] = v[10] = v[11] = 0;
++}
++
++fastcall static void
++gen9_render_fill_op_boxes(struct sna *sna,
++ const struct sna_fill_op *op,
++ const BoxRec *box,
++ int nbox)
++{
++ DBG(("%s: (%d, %d),(%d, %d)... x %d\n", __FUNCTION__,
++ box->x1, box->y1, box->x2, box->y2, nbox));
++
++ do {
++ int nbox_this_time;
++ int16_t *v;
++
++ nbox_this_time = gen9_get_rectangles(sna, &op->base, nbox,
++ gen9_emit_fill_state);
++ nbox -= nbox_this_time;
++
++ v = (int16_t *)&sna->render.vertices[sna->render.vertex_used];
++ sna->render.vertex_used += 6 * nbox_this_time;
++ assert(sna->render.vertex_used <= sna->render.vertex_size);
++
++ do {
++ v[0] = box->x2;
++ v[8] = v[4] = box->x1;
++ v[5] = v[1] = box->y2;
++ v[9] = box->y1;
++ v[7] = v[2] = v[3] = 1;
++ v[6] = v[10] = v[11] = 0;
++ box++; v += 12;
++ } while (--nbox_this_time);
++ } while (nbox);
++}
++
++static void
++gen9_render_fill_op_done(struct sna *sna, const struct sna_fill_op *op)
++{
++ if (sna->render.vertex_offset)
++ gen8_vertex_flush(sna);
++ kgem_bo_destroy(&sna->kgem, op->base.src.bo);
++}
++
++static bool
++gen9_render_fill(struct sna *sna, uint8_t alu,
++ PixmapPtr dst, struct kgem_bo *dst_bo,
++ uint32_t color, unsigned flags,
++ struct sna_fill_op *op)
++{
++ DBG(("%s: (alu=%d, color=%x)\n", __FUNCTION__, alu, color));
++
++ if (prefer_blt_fill(sna, dst_bo, flags) &&
++ sna_blt_fill(sna, alu,
++ dst_bo, dst->drawable.bitsPerPixel,
++ color,
++ op))
++ return true;
++
++ if (!(alu == GXcopy || alu == GXclear) ||
++ too_large(dst->drawable.width, dst->drawable.height) ||
++ unaligned(dst_bo, dst->drawable.bitsPerPixel))
++ return sna_blt_fill(sna, alu,
++ dst_bo, dst->drawable.bitsPerPixel,
++ color,
++ op);
++
++ if (alu == GXclear)
++ color = 0;
++
++ op->base.dst.pixmap = dst;
++ op->base.dst.width = dst->drawable.width;
++ op->base.dst.height = dst->drawable.height;
++ op->base.dst.format = sna_format_for_depth(dst->drawable.depth);
++ op->base.dst.bo = dst_bo;
++ op->base.dst.x = op->base.dst.y = 0;
++
++ op->base.src.bo =
++ sna_render_get_solid(sna,
++ sna_rgba_for_color(color,
++ dst->drawable.depth));
++ op->base.mask.bo = NULL;
++
++ op->base.need_magic_ca_pass = false;
++ op->base.floats_per_vertex = 2;
++ op->base.floats_per_rect = 6;
++
++ op->base.u.gen9.flags = FILL_FLAGS_NOBLEND;
++
++ kgem_set_mode(&sna->kgem, KGEM_RENDER, dst_bo);
++ if (!kgem_check_bo(&sna->kgem, dst_bo, NULL)) {
++ kgem_submit(&sna->kgem);
++ if (!kgem_check_bo(&sna->kgem, dst_bo, NULL)) {
++ kgem_bo_destroy(&sna->kgem, op->base.src.bo);
++ return false;
++ }
++
++ _kgem_set_mode(&sna->kgem, KGEM_RENDER);
++ }
++
++ gen9_align_vertex(sna, &op->base);
++ gen9_emit_fill_state(sna, &op->base);
++
++ op->blt = gen9_render_fill_op_blt;
++ op->box = gen9_render_fill_op_box;
++ op->boxes = gen9_render_fill_op_boxes;
++ op->points = NULL;
++ op->done = gen9_render_fill_op_done;
++ return true;
++}
++
++static bool
++gen9_render_fill_one_try_blt(struct sna *sna, PixmapPtr dst, struct kgem_bo *bo,
++ uint32_t color,
++ int16_t x1, int16_t y1, int16_t x2, int16_t y2,
++ uint8_t alu)
++{
++ BoxRec box;
++
++ box.x1 = x1;
++ box.y1 = y1;
++ box.x2 = x2;
++ box.y2 = y2;
++
++ return sna_blt_fill_boxes(sna, alu,
++ bo, dst->drawable.bitsPerPixel,
++ color, &box, 1);
++}
++
++static bool
++gen9_render_fill_one(struct sna *sna, PixmapPtr dst, struct kgem_bo *bo,
++ uint32_t color,
++ int16_t x1, int16_t y1,
++ int16_t x2, int16_t y2,
++ uint8_t alu)
++{
++ struct sna_composite_op tmp;
++ int16_t *v;
++
++ /* Prefer to use the BLT if already engaged */
++ if (prefer_blt_fill(sna, bo, FILL_BOXES) &&
++ gen9_render_fill_one_try_blt(sna, dst, bo, color,
++ x1, y1, x2, y2, alu))
++ return true;
++
++ /* Must use the BLT if we can't RENDER... */
++ if (!(alu == GXcopy || alu == GXclear) ||
++ too_large(dst->drawable.width, dst->drawable.height) ||
++ unaligned(bo, dst->drawable.bitsPerPixel))
++ return gen9_render_fill_one_try_blt(sna, dst, bo, color,
++ x1, y1, x2, y2, alu);
++
++ if (alu == GXclear)
++ color = 0;
++
++ tmp.dst.pixmap = dst;
++ tmp.dst.width = dst->drawable.width;
++ tmp.dst.height = dst->drawable.height;
++ tmp.dst.format = sna_format_for_depth(dst->drawable.depth);
++ tmp.dst.bo = bo;
++ tmp.dst.x = tmp.dst.y = 0;
++
++ tmp.src.bo =
++ sna_render_get_solid(sna,
++ sna_rgba_for_color(color,
++ dst->drawable.depth));
++ tmp.mask.bo = NULL;
++
++ tmp.floats_per_vertex = 2;
++ tmp.floats_per_rect = 6;
++ tmp.need_magic_ca_pass = false;
++
++ tmp.u.gen9.flags = FILL_FLAGS_NOBLEND;
++
++ kgem_set_mode(&sna->kgem, KGEM_RENDER, bo);
++ if (!kgem_check_bo(&sna->kgem, bo, NULL)) {
++ kgem_submit(&sna->kgem);
++ if (kgem_check_bo(&sna->kgem, bo, NULL)) {
++ kgem_bo_destroy(&sna->kgem, tmp.src.bo);
++ return false;
++ }
++ _kgem_set_mode(&sna->kgem, KGEM_RENDER);
++ }
++
++ gen9_align_vertex(sna, &tmp);
++ gen9_emit_fill_state(sna, &tmp);
++
++ gen9_get_rectangles(sna, &tmp, 1, gen9_emit_fill_state);
++
++ DBG((" (%d, %d), (%d, %d)\n", x1, y1, x2, y2));
++
++ v = (int16_t *)&sna->render.vertices[sna->render.vertex_used];
++ sna->render.vertex_used += 6;
++ assert(sna->render.vertex_used <= sna->render.vertex_size);
++
++ v[0] = x2;
++ v[8] = v[4] = x1;
++ v[5] = v[1] = y2;
++ v[9] = y1;
++ v[7] = v[2] = v[3] = 1;
++ v[6] = v[10] = v[11] = 0;
++
++ gen8_vertex_flush(sna);
++ kgem_bo_destroy(&sna->kgem, tmp.src.bo);
++
++ return true;
++}
++
++static bool
++gen9_render_clear_try_blt(struct sna *sna, PixmapPtr dst, struct kgem_bo *bo)
++{
++ BoxRec box;
++
++ box.x1 = 0;
++ box.y1 = 0;
++ box.x2 = dst->drawable.width;
++ box.y2 = dst->drawable.height;
++
++ return sna_blt_fill_boxes(sna, GXclear,
++ bo, dst->drawable.bitsPerPixel,
++ 0, &box, 1);
++}
++
++static bool
++gen9_render_clear(struct sna *sna, PixmapPtr dst, struct kgem_bo *bo)
++{
++ struct sna_composite_op tmp;
++ int16_t *v;
++
++ DBG(("%s: %dx%d\n",
++ __FUNCTION__,
++ dst->drawable.width,
++ dst->drawable.height));
++
++ /* Prefer to use the BLT if already engaged */
++ if (sna->kgem.mode == KGEM_BLT &&
++ gen9_render_clear_try_blt(sna, dst, bo))
++ return true;
++
++ /* Must use the BLT if we can't RENDER... */
++ if (too_large(dst->drawable.width, dst->drawable.height) ||
++ unaligned(bo, dst->drawable.bitsPerPixel))
++ return gen9_render_clear_try_blt(sna, dst, bo);
++
++ tmp.dst.pixmap = dst;
++ tmp.dst.width = dst->drawable.width;
++ tmp.dst.height = dst->drawable.height;
++ tmp.dst.format = sna_format_for_depth(dst->drawable.depth);
++ tmp.dst.bo = bo;
++ tmp.dst.x = tmp.dst.y = 0;
++
++ tmp.src.bo = sna_render_get_solid(sna, 0);
++ tmp.mask.bo = NULL;
++
++ tmp.floats_per_vertex = 2;
++ tmp.floats_per_rect = 6;
++ tmp.need_magic_ca_pass = false;
++
++ tmp.u.gen9.flags = FILL_FLAGS_NOBLEND;
++
++ kgem_set_mode(&sna->kgem, KGEM_RENDER, bo);
++ if (!kgem_check_bo(&sna->kgem, bo, NULL)) {
++ kgem_submit(&sna->kgem);
++ if (!kgem_check_bo(&sna->kgem, bo, NULL)) {
++ kgem_bo_destroy(&sna->kgem, tmp.src.bo);
++ return false;
++ }
++ _kgem_set_mode(&sna->kgem, KGEM_RENDER);
++ }
++
++ gen9_align_vertex(sna, &tmp);
++ gen9_emit_fill_state(sna, &tmp);
++
++ gen9_get_rectangles(sna, &tmp, 1, gen9_emit_fill_state);
++
++ v = (int16_t *)&sna->render.vertices[sna->render.vertex_used];
++ sna->render.vertex_used += 6;
++ assert(sna->render.vertex_used <= sna->render.vertex_size);
++
++ v[0] = dst->drawable.width;
++ v[5] = v[1] = dst->drawable.height;
++ v[8] = v[4] = 0;
++ v[9] = 0;
++
++ v[7] = v[2] = v[3] = 1;
++ v[6] = v[10] = v[11] = 0;
++
++ gen8_vertex_flush(sna);
++ kgem_bo_destroy(&sna->kgem, tmp.src.bo);
++
++ return true;
++}
++
++#if !NO_VIDEO
++static uint32_t gen9_bind_video_source(struct sna *sna,
++ struct kgem_bo *bo,
++ uint32_t delta,
++ int width,
++ int height,
++ int pitch,
++ uint32_t format)
++{
++ uint32_t *ss;
++ int offset;
++
++ offset = sna->kgem.surface -= SURFACE_DW;
++ ss = sna->kgem.batch + offset;
++ ss[0] = (SURFACE_2D << SURFACE_TYPE_SHIFT |
++ gen9_tiling_bits(bo->tiling) |
++ format << SURFACE_FORMAT_SHIFT |
++ SURFACE_VALIGN_4 | SURFACE_HALIGN_4);
++ ss[1] = 0;
++ ss[2] = ((width - 1) << SURFACE_WIDTH_SHIFT |
++ (height - 1) << SURFACE_HEIGHT_SHIFT);
++ ss[3] = (pitch - 1) << SURFACE_PITCH_SHIFT;
++ ss[4] = 0;
++ ss[5] = 0;
++ ss[6] = 0;
++ ss[7] = SURFACE_SWIZZLE(RED, GREEN, BLUE, ALPHA);
++ *(uint64_t *)(ss+8) =
++ kgem_add_reloc64(&sna->kgem, offset + 8, bo,
++ I915_GEM_DOMAIN_SAMPLER << 16,
++ delta);
++ ss[10] = 0;
++ ss[11] = 0;
++ ss[12] = 0;
++ ss[13] = 0;
++ ss[14] = 0;
++ ss[15] = 0;
++
++ DBG(("[%x] bind bo(handle=%d, addr=%d), format=%d, width=%d, height=%d, pitch=%d, tiling=%d -> sampler\n",
++ offset, bo->handle, ss[1],
++ format, width, height, bo->pitch, bo->tiling));
++
++ return offset * sizeof(uint32_t);
++}
++
++static void gen9_emit_video_state(struct sna *sna,
++ const struct sna_composite_op *op)
++{
++ struct sna_video_frame *frame = op->priv;
++ uint32_t src_surf_format;
++ uint32_t src_surf_base[6];
++ int src_width[6];
++ int src_height[6];
++ int src_pitch[6];
++ uint32_t *binding_table;
++ uint16_t offset;
++ int n_src, n;
++
++ /* XXX VeBox, bicubic */
++
++ gen9_get_batch(sna, op);
++
++ src_surf_base[0] = 0;
++ src_surf_base[1] = 0;
++ src_surf_base[2] = frame->VBufOffset;
++ src_surf_base[3] = frame->VBufOffset;
++ src_surf_base[4] = frame->UBufOffset;
++ src_surf_base[5] = frame->UBufOffset;
++
++ if (is_planar_fourcc(frame->id)) {
++ src_surf_format = SURFACEFORMAT_R8_UNORM;
++ src_width[1] = src_width[0] = frame->width;
++ src_height[1] = src_height[0] = frame->height;
++ src_pitch[1] = src_pitch[0] = frame->pitch[1];
++ src_width[4] = src_width[5] = src_width[2] = src_width[3] =
++ frame->width / 2;
++ src_height[4] = src_height[5] = src_height[2] = src_height[3] =
++ frame->height / 2;
++ src_pitch[4] = src_pitch[5] = src_pitch[2] = src_pitch[3] =
++ frame->pitch[0];
++ n_src = 6;
++ } else {
++ if (frame->id == FOURCC_RGB888)
++ src_surf_format = SURFACEFORMAT_B8G8R8X8_UNORM;
++ else if (frame->id == FOURCC_UYVY)
++ src_surf_format = SURFACEFORMAT_YCRCB_SWAPY;
++ else
++ src_surf_format = SURFACEFORMAT_YCRCB_NORMAL;
++
++ src_width[0] = frame->width;
++ src_height[0] = frame->height;
++ src_pitch[0] = frame->pitch[0];
++ n_src = 1;
++ }
++
++ binding_table = gen9_composite_get_binding_table(sna, &offset);
++
++ binding_table[0] =
++ gen9_bind_bo(sna,
++ op->dst.bo, op->dst.width, op->dst.height,
++ gen9_get_dest_format(op->dst.format),
++ true);
++ for (n = 0; n < n_src; n++) {
++ binding_table[1+n] =
++ gen9_bind_video_source(sna,
++ frame->bo,
++ src_surf_base[n],
++ src_width[n],
++ src_height[n],
++ src_pitch[n],
++ src_surf_format);
++ }
++
++ gen9_emit_state(sna, op, offset);
++}
++
++static unsigned select_video_kernel(const struct sna_video_frame *frame)
++{
++ switch (frame->id) {
++ case FOURCC_YV12:
++ case FOURCC_I420:
++ case FOURCC_XVMC:
++ return GEN9_WM_KERNEL_VIDEO_PLANAR;
++
++ case FOURCC_RGB888:
++ case FOURCC_RGB565:
++ return GEN9_WM_KERNEL_VIDEO_RGB;
++
++ default:
++ return GEN9_WM_KERNEL_VIDEO_PACKED;
++ }
++}
++
++static bool
++gen9_render_video(struct sna *sna,
++ struct sna_video *video,
++ struct sna_video_frame *frame,
++ RegionPtr dstRegion,
++ PixmapPtr pixmap)
++{
++ struct sna_composite_op tmp;
++ struct sna_pixmap *priv = sna_pixmap(pixmap);
++ int dst_width = dstRegion->extents.x2 - dstRegion->extents.x1;
++ int dst_height = dstRegion->extents.y2 - dstRegion->extents.y1;
++ int src_width = frame->src.x2 - frame->src.x1;
++ int src_height = frame->src.y2 - frame->src.y1;
++ float src_offset_x, src_offset_y;
++ float src_scale_x, src_scale_y;
++ unsigned filter;
++ const BoxRec *box;
++ int nbox;
++
++ DBG(("%s: src=(%d, %d), dst=(%d, %d), %dx[(%d, %d), (%d, %d)...]\n",
++ __FUNCTION__,
++ src_width, src_height, dst_width, dst_height,
++ region_num_rects(dstRegion),
++ REGION_EXTENTS(NULL, dstRegion)->x1,
++ REGION_EXTENTS(NULL, dstRegion)->y1,
++ REGION_EXTENTS(NULL, dstRegion)->x2,
++ REGION_EXTENTS(NULL, dstRegion)->y2));
++
++ assert(priv->gpu_bo);
++ assert(!too_large(pixmap->drawable.width, pixmap->drawable.height));
++ assert(!unaligned(priv->gpu_bo, pixmap->drawable.bitsPerPixel));
++
++ memset(&tmp, 0, sizeof(tmp));
++
++ tmp.dst.pixmap = pixmap;
++ tmp.dst.width = pixmap->drawable.width;
++ tmp.dst.height = pixmap->drawable.height;
++ tmp.dst.format = sna_render_format_for_depth(pixmap->drawable.depth);
++ tmp.dst.bo = priv->gpu_bo;
++
++ tmp.src.bo = frame->bo;
++ tmp.mask.bo = NULL;
++
++ tmp.floats_per_vertex = 3;
++ tmp.floats_per_rect = 9;
++
++ DBG(("%s: scaling?=%d, planar?=%d [%x]\n",
++ __FUNCTION__,
++ src_width != dst_width || src_height != dst_height,
++ is_planar_fourcc(frame->id), frame->id));
++
++ if (src_width == dst_width && src_height == dst_height)
++ filter = SAMPLER_FILTER_NEAREST;
++ else
++ filter = SAMPLER_FILTER_BILINEAR;
++
++ tmp.u.gen9.flags =
++ GEN9_SET_FLAGS(SAMPLER_OFFSET(filter, SAMPLER_EXTEND_PAD,
++ SAMPLER_FILTER_NEAREST, SAMPLER_EXTEND_NONE),
++ NO_BLEND,
++ select_video_kernel(frame),
++ 2);
++ tmp.priv = frame;
++
++ kgem_set_mode(&sna->kgem, KGEM_RENDER, tmp.dst.bo);
++ if (!kgem_check_bo(&sna->kgem, tmp.dst.bo, frame->bo, NULL)) {
++ kgem_submit(&sna->kgem);
++ if (!kgem_check_bo(&sna->kgem, tmp.dst.bo, frame->bo, NULL))
++ return false;
++
++ _kgem_set_mode(&sna->kgem, KGEM_RENDER);
++ }
++
++ gen9_align_vertex(sna, &tmp);
++ gen9_emit_video_state(sna, &tmp);
++
++ DBG(("%s: src=(%d, %d)x(%d, %d); frame=(%dx%d), dst=(%dx%d)\n",
++ __FUNCTION__,
++ frame->src.x1, frame->src.y1,
++ src_width, src_height,
++ dst_width, dst_height,
++ frame->width, frame->height));
++
++ src_scale_x = (float)src_width / dst_width / frame->width;
++ src_offset_x = (float)frame->src.x1 / frame->width - dstRegion->extents.x1 * src_scale_x;
++
++ src_scale_y = (float)src_height / dst_height / frame->height;
++ src_offset_y = (float)frame->src.y1 / frame->height - dstRegion->extents.y1 * src_scale_y;
++
++ DBG(("%s: scale=(%f, %f), offset=(%f, %f)\n",
++ __FUNCTION__,
++ src_scale_x, src_scale_y,
++ src_offset_x, src_offset_y));
++
++ box = region_rects(dstRegion);
++ nbox = region_num_rects(dstRegion);
++ while (nbox--) {
++ DBG(("%s: dst=(%d, %d), (%d, %d) + (%d, %d); src=(%f, %f), (%f, %f)\n",
++ __FUNCTION__,
++ box->x1, box->y1,
++ box->x2, box->y2,
++ box->x1 * src_scale_x + src_offset_x,
++ box->y1 * src_scale_y + src_offset_y,
++ box->x2 * src_scale_x + src_offset_x,
++ box->y2 * src_scale_y + src_offset_y));
++
++ gen9_get_rectangles(sna, &tmp, 1, gen9_emit_video_state);
++
++ OUT_VERTEX(box->x2, box->y2);
++ OUT_VERTEX_F(box->x2 * src_scale_x + src_offset_x);
++ OUT_VERTEX_F(box->y2 * src_scale_y + src_offset_y);
++
++ OUT_VERTEX(box->x1, box->y2);
++ OUT_VERTEX_F(box->x1 * src_scale_x + src_offset_x);
++ OUT_VERTEX_F(box->y2 * src_scale_y + src_offset_y);
++
++ OUT_VERTEX(box->x1, box->y1);
++ OUT_VERTEX_F(box->x1 * src_scale_x + src_offset_x);
++ OUT_VERTEX_F(box->y1 * src_scale_y + src_offset_y);
++
++ box++;
++ }
++ gen8_vertex_flush(sna);
++
++ if (!DAMAGE_IS_ALL(priv->gpu_damage))
++ sna_damage_add(&priv->gpu_damage, dstRegion);
++
++ return true;
++}
++#endif
++
++static void gen9_render_flush(struct sna *sna)
++{
++ gen8_vertex_close(sna);
++
++ assert(sna->render.vb_id == 0);
++ assert(sna->render.vertex_offset == 0);
++}
++
++static void gen9_render_reset(struct sna *sna)
++{
++ sna->render_state.gen9.emit_flush = false;
++ sna->render_state.gen9.needs_invariant = true;
++ sna->render_state.gen9.ve_id = 3 << 2;
++ sna->render_state.gen9.ve_dirty = false;
++ sna->render_state.gen9.last_primitive = -1;
++
++ sna->render_state.gen9.num_sf_outputs = 0;
++ sna->render_state.gen9.samplers = -1;
++ sna->render_state.gen9.blend = -1;
++ sna->render_state.gen9.kernel = -1;
++ sna->render_state.gen9.drawrect_offset = -1;
++ sna->render_state.gen9.drawrect_limit = -1;
++ sna->render_state.gen9.surface_table = 0;
++
++ if (sna->render.vbo && !kgem_bo_can_map(&sna->kgem, sna->render.vbo)) {
++ DBG(("%s: discarding unmappable vbo\n", __FUNCTION__));
++ discard_vbo(sna);
++ }
++
++ sna->render.vertex_offset = 0;
++ sna->render.nvertex_reloc = 0;
++ sna->render.vb_id = 0;
++}
++
++static void gen9_render_fini(struct sna *sna)
++{
++ kgem_bo_destroy(&sna->kgem, sna->render_state.gen9.general_bo);
++}
++
++static bool gen9_render_setup(struct sna *sna)
++{
++ struct gen9_render_state *state = &sna->render_state.gen9;
++ struct sna_static_stream general;
++ struct gen9_sampler_state *ss;
++ int i, j, k, l, m;
++ uint32_t devid;
++
++ devid = intel_get_device_id(sna->dev);
++ if (devid & 0xf)
++ state->gt = GEN9_GT_BIAS + ((devid >> 4) & 0xf) + 1;
++ DBG(("%s: gt=%d\n", __FUNCTION__, state->gt));
++
++ state->info = &min_gt_info;
++ if (is_skl(sna))
++ state->info = &skl_gt_info;
++ if (is_bxt(sna))
++ state->info = &bxt_gt_info;
++ if (is_kbl(sna))
++ state->info = &kbl_gt_info;
++ if (is_glk(sna))
++ state->info = &glk_gt_info;
++
++ sna_static_stream_init(&general);
++
++ /* Zero pad the start. If you see an offset of 0x0 in the batchbuffer
++ * dumps, you know it points to zero.
++ */
++ null_create(&general);
++
++ for (m = 0; m < ARRAY_SIZE(wm_kernels); m++) {
++ if (wm_kernels[m].size) {
++ state->wm_kernel[m][1] =
++ sna_static_stream_add(&general,
++ wm_kernels[m].data,
++ wm_kernels[m].size,
++ 64);
++ } else {
++ if (USE_8_PIXEL_DISPATCH) {
++ state->wm_kernel[m][0] =
++ sna_static_stream_compile_wm(sna, &general,
++ wm_kernels[m].data, 8);
++ }
++
++ if (USE_16_PIXEL_DISPATCH) {
++ state->wm_kernel[m][1] =
++ sna_static_stream_compile_wm(sna, &general,
++ wm_kernels[m].data, 16);
++ }
++
++ if (USE_32_PIXEL_DISPATCH) {
++ state->wm_kernel[m][2] =
++ sna_static_stream_compile_wm(sna, &general,
++ wm_kernels[m].data, 32);
++ }
++ }
++ assert(state->wm_kernel[m][0]|state->wm_kernel[m][1]|state->wm_kernel[m][2]);
++ }
++
++ COMPILE_TIME_ASSERT(SAMPLER_OFFSET(FILTER_COUNT, EXTEND_COUNT, FILTER_COUNT, EXTEND_COUNT) <= 0x7ff);
++ ss = sna_static_stream_map(&general,
++ 2 * sizeof(*ss) *
++ (2 +
++ FILTER_COUNT * EXTEND_COUNT *
++ FILTER_COUNT * EXTEND_COUNT),
++ 32);
++ state->wm_state = sna_static_stream_offsetof(&general, ss);
++ sampler_copy_init(ss); ss += 2;
++ sampler_fill_init(ss); ss += 2;
++ for (i = 0; i < FILTER_COUNT; i++) {
++ for (j = 0; j < EXTEND_COUNT; j++) {
++ for (k = 0; k < FILTER_COUNT; k++) {
++ for (l = 0; l < EXTEND_COUNT; l++) {
++ sampler_state_init(ss++, i, j);
++ sampler_state_init(ss++, k, l);
++ }
++ }
++ }
++ }
++
++ state->cc_blend = gen9_create_blend_state(&general);
++
++ state->general_bo = sna_static_stream_fini(sna, &general);
++ return state->general_bo != NULL;
++}
++
++const char *gen9_render_init(struct sna *sna, const char *backend)
++{
++ if (!gen9_render_setup(sna))
++ return backend;
++
++ sna->kgem.context_switch = gen6_render_context_switch;
++ sna->kgem.retire = gen6_render_retire;
++ sna->kgem.expire = gen4_render_expire;
++
++#if !NO_COMPOSITE
++ sna->render.composite = gen9_render_composite;
++ sna->render.prefer_gpu |= PREFER_GPU_RENDER;
++#endif
++#if !NO_COMPOSITE_SPANS
++ sna->render.check_composite_spans = gen9_check_composite_spans;
++ sna->render.composite_spans = gen9_render_composite_spans;
++ sna->render.prefer_gpu |= PREFER_GPU_SPANS;
++#endif
++#if !NO_VIDEO
++ sna->render.video = gen9_render_video;
++#endif
++
++#if !NO_COPY_BOXES
++ sna->render.copy_boxes = gen9_render_copy_boxes;
++#endif
++#if !NO_COPY
++ sna->render.copy = gen9_render_copy;
++#endif
++
++#if !NO_FILL_BOXES
++ sna->render.fill_boxes = gen9_render_fill_boxes;
++#endif
++#if !NO_FILL
++ sna->render.fill = gen9_render_fill;
++#endif
++#if !NO_FILL_ONE
++ sna->render.fill_one = gen9_render_fill_one;
++#endif
++#if !NO_FILL_CLEAR
++ sna->render.clear = gen9_render_clear;
++#endif
++
++ sna->render.flush = gen9_render_flush;
++ sna->render.reset = gen9_render_reset;
++ sna->render.fini = gen9_render_fini;
++
++ sna->render.max_3d_size = GEN9_MAX_SIZE;
++ sna->render.max_3d_pitch = 1 << 18;
++ return sna->render_state.gen9.info->name;
++}
+diff --git a/src/sna/gen9_render.h b/src/sna/gen9_render.h
+new file mode 100644
+index 00000000..e3cb3f93
+--- /dev/null
++++ b/src/sna/gen9_render.h
+@@ -0,0 +1,1130 @@
++#ifndef GEN9_RENDER_H
++#define GEN9_RENDER_H
++
++#define INTEL_MASK(high, low) (((1 << ((high) - (low) + 1)) - 1) << (low))
++
++#define GEN9_3D(pipeline,op,sub) \
++ ((3 << 29) | ((pipeline) << 27) | ((op) << 24) | ((sub) << 16))
++
++#define GEN9_STATE_BASE_ADDRESS GEN9_3D(0, 1, 1)
++# define BASE_ADDRESS_MODIFY (1 << 0)
++
++#define GEN9_STATE_SIP GEN9_3D(0, 1, 2)
++
++#define GEN9_3DSTATE_VF_STATISTICS GEN9_3D(1, 0, 0xb)
++#define GEN9_PIPELINE_SELECT GEN9_3D(1, 1, 4)
++# define PIPELINE_SELECT_3D 0
++# define PIPELINE_SELECT_MEDIA 1
++#define PIPELINE_SELECTION_MASK (3 << 8)
++
++#define GEN9_MEDIA_STATE_POINTERS GEN9_3D(2, 0, 0)
++#define GEN9_MEDIA_OBJECT GEN9_3D(2, 1, 0)
++
++#define GEN9_3DSTATE_CLEAR_PARAMS GEN9_3D(3, 0, 0x04)
++#define GEN9_3DSTATE_DEPTH_BUFFER GEN9_3D(3, 0, 0x05)
++# define DEPTH_BUFFER_TYPE_SHIFT 29
++# define DEPTH_BUFFER_FORMAT_SHIFT 18
++
++#define GEN9_3DSTATE_STENCIL_BUFFER GEN9_3D(3, 0, 0x06)
++#define GEN9_3DSTATE_HIER_DEPTH_BUFFER GEN9_3D(3, 0, 0x07)
++#define GEN9_3DSTATE_VERTEX_BUFFERS GEN9_3D(3, 0, 0x08)
++# define VB_INDEX_SHIFT 26
++# define VB_MODIFY_ENABLE (1 << 14)
++#define GEN9_3DSTATE_VERTEX_ELEMENTS GEN9_3D(3, 0, 0x09)
++# define VE_INDEX_SHIFT 26
++# define VE_VALID (1 << 25)
++# define VE_FORMAT_SHIFT 16
++# define VE_OFFSET_SHIFT 0
++# define VE_COMPONENT_0_SHIFT 28
++# define VE_COMPONENT_1_SHIFT 24
++# define VE_COMPONENT_2_SHIFT 20
++# define VE_COMPONENT_3_SHIFT 16
++#define GEN9_3DSTATE_INDEX_BUFFER GEN9_3D(3, 0, 0x0a)
++#define GEN9_3DSTATE_VF GEN9_3D(3, 0, 0x0c)
++
++#define GEN9_3DSTATE_MULTISAMPLE GEN9_3D(3, 0, 0x0d)
++/* DW1 */
++# define MULTISAMPLE_PIXEL_LOCATION_CENTER (0 << 4)
++# define MULTISAMPLE_PIXEL_LOCATION_UPPER_LEFT (1 << 4)
++# define MULTISAMPLE_NUMSAMPLES_1 (0 << 1)
++# define MULTISAMPLE_NUMSAMPLES_4 (2 << 1)
++# define MULTISAMPLE_NUMSAMPLES_8 (3 << 1)
++
++#define GEN9_3DSTATE_CC_STATE_POINTERS GEN9_3D(3, 0, 0x0e)
++#define GEN9_3DSTATE_SCISSOR_STATE_POINTERS GEN9_3D(3, 0, 0x0f)
++
++#define GEN9_3DSTATE_VS GEN9_3D(3, 0, 0x10)
++#define GEN9_3DSTATE_GS GEN9_3D(3, 0, 0x11)
++#define GEN9_3DSTATE_CLIP GEN9_3D(3, 0, 0x12)
++#define GEN9_3DSTATE_SF GEN9_3D(3, 0, 0x13)
++# define SF_TRI_PROVOKE_SHIFT 29
++# define SF_LINE_PROVOKE_SHIFT 27
++# define SF_FAN_PROVOKE_SHIFT 25
++
++#define GEN9_3DSTATE_WM GEN9_3D(3, 0, 0x14)
++/* DW1 */
++# define WM_STATISTICS_ENABLE (1 << 31)
++# define WM_DEPTH_CLEAR (1 << 30)
++# define WM_DEPTH_RESOLVE (1 << 28)
++# define WM_HIERARCHICAL_DEPTH_RESOLVE (1 << 27)
++# define WM_KILL_ENABLE (1 << 25)
++# define WM_POSITION_ZW_PIXEL (0 << 17)
++# define WM_POSITION_ZW_CENTROID (2 << 17)
++# define WM_POSITION_ZW_SAMPLE (3 << 17)
++# define WM_NONPERSPECTIVE_SAMPLE_BARYCENTRIC (1 << 16)
++# define WM_NONPERSPECTIVE_CENTROID_BARYCENTRIC (1 << 15)
++# define WM_NONPERSPECTIVE_PIXEL_BARYCENTRIC (1 << 14)
++# define WM_PERSPECTIVE_SAMPLE_BARYCENTRIC (1 << 13)
++# define WM_PERSPECTIVE_CENTROID_BARYCENTRIC (1 << 12)
++# define WM_PERSPECTIVE_PIXEL_BARYCENTRIC (1 << 11)
++# define WM_LINE_END_CAP_AA_WIDTH_0_5 (0 << 8)
++# define WM_LINE_END_CAP_AA_WIDTH_1_0 (1 << 8)
++# define WM_LINE_END_CAP_AA_WIDTH_2_0 (2 << 8)
++# define WM_LINE_END_CAP_AA_WIDTH_4_0 (3 << 8)
++# define WM_LINE_AA_WIDTH_0_5 (0 << 6)
++# define WM_LINE_AA_WIDTH_1_0 (1 << 6)
++# define WM_LINE_AA_WIDTH_2_0 (2 << 6)
++# define WM_LINE_AA_WIDTH_4_0 (3 << 6)
++# define WM_POLYGON_STIPPLE_ENABLE (1 << 4)
++# define WM_LINE_STIPPLE_ENABLE (1 << 3)
++# define WM_POINT_RASTRULE_UPPER_RIGHT (1 << 2)
++# define WM_MSRAST_OFF_PIXEL (0 << 0)
++# define WM_MSRAST_OFF_PATTERN (1 << 0)
++# define WM_MSRAST_ON_PIXEL (2 << 0)
++# define WM_MSRAST_ON_PATTERN (3 << 0)
++
++#define GEN9_3DSTATE_CONSTANT_VS GEN9_3D(3, 0, 0x15)
++#define GEN9_3DSTATE_CONSTANT_GS GEN9_3D(3, 0, 0x16)
++#define GEN9_3DSTATE_CONSTANT_PS GEN9_3D(3, 0, 0x17)
++
++#define GEN9_3DSTATE_SAMPLE_MASK GEN9_3D(3, 0, 0x18)
++
++#define GEN9_3DSTATE_CONSTANT_HS GEN9_3D(3, 0, 0x19)
++#define GEN9_3DSTATE_CONSTANT_DS GEN9_3D(3, 0, 0x1a)
++
++#define GEN9_3DSTATE_HS GEN9_3D(3, 0, 0x1b)
++#define GEN9_3DSTATE_TE GEN9_3D(3, 0, 0x1c)
++#define GEN9_3DSTATE_DS GEN9_3D(3, 0, 0x1d)
++#define GEN9_3DSTATE_STREAMOUT GEN9_3D(3, 0, 0x1e)
++
++#define GEN9_3DSTATE_SBE GEN9_3D(3, 0, 0x1f)
++/* DW1 */
++# define SBE_FORCE_VERTEX_URB_READ_LENGTH (1<<29)
++# define SBE_FORCE_VERTEX_URB_READ_OFFSET (1<<28)
++# define SBE_NUM_OUTPUTS_SHIFT 22
++# define SBE_SWIZZLE_ENABLE (1 << 21)
++# define SBE_POINT_SPRITE_LOWERLEFT (1 << 20)
++# define SBE_URB_ENTRY_READ_LENGTH_SHIFT 11
++# define SBE_URB_ENTRY_READ_OFFSET_SHIFT 5
++#define SBE_ACTIVE_COMPONENT_NONE 0
++#define SBE_ACTIVE_COMPONENT_XY 1
++#define SBE_ACTIVE_COMPONENT_XYZ 2
++#define SBE_ACTIVE_COMPONENT_XYZW 3
++
++
++#define GEN9_3DSTATE_PS GEN9_3D(3, 0, 0x20)
++/* DW1:DW2 kernel pointer */
++/* DW3 */
++# define PS_SPF_MODE (1 << 31)
++# define PS_VECTOR_MASK_ENABLE (1 << 30)
++# define PS_SAMPLER_COUNT_SHIFT 27
++# define PS_BINDING_TABLE_ENTRY_COUNT_SHIFT 18
++# define PS_FLOATING_POINT_MODE_IEEE_754 (0 << 16)
++# define PS_FLOATING_POINT_MODE_ALT (1 << 16)
++/* DW4:DW5: scratch space */
++/* DW6 */
++# define PS_MAX_THREADS_SHIFT 23
++# define PS_MAX_THREADS (63 << PS_MAX_THREADS_SHIFT)
++# define PS_PUSH_CONSTANT_ENABLE (1 << 11)
++# define PS_RENDER_TARGET_CLEAR (1 << 8)
++# define PS_RENDER_TARGET_RESOLVE (1 << 6)
++# define PS_POSOFFSET_NONE (0 << 3)
++# define PS_POSOFFSET_CENTROID (2 << 3)
++# define PS_POSOFFSET_SAMPLE (3 << 3)
++# define PS_32_DISPATCH_ENABLE (1 << 2)
++# define PS_16_DISPATCH_ENABLE (1 << 1)
++# define PS_8_DISPATCH_ENABLE (1 << 0)
++/* DW7 */
++# define PS_DISPATCH_START_GRF_SHIFT_0 16
++# define PS_DISPATCH_START_GRF_SHIFT_1 8
++# define PS_DISPATCH_START_GRF_SHIFT_2 0
++/* DW8:D9: kernel 1 pointer */
++/* DW10:D11: kernel 2 pointer */
++
++#define GEN9_3DSTATE_VIEWPORT_STATE_POINTERS_SF_CLIP GEN9_3D(3, 0, 0x21)
++#define GEN9_3DSTATE_VIEWPORT_STATE_POINTERS_CC GEN9_3D(3, 0, 0x23)
++
++#define GEN9_3DSTATE_BLEND_STATE_POINTERS GEN9_3D(3, 0, 0x24)
++
++#define GEN9_3DSTATE_BINDING_TABLE_POINTERS_VS GEN9_3D(3, 0, 0x26)
++#define GEN9_3DSTATE_BINDING_TABLE_POINTERS_HS GEN9_3D(3, 0, 0x27)
++#define GEN9_3DSTATE_BINDING_TABLE_POINTERS_DS GEN9_3D(3, 0, 0x28)
++#define GEN9_3DSTATE_BINDING_TABLE_POINTERS_GS GEN9_3D(3, 0, 0x29)
++#define GEN9_3DSTATE_BINDING_TABLE_POINTERS_PS GEN9_3D(3, 0, 0x2a)
++
++#define GEN9_3DSTATE_SAMPLER_STATE_POINTERS_VS GEN9_3D(3, 0, 0x2b)
++#define GEN9_3DSTATE_SAMPLER_STATE_POINTERS_HS GEN9_3D(3, 0, 0x2c)
++#define GEN9_3DSTATE_SAMPLER_STATE_POINTERS_DS GEN9_3D(3, 0, 0x2d)
++#define GEN9_3DSTATE_SAMPLER_STATE_POINTERS_GS GEN9_3D(3, 0, 0x2e)
++#define GEN9_3DSTATE_SAMPLER_STATE_POINTERS_PS GEN9_3D(3, 0, 0x2f)
++
++#define GEN9_3DSTATE_URB_VS GEN9_3D(3, 0, 0x30)
++#define GEN9_3DSTATE_URB_HS GEN9_3D(3, 0, 0x31)
++#define GEN9_3DSTATE_URB_DS GEN9_3D(3, 0, 0x32)
++#define GEN9_3DSTATE_URB_GS GEN9_3D(3, 0, 0x33)
++/* DW1 */
++# define URB_ENTRY_NUMBER_SHIFT 0
++# define URB_ENTRY_SIZE_SHIFT 16
++# define URB_STARTING_ADDRESS_SHIFT 25
++
++#define GEN9_3DSTATE_GATHER_CONSTANT_VS GEN9_3D(3, 0, 0x34)
++#define GEN9_3DSTATE_GATHER_CONSTANT_GS GEN9_3D(3, 0, 0x35)
++#define GEN9_3DSTATE_GATHER_CONSTANT_HS GEN9_3D(3, 0, 0x36)
++#define GEN9_3DSTATE_GATHER_CONSTANT_DS GEN9_3D(3, 0, 0x37)
++#define GEN9_3DSTATE_GATHER_CONSTANT_PS GEN9_3D(3, 0, 0x38)
++
++#define GEN9_3DSTATE_DX9_CONSTANTF_VS GEN9_3D(3, 0, 0x39)
++#define GEN9_3DSTATE_DX9_CONSTANTF_PS GEN9_3D(3, 0, 0x3a)
++#define GEN9_3DSTATE_DX9_CONSTANTI_VS GEN9_3D(3, 0, 0x3b)
++#define GEN9_3DSTATE_DX9_CONSTANTI_PS GEN9_3D(3, 0, 0x3c)
++#define GEN9_3DSTATE_DX9_CONSTANTB_VS GEN9_3D(3, 0, 0x3d)
++#define GEN9_3DSTATE_DX9_CONSTANTB_PS GEN9_3D(3, 0, 0x3e)
++#define GEN9_3DSTATE_DX9_LOCAL_VALID_VS GEN9_3D(3, 0, 0x3f)
++#define GEN9_3DSTATE_DX9_LOCAL_VALID_PS GEN9_3D(3, 0, 0x40)
++#define GEN9_3DSTATE_DX9_GENERATE_ACTIVE_VS GEN9_3D(3, 0, 0x41)
++#define GEN9_3DSTATE_DX9_GENERATE_ACTIVE_PS GEN9_3D(3, 0, 0x42)
++
++#define GEN9_3DSTATE_BINDING_TABLE_EDIT_VS GEN9_3D(3, 0, 0x43)
++#define GEN9_3DSTATE_BINDING_TABLE_EDIT_GS GEN9_3D(3, 0, 0x44)
++#define GEN9_3DSTATE_BINDING_TABLE_EDIT_HS GEN9_3D(3, 0, 0x45)
++#define GEN9_3DSTATE_BINDING_TABLE_EDIT_DS GEN9_3D(3, 0, 0x46)
++#define GEN9_3DSTATE_BINDING_TABLE_EDIT_PS GEN9_3D(3, 0, 0x47)
++
++#define GEN9_3DSTATE_VF_INSTANCING GEN9_3D(3, 0, 0x49)
++#define GEN9_3DSTATE_VF_SGVS GEN9_3D(3, 0, 0x4a)
++# define SGVS_ENABLE_INSTANCE_ID (1 << 31)
++# define SGVS_INSTANCE_ID_COMPONENT_SHIFT 29
++# define SGVS_INSTANCE_ID_ELEMENT_OFFSET_SHIFT 16
++# define SGVS_ENABLE_VERTEX_ID (1 << 15)
++# define SGVS_VERTEX_ID_COMPONENT_SHIFT 13
++# define SGVS_VERTEX_ID_ELEMENT_OFFSET_SHIFT 0
++#define GEN9_3DSTATE_VF_TOPOLOGY GEN9_3D(3, 0, 0x4b)
++# define POINTLIST 0x01
++# define LINELIST 0x02
++# define LINESTRIP 0x03
++# define TRILIST 0x04
++# define TRISTRIP 0x05
++# define TRIFAN 0x06
++# define QUADLIST 0x07
++# define QUADSTRIP 0x08
++# define LINELIST_ADJ 0x09
++# define LINESTRIP_ADJ 0x0A
++# define TRILIST_ADJ 0x0B
++# define TRISTRIP_ADJ 0x0C
++# define TRISTRIP_REVERSE 0x0D
++# define POLYGON 0x0E
++# define RECTLIST 0x0F
++# define LINELOOP 0x10
++# define POINTLIST_BF 0x11
++# define LINESTRIP_CONT 0x12
++# define LINESTRIP_BF 0x13
++# define LINESTRIP_CONT_BF 0x14
++# define TRIFAN_NOSTIPPLE 0x15
++
++#define GEN9_3DSTATE_WM_CHROMAKEY GEN9_3D(3, 0, 0x4c)
++
++#define GEN9_3DSTATE_PS_BLEND GEN9_3D(3, 0, 0x4d)
++# define PS_BLEND_ALPHA_TO_COVERAGE_ENABLE (1 << 31)
++# define PS_BLEND_HAS_WRITEABLE_RT (1 << 30)
++# define PS_BLEND_COLOR_BLEND_ENABLE (1 << 29)
++# define PS_BLEND_SRC_ALPHA_SHIFT 24
++# define PS_BLEND_DST_ALPHA_SHIFT 19
++# define PS_BLEND_SRC_SHIFT 14
++# define PS_BLEND_DST_SHIFT 9
++# define PS_BLEND_ALPHA_TEST_ENABLE (1 << 8)
++# define PS_BLEND_INDEPENDENT_ALPHA_BLEND_ENABLE (1 << 7)
++
++#define GEN9_3DSTATE_WM_DEPTH_STENCIL GEN9_3D(3, 0, 0x4e)
++/* DW1 */
++# define WM_DS_STENCIL_TEST_MASK_MASK INTEL_MASK(31, 24)
++# define WM_DS_STENCIL_TEST_MASK_SHIFT 24
++# define WM_DS_STENCIL_WRITE_MASK_MASK INTEL_MASK(23, 16)
++# define WM_DS_STENCIL_WRITE_MASK_SHIFT 16
++# define WM_DS_BF_STENCIL_TEST_MASK_MASK INTEL_MASK(15, 8)
++# define WM_DS_BF_STENCIL_TEST_MASK_SHIFT 8
++# define WM_DS_BF_STENCIL_WRITE_MASK_MASK INTEL_MASK(7, 0)
++# define WM_DS_DEPTH_FUNC_SHIFT 5
++# define WM_DS_DOUBLE_SIDED_STENCIL_ENABLE (1 << 4)
++# define WM_DS_STENCIL_TEST_ENABLE (1 << 3)
++# define WM_DS_STENCIL_BUFFER_WRITE_ENABLE (1 << 2)
++# define WM_DS_DEPTH_TEST_ENABLE (1 << 1)
++# define WM_DS_DEPTH_BUFFER_WRITE_ENABLE (1 << 0)
++/* DW2 */
++# define WM_DS_STENCIL_TEST_MASK_MASK INTEL_MASK(31, 24)
++# define WM_DS_STENCIL_TEST_MASK_SHIFT 24
++# define WM_DS_STENCIL_WRITE_MASK_MASK INTEL_MASK(23, 16)
++# define WM_DS_STENCIL_WRITE_MASK_SHIFT 16
++# define WM_DS_BF_STENCIL_TEST_MASK_MASK INTEL_MASK(15, 8)
++# define WM_DS_BF_STENCIL_TEST_MASK_SHIFT 8
++# define WM_DS_BF_STENCIL_WRITE_MASK_MASK INTEL_MASK(7, 0)
++# define WM_DS_BF_STENCIL_WRITE_MASK_SHIFT 0
++
++#define GEN9_3DSTATE_PS_EXTRA GEN9_3D(3, 0, 0x4f)
++# define PSX_PIXEL_SHADER_VALID (1 << 31)
++# define PSX_PIXEL_SHADER_NO_RT_WRITE (1 << 30)
++# define PSX_OMASK_TO_RENDER_TARGET (1 << 29)
++# define PSX_KILL_ENABLE (1 << 28)
++# define PSX_PSCDEPTH_OFF (0 << 26)
++# define PSX_PSCDEPTH_ON (1 << 26)
++# define PSX_PSCDEPTH_ON_GE (2 << 26)
++# define PSX_PSCDEPTH_ON_LE (3 << 26)
++# define PSX_FORCE_COMPUTED_DEPTH (1 << 25)
++# define PSX_USES_SOURCE_DEPTH (1 << 24)
++# define PSX_USES_SOURCE_W (1 << 23)
++# define PSX_ATTRIBUTE_ENABLE (1 << 8)
++# define PSX_SHADER_DISABLES_ALPHA_TO_COVERAGE (1 << 7)
++# define PSX_SHADER_IS_PER_SAMPLE (1 << 6)
++# define PSX_SHADER_HAS_UAV (1 << 2)
++# define PSX_SHADER_USES_INPUT_COVERAGE_MASK (1 << 1)
++
++#define GEN9_3DSTATE_RASTER GEN9_3D(3, 0, 0x50)
++/* DW1 */
++# define RASTER_FRONT_WINDING_CCW (1 << 21)
++# define RASTER_CULL_BOTH (0 << 16)
++# define RASTER_CULL_NONE (1 << 16)
++# define RASTER_CULL_FRONT (2 << 16)
++# define RASTER_CULL_BACK (3 << 16)
++# define RASTER_SMOOTH_POINT_ENABLE (1 << 13)
++# define RASTER_LINE_AA_ENABLE (1 << 2)
++# define RASTER_VIEWPORT_Z_CLIP_TEST_ENABLE (1 << 0)
++
++#define GEN9_3DSTATE_SBE_SWIZ GEN9_3D(3, 0, 0x51)
++#define GEN9_3DSTATE_WM_HZ_OP GEN9_3D(3, 0, 0x52)
++
++#define GEN9_3DSTATE_COMPONENT_PACKING GEN6_3D(3, 0, 0x55)
++
++
++
++#define GEN9_3DSTATE_DRAWING_RECTANGLE GEN9_3D(3, 1, 0x00)
++#define GEN9_3DSTATE_SAMPLER_PALETTE_LOAD GEN9_3D(3, 1, 0x02)
++#define GEN9_3DSTATE_CHROMA_KEY GEN9_3D(3, 1, 0x04)
++
++#define GEN9_3DSTATE_POLY_STIPPLE_OFFSET GEN9_3D(3, 1, 0x06)
++#define GEN9_3DSTATE_POLY_STIPPLE_PATTERN GEN9_3D(3, 1, 0x07)
++#define GEN9_3DSTATE_LINE_STIPPLE GEN9_3D(3, 1, 0x08)
++#define GEN9_3DSTATE_AA_LINE_PARAMS GEN9_3D(3, 1, 0x0a)
++#define GEN9_3DSTATE_SAMPLER_PALETTE_LOAD1 GEN9_3D(3, 1, 0x0c)
++#define GEN9_3DSTATE_MONOFILTER_SIZE GEN9_3D(3, 1, 0x11)
++#define GEN9_3DSTATE_PUSH_CONSTANT_ALLOC_VS GEN9_3D(3, 1, 0x12)
++#define GEN9_3DSTATE_PUSH_CONSTANT_ALLOC_HS GEN9_3D(3, 1, 0x13)
++#define GEN9_3DSTATE_PUSH_CONSTANT_ALLOC_DS GEN9_3D(3, 1, 0x14)
++#define GEN9_3DSTATE_PUSH_CONSTANT_ALLOC_GS GEN9_3D(3, 1, 0x15)
++#define GEN9_3DSTATE_PUSH_CONSTANT_ALLOC_PS GEN9_3D(3, 1, 0x16)
++/* DW1 */
++# define PUSH_CONSTANT_BUFFER_OFFSET_SHIFT 16
++# define PUSH_CONSTANT_BUFFER_SIZE_SHIFT 0
++
++#define GEN9_3DSTATE_SO_DECL_LIST GEN9_3D(3, 1, 0x17)
++#define GEN9_3DSTATE_SO_BUFFER GEN9_3D(3, 1, 0x18)
++#define GEN9_3DSTATE_BINDING_TABLE_POOL_ALLOC GEN9_3D(3, 1, 0x19)
++#define GEN9_3DSTATE_GATHER_BUFFER_POOL_ALLOC GEN9_3D(3, 1, 0x1a)
++#define GEN9_3DSTATE_DX9_CONSTANT_BUFFER_POOL_ALLOC GEN9_3D(3, 1, 0x1b)
++#define GEN9_3DSTATE_SAMPLE_PATTERN GEN9_3D(3, 1, 0x1c)
++
++
++/* for GEN9_PIPE_CONTROL */
++#define GEN9_PIPE_CONTROL GEN9_3D(3, 2, 0)
++#define PIPE_CONTROL_CS_STALL (1 << 20)
++#define PIPE_CONTROL_NOWRITE (0 << 14)
++#define PIPE_CONTROL_WRITE_QWORD (1 << 14)
++#define PIPE_CONTROL_WRITE_DEPTH (2 << 14)
++#define PIPE_CONTROL_WRITE_TIME (3 << 14)
++#define PIPE_CONTROL_DEPTH_STALL (1 << 13)
++#define PIPE_CONTROL_WC_FLUSH (1 << 12)
++#define PIPE_CONTROL_IS_FLUSH (1 << 11)
++#define PIPE_CONTROL_TC_FLUSH (1 << 10)
++#define PIPE_CONTROL_NOTIFY_ENABLE (1 << 8)
++#define PIPE_CONTROL_FLUSH (1 << 7)
++#define PIPE_CONTROL_GLOBAL_GTT (1 << 2)
++#define PIPE_CONTROL_LOCAL_PGTT (0 << 2)
++#define PIPE_CONTROL_STALL_AT_SCOREBOARD (1 << 1)
++#define PIPE_CONTROL_DEPTH_CACHE_FLUSH (1 << 0)
++
++
++#define GEN9_3DPRIMITIVE GEN9_3D(3, 3, 0)
++
++/* 3DPRIMITIVE bits */
++#define VERTEX_SEQUENTIAL (0 << 15)
++#define VERTEX_RANDOM (1 << 15)
++
++#define ANISORATIO_2 0
++#define ANISORATIO_4 1
++#define ANISORATIO_6 2
++#define ANISORATIO_8 3
++#define ANISORATIO_10 4
++#define ANISORATIO_12 5
++#define ANISORATIO_14 6
++#define ANISORATIO_16 7
++
++#define BLENDFACTOR_ONE 0x1
++#define BLENDFACTOR_SRC_COLOR 0x2
++#define BLENDFACTOR_SRC_ALPHA 0x3
++#define BLENDFACTOR_DST_ALPHA 0x4
++#define BLENDFACTOR_DST_COLOR 0x5
++#define BLENDFACTOR_SRC_ALPHA_SATURATE 0x6
++#define BLENDFACTOR_CONST_COLOR 0x7
++#define BLENDFACTOR_CONST_ALPHA 0x8
++#define BLENDFACTOR_SRC1_COLOR 0x9
++#define BLENDFACTOR_SRC1_ALPHA 0x0A
++#define BLENDFACTOR_ZERO 0x11
++#define BLENDFACTOR_INV_SRC_COLOR 0x12
++#define BLENDFACTOR_INV_SRC_ALPHA 0x13
++#define BLENDFACTOR_INV_DST_ALPHA 0x14
++#define BLENDFACTOR_INV_DST_COLOR 0x15
++#define BLENDFACTOR_INV_CONST_COLOR 0x17
++#define BLENDFACTOR_INV_CONST_ALPHA 0x18
++#define BLENDFACTOR_INV_SRC1_COLOR 0x19
++#define BLENDFACTOR_INV_SRC1_ALPHA 0x1A
++
++#define BLENDFUNCTION_ADD 0
++#define BLENDFUNCTION_SUBTRACT 1
++#define BLENDFUNCTION_REVERSE_SUBTRACT 2
++#define GEN9_BLENDFUNCTION_MIN 3
++#define BLENDFUNCTION_MAX 4
++
++#define ALPHATEST_FORMAT_UNORM8 0
++#define ALPHATEST_FORMAT_FLOAT32 1
++
++#define CHROMAKEY_KILL_ON_ANY_MATCH 0
++#define CHROMAKEY_REPLACE_BLACK 1
++
++#define CLIP_API_OGL 0
++#define CLIP_API_DX 1
++
++#define CLIPMODE_NORMAL 0
++#define CLIPMODE_CLIP_ALL 1
++#define CLIPMODE_CLIP_NON_REJECTED 2
++#define CLIPMODE_REJECT_ALL 3
++#define CLIPMODE_ACCEPT_ALL 4
++
++#define CLIP_NDCSPACE 0
++#define CLIP_SCREENSPACE 1
++
++#define COMPAREFUNCTION_ALWAYS 0
++#define COMPAREFUNCTION_NEVER 1
++#define COMPAREFUNCTION_LESS 2
++#define COMPAREFUNCTION_EQUAL 3
++#define COMPAREFUNCTION_LEQUAL 4
++#define COMPAREFUNCTION_GREATER 5
++#define COMPAREFUNCTION_NOTEQUAL 6
++#define COMPAREFUNCTION_GEQUAL 7
++
++#define COVERAGE_PIXELS_HALF 0
++#define COVERAGE_PIXELS_1 1
++#define COVERAGE_PIXELS_2 2
++#define COVERAGE_PIXELS_4 3
++
++#define DEPTHFORMAT_D32_FLOAT_S8X24_UINT 0
++#define DEPTHFORMAT_D32_FLOAT 1
++#define DEPTHFORMAT_D24_UNORM_S8_UINT 2
++#define DEPTHFORMAT_D16_UNORM 5
++
++#define FLOATING_POINT_IEEE_754 0
++#define FLOATING_POINT_NON_IEEE_754 1
++
++#define INDEX_BYTE 0
++#define INDEX_WORD 1
++#define INDEX_DWORD 2
++
++#define LOGICOPFUNCTION_CLEAR 0
++#define LOGICOPFUNCTION_NOR 1
++#define LOGICOPFUNCTION_AND_INVERTED 2
++#define LOGICOPFUNCTION_COPY_INVERTED 3
++#define LOGICOPFUNCTION_AND_REVERSE 4
++#define LOGICOPFUNCTION_INVERT 5
++#define LOGICOPFUNCTION_XOR 6
++#define LOGICOPFUNCTION_NAND 7
++#define LOGICOPFUNCTION_AND 8
++#define LOGICOPFUNCTION_EQUIV 9
++#define LOGICOPFUNCTION_NOOP 10
++#define LOGICOPFUNCTION_OR_INVERTED 11
++#define LOGICOPFUNCTION_COPY 12
++#define LOGICOPFUNCTION_OR_REVERSE 13
++#define LOGICOPFUNCTION_OR 14
++#define LOGICOPFUNCTION_SET 15
++
++#define MAPFILTER_NEAREST 0x0
++#define MAPFILTER_LINEAR 0x1
++#define MAPFILTER_ANISOTROPIC 0x2
++#define MAPFILTER_FLEXIBLE 0x3
++#define MAPFILTER_MONO 0x6
++
++#define MIPFILTER_NONE 0
++#define MIPFILTER_NEAREST 1
++#define MIPFILTER_LINEAR 3
++
++#define POLYGON_FRONT_FACING 0
++#define POLYGON_BACK_FACING 1
++
++#define PREFILTER_ALWAYS 0x0
++#define PREFILTER_NEVER 0x1
++#define PREFILTER_LESS 0x2
++#define PREFILTER_EQUAL 0x3
++#define PREFILTER_LEQUAL 0x4
++#define PREFILTER_GREATER 0x5
++#define PREFILTER_NOTEQUAL 0x6
++#define PREFILTER_GEQUAL 0x7
++
++#define RASTRULE_UPPER_LEFT 0
++#define RASTRULE_UPPER_RIGHT 1
++
++#define STENCILOP_KEEP 0
++#define STENCILOP_ZERO 1
++#define STENCILOP_REPLACE 2
++#define STENCILOP_INCRSAT 3
++#define STENCILOP_DECRSAT 4
++#define STENCILOP_INCR 5
++#define STENCILOP_DECR 6
++#define STENCILOP_INVERT 7
++
++#define SURFACE_MIPMAPLAYOUT_BELOW 0
++#define SURFACE_MIPMAPLAYOUT_RIGHT 1
++
++#define SURFACEFORMAT_R32G32B32A32_FLOAT 0x000
++#define SURFACEFORMAT_R32G32B32A32_SINT 0x001
++#define SURFACEFORMAT_R32G32B32A32_UINT 0x002
++#define SURFACEFORMAT_R32G32B32A32_UNORM 0x003
++#define SURFACEFORMAT_R32G32B32A32_SNORM 0x004
++#define SURFACEFORMAT_R64G64_FLOAT 0x005
++#define SURFACEFORMAT_R32G32B32X32_FLOAT 0x006
++#define SURFACEFORMAT_R32G32B32A32_SSCALED 0x007
++#define SURFACEFORMAT_R32G32B32A32_USCALED 0x008
++#define SURFACEFORMAT_R32G32B32_FLOAT 0x040
++#define SURFACEFORMAT_R32G32B32_SINT 0x041
++#define SURFACEFORMAT_R32G32B32_UINT 0x042
++#define SURFACEFORMAT_R32G32B32_UNORM 0x043
++#define SURFACEFORMAT_R32G32B32_SNORM 0x044
++#define SURFACEFORMAT_R32G32B32_SSCALED 0x045
++#define SURFACEFORMAT_R32G32B32_USCALED 0x046
++#define SURFACEFORMAT_R16G16B16A16_UNORM 0x080
++#define SURFACEFORMAT_R16G16B16A16_SNORM 0x081
++#define SURFACEFORMAT_R16G16B16A16_SINT 0x082
++#define SURFACEFORMAT_R16G16B16A16_UINT 0x083
++#define SURFACEFORMAT_R16G16B16A16_FLOAT 0x084
++#define SURFACEFORMAT_R32G32_FLOAT 0x085
++#define SURFACEFORMAT_R32G32_SINT 0x086
++#define SURFACEFORMAT_R32G32_UINT 0x087
++#define SURFACEFORMAT_R32_FLOAT_X8X24_TYPELESS 0x088
++#define SURFACEFORMAT_X32_TYPELESS_G8X24_UINT 0x089
++#define SURFACEFORMAT_L32A32_FLOAT 0x08A
++#define SURFACEFORMAT_R32G32_UNORM 0x08B
++#define SURFACEFORMAT_R32G32_SNORM 0x08C
++#define SURFACEFORMAT_R64_FLOAT 0x08D
++#define SURFACEFORMAT_R16G16B16X16_UNORM 0x08E
++#define SURFACEFORMAT_R16G16B16X16_FLOAT 0x08F
++#define SURFACEFORMAT_A32X32_FLOAT 0x090
++#define SURFACEFORMAT_L32X32_FLOAT 0x091
++#define SURFACEFORMAT_I32X32_FLOAT 0x092
++#define SURFACEFORMAT_R16G16B16A16_SSCALED 0x093
++#define SURFACEFORMAT_R16G16B16A16_USCALED 0x094
++#define SURFACEFORMAT_R32G32_SSCALED 0x095
++#define SURFACEFORMAT_R32G32_USCALED 0x096
++#define SURFACEFORMAT_B8G8R8A8_UNORM 0x0C0
++#define SURFACEFORMAT_B8G8R8A8_UNORM_SRGB 0x0C1
++#define SURFACEFORMAT_R10G10B10A2_UNORM 0x0C2
++#define SURFACEFORMAT_R10G10B10A2_UNORM_SRGB 0x0C3
++#define SURFACEFORMAT_R10G10B10A2_UINT 0x0C4
++#define SURFACEFORMAT_R10G10B10_SNORM_A2_UNORM 0x0C5
++#define SURFACEFORMAT_R8G8B8A8_UNORM 0x0C7
++#define SURFACEFORMAT_R8G8B8A8_UNORM_SRGB 0x0C8
++#define SURFACEFORMAT_R8G8B8A8_SNORM 0x0C9
++#define SURFACEFORMAT_R8G8B8A8_SINT 0x0CA
++#define SURFACEFORMAT_R8G8B8A8_UINT 0x0CB
++#define SURFACEFORMAT_R16G16_UNORM 0x0CC
++#define SURFACEFORMAT_R16G16_SNORM 0x0CD
++#define SURFACEFORMAT_R16G16_SINT 0x0CE
++#define SURFACEFORMAT_R16G16_UINT 0x0CF
++#define SURFACEFORMAT_R16G16_FLOAT 0x0D0
++#define SURFACEFORMAT_B10G10R10A2_UNORM 0x0D1
++#define SURFACEFORMAT_B10G10R10A2_UNORM_SRGB 0x0D2
++#define SURFACEFORMAT_R11G11B10_FLOAT 0x0D3
++#define SURFACEFORMAT_R32_SINT 0x0D6
++#define SURFACEFORMAT_R32_UINT 0x0D7
++#define SURFACEFORMAT_R32_FLOAT 0x0D8
++#define SURFACEFORMAT_R24_UNORM_X8_TYPELESS 0x0D9
++#define SURFACEFORMAT_X24_TYPELESS_G8_UINT 0x0DA
++#define SURFACEFORMAT_L16A16_UNORM 0x0DF
++#define SURFACEFORMAT_I24X8_UNORM 0x0E0
++#define SURFACEFORMAT_L24X8_UNORM 0x0E1
++#define SURFACEFORMAT_A24X8_UNORM 0x0E2
++#define SURFACEFORMAT_I32_FLOAT 0x0E3
++#define SURFACEFORMAT_L32_FLOAT 0x0E4
++#define SURFACEFORMAT_A32_FLOAT 0x0E5
++#define SURFACEFORMAT_B8G8R8X8_UNORM 0x0E9
++#define SURFACEFORMAT_B8G8R8X8_UNORM_SRGB 0x0EA
++#define SURFACEFORMAT_R8G8B8X8_UNORM 0x0EB
++#define SURFACEFORMAT_R8G8B8X8_UNORM_SRGB 0x0EC
++#define SURFACEFORMAT_R9G9B9E5_SHAREDEXP 0x0ED
++#define SURFACEFORMAT_B10G10R10X2_UNORM 0x0EE
++#define SURFACEFORMAT_L16A16_FLOAT 0x0F0
++#define SURFACEFORMAT_R32_UNORM 0x0F1
++#define SURFACEFORMAT_R32_SNORM 0x0F2
++#define SURFACEFORMAT_R10G10B10X2_USCALED 0x0F3
++#define SURFACEFORMAT_R8G8B8A8_SSCALED 0x0F4
++#define SURFACEFORMAT_R8G8B8A8_USCALED 0x0F5
++#define SURFACEFORMAT_R16G16_SSCALED 0x0F6
++#define SURFACEFORMAT_R16G16_USCALED 0x0F7
++#define SURFACEFORMAT_R32_SSCALED 0x0F8
++#define SURFACEFORMAT_R32_USCALED 0x0F9
++#define SURFACEFORMAT_B5G6R5_UNORM 0x100
++#define SURFACEFORMAT_B5G6R5_UNORM_SRGB 0x101
++#define SURFACEFORMAT_B5G5R5A1_UNORM 0x102
++#define SURFACEFORMAT_B5G5R5A1_UNORM_SRGB 0x103
++#define SURFACEFORMAT_B4G4R4A4_UNORM 0x104
++#define SURFACEFORMAT_B4G4R4A4_UNORM_SRGB 0x105
++#define SURFACEFORMAT_R8G8_UNORM 0x106
++#define SURFACEFORMAT_R8G8_SNORM 0x107
++#define SURFACEFORMAT_R8G8_SINT 0x108
++#define SURFACEFORMAT_R8G8_UINT 0x109
++#define SURFACEFORMAT_R16_UNORM 0x10A
++#define SURFACEFORMAT_R16_SNORM 0x10B
++#define SURFACEFORMAT_R16_SINT 0x10C
++#define SURFACEFORMAT_R16_UINT 0x10D
++#define SURFACEFORMAT_R16_FLOAT 0x10E
++#define SURFACEFORMAT_I16_UNORM 0x111
++#define SURFACEFORMAT_L16_UNORM 0x112
++#define SURFACEFORMAT_A16_UNORM 0x113
++#define SURFACEFORMAT_L8A8_UNORM 0x114
++#define SURFACEFORMAT_I16_FLOAT 0x115
++#define SURFACEFORMAT_L16_FLOAT 0x116
++#define SURFACEFORMAT_A16_FLOAT 0x117
++#define SURFACEFORMAT_R5G5_SNORM_B6_UNORM 0x119
++#define SURFACEFORMAT_B5G5R5X1_UNORM 0x11A
++#define SURFACEFORMAT_B5G5R5X1_UNORM_SRGB 0x11B
++#define SURFACEFORMAT_R8G8_SSCALED 0x11C
++#define SURFACEFORMAT_R8G8_USCALED 0x11D
++#define SURFACEFORMAT_R16_SSCALED 0x11E
++#define SURFACEFORMAT_R16_USCALED 0x11F
++#define SURFACEFORMAT_R8_UNORM 0x140
++#define SURFACEFORMAT_R8_SNORM 0x141
++#define SURFACEFORMAT_R8_SINT 0x142
++#define SURFACEFORMAT_R8_UINT 0x143
++#define SURFACEFORMAT_A8_UNORM 0x144
++#define SURFACEFORMAT_I8_UNORM 0x145
++#define SURFACEFORMAT_L8_UNORM 0x146
++#define SURFACEFORMAT_P4A4_UNORM 0x147
++#define SURFACEFORMAT_A4P4_UNORM 0x148
++#define SURFACEFORMAT_R8_SSCALED 0x149
++#define SURFACEFORMAT_R8_USCALED 0x14A
++#define SURFACEFORMAT_R1_UINT 0x181
++#define SURFACEFORMAT_YCRCB_NORMAL 0x182
++#define SURFACEFORMAT_YCRCB_SWAPUVY 0x183
++#define SURFACEFORMAT_BC1_UNORM 0x186
++#define SURFACEFORMAT_BC2_UNORM 0x187
++#define SURFACEFORMAT_BC3_UNORM 0x188
++#define SURFACEFORMAT_BC4_UNORM 0x189
++#define SURFACEFORMAT_BC5_UNORM 0x18A
++#define SURFACEFORMAT_BC1_UNORM_SRGB 0x18B
++#define SURFACEFORMAT_BC2_UNORM_SRGB 0x18C
++#define SURFACEFORMAT_BC3_UNORM_SRGB 0x18D
++#define SURFACEFORMAT_MONO8 0x18E
++#define SURFACEFORMAT_YCRCB_SWAPUV 0x18F
++#define SURFACEFORMAT_YCRCB_SWAPY 0x190
++#define SURFACEFORMAT_DXT1_RGB 0x191
++#define SURFACEFORMAT_FXT1 0x192
++#define SURFACEFORMAT_R8G8B8_UNORM 0x193
++#define SURFACEFORMAT_R8G8B8_SNORM 0x194
++#define SURFACEFORMAT_R8G8B8_SSCALED 0x195
++#define SURFACEFORMAT_R8G8B8_USCALED 0x196
++#define SURFACEFORMAT_R64G64B64A64_FLOAT 0x197
++#define SURFACEFORMAT_R64G64B64_FLOAT 0x198
++#define SURFACEFORMAT_BC4_SNORM 0x199
++#define SURFACEFORMAT_BC5_SNORM 0x19A
++#define SURFACEFORMAT_R16G16B16_UNORM 0x19C
++#define SURFACEFORMAT_R16G16B16_SNORM 0x19D
++#define SURFACEFORMAT_R16G16B16_SSCALED 0x19E
++#define SURFACEFORMAT_R16G16B16_USCALED 0x19F
++
++#define SURFACE_1D 0
++#define SURFACE_2D 1
++#define SURFACE_3D 2
++#define SURFACE_CUBE 3
++#define SURFACE_BUFFER 4
++#define SURFACE_NULL 7
++
++#define TEXCOORDMODE_WRAP 0
++#define TEXCOORDMODE_MIRROR 1
++#define TEXCOORDMODE_CLAMP 2
++#define TEXCOORDMODE_CUBE 3
++#define TEXCOORDMODE_CLAMP_BORDER 4
++#define TEXCOORDMODE_MIRROR_ONCE 5
++
++#define THREAD_PRIORITY_NORMAL 0
++#define THREAD_PRIORITY_HIGH 1
++
++#define VERTEX_SUBPIXEL_PRECISION_8BITS 0
++#define VERTEX_SUBPIXEL_PRECISION_4BITS 1
++
++#define COMPONENT_NOSTORE 0
++#define COMPONENT_STORE_SRC 1
++#define COMPONENT_STORE_0 2
++#define COMPONENT_STORE_1_FLT 3
++#define COMPONENT_STORE_1_INT 4
++#define COMPONENT_STORE_VID 5
++#define COMPONENT_STORE_IID 6
++#define COMPONENT_STORE_PID 7
++
++/* Execution Unit (EU) defines
++ */
++
++#define GEN9_ALIGN_1 0
++#define GEN9_ALIGN_16 1
++
++#define GEN9_ADDRESS_DIRECT 0
++#define GEN9_ADDRESS_REGISTER_INDIRECT_REGISTER 1
++
++#define GEN9_CHANNEL_X 0
++#define GEN9_CHANNEL_Y 1
++#define GEN9_CHANNEL_Z 2
++#define GEN9_CHANNEL_W 3
++
++#define GEN9_COMPRESSION_NONE 0
++#define GEN9_COMPRESSION_2NDHALF 1
++#define GEN9_COMPRESSION_COMPRESSED 2
++
++#define GEN9_CONDITIONAL_NONE 0
++#define GEN9_CONDITIONAL_Z 1
++#define GEN9_CONDITIONAL_NZ 2
++#define GEN9_CONDITIONAL_EQ 1 /* Z */
++#define GEN9_CONDITIONAL_NEQ 2 /* NZ */
++#define GEN9_CONDITIONAL_G 3
++#define GEN9_CONDITIONAL_GE 4
++#define GEN9_CONDITIONAL_L 5
++#define GEN9_CONDITIONAL_LE 6
++#define GEN9_CONDITIONAL_C 7
++#define GEN9_CONDITIONAL_O 8
++
++#define GEN9_DEBUG_NONE 0
++#define GEN9_DEBUG_BREAKPOINT 1
++
++#define GEN9_DEPENDENCY_NORMAL 0
++#define GEN9_DEPENDENCY_NOTCLEARED 1
++#define GEN9_DEPENDENCY_NOTCHECKED 2
++#define GEN9_DEPENDENCY_DISABLE 3
++
++#define GEN9_EXECUTE_1 0
++#define GEN9_EXECUTE_2 1
++#define GEN9_EXECUTE_4 2
++#define GEN9_EXECUTE_8 3
++#define GEN9_EXECUTE_16 4
++#define GEN9_EXECUTE_32 5
++
++#define GEN9_HORIZONTAL_STRIDE_0 0
++#define GEN9_HORIZONTAL_STRIDE_1 1
++#define GEN9_HORIZONTAL_STRIDE_2 2
++#define GEN9_HORIZONTAL_STRIDE_4 3
++
++#define GEN9_INSTRUCTION_NORMAL 0
++#define GEN9_INSTRUCTION_SATURATE 1
++
++#define GEN9_OPCODE_MOV 1
++#define GEN9_OPCODE_SEL 2
++#define GEN9_OPCODE_NOT 4
++#define GEN9_OPCODE_AND 5
++#define GEN9_OPCODE_OR 6
++#define GEN9_OPCODE_XOR 7
++#define GEN9_OPCODE_SHR 8
++#define GEN9_OPCODE_SHL 9
++#define GEN9_OPCODE_RSR 10
++#define GEN9_OPCODE_RSL 11
++#define GEN9_OPCODE_ASR 12
++#define GEN9_OPCODE_CMP 16
++#define GEN9_OPCODE_JMPI 32
++#define GEN9_OPCODE_IF 34
++#define GEN9_OPCODE_IFF 35
++#define GEN9_OPCODE_ELSE 36
++#define GEN9_OPCODE_ENDIF 37
++#define GEN9_OPCODE_DO 38
++#define GEN9_OPCODE_WHILE 39
++#define GEN9_OPCODE_BREAK 40
++#define GEN9_OPCODE_CONTINUE 41
++#define GEN9_OPCODE_HALT 42
++#define GEN9_OPCODE_MSAVE 44
++#define GEN9_OPCODE_MRESTORE 45
++#define GEN9_OPCODE_PUSH 46
++#define GEN9_OPCODE_POP 47
++#define GEN9_OPCODE_WAIT 48
++#define GEN9_OPCODE_SEND 49
++#define GEN9_OPCODE_ADD 64
++#define GEN9_OPCODE_MUL 65
++#define GEN9_OPCODE_AVG 66
++#define GEN9_OPCODE_FRC 67
++#define GEN9_OPCODE_RNDU 68
++#define GEN9_OPCODE_RNDD 69
++#define GEN9_OPCODE_RNDE 70
++#define GEN9_OPCODE_RNDZ 71
++#define GEN9_OPCODE_MAC 72
++#define GEN9_OPCODE_MACH 73
++#define GEN9_OPCODE_LZD 74
++#define GEN9_OPCODE_SAD2 80
++#define GEN9_OPCODE_SADA2 81
++#define GEN9_OPCODE_DP4 84
++#define GEN9_OPCODE_DPH 85
++#define GEN9_OPCODE_DP3 86
++#define GEN9_OPCODE_DP2 87
++#define GEN9_OPCODE_DPA2 88
++#define GEN9_OPCODE_LINE 89
++#define GEN9_OPCODE_NOP 126
++
++#define GEN9_PREDICATE_NONE 0
++#define GEN9_PREDICATE_NORMAL 1
++#define GEN9_PREDICATE_ALIGN1_ANYV 2
++#define GEN9_PREDICATE_ALIGN1_ALLV 3
++#define GEN9_PREDICATE_ALIGN1_ANY2H 4
++#define GEN9_PREDICATE_ALIGN1_ALL2H 5
++#define GEN9_PREDICATE_ALIGN1_ANY4H 6
++#define GEN9_PREDICATE_ALIGN1_ALL4H 7
++#define GEN9_PREDICATE_ALIGN1_ANY8H 8
++#define GEN9_PREDICATE_ALIGN1_ALL8H 9
++#define GEN9_PREDICATE_ALIGN1_ANY16H 10
++#define GEN9_PREDICATE_ALIGN1_ALL16H 11
++#define GEN9_PREDICATE_ALIGN16_REPLICATE_X 2
++#define GEN9_PREDICATE_ALIGN16_REPLICATE_Y 3
++#define GEN9_PREDICATE_ALIGN16_REPLICATE_Z 4
++#define GEN9_PREDICATE_ALIGN16_REPLICATE_W 5
++#define GEN9_PREDICATE_ALIGN16_ANY4H 6
++#define GEN9_PREDICATE_ALIGN16_ALL4H 7
++
++#define GEN9_ARCHITECTURE_REGISTER_FILE 0
++#define GEN9_GENERAL_REGISTER_FILE 1
++#define GEN9_MESSAGE_REGISTER_FILE 2
++#define GEN9_IMMEDIATE_VALUE 3
++
++#define GEN9_REGISTER_TYPE_UD 0
++#define GEN9_REGISTER_TYPE_D 1
++#define GEN9_REGISTER_TYPE_UW 2
++#define GEN9_REGISTER_TYPE_W 3
++#define GEN9_REGISTER_TYPE_UB 4
++#define GEN9_REGISTER_TYPE_B 5
++#define GEN9_REGISTER_TYPE_VF 5 /* packed float vector, immediates only? */
++#define GEN9_REGISTER_TYPE_HF 6
++#define GEN9_REGISTER_TYPE_V 6 /* packed int vector, immediates only, uword dest only */
++#define GEN9_REGISTER_TYPE_F 7
++
++#define GEN9_ARF_NULL 0x00
++#define GEN9_ARF_ADDRESS 0x10
++#define GEN9_ARF_ACCUMULATOR 0x20
++#define GEN9_ARF_FLAG 0x30
++#define GEN9_ARF_MASK 0x40
++#define GEN9_ARF_MASK_STACK 0x50
++#define GEN9_ARF_MASK_STACK_DEPTH 0x60
++#define GEN9_ARF_STATE 0x70
++#define GEN9_ARF_CONTROL 0x80
++#define GEN9_ARF_NOTIFICATION_COUNT 0x90
++#define GEN9_ARF_IP 0xA0
++
++#define GEN9_AMASK 0
++#define GEN9_IMASK 1
++#define GEN9_LMASK 2
++#define GEN9_CMASK 3
++
++#define GEN9_THREAD_NORMAL 0
++#define GEN9_THREAD_ATOMIC 1
++#define GEN9_THREAD_SWITCH 2
++
++#define GEN9_VERTICAL_STRIDE_0 0
++#define GEN9_VERTICAL_STRIDE_1 1
++#define GEN9_VERTICAL_STRIDE_2 2
++#define GEN9_VERTICAL_STRIDE_4 3
++#define GEN9_VERTICAL_STRIDE_8 4
++#define GEN9_VERTICAL_STRIDE_16 5
++#define GEN9_VERTICAL_STRIDE_32 6
++#define GEN9_VERTICAL_STRIDE_64 7
++#define GEN9_VERTICAL_STRIDE_128 8
++#define GEN9_VERTICAL_STRIDE_256 9
++#define GEN9_VERTICAL_STRIDE_ONE_DIMENSIONAL 0xF
++
++#define GEN9_WIDTH_1 0
++#define GEN9_WIDTH_2 1
++#define GEN9_WIDTH_4 2
++#define GEN9_WIDTH_8 3
++#define GEN9_WIDTH_16 4
++
++#define GEN9_STATELESS_BUFFER_BOUNDARY_1K 0
++#define GEN9_STATELESS_BUFFER_BOUNDARY_2K 1
++#define GEN9_STATELESS_BUFFER_BOUNDARY_4K 2
++#define GEN9_STATELESS_BUFFER_BOUNDARY_8K 3
++#define GEN9_STATELESS_BUFFER_BOUNDARY_16K 4
++#define GEN9_STATELESS_BUFFER_BOUNDARY_32K 5
++#define GEN9_STATELESS_BUFFER_BOUNDARY_64K 6
++#define GEN9_STATELESS_BUFFER_BOUNDARY_128K 7
++#define GEN9_STATELESS_BUFFER_BOUNDARY_256K 8
++#define GEN9_STATELESS_BUFFER_BOUNDARY_512K 9
++#define GEN9_STATELESS_BUFFER_BOUNDARY_1M 10
++#define GEN9_STATELESS_BUFFER_BOUNDARY_2M 11
++
++#define GEN9_POLYGON_FACING_FRONT 0
++#define GEN9_POLYGON_FACING_BACK 1
++
++#define GEN9_MESSAGE_TARGET_NULL 0
++#define GEN9_MESSAGE_TARGET_MATH 1
++#define GEN9_MESSAGE_TARGET_SAMPLER 2
++#define GEN9_MESSAGE_TARGET_GATEWAY 3
++#define GEN9_MESSAGE_TARGET_DATAPORT_READ 4
++#define GEN9_MESSAGE_TARGET_DATAPORT_WRITE 5
++#define GEN9_MESSAGE_TARGET_URB 6
++#define GEN9_MESSAGE_TARGET_THREAD_SPAWNER 7
++
++#define GEN9_SAMPLER_RETURN_FORMAT_FLOAT32 0
++#define GEN9_SAMPLER_RETURN_FORMAT_UINT32 2
++#define GEN9_SAMPLER_RETURN_FORMAT_SINT32 3
++
++#define GEN9_SAMPLER_MESSAGE_SIMD8_SAMPLE 0
++#define GEN9_SAMPLER_MESSAGE_SIMD16_SAMPLE 0
++#define GEN9_SAMPLER_MESSAGE_SIMD16_SAMPLE_BIAS 0
++#define GEN9_SAMPLER_MESSAGE_SIMD8_KILLPIX 1
++#define GEN9_SAMPLER_MESSAGE_SIMD4X2_SAMPLE_LOD 1
++#define GEN9_SAMPLER_MESSAGE_SIMD16_SAMPLE_LOD 1
++#define GEN9_SAMPLER_MESSAGE_SIMD4X2_SAMPLE_GRADIENTS 2
++#define GEN9_SAMPLER_MESSAGE_SIMD8_SAMPLE_GRADIENTS 2
++#define GEN9_SAMPLER_MESSAGE_SIMD4X2_SAMPLE_COMPARE 0
++#define GEN9_SAMPLER_MESSAGE_SIMD16_SAMPLE_COMPARE 2
++#define GEN9_SAMPLER_MESSAGE_SIMD4X2_RESINFO 2
++#define GEN9_SAMPLER_MESSAGE_SIMD8_RESINFO 2
++#define GEN9_SAMPLER_MESSAGE_SIMD16_RESINFO 2
++#define GEN9_SAMPLER_MESSAGE_SIMD4X2_LD 3
++#define GEN9_SAMPLER_MESSAGE_SIMD8_LD 3
++#define GEN9_SAMPLER_MESSAGE_SIMD16_LD 3
++
++#define GEN9_DATAPORT_OWORD_BLOCK_1_OWORDLOW 0
++#define GEN9_DATAPORT_OWORD_BLOCK_1_OWORDHIGH 1
++#define GEN9_DATAPORT_OWORD_BLOCK_2_OWORDS 2
++#define GEN9_DATAPORT_OWORD_BLOCK_4_OWORDS 3
++#define GEN9_DATAPORT_OWORD_BLOCK_8_OWORDS 4
++
++#define GEN9_DATAPORT_OWORD_DUAL_BLOCK_1OWORD 0
++#define GEN9_DATAPORT_OWORD_DUAL_BLOCK_4OWORDS 2
++
++#define GEN9_DATAPORT_DWORD_SCATTERED_BLOCK_8DWORDS 2
++#define GEN9_DATAPORT_DWORD_SCATTERED_BLOCK_16DWORDS 3
++
++#define GEN9_DATAPORT_READ_MESSAGE_OWORD_BLOCK_READ 0
++#define GEN9_DATAPORT_READ_MESSAGE_OWORD_DUAL_BLOCK_READ 1
++#define GEN9_DATAPORT_READ_MESSAGE_DWORD_BLOCK_READ 2
++#define GEN9_DATAPORT_READ_MESSAGE_DWORD_SCATTERED_READ 3
++
++#define GEN9_DATAPORT_READ_TARGET_DATA_CACHE 0
++#define GEN9_DATAPORT_READ_TARGET_RENDER_CACHE 1
++#define GEN9_DATAPORT_READ_TARGET_SAMPLER_CACHE 2
++
++#define GEN9_DATAPORT_RENDER_TARGET_WRITE_SIMD16_SINGLE_SOURCE 0
++#define GEN9_DATAPORT_RENDER_TARGET_WRITE_SIMD16_SINGLE_SOURCE_REPLICATED 1
++#define GEN9_DATAPORT_RENDER_TARGET_WRITE_SIMD8_DUAL_SOURCE_SUBSPAN01 2
++#define GEN9_DATAPORT_RENDER_TARGET_WRITE_SIMD8_DUAL_SOURCE_SUBSPAN23 3
++#define GEN9_DATAPORT_RENDER_TARGET_WRITE_SIMD8_SINGLE_SOURCE_SUBSPAN01 4
++
++#define GEN9_DATAPORT_WRITE_MESSAGE_OWORD_BLOCK_WRITE 0
++#define GEN9_DATAPORT_WRITE_MESSAGE_OWORD_DUAL_BLOCK_WRITE 1
++#define GEN9_DATAPORT_WRITE_MESSAGE_DWORD_BLOCK_WRITE 2
++#define GEN9_DATAPORT_WRITE_MESSAGE_DWORD_SCATTERED_WRITE 3
++#define GEN9_DATAPORT_WRITE_MESSAGE_RENDER_TARGET_WRITE 4
++#define GEN9_DATAPORT_WRITE_MESSAGE_STREAMED_VERTEX_BUFFER_WRITE 5
++#define GEN9_DATAPORT_WRITE_MESSAGE_FLUSH_RENDER_CACHE 7
++
++#define GEN9_MATH_FUNCTION_INV 1
++#define GEN9_MATH_FUNCTION_LOG 2
++#define GEN9_MATH_FUNCTION_EXP 3
++#define GEN9_MATH_FUNCTION_SQRT 4
++#define GEN9_MATH_FUNCTION_RSQ 5
++#define GEN9_MATH_FUNCTION_SIN 6 /* was 7 */
++#define GEN9_MATH_FUNCTION_COS 7 /* was 8 */
++#define GEN9_MATH_FUNCTION_SINCOS 8 /* was 6 */
++#define GEN9_MATH_FUNCTION_TAN 9
++#define GEN9_MATH_FUNCTION_POW 10
++#define GEN9_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER 11
++#define GEN9_MATH_FUNCTION_INT_DIV_QUOTIENT 12
++#define GEN9_MATH_FUNCTION_INT_DIV_REMAINDER 13
++
++#define GEN9_MATH_INTEGER_UNSIGNED 0
++#define GEN9_MATH_INTEGER_SIGNED 1
++
++#define GEN9_MATH_PRECISION_FULL 0
++#define GEN9_MATH_PRECISION_PARTIAL 1
++
++#define GEN9_MATH_SATURATE_NONE 0
++#define GEN9_MATH_SATURATE_SATURATE 1
++
++#define GEN9_MATH_DATA_VECTOR 0
++#define GEN9_MATH_DATA_SCALAR 1
++
++#define GEN9_URB_OPCODE_WRITE 0
++
++#define GEN9_URB_SWIZZLE_NONE 0
++#define GEN9_URB_SWIZZLE_INTERLEAVE 1
++#define GEN9_URB_SWIZZLE_TRANSPOSE 2
++
++#define GEN9_SCRATCH_SPACE_SIZE_1K 0
++#define GEN9_SCRATCH_SPACE_SIZE_2K 1
++#define GEN9_SCRATCH_SPACE_SIZE_4K 2
++#define GEN9_SCRATCH_SPACE_SIZE_8K 3
++#define GEN9_SCRATCH_SPACE_SIZE_16K 4
++#define GEN9_SCRATCH_SPACE_SIZE_32K 5
++#define GEN9_SCRATCH_SPACE_SIZE_64K 6
++#define GEN9_SCRATCH_SPACE_SIZE_128K 7
++#define GEN9_SCRATCH_SPACE_SIZE_256K 8
++#define GEN9_SCRATCH_SPACE_SIZE_512K 9
++#define GEN9_SCRATCH_SPACE_SIZE_1M 10
++#define GEN9_SCRATCH_SPACE_SIZE_2M 11
++
++struct gen9_blend_state {
++ struct {
++ /* 00 */ uint32_t pad:19;
++ /* 19 */ uint32_t y_dither_offset:2;
++ /* 21 */ uint32_t x_dither_offset:2;
++ /* 23 */ uint32_t color_dither_enable:1;
++ /* 24 */ uint32_t alpha_test_function:3;
++ /* 27 */ uint32_t alpha_test:1;
++ /* 28 */ uint32_t alpha_to_coverage_dither:1;
++ /* 29 */ uint32_t alpha_to_one:1;
++ /* 30 */ uint32_t ia_blend:1;
++ /* 31 */ uint32_t alpha_to_coverage:1;
++ } common;
++
++ struct {
++ /* 00 */ uint32_t write_disable_blue:1;
++ /* 01 */ uint32_t write_disable_green:1;
++ /* 02 */ uint32_t write_disable_red:1;
++ /* 03 */ uint32_t write_disable_alpha:1;
++ /* 04 */ uint32_t pad0:1;
++ /* 05 */ uint32_t alpha_blend_function:3;
++ /* 08 */ uint32_t dest_alpha_blend_factor:5;
++ /* 13 */ uint32_t source_alpha_blend_factor:5;
++ /* 18 */ uint32_t color_blend_function:3;
++ /* 21 */ uint32_t dest_blend_factor:5;
++ /* 26 */ uint32_t source_blend_factor:5;
++ /* 31 */ uint32_t color_blend:1;
++ /* 32 */ uint32_t post_blend_clamp:1;
++ /* 33 */ uint32_t pre_blend_clamp:1;
++ /* 34 */ uint32_t color_clamp_range:2;
++ /* 36 */ uint32_t pre_blend_source_only_clamp:1;
++ /* 37 */ uint32_t pad1:22;
++ /* 59 */ uint32_t logic_op_function:4;
++ /* 63 */ uint32_t logic_op:1;
++ } rt;
++};
++
++struct gen9_color_calc_state {
++ struct {
++ /* 00 */ uint32_t alpha_test_format:1;
++ /* 01 */ uint32_t pad0:14;
++ /* 15 */ uint32_t round_disable:1;
++ /* 16 */ uint32_t bf_stencil_ref:8;
++ /* 24 */ uint32_t stencil_ref:8;
++ } cc0;
++
++ union {
++ float alpha_ref_f;
++ struct {
++ uint32_t ui:8;
++ uint32_t pad0:24;
++ } alpha_ref_fi;
++ } cc1;
++
++ float constant_r;
++ float constant_g;
++ float constant_b;
++ float constant_a;
++};
++
++struct gen9_sampler_state {
++ struct {
++ /* 00 */ unsigned int aniso_algorithm:1;
++ /* 01 */ unsigned int lod_bias:13;
++ /* 14 */ unsigned int min_filter:3;
++ /* 17 */ unsigned int mag_filter:3;
++ /* 20 */ unsigned int mip_filter:2;
++ /* 22 */ unsigned int base_level:5;
++ /* 27 */ unsigned int lod_preclamp:2;
++ /* 29 */ unsigned int default_color_mode:1;
++ /* 30 */ unsigned int flexible_filter_clamp:1;
++ /* 31 */ unsigned int disable:1;
++ } ss0;
++
++ struct {
++ /* 00 */ unsigned int cube_control_mode:1;
++ /* 01 */ unsigned int shadow_function:3;
++ /* 04 */ unsigned int chroma_key_mode:1;
++ /* 05 */ unsigned int chroma_key_index:2;
++ /* 07 */ unsigned int chroma_key_enable:1;
++ /* 08 */ unsigned int max_lod:12;
++ /* 20 */ unsigned int min_lod:12;
++ } ss1;
++
++ struct {
++ unsigned int pad:6;
++ unsigned int default_color_pointer:26;
++ } ss2;
++
++ struct {
++ /* 00 */ unsigned int r_wrap_mode:3;
++ /* 03 */ unsigned int t_wrap_mode:3;
++ /* 06 */ unsigned int s_wrap_mode:3;
++ /* 09 */ unsigned int pad:1;
++ /* 10 */ unsigned int non_normalized_coord:1;
++ /* 11 */ unsigned int trilinear_quality:2;
++ /* 13 */ unsigned int address_round:6;
++ /* 19 */ unsigned int max_aniso:3;
++ /* 22 */ unsigned int pad0:2;
++ /* 24 */ unsigned int non_separable_filter:8;
++ } ss3;
++};
++
++/* Surface state DW0 */
++#define SURFACE_RC_READ_WRITE (1 << 8)
++#define SURFACE_TILED (1 << 13)
++#define SURFACE_TILED_Y (1 << 12)
++#define SURFACE_FORMAT_SHIFT 18
++#define SURFACE_VALIGN_1 (0 << 16) /* reserved! */
++#define SURFACE_VALIGN_4 (1 << 16)
++#define SURFACE_VALIGN_8 (2 << 16)
++#define SURFACE_VALIGN_16 (3 << 16)
++#define SURFACE_HALIGN_1 (0 << 14) /* reserved! */
++#define SURFACE_HALIGN_4 (1 << 14)
++#define SURFACE_HALIGN_8 (2 << 14)
++#define SURFACE_HALIGN_16 (3 << 14)
++#define SURFACE_TYPE_SHIFT 29
++
++/* Surface state DW2 */
++#define SURFACE_HEIGHT_SHIFT 16
++#define SURFACE_WIDTH_SHIFT 0
++
++/* Surface state DW3 */
++#define SURFACE_DEPTH_SHIFT 21
++#define SURFACE_PITCH_SHIFT 0
++
++#define SWIZZLE_ZERO 0
++#define SWIZZLE_ONE 1
++#define SWIZZLE_RED 4
++#define SWIZZLE_GREEN 5
++#define SWIZZLE_BLUE 6
++#define SWIZZLE_ALPHA 7
++#define __SURFACE_SWIZZLE(r,g,b,a) \
++ ((a) << 16 | (b) << 19 | (g) << 22 | (r) << 25)
++#define SURFACE_SWIZZLE(r,g,b,a) \
++ __SURFACE_SWIZZLE(SWIZZLE_##r, SWIZZLE_##g, SWIZZLE_##b, SWIZZLE_##a)
++
++typedef enum {
++ SAMPLER_FILTER_NEAREST = 0,
++ SAMPLER_FILTER_BILINEAR,
++ FILTER_COUNT
++} sampler_filter_t;
++
++typedef enum {
++ SAMPLER_EXTEND_NONE = 0,
++ SAMPLER_EXTEND_REPEAT,
++ SAMPLER_EXTEND_PAD,
++ SAMPLER_EXTEND_REFLECT,
++ EXTEND_COUNT
++} sampler_extend_t;
++
++#endif
+diff --git a/src/sna/kgem.c b/src/sna/kgem.c
+index 78ed5407..f0d171ac 100644
+--- a/src/sna/kgem.c
++++ b/src/sna/kgem.c
+@@ -84,6 +84,10 @@ search_snoop_cache(struct kgem *kgem, unsigned int num_pages, unsigned flags);
+ #define DBG_NO_HANDLE_LUT 0
+ #define DBG_NO_WT 0
+ #define DBG_NO_WC_MMAP 0
++#define DBG_NO_BLT_Y 0
++#define DBG_NO_SCANOUT_Y 0
++#define DBG_NO_DIRTYFB 0
++#define DBG_NO_DETILING 0
+ #define DBG_DUMP 0
+ #define DBG_NO_MALLOC_CACHE 0
+
+@@ -96,11 +100,6 @@ search_snoop_cache(struct kgem *kgem, unsigned int num_pages, unsigned flags);
+ #define SHOW_BATCH_BEFORE 0
+ #define SHOW_BATCH_AFTER 0
+
+-#if !USE_WC_MMAP
+-#undef DBG_NO_WC_MMAP
+-#define DBG_NO_WC_MMAP 1
+-#endif
+-
+ #if 0
+ #define ASSERT_IDLE(kgem__, handle__) assert(!__kgem_busy(kgem__, handle__))
+ #define ASSERT_MAYBE_IDLE(kgem__, handle__, expect__) assert(!(expect__) || !__kgem_busy(kgem__, handle__))
+@@ -187,6 +186,15 @@ struct local_i915_gem_caching {
+ #define LOCAL_IOCTL_I915_GEM_SET_CACHING DRM_IOW(DRM_COMMAND_BASE + LOCAL_I915_GEM_SET_CACHING, struct local_i915_gem_caching)
+ #define LOCAL_IOCTL_I915_GEM_GET_CACHING DRM_IOW(DRM_COMMAND_BASE + LOCAL_I915_GEM_GET_CACHING, struct local_i915_gem_caching)
+
++struct local_i915_gem_mmap {
++ uint32_t handle;
++ uint32_t pad;
++ uint64_t offset;
++ uint64_t size;
++ uint64_t addr_ptr;
++};
++#define LOCAL_IOCTL_I915_GEM_MMAP DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_GEM_MMAP, struct local_i915_gem_mmap)
++
+ struct local_i915_gem_mmap2 {
+ uint32_t handle;
+ uint32_t pad;
+@@ -216,6 +224,12 @@ static struct kgem_bo *__kgem_freed_bo;
+ static struct kgem_request *__kgem_freed_request;
+ static struct drm_i915_gem_exec_object2 _kgem_dummy_exec;
+
++static inline struct sna *__to_sna(struct kgem *kgem)
++{
++ /* minor layering violations */
++ return container_of(kgem, struct sna, kgem);
++}
++
+ static inline int bytes(struct kgem_bo *bo)
+ {
+ return __kgem_bo_size(bo);
+@@ -224,25 +238,31 @@ static inline int bytes(struct kgem_bo *bo)
+ #define bucket(B) (B)->size.pages.bucket
+ #define num_pages(B) (B)->size.pages.count
+
+-static int do_ioctl(int fd, unsigned long req, void *arg)
++static int __do_ioctl(int fd, unsigned long req, void *arg)
+ {
+- int err;
+-
+-restart:
+- if (ioctl(fd, req, arg) == 0)
+- return 0;
++ do {
++ int err;
+
+- err = errno;
++ switch ((err = errno)) {
++ case EAGAIN:
++ sched_yield();
++ case EINTR:
++ break;
++ default:
++ return -err;
++ }
+
+- if (err == EINTR)
+- goto restart;
++ if (likely(ioctl(fd, req, arg) == 0))
++ return 0;
++ } while (1);
++}
+
+- if (err == EAGAIN) {
+- sched_yield();
+- goto restart;
+- }
++inline static int do_ioctl(int fd, unsigned long req, void *arg)
++{
++ if (likely(ioctl(fd, req, arg) == 0))
++ return 0;
+
+- return -err;
++ return __do_ioctl(fd, req, arg);
+ }
+
+ #ifdef DEBUG_MEMORY
+@@ -266,6 +286,9 @@ static void assert_tiling(struct kgem *kgem, struct kgem_bo *bo)
+
+ assert(bo);
+
++ if (!kgem->can_fence && kgem->gen >= 040 && bo->tiling)
++ return; /* lies */
++
+ VG_CLEAR(tiling);
+ tiling.handle = bo->handle;
+ tiling.tiling_mode = bo->tiling;
+@@ -273,7 +296,7 @@ static void assert_tiling(struct kgem *kgem, struct kgem_bo *bo)
+ assert(tiling.tiling_mode == bo->tiling);
+ }
+
+-static void assert_cacheing(struct kgem *kgem, struct kgem_bo *bo)
++static void assert_caching(struct kgem *kgem, struct kgem_bo *bo)
+ {
+ struct local_i915_gem_caching arg;
+ int expect = kgem->has_llc ? SNOOPED : UNCACHED;
+@@ -294,24 +317,117 @@ static void assert_bo_retired(struct kgem_bo *bo)
+ assert(bo->refcnt);
+ assert(bo->rq == NULL);
+ assert(bo->exec == NULL);
++ assert(!bo->needs_flush);
+ assert(list_is_empty(&bo->request));
+ }
+ #else
+ #define assert_tiling(kgem, bo)
+-#define assert_cacheing(kgem, bo)
++#define assert_caching(kgem, bo)
+ #define assert_bo_retired(bo)
+ #endif
+
++static int __find_debugfs(struct kgem *kgem)
++{
++ int i;
++
++ for (i = 0; i < DRM_MAX_MINOR; i++) {
++ char path[80];
++
++ sprintf(path, "/sys/kernel/debug/dri/%d/i915_wedged", i);
++ if (access(path, R_OK) == 0)
++ return i;
++
++ sprintf(path, "/debug/dri/%d/i915_wedged", i);
++ if (access(path, R_OK) == 0)
++ return i;
++ }
++
++ return -1;
++}
++
++static int kgem_get_minor(struct kgem *kgem)
++{
++ struct stat st;
++
++ if (fstat(kgem->fd, &st))
++ return __find_debugfs(kgem);
++
++ if (!S_ISCHR(st.st_mode))
++ return __find_debugfs(kgem);
++
++ return st.st_rdev & 0x63;
++}
++
++static bool find_hang_state(struct kgem *kgem, char *path, int maxlen)
++{
++ int minor = kgem_get_minor(kgem);
++
++ /* Search for our hang state in a few canonical locations.
++ * In the unlikely event of having multiple devices, we
++ * will need to check which minor actually corresponds to ours.
++ */
++
++ snprintf(path, maxlen, "/sys/class/drm/card%d/error", minor);
++ if (access(path, R_OK) == 0)
++ return true;
++
++ snprintf(path, maxlen, "/sys/kernel/debug/dri/%d/i915_error_state", minor);
++ if (access(path, R_OK) == 0)
++ return true;
++
++ snprintf(path, maxlen, "/debug/dri/%d/i915_error_state", minor);
++ if (access(path, R_OK) == 0)
++ return true;
++
++ path[0] = '\0';
++ return false;
++}
++
++static bool has_error_state(struct kgem *kgem, char *path)
++{
++ bool ret = false;
++ char no;
++ int fd;
++
++ fd = open(path, O_RDONLY);
++ if (fd >= 0) {
++ ret = read(fd, &no, 1) == 1 && no != 'N';
++ close(fd);
++ }
++
++ return ret;
++}
++
++static int kgem_get_screen_index(struct kgem *kgem)
++{
++ return __to_sna(kgem)->scrn->scrnIndex;
++}
++
+ static void
+ __kgem_set_wedged(struct kgem *kgem)
+ {
++ static int once;
++ char path[256];
++
++ if (kgem->wedged)
++ return;
++
++ if (!once &&
++ find_hang_state(kgem, path, sizeof(path)) &&
++ has_error_state(kgem, path)) {
++ xf86DrvMsg(kgem_get_screen_index(kgem), X_ERROR,
++ "When reporting this, please include %s and the full dmesg.\n",
++ path);
++ once = 1;
++ }
++
+ kgem->wedged = true;
+- sna_render_mark_wedged(container_of(kgem, struct sna, kgem));
++ sna_render_mark_wedged(__to_sna(kgem));
+ }
+
+ static void kgem_sna_reset(struct kgem *kgem)
+ {
+- struct sna *sna = container_of(kgem, struct sna, kgem);
++ struct sna *sna = __to_sna(kgem);
+
+ sna->render.reset(sna);
+ sna->blt_state.fill_bo = 0;
+@@ -319,7 +435,7 @@ static void kgem_sna_reset(struct kgem *kgem)
+
+ static void kgem_sna_flush(struct kgem *kgem)
+ {
+- struct sna *sna = container_of(kgem, struct sna, kgem);
++ struct sna *sna = __to_sna(kgem);
+
+ sna->render.flush(sna);
+
+@@ -327,22 +443,53 @@ static void kgem_sna_flush(struct kgem *kgem)
+ sna_render_flush_solid(sna);
+ }
+
+-static bool gem_set_tiling(int fd, uint32_t handle, int tiling, int stride)
++static bool kgem_bo_rmfb(struct kgem *kgem, struct kgem_bo *bo)
++{
++ if (bo->scanout && bo->delta) {
++ DBG(("%s: releasing fb=%d for handle=%d\n",
++ __FUNCTION__, bo->delta, bo->handle));
++ /* XXX will leak if we are not DRM_MASTER. *shrug* */
++ do_ioctl(kgem->fd, DRM_IOCTL_MODE_RMFB, &bo->delta);
++ bo->delta = 0;
++ return true;
++ } else
++ return false;
++}
++
++static bool kgem_set_tiling(struct kgem *kgem, struct kgem_bo *bo,
++ int tiling, int stride)
+ {
+ struct drm_i915_gem_set_tiling set_tiling;
+ int err;
+
++ if (tiling == bo->tiling) {
++ if (tiling == I915_TILING_NONE) {
++ bo->pitch = stride;
++ return true;
++ }
++ if (stride == bo->pitch)
++ return true;
++ }
++
+ if (DBG_NO_TILING)
+ return false;
+
+ VG_CLEAR(set_tiling);
+ restart:
+- set_tiling.handle = handle;
++ set_tiling.handle = bo->handle;
+ set_tiling.tiling_mode = tiling;
+- set_tiling.stride = stride;
++ set_tiling.stride = tiling ? stride : 0;
+
+- if (ioctl(fd, DRM_IOCTL_I915_GEM_SET_TILING, &set_tiling) == 0)
+- return true;
++ if (ioctl(kgem->fd, DRM_IOCTL_I915_GEM_SET_TILING, &set_tiling) == 0) {
++ bo->tiling = set_tiling.tiling_mode;
++ bo->pitch = set_tiling.tiling_mode ? set_tiling.stride : stride;
++ DBG(("%s: handle=%d, tiling=%d [%d], pitch=%d [%d]: %d\n",
++ __FUNCTION__, bo->handle,
++ bo->tiling, tiling,
++ bo->pitch, stride,
++ set_tiling.tiling_mode == tiling));
++ return set_tiling.tiling_mode == tiling;
++ }
+
+ err = errno;
+ if (err == EINTR)
+@@ -353,6 +500,11 @@ restart:
+ goto restart;
+ }
+
++ if (err == EBUSY && kgem_bo_rmfb(kgem, bo))
++ goto restart;
++
++ ERR(("%s: failed to set-tiling(tiling=%d, pitch=%d) for handle=%d: %d\n",
++ __FUNCTION__, tiling, stride, bo->handle, err));
+ return false;
+ }
+
+@@ -437,10 +589,15 @@ static void *__kgem_bo_map__gtt(struct kgem *kgem, struct kgem_bo *bo)
+ DBG(("%s(handle=%d, size=%d)\n", __FUNCTION__,
+ bo->handle, bytes(bo)));
+
++ if (bo->tiling && !kgem->can_fence)
++ return NULL;
++
+ VG_CLEAR(gtt);
+ retry_gtt:
+ gtt.handle = bo->handle;
+ if ((err = do_ioctl(kgem->fd, DRM_IOCTL_I915_GEM_MMAP_GTT, &gtt))) {
++ DBG(("%s: failed %d, throttling/cleaning caches\n",
++ __FUNCTION__, err));
+ assert(err != EINVAL);
+
+ (void)__kgem_throttle_retire(kgem, 0);
+@@ -460,6 +617,8 @@ retry_mmap:
+ kgem->fd, gtt.offset);
+ if (ptr == MAP_FAILED) {
+ err = errno;
++ DBG(("%s: failed %d, throttling/cleaning caches\n",
++ __FUNCTION__, err));
+ assert(err != EINVAL);
+
+ if (__kgem_throttle_retire(kgem, 0))
+@@ -498,6 +657,8 @@ retry_wc:
+ wc.size = bytes(bo);
+ wc.flags = I915_MMAP_WC;
+ if ((err = do_ioctl(kgem->fd, LOCAL_IOCTL_I915_GEM_MMAP_v2, &wc))) {
++ DBG(("%s: failed %d, throttling/cleaning caches\n",
++ __FUNCTION__, err));
+ assert(err != EINVAL);
+
+ if (__kgem_throttle_retire(kgem, 0))
+@@ -519,16 +680,19 @@ retry_wc:
+
+ static void *__kgem_bo_map__cpu(struct kgem *kgem, struct kgem_bo *bo)
+ {
+- struct drm_i915_gem_mmap mmap_arg;
++ struct local_i915_gem_mmap arg;
+ int err;
+
++ VG_CLEAR(arg);
++ arg.offset = 0;
++
+ retry:
+- VG_CLEAR(mmap_arg);
+- mmap_arg.handle = bo->handle;
+- mmap_arg.offset = 0;
+- mmap_arg.size = bytes(bo);
+- if ((err = do_ioctl(kgem->fd, DRM_IOCTL_I915_GEM_MMAP, &mmap_arg))) {
+- assert(err != EINVAL);
++ arg.handle = bo->handle;
++ arg.size = bytes(bo);
++ if ((err = do_ioctl(kgem->fd, LOCAL_IOCTL_I915_GEM_MMAP, &arg))) {
++ DBG(("%s: failed %d, throttling/cleaning caches\n",
++ __FUNCTION__, err));
++ assert(err != -EINVAL || bo->prime);
+
+ if (__kgem_throttle_retire(kgem, 0))
+ goto retry;
+@@ -536,15 +700,16 @@ retry:
+ if (kgem_cleanup_cache(kgem))
+ goto retry;
+
+- ERR(("%s: failed to mmap handle=%d, %d bytes, into CPU domain: %d\n",
+- __FUNCTION__, bo->handle, bytes(bo), -err));
++ ERR(("%s: failed to mmap handle=%d (prime? %d), %d bytes, into CPU domain: %d\n",
++ __FUNCTION__, bo->handle, bo->prime, bytes(bo), -err));
++ bo->purged = 1;
+ return NULL;
+ }
+
+- VG(VALGRIND_MAKE_MEM_DEFINED(mmap_arg.addr_ptr, bytes(bo)));
++ VG(VALGRIND_MAKE_MEM_DEFINED(arg.addr_ptr, bytes(bo)));
+
+ DBG(("%s: caching CPU vma for %d\n", __FUNCTION__, bo->handle));
+- return bo->map__cpu = (void *)(uintptr_t)mmap_arg.addr_ptr;
++ return bo->map__cpu = (void *)(uintptr_t)arg.addr_ptr;
+ }
+
+ static int gem_write(int fd, uint32_t handle,
+@@ -634,16 +799,10 @@ static void kgem_bo_retire(struct kgem *kgem, struct kgem_bo *bo)
+ assert(bo->exec == NULL);
+ assert(list_is_empty(&bo->vma));
+
+- if (bo->rq) {
+- __kgem_bo_clear_busy(bo);
+- kgem_retire(kgem);
+- assert_bo_retired(bo);
+- } else {
+- assert(bo->exec == NULL);
+- assert(list_is_empty(&bo->request));
+- assert(!bo->needs_flush);
+- ASSERT_IDLE(kgem, bo->handle);
+- }
++ if (bo->rq)
++ __kgem_retire_requests_upto(kgem, bo);
++ ASSERT_IDLE(kgem, bo->handle);
++ assert_bo_retired(bo);
+ }
+
+ static void kgem_bo_maybe_retire(struct kgem *kgem, struct kgem_bo *bo)
+@@ -655,10 +814,8 @@ static void kgem_bo_maybe_retire(struct kgem *kgem, struct kgem_bo *bo)
+ assert(list_is_empty(&bo->vma));
+
+ if (bo->rq) {
+- if (!__kgem_busy(kgem, bo->handle)) {
+- __kgem_bo_clear_busy(bo);
+- kgem_retire(kgem);
+- }
++ if (!__kgem_busy(kgem, bo->handle))
++ __kgem_retire_requests_upto(kgem, bo);
+ } else {
+ assert(!bo->needs_flush);
+ ASSERT_IDLE(kgem, bo->handle);
+@@ -694,6 +851,8 @@ retry:
+ }
+
+ if ((err = gem_write(kgem->fd, bo->handle, 0, length, data))) {
++ DBG(("%s: failed %d, throttling/cleaning caches\n",
++ __FUNCTION__, err));
+ assert(err != EINVAL);
+
+ (void)__kgem_throttle_retire(kgem, 0);
+@@ -728,27 +887,21 @@ static uint32_t gem_create(int fd, int num_pages)
+ return create.handle;
+ }
+
+-static bool
++static void
+ kgem_bo_set_purgeable(struct kgem *kgem, struct kgem_bo *bo)
+ {
+-#if DBG_NO_MADV
+- return true;
+-#else
++#if !DBG_NO_MADV
+ struct drm_i915_gem_madvise madv;
+
+ assert(bo->exec == NULL);
+- assert(!bo->purged);
+
+ VG_CLEAR(madv);
+ madv.handle = bo->handle;
+ madv.madv = I915_MADV_DONTNEED;
+ if (do_ioctl(kgem->fd, DRM_IOCTL_I915_GEM_MADVISE, &madv) == 0) {
+- bo->purged = 1;
+- kgem->need_purge |= !madv.retained && bo->domain == DOMAIN_GPU;
+- return madv.retained;
++ bo->purged = true;
++ kgem->need_purge |= !madv.retained && bo->domain != DOMAIN_CPU;
+ }
+-
+- return true;
+ #endif
+ }
+
+@@ -788,7 +941,7 @@ kgem_bo_clear_purgeable(struct kgem *kgem, struct kgem_bo *bo)
+ madv.madv = I915_MADV_WILLNEED;
+ if (do_ioctl(kgem->fd, DRM_IOCTL_I915_GEM_MADVISE, &madv) == 0) {
+ bo->purged = !madv.retained;
+- kgem->need_purge |= !madv.retained && bo->domain == DOMAIN_GPU;
++ kgem->need_purge |= !madv.retained && bo->domain != DOMAIN_CPU;
+ return madv.retained;
+ }
+
+@@ -869,13 +1022,17 @@ static struct kgem_request *__kgem_request_alloc(struct kgem *kgem)
+ {
+ struct kgem_request *rq;
+
+- rq = __kgem_freed_request;
+- if (rq) {
+- __kgem_freed_request = *(struct kgem_request **)rq;
++ if (unlikely(kgem->wedged)) {
++ rq = &kgem->static_request;
+ } else {
+- rq = malloc(sizeof(*rq));
+- if (rq == NULL)
+- rq = &kgem->static_request;
++ rq = __kgem_freed_request;
++ if (rq) {
++ __kgem_freed_request = *(struct kgem_request **)rq;
++ } else {
++ rq = malloc(sizeof(*rq));
++ if (rq == NULL)
++ rq = &kgem->static_request;
++ }
+ }
+
+ list_init(&rq->buffers);
+@@ -925,11 +1082,11 @@ total_ram_size(void)
+ #ifdef HAVE_STRUCT_SYSINFO_TOTALRAM
+ struct sysinfo info;
+ if (sysinfo(&info) == 0)
+- return info.totalram * info.mem_unit;
++ return (size_t)info.totalram * info.mem_unit;
+ #endif
+
+ #ifdef _SC_PHYS_PAGES
+- return sysconf(_SC_PHYS_PAGES) * sysconf(_SC_PAGE_SIZE);
++ return (size_t)sysconf(_SC_PHYS_PAGES) * sysconf(_SC_PAGE_SIZE);
+ #endif
+
+ return 0;
+@@ -1150,6 +1307,10 @@ static bool test_has_wc_mmap(struct kgem *kgem)
+ if (DBG_NO_WC_MMAP)
+ return false;
+
++ /* XXX See https://bugs.freedesktop.org/show_bug.cgi?id=90841 */
++ if (kgem->gen < 033)
++ return false;
++
+ if (gem_param(kgem, LOCAL_I915_PARAM_MMAP_VERSION) < 1)
+ return false;
+
+@@ -1187,7 +1348,7 @@ static bool test_has_caching(struct kgem *kgem)
+
+ static bool test_has_userptr(struct kgem *kgem)
+ {
+- uint32_t handle;
++ struct local_i915_gem_userptr arg;
+ void *ptr;
+
+ if (DBG_NO_USERPTR)
+@@ -1200,11 +1361,23 @@ static bool test_has_userptr(struct kgem *kgem)
+ if (posix_memalign(&ptr, PAGE_SIZE, PAGE_SIZE))
+ return false;
+
+- handle = gem_userptr(kgem->fd, ptr, PAGE_SIZE, false);
+- gem_close(kgem->fd, handle);
+- free(ptr);
++ VG_CLEAR(arg);
++ arg.user_ptr = (uintptr_t)ptr;
++ arg.user_size = PAGE_SIZE;
++ arg.flags = I915_USERPTR_UNSYNCHRONIZED;
+
+- return handle != 0;
++ if (DBG_NO_UNSYNCHRONIZED_USERPTR ||
++ do_ioctl(kgem->fd, LOCAL_IOCTL_I915_GEM_USERPTR, &arg)) {
++ arg.flags &= ~I915_USERPTR_UNSYNCHRONIZED;
++ if (do_ioctl(kgem->fd, LOCAL_IOCTL_I915_GEM_USERPTR, &arg))
++ arg.handle = 0;
++ /* Leak the userptr bo to keep the mmu_notifier alive */
++ } else {
++ gem_close(kgem->fd, arg.handle);
++ free(ptr);
++ }
++
++ return arg.handle != 0;
+ }
+
+ static bool test_has_create2(struct kgem *kgem)
+@@ -1227,67 +1400,187 @@ static bool test_has_create2(struct kgem *kgem)
+ #endif
+ }
+
+-static bool test_has_secure_batches(struct kgem *kgem)
++static bool test_can_blt_y(struct kgem *kgem)
+ {
+- if (DBG_NO_SECURE_BATCHES)
++ struct drm_i915_gem_exec_object2 object;
++ uint32_t batch[] = {
++#define MI_LOAD_REGISTER_IMM (0x22<<23 | (3-2))
++#define BCS_SWCTRL 0x22200
++#define BCS_SRC_Y (1 << 0)
++#define BCS_DST_Y (1 << 1)
++ MI_LOAD_REGISTER_IMM,
++ BCS_SWCTRL,
++ (BCS_SRC_Y | BCS_DST_Y) << 16 | (BCS_SRC_Y | BCS_DST_Y),
++
++ MI_LOAD_REGISTER_IMM,
++ BCS_SWCTRL,
++ (BCS_SRC_Y | BCS_DST_Y) << 16,
++
++ MI_BATCH_BUFFER_END,
++ 0,
++ };
++ int ret;
++
++ if (DBG_NO_BLT_Y)
+ return false;
+
+- return gem_param(kgem, LOCAL_I915_PARAM_HAS_SECURE_BATCHES) > 0;
++ if (kgem->gen < 060)
++ return false;
++
++ memset(&object, 0, sizeof(object));
++ object.handle = gem_create(kgem->fd, 1);
++
++ ret = gem_write(kgem->fd, object.handle, 0, sizeof(batch), batch);
++ if (ret == 0) {
++ struct drm_i915_gem_execbuffer2 execbuf;
++
++ memset(&execbuf, 0, sizeof(execbuf));
++ execbuf.buffers_ptr = (uintptr_t)&object;
++ execbuf.buffer_count = 1;
++ execbuf.flags = KGEM_BLT;
++
++ ret = do_ioctl(kgem->fd,
++ DRM_IOCTL_I915_GEM_EXECBUFFER2,
++ &execbuf);
++ }
++ gem_close(kgem->fd, object.handle);
++
++ return ret == 0;
+ }
+
+-static bool test_has_pinned_batches(struct kgem *kgem)
++static bool gem_set_tiling(int fd, uint32_t handle, int tiling, int stride)
+ {
+- if (DBG_NO_PINNED_BATCHES)
++ struct drm_i915_gem_set_tiling set_tiling;
++
++ if (DBG_NO_TILING)
+ return false;
+
+- return gem_param(kgem, LOCAL_I915_PARAM_HAS_PINNED_BATCHES) > 0;
++ VG_CLEAR(set_tiling);
++ set_tiling.handle = handle;
++ set_tiling.tiling_mode = tiling;
++ set_tiling.stride = stride;
++
++ if (ioctl(fd, DRM_IOCTL_I915_GEM_SET_TILING, &set_tiling) == 0)
++ return set_tiling.tiling_mode == tiling;
++
++ return false;
+ }
+
+-static int kgem_get_screen_index(struct kgem *kgem)
++static bool test_can_scanout_y(struct kgem *kgem)
+ {
+- struct sna *sna = container_of(kgem, struct sna, kgem);
+- return sna->scrn->scrnIndex;
++ struct drm_mode_fb_cmd arg;
++ bool ret = false;
++
++ if (DBG_NO_SCANOUT_Y)
++ return false;
++
++ VG_CLEAR(arg);
++ arg.width = 32;
++ arg.height = 32;
++ arg.pitch = 4*32;
++ arg.bpp = 32;
++ arg.depth = 24;
++ arg.handle = gem_create(kgem->fd, 1);
++
++ if (gem_set_tiling(kgem->fd, arg.handle, I915_TILING_Y, arg.pitch))
++ ret = do_ioctl(kgem->fd, DRM_IOCTL_MODE_ADDFB, &arg) == 0;
++ if (!ret) {
++ struct local_mode_fb_cmd2 {
++ uint32_t fb_id;
++ uint32_t width, height;
++ uint32_t pixel_format;
++ uint32_t flags;
++
++ uint32_t handles[4];
++ uint32_t pitches[4];
++ uint32_t offsets[4];
++ uint64_t modifiers[4];
++ } f;
++#define LOCAL_IOCTL_MODE_ADDFB2 DRM_IOWR(0xb8, struct local_mode_fb_cmd2)
++ memset(&f, 0, sizeof(f));
++ f.width = arg.width;
++ f.height = arg.height;
++ f.handles[0] = arg.handle;
++ f.pitches[0] = arg.pitch;
++ f.modifiers[0] = (uint64_t)1 << 56 | 2; /* MOD_Y_TILED */
++ f.pixel_format = 'X' | 'R' << 8 | '2' << 16 | '4' << 24; /* XRGB8888 */
++ f.flags = 1 << 1; /* + modifier */
++ if (drmIoctl(kgem->fd, LOCAL_IOCTL_MODE_ADDFB2, &f) == 0) {
++ ret = true;
++ arg.fb_id = f.fb_id;
++ }
++ }
++ do_ioctl(kgem->fd, DRM_IOCTL_MODE_RMFB, &arg.fb_id);
++ gem_close(kgem->fd, arg.handle);
++
++ return ret;
+ }
+
+-static int __find_debugfs(struct kgem *kgem)
++static bool test_has_dirtyfb(struct kgem *kgem)
+ {
+- int i;
++ struct drm_mode_fb_cmd create;
++ bool ret = false;
+
+- for (i = 0; i < DRM_MAX_MINOR; i++) {
+- char path[80];
++ if (DBG_NO_DIRTYFB)
++ return false;
+
+- sprintf(path, "/sys/kernel/debug/dri/%d/i915_wedged", i);
+- if (access(path, R_OK) == 0)
+- return i;
++ VG_CLEAR(create);
++ create.width = 32;
++ create.height = 32;
++ create.pitch = 4*32;
++ create.bpp = 32;
++ create.depth = 32;
++ create.handle = gem_create(kgem->fd, 1);
++ if (create.handle == 0)
++ return false;
+
+- sprintf(path, "/debug/dri/%d/i915_wedged", i);
+- if (access(path, R_OK) == 0)
+- return i;
++ if (drmIoctl(kgem->fd, DRM_IOCTL_MODE_ADDFB, &create) == 0) {
++ struct drm_mode_fb_dirty_cmd dirty;
++
++ memset(&dirty, 0, sizeof(dirty));
++ dirty.fb_id = create.fb_id;
++ ret = drmIoctl(kgem->fd,
++ DRM_IOCTL_MODE_DIRTYFB,
++ &dirty) == 0;
++
++ /* XXX There may be multiple levels of DIRTYFB, depending on
++ * whether the kernel thinks tracking dirty regions is
++ * beneficial vs flagging the whole fb as dirty.
++ */
++
++ drmIoctl(kgem->fd,
++ DRM_IOCTL_MODE_RMFB,
++ &create.fb_id);
+ }
++ gem_close(kgem->fd, create.handle);
+
+- return -1;
++ return ret;
+ }
+
+-static int kgem_get_minor(struct kgem *kgem)
++static bool test_has_secure_batches(struct kgem *kgem)
+ {
+- struct stat st;
++ if (DBG_NO_SECURE_BATCHES)
++ return false;
+
+- if (fstat(kgem->fd, &st))
+- return __find_debugfs(kgem);
++ return gem_param(kgem, LOCAL_I915_PARAM_HAS_SECURE_BATCHES) > 0;
++}
+
+- if (!S_ISCHR(st.st_mode))
+- return __find_debugfs(kgem);
++static bool test_has_pinned_batches(struct kgem *kgem)
++{
++ if (DBG_NO_PINNED_BATCHES)
++ return false;
+
+- return st.st_rdev & 0x63;
++ return gem_param(kgem, LOCAL_I915_PARAM_HAS_PINNED_BATCHES) > 0;
+ }
+
+ static bool kgem_init_pinned_batches(struct kgem *kgem)
+ {
+ int count[2] = { 16, 4 };
+ int size[2] = { 1, 4 };
++ int ret = 0;
+ int n, i;
+
+- if (kgem->wedged)
++ if (unlikely(kgem->wedged))
+ return true;
+
+ for (n = 0; n < ARRAY_SIZE(count); n++) {
+@@ -1311,7 +1604,8 @@ static bool kgem_init_pinned_batches(struct kgem *kgem)
+ }
+
+ pin.alignment = 0;
+- if (do_ioctl(kgem->fd, DRM_IOCTL_I915_GEM_PIN, &pin)) {
++ ret = do_ioctl(kgem->fd, DRM_IOCTL_I915_GEM_PIN, &pin);
++ if (ret) {
+ gem_close(kgem->fd, pin.handle);
+ free(bo);
+ goto err;
+@@ -1333,6 +1627,16 @@ err:
+ }
+ }
+
++ /* If we fail to pin some memory for 830gm/845g, we need to disable
++ * acceleration as otherwise the machine will eventually fail. However,
++ * the kernel started arbitrarily rejecting PIN, so hope for the best
++ * if the ioctl no longer works.
++ */
++ if (ret != -ENODEV && kgem->gen == 020)
++ return false;
++
++ kgem->has_pinned_batches = false;
++
+ /* For simplicity populate the lists with a single unpinned bo */
+ for (n = 0; n < ARRAY_SIZE(count); n++) {
+ struct kgem_bo *bo;
+@@ -1340,18 +1644,18 @@ err:
+
+ handle = gem_create(kgem->fd, size[n]);
+ if (handle == 0)
+- break;
++ return false;
+
+ bo = __kgem_bo_alloc(handle, size[n]);
+ if (bo == NULL) {
+ gem_close(kgem->fd, handle);
+- break;
++ return false;
+ }
+
+ debug_alloc__bo(kgem, bo);
+ list_add(&bo->list, &kgem->pinned_batches[n]);
+ }
+- return false;
++ return true;
+ }
+
+ static void kgem_init_swizzling(struct kgem *kgem)
+@@ -1364,7 +1668,7 @@ static void kgem_init_swizzling(struct kgem *kgem)
+ } tiling;
+ #define LOCAL_IOCTL_I915_GEM_GET_TILING DRM_IOWR (DRM_COMMAND_BASE + DRM_I915_GEM_GET_TILING, struct local_i915_gem_get_tiling_v2)
+
+- VG_CLEAR(tiling);
++ memset(&tiling, 0, sizeof(tiling));
+ tiling.handle = gem_create(kgem->fd, 1);
+ if (!tiling.handle)
+ return;
+@@ -1375,12 +1679,23 @@ static void kgem_init_swizzling(struct kgem *kgem)
+ if (do_ioctl(kgem->fd, LOCAL_IOCTL_I915_GEM_GET_TILING, &tiling))
+ goto out;
+
+- if (kgem->gen < 50 && tiling.phys_swizzle_mode != tiling.swizzle_mode)
++ DBG(("%s: swizzle_mode=%d, phys_swizzle_mode=%d\n",
++ __FUNCTION__, tiling.swizzle_mode, tiling.phys_swizzle_mode));
++
++ kgem->can_fence =
++ !DBG_NO_TILING &&
++ tiling.swizzle_mode != I915_BIT_6_SWIZZLE_UNKNOWN;
++
++ if (kgem->gen < 050 && tiling.phys_swizzle_mode != tiling.swizzle_mode)
+ goto out;
+
+- choose_memcpy_tiled_x(kgem, tiling.swizzle_mode);
++ if (!DBG_NO_DETILING)
++ choose_memcpy_tiled_x(kgem,
++ tiling.swizzle_mode,
++ __to_sna(kgem)->cpu_features);
+ out:
+ gem_close(kgem->fd, tiling.handle);
++ DBG(("%s: can fence?=%d\n", __FUNCTION__, kgem->can_fence));
+ }
+
+ static void kgem_fixup_relocs(struct kgem *kgem, struct kgem_bo *bo, int shrink)
+@@ -1399,6 +1714,7 @@ static void kgem_fixup_relocs(struct kgem *kgem, struct kgem_bo *bo, int shrink)
+ bo->handle, (long long)bo->presumed_offset));
+ for (n = 0; n < kgem->nreloc__self; n++) {
+ int i = kgem->reloc__self[n];
++ uint64_t addr;
+
+ assert(kgem->reloc[i].target_handle == ~0U);
+ kgem->reloc[i].target_handle = bo->target_handle;
+@@ -1412,13 +1728,17 @@ static void kgem_fixup_relocs(struct kgem *kgem, struct kgem_bo *bo, int shrink)
+
+ kgem->reloc[i].delta -= shrink;
+ }
+- kgem->batch[kgem->reloc[i].offset/sizeof(uint32_t)] =
+- kgem->reloc[i].delta + bo->presumed_offset;
++ addr = (int)kgem->reloc[i].delta + bo->presumed_offset;
++ kgem->batch[kgem->reloc[i].offset/sizeof(uint32_t)] = addr;
++ if (kgem->gen >= 0100)
++ kgem->batch[kgem->reloc[i].offset/sizeof(uint32_t) + 1] = addr >> 32;
+ }
+
+ if (n == 256) {
+ for (n = kgem->reloc__self[255]; n < kgem->nreloc; n++) {
+ if (kgem->reloc[n].target_handle == ~0U) {
++ uint64_t addr;
++
+ kgem->reloc[n].target_handle = bo->target_handle;
+ kgem->reloc[n].presumed_offset = bo->presumed_offset;
+
+@@ -1429,8 +1749,11 @@ static void kgem_fixup_relocs(struct kgem *kgem, struct kgem_bo *bo, int shrink)
+ kgem->reloc[n].delta - shrink));
+ kgem->reloc[n].delta -= shrink;
+ }
+- kgem->batch[kgem->reloc[n].offset/sizeof(uint32_t)] =
+- kgem->reloc[n].delta + bo->presumed_offset;
++
++ addr = (int)kgem->reloc[n].delta + bo->presumed_offset;
++ kgem->batch[kgem->reloc[n].offset/sizeof(uint32_t)] = addr;
++ if (kgem->gen >= 0100)
++ kgem->batch[kgem->reloc[n].offset/sizeof(uint32_t) + 1] = addr >> 32;
+ }
+ }
+ }
+@@ -1444,6 +1767,44 @@ static void kgem_fixup_relocs(struct kgem *kgem, struct kgem_bo *bo, int shrink)
+ }
+ }
+
++static int kgem_bo_wait(struct kgem *kgem, struct kgem_bo *bo)
++{
++ struct local_i915_gem_wait {
++ uint32_t handle;
++ uint32_t flags;
++ int64_t timeout;
++ } wait;
++#define LOCAL_I915_GEM_WAIT 0x2c
++#define LOCAL_IOCTL_I915_GEM_WAIT DRM_IOWR(DRM_COMMAND_BASE + LOCAL_I915_GEM_WAIT, struct local_i915_gem_wait)
++ int ret;
++
++ DBG(("%s: waiting for handle=%d\n", __FUNCTION__, bo->handle));
++ if (bo->rq == NULL)
++ return 0;
++
++ VG_CLEAR(wait);
++ wait.handle = bo->handle;
++ wait.flags = 0;
++ wait.timeout = -1;
++ ret = do_ioctl(kgem->fd, LOCAL_IOCTL_I915_GEM_WAIT, &wait);
++ if (ret) {
++ struct drm_i915_gem_set_domain set_domain;
++
++ VG_CLEAR(set_domain);
++ set_domain.handle = bo->handle;
++ set_domain.read_domains = I915_GEM_DOMAIN_GTT;
++ set_domain.write_domain = I915_GEM_DOMAIN_GTT;
++ ret = do_ioctl(kgem->fd,
++ DRM_IOCTL_I915_GEM_SET_DOMAIN,
++ &set_domain);
++ }
++
++ if (ret == 0)
++ __kgem_retire_requests_upto(kgem, bo);
++
++ return ret;
++}
++
+ static struct kgem_bo *kgem_new_batch(struct kgem *kgem)
+ {
+ struct kgem_bo *last;
+@@ -1464,20 +1825,41 @@ static struct kgem_bo *kgem_new_batch(struct kgem *kgem)
+ if (!kgem->has_llc)
+ flags |= CREATE_UNCACHED;
+
++restart:
+ kgem->batch_bo = kgem_create_linear(kgem,
+ sizeof(uint32_t)*kgem->batch_size,
+ flags);
+ if (kgem->batch_bo)
+ kgem->batch = kgem_bo_map__cpu(kgem, kgem->batch_bo);
+ if (kgem->batch == NULL) {
+- DBG(("%s: unable to map batch bo, mallocing(size=%d)\n",
+- __FUNCTION__,
+- sizeof(uint32_t)*kgem->batch_size));
++ int ring = kgem->ring == KGEM_BLT;
++ assert(ring < ARRAY_SIZE(kgem->requests));
++
+ if (kgem->batch_bo) {
+ kgem_bo_destroy(kgem, kgem->batch_bo);
+ kgem->batch_bo = NULL;
+ }
+
++ if (!list_is_empty(&kgem->requests[ring])) {
++ struct kgem_request *rq;
++
++ rq = list_first_entry(&kgem->requests[ring],
++ struct kgem_request, list);
++ assert(rq->ring == ring);
++ assert(rq->bo);
++ assert(RQ(rq->bo->rq) == rq);
++ if (kgem_bo_wait(kgem, rq->bo) == 0)
++ goto restart;
++ }
++
++ if (flags & CREATE_NO_THROTTLE) {
++ flags &= ~CREATE_NO_THROTTLE;
++ if (kgem_cleanup_cache(kgem))
++ goto restart;
++ }
++
++ DBG(("%s: unable to map batch bo, mallocing(size=%d)\n",
++ __FUNCTION__, sizeof(uint32_t)*kgem->batch_size));
+ if (posix_memalign((void **)&kgem->batch, PAGE_SIZE,
+ ALIGN(sizeof(uint32_t) * kgem->batch_size, PAGE_SIZE))) {
+ ERR(("%s: batch allocation failed, disabling acceleration\n", __FUNCTION__));
+@@ -1495,18 +1877,79 @@ static struct kgem_bo *kgem_new_batch(struct kgem *kgem)
+ return last;
+ }
+
+-void kgem_init(struct kgem *kgem, int fd, struct pci_device *dev, unsigned gen)
++static void
++no_retire(struct kgem *kgem)
++{
++ (void)kgem;
++}
++
++static void
++no_expire(struct kgem *kgem)
++{
++ (void)kgem;
++}
++
++static void
++no_context_switch(struct kgem *kgem, int new_mode)
++{
++ (void)kgem;
++ (void)new_mode;
++}
++
++static uint64_t get_gtt_size(int fd)
+ {
+ struct drm_i915_gem_get_aperture aperture;
++ struct local_i915_gem_context_param {
++ uint32_t context;
++ uint32_t size;
++ uint64_t param;
++#define LOCAL_CONTEXT_PARAM_BAN_PERIOD 0x1
++#define LOCAL_CONTEXT_PARAM_NO_ZEROMAP 0x2
++#define LOCAL_CONTEXT_PARAM_GTT_SIZE 0x3
++ uint64_t value;
++ } p;
++#define LOCAL_I915_GEM_CONTEXT_GETPARAM 0x34
++#define LOCAL_IOCTL_I915_GEM_CONTEXT_GETPARAM DRM_IOWR (DRM_COMMAND_BASE + LOCAL_I915_GEM_CONTEXT_GETPARAM, struct local_i915_gem_context_param)
++
++ memset(&aperture, 0, sizeof(aperture));
++
++ memset(&p, 0, sizeof(p));
++ p.param = LOCAL_CONTEXT_PARAM_GTT_SIZE;
++ if (drmIoctl(fd, LOCAL_IOCTL_I915_GEM_CONTEXT_GETPARAM, &p) == 0)
++ aperture.aper_size = p.value;
++ if (aperture.aper_size == 0)
++ (void)drmIoctl(fd, DRM_IOCTL_I915_GEM_GET_APERTURE, &aperture);
++ if (aperture.aper_size == 0)
++ aperture.aper_size = 64*1024*1024;
++
++ DBG(("%s: aperture size %lld, available now %lld\n",
++ __FUNCTION__,
++ (long long)aperture.aper_size,
++ (long long)aperture.aper_available_size));
++
++ /* clamp aperture to uint32_t for simplicity */
++ if (aperture.aper_size > 0xc0000000)
++ aperture.aper_size = 0xc0000000;
++
++ return aperture.aper_size;
++}
++
++void kgem_init(struct kgem *kgem, int fd, struct pci_device *dev, unsigned gen)
++{
+ size_t totalram;
+ unsigned half_gpu_max;
+ unsigned int i, j;
++ uint64_t gtt_size;
+
+ DBG(("%s: fd=%d, gen=%d\n", __FUNCTION__, fd, gen));
+
+ kgem->fd = fd;
+ kgem->gen = gen;
+
++ kgem->retire = no_retire;
++ kgem->expire = no_expire;
++ kgem->context_switch = no_context_switch;
++
+ list_init(&kgem->requests[0]);
+ list_init(&kgem->requests[1]);
+ list_init(&kgem->batch_buffers);
+@@ -1586,10 +2029,21 @@ void kgem_init(struct kgem *kgem, int fd, struct pci_device *dev, unsigned gen)
+ DBG(("%s: can blt to cpu? %d\n", __FUNCTION__,
+ kgem->can_blt_cpu));
+
++ kgem->can_blt_y = test_can_blt_y(kgem);
++ DBG(("%s: can blit to Y-tiled surfaces? %d\n", __FUNCTION__,
++ kgem->can_blt_y));
++
+ kgem->can_render_y = gen != 021 && (gen >> 3) != 4;
+ DBG(("%s: can render to Y-tiled surfaces? %d\n", __FUNCTION__,
+ kgem->can_render_y));
+
++ kgem->can_scanout_y = test_can_scanout_y(kgem);
++ DBG(("%s: can scanout Y-tiled surfaces? %d\n", __FUNCTION__,
++ kgem->can_scanout_y));
++
++ kgem->has_dirtyfb = test_has_dirtyfb(kgem);
++ DBG(("%s: has dirty fb? %d\n", __FUNCTION__, kgem->has_dirtyfb));
++
+ kgem->has_secure_batches = test_has_secure_batches(kgem);
+ DBG(("%s: can use privileged batchbuffers? %d\n", __FUNCTION__,
+ kgem->has_secure_batches));
+@@ -1620,7 +2074,7 @@ void kgem_init(struct kgem *kgem, int fd, struct pci_device *dev, unsigned gen)
+ if (!kgem->has_relaxed_delta && kgem->batch_size > 4*1024)
+ kgem->batch_size = 4*1024;
+
+- if (!kgem_init_pinned_batches(kgem) && gen == 020) {
++ if (!kgem_init_pinned_batches(kgem)) {
+ xf86DrvMsg(kgem_get_screen_index(kgem), X_WARNING,
+ "Unable to reserve memory for GPU, disabling acceleration.\n");
+ __kgem_set_wedged(kgem);
+@@ -1640,35 +2094,24 @@ void kgem_init(struct kgem *kgem, int fd, struct pci_device *dev, unsigned gen)
+ !DBG_NO_CPU && (kgem->has_llc | kgem->has_userptr | kgem->has_caching),
+ kgem->has_llc, kgem->has_caching, kgem->has_userptr));
+
+- VG_CLEAR(aperture);
+- aperture.aper_size = 0;
+- (void)do_ioctl(fd, DRM_IOCTL_I915_GEM_GET_APERTURE, &aperture);
+- if (aperture.aper_size == 0)
+- aperture.aper_size = 64*1024*1024;
+-
+- DBG(("%s: aperture size %lld, available now %lld\n",
+- __FUNCTION__,
+- (long long)aperture.aper_size,
+- (long long)aperture.aper_available_size));
+-
+- kgem->aperture_total = aperture.aper_size;
+- kgem->aperture_high = aperture.aper_size * 3/4;
+- kgem->aperture_low = aperture.aper_size * 1/3;
++ gtt_size = get_gtt_size(fd);
++ kgem->aperture_total = gtt_size;
++ kgem->aperture_high = gtt_size * 3/4;
++ kgem->aperture_low = gtt_size * 1/3;
+ if (gen < 033) {
+ /* Severe alignment penalties */
+ kgem->aperture_high /= 2;
+ kgem->aperture_low /= 2;
+ }
+- DBG(("%s: aperture low=%d [%d], high=%d [%d]\n", __FUNCTION__,
++ DBG(("%s: aperture low=%u [%u], high=%u [%u]\n", __FUNCTION__,
+ kgem->aperture_low, kgem->aperture_low / (1024*1024),
+ kgem->aperture_high, kgem->aperture_high / (1024*1024)));
+
+ kgem->aperture_mappable = 256 * 1024 * 1024;
+ if (dev != NULL)
+ kgem->aperture_mappable = agp_aperture_size(dev, gen);
+- if (kgem->aperture_mappable == 0 ||
+- kgem->aperture_mappable > aperture.aper_size)
+- kgem->aperture_mappable = aperture.aper_size;
++ if (kgem->aperture_mappable == 0 || kgem->aperture_mappable > gtt_size)
++ kgem->aperture_mappable = gtt_size;
+ DBG(("%s: aperture mappable=%d [%d MiB]\n", __FUNCTION__,
+ kgem->aperture_mappable, kgem->aperture_mappable / (1024*1024)));
+
+@@ -1697,7 +2140,7 @@ void kgem_init(struct kgem *kgem, int fd, struct pci_device *dev, unsigned gen)
+ __FUNCTION__));
+ totalram = kgem->aperture_total;
+ }
+- DBG(("%s: total ram=%ld\n", __FUNCTION__, (long)totalram));
++ DBG(("%s: total ram=%lld\n", __FUNCTION__, (long long)totalram));
+ if (kgem->max_object_size > totalram / 2)
+ kgem->max_object_size = totalram / 2;
+ if (kgem->max_gpu_size > totalram / 4)
+@@ -1749,11 +2192,11 @@ void kgem_init(struct kgem *kgem, int fd, struct pci_device *dev, unsigned gen)
+ if (DBG_NO_CPU)
+ kgem->max_cpu_size = 0;
+
+- DBG(("%s: maximum object size=%d\n",
++ DBG(("%s: maximum object size=%u\n",
+ __FUNCTION__, kgem->max_object_size));
+- DBG(("%s: large object thresold=%d\n",
++ DBG(("%s: large object thresold=%u\n",
+ __FUNCTION__, kgem->large_object_size));
+- DBG(("%s: max object sizes (gpu=%d, cpu=%d, tile upload=%d, copy=%d)\n",
++ DBG(("%s: max object sizes (gpu=%u, cpu=%u, tile upload=%u, copy=%u)\n",
+ __FUNCTION__,
+ kgem->max_gpu_size, kgem->max_cpu_size,
+ kgem->max_upload_tile_size, kgem->max_copy_tile_size));
+@@ -2043,8 +2486,34 @@ static void kgem_add_bo(struct kgem *kgem, struct kgem_bo *bo)
+ kgem->flush |= bo->flush;
+ }
+
++static void kgem_clear_swctrl(struct kgem *kgem)
++{
++ uint32_t *b;
++
++ if (kgem->bcs_state == 0)
++ return;
++
++ DBG(("%s: clearin SWCTRL LRI from %x\n",
++ __FUNCTION__, kgem->bcs_state));
++
++ b = kgem->batch + kgem->nbatch;
++ kgem->nbatch += 7;
++
++ *b++ = MI_FLUSH_DW;
++ *b++ = 0;
++ *b++ = 0;
++ *b++ = 0;
++
++ *b++ = MI_LOAD_REGISTER_IMM;
++ *b++ = BCS_SWCTRL;
++ *b++ = (BCS_SRC_Y | BCS_DST_Y) << 16;
++
++ kgem->bcs_state = 0;
++}
++
+ static uint32_t kgem_end_batch(struct kgem *kgem)
+ {
++ kgem_clear_swctrl(kgem);
+ kgem->batch[kgem->nbatch++] = MI_BATCH_BUFFER_END;
+ if (kgem->nbatch & 1)
+ kgem->batch[kgem->nbatch++] = MI_NOOP;
+@@ -2064,17 +2533,6 @@ static void kgem_bo_binding_free(struct kgem *kgem, struct kgem_bo *bo)
+ }
+ }
+
+-static void kgem_bo_rmfb(struct kgem *kgem, struct kgem_bo *bo)
+-{
+- if (bo->scanout && bo->delta) {
+- DBG(("%s: releasing fb=%d for handle=%d\n",
+- __FUNCTION__, bo->delta, bo->handle));
+- /* XXX will leak if we are not DRM_MASTER. *shrug* */
+- do_ioctl(kgem->fd, DRM_IOCTL_MODE_RMFB, &bo->delta);
+- bo->delta = 0;
+- }
+-}
+-
+ static void kgem_bo_free(struct kgem *kgem, struct kgem_bo *bo)
+ {
+ DBG(("%s: handle=%d, size=%d\n", __FUNCTION__, bo->handle, bytes(bo)));
+@@ -2150,13 +2608,16 @@ inline static void kgem_bo_move_to_inactive(struct kgem *kgem,
+ assert(!bo->snoop);
+ assert(!bo->flush);
+ assert(!bo->needs_flush);
++ assert(!bo->delta);
+ assert(list_is_empty(&bo->vma));
+ assert_tiling(kgem, bo);
+- assert_cacheing(kgem, bo);
++ assert_caching(kgem, bo);
+ ASSERT_IDLE(kgem, bo->handle);
+
+ if (bucket(bo) >= NUM_CACHE_BUCKETS) {
+ if (bo->map__gtt) {
++ DBG(("%s: relinquishing large GTT mapping for handle=%d\n",
++ __FUNCTION__, bo->handle));
+ munmap(bo->map__gtt, bytes(bo));
+ bo->map__gtt = NULL;
+ }
+@@ -2167,6 +2628,8 @@ inline static void kgem_bo_move_to_inactive(struct kgem *kgem,
+ assert(list_is_empty(&bo->vma));
+ list_move(&bo->list, &kgem->inactive[bucket(bo)]);
+ if (bo->map__gtt && !kgem_bo_can_map(kgem, bo)) {
++ DBG(("%s: relinquishing old GTT mapping for handle=%d\n",
++ __FUNCTION__, bo->handle));
+ munmap(bo->map__gtt, bytes(bo));
+ bo->map__gtt = NULL;
+ }
+@@ -2191,6 +2654,10 @@ static struct kgem_bo *kgem_bo_replace_io(struct kgem_bo *bo)
+ return bo;
+
+ assert(!bo->snoop);
++ assert(!bo->purged);
++ assert(!bo->scanout);
++ assert(!bo->delta);
++
+ if (__kgem_freed_bo) {
+ base = __kgem_freed_bo;
+ __kgem_freed_bo = *(struct kgem_bo **)base;
+@@ -2221,6 +2688,7 @@ inline static void kgem_bo_remove_from_inactive(struct kgem *kgem,
+ list_del(&bo->list);
+ assert(bo->rq == NULL);
+ assert(bo->exec == NULL);
++ assert(!bo->purged);
+ if (!list_is_empty(&bo->vma)) {
+ assert(bo->map__gtt || bo->map__wc || bo->map__cpu);
+ list_del(&bo->vma);
+@@ -2305,7 +2773,6 @@ static void kgem_bo_move_to_scanout(struct kgem *kgem, struct kgem_bo *bo)
+ list_move(&bo->list, &kgem->scanout);
+
+ kgem->need_expire = true;
+-
+ }
+
+ static void kgem_bo_move_to_snoop(struct kgem *kgem, struct kgem_bo *bo)
+@@ -2316,6 +2783,8 @@ static void kgem_bo_move_to_snoop(struct kgem *kgem, struct kgem_bo *bo)
+ assert(!bo->needs_flush);
+ assert(bo->refcnt == 0);
+ assert(bo->exec == NULL);
++ assert(!bo->purged);
++ assert(!bo->delta);
+
+ if (DBG_NO_SNOOP_CACHE) {
+ kgem_bo_free(kgem, bo);
+@@ -2351,8 +2820,7 @@ static bool kgem_bo_move_to_cache(struct kgem *kgem, struct kgem_bo *bo)
+ kgem_bo_move_to_snoop(kgem, bo);
+ } else if (bo->scanout) {
+ kgem_bo_move_to_scanout(kgem, bo);
+- } else if ((bo = kgem_bo_replace_io(bo))->reusable &&
+- kgem_bo_set_purgeable(kgem, bo)) {
++ } else if ((bo = kgem_bo_replace_io(bo))->reusable) {
+ kgem_bo_move_to_inactive(kgem, bo);
+ retired = true;
+ } else
+@@ -2429,7 +2897,7 @@ void kgem_bo_undo(struct kgem *kgem, struct kgem_bo *bo)
+ DBG(("%s: only handle in batch, discarding last operations for handle=%d\n",
+ __FUNCTION__, bo->handle));
+
+- assert(bo->exec == &kgem->exec[0]);
++ assert(bo->exec == &_kgem_dummy_exec || bo->exec == &kgem->exec[0]);
+ assert(kgem->exec[0].handle == bo->handle);
+ assert(RQ(bo->rq) == kgem->next_request);
+
+@@ -2457,16 +2925,23 @@ void kgem_bo_pair_undo(struct kgem *kgem, struct kgem_bo *a, struct kgem_bo *b)
+
+ if (a == NULL || b == NULL)
+ return;
++ assert(a != b);
+ if (a->exec == NULL || b->exec == NULL)
+ return;
+
+- DBG(("%s: only handles in batch, discarding last operations for handle=%d and handle=%d\n",
+- __FUNCTION__, a->handle, b->handle));
++ DBG(("%s: only handles in batch, discarding last operations for handle=%d (index=%d) and handle=%d (index=%d)\n",
++ __FUNCTION__,
++ a->handle, a->proxy ? -1 : a->exec - kgem->exec,
++ b->handle, b->proxy ? -1 : b->exec - kgem->exec));
+
+- assert(a->exec == &kgem->exec[0] || a->exec == &kgem->exec[1]);
++ assert(a->exec == &_kgem_dummy_exec ||
++ a->exec == &kgem->exec[0] ||
++ a->exec == &kgem->exec[1]);
+ assert(a->handle == kgem->exec[0].handle || a->handle == kgem->exec[1].handle);
+ assert(RQ(a->rq) == kgem->next_request);
+- assert(b->exec == &kgem->exec[0] || b->exec == &kgem->exec[1]);
++ assert(b->exec == &_kgem_dummy_exec ||
++ b->exec == &kgem->exec[0] ||
++ b->exec == &kgem->exec[1]);
+ assert(b->handle == kgem->exec[0].handle || b->handle == kgem->exec[1].handle);
+ assert(RQ(b->rq) == kgem->next_request);
+
+@@ -2487,6 +2962,7 @@ static void __kgem_bo_destroy(struct kgem *kgem, struct kgem_bo *bo)
+ DBG(("%s: handle=%d, size=%d\n", __FUNCTION__, bo->handle, bytes(bo)));
+
+ assert(list_is_empty(&bo->list));
++ assert(list_is_empty(&bo->vma));
+ assert(bo->refcnt == 0);
+ assert(bo->proxy == NULL);
+ assert(bo->active_scanout == 0);
+@@ -2532,7 +3008,7 @@ static void __kgem_bo_destroy(struct kgem *kgem, struct kgem_bo *bo)
+ assert(bo->snoop == false);
+ assert(bo->io == false);
+ assert(bo->scanout == false);
+- assert_cacheing(kgem, bo);
++ assert_caching(kgem, bo);
+
+ kgem_bo_undo(kgem, bo);
+ assert(bo->refcnt == 0);
+@@ -2556,9 +3032,6 @@ static void __kgem_bo_destroy(struct kgem *kgem, struct kgem_bo *bo)
+ assert(list_is_empty(&bo->request));
+
+ if (bo->map__cpu == NULL || bucket(bo) >= NUM_CACHE_BUCKETS) {
+- if (!kgem_bo_set_purgeable(kgem, bo))
+- goto destroy;
+-
+ if (!kgem->has_llc && bo->domain == DOMAIN_CPU)
+ goto destroy;
+
+@@ -2647,7 +3120,7 @@ static bool kgem_retire__flushing(struct kgem *kgem)
+ int count = 0;
+ list_for_each_entry(bo, &kgem->flushing, request)
+ count++;
+- DBG(("%s: %d bo on flushing list\n", __FUNCTION__, count));
++ DBG(("%s: %d bo on flushing list, retired? %d\n", __FUNCTION__, count, retired));
+ }
+ #endif
+
+@@ -2656,6 +3129,34 @@ static bool kgem_retire__flushing(struct kgem *kgem)
+ return retired;
+ }
+
++static bool __kgem_bo_flush(struct kgem *kgem, struct kgem_bo *bo)
++{
++ struct drm_i915_gem_busy busy;
++
++ if (!bo->needs_flush)
++ return false;
++
++ bo->needs_flush = false;
++
++ VG_CLEAR(busy);
++ busy.handle = bo->handle;
++ busy.busy = !kgem->wedged;
++ (void)do_ioctl(kgem->fd, DRM_IOCTL_I915_GEM_BUSY, &busy);
++ DBG(("%s: handle=%d, busy=%d, wedged=%d\n",
++ __FUNCTION__, bo->handle, busy.busy, kgem->wedged));
++
++ if (busy.busy == 0)
++ return false;
++
++ DBG(("%s: moving %d to flushing\n",
++ __FUNCTION__, bo->handle));
++ list_add(&bo->request, &kgem->flushing);
++ bo->rq = MAKE_REQUEST(kgem, !!(busy.busy & ~0x1ffff));
++ bo->needs_flush = busy.busy & 0xffff;
++ kgem->need_retire = true;
++ return true;
++}
++
+ static bool __kgem_retire_rq(struct kgem *kgem, struct kgem_request *rq)
+ {
+ bool retired = false;
+@@ -2663,6 +3164,8 @@ static bool __kgem_retire_rq(struct kgem *kgem, struct kgem_request *rq)
+ DBG(("%s: request %d complete\n",
+ __FUNCTION__, rq->bo->handle));
+ assert(RQ(rq->bo->rq) == rq);
++ assert(rq != (struct kgem_request *)kgem);
++ assert(rq != &kgem->static_request);
+
+ if (rq == kgem->fence[rq->ring])
+ kgem->fence[rq->ring] = NULL;
+@@ -2680,19 +3183,14 @@ static bool __kgem_retire_rq(struct kgem *kgem, struct kgem_request *rq)
+
+ list_del(&bo->request);
+
+- if (bo->needs_flush)
+- bo->needs_flush = __kgem_busy(kgem, bo->handle);
+- if (bo->needs_flush) {
+- DBG(("%s: moving %d to flushing\n",
++ if (unlikely(__kgem_bo_flush(kgem, bo))) {
++ assert(bo != rq->bo);
++ DBG(("%s: movied %d to flushing\n",
+ __FUNCTION__, bo->handle));
+- list_add(&bo->request, &kgem->flushing);
+- bo->rq = MAKE_REQUEST(kgem, RQ_RING(bo->rq));
+- kgem->need_retire = true;
+ continue;
+ }
+
+ bo->domain = DOMAIN_NONE;
+- bo->gtt_dirty = false;
+ bo->rq = NULL;
+ if (bo->refcnt)
+ continue;
+@@ -2706,14 +3204,8 @@ static bool __kgem_retire_rq(struct kgem *kgem, struct kgem_request *rq)
+ assert(rq->bo->refcnt > 0);
+
+ if (--rq->bo->refcnt == 0) {
+- if (kgem_bo_set_purgeable(kgem, rq->bo)) {
+- kgem_bo_move_to_inactive(kgem, rq->bo);
+- retired = true;
+- } else {
+- DBG(("%s: closing %d\n",
+- __FUNCTION__, rq->bo->handle));
+- kgem_bo_free(kgem, rq->bo);
+- }
++ kgem_bo_move_to_inactive(kgem, rq->bo);
++ retired = true;
+ }
+
+ __kgem_request_free(rq);
+@@ -2724,13 +3216,18 @@ static bool kgem_retire__requests_ring(struct kgem *kgem, int ring)
+ {
+ bool retired = false;
+
++ assert(ring < ARRAY_SIZE(kgem->requests));
+ while (!list_is_empty(&kgem->requests[ring])) {
+ struct kgem_request *rq;
+
++ DBG(("%s: retiring ring %d\n", __FUNCTION__, ring));
++
+ rq = list_first_entry(&kgem->requests[ring],
+ struct kgem_request,
+ list);
+ assert(rq->ring == ring);
++ assert(rq->bo);
++ assert(RQ(rq->bo->rq) == rq);
+ if (__kgem_busy(kgem, rq->bo->handle))
+ break;
+
+@@ -2751,8 +3248,8 @@ static bool kgem_retire__requests_ring(struct kgem *kgem, int ring)
+ struct kgem_request,
+ list)->bo;
+
+- DBG(("%s: ring=%d, %d outstanding requests, oldest=%d\n",
+- __FUNCTION__, ring, count, bo ? bo->handle : 0));
++ DBG(("%s: ring=%d, %d outstanding requests, oldest=%d, retired? %d\n",
++ __FUNCTION__, ring, count, bo ? bo->handle : 0, retired));
+ }
+ #endif
+
+@@ -2824,6 +3321,8 @@ bool __kgem_ring_is_idle(struct kgem *kgem, int ring)
+ rq = list_last_entry(&kgem->requests[ring],
+ struct kgem_request, list);
+ assert(rq->ring == ring);
++ assert(rq->bo);
++ assert(RQ(rq->bo->rq) == rq);
+ if (__kgem_busy(kgem, rq->bo->handle)) {
+ DBG(("%s: last requests handle=%d still busy\n",
+ __FUNCTION__, rq->bo->handle));
+@@ -2845,23 +3344,30 @@ bool __kgem_ring_is_idle(struct kgem *kgem, int ring)
+ return true;
+ }
+
+-void __kgem_retire_requests_upto(struct kgem *kgem, struct kgem_bo *bo)
++bool __kgem_retire_requests_upto(struct kgem *kgem, struct kgem_bo *bo)
+ {
+- struct kgem_request *rq = bo->rq, *tmp;
+- struct list *requests = &kgem->requests[RQ_RING(rq) == I915_EXEC_BLT];
++ struct kgem_request * const rq = RQ(bo->rq), *tmp;
++ struct list *requests = &kgem->requests[rq->ring];
++
++ DBG(("%s(handle=%d, ring=%d)\n", __FUNCTION__, bo->handle, rq->ring));
+
+- rq = RQ(rq);
+ assert(rq != &kgem->static_request);
+ if (rq == (struct kgem_request *)kgem) {
+ __kgem_bo_clear_busy(bo);
+- return;
++ return false;
+ }
+
++ assert(rq->ring < ARRAY_SIZE(kgem->requests));
+ do {
+ tmp = list_first_entry(requests, struct kgem_request, list);
+ assert(tmp->ring == rq->ring);
+ __kgem_retire_rq(kgem, tmp);
+ } while (tmp != rq);
++
++ assert(bo->needs_flush || bo->rq == NULL);
++ assert(bo->needs_flush || list_is_empty(&bo->request));
++ assert(bo->needs_flush || bo->domain == DOMAIN_NONE);
++ return bo->rq;
+ }
+
+ #if 0
+@@ -2932,6 +3438,7 @@ static void kgem_commit(struct kgem *kgem)
+ bo->binding.offset = 0;
+ bo->domain = DOMAIN_GPU;
+ bo->gpu_dirty = false;
++ bo->gtt_dirty = false;
+
+ if (bo->proxy) {
+ /* proxies are not used for domain tracking */
+@@ -2955,6 +3462,23 @@ static void kgem_commit(struct kgem *kgem)
+ kgem_throttle(kgem);
+ }
+
++ while (!list_is_empty(&rq->buffers)) {
++ bo = list_first_entry(&rq->buffers,
++ struct kgem_bo,
++ request);
++
++ assert(RQ(bo->rq) == rq);
++ assert(bo->exec == NULL);
++ assert(bo->domain == DOMAIN_GPU);
++
++ list_del(&bo->request);
++ bo->domain = DOMAIN_NONE;
++ bo->rq = NULL;
++
++ if (bo->refcnt == 0)
++ _kgem_bo_destroy(kgem, bo);
++ }
++
+ kgem_retire(kgem);
+ assert(list_is_empty(&rq->buffers));
+
+@@ -2964,7 +3488,9 @@ static void kgem_commit(struct kgem *kgem)
+ gem_close(kgem->fd, rq->bo->handle);
+ kgem_cleanup_cache(kgem);
+ } else {
++ assert(rq != (struct kgem_request *)kgem);
+ assert(rq->ring < ARRAY_SIZE(kgem->requests));
++ assert(rq->bo);
+ list_add_tail(&rq->list, &kgem->requests[rq->ring]);
+ kgem->need_throttle = kgem->need_retire = 1;
+
+@@ -2988,8 +3514,10 @@ static void kgem_close_inactive(struct kgem *kgem)
+ {
+ unsigned int i;
+
+- for (i = 0; i < ARRAY_SIZE(kgem->inactive); i++)
++ for (i = 0; i < ARRAY_SIZE(kgem->inactive); i++) {
+ kgem_close_list(kgem, &kgem->inactive[i]);
++ assert(list_is_empty(&kgem->inactive[i]));
++ }
+ }
+
+ static void kgem_finish_buffers(struct kgem *kgem)
+@@ -3079,10 +3607,13 @@ static void kgem_finish_buffers(struct kgem *kgem)
+ kgem->has_handle_lut ? bo->base.target_handle : shrink->handle;
+ for (n = 0; n < kgem->nreloc; n++) {
+ if (kgem->reloc[n].target_handle == bo->base.target_handle) {
++ uint64_t addr = (int)kgem->reloc[n].delta + shrink->presumed_offset;
++ kgem->batch[kgem->reloc[n].offset/sizeof(kgem->batch[0])] = addr;
++ if (kgem->gen >= 0100)
++ kgem->batch[kgem->reloc[n].offset/sizeof(kgem->batch[0]) + 1] = addr >> 32;
++
+ kgem->reloc[n].target_handle = shrink->target_handle;
+ kgem->reloc[n].presumed_offset = shrink->presumed_offset;
+- kgem->batch[kgem->reloc[n].offset/sizeof(kgem->batch[0])] =
+- kgem->reloc[n].delta + shrink->presumed_offset;
+ }
+ }
+
+@@ -3124,10 +3655,13 @@ static void kgem_finish_buffers(struct kgem *kgem)
+ kgem->has_handle_lut ? bo->base.target_handle : shrink->handle;
+ for (n = 0; n < kgem->nreloc; n++) {
+ if (kgem->reloc[n].target_handle == bo->base.target_handle) {
++ uint64_t addr = (int)kgem->reloc[n].delta + shrink->presumed_offset;
++ kgem->batch[kgem->reloc[n].offset/sizeof(kgem->batch[0])] = addr;
++ if (kgem->gen >= 0100)
++ kgem->batch[kgem->reloc[n].offset/sizeof(kgem->batch[0]) + 1] = addr >> 32;
++
+ kgem->reloc[n].target_handle = shrink->target_handle;
+ kgem->reloc[n].presumed_offset = shrink->presumed_offset;
+- kgem->batch[kgem->reloc[n].offset/sizeof(kgem->batch[0])] =
+- kgem->reloc[n].delta + shrink->presumed_offset;
+ }
+ }
+
+@@ -3195,6 +3729,9 @@ static void kgem_cleanup(struct kgem *kgem)
+ kgem_bo_free(kgem, bo);
+ }
+
++ if (--rq->bo->refcnt == 0)
++ kgem_bo_free(kgem, rq->bo);
++
+ __kgem_request_free(rq);
+ }
+ }
+@@ -3210,7 +3747,9 @@ kgem_batch_write(struct kgem *kgem,
+ char *ptr;
+ int ret;
+
+- ASSERT_IDLE(kgem, bo->handle);
++ assert(bo->exec == NULL);
++ assert(bo->rq == NULL);
++ assert(!__kgem_busy(kgem, bo->handle));
+
+ #if DBG_NO_EXEC
+ {
+@@ -3371,55 +3910,54 @@ static int compact_batch_surface(struct kgem *kgem, int *shrink)
+ return size * sizeof(uint32_t);
+ }
+
++static struct kgem_bo *first_available(struct kgem *kgem, struct list *list)
++{
++ struct kgem_bo *bo;
++
++ list_for_each_entry(bo, list, list) {
++ assert(bo->refcnt > 0);
++
++ if (bo->rq) {
++ assert(RQ(bo->rq)->bo == bo);
++ if (__kgem_busy(kgem, bo->handle))
++ break;
++
++ __kgem_retire_rq(kgem, RQ(bo->rq));
++ assert(bo->rq == NULL);
++ }
++
++ if (bo->refcnt > 1)
++ continue;
++
++ list_move_tail(&bo->list, list);
++ return kgem_bo_reference(bo);
++ }
++
++ return NULL;
++}
++
+ static struct kgem_bo *
+ kgem_create_batch(struct kgem *kgem)
+ {
+-#if !DBG_NO_SHRINK_BATCHES
+- struct drm_i915_gem_set_domain set_domain;
+ struct kgem_bo *bo;
+- int shrink = 0;
+- int size;
++ int size, shrink = 0;
+
++#if !DBG_NO_SHRINK_BATCHES
+ if (kgem->surface != kgem->batch_size)
+ size = compact_batch_surface(kgem, &shrink);
+ else
+ size = kgem->nbatch * sizeof(uint32_t);
+
+ if (size <= 4096) {
+- bo = list_first_entry(&kgem->pinned_batches[0],
+- struct kgem_bo,
+- list);
+- if (!bo->rq) {
+-out_4096:
+- assert(bo->refcnt > 0);
+- list_move_tail(&bo->list, &kgem->pinned_batches[0]);
+- bo = kgem_bo_reference(bo);
++ bo = first_available(kgem, &kgem->pinned_batches[0]);
++ if (bo)
+ goto write;
+- }
+-
+- if (!__kgem_busy(kgem, bo->handle)) {
+- assert(RQ(bo->rq)->bo == bo);
+- __kgem_retire_rq(kgem, RQ(bo->rq));
+- goto out_4096;
+- }
+ }
+
+- if (size <= 16384) {
+- bo = list_first_entry(&kgem->pinned_batches[1],
+- struct kgem_bo,
+- list);
+- if (!bo->rq) {
+-out_16384:
+- assert(bo->refcnt > 0);
+- list_move_tail(&bo->list, &kgem->pinned_batches[1]);
+- bo = kgem_bo_reference(bo);
+- goto write;
+- }
+-
+- if (!__kgem_busy(kgem, bo->handle)) {
+- __kgem_retire_rq(kgem, RQ(bo->rq));
+- goto out_16384;
+- }
++ if (size <= 16384) {
++ bo = first_available(kgem, &kgem->pinned_batches[1]);
++ if (bo)
++ goto write;
+ }
+
+ if (kgem->gen == 020) {
+@@ -3443,16 +3981,8 @@ out_16384:
+ list_move_tail(&bo->list, &kgem->pinned_batches[size > 4096]);
+
+ DBG(("%s: syncing due to busy batches\n", __FUNCTION__));
+-
+- VG_CLEAR(set_domain);
+- set_domain.handle = bo->handle;
+- set_domain.read_domains = I915_GEM_DOMAIN_GTT;
+- set_domain.write_domain = I915_GEM_DOMAIN_GTT;
+- if (do_ioctl(kgem->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &set_domain)) {
+- DBG(("%s: sync: GPU hang detected\n", __FUNCTION__));
+- kgem_throttle(kgem);
++ if (kgem_bo_wait(kgem, bo))
+ return NULL;
+- }
+
+ kgem_retire(kgem);
+ assert(bo->rq == NULL);
+@@ -3460,9 +3990,14 @@ out_16384:
+ goto write;
+ }
+ }
++#else
++ if (kgem->surface != kgem->batch_size)
++ size = kgem->batch_size * sizeof(uint32_t);
++ else
++ size = kgem->nbatch * sizeof(uint32_t);
++#endif
+
+- bo = NULL;
+- if (!kgem->has_llc) {
++ if (!kgem->batch_bo || !kgem->has_llc) {
+ bo = kgem_create_linear(kgem, size, CREATE_NO_THROTTLE);
+ if (bo) {
+ write:
+@@ -3471,14 +4006,11 @@ write:
+ kgem_bo_destroy(kgem, bo);
+ return NULL;
+ }
++ return bo;
+ }
+ }
+- if (bo == NULL)
+- bo = kgem_new_batch(kgem);
+- return bo;
+-#else
++
+ return kgem_new_batch(kgem);
+-#endif
+ }
+
+ #if !NDEBUG
+@@ -3530,7 +4062,7 @@ static void dump_fence_regs(struct kgem *kgem)
+
+ static int do_execbuf(struct kgem *kgem, struct drm_i915_gem_execbuffer2 *execbuf)
+ {
+- int ret, err;
++ int ret;
+
+ retry:
+ ret = do_ioctl(kgem->fd, DRM_IOCTL_I915_GEM_EXECBUFFER2, execbuf);
+@@ -3547,26 +4079,25 @@ retry:
+
+ /* last gasp */
+ ret = do_ioctl(kgem->fd, DRM_IOCTL_I915_GEM_EXECBUFFER2, execbuf);
+- if (ret == 0)
+- return 0;
++ if (ret != -ENOSPC)
++ return ret;
++
++ /* One final trick up our sleeve for when we run out of space.
++ * We turn everything off to free up our pinned framebuffers,
++ * sprites and cursors, and try just one more time.
++ */
+
+ xf86DrvMsg(kgem_get_screen_index(kgem), X_WARNING,
+ "Failed to submit rendering commands, trying again with outputs disabled.\n");
+
+- /* One last trick up our sleeve for when we run out of space.
+- * We turn everything off to free up our pinned framebuffers,
+- * sprites and cursors, and try one last time.
+- */
+- err = errno;
+- if (sna_mode_disable(container_of(kgem, struct sna, kgem))) {
++ if (sna_mode_disable(__to_sna(kgem))) {
+ kgem_cleanup_cache(kgem);
+ ret = do_ioctl(kgem->fd,
+ DRM_IOCTL_I915_GEM_EXECBUFFER2,
+ execbuf);
+ DBG(("%s: last_gasp ret=%d\n", __FUNCTION__, ret));
+- sna_mode_enable(container_of(kgem, struct sna, kgem));
++ sna_mode_enable(__to_sna(kgem));
+ }
+- errno = err;
+
+ return ret;
+ }
+@@ -3575,6 +4106,7 @@ void _kgem_submit(struct kgem *kgem)
+ {
+ struct kgem_request *rq;
+ uint32_t batch_end;
++ int i, ret;
+
+ assert(!DBG_NO_HW);
+ assert(!kgem->wedged);
+@@ -3609,7 +4141,6 @@ void _kgem_submit(struct kgem *kgem)
+ rq->bo = kgem_create_batch(kgem);
+ if (rq->bo) {
+ struct drm_i915_gem_execbuffer2 execbuf;
+- int i, ret;
+
+ assert(!rq->bo->needs_flush);
+
+@@ -3619,7 +4150,8 @@ void _kgem_submit(struct kgem *kgem)
+ kgem->exec[i].relocs_ptr = (uintptr_t)kgem->reloc;
+ kgem->exec[i].alignment = 0;
+ kgem->exec[i].offset = rq->bo->presumed_offset;
+- kgem->exec[i].flags = 0;
++ /* Make sure the kernel releases any fence, ignored if gen4+ */
++ kgem->exec[i].flags = EXEC_OBJECT_NEEDS_FENCE;
+ kgem->exec[i].rsvd1 = 0;
+ kgem->exec[i].rsvd2 = 0;
+
+@@ -3631,7 +4163,8 @@ void _kgem_submit(struct kgem *kgem)
+ memset(&execbuf, 0, sizeof(execbuf));
+ execbuf.buffers_ptr = (uintptr_t)kgem->exec;
+ execbuf.buffer_count = kgem->nexec;
+- execbuf.batch_len = batch_end*sizeof(uint32_t);
++ if (kgem->gen < 030)
++ execbuf.batch_len = batch_end*sizeof(uint32_t);
+ execbuf.flags = kgem->ring | kgem->batch_flags;
+
+ if (DBG_DUMP) {
+@@ -3645,91 +4178,98 @@ void _kgem_submit(struct kgem *kgem)
+ }
+
+ ret = do_execbuf(kgem, &execbuf);
+- if (DEBUG_SYNC && ret == 0) {
+- struct drm_i915_gem_set_domain set_domain;
+-
+- VG_CLEAR(set_domain);
+- set_domain.handle = rq->bo->handle;
+- set_domain.read_domains = I915_GEM_DOMAIN_GTT;
+- set_domain.write_domain = I915_GEM_DOMAIN_GTT;
++ } else
++ ret = -ENOMEM;
+
+- ret = do_ioctl(kgem->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &set_domain);
++ if (ret < 0) {
++ kgem_throttle(kgem);
++ if (!kgem->wedged) {
++ xf86DrvMsg(kgem_get_screen_index(kgem), X_ERROR,
++ "Failed to submit rendering commands (%s), disabling acceleration.\n",
++ strerror(-ret));
++ __kgem_set_wedged(kgem);
+ }
+- if (ret < 0) {
+- kgem_throttle(kgem);
+- if (!kgem->wedged) {
+- xf86DrvMsg(kgem_get_screen_index(kgem), X_ERROR,
+- "Failed to submit rendering commands, disabling acceleration.\n");
+- __kgem_set_wedged(kgem);
+- }
+
+ #if !NDEBUG
+- ErrorF("batch[%d/%d]: %d %d %d, nreloc=%d, nexec=%d, nfence=%d, aperture=%d, fenced=%d, high=%d,%d: errno=%d\n",
+- kgem->mode, kgem->ring, batch_end, kgem->nbatch, kgem->surface,
+- kgem->nreloc, kgem->nexec, kgem->nfence, kgem->aperture, kgem->aperture_fenced, kgem->aperture_high, kgem->aperture_total, -ret);
++ ErrorF("batch[%d/%d]: %d %d %d, nreloc=%d, nexec=%d, nfence=%d, aperture=%d, fenced=%d, high=%d,%d: errno=%d\n",
++ kgem->mode, kgem->ring, batch_end, kgem->nbatch, kgem->surface,
++ kgem->nreloc, kgem->nexec, kgem->nfence, kgem->aperture, kgem->aperture_fenced, kgem->aperture_high, kgem->aperture_total, -ret);
+
+- for (i = 0; i < kgem->nexec; i++) {
+- struct kgem_bo *bo, *found = NULL;
++ for (i = 0; i < kgem->nexec; i++) {
++ struct kgem_bo *bo, *found = NULL;
+
+- list_for_each_entry(bo, &kgem->next_request->buffers, request) {
+- if (bo->handle == kgem->exec[i].handle) {
+- found = bo;
+- break;
+- }
++ list_for_each_entry(bo, &kgem->next_request->buffers, request) {
++ if (bo->handle == kgem->exec[i].handle) {
++ found = bo;
++ break;
+ }
+- ErrorF("exec[%d] = handle:%d, presumed offset: %x, size: %d, tiling %d, fenced %d, snooped %d, deleted %d\n",
+- i,
+- kgem->exec[i].handle,
+- (int)kgem->exec[i].offset,
+- found ? kgem_bo_size(found) : -1,
+- found ? found->tiling : -1,
+- (int)(kgem->exec[i].flags & EXEC_OBJECT_NEEDS_FENCE),
+- found ? found->snoop : -1,
+- found ? found->purged : -1);
+ }
+- for (i = 0; i < kgem->nreloc; i++) {
+- ErrorF("reloc[%d] = pos:%d, target:%d, delta:%d, read:%x, write:%x, offset:%x\n",
+- i,
+- (int)kgem->reloc[i].offset,
+- kgem->reloc[i].target_handle,
+- kgem->reloc[i].delta,
+- kgem->reloc[i].read_domains,
+- kgem->reloc[i].write_domain,
+- (int)kgem->reloc[i].presumed_offset);
++ ErrorF("exec[%d] = handle:%d, presumed offset: %x, size: %d, tiling %d, fenced %d, snooped %d, deleted %d\n",
++ i,
++ kgem->exec[i].handle,
++ (int)kgem->exec[i].offset,
++ found ? kgem_bo_size(found) : -1,
++ found ? found->tiling : -1,
++ (int)(kgem->exec[i].flags & EXEC_OBJECT_NEEDS_FENCE),
++ found ? found->snoop : -1,
++ found ? found->purged : -1);
++ }
++ for (i = 0; i < kgem->nreloc; i++) {
++ ErrorF("reloc[%d] = pos:%d, target:%d, delta:%d, read:%x, write:%x, offset:%x\n",
++ i,
++ (int)kgem->reloc[i].offset,
++ kgem->reloc[i].target_handle,
++ kgem->reloc[i].delta,
++ kgem->reloc[i].read_domains,
++ kgem->reloc[i].write_domain,
++ (int)kgem->reloc[i].presumed_offset);
++ }
++
++ {
++ struct drm_i915_gem_get_aperture aperture;
++ if (do_ioctl(kgem->fd, DRM_IOCTL_I915_GEM_GET_APERTURE, &aperture) == 0)
++ ErrorF("Aperture size %lld, available %lld\n",
++ (long long)aperture.aper_size,
++ (long long)aperture.aper_available_size);
++ }
++
++ if (ret == -ENOSPC)
++ dump_gtt_info(kgem);
++ if (ret == -EDEADLK)
++ dump_fence_regs(kgem);
++
++ if (DEBUG_SYNC) {
++ int fd = open("/tmp/batchbuffer", O_WRONLY | O_CREAT | O_APPEND, 0666);
++ if (fd != -1) {
++ int ignored = write(fd, kgem->batch, batch_end*sizeof(uint32_t));
++ assert(ignored == batch_end*sizeof(uint32_t));
++ close(fd);
+ }
+
+- {
+- struct drm_i915_gem_get_aperture aperture;
+- if (do_ioctl(kgem->fd, DRM_IOCTL_I915_GEM_GET_APERTURE, &aperture) == 0)
+- ErrorF("Aperture size %lld, available %lld\n",
+- (long long)aperture.aper_size,
+- (long long)aperture.aper_available_size);
+- }
++ FatalError("SNA: failed to submit batchbuffer, errno=%d\n", -ret);
++ }
++#endif
++ } else {
++ if (DEBUG_SYNC) {
++ struct drm_i915_gem_set_domain set_domain;
+
+- if (ret == -ENOSPC)
+- dump_gtt_info(kgem);
+- if (ret == -EDEADLK)
+- dump_fence_regs(kgem);
+-
+- if (DEBUG_SYNC) {
+- int fd = open("/tmp/batchbuffer", O_WRONLY | O_CREAT | O_APPEND, 0666);
+- if (fd != -1) {
+- int ignored = write(fd, kgem->batch, batch_end*sizeof(uint32_t));
+- assert(ignored == batch_end*sizeof(uint32_t));
+- close(fd);
+- }
++ VG_CLEAR(set_domain);
++ set_domain.handle = rq->bo->handle;
++ set_domain.read_domains = I915_GEM_DOMAIN_GTT;
++ set_domain.write_domain = I915_GEM_DOMAIN_GTT;
+
+- FatalError("SNA: failed to submit batchbuffer, errno=%d\n", -ret);
+- }
+-#endif
++ ret = do_ioctl(kgem->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &set_domain);
+ }
+- }
++
+ #if SHOW_BATCH_AFTER
+- if (gem_read(kgem->fd, rq->bo->handle, kgem->batch, 0, batch_end*sizeof(uint32_t)) == 0)
+- __kgem_batch_debug(kgem, batch_end);
++ if (gem_read(kgem->fd, rq->bo->handle, kgem->batch, 0, batch_end*sizeof(uint32_t)) == 0)
++ __kgem_batch_debug(kgem, batch_end);
+ #endif
+- kgem_commit(kgem);
+- if (kgem->wedged)
++
++ kgem_commit(kgem);
++ }
++
++ if (unlikely(kgem->wedged))
+ kgem_cleanup(kgem);
+
+ kgem_reset(kgem);
+@@ -3737,49 +4277,14 @@ void _kgem_submit(struct kgem *kgem)
+ assert(kgem->next_request != NULL);
+ }
+
+-static bool find_hang_state(struct kgem *kgem, char *path, int maxlen)
+-{
+- int minor = kgem_get_minor(kgem);
+-
+- /* Search for our hang state in a few canonical locations.
+- * In the unlikely event of having multiple devices, we
+- * will need to check which minor actually corresponds to ours.
+- */
+-
+- snprintf(path, maxlen, "/sys/class/drm/card%d/error", minor);
+- if (access(path, R_OK) == 0)
+- return true;
+-
+- snprintf(path, maxlen, "/sys/kernel/debug/dri/%d/i915_error_state", minor);
+- if (access(path, R_OK) == 0)
+- return true;
+-
+- snprintf(path, maxlen, "/debug/dri/%d/i915_error_state", minor);
+- if (access(path, R_OK) == 0)
+- return true;
+-
+- path[0] = '\0';
+- return false;
+-}
+-
+ void kgem_throttle(struct kgem *kgem)
+ {
+- if (kgem->wedged)
++ if (unlikely(kgem->wedged))
+ return;
+
+ if (__kgem_throttle(kgem, true)) {
+- static int once;
+- char path[128];
+-
+ xf86DrvMsg(kgem_get_screen_index(kgem), X_ERROR,
+ "Detected a hung GPU, disabling acceleration.\n");
+- if (!once && find_hang_state(kgem, path, sizeof(path))) {
+- xf86DrvMsg(kgem_get_screen_index(kgem), X_ERROR,
+- "When reporting this, please include %s and the full dmesg.\n",
+- path);
+- once = 1;
+- }
+-
+ __kgem_set_wedged(kgem);
+ kgem->need_throttle = false;
+ }
+@@ -3860,7 +4365,8 @@ bool kgem_expire_cache(struct kgem *kgem)
+ bool idle;
+ unsigned int i;
+
+- time(&now);
++ if (!time(&now))
++ return false;
+
+ while (__kgem_freed_bo) {
+ bo = __kgem_freed_bo;
+@@ -3875,7 +4381,7 @@ bool kgem_expire_cache(struct kgem *kgem)
+ }
+
+ kgem_clean_large_cache(kgem);
+- if (container_of(kgem, struct sna, kgem)->scrn->vtSema)
++ if (__to_sna(kgem)->scrn->vtSema)
+ kgem_clean_scanout_cache(kgem);
+
+ expire = 0;
+@@ -3885,6 +4391,7 @@ bool kgem_expire_cache(struct kgem *kgem)
+ break;
+ }
+
++ assert(now);
+ bo->delta = now;
+ }
+ if (expire) {
+@@ -3909,7 +4416,7 @@ bool kgem_expire_cache(struct kgem *kgem)
+ #endif
+
+ kgem_retire(kgem);
+- if (kgem->wedged)
++ if (unlikely(kgem->wedged))
+ kgem_cleanup(kgem);
+
+ kgem->expire(kgem);
+@@ -3930,6 +4437,8 @@ bool kgem_expire_cache(struct kgem *kgem)
+ break;
+ }
+
++ assert(now);
++ kgem_bo_set_purgeable(kgem, bo);
+ bo->delta = now;
+ }
+ }
+@@ -3960,16 +4469,11 @@ bool kgem_expire_cache(struct kgem *kgem)
+ count++;
+ size += bytes(bo);
+ kgem_bo_free(kgem, bo);
+- DBG(("%s: expiring %d\n",
++ DBG(("%s: expiring handle=%d\n",
+ __FUNCTION__, bo->handle));
+ }
+ }
+- if (!list_is_empty(&preserve)) {
+- preserve.prev->next = kgem->inactive[i].next;
+- kgem->inactive[i].next->prev = preserve.prev;
+- kgem->inactive[i].next = preserve.next;
+- preserve.next->prev = &kgem->inactive[i];
+- }
++ list_splice_tail(&preserve, &kgem->inactive[i]);
+ }
+
+ #ifdef DEBUG_MEMORY
+@@ -3998,31 +4502,30 @@ bool kgem_cleanup_cache(struct kgem *kgem)
+ unsigned int i;
+ int n;
+
++ DBG(("%s\n", __FUNCTION__));
++
+ /* sync to the most recent request */
+ for (n = 0; n < ARRAY_SIZE(kgem->requests); n++) {
+ if (!list_is_empty(&kgem->requests[n])) {
+ struct kgem_request *rq;
+- struct drm_i915_gem_set_domain set_domain;
+
+- rq = list_first_entry(&kgem->requests[n],
+- struct kgem_request,
+- list);
++ rq = list_last_entry(&kgem->requests[n],
++ struct kgem_request,
++ list);
+
+ DBG(("%s: sync on cleanup\n", __FUNCTION__));
+-
+- VG_CLEAR(set_domain);
+- set_domain.handle = rq->bo->handle;
+- set_domain.read_domains = I915_GEM_DOMAIN_GTT;
+- set_domain.write_domain = I915_GEM_DOMAIN_GTT;
+- (void)do_ioctl(kgem->fd,
+- DRM_IOCTL_I915_GEM_SET_DOMAIN,
+- &set_domain);
++ assert(rq->ring == n);
++ assert(rq->bo);
++ assert(RQ(rq->bo->rq) == rq);
++ kgem_bo_wait(kgem, rq->bo);
+ }
++ assert(list_is_empty(&kgem->requests[n]));
+ }
+
+ kgem_retire(kgem);
+ kgem_cleanup(kgem);
+
++ DBG(("%s: need_expire?=%d\n", __FUNCTION__, kgem->need_expire));
+ if (!kgem->need_expire)
+ return false;
+
+@@ -4049,6 +4552,8 @@ bool kgem_cleanup_cache(struct kgem *kgem)
+
+ kgem->need_purge = false;
+ kgem->need_expire = false;
++
++ DBG(("%s: complete\n", __FUNCTION__));
+ return true;
+ }
+
+@@ -4079,16 +4584,15 @@ retry_large:
+ goto discard;
+
+ if (bo->tiling != I915_TILING_NONE) {
+- if (use_active)
++ if (use_active && kgem->gen < 040)
+ goto discard;
+
+- if (!gem_set_tiling(kgem->fd, bo->handle,
++ if (!kgem_set_tiling(kgem, bo,
+ I915_TILING_NONE, 0))
+ goto discard;
+-
+- bo->tiling = I915_TILING_NONE;
+- bo->pitch = 0;
+ }
++ assert(bo->tiling == I915_TILING_NONE);
++ bo->pitch = 0;
+
+ if (bo->purged && !kgem_bo_clear_purgeable(kgem, bo))
+ goto discard;
+@@ -4169,17 +4673,17 @@ discard:
+ break;
+ }
+
+- if (I915_TILING_NONE != bo->tiling &&
+- !gem_set_tiling(kgem->fd, bo->handle,
+- I915_TILING_NONE, 0))
+- continue;
++ if (!kgem_set_tiling(kgem, bo, I915_TILING_NONE, 0)) {
++ kgem_bo_free(kgem, bo);
++ break;
++ }
+
+ kgem_bo_remove_from_inactive(kgem, bo);
+ assert(list_is_empty(&bo->vma));
+ assert(list_is_empty(&bo->list));
+
+- bo->tiling = I915_TILING_NONE;
+- bo->pitch = 0;
++ assert(bo->tiling == I915_TILING_NONE);
++ assert(bo->pitch == 0);
+ bo->delta = 0;
+ DBG((" %s: found handle=%d (num_pages=%d) in linear vma cache\n",
+ __FUNCTION__, bo->handle, num_pages(bo)));
+@@ -4225,13 +4729,13 @@ discard:
+ if (first)
+ continue;
+
+- if (!gem_set_tiling(kgem->fd, bo->handle,
+- I915_TILING_NONE, 0))
+- continue;
+-
+- bo->tiling = I915_TILING_NONE;
+- bo->pitch = 0;
++ if (!kgem_set_tiling(kgem, bo, I915_TILING_NONE, 0)) {
++ kgem_bo_free(kgem, bo);
++ break;
++ }
+ }
++ assert(bo->tiling == I915_TILING_NONE);
++ bo->pitch = 0;
+
+ if (bo->map__gtt || bo->map__wc || bo->map__cpu) {
+ if (flags & (CREATE_CPU_MAP | CREATE_GTT_MAP)) {
+@@ -4269,7 +4773,7 @@ discard:
+ kgem_bo_remove_from_inactive(kgem, bo);
+
+ assert(bo->tiling == I915_TILING_NONE);
+- bo->pitch = 0;
++ assert(bo->pitch == 0);
+ bo->delta = 0;
+ DBG((" %s: found handle=%d (num_pages=%d) in linear %s cache\n",
+ __FUNCTION__, bo->handle, num_pages(bo),
+@@ -4340,9 +4844,9 @@ struct kgem_bo *kgem_create_for_name(struct kgem *kgem, uint32_t name)
+
+ bo->unique_id = kgem_get_unique_id(kgem);
+ bo->tiling = tiling.tiling_mode;
+- bo->reusable = false;
+ bo->prime = true;
+- bo->purged = true; /* no coherency guarantees */
++ bo->reusable = false;
++ kgem_bo_unclean(kgem, bo);
+
+ debug_alloc__bo(kgem, bo);
+ return bo;
+@@ -4448,6 +4952,8 @@ int kgem_bo_export_to_prime(struct kgem *kgem, struct kgem_bo *bo)
+ #if defined(DRM_IOCTL_PRIME_HANDLE_TO_FD) && defined(O_CLOEXEC)
+ struct drm_prime_handle args;
+
++ assert(kgem_bo_is_fenced(kgem, bo));
++
+ VG_CLEAR(args);
+ args.handle = bo->handle;
+ args.flags = O_CLOEXEC;
+@@ -4479,6 +4985,8 @@ struct kgem_bo *kgem_create_linear(struct kgem *kgem, int size, unsigned flags)
+ if ((flags & CREATE_UNCACHED) == 0) {
+ bo = search_linear_cache(kgem, size, CREATE_INACTIVE | flags);
+ if (bo) {
++ assert(!bo->purged);
++ assert(!bo->delta);
+ assert(bo->domain != DOMAIN_GPU);
+ ASSERT_IDLE(kgem, bo->handle);
+ bo->refcnt = 1;
+@@ -4760,8 +5268,7 @@ static void __kgem_bo_make_scanout(struct kgem *kgem,
+ struct kgem_bo *bo,
+ int width, int height)
+ {
+- ScrnInfoPtr scrn =
+- container_of(kgem, struct sna, kgem)->scrn;
++ ScrnInfoPtr scrn = __to_sna(kgem)->scrn;
+ struct drm_mode_fb_cmd arg;
+
+ assert(bo->proxy == NULL);
+@@ -4809,6 +5316,48 @@ static void __kgem_bo_make_scanout(struct kgem *kgem,
+ }
+ }
+
++static bool tiling_changed(struct kgem_bo *bo, int tiling, int pitch)
++{
++ if (tiling != bo->tiling)
++ return true;
++
++ return tiling != I915_TILING_NONE && pitch != bo->pitch;
++}
++
++static void set_gpu_tiling(struct kgem *kgem,
++ struct kgem_bo *bo,
++ int tiling, int pitch)
++{
++ DBG(("%s: handle=%d, tiling=%d, pitch=%d\n",
++ __FUNCTION__, bo->handle, tiling, pitch));
++
++ if (tiling_changed(bo, tiling, pitch) && bo->map__gtt) {
++ if (!list_is_empty(&bo->vma)) {
++ list_del(&bo->vma);
++ kgem->vma[0].count--;
++ }
++ munmap(bo->map__gtt, bytes(bo));
++ bo->map__gtt = NULL;
++ }
++
++ bo->tiling = tiling;
++ bo->pitch = pitch;
++}
++
++bool kgem_bo_is_fenced(struct kgem *kgem, struct kgem_bo *bo)
++{
++ struct drm_i915_gem_get_tiling tiling;
++
++ assert(kgem);
++ assert(bo);
++
++ VG_CLEAR(tiling);
++ tiling.handle = bo->handle;
++ tiling.tiling_mode = bo->tiling;
++ (void)do_ioctl(kgem->fd, DRM_IOCTL_I915_GEM_GET_TILING, &tiling);
++ return tiling.tiling_mode == bo->tiling; /* assume pitch is fine! */
++}
++
+ struct kgem_bo *kgem_create_2d(struct kgem *kgem,
+ int width,
+ int height,
+@@ -4892,8 +5441,8 @@ struct kgem_bo *kgem_create_2d(struct kgem *kgem,
+ return last;
+ }
+
+- if (container_of(kgem, struct sna, kgem)->scrn->vtSema) {
+- ScrnInfoPtr scrn = container_of(kgem, struct sna, kgem)->scrn;
++ if (__to_sna(kgem)->scrn->vtSema) {
++ ScrnInfoPtr scrn = __to_sna(kgem)->scrn;
+
+ list_for_each_entry_reverse(bo, &kgem->scanout, list) {
+ struct drm_mode_fb_cmd arg;
+@@ -4915,11 +5464,8 @@ struct kgem_bo *kgem_create_2d(struct kgem *kgem,
+ bo->delta = 0;
+ }
+
+- if (gem_set_tiling(kgem->fd, bo->handle,
+- tiling, pitch)) {
+- bo->tiling = tiling;
+- bo->pitch = pitch;
+- } else {
++ if (!kgem_set_tiling(kgem, bo,
++ tiling, pitch)) {
+ kgem_bo_free(kgem, bo);
+ break;
+ }
+@@ -4950,6 +5496,9 @@ struct kgem_bo *kgem_create_2d(struct kgem *kgem,
+ }
+ }
+
++ if (flags & CREATE_CACHED)
++ return NULL;
++
+ bo = __kgem_bo_create_as_display(kgem, size, tiling, pitch);
+ if (bo)
+ return bo;
+@@ -4987,14 +5536,9 @@ struct kgem_bo *kgem_create_2d(struct kgem *kgem,
+ if (num_pages(bo) < size)
+ continue;
+
+- if (bo->pitch != pitch || bo->tiling != tiling) {
+- if (!gem_set_tiling(kgem->fd, bo->handle,
+- tiling, pitch))
+- continue;
+-
+- bo->pitch = pitch;
+- bo->tiling = tiling;
+- }
++ if (!kgem_set_tiling(kgem, bo, tiling, pitch) &&
++ !exact)
++ set_gpu_tiling(kgem, bo, tiling, pitch);
+ }
+
+ kgem_bo_remove_from_active(kgem, bo);
+@@ -5020,14 +5564,11 @@ large_inactive:
+ if (size > num_pages(bo))
+ continue;
+
+- if (bo->tiling != tiling ||
+- (tiling != I915_TILING_NONE && bo->pitch != pitch)) {
+- if (!gem_set_tiling(kgem->fd, bo->handle,
+- tiling, pitch))
++ if (!kgem_set_tiling(kgem, bo, tiling, pitch)) {
++ if (kgem->gen >= 040 && !exact)
++ set_gpu_tiling(kgem, bo, tiling, pitch);
++ else
+ continue;
+-
+- bo->tiling = tiling;
+- bo->pitch = pitch;
+ }
+
+ if (bo->purged && !kgem_bo_clear_purgeable(kgem, bo)) {
+@@ -5039,7 +5580,6 @@ large_inactive:
+
+ assert(bo->domain != DOMAIN_GPU);
+ bo->unique_id = kgem_get_unique_id(kgem);
+- bo->pitch = pitch;
+ bo->delta = 0;
+ DBG((" 1:from large inactive: pitch=%d, tiling=%d, handle=%d, id=%d\n",
+ bo->pitch, bo->tiling, bo->handle, bo->unique_id));
+@@ -5088,14 +5628,13 @@ large_inactive:
+ if (bo->tiling != tiling ||
+ (tiling != I915_TILING_NONE && bo->pitch != pitch)) {
+ if (bo->map__gtt ||
+- !gem_set_tiling(kgem->fd, bo->handle,
+- tiling, pitch)) {
++ !kgem_set_tiling(kgem, bo,
++ tiling, pitch)) {
+ DBG(("inactive GTT vma with wrong tiling: %d < %d\n",
+ bo->tiling, tiling));
+- continue;
++ kgem_bo_free(kgem, bo);
++ break;
+ }
+- bo->tiling = tiling;
+- bo->pitch = pitch;
+ }
+
+ if (bo->purged && !kgem_bo_clear_purgeable(kgem, bo)) {
+@@ -5103,8 +5642,11 @@ large_inactive:
+ break;
+ }
+
++ if (tiling == I915_TILING_NONE)
++ bo->pitch = pitch;
++
+ assert(bo->tiling == tiling);
+- bo->pitch = pitch;
++ assert(bo->pitch >= pitch);
+ bo->delta = 0;
+ bo->unique_id = kgem_get_unique_id(kgem);
+
+@@ -5170,15 +5712,12 @@ search_active:
+ if (num_pages(bo) < size)
+ continue;
+
+- if (bo->pitch != pitch) {
+- if (!gem_set_tiling(kgem->fd,
+- bo->handle,
+- tiling, pitch))
+- continue;
+-
+- bo->pitch = pitch;
+- }
++ if (!kgem_set_tiling(kgem, bo, tiling, pitch) &&
++ !exact)
++ set_gpu_tiling(kgem, bo, tiling, pitch);
+ }
++ assert(bo->tiling == tiling);
++ assert(bo->pitch >= pitch);
+
+ kgem_bo_remove_from_active(kgem, bo);
+
+@@ -5233,19 +5772,21 @@ search_active:
+ if (num_pages(bo) < size)
+ continue;
+
+- if (bo->tiling != tiling ||
+- (tiling != I915_TILING_NONE && bo->pitch != pitch)) {
+- if (!gem_set_tiling(kgem->fd,
+- bo->handle,
+- tiling, pitch))
+- continue;
++ if (!kgem_set_tiling(kgem, bo, tiling, pitch)) {
++ if (kgem->gen >= 040 && !exact) {
++ set_gpu_tiling(kgem, bo,
++ tiling, pitch);
++ } else {
++ kgem_bo_free(kgem, bo);
++ break;
++ }
+ }
++ assert(bo->tiling == tiling);
++ assert(bo->pitch >= pitch);
+
+ kgem_bo_remove_from_active(kgem, bo);
+
+ bo->unique_id = kgem_get_unique_id(kgem);
+- bo->pitch = pitch;
+- bo->tiling = tiling;
+ bo->delta = 0;
+ DBG((" 1:from active: pitch=%d, tiling=%d, handle=%d, id=%d\n",
+ bo->pitch, bo->tiling, bo->handle, bo->unique_id));
+@@ -5323,11 +5864,13 @@ search_inactive:
+ continue;
+ }
+
+- if (bo->tiling != tiling ||
+- (tiling != I915_TILING_NONE && bo->pitch != pitch)) {
+- if (!gem_set_tiling(kgem->fd, bo->handle,
+- tiling, pitch))
+- continue;
++ if (!kgem_set_tiling(kgem, bo, tiling, pitch)) {
++ if (kgem->gen >= 040 && !exact) {
++ set_gpu_tiling(kgem, bo, tiling, pitch);
++ } else {
++ kgem_bo_free(kgem, bo);
++ break;
++ }
+ }
+
+ if (bo->purged && !kgem_bo_clear_purgeable(kgem, bo)) {
+@@ -5338,9 +5881,8 @@ search_inactive:
+ kgem_bo_remove_from_inactive(kgem, bo);
+ assert(list_is_empty(&bo->list));
+ assert(list_is_empty(&bo->vma));
+-
+- bo->pitch = pitch;
+- bo->tiling = tiling;
++ assert(bo->tiling == tiling);
++ assert(bo->pitch >= pitch);
+
+ bo->delta = 0;
+ bo->unique_id = kgem_get_unique_id(kgem);
+@@ -5388,14 +5930,17 @@ search_inactive:
+ kgem_bo_remove_from_active(kgem, bo);
+ __kgem_bo_clear_busy(bo);
+
+- if (tiling != I915_TILING_NONE && bo->pitch != pitch) {
+- if (!gem_set_tiling(kgem->fd, bo->handle, tiling, pitch)) {
++ if (!kgem_set_tiling(kgem, bo, tiling, pitch)) {
++ if (kgem->gen >= 040 && !exact) {
++ set_gpu_tiling(kgem, bo, tiling, pitch);
++ } else {
+ kgem_bo_free(kgem, bo);
+ goto no_retire;
+ }
+ }
++ assert(bo->tiling == tiling);
++ assert(bo->pitch >= pitch);
+
+- bo->pitch = pitch;
+ bo->unique_id = kgem_get_unique_id(kgem);
+ bo->delta = 0;
+ DBG((" 2:from active: pitch=%d, tiling=%d, handle=%d, id=%d\n",
+@@ -5440,18 +5985,21 @@ create:
+ }
+
+ bo->unique_id = kgem_get_unique_id(kgem);
+- if (tiling == I915_TILING_NONE ||
+- gem_set_tiling(kgem->fd, handle, tiling, pitch)) {
+- bo->tiling = tiling;
+- bo->pitch = pitch;
++ if (kgem_set_tiling(kgem, bo, tiling, pitch)) {
+ if (flags & CREATE_SCANOUT)
+ __kgem_bo_make_scanout(kgem, bo, width, height);
+ } else {
+- if (flags & CREATE_EXACT) {
+- DBG(("%s: failed to set exact tiling (gem_set_tiling)\n", __FUNCTION__));
+- gem_close(kgem->fd, handle);
+- free(bo);
+- return NULL;
++ if (kgem->gen >= 040) {
++ assert(!kgem->can_fence);
++ bo->tiling = tiling;
++ bo->pitch = pitch;
++ } else {
++ if (flags & CREATE_EXACT) {
++ DBG(("%s: failed to set exact tiling (gem_set_tiling)\n", __FUNCTION__));
++ gem_close(kgem->fd, handle);
++ free(bo);
++ return NULL;
++ }
+ }
+ }
+
+@@ -5608,7 +6156,7 @@ static void __kgem_flush(struct kgem *kgem, struct kgem_bo *bo)
+
+ void kgem_scanout_flush(struct kgem *kgem, struct kgem_bo *bo)
+ {
+- if (!bo->needs_flush)
++ if (!bo->needs_flush && !bo->gtt_dirty)
+ return;
+
+ kgem_bo_submit(kgem, bo);
+@@ -5621,18 +6169,24 @@ void kgem_scanout_flush(struct kgem *kgem, struct kgem_bo *bo)
+ if (bo->rq)
+ __kgem_flush(kgem, bo);
+
++ if (bo->scanout && kgem->needs_dirtyfb) {
++ struct drm_mode_fb_dirty_cmd cmd;
++ memset(&cmd, 0, sizeof(cmd));
++ cmd.fb_id = bo->delta;
++ (void)drmIoctl(kgem->fd, DRM_IOCTL_MODE_DIRTYFB, &cmd);
++ }
++
+ /* Whatever actually happens, we can regard the GTT write domain
+ * as being flushed.
+ */
+- bo->gtt_dirty = false;
+- bo->needs_flush = false;
+- bo->domain = DOMAIN_NONE;
++ __kgem_bo_clear_dirty(bo);
+ }
+
+ inline static bool nearly_idle(struct kgem *kgem)
+ {
+ int ring = kgem->ring == KGEM_BLT;
+
++ assert(ring < ARRAY_SIZE(kgem->requests));
+ if (list_is_singular(&kgem->requests[ring]))
+ return true;
+
+@@ -5720,7 +6274,7 @@ static inline bool kgem_flush(struct kgem *kgem, bool flush)
+ if (kgem->nreloc == 0)
+ return true;
+
+- if (container_of(kgem, struct sna, kgem)->flags & SNA_POWERSAVE)
++ if (__to_sna(kgem)->flags & SNA_POWERSAVE)
+ return true;
+
+ if (kgem->flush == flush && kgem->aperture < kgem->aperture_low)
+@@ -5982,6 +6536,55 @@ bool kgem_check_many_bo_fenced(struct kgem *kgem, ...)
+ return kgem_flush(kgem, flush);
+ }
+
++void __kgem_bcs_set_tiling(struct kgem *kgem,
++ struct kgem_bo *src,
++ struct kgem_bo *dst)
++{
++ uint32_t state, *b;
++
++ DBG(("%s: src handle=%d:tiling=%d, dst handle=%d:tiling=%d\n",
++ __FUNCTION__,
++ src ? src->handle : 0, src ? src->tiling : 0,
++ dst ? dst->handle : 0, dst ? dst->tiling : 0));
++ assert(kgem->mode == KGEM_BLT);
++ assert(dst == NULL || kgem_bo_can_blt(kgem, dst));
++ assert(src == NULL || kgem_bo_can_blt(kgem, src));
++
++ state = 0;
++ if (dst && dst->tiling == I915_TILING_Y)
++ state |= BCS_DST_Y;
++ if (src && src->tiling == I915_TILING_Y)
++ state |= BCS_SRC_Y;
++
++ if (kgem->bcs_state == state)
++ return;
++
++ DBG(("%s: updating SWCTRL %x -> %x\n", __FUNCTION__,
++ kgem->bcs_state, state));
++
++ /* Over-estimate space in case we need to re-emit the cmd packet */
++ if (!kgem_check_batch(kgem, 24)) {
++ _kgem_submit(kgem);
++ _kgem_set_mode(kgem, KGEM_BLT);
++ if (state == 0)
++ return;
++ }
++
++ b = kgem->batch + kgem->nbatch;
++ if (kgem->nbatch) {
++ *b++ = MI_FLUSH_DW;
++ *b++ = 0;
++ *b++ = 0;
++ *b++ = 0;
++ }
++ *b++ = MI_LOAD_REGISTER_IMM;
++ *b++ = BCS_SWCTRL;
++ *b++ = (BCS_SRC_Y | BCS_DST_Y) << 16 | state;
++ kgem->nbatch = b - kgem->batch;
++
++ kgem->bcs_state = state;
++}
++
+ uint32_t kgem_add_reloc(struct kgem *kgem,
+ uint32_t pos,
+ struct kgem_bo *bo,
+@@ -6195,12 +6798,6 @@ static void kgem_trim_vma_cache(struct kgem *kgem, int type, int bucket)
+
+ list_del(&bo->vma);
+ kgem->vma[type].count--;
+-
+- if (!bo->purged && !kgem_bo_set_purgeable(kgem, bo)) {
+- DBG(("%s: freeing unpurgeable old mapping\n",
+- __FUNCTION__));
+- kgem_bo_free(kgem, bo);
+- }
+ }
+ }
+
+@@ -6216,8 +6813,8 @@ static void *__kgem_bo_map__gtt_or_wc(struct kgem *kgem, struct kgem_bo *bo)
+ kgem_trim_vma_cache(kgem, MAP_GTT, bucket(bo));
+
+ if (bo->tiling || !kgem->has_wc_mmap) {
+- assert(num_pages(bo) <= kgem->aperture_mappable / 2);
+ assert(kgem->gen != 021 || bo->tiling != I915_TILING_Y);
++ warn_unless(num_pages(bo) <= kgem->aperture_mappable / 2);
+
+ ptr = bo->map__gtt;
+ if (ptr == NULL)
+@@ -6291,6 +6888,7 @@ void *kgem_bo_map(struct kgem *kgem, struct kgem_bo *bo)
+ DBG(("%s: sync: GPU hang detected\n", __FUNCTION__));
+ kgem_throttle(kgem);
+ }
++ bo->needs_flush = false;
+ kgem_bo_retire(kgem, bo);
+ bo->domain = DOMAIN_GTT;
+ bo->gtt_dirty = true;
+@@ -6319,14 +6917,16 @@ void *kgem_bo_map__wc(struct kgem *kgem, struct kgem_bo *bo)
+ bo->handle, (long)bo->presumed_offset, bo->tiling, bo->map__gtt, bo->map__cpu, bo->domain));
+
+ assert(bo->proxy == NULL);
+- assert(bo->exec == NULL);
+ assert(list_is_empty(&bo->list));
+ assert_tiling(kgem, bo);
+ assert(!bo->purged || bo->reusable);
+
+ if (bo->map__wc)
+ return bo->map__wc;
++ if (!kgem->has_wc_mmap)
++ return NULL;
+
++ kgem_trim_vma_cache(kgem, MAP_GTT, bucket(bo));
+ return __kgem_bo_map__wc(kgem, bo);
+ }
+
+@@ -6373,6 +6973,8 @@ uint32_t kgem_bo_flink(struct kgem *kgem, struct kgem_bo *bo)
+ {
+ struct drm_gem_flink flink;
+
++ assert(kgem_bo_is_fenced(kgem, bo));
++
+ VG_CLEAR(flink);
+ flink.handle = bo->handle;
+ if (do_ioctl(kgem->fd, DRM_IOCTL_GEM_FLINK, &flink))
+@@ -6387,7 +6989,6 @@ uint32_t kgem_bo_flink(struct kgem *kgem, struct kgem_bo *bo)
+ * party, we track the lifetime accurately.
+ */
+ bo->reusable = false;
+-
+ kgem_bo_unclean(kgem, bo);
+
+ return flink.name;
+@@ -6411,16 +7012,34 @@ struct kgem_bo *kgem_create_map(struct kgem *kgem,
+ first_page = (uintptr_t)ptr;
+ last_page = first_page + size + PAGE_SIZE - 1;
+
+- first_page &= ~(PAGE_SIZE-1);
+- last_page &= ~(PAGE_SIZE-1);
++ first_page &= ~(uintptr_t)(PAGE_SIZE-1);
++ last_page &= ~(uintptr_t)(PAGE_SIZE-1);
+ assert(last_page > first_page);
+
+ handle = gem_userptr(kgem->fd,
+ (void *)first_page, last_page-first_page,
+ read_only);
+ if (handle == 0) {
+- DBG(("%s: import failed, errno=%d\n", __FUNCTION__, errno));
+- return NULL;
++ if (read_only && kgem->has_wc_mmap) {
++ struct drm_i915_gem_set_domain set_domain;
++
++ handle = gem_userptr(kgem->fd,
++ (void *)first_page, last_page-first_page,
++ false);
++
++ VG_CLEAR(set_domain);
++ set_domain.handle = handle;
++ set_domain.read_domains = I915_GEM_DOMAIN_GTT;
++ set_domain.write_domain = 0;
++ if (do_ioctl(kgem->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &set_domain)) {
++ gem_close(kgem->fd, handle);
++ handle = 0;
++ }
++ }
++ if (handle == 0) {
++ DBG(("%s: import failed, errno=%d\n", __FUNCTION__, errno));
++ return NULL;
++ }
+ }
+
+ bo = __kgem_bo_alloc(handle, (last_page - first_page) / PAGE_SIZE);
+@@ -6483,8 +7102,10 @@ void kgem_bo_sync__cpu(struct kgem *kgem, struct kgem_bo *bo)
+ DBG(("%s: sync: GPU hang detected\n", __FUNCTION__));
+ kgem_throttle(kgem);
+ }
++ bo->needs_flush = false;
+ kgem_bo_retire(kgem, bo);
+ bo->domain = DOMAIN_CPU;
++ bo->gtt_dirty = true;
+ }
+ }
+
+@@ -6505,6 +7126,9 @@ void kgem_bo_sync__cpu_full(struct kgem *kgem, struct kgem_bo *bo, bool write)
+ assert(bo->refcnt);
+ assert(!bo->purged);
+
++ if (bo->rq == NULL && (kgem->has_llc || bo->snoop) && !write)
++ return;
++
+ if (bo->domain != DOMAIN_CPU || FORCE_MMAP_SYNC & (1 << DOMAIN_CPU)) {
+ struct drm_i915_gem_set_domain set_domain;
+
+@@ -6522,9 +7146,11 @@ void kgem_bo_sync__cpu_full(struct kgem *kgem, struct kgem_bo *bo, bool write)
+ DBG(("%s: sync: GPU hang detected\n", __FUNCTION__));
+ kgem_throttle(kgem);
+ }
++ bo->needs_flush = false;
+ if (write) {
+ kgem_bo_retire(kgem, bo);
+ bo->domain = DOMAIN_CPU;
++ bo->gtt_dirty = true;
+ } else {
+ if (bo->exec == NULL)
+ kgem_bo_maybe_retire(kgem, bo);
+@@ -6539,6 +7165,7 @@ void kgem_bo_sync__gtt(struct kgem *kgem, struct kgem_bo *bo)
+ assert(bo->refcnt);
+ assert(bo->proxy == NULL);
+ assert_tiling(kgem, bo);
++ assert(!bo->snoop);
+
+ kgem_bo_submit(kgem, bo);
+
+@@ -6559,6 +7186,7 @@ void kgem_bo_sync__gtt(struct kgem *kgem, struct kgem_bo *bo)
+ DBG(("%s: sync: GPU hang detected\n", __FUNCTION__));
+ kgem_throttle(kgem);
+ }
++ bo->needs_flush = false;
+ kgem_bo_retire(kgem, bo);
+ bo->domain = DOMAIN_GTT;
+ bo->gtt_dirty = true;
+@@ -7485,6 +8113,7 @@ kgem_replace_bo(struct kgem *kgem,
+ }
+ _kgem_set_mode(kgem, KGEM_BLT);
+ }
++ kgem_bcs_set_tiling(kgem, src, dst);
+
+ br00 = XY_SRC_COPY_BLT_CMD;
+ br13 = pitch;
+@@ -7553,6 +8182,9 @@ bool kgem_bo_convert_to_gpu(struct kgem *kgem,
+ __FUNCTION__, bo->handle, flags, __kgem_bo_is_busy(kgem, bo)));
+ assert(bo->tiling == I915_TILING_NONE);
+
++ if (flags & (__MOVE_PRIME | __MOVE_SCANOUT))
++ return false;
++
+ if (kgem->has_llc)
+ return true;
+
+diff --git a/src/sna/kgem.h b/src/sna/kgem.h
+index 2267bacf..08b4eb20 100644
+--- a/src/sna/kgem.h
++++ b/src/sna/kgem.h
+@@ -42,6 +42,7 @@ struct kgem_bo {
+ #define RQ(rq) ((struct kgem_request *)((uintptr_t)(rq) & ~3))
+ #define RQ_RING(rq) ((uintptr_t)(rq) & 3)
+ #define RQ_IS_BLT(rq) (RQ_RING(rq) == KGEM_BLT)
++#define RQ_IS_RENDER(rq) (RQ_RING(rq) == KGEM_RENDER)
+ #define MAKE_REQUEST(rq, ring) ((struct kgem_request *)((uintptr_t)(rq) | (ring)))
+
+ struct drm_i915_gem_exec_object2 *exec;
+@@ -103,7 +104,7 @@ struct kgem_request {
+ struct list list;
+ struct kgem_bo *bo;
+ struct list buffers;
+- int ring;
++ unsigned ring;
+ };
+
+ enum {
+@@ -112,6 +113,12 @@ enum {
+ NUM_MAP_TYPES,
+ };
+
++typedef void (*memcpy_box_func)(const void *src, void *dst, int bpp,
++ int32_t src_stride, int32_t dst_stride,
++ int16_t src_x, int16_t src_y,
++ int16_t dst_x, int16_t dst_y,
++ uint16_t width, uint16_t height);
++
+ struct kgem {
+ unsigned wedged;
+ int fd;
+@@ -157,6 +164,8 @@ struct kgem {
+ int16_t count;
+ } vma[NUM_MAP_TYPES];
+
++ uint32_t bcs_state;
++
+ uint32_t batch_flags;
+ uint32_t batch_flags_base;
+ #define I915_EXEC_SECURE (1<<9)
+@@ -186,9 +195,15 @@ struct kgem {
+ uint32_t has_no_reloc :1;
+ uint32_t has_handle_lut :1;
+ uint32_t has_wc_mmap :1;
++ uint32_t has_dirtyfb :1;
+
++ uint32_t can_fence :1;
+ uint32_t can_blt_cpu :1;
++ uint32_t can_blt_y :1;
+ uint32_t can_render_y :1;
++ uint32_t can_scanout_y :1;
++
++ uint32_t needs_dirtyfb :1;
+
+ uint16_t fence_max;
+ uint16_t half_cpu_cache_pages;
+@@ -203,16 +218,9 @@ struct kgem {
+ void (*retire)(struct kgem *kgem);
+ void (*expire)(struct kgem *kgem);
+
+- void (*memcpy_to_tiled_x)(const void *src, void *dst, int bpp,
+- int32_t src_stride, int32_t dst_stride,
+- int16_t src_x, int16_t src_y,
+- int16_t dst_x, int16_t dst_y,
+- uint16_t width, uint16_t height);
+- void (*memcpy_from_tiled_x)(const void *src, void *dst, int bpp,
+- int32_t src_stride, int32_t dst_stride,
+- int16_t src_x, int16_t src_y,
+- int16_t dst_x, int16_t dst_y,
+- uint16_t width, uint16_t height);
++ memcpy_box_func memcpy_to_tiled_x;
++ memcpy_box_func memcpy_from_tiled_x;
++ memcpy_box_func memcpy_between_tiled_x;
+
+ struct kgem_bo *batch_bo;
+
+@@ -230,7 +238,7 @@ struct kgem {
+
+ #define KGEM_MAX_DEFERRED_VBO 16
+
+-#define KGEM_BATCH_RESERVED 1
++#define KGEM_BATCH_RESERVED 8 /* LRI(SWCTRL) + END */
+ #define KGEM_RELOC_RESERVED (KGEM_MAX_DEFERRED_VBO)
+ #define KGEM_EXEC_RESERVED (1+KGEM_MAX_DEFERRED_VBO)
+
+@@ -317,6 +325,7 @@ bool kgem_bo_convert_to_gpu(struct kgem *kgem,
+ struct kgem_bo *bo,
+ unsigned flags);
+
++bool kgem_bo_is_fenced(struct kgem *kgem, struct kgem_bo *bo);
+ uint32_t kgem_bo_get_binding(struct kgem_bo *bo, uint32_t format);
+ void kgem_bo_set_binding(struct kgem_bo *bo, uint32_t format, uint16_t offset);
+
+@@ -342,6 +351,11 @@ static inline bool kgem_ring_is_idle(struct kgem *kgem, int ring)
+ {
+ ring = ring == KGEM_BLT;
+
++ if (kgem->needs_semaphore &&
++ !list_is_empty(&kgem->requests[!ring]) &&
++ !__kgem_ring_is_idle(kgem, !ring))
++ return false;
++
+ if (list_is_empty(&kgem->requests[ring]))
+ return true;
+
+@@ -390,6 +404,7 @@ void _kgem_bo_destroy(struct kgem *kgem, struct kgem_bo *bo);
+ static inline void kgem_bo_destroy(struct kgem *kgem, struct kgem_bo *bo)
+ {
+ assert(bo->refcnt);
++ assert(bo->refcnt > bo->active_scanout);
+ if (--bo->refcnt == 0)
+ _kgem_bo_destroy(kgem, bo);
+ }
+@@ -400,13 +415,13 @@ static inline void kgem_set_mode(struct kgem *kgem,
+ enum kgem_mode mode,
+ struct kgem_bo *bo)
+ {
+- assert(!kgem->wedged);
++ warn_unless(!kgem->wedged);
+
+ #if DEBUG_FLUSH_BATCH
+ kgem_submit(kgem);
+ #endif
+
+- if (kgem->nreloc && bo->exec == NULL && kgem_ring_is_idle(kgem, kgem->ring)) {
++ if (kgem->nreloc && bo->rq == NULL && kgem_ring_is_idle(kgem, kgem->ring)) {
+ DBG(("%s: flushing before new bo\n", __FUNCTION__));
+ _kgem_submit(kgem);
+ }
+@@ -422,7 +437,7 @@ static inline void _kgem_set_mode(struct kgem *kgem, enum kgem_mode mode)
+ {
+ assert(kgem->mode == KGEM_NONE);
+ assert(kgem->nbatch == 0);
+- assert(!kgem->wedged);
++ warn_unless(!kgem->wedged);
+ kgem->context_switch(kgem, mode);
+ kgem->mode = mode;
+ }
+@@ -566,7 +581,7 @@ static inline bool kgem_bo_can_blt(struct kgem *kgem,
+ {
+ assert(bo->refcnt);
+
+- if (bo->tiling == I915_TILING_Y) {
++ if (bo->tiling == I915_TILING_Y && !kgem->can_blt_y) {
+ DBG(("%s: can not blt to handle=%d, tiling=Y\n",
+ __FUNCTION__, bo->handle));
+ return false;
+@@ -581,6 +596,22 @@ static inline bool kgem_bo_can_blt(struct kgem *kgem,
+ return kgem_bo_blt_pitch_is_ok(kgem, bo);
+ }
+
++void __kgem_bcs_set_tiling(struct kgem *kgem,
++ struct kgem_bo *src,
++ struct kgem_bo *dst);
++
++inline static void kgem_bcs_set_tiling(struct kgem *kgem,
++ struct kgem_bo *src,
++ struct kgem_bo *dst)
++{
++ assert(kgem->mode == KGEM_BLT);
++
++ if (!kgem->can_blt_y)
++ return;
++
++ __kgem_bcs_set_tiling(kgem, src, dst);
++}
++
+ static inline bool kgem_bo_is_snoop(struct kgem_bo *bo)
+ {
+ assert(bo->refcnt);
+@@ -607,17 +638,24 @@ static inline void kgem_bo_mark_busy(struct kgem *kgem, struct kgem_bo *bo, int
+ }
+ }
+
+-inline static void __kgem_bo_clear_busy(struct kgem_bo *bo)
++static inline void __kgem_bo_clear_dirty(struct kgem_bo *bo)
+ {
+ DBG(("%s: handle=%d\n", __FUNCTION__, bo->handle));
+- bo->rq = NULL;
+- list_del(&bo->request);
+
+ bo->domain = DOMAIN_NONE;
+ bo->needs_flush = false;
+ bo->gtt_dirty = false;
+ }
+
++inline static void __kgem_bo_clear_busy(struct kgem_bo *bo)
++{
++ DBG(("%s: handle=%d\n", __FUNCTION__, bo->handle));
++ bo->rq = NULL;
++ list_del(&bo->request);
++
++ __kgem_bo_clear_dirty(bo);
++}
++
+ static inline bool kgem_bo_is_busy(struct kgem_bo *bo)
+ {
+ DBG(("%s: handle=%d, domain: %d exec? %d, rq? %d\n", __FUNCTION__,
+@@ -626,7 +664,7 @@ static inline bool kgem_bo_is_busy(struct kgem_bo *bo)
+ return bo->rq;
+ }
+
+-void __kgem_retire_requests_upto(struct kgem *kgem, struct kgem_bo *bo);
++bool __kgem_retire_requests_upto(struct kgem *kgem, struct kgem_bo *bo);
+ static inline bool __kgem_bo_is_busy(struct kgem *kgem, struct kgem_bo *bo)
+ {
+ DBG(("%s: handle=%d, domain: %d exec? %d, rq? %d\n", __FUNCTION__,
+@@ -636,14 +674,13 @@ static inline bool __kgem_bo_is_busy(struct kgem *kgem, struct kgem_bo *bo)
+ if (bo->exec)
+ return true;
+
+- if (bo->rq && !__kgem_busy(kgem, bo->handle)) {
+- __kgem_retire_requests_upto(kgem, bo);
+- assert(list_is_empty(&bo->request));
+- assert(bo->rq == NULL);
+- assert(bo->domain == DOMAIN_NONE);
+- }
++ if (bo->rq == NULL)
++ return false;
++
++ if (__kgem_busy(kgem, bo->handle))
++ return true;
+
+- return kgem_bo_is_busy(bo);
++ return __kgem_retire_requests_upto(kgem, bo);
+ }
+
+ static inline bool kgem_bo_is_render(struct kgem_bo *bo)
+@@ -651,7 +688,15 @@ static inline bool kgem_bo_is_render(struct kgem_bo *bo)
+ DBG(("%s: handle=%d, rq? %d [%d]\n", __FUNCTION__,
+ bo->handle, bo->rq != NULL, (int)RQ_RING(bo->rq)));
+ assert(bo->refcnt);
+- return bo->rq && RQ_RING(bo->rq) == I915_EXEC_RENDER;
++ return bo->rq && RQ_RING(bo->rq) != KGEM_BLT;
++}
++
++static inline bool kgem_bo_is_blt(struct kgem_bo *bo)
++{
++ DBG(("%s: handle=%d, rq? %d\n", __FUNCTION__,
++ bo->handle, bo->rq != NULL, (int)RQ_RING(bo->rq)));
++ assert(bo->refcnt);
++ return RQ_RING(bo->rq) == KGEM_BLT;
+ }
+
+ static inline void kgem_bo_mark_unreusable(struct kgem_bo *bo)
+@@ -852,6 +897,6 @@ memcpy_from_tiled_x(struct kgem *kgem,
+ width, height);
+ }
+
+-void choose_memcpy_tiled_x(struct kgem *kgem, int swizzling);
++void choose_memcpy_tiled_x(struct kgem *kgem, int swizzling, unsigned cpu);
+
+ #endif /* KGEM_H */
+diff --git a/src/sna/kgem_debug_gen4.c b/src/sna/kgem_debug_gen4.c
+index 9b80dc88..8e6e47b6 100644
+--- a/src/sna/kgem_debug_gen4.c
++++ b/src/sna/kgem_debug_gen4.c
+@@ -598,7 +598,7 @@ int kgem_gen4_decode_3d(struct kgem *kgem, uint32_t offset)
+ assert(len == 7);
+ kgem_debug_print(data, offset, 0,
+ "3DSTATE_DEPTH_BUFFER\n");
+- kgem_debug_print(data, offset, 1, "%s, %s, pitch = %d bytes, %stiled, HiZ %d, Seperate Stencil %d\n",
++ kgem_debug_print(data, offset, 1, "%s, %s, pitch = %d bytes, %stiled, HiZ %d, Separate Stencil %d\n",
+ get_965_surfacetype(data[1] >> 29),
+ get_965_depthformat((data[1] >> 18) & 0x7),
+ (data[1] & 0x0001ffff) + 1,
+diff --git a/src/sna/kgem_debug_gen5.c b/src/sna/kgem_debug_gen5.c
+index 8b55dd91..f1b1275f 100644
+--- a/src/sna/kgem_debug_gen5.c
++++ b/src/sna/kgem_debug_gen5.c
+@@ -573,7 +573,7 @@ int kgem_gen5_decode_3d(struct kgem *kgem, uint32_t offset)
+ assert(len == 7);
+ kgem_debug_print(data, offset, 0,
+ "3DSTATE_DEPTH_BUFFER\n");
+- kgem_debug_print(data, offset, 1, "%s, %s, pitch = %d bytes, %stiled, HiZ %d, Seperate Stencil %d\n",
++ kgem_debug_print(data, offset, 1, "%s, %s, pitch = %d bytes, %stiled, HiZ %d, Separate Stencil %d\n",
+ get_965_surfacetype(data[1] >> 29),
+ get_965_depthformat((data[1] >> 18) & 0x7),
+ (data[1] & 0x0001ffff) + 1,
+diff --git a/src/sna/kgem_debug_gen6.c b/src/sna/kgem_debug_gen6.c
+index 7ef55d38..579c5d54 100644
+--- a/src/sna/kgem_debug_gen6.c
++++ b/src/sna/kgem_debug_gen6.c
+@@ -985,7 +985,7 @@ int kgem_gen6_decode_3d(struct kgem *kgem, uint32_t offset)
+ assert(len == 7);
+ kgem_debug_print(data, offset, 0,
+ "3DSTATE_DEPTH_BUFFER\n");
+- kgem_debug_print(data, offset, 1, "%s, %s, pitch = %d bytes, %stiled, HiZ %d, Seperate Stencil %d\n",
++ kgem_debug_print(data, offset, 1, "%s, %s, pitch = %d bytes, %stiled, HiZ %d, Separate Stencil %d\n",
+ get_965_surfacetype(data[1] >> 29),
+ get_965_depthformat((data[1] >> 18) & 0x7),
+ (data[1] & 0x0001ffff) + 1,
+diff --git a/src/sna/sna.h b/src/sna/sna.h
+index 18425e30..7861110a 100644
+--- a/src/sna/sna.h
++++ b/src/sna/sna.h
+@@ -154,6 +154,8 @@ struct sna_pixmap {
+ #define MAPPED_GTT 1
+ #define MAPPED_CPU 2
+ uint8_t flush :2;
++#define FLUSH_READ 1
++#define FLUSH_WRITE 2
+ uint8_t shm :1;
+ uint8_t clear :1;
+ uint8_t header :1;
+@@ -179,18 +181,31 @@ static inline WindowPtr get_root_window(ScreenPtr screen)
+ #endif
+ }
+
++#if !NDEBUG
++static PixmapPtr check_pixmap(PixmapPtr pixmap)
++{
++ if (pixmap != NULL) {
++ assert(pixmap->refcnt >= 1);
++ assert(pixmap->devKind != 0xdeadbeef);
++ }
++ return pixmap;
++}
++#else
++#define check_pixmap(p) p
++#endif
++
+ static inline PixmapPtr get_window_pixmap(WindowPtr window)
+ {
+ assert(window);
+ assert(window->drawable.type != DRAWABLE_PIXMAP);
+- return fbGetWindowPixmap(window);
++ return check_pixmap(fbGetWindowPixmap(window));
+ }
+
+ static inline PixmapPtr get_drawable_pixmap(DrawablePtr drawable)
+ {
+ assert(drawable);
+ if (drawable->type == DRAWABLE_PIXMAP)
+- return (PixmapPtr)drawable;
++ return check_pixmap((PixmapPtr)drawable);
+ else
+ return get_window_pixmap((WindowPtr)drawable);
+ }
+@@ -244,11 +259,12 @@ struct sna {
+ #define SNA_NO_VSYNC 0x40
+ #define SNA_TRIPLE_BUFFER 0x80
+ #define SNA_TEAR_FREE 0x100
+-#define SNA_FORCE_SHADOW 0x200
+-#define SNA_FLUSH_GTT 0x400
++#define SNA_WANT_TEAR_FREE 0x200
++#define SNA_FORCE_SHADOW 0x400
++#define SNA_FLUSH_GTT 0x800
+ #define SNA_PERFORMANCE 0x1000
+ #define SNA_POWERSAVE 0x2000
+-#define SNA_REMOVE_OUTPUTS 0x4000
++#define SNA_NO_DPMS 0x4000
+ #define SNA_HAS_FLIP 0x10000
+ #define SNA_HAS_ASYNC_FLIP 0x20000
+ #define SNA_LINEAR_FB 0x40000
+@@ -265,7 +281,13 @@ struct sna {
+ #define AVX 0x80
+ #define AVX2 0x100
+
+- unsigned watch_flush;
++ bool ignore_copy_area : 1;
++
++ unsigned watch_shm_flush;
++ unsigned watch_dri_flush;
++ unsigned damage_event;
++ bool needs_shm_flush;
++ bool needs_dri_flush;
+
+ struct timeval timer_tv;
+ uint32_t timer_expire[NUM_TIMERS];
+@@ -284,9 +306,17 @@ struct sna {
+ struct kgem_bo *shadow;
+ unsigned front_active;
+ unsigned shadow_active;
++ unsigned rr_active;
+ unsigned flip_active;
++ unsigned hidden;
++ bool shadow_enabled;
++ bool shadow_wait;
+ bool dirty;
+
++ struct drm_event_vblank *shadow_events;
++ int shadow_nevent;
++ int shadow_size;
++
+ int max_crtc_width, max_crtc_height;
+ RegionRec shadow_region;
+ RegionRec shadow_cancel;
+@@ -318,7 +348,8 @@ struct sna {
+ uint32_t fg, bg;
+ int size;
+
+- int active;
++ bool disable;
++ bool active;
+ int last_x;
+ int last_y;
+
+@@ -331,8 +362,9 @@ struct sna {
+ } cursor;
+
+ struct sna_dri2 {
+- bool available;
+- bool open;
++ bool available : 1;
++ bool enable : 1;
++ bool open : 1;
+
+ #if HAVE_DRI2
+ void *flip_pending;
+@@ -341,8 +373,11 @@ struct sna {
+ } dri2;
+
+ struct sna_dri3 {
+- bool available;
+- bool open;
++ bool available :1;
++ bool override : 1;
++ bool enable : 1;
++ bool open :1;
++
+ #if HAVE_DRI3
+ SyncScreenCreateFenceFunc create_fence;
+ struct list pixmaps;
+@@ -353,6 +388,9 @@ struct sna {
+ bool available;
+ bool open;
+ #if HAVE_PRESENT
++ struct list vblank_queue;
++ uint64_t unflip;
++ void *freed_info;
+ #endif
+ } present;
+
+@@ -364,8 +402,10 @@ struct sna {
+ EntityInfoPtr pEnt;
+ const struct intel_device_info *info;
+
++#if !HAVE_NOTIFY_FD
+ ScreenBlockHandlerProcPtr BlockHandler;
+ ScreenWakeupHandlerProcPtr WakeupHandler;
++#endif
+ CloseScreenProcPtr CloseScreen;
+
+ PicturePtr clear;
+@@ -383,6 +423,7 @@ struct sna {
+ struct gen6_render_state gen6;
+ struct gen7_render_state gen7;
+ struct gen8_render_state gen8;
++ struct gen9_render_state gen9;
+ } render_state;
+
+ /* Broken-out options. */
+@@ -420,7 +461,7 @@ bool sna_mode_pre_init(ScrnInfoPtr scrn, struct sna *sna);
+ bool sna_mode_fake_init(struct sna *sna, int num_fake);
+ bool sna_mode_wants_tear_free(struct sna *sna);
+ void sna_mode_adjust_frame(struct sna *sna, int x, int y);
+-extern void sna_mode_discover(struct sna *sna);
++extern void sna_mode_discover(struct sna *sna, bool tell);
+ extern void sna_mode_check(struct sna *sna);
+ extern bool sna_mode_disable(struct sna *sna);
+ extern void sna_mode_enable(struct sna *sna);
+@@ -434,6 +475,7 @@ extern void sna_shadow_unset_crtc(struct sna *sna, xf86CrtcPtr crtc);
+ extern bool sna_pixmap_discard_shadow_damage(struct sna_pixmap *priv,
+ const RegionRec *region);
+ extern void sna_mode_set_primary(struct sna *sna);
++extern bool sna_mode_find_hotplug_connector(struct sna *sna, unsigned id);
+ extern void sna_mode_close(struct sna *sna);
+ extern void sna_mode_fini(struct sna *sna);
+
+@@ -444,6 +486,7 @@ extern bool sna_cursors_init(ScreenPtr screen, struct sna *sna);
+ typedef void (*sna_flip_handler_t)(struct drm_event_vblank *e,
+ void *data);
+
++extern bool sna_needs_page_flip(struct sna *sna, struct kgem_bo *bo);
+ extern int sna_page_flip(struct sna *sna,
+ struct kgem_bo *bo,
+ sna_flip_handler_t handler,
+@@ -461,6 +504,11 @@ to_sna_from_screen(ScreenPtr screen)
+ return to_sna(xf86ScreenToScrn(screen));
+ }
+
++pure static inline ScreenPtr to_screen_from_sna(struct sna *sna)
++{
++ return xf86ScrnToScreen(sna->scrn);
++}
++
+ pure static inline struct sna *
+ to_sna_from_pixmap(PixmapPtr pixmap)
+ {
+@@ -498,12 +546,11 @@ to_sna_from_kgem(struct kgem *kgem)
+ extern xf86CrtcPtr sna_covering_crtc(struct sna *sna,
+ const BoxRec *box,
+ xf86CrtcPtr desired);
++extern xf86CrtcPtr sna_primary_crtc(struct sna *sna);
+
+ extern bool sna_wait_for_scanline(struct sna *sna, PixmapPtr pixmap,
+ xf86CrtcPtr crtc, const BoxRec *clip);
+
+-xf86CrtcPtr sna_mode_first_crtc(struct sna *sna);
+-
+ const struct ust_msc {
+ uint64_t msc;
+ int tv_sec;
+@@ -536,6 +583,11 @@ static inline uint64_t ust64(int tv_sec, int tv_usec)
+ return (uint64_t)tv_sec * 1000000 + tv_usec;
+ }
+
++static inline uint64_t swap_ust(const struct ust_msc *swap)
++{
++ return ust64(swap->tv_sec, swap->tv_usec);
++}
++
+ #if HAVE_DRI2
+ bool sna_dri2_open(struct sna *sna, ScreenPtr pScreen);
+ void sna_dri2_page_flip_handler(struct sna *sna, struct drm_event_vblank *event);
+@@ -567,20 +619,59 @@ bool sna_present_open(struct sna *sna, ScreenPtr pScreen);
+ void sna_present_update(struct sna *sna);
+ void sna_present_close(struct sna *sna, ScreenPtr pScreen);
+ void sna_present_vblank_handler(struct drm_event_vblank *event);
++void sna_present_cancel_flip(struct sna *sna);
+ #else
+ static inline bool sna_present_open(struct sna *sna, ScreenPtr pScreen) { return false; }
+ static inline void sna_present_update(struct sna *sna) { }
+ static inline void sna_present_close(struct sna *sna, ScreenPtr pScreen) { }
+ static inline void sna_present_vblank_handler(struct drm_event_vblank *event) { }
++static inline void sna_present_cancel_flip(struct sna *sna) { }
+ #endif
+
+-extern bool sna_crtc_set_sprite_rotation(xf86CrtcPtr crtc, uint32_t rotation);
+-extern int sna_crtc_to_pipe(xf86CrtcPtr crtc);
+-extern uint32_t sna_crtc_to_sprite(xf86CrtcPtr crtc);
+-extern uint32_t sna_crtc_id(xf86CrtcPtr crtc);
+-extern bool sna_crtc_is_on(xf86CrtcPtr crtc);
++extern unsigned sna_crtc_count_sprites(xf86CrtcPtr crtc);
++extern bool sna_crtc_set_sprite_rotation(xf86CrtcPtr crtc, unsigned idx, uint32_t rotation);
++extern uint32_t sna_crtc_to_sprite(xf86CrtcPtr crtc, unsigned idx);
+ extern bool sna_crtc_is_transformed(xf86CrtcPtr crtc);
+
++#define CRTC_VBLANK 0x3
++#define CRTC_ON 0x80000000
++
++uint32_t sna_crtc_id(xf86CrtcPtr crtc);
++
++static inline unsigned long *sna_crtc_flags(xf86CrtcPtr crtc)
++{
++ unsigned long *flags = crtc->driver_private;
++ assert(flags);
++ return flags;
++}
++
++static inline unsigned sna_crtc_pipe(xf86CrtcPtr crtc)
++{
++ return *sna_crtc_flags(crtc) >> 8 & 0xff;
++}
++
++static inline bool sna_crtc_is_on(xf86CrtcPtr crtc)
++{
++ return *sna_crtc_flags(crtc) & CRTC_ON;
++}
++
++static inline void sna_crtc_set_vblank(xf86CrtcPtr crtc)
++{
++ assert((*sna_crtc_flags(crtc) & CRTC_VBLANK) < 3);
++ ++*sna_crtc_flags(crtc);
++}
++
++static inline void sna_crtc_clear_vblank(xf86CrtcPtr crtc)
++{
++ assert(*sna_crtc_flags(crtc) & CRTC_VBLANK);
++ --*sna_crtc_flags(crtc);
++}
++
++static inline bool sna_crtc_has_vblank(xf86CrtcPtr crtc)
++{
++ return *sna_crtc_flags(crtc) & CRTC_VBLANK;
++}
++
+ CARD32 sna_format_for_depth(int depth);
+ CARD32 sna_render_format_for_depth(int depth);
+
+@@ -998,15 +1089,14 @@ static inline uint32_t pixmap_size(PixmapPtr pixmap)
+
+ bool sna_accel_init(ScreenPtr sreen, struct sna *sna);
+ void sna_accel_create(struct sna *sna);
+-void sna_accel_block_handler(struct sna *sna, struct timeval **tv);
+-void sna_accel_wakeup_handler(struct sna *sna);
+-void sna_accel_watch_flush(struct sna *sna, int enable);
++void sna_accel_block(struct sna *sna, struct timeval **tv);
+ void sna_accel_flush(struct sna *sna);
+ void sna_accel_enter(struct sna *sna);
+ void sna_accel_leave(struct sna *sna);
+ void sna_accel_close(struct sna *sna);
+ void sna_accel_free(struct sna *sna);
+
++void sna_watch_flush(struct sna *sna, int enable);
+ void sna_copy_fbcon(struct sna *sna);
+
+ bool sna_composite_create(struct sna *sna);
+@@ -1127,6 +1217,16 @@ memcpy_blt(const void *src, void *dst, int bpp,
+ uint16_t width, uint16_t height);
+
+ void
++affine_blt(const void *src, void *dst, int bpp,
++ int16_t src_x, int16_t src_y,
++ int16_t src_width, int16_t src_height,
++ int32_t src_stride,
++ int16_t dst_x, int16_t dst_y,
++ uint16_t dst_width, uint16_t dst_height,
++ int32_t dst_stride,
++ const struct pixman_f_transform *t);
++
++void
+ memmove_box(const void *src, void *dst,
+ int bpp, int32_t stride,
+ const BoxRec *box,
+@@ -1182,6 +1282,31 @@ box_intersect(BoxPtr a, const BoxRec *b)
+ return true;
+ }
+
++const BoxRec *
++__find_clip_box_for_y(const BoxRec *begin, const BoxRec *end, int16_t y);
++inline static const BoxRec *
++find_clip_box_for_y(const BoxRec *begin, const BoxRec *end, int16_t y)
++{
++ /* Special case for incremental trapezoid clipping */
++ if (begin == end)
++ return end;
++
++ /* Quick test if scanline is within range of clip boxes */
++ if (begin->y2 > y) {
++ assert(end == begin + 1 ||
++ __find_clip_box_for_y(begin, end, y) == begin);
++ return begin;
++ }
++ if (y >= end[-1].y2) {
++ assert(end == begin + 1 ||
++ __find_clip_box_for_y(begin, end, y) == end);
++ return end;
++ }
++
++ /* Otherwise bisect to find the first box crossing y */
++ return __find_clip_box_for_y(begin, end, y);
++}
++
+ unsigned sna_cpu_detect(void);
+ char *sna_cpu_features_to_string(unsigned features, char *line);
+
+@@ -1237,4 +1362,17 @@ static inline void sigtrap_put(void)
+ extern int getline(char **line, size_t *len, FILE *file);
+ #endif
+
++static inline void add_shm_flush(struct sna *sna, struct sna_pixmap *priv)
++{
++ if (!priv->shm)
++ return;
++
++ DBG(("%s: marking handle=%d for SHM flush\n",
++ __FUNCTION__, priv->cpu_bo->handle));
++
++ assert(!priv->flush);
++ sna_add_flush_pixmap(sna, priv, priv->cpu_bo);
++ sna->needs_shm_flush = true;
++}
++
+ #endif /* _SNA_H */
+diff --git a/src/sna/sna_accel.c b/src/sna/sna_accel.c
+index baf5f609..25a075cf 100644
+--- a/src/sna/sna_accel.c
++++ b/src/sna/sna_accel.c
+@@ -50,8 +50,11 @@
+ #endif
+ #include <shmint.h>
+
++#include <X11/extensions/damageproto.h>
++
+ #include <sys/time.h>
+ #include <sys/mman.h>
++#include <sys/ioctl.h>
+ #include <unistd.h>
+
+ #ifdef HAVE_VALGRIND
+@@ -66,7 +69,8 @@
+ #define FORCE_FLUSH 0
+ #define FORCE_FULL_SYNC 0 /* https://bugs.freedesktop.org/show_bug.cgi?id=61628 */
+
+-#define DEFAULT_TILING I915_TILING_X
++#define DEFAULT_PIXMAP_TILING I915_TILING_X
++#define DEFAULT_SCANOUT_TILING I915_TILING_X
+
+ #define USE_INPLACE 1
+ #define USE_SPANS 0 /* -1 force CPU, 1 force GPU */
+@@ -115,6 +119,11 @@
+ #define RECTILINEAR 0x4
+ #define OVERWRITES 0x8
+
++#if XFONT2_CLIENT_FUNCS_VERSION >= 1
++#define AllocateFontPrivateIndex() xfont2_allocate_font_private_index()
++#define FontSetPrivate(font, idx, data) xfont2_font_set_private(font, idx, data)
++#endif
++
+ #if 0
+ static void __sna_fallback_flush(DrawablePtr d)
+ {
+@@ -213,6 +222,7 @@ static GCOps sna_gc_ops__tmp;
+ static const GCFuncs sna_gc_funcs;
+ static const GCFuncs sna_gc_funcs__cpu;
+
++static void sna_shm_watch_flush(struct sna *sna, int enable);
+ static void
+ sna_poly_fill_rect__gpu(DrawablePtr draw, GCPtr gc, int n, xRectangle *rect);
+
+@@ -527,10 +537,10 @@ sna_pixmap_alloc_cpu(struct sna *sna,
+ DBG(("%s: allocating CPU buffer (%dx%d)\n", __FUNCTION__,
+ pixmap->drawable.width, pixmap->drawable.height));
+
+- hint = 0;
+- if ((flags & MOVE_ASYNC_HINT) == 0 &&
+- ((flags & MOVE_READ) == 0 || (priv->gpu_damage && !priv->clear && !sna->kgem.has_llc)))
+- hint = CREATE_CPU_MAP | CREATE_INACTIVE | CREATE_NO_THROTTLE;
++ hint = CREATE_CPU_MAP | CREATE_INACTIVE | CREATE_NO_THROTTLE;
++ if ((flags & MOVE_ASYNC_HINT) ||
++ (priv->gpu_damage && !priv->clear && kgem_bo_is_busy(priv->gpu_bo) && sna->kgem.can_blt_cpu))
++ hint = 0;
+
+ priv->cpu_bo = kgem_create_cpu_2d(&sna->kgem,
+ pixmap->drawable.width,
+@@ -580,7 +590,7 @@ static void __sna_pixmap_free_cpu(struct sna *sna, struct sna_pixmap *priv)
+ if (priv->cpu_bo->flush) {
+ assert(!priv->cpu_bo->reusable);
+ kgem_bo_sync__cpu(&sna->kgem, priv->cpu_bo);
+- sna_accel_watch_flush(sna, -1);
++ sna_shm_watch_flush(sna, -1);
+ }
+ kgem_bo_destroy(&sna->kgem, priv->cpu_bo);
+ } else if (!IS_STATIC_PTR(priv->ptr))
+@@ -612,9 +622,9 @@ static bool sna_pixmap_free_cpu(struct sna *sna, struct sna_pixmap *priv, bool a
+
+ static inline uint32_t default_tiling(struct sna *sna, PixmapPtr pixmap)
+ {
+-#if DEFAULT_TILING == I915_TILING_NONE
++#if DEFAULT_PIXMAP_TILING == I915_TILING_NONE
+ return I915_TILING_NONE;
+-#elif DEFAULT_TILING == I915_TILING_X
++#elif DEFAULT_PIXMAP_TILING == I915_TILING_X
+ return I915_TILING_X;
+ #else
+ /* Try to avoid hitting the Y-tiling GTT mapping bug on 855GM */
+@@ -630,15 +640,6 @@ static inline uint32_t default_tiling(struct sna *sna, PixmapPtr pixmap)
+ pixmap->drawable.height > sna->render.max_3d_size))
+ return I915_TILING_X;
+
+- if (sna_damage_is_all(&sna_pixmap(pixmap)->cpu_damage,
+- pixmap->drawable.width,
+- pixmap->drawable.height)) {
+- DBG(("%s: entire source is damaged, using Y-tiling\n",
+- __FUNCTION__));
+- sna_damage_destroy(&sna_pixmap(priv)->gpu_damage);
+- return I915_TILING_Y;
+- }
+-
+ return I915_TILING_Y;
+ #endif
+ }
+@@ -666,6 +667,7 @@ struct kgem_bo *sna_pixmap_change_tiling(PixmapPtr pixmap, uint32_t tiling)
+ __FUNCTION__, priv->gpu_bo->tiling, tiling,
+ pixmap->drawable.width, pixmap->drawable.height));
+ assert(priv->gpu_damage == NULL || priv->gpu_bo);
++ assert(priv->gpu_bo->tiling != tiling);
+
+ if (priv->pinned) {
+ DBG(("%s: can't convert pinned bo\n", __FUNCTION__));
+@@ -690,6 +692,12 @@ struct kgem_bo *sna_pixmap_change_tiling(PixmapPtr pixmap, uint32_t tiling)
+ return NULL;
+ }
+
++ if (bo->tiling == priv->gpu_bo->tiling) {
++ DBG(("%s: tiling request failed\n", __FUNCTION__));
++ kgem_bo_destroy(&sna->kgem, bo);
++ return NULL;
++ }
++
+ box.x1 = box.y1 = 0;
+ box.x2 = pixmap->drawable.width;
+ box.y2 = pixmap->drawable.height;
+@@ -824,8 +832,8 @@ create_pixmap(struct sna *sna, ScreenPtr screen,
+ datasize += adjust;
+ }
+
+- DBG(("%s: allocating pixmap %dx%d, depth=%d, size=%ld\n",
+- __FUNCTION__, width, height, depth, (long)datasize));
++ DBG(("%s: allocating pixmap %dx%d, depth=%d/%d, size=%ld\n",
++ __FUNCTION__, width, height, depth, bpp, (long)datasize));
+ pixmap = AllocatePixmap(screen, datasize);
+ if (!pixmap)
+ return NullPixmap;
+@@ -878,7 +886,11 @@ __pop_freed_pixmap(struct sna *sna)
+ pixmap = sna->freed_pixmap;
+ sna->freed_pixmap = pixmap->devPrivate.ptr;
+
++ DBG(("%s: reusing freed pixmap=%ld header\n",
++ __FUNCTION__, pixmap->drawable.serialNumber));
++
+ assert(pixmap->refcnt == 0);
++ assert(pixmap->devKind = 0xdeadbeef);
+ assert(sna_pixmap(pixmap));
+ assert(sna_pixmap(pixmap)->header);
+
+@@ -990,7 +1002,7 @@ fallback:
+ }
+ priv->cpu_bo->pitch = pitch;
+ kgem_bo_mark_unreusable(priv->cpu_bo);
+- sna_accel_watch_flush(sna, 1);
++ sna_shm_watch_flush(sna, 1);
+ #ifdef DEBUG_MEMORY
+ sna->debug_memory.cpu_bo_allocs++;
+ sna->debug_memory.cpu_bo_bytes += kgem_bo_size(priv->cpu_bo);
+@@ -1081,6 +1093,18 @@ sna_pixmap_create_scratch(ScreenPtr screen,
+ return pixmap;
+ }
+
++static unsigned small_copy(const RegionRec *region)
++{
++ if ((region->extents.x2 - region->extents.x1)*(region->extents.y2 - region->extents.y1) < 1024) {
++ DBG(("%s: region:%dx%d\n", __FUNCTION__,
++ (region->extents.x2 - region->extents.x1),
++ (region->extents.y2 - region->extents.y1)));
++ return COPY_SMALL;
++ }
++
++ return 0;
++}
++
+ #ifdef CREATE_PIXMAP_USAGE_SHARED
+ static Bool
+ sna_share_pixmap_backing(PixmapPtr pixmap, ScreenPtr slave, void **fd_handle)
+@@ -1124,7 +1148,7 @@ sna_share_pixmap_backing(PixmapPtr pixmap, ScreenPtr slave, void **fd_handle)
+ pixmap->drawable.height,
+ pixmap->drawable.bitsPerPixel,
+ I915_TILING_NONE,
+- CREATE_GTT_MAP | CREATE_PRIME | CREATE_EXACT);
++ CREATE_GTT_MAP | CREATE_SCANOUT | CREATE_PRIME | CREATE_EXACT);
+ if (bo == NULL) {
+ DBG(("%s: allocation failed\n", __FUNCTION__));
+ return FALSE;
+@@ -1243,7 +1267,7 @@ sna_create_pixmap_shared(struct sna *sna, ScreenPtr screen,
+ width, height,
+ pixmap->drawable.bitsPerPixel,
+ I915_TILING_NONE,
+- CREATE_GTT_MAP | CREATE_PRIME | CREATE_EXACT);
++ CREATE_GTT_MAP | CREATE_SCANOUT | CREATE_PRIME | CREATE_EXACT);
+ if (priv->gpu_bo == NULL) {
+ free(priv);
+ FreePixmap(pixmap);
+@@ -1311,7 +1335,7 @@ static PixmapPtr sna_create_pixmap(ScreenPtr screen,
+
+ if (unlikely((sna->render.prefer_gpu & PREFER_GPU_RENDER) == 0))
+ flags &= ~KGEM_CAN_CREATE_GPU;
+- if (wedged(sna))
++ if (wedged(sna) && usage != SNA_CREATE_FB)
+ flags &= ~KGEM_CAN_CREATE_GTT;
+
+ DBG(("%s: usage=%d, flags=%x\n", __FUNCTION__, usage, flags));
+@@ -1417,10 +1441,13 @@ static void __sna_free_pixmap(struct sna *sna,
+ __sna_pixmap_free_cpu(sna, priv);
+
+ if (priv->flush)
+- sna_accel_watch_flush(sna, -1);
++ sna_watch_flush(sna, -1);
+
++#if !NDEBUG
++ pixmap->devKind = 0xdeadbeef;
++#endif
+ if (priv->header) {
+- assert(pixmap->drawable.pScreen == sna->scrn->pScreen);
++ assert(pixmap->drawable.pScreen == to_screen_from_sna(sna));
+ assert(!priv->shm);
+ pixmap->devPrivate.ptr = sna->freed_pixmap;
+ sna->freed_pixmap = pixmap;
+@@ -1485,7 +1512,7 @@ static Bool sna_destroy_pixmap(PixmapPtr pixmap)
+ if (priv->shm && kgem_bo_is_busy(priv->cpu_bo)) {
+ DBG(("%s: deferring release of active SHM pixmap=%ld\n",
+ __FUNCTION__, pixmap->drawable.serialNumber));
+- sna_add_flush_pixmap(sna, priv, priv->cpu_bo);
++ add_shm_flush(sna, priv);
+ kgem_bo_submit(&sna->kgem, priv->cpu_bo); /* XXX ShmDetach */
+ } else
+ __sna_free_pixmap(sna, pixmap, priv);
+@@ -1529,7 +1556,7 @@ static inline bool has_coherent_ptr(struct sna *sna, struct sna_pixmap *priv, un
+ if (!priv->cpu_bo)
+ return true;
+
+- assert(!priv->cpu_bo->needs_flush);
++ assert(!priv->cpu_bo->needs_flush || (flags & MOVE_WRITE) == 0);
+ assert(priv->pixmap->devKind == priv->cpu_bo->pitch);
+ return priv->pixmap->devPrivate.ptr == MAP(priv->cpu_bo->map__cpu);
+ }
+@@ -1557,6 +1584,11 @@ static inline bool has_coherent_ptr(struct sna *sna, struct sna_pixmap *priv, un
+ return true;
+ }
+
++ if (priv->pixmap->devPrivate.ptr == MAP(priv->gpu_bo->map__wc)) {
++ assert(priv->mapped == MAPPED_GTT);
++ return true;
++ }
++
+ return false;
+ }
+
+@@ -1577,6 +1609,16 @@ static inline bool pixmap_inplace(struct sna *sna,
+ return false;
+
+ if (priv->gpu_bo && kgem_bo_is_busy(priv->gpu_bo)) {
++ if (priv->clear) {
++ DBG(("%s: no, clear GPU bo is busy\n", __FUNCTION__));
++ return false;
++ }
++
++ if (flags & MOVE_ASYNC_HINT) {
++ DBG(("%s: no, async hint and GPU bo is busy\n", __FUNCTION__));
++ return false;
++ }
++
+ if ((flags & (MOVE_WRITE | MOVE_READ)) == (MOVE_WRITE | MOVE_READ)) {
+ DBG(("%s: no, GPU bo is busy\n", __FUNCTION__));
+ return false;
+@@ -1624,7 +1666,7 @@ static bool sna_pixmap_alloc_gpu(struct sna *sna,
+ if (pixmap->usage_hint == SNA_CREATE_FB && (sna->flags & SNA_LINEAR_FB) == 0) {
+ flags |= CREATE_SCANOUT;
+ tiling = kgem_choose_tiling(&sna->kgem,
+- -I915_TILING_X,
++ -DEFAULT_SCANOUT_TILING,
+ pixmap->drawable.width,
+ pixmap->drawable.height,
+ pixmap->drawable.bitsPerPixel);
+@@ -1861,7 +1903,9 @@ sna_pixmap_undo_cow(struct sna *sna, struct sna_pixmap *priv, unsigned flags)
+ assert(priv->gpu_bo == cow->bo);
+ assert(cow->refcnt);
+
+- if (flags && (flags & MOVE_WRITE) == 0 && IS_COW_OWNER(priv->cow))
++ if (flags && /* flags == 0 => force decouple */
++ (flags & MOVE_WRITE) == 0 &&
++ (((flags & __MOVE_FORCE) == 0) || IS_COW_OWNER(priv->cow)))
+ return true;
+
+ if (!IS_COW_OWNER(priv->cow))
+@@ -1933,7 +1977,7 @@ sna_pixmap_undo_cow(struct sna *sna, struct sna_pixmap *priv, unsigned flags)
+ box.y2 = pixmap->drawable.height;
+
+ if (flags & __MOVE_PRIME) {
+- create = CREATE_GTT_MAP | CREATE_PRIME | CREATE_EXACT;
++ create = CREATE_GTT_MAP | CREATE_SCANOUT | CREATE_PRIME | CREATE_EXACT;
+ tiling = I915_TILING_NONE;
+ } else {
+ create = 0;
+@@ -2021,6 +2065,10 @@ sna_pixmap_make_cow(struct sna *sna,
+ cow->bo->handle));
+
+ src_priv->cow = MAKE_COW_OWNER(cow);
++ if (src_priv->flush & FLUSH_WRITE) {
++ assert(!src_priv->shm);
++ sna_add_flush_pixmap(sna, src_priv, src_priv->gpu_bo);
++ }
+ }
+
+ if (cow == COW(dst_priv->cow)) {
+@@ -2267,6 +2315,7 @@ skip_inplace_map:
+ (flags & MOVE_WRITE ? (void *)priv->gpu_bo : (void *)priv->gpu_damage) && priv->cpu_damage == NULL &&
+ priv->gpu_bo->tiling == I915_TILING_NONE &&
+ (flags & MOVE_READ || kgem_bo_can_map__cpu(&sna->kgem, priv->gpu_bo, flags & MOVE_WRITE)) &&
++ (!priv->clear || !kgem_bo_is_busy(priv->gpu_bo)) &&
+ ((flags & (MOVE_WRITE | MOVE_ASYNC_HINT)) == 0 ||
+ (!priv->cow && !priv->move_to_gpu && !__kgem_bo_is_busy(&sna->kgem, priv->gpu_bo)))) {
+ void *ptr;
+@@ -2330,7 +2379,9 @@ skip_inplace_map:
+ pixmap->devKind, pixmap->devKind * pixmap->drawable.height));
+
+ if (priv->cpu_bo) {
++ kgem_bo_undo(&sna->kgem, priv->cpu_bo);
+ if ((flags & MOVE_ASYNC_HINT || priv->cpu_bo->exec) &&
++ sna->kgem.can_blt_cpu &&
+ sna->render.fill_one(sna,
+ pixmap, priv->cpu_bo, priv->clear_color,
+ 0, 0,
+@@ -2344,21 +2395,26 @@ skip_inplace_map:
+ assert(pixmap->devPrivate.ptr == MAP(priv->cpu_bo->map__cpu));
+ }
+
+- assert(pixmap->devKind);
+- if (priv->clear_color == 0 ||
+- pixmap->drawable.bitsPerPixel == 8 ||
+- priv->clear_color == (1 << pixmap->drawable.depth) - 1) {
+- memset(pixmap->devPrivate.ptr, priv->clear_color,
+- (size_t)pixmap->devKind * pixmap->drawable.height);
+- } else {
+- pixman_fill(pixmap->devPrivate.ptr,
+- pixmap->devKind/sizeof(uint32_t),
+- pixmap->drawable.bitsPerPixel,
+- 0, 0,
+- pixmap->drawable.width,
+- pixmap->drawable.height,
+- priv->clear_color);
+- }
++ if (sigtrap_get() == 0) {
++ assert(pixmap->devKind);
++ sigtrap_assert_active();
++ if (priv->clear_color == 0 ||
++ pixmap->drawable.bitsPerPixel == 8 ||
++ priv->clear_color == (1 << pixmap->drawable.depth) - 1) {
++ memset(pixmap->devPrivate.ptr, priv->clear_color,
++ (size_t)pixmap->devKind * pixmap->drawable.height);
++ } else {
++ pixman_fill(pixmap->devPrivate.ptr,
++ pixmap->devKind/sizeof(uint32_t),
++ pixmap->drawable.bitsPerPixel,
++ 0, 0,
++ pixmap->drawable.width,
++ pixmap->drawable.height,
++ priv->clear_color);
++ }
++ sigtrap_put();
++ } else
++ return false;
+
+ clear_done:
+ sna_damage_all(&priv->cpu_damage, pixmap);
+@@ -2414,6 +2470,10 @@ done:
+ DBG(("%s: discarding idle GPU bo\n", __FUNCTION__));
+ sna_pixmap_free_gpu(sna, priv);
+ }
++ if (priv->flush) {
++ assert(!priv->shm);
++ sna_add_flush_pixmap(sna, priv, priv->gpu_bo);
++ }
+ priv->source_count = SOURCE_BIAS;
+ }
+
+@@ -2531,6 +2591,9 @@ static bool cpu_clear_boxes(struct sna *sna,
+ {
+ struct sna_fill_op fill;
+
++ if (!sna->kgem.can_blt_cpu)
++ return false;
++
+ if (!sna_fill_init_blt(&fill, sna,
+ pixmap, priv->cpu_bo,
+ GXcopy, priv->clear_color,
+@@ -2659,6 +2722,10 @@ sna_drawable_move_region_to_cpu(DrawablePtr drawable,
+ }
+ }
+ sna_damage_add_to_pixmap(&priv->cpu_damage, region, pixmap);
++ if (priv->flush) {
++ assert(!priv->shm);
++ sna_add_flush_pixmap(sna, priv, priv->gpu_bo);
++ }
+
+ if (dx | dy)
+ RegionTranslate(region, -dx, -dy);
+@@ -2904,17 +2971,22 @@ move_to_cpu:
+ assert(pixmap->devPrivate.ptr == MAP(priv->cpu_bo->map__cpu));
+ }
+
+- assert(pixmap->devKind);
+- do {
+- pixman_fill(pixmap->devPrivate.ptr,
+- pixmap->devKind/sizeof(uint32_t),
+- pixmap->drawable.bitsPerPixel,
+- box->x1, box->y1,
+- box->x2 - box->x1,
+- box->y2 - box->y1,
+- priv->clear_color);
+- box++;
+- } while (--n);
++ if (sigtrap_get() == 0) {
++ assert(pixmap->devKind);
++ sigtrap_assert_active();
++ do {
++ pixman_fill(pixmap->devPrivate.ptr,
++ pixmap->devKind/sizeof(uint32_t),
++ pixmap->drawable.bitsPerPixel,
++ box->x1, box->y1,
++ box->x2 - box->x1,
++ box->y2 - box->y1,
++ priv->clear_color);
++ box++;
++ } while (--n);
++ sigtrap_put();
++ } else
++ return false;
+
+ clear_done:
+ if (flags & MOVE_WRITE ||
+@@ -3209,13 +3281,14 @@ __sna_pixmap_for_gpu(struct sna *sna, PixmapPtr pixmap, unsigned flags)
+ {
+ struct sna_pixmap *priv;
+
++ assert(flags & (MOVE_READ | MOVE_WRITE | __MOVE_FORCE));
+ if ((flags & __MOVE_FORCE) == 0 && wedged(sna))
+ return NULL;
+
+ priv = sna_pixmap(pixmap);
+ if (priv == NULL) {
+ DBG(("%s: not attached\n", __FUNCTION__));
+- if ((flags & __MOVE_DRI) == 0)
++ if ((flags & (__MOVE_DRI | __MOVE_SCANOUT)) == 0)
+ return NULL;
+
+ if (pixmap->usage_hint == -1) {
+@@ -3238,6 +3311,44 @@ __sna_pixmap_for_gpu(struct sna *sna, PixmapPtr pixmap, unsigned flags)
+ return priv;
+ }
+
++inline static void sna_pixmap_unclean(struct sna *sna,
++ struct sna_pixmap *priv,
++ unsigned flags)
++{
++ struct drm_i915_gem_busy busy;
++
++ assert(DAMAGE_IS_ALL(priv->gpu_damage));
++ assert(priv->gpu_bo);
++ assert(priv->gpu_bo->proxy == NULL);
++ assert_pixmap_map(priv->pixmap, priv);
++
++ sna_damage_destroy(&priv->cpu_damage);
++ list_del(&priv->flush_list);
++
++ if (flags & (__MOVE_DRI | __MOVE_SCANOUT))
++ return;
++
++ if (!priv->flush || priv->gpu_bo->exec)
++ return;
++
++ busy.handle = priv->gpu_bo->handle;
++ busy.busy = 0;
++ ioctl(sna->kgem.fd, DRM_IOCTL_I915_GEM_BUSY, &busy);
++
++ DBG(("%s(pixmap=%ld): cleaning foreign bo handle=%u, busy=%x [ring=%d]\n",
++ __FUNCTION__,
++ priv->pixmap->drawable.serialNumber,
++ busy.handle, busy.busy, !!(busy.busy & (0xfffe << 16))));
++
++ if (busy.busy) {
++ unsigned mode = KGEM_RENDER;
++ if (busy.busy & (0xfffe << 16))
++ mode = KGEM_BLT;
++ kgem_bo_mark_busy(&sna->kgem, priv->gpu_bo, mode);
++ } else
++ __kgem_bo_clear_busy(priv->gpu_bo);
++}
++
+ struct sna_pixmap *
+ sna_pixmap_move_area_to_gpu(PixmapPtr pixmap, const BoxRec *box, unsigned int flags)
+ {
+@@ -3287,12 +3398,14 @@ sna_pixmap_move_area_to_gpu(PixmapPtr pixmap, const BoxRec *box, unsigned int fl
+ if (priv->cow) {
+ unsigned cow = flags & (MOVE_READ | MOVE_WRITE | __MOVE_FORCE);
+
++ assert(cow);
++
+ if ((flags & MOVE_READ) == 0) {
+ if (priv->gpu_damage) {
+ r.extents = *box;
+ r.data = NULL;
+ if (!region_subsumes_damage(&r, priv->gpu_damage))
+- cow |= MOVE_READ;
++ cow |= MOVE_READ | __MOVE_FORCE;
+ }
+ } else {
+ if (priv->cpu_damage) {
+@@ -3303,22 +3416,18 @@ sna_pixmap_move_area_to_gpu(PixmapPtr pixmap, const BoxRec *box, unsigned int fl
+ }
+ }
+
+- if (cow) {
+- if (!sna_pixmap_undo_cow(sna, priv, cow))
+- return NULL;
++ if (!sna_pixmap_undo_cow(sna, priv, cow))
++ return NULL;
+
+- if (priv->gpu_bo == NULL)
+- sna_damage_destroy(&priv->gpu_damage);
+- }
++ if (priv->gpu_bo == NULL)
++ sna_damage_destroy(&priv->gpu_damage);
+ }
+
+ if (sna_damage_is_all(&priv->gpu_damage,
+ pixmap->drawable.width,
+ pixmap->drawable.height)) {
+- assert(priv->gpu_bo);
+- assert(priv->gpu_bo->proxy == NULL);
+- sna_damage_destroy(&priv->cpu_damage);
+- list_del(&priv->flush_list);
++ DBG(("%s: already all-damaged\n", __FUNCTION__));
++ sna_pixmap_unclean(sna, priv, flags);
+ goto done;
+ }
+
+@@ -3360,10 +3469,7 @@ sna_pixmap_move_area_to_gpu(PixmapPtr pixmap, const BoxRec *box, unsigned int fl
+ return priv;
+ }
+
+- if (priv->shm) {
+- assert(!priv->flush);
+- sna_add_flush_pixmap(sna, priv, priv->cpu_bo);
+- }
++ add_shm_flush(sna, priv);
+
+ assert(priv->cpu_damage);
+ region_set(&r, box);
+@@ -3527,7 +3633,8 @@ sna_drawable_use_bo(DrawablePtr drawable, unsigned flags, const BoxRec *box,
+ }
+
+ if (priv->cow) {
+- unsigned cow = MOVE_WRITE | MOVE_READ;
++ unsigned cow = MOVE_WRITE | MOVE_READ | __MOVE_FORCE;
++ assert(cow);
+
+ if (flags & IGNORE_DAMAGE) {
+ if (priv->gpu_damage) {
+@@ -3717,8 +3824,11 @@ create_gpu_bo:
+ else
+ move = MOVE_WRITE | MOVE_READ | MOVE_ASYNC_HINT;
+
+- if (sna_pixmap_move_to_gpu(pixmap, move))
++ if (sna_pixmap_move_to_gpu(pixmap, move)) {
++ sna_damage_all(&priv->gpu_damage,
++ pixmap);
+ goto use_gpu_bo;
++ }
+ }
+
+ if (DAMAGE_IS_ALL(priv->gpu_damage) ||
+@@ -3934,26 +4044,28 @@ prefer_gpu_bo:
+ goto move_to_gpu;
+ }
+
+- if ((priv->cpu_damage == NULL || flags & IGNORE_DAMAGE)) {
+- if (priv->gpu_bo && priv->gpu_bo->tiling) {
+- DBG(("%s: prefer to use GPU bo for rendering large pixmaps\n", __FUNCTION__));
+- goto prefer_gpu_bo;
++ if (!priv->shm) {
++ if ((priv->cpu_damage == NULL || flags & IGNORE_DAMAGE)) {
++ if (priv->gpu_bo && priv->gpu_bo->tiling) {
++ DBG(("%s: prefer to use GPU bo for rendering large pixmaps\n", __FUNCTION__));
++ goto prefer_gpu_bo;
++ }
++
++ if (priv->cpu_bo->pitch >= 4096) {
++ DBG(("%s: prefer to use GPU bo for rendering wide pixmaps\n", __FUNCTION__));
++ goto prefer_gpu_bo;
++ }
+ }
+
+- if (priv->cpu_bo->pitch >= 4096) {
+- DBG(("%s: prefer to use GPU bo for rendering wide pixmaps\n", __FUNCTION__));
++ if ((flags & IGNORE_DAMAGE) == 0 && priv->cpu_bo->snoop) {
++ DBG(("%s: prefer to use GPU bo for reading from snooped target bo\n", __FUNCTION__));
+ goto prefer_gpu_bo;
+ }
+- }
+-
+- if ((flags & IGNORE_DAMAGE) == 0 && priv->cpu_bo->snoop) {
+- DBG(("%s: prefer to use GPU bo for reading from snooped target bo\n", __FUNCTION__));
+- goto prefer_gpu_bo;
+- }
+
+- if (!sna->kgem.can_blt_cpu) {
+- DBG(("%s: can't render to CPU bo, try to use GPU bo\n", __FUNCTION__));
+- goto prefer_gpu_bo;
++ if (!sna->kgem.can_blt_cpu) {
++ DBG(("%s: can't render to CPU bo, try to use GPU bo\n", __FUNCTION__));
++ goto prefer_gpu_bo;
++ }
+ }
+ }
+
+@@ -3967,9 +4079,7 @@ prefer_gpu_bo:
+ }
+
+ if (priv->shm) {
+- assert(!priv->flush);
+- sna_add_flush_pixmap(sna, priv, priv->cpu_bo);
+-
++ add_shm_flush(sna, priv);
+ /* As we may have flushed and retired,, recheck for busy bo */
+ if ((flags & FORCE_GPU) == 0 && !kgem_bo_is_busy(priv->cpu_bo))
+ return NULL;
+@@ -4019,7 +4129,7 @@ sna_pixmap_create_upload(ScreenPtr screen,
+ assert(width);
+ assert(height);
+
+- if (depth == 1)
++ if (depth < 8)
+ return create_pixmap(sna, screen, width, height, depth,
+ CREATE_PIXMAP_USAGE_SCRATCH);
+
+@@ -4121,27 +4231,21 @@ sna_pixmap_move_to_gpu(PixmapPtr pixmap, unsigned flags)
+
+ if (priv->cow) {
+ unsigned cow = flags & (MOVE_READ | MOVE_WRITE | __MOVE_FORCE);
++ assert(cow);
+ if (flags & MOVE_READ && priv->cpu_damage)
+ cow |= MOVE_WRITE;
+- if (cow) {
+- if (!sna_pixmap_undo_cow(sna, priv, cow))
+- return NULL;
++ if (!sna_pixmap_undo_cow(sna, priv, cow))
++ return NULL;
+
+- if (priv->gpu_bo == NULL)
+- sna_damage_destroy(&priv->gpu_damage);
+- }
++ if (priv->gpu_bo == NULL)
++ sna_damage_destroy(&priv->gpu_damage);
+ }
+
+ if (sna_damage_is_all(&priv->gpu_damage,
+ pixmap->drawable.width,
+ pixmap->drawable.height)) {
+ DBG(("%s: already all-damaged\n", __FUNCTION__));
+- assert(DAMAGE_IS_ALL(priv->gpu_damage));
+- assert(priv->gpu_bo);
+- assert(priv->gpu_bo->proxy == NULL);
+- assert_pixmap_map(pixmap, priv);
+- sna_damage_destroy(&priv->cpu_damage);
+- list_del(&priv->flush_list);
++ sna_pixmap_unclean(sna, priv, flags);
+ goto active;
+ }
+
+@@ -4206,7 +4310,7 @@ sna_pixmap_move_to_gpu(PixmapPtr pixmap, unsigned flags)
+ if (flags & MOVE_INPLACE_HINT || (priv->cpu_damage && priv->cpu_bo == NULL))
+ create = CREATE_GTT_MAP | CREATE_INACTIVE;
+ if (flags & __MOVE_PRIME)
+- create |= CREATE_GTT_MAP | CREATE_PRIME | CREATE_EXACT;
++ create |= CREATE_GTT_MAP | CREATE_SCANOUT | CREATE_PRIME | CREATE_EXACT;
+
+ sna_pixmap_alloc_gpu(sna, pixmap, priv, create);
+ }
+@@ -4282,10 +4386,7 @@ sna_pixmap_move_to_gpu(PixmapPtr pixmap, unsigned flags)
+ goto done;
+ }
+
+- if (priv->shm) {
+- assert(!priv->flush);
+- sna_add_flush_pixmap(sna, priv, priv->cpu_bo);
+- }
++ add_shm_flush(sna, priv);
+
+ n = sna_damage_get_boxes(priv->cpu_damage, &box);
+ assert(n);
+@@ -4534,7 +4635,7 @@ static inline bool box32_trim_and_translate(Box32Rec *box, DrawablePtr d, GCPtr
+ return box32_clip(box, gc);
+ }
+
+-static inline void box_add_pt(BoxPtr box, int16_t x, int16_t y)
++static inline void box_add_xy(BoxPtr box, int16_t x, int16_t y)
+ {
+ if (box->x1 > x)
+ box->x1 = x;
+@@ -4547,6 +4648,11 @@ static inline void box_add_pt(BoxPtr box, int16_t x, int16_t y)
+ box->y2 = y;
+ }
+
++static inline void box_add_pt(BoxPtr box, const DDXPointRec *pt)
++{
++ box_add_xy(box, pt->x, pt->y);
++}
++
+ static inline bool box32_to_box16(const Box32Rec *b32, BoxRec *b16)
+ {
+ b16->x1 = b32->x1;
+@@ -4864,6 +4970,7 @@ try_upload__inplace(PixmapPtr pixmap, RegionRec *region,
+ pixmap->devPrivate.ptr = dst;
+ pixmap->devKind = priv->gpu_bo->pitch;
+ priv->mapped = dst == MAP(priv->gpu_bo->map__cpu) ? MAPPED_CPU : MAPPED_GTT;
++ priv->cpu &= priv->mapped == MAPPED_CPU;
+ assert(has_coherent_ptr(sna, priv, MOVE_WRITE));
+
+ box = region_rects(region);
+@@ -4923,8 +5030,7 @@ done:
+ sna_damage_all(&priv->gpu_damage, pixmap);
+ }
+
+- if (priv->shm)
+- sna_add_flush_pixmap(sna, priv, priv->cpu_bo);
++ add_shm_flush(sna, priv);
+ }
+
+ assert(!priv->clear);
+@@ -5172,6 +5278,16 @@ static inline uint8_t blt_depth(int depth)
+ }
+ }
+
++inline static void blt_done(struct sna *sna)
++{
++ sna->blt_state.fill_bo = 0;
++ if (sna->kgem.nbatch && __kgem_ring_empty(&sna->kgem)) {
++ DBG(("%s: flushing BLT operation on empty ring\n",
++ __FUNCTION__));
++ _kgem_submit(&sna->kgem);
++ }
++}
++
+ static bool
+ sna_put_xybitmap_blt(DrawablePtr drawable, GCPtr gc, RegionPtr region,
+ int x, int y, int w, int h, char *bits)
+@@ -5217,6 +5333,7 @@ sna_put_xybitmap_blt(DrawablePtr drawable, GCPtr gc, RegionPtr region,
+
+ kgem_set_mode(&sna->kgem, KGEM_BLT, bo);
+ assert(kgem_bo_can_blt(&sna->kgem, bo));
++ kgem_bcs_set_tiling(&sna->kgem, NULL, bo);
+
+ /* Region is pre-clipped and translated into pixmap space */
+ box = region_rects(region);
+@@ -5238,6 +5355,7 @@ sna_put_xybitmap_blt(DrawablePtr drawable, GCPtr gc, RegionPtr region,
+ return false;
+ _kgem_set_mode(&sna->kgem, KGEM_BLT);
+ }
++ kgem_bcs_set_tiling(&sna->kgem, NULL, bo);
+
+ upload = kgem_create_buffer(&sna->kgem,
+ bstride*bh,
+@@ -5331,7 +5449,7 @@ sna_put_xybitmap_blt(DrawablePtr drawable, GCPtr gc, RegionPtr region,
+ box++;
+ } while (--n);
+
+- sna->blt_state.fill_bo = 0;
++ blt_done(sna);
+ return true;
+ }
+
+@@ -5381,6 +5499,7 @@ sna_put_xypixmap_blt(DrawablePtr drawable, GCPtr gc, RegionPtr region,
+
+ kgem_set_mode(&sna->kgem, KGEM_BLT, bo);
+ assert(kgem_bo_can_blt(&sna->kgem, bo));
++ kgem_bcs_set_tiling(&sna->kgem, NULL, bo);
+
+ skip = h * BitmapBytePad(w + left);
+ for (i = 1 << (gc->depth-1); i; i >>= 1, bits += skip) {
+@@ -5408,6 +5527,7 @@ sna_put_xypixmap_blt(DrawablePtr drawable, GCPtr gc, RegionPtr region,
+ return false;
+ _kgem_set_mode(&sna->kgem, KGEM_BLT);
+ }
++ kgem_bcs_set_tiling(&sna->kgem, NULL, bo);
+
+ upload = kgem_create_buffer(&sna->kgem,
+ bstride*bh,
+@@ -5509,7 +5629,7 @@ sna_put_xypixmap_blt(DrawablePtr drawable, GCPtr gc, RegionPtr region,
+ } while (--n);
+ }
+
+- sna->blt_state.fill_bo = 0;
++ blt_done(sna);
+ return true;
+ }
+
+@@ -5837,7 +5957,7 @@ sna_self_copy_boxes(DrawablePtr src, DrawablePtr dst, GCPtr gc,
+ if (!sna->render.copy_boxes(sna, alu,
+ &pixmap->drawable, priv->gpu_bo, sx, sy,
+ &pixmap->drawable, priv->gpu_bo, tx, ty,
+- box, n, 0)) {
++ box, n, small_copy(region))) {
+ DBG(("%s: fallback - accelerated copy boxes failed\n",
+ __FUNCTION__));
+ goto fallback;
+@@ -6098,6 +6218,9 @@ sna_copy_boxes__inplace(struct sna *sna, RegionPtr region, int alu,
+
+ kgem_bo_sync__cpu_full(&sna->kgem, src_priv->gpu_bo, FORCE_FULL_SYNC);
+
++ if (sigtrap_get())
++ return false;
++
+ box = region_rects(region);
+ n = region_num_rects(region);
+ if (src_priv->gpu_bo->tiling) {
+@@ -6137,6 +6260,8 @@ sna_copy_boxes__inplace(struct sna *sna, RegionPtr region, int alu,
+ }
+ }
+
++ sigtrap_put();
++
+ return true;
+
+ upload_inplace:
+@@ -6234,6 +6359,9 @@ upload_inplace:
+
+ assert(has_coherent_ptr(sna, src_priv, MOVE_READ));
+
++ if (sigtrap_get())
++ return false;
++
+ box = region_rects(region);
+ n = region_num_rects(region);
+ if (dst_priv->gpu_bo->tiling) {
+@@ -6265,15 +6393,19 @@ upload_inplace:
+ } while (--n);
+
+ if (!dst_priv->shm) {
+- assert(ptr == MAP(dst_priv->gpu_bo->map__cpu));
+ dst_pixmap->devPrivate.ptr = ptr;
+ dst_pixmap->devKind = dst_priv->gpu_bo->pitch;
+- dst_priv->mapped = MAPPED_CPU;
++ if (ptr == MAP(dst_priv->gpu_bo->map__cpu)) {
++ dst_priv->mapped = MAPPED_CPU;
++ dst_priv->cpu = true;
++ } else
++ dst_priv->mapped = MAPPED_GTT;
+ assert_pixmap_map(dst_pixmap, dst_priv);
+- dst_priv->cpu = true;
+ }
+ }
+
++ sigtrap_put();
++
+ return true;
+ }
+
+@@ -6326,6 +6458,16 @@ sna_copy_boxes(DrawablePtr src, DrawablePtr dst, GCPtr gc,
+
+ assert(region_num_rects(region));
+
++ if (src_priv &&
++ src_priv->gpu_bo == NULL &&
++ src_priv->cpu_bo == NULL &&
++ src_priv->ptr == NULL) {
++ /* Rare but still happens, nothing to copy */
++ DBG(("%s: src pixmap=%ld is empty\n",
++ __FUNCTION__, src_pixmap->drawable.serialNumber));
++ return;
++ }
++
+ if (src_pixmap == dst_pixmap)
+ return sna_self_copy_boxes(src, dst, gc,
+ region, dx, dy,
+@@ -6491,15 +6633,14 @@ discard_cow:
+ sna_damage_all(&dst_priv->gpu_damage, dst_pixmap);
+ sna_damage_destroy(&dst_priv->cpu_damage);
+ list_del(&dst_priv->flush_list);
+- if (dst_priv->shm)
+- sna_add_flush_pixmap(sna, dst_priv, dst_priv->cpu_bo);
++ add_shm_flush(sna, dst_priv);
+ return;
+ }
+ }
+ if (!sna->render.copy_boxes(sna, alu,
+ &src_pixmap->drawable, src_priv->gpu_bo, src_dx, src_dy,
+ &dst_pixmap->drawable, bo, 0, 0,
+- box, n, 0)) {
++ box, n, small_copy(region))) {
+ DBG(("%s: fallback - accelerated copy boxes failed\n",
+ __FUNCTION__));
+ goto fallback;
+@@ -6536,7 +6677,7 @@ discard_cow:
+ if (!sna->render.copy_boxes(sna, alu,
+ &src_pixmap->drawable, src_priv->gpu_bo, src_dx, src_dy,
+ &dst_pixmap->drawable, bo, 0, 0,
+- box, n, 0)) {
++ box, n, small_copy(region))) {
+ DBG(("%s: fallback - accelerated copy boxes failed\n",
+ __FUNCTION__));
+ goto fallback;
+@@ -6571,15 +6712,12 @@ discard_cow:
+ if (replaces && UNDO)
+ kgem_bo_pair_undo(&sna->kgem, dst_priv->gpu_bo, dst_priv->cpu_bo);
+
+- if (src_priv->shm) {
+- assert(!src_priv->flush);
+- sna_add_flush_pixmap(sna, src_priv, src_priv->cpu_bo);
+- }
++ add_shm_flush(sna, src_priv);
+
+ if (!sna->render.copy_boxes(sna, alu,
+ &src_pixmap->drawable, src_priv->cpu_bo, src_dx, src_dy,
+ &dst_pixmap->drawable, bo, 0, 0,
+- box, n, src_priv->shm ? COPY_LAST : 0)) {
++ box, n, small_copy(region) | (src_priv->shm ? COPY_LAST : 0))) {
+ DBG(("%s: fallback - accelerated copy boxes failed\n",
+ __FUNCTION__));
+ goto fallback;
+@@ -6631,8 +6769,7 @@ discard_cow:
+ ok = sna->render.copy_boxes(sna, alu,
+ &src_pixmap->drawable, src_bo, src_dx, src_dy,
+ &dst_pixmap->drawable, bo, 0, 0,
+- box, n, COPY_LAST);
+-
++ box, n, small_copy(region) | COPY_LAST);
+ kgem_bo_sync__cpu(&sna->kgem, src_bo);
+ assert(src_bo->rq == NULL);
+ kgem_bo_destroy(&sna->kgem, src_bo);
+@@ -6780,18 +6917,22 @@ fallback:
+ return;
+ }
+
+- assert(dst_pixmap->devPrivate.ptr);
+- assert(dst_pixmap->devKind);
+- do {
+- pixman_fill(dst_pixmap->devPrivate.ptr,
+- dst_pixmap->devKind/sizeof(uint32_t),
+- dst_pixmap->drawable.bitsPerPixel,
+- box->x1, box->y1,
+- box->x2 - box->x1,
+- box->y2 - box->y1,
+- src_priv->clear_color);
+- box++;
+- } while (--n);
++ if (sigtrap_get() == 0) {
++ assert(dst_pixmap->devPrivate.ptr);
++ assert(dst_pixmap->devKind);
++ sigtrap_assert_active();
++ do {
++ pixman_fill(dst_pixmap->devPrivate.ptr,
++ dst_pixmap->devKind/sizeof(uint32_t),
++ dst_pixmap->drawable.bitsPerPixel,
++ box->x1, box->y1,
++ box->x2 - box->x1,
++ box->y2 - box->y1,
++ src_priv->clear_color);
++ box++;
++ } while (--n);
++ sigtrap_put();
++ }
+ } else if (!sna_copy_boxes__inplace(sna, region, alu,
+ src_pixmap, src_priv,
+ src_dx, src_dy,
+@@ -6848,36 +6989,39 @@ fallback:
+ ((char *)src_pixmap->devPrivate.ptr +
+ src_dy * src_stride + src_dx * bpp / 8);
+
+- do {
+- DBG(("%s: memcpy_blt(box=(%d, %d), (%d, %d), src=(%d, %d), pitches=(%d, %d))\n",
+- __FUNCTION__,
+- box->x1, box->y1,
+- box->x2 - box->x1,
+- box->y2 - box->y1,
+- src_dx, src_dy,
+- src_stride, dst_stride));
+-
+- assert(box->x1 >= 0);
+- assert(box->y1 >= 0);
+- assert(box->x2 <= dst_pixmap->drawable.width);
+- assert(box->y2 <= dst_pixmap->drawable.height);
+-
+- assert(box->x1 + src_dx >= 0);
+- assert(box->y1 + src_dy >= 0);
+- assert(box->x2 + src_dx <= src_pixmap->drawable.width);
+- assert(box->y2 + src_dy <= src_pixmap->drawable.height);
+- assert(has_coherent_ptr(sna, src_priv, MOVE_READ));
+- assert(has_coherent_ptr(sna, dst_priv, MOVE_WRITE));
+- assert(src_stride);
+- assert(dst_stride);
+- memcpy_blt(src_bits, dst_bits, bpp,
+- src_stride, dst_stride,
+- box->x1, box->y1,
+- box->x1, box->y1,
+- box->x2 - box->x1,
+- box->y2 - box->y1);
+- box++;
+- } while (--n);
++ if (sigtrap_get() == 0) {
++ do {
++ DBG(("%s: memcpy_blt(box=(%d, %d), (%d, %d), src=(%d, %d), pitches=(%d, %d))\n",
++ __FUNCTION__,
++ box->x1, box->y1,
++ box->x2 - box->x1,
++ box->y2 - box->y1,
++ src_dx, src_dy,
++ src_stride, dst_stride));
++
++ assert(box->x1 >= 0);
++ assert(box->y1 >= 0);
++ assert(box->x2 <= dst_pixmap->drawable.width);
++ assert(box->y2 <= dst_pixmap->drawable.height);
++
++ assert(box->x1 + src_dx >= 0);
++ assert(box->y1 + src_dy >= 0);
++ assert(box->x2 + src_dx <= src_pixmap->drawable.width);
++ assert(box->y2 + src_dy <= src_pixmap->drawable.height);
++ assert(has_coherent_ptr(sna, src_priv, MOVE_READ));
++ assert(has_coherent_ptr(sna, dst_priv, MOVE_WRITE));
++ assert(src_stride);
++ assert(dst_stride);
++ memcpy_blt(src_bits, dst_bits, bpp,
++ src_stride, dst_stride,
++ box->x1, box->y1,
++ box->x1, box->y1,
++ box->x2 - box->x1,
++ box->y2 - box->y1);
++ box++;
++ } while (--n);
++ sigtrap_put();
++ }
+ } else {
+ DBG(("%s: fallback -- miCopyRegion\n", __FUNCTION__));
+
+@@ -6931,7 +7075,8 @@ sna_do_copy(DrawablePtr src, DrawablePtr dst, GCPtr gc,
+
+ /* Short cut for unmapped windows */
+ if (dst->type == DRAWABLE_WINDOW && !((WindowPtr)dst)->realized) {
+- DBG(("%s: unmapped\n", __FUNCTION__));
++ DBG(("%s: unmapped/unrealized dst (pixmap=%ld)\n",
++ __FUNCTION__, get_window_pixmap((WindowPtr)dst)));
+ return NULL;
+ }
+
+@@ -7115,19 +7260,28 @@ sna_copy_area(DrawablePtr src, DrawablePtr dst, GCPtr gc,
+ if (gc->planemask == 0)
+ return NULL;
+
+- DBG(("%s: src=(%d, %d)x(%d, %d)+(%d, %d) -> dst=(%d, %d)+(%d, %d); alu=%d, pm=%lx, depth=%d\n",
++ if (sna->ignore_copy_area)
++ return NULL;
++
++ DBG(("%s: src=pixmap=%ld:(%d, %d)x(%d, %d)+(%d, %d) -> dst=pixmap=%ld:(%d, %d)+(%d, %d); alu=%d, pm=%lx, depth=%d\n",
+ __FUNCTION__,
++ get_drawable_pixmap(src)->drawable.serialNumber,
+ src_x, src_y, width, height, src->x, src->y,
++ get_drawable_pixmap(dst)->drawable.serialNumber,
+ dst_x, dst_y, dst->x, dst->y,
+ gc->alu, gc->planemask, gc->depth));
+
+ if (FORCE_FALLBACK || !ACCEL_COPY_AREA || wedged(sna) ||
+- !PM_IS_SOLID(dst, gc->planemask) || gc->depth < 8)
++ !PM_IS_SOLID(dst, gc->planemask) || gc->depth < 8) {
++ DBG(("%s: fallback copy\n", __FUNCTION__));
+ copy = sna_fallback_copy_boxes;
+- else if (src == dst)
++ } else if (src == dst) {
++ DBG(("%s: self copy\n", __FUNCTION__));
+ copy = sna_self_copy_boxes;
+- else
++ } else {
++ DBG(("%s: normal copy\n", __FUNCTION__));
+ copy = sna_copy_boxes;
++ }
+
+ return sna_do_copy(src, dst, gc,
+ src_x, src_y,
+@@ -7136,30 +7290,21 @@ sna_copy_area(DrawablePtr src, DrawablePtr dst, GCPtr gc,
+ copy, 0, NULL);
+ }
+
+-static const BoxRec *
+-find_clip_box_for_y(const BoxRec *begin, const BoxRec *end, int16_t y)
++const BoxRec *
++__find_clip_box_for_y(const BoxRec *begin, const BoxRec *end, int16_t y)
+ {
+- const BoxRec *mid;
+-
+- if (end == begin)
+- return end;
+-
+- if (end - begin == 1) {
++ assert(end - begin > 1);
++ do {
++ const BoxRec *mid = begin + (end - begin) / 2;
++ if (mid->y2 > y)
++ end = mid;
++ else
++ begin = mid;
++ } while (end > begin + 1);
+ if (begin->y2 > y)
+- return begin;
++ return begin;
+ else
+- return end;
+- }
+-
+- mid = begin + (end - begin) / 2;
+- if (mid->y2 > y)
+- /* If no box is found in [begin, mid], the function
+- * will return @mid, which is then known to be the
+- * correct answer.
+- */
+- return find_clip_box_for_y(begin, mid, y);
+- else
+- return find_clip_box_for_y(mid, end, y);
++ return end;
+ }
+
+ struct sna_fill_spans {
+@@ -8223,6 +8368,8 @@ sna_copy_bitmap_blt(DrawablePtr _bitmap, DrawablePtr drawable, GCPtr gc,
+ }
+ br13 |= blt_depth(drawable->depth) << 24;
+ br13 |= copy_ROP[gc->alu] << 16;
++ DBG(("%s: target-depth=%d, alu=%d, bg=%08x, fg=%08x\n",
++ __FUNCTION__, drawable->depth, gc->alu, gc->bgPixel, gc->fgPixel));
+
+ kgem_set_mode(&sna->kgem, KGEM_BLT, arg->bo);
+ assert(kgem_bo_can_blt(&sna->kgem, arg->bo));
+@@ -8255,6 +8402,7 @@ sna_copy_bitmap_blt(DrawablePtr _bitmap, DrawablePtr drawable, GCPtr gc,
+ return; /* XXX fallback? */
+ _kgem_set_mode(&sna->kgem, KGEM_BLT);
+ }
++ kgem_bcs_set_tiling(&sna->kgem, NULL, arg->bo);
+
+ assert(sna->kgem.mode == KGEM_BLT);
+ if (sna->kgem.gen >= 0100) {
+@@ -8270,8 +8418,8 @@ sna_copy_bitmap_blt(DrawablePtr _bitmap, DrawablePtr drawable, GCPtr gc,
+ I915_GEM_DOMAIN_RENDER |
+ KGEM_RELOC_FENCED,
+ 0);
+- b[5] = gc->bgPixel;
+- b[6] = gc->fgPixel;
++ b[6] = gc->bgPixel;
++ b[7] = gc->fgPixel;
+
+ dst = (uint8_t *)&b[8];
+ sna->kgem.nbatch += 8 + src_stride;
+@@ -8322,6 +8470,7 @@ sna_copy_bitmap_blt(DrawablePtr _bitmap, DrawablePtr drawable, GCPtr gc,
+ return; /* XXX fallback? */
+ _kgem_set_mode(&sna->kgem, KGEM_BLT);
+ }
++ kgem_bcs_set_tiling(&sna->kgem, NULL, arg->bo);
+
+ upload = kgem_create_buffer(&sna->kgem,
+ bstride*bh,
+@@ -8408,7 +8557,7 @@ sna_copy_bitmap_blt(DrawablePtr _bitmap, DrawablePtr drawable, GCPtr gc,
+ sna_damage_add_to_pixmap(arg->damage, region, pixmap);
+ }
+ assert_pixmap_damage(pixmap);
+- sna->blt_state.fill_bo = 0;
++ blt_done(sna);
+ }
+
+ static void
+@@ -8472,6 +8621,7 @@ sna_copy_plane_blt(DrawablePtr source, DrawablePtr drawable, GCPtr gc,
+ return; /* XXX fallback? */
+ _kgem_set_mode(&sna->kgem, KGEM_BLT);
+ }
++ kgem_bcs_set_tiling(&sna->kgem, NULL, arg->bo);
+
+ upload = kgem_create_buffer(&sna->kgem,
+ bstride*bh,
+@@ -8588,6 +8738,8 @@ sna_copy_plane_blt(DrawablePtr source, DrawablePtr drawable, GCPtr gc,
+ }
+ }
+
++ kgem_bcs_set_tiling(&sna->kgem, upload, arg->bo);
++
+ assert(sna->kgem.mode == KGEM_BLT);
+ b = sna->kgem.batch + sna->kgem.nbatch;
+ if (sna->kgem.gen >= 0100) {
+@@ -8641,7 +8793,7 @@ sna_copy_plane_blt(DrawablePtr source, DrawablePtr drawable, GCPtr gc,
+ sna_damage_add_to_pixmap(arg->damage, region, dst_pixmap);
+ }
+ assert_pixmap_damage(dst_pixmap);
+- sna->blt_state.fill_bo = 0;
++ blt_done(sna);
+ }
+
+ static RegionPtr
+@@ -8895,36 +9047,11 @@ sna_poly_point_extents(DrawablePtr drawable, GCPtr gc,
+ last.x += pt->x;
+ last.y += pt->y;
+ pt++;
+- box_add_pt(&box, last.x, last.y);
++ box_add_xy(&box, last.x, last.y);
+ }
+ } else {
+- --n; ++pt;
+- while (n >= 8) {
+- box_add_pt(&box, pt[0].x, pt[0].y);
+- box_add_pt(&box, pt[1].x, pt[1].y);
+- box_add_pt(&box, pt[2].x, pt[2].y);
+- box_add_pt(&box, pt[3].x, pt[3].y);
+- box_add_pt(&box, pt[4].x, pt[4].y);
+- box_add_pt(&box, pt[5].x, pt[5].y);
+- box_add_pt(&box, pt[6].x, pt[6].y);
+- box_add_pt(&box, pt[7].x, pt[7].y);
+- pt += 8;
+- n -= 8;
+- }
+- if (n & 4) {
+- box_add_pt(&box, pt[0].x, pt[0].y);
+- box_add_pt(&box, pt[1].x, pt[1].y);
+- box_add_pt(&box, pt[2].x, pt[2].y);
+- box_add_pt(&box, pt[3].x, pt[3].y);
+- pt += 4;
+- }
+- if (n & 2) {
+- box_add_pt(&box, pt[0].x, pt[0].y);
+- box_add_pt(&box, pt[1].x, pt[1].y);
+- pt += 2;
+- }
+- if (n & 1)
+- box_add_pt(&box, pt[0].x, pt[0].y);
++ while (--n)
++ box_add_pt(&box, ++pt);
+ }
+ box.x2++;
+ box.y2++;
+@@ -9636,7 +9763,7 @@ sna_poly_line_extents(DrawablePtr drawable, GCPtr gc,
+ y += pt->y;
+ if (blt)
+ blt &= pt->x == 0 || pt->y == 0;
+- box_add_pt(&box, x, y);
++ box_add_xy(&box, x, y);
+ }
+ } else {
+ int x = box.x1;
+@@ -9648,7 +9775,7 @@ sna_poly_line_extents(DrawablePtr drawable, GCPtr gc,
+ x = pt->x;
+ y = pt->y;
+ }
+- box_add_pt(&box, pt->x, pt->y);
++ box_add_pt(&box, pt);
+ }
+ }
+ box.x2++;
+@@ -10037,7 +10164,7 @@ out:
+ RegionUninit(&data.region);
+ }
+
+-static inline void box_from_seg(BoxPtr b, const xSegment *seg, GCPtr gc)
++static inline bool box_from_seg(BoxPtr b, const xSegment *seg, GCPtr gc)
+ {
+ if (seg->x1 == seg->x2) {
+ if (seg->y1 > seg->y2) {
+@@ -10051,6 +10178,9 @@ static inline void box_from_seg(BoxPtr b, const xSegment *seg, GCPtr gc)
+ if (gc->capStyle != CapNotLast)
+ b->y2++;
+ }
++ if (b->y1 >= b->y2)
++ return false;
++
+ b->x1 = seg->x1;
+ b->x2 = seg->x1 + 1;
+ } else {
+@@ -10065,6 +10195,9 @@ static inline void box_from_seg(BoxPtr b, const xSegment *seg, GCPtr gc)
+ if (gc->capStyle != CapNotLast)
+ b->x2++;
+ }
++ if (b->x1 >= b->x2)
++ return false;
++
+ b->y1 = seg->y1;
+ b->y2 = seg->y1 + 1;
+ }
+@@ -10073,6 +10206,7 @@ static inline void box_from_seg(BoxPtr b, const xSegment *seg, GCPtr gc)
+ __FUNCTION__,
+ seg->x1, seg->y1, seg->x2, seg->y2,
+ b->x1, b->y1, b->x2, b->y2));
++ return true;
+ }
+
+ static bool
+@@ -10107,12 +10241,13 @@ sna_poly_segment_blt(DrawablePtr drawable,
+ nbox = ARRAY_SIZE(boxes);
+ n -= nbox;
+ do {
+- box_from_seg(b, seg++, gc);
+- if (b->y2 > b->y1 && b->x2 > b->x1) {
++ if (box_from_seg(b, seg++, gc)) {
++ assert(!box_empty(b));
+ b->x1 += dx;
+ b->x2 += dx;
+ b->y1 += dy;
+ b->y2 += dy;
++ assert(!box_empty(b));
+ b++;
+ }
+ } while (--nbox);
+@@ -10131,7 +10266,10 @@ sna_poly_segment_blt(DrawablePtr drawable,
+ nbox = ARRAY_SIZE(boxes);
+ n -= nbox;
+ do {
+- box_from_seg(b++, seg++, gc);
++ if (box_from_seg(b, seg++, gc)) {
++ assert(!box_empty(b));
++ b++;
++ }
+ } while (--nbox);
+
+ if (b != boxes) {
+@@ -10156,7 +10294,10 @@ sna_poly_segment_blt(DrawablePtr drawable,
+ do {
+ BoxRec box;
+
+- box_from_seg(&box, seg++, gc);
++ if (!box_from_seg(&box, seg++, gc))
++ continue;
++
++ assert(!box_empty(&box));
+ box.x1 += drawable->x;
+ box.x2 += drawable->x;
+ box.y1 += drawable->y;
+@@ -10174,6 +10315,7 @@ sna_poly_segment_blt(DrawablePtr drawable,
+ b->x2 += dx;
+ b->y1 += dy;
+ b->y2 += dy;
++ assert(!box_empty(b));
+ if (++b == last_box) {
+ fill.boxes(sna, &fill, boxes, last_box-boxes);
+ if (damage)
+@@ -10185,7 +10327,10 @@ sna_poly_segment_blt(DrawablePtr drawable,
+ } while (--n);
+ } else {
+ do {
+- box_from_seg(b, seg++, gc);
++ if (!box_from_seg(b, seg++, gc))
++ continue;
++
++ assert(!box_empty(b));
+ b->x1 += drawable->x;
+ b->x2 += drawable->x;
+ b->y1 += drawable->y;
+@@ -10195,6 +10340,7 @@ sna_poly_segment_blt(DrawablePtr drawable,
+ b->x2 += dx;
+ b->y1 += dy;
+ b->y2 += dy;
++ assert(!box_empty(b));
+ if (++b == last_box) {
+ fill.boxes(sna, &fill, boxes, last_box-boxes);
+ if (damage)
+@@ -10319,8 +10465,11 @@ sna_poly_zero_segment_blt(DrawablePtr drawable,
+ }
+ b->x2++;
+ b->y2++;
+- if (oc1 | oc2)
+- box_intersect(b, extents);
++
++ if ((oc1 | oc2) && !box_intersect(b, extents))
++ continue;
++
++ assert(!box_empty(b));
+ if (++b == last_box) {
+ ret = &&rectangle_continue;
+ goto *jump;
+@@ -10383,6 +10532,7 @@ rectangle_continue:
+ __FUNCTION__, x1, y1,
+ b->x1, b->y1, b->x2, b->y2));
+
++ assert(!box_empty(b));
+ if (++b == last_box) {
+ ret = &&X_continue;
+ goto *jump;
+@@ -10407,6 +10557,7 @@ X_continue:
+ b->x2 = x1 + 1;
+ b->y2 = b->y1 + 1;
+
++ assert(!box_empty(b));
+ if (++b == last_box) {
+ ret = &&X2_continue;
+ goto *jump;
+@@ -10468,6 +10619,7 @@ X2_continue:
+ b->y2 = y1 + 1;
+ b->x2 = x1 + 1;
+
++ assert(!box_empty(b));
+ if (++b == last_box) {
+ ret = &&Y_continue;
+ goto *jump;
+@@ -10491,6 +10643,7 @@ Y_continue:
+ b->y2 = y1 + 1;
+ b->x2 = x1 + 1;
+
++ assert(!box_empty(b));
+ if (++b == last_box) {
+ ret = &&Y2_continue;
+ goto *jump;
+@@ -11785,14 +11938,29 @@ sna_poly_fill_rect_blt(DrawablePtr drawable,
+ if (nbox > ARRAY_SIZE(boxes))
+ nbox = ARRAY_SIZE(boxes);
+ n -= nbox;
+- do {
++ while (nbox >= 2) {
++ b[0].x1 = rect[0].x + dx;
++ b[0].y1 = rect[0].y + dy;
++ b[0].x2 = b[0].x1 + rect[0].width;
++ b[0].y2 = b[0].y1 + rect[0].height;
++
++ b[1].x1 = rect[1].x + dx;
++ b[1].y1 = rect[1].y + dy;
++ b[1].x2 = b[1].x1 + rect[1].width;
++ b[1].y2 = b[1].y1 + rect[1].height;
++
++ b += 2;
++ rect += 2;
++ nbox -= 2;
++ }
++ if (nbox) {
+ b->x1 = rect->x + dx;
+ b->y1 = rect->y + dy;
+ b->x2 = b->x1 + rect->width;
+ b->y2 = b->y1 + rect->height;
+ b++;
+ rect++;
+- } while (--nbox);
++ }
+ fill.boxes(sna, &fill, boxes, b-boxes);
+ b = boxes;
+ } while (n);
+@@ -11802,14 +11970,29 @@ sna_poly_fill_rect_blt(DrawablePtr drawable,
+ if (nbox > ARRAY_SIZE(boxes))
+ nbox = ARRAY_SIZE(boxes);
+ n -= nbox;
+- do {
++ while (nbox >= 2) {
++ b[0].x1 = rect[0].x;
++ b[0].y1 = rect[0].y;
++ b[0].x2 = b[0].x1 + rect[0].width;
++ b[0].y2 = b[0].y1 + rect[0].height;
++
++ b[1].x1 = rect[1].x;
++ b[1].y1 = rect[1].y;
++ b[1].x2 = b[1].x1 + rect[1].width;
++ b[1].y2 = b[1].y1 + rect[1].height;
++
++ b += 2;
++ rect += 2;
++ nbox -= 2;
++ }
++ if (nbox) {
+ b->x1 = rect->x;
+ b->y1 = rect->y;
+ b->x2 = b->x1 + rect->width;
+ b->y2 = b->y1 + rect->height;
+ b++;
+ rect++;
+- } while (--nbox);
++ }
+ fill.boxes(sna, &fill, boxes, b-boxes);
+ b = boxes;
+ } while (n);
+@@ -12192,6 +12375,7 @@ sna_poly_fill_rect_tiled_8x8_blt(DrawablePtr drawable,
+ return false;
+ _kgem_set_mode(&sna->kgem, KGEM_BLT);
+ }
++ kgem_bcs_set_tiling(&sna->kgem, tile_bo, bo);
+
+ get_drawable_deltas(drawable, pixmap, &dx, &dy);
+ assert(extents->x1 + dx >= 0);
+@@ -12335,6 +12519,7 @@ sna_poly_fill_rect_tiled_8x8_blt(DrawablePtr drawable,
+
+ _kgem_submit(&sna->kgem);
+ _kgem_set_mode(&sna->kgem, KGEM_BLT);
++ kgem_bcs_set_tiling(&sna->kgem, tile_bo, bo);
+ } while (1);
+ } else {
+ RegionRec clip;
+@@ -12403,6 +12588,7 @@ sna_poly_fill_rect_tiled_8x8_blt(DrawablePtr drawable,
+ if (!kgem_check_batch(&sna->kgem, 3)) {
+ _kgem_submit(&sna->kgem);
+ _kgem_set_mode(&sna->kgem, KGEM_BLT);
++ kgem_bcs_set_tiling(&sna->kgem, tile_bo, bo);
+
+ unwind_batch = sna->kgem.nbatch;
+ unwind_reloc = sna->kgem.nreloc;
+@@ -12499,6 +12685,7 @@ sna_poly_fill_rect_tiled_8x8_blt(DrawablePtr drawable,
+ DBG(("%s: emitting split batch\n", __FUNCTION__));
+ _kgem_submit(&sna->kgem);
+ _kgem_set_mode(&sna->kgem, KGEM_BLT);
++ kgem_bcs_set_tiling(&sna->kgem, tile_bo, bo);
+
+ unwind_batch = sna->kgem.nbatch;
+ unwind_reloc = sna->kgem.nreloc;
+@@ -12572,7 +12759,7 @@ sna_poly_fill_rect_tiled_8x8_blt(DrawablePtr drawable,
+ }
+ done:
+ assert_pixmap_damage(pixmap);
+- sna->blt_state.fill_bo = 0;
++ blt_done(sna);
+ return true;
+ }
+
+@@ -13128,6 +13315,7 @@ sna_poly_fill_rect_stippled_8x8_blt(DrawablePtr drawable,
+ return false;
+ _kgem_set_mode(&sna->kgem, KGEM_BLT);
+ }
++ kgem_bcs_set_tiling(&sna->kgem, NULL, bo);
+
+ if (!clipped) {
+ dx += drawable->x;
+@@ -13240,6 +13428,7 @@ sna_poly_fill_rect_stippled_8x8_blt(DrawablePtr drawable,
+
+ _kgem_submit(&sna->kgem);
+ _kgem_set_mode(&sna->kgem, KGEM_BLT);
++ kgem_bcs_set_tiling(&sna->kgem, NULL, bo);
+ } while (1);
+ } else {
+ RegionRec clip;
+@@ -13297,6 +13486,7 @@ sna_poly_fill_rect_stippled_8x8_blt(DrawablePtr drawable,
+ if (!kgem_check_batch(&sna->kgem, 3)) {
+ _kgem_submit(&sna->kgem);
+ _kgem_set_mode(&sna->kgem, KGEM_BLT);
++ kgem_bcs_set_tiling(&sna->kgem, NULL, bo);
+
+ assert(sna->kgem.mode == KGEM_BLT);
+ b = sna->kgem.batch + sna->kgem.nbatch;
+@@ -13369,6 +13559,7 @@ sna_poly_fill_rect_stippled_8x8_blt(DrawablePtr drawable,
+ if (!kgem_check_batch(&sna->kgem, 3)) {
+ _kgem_submit(&sna->kgem);
+ _kgem_set_mode(&sna->kgem, KGEM_BLT);
++ kgem_bcs_set_tiling(&sna->kgem, NULL, bo);
+
+ assert(sna->kgem.mode == KGEM_BLT);
+ b = sna->kgem.batch + sna->kgem.nbatch;
+@@ -13419,7 +13610,7 @@ sna_poly_fill_rect_stippled_8x8_blt(DrawablePtr drawable,
+ }
+
+ assert_pixmap_damage(pixmap);
+- sna->blt_state.fill_bo = 0;
++ blt_done(sna);
+ return true;
+ }
+
+@@ -13499,6 +13690,7 @@ sna_poly_fill_rect_stippled_1_blt(DrawablePtr drawable,
+ get_drawable_deltas(drawable, pixmap, &dx, &dy);
+ kgem_set_mode(&sna->kgem, KGEM_BLT, bo);
+ assert(kgem_bo_can_blt(&sna->kgem, bo));
++ kgem_bcs_set_tiling(&sna->kgem, NULL, bo);
+
+ br00 = 3 << 20;
+ br13 = bo->pitch;
+@@ -13543,6 +13735,7 @@ sna_poly_fill_rect_stippled_1_blt(DrawablePtr drawable,
+ return false;
+ _kgem_set_mode(&sna->kgem, KGEM_BLT);
+ }
++ kgem_bcs_set_tiling(&sna->kgem, NULL, bo);
+
+ assert(sna->kgem.mode == KGEM_BLT);
+ b = sna->kgem.batch + sna->kgem.nbatch;
+@@ -13606,6 +13799,7 @@ sna_poly_fill_rect_stippled_1_blt(DrawablePtr drawable,
+ return false;
+ _kgem_set_mode(&sna->kgem, KGEM_BLT);
+ }
++ kgem_bcs_set_tiling(&sna->kgem, NULL, bo);
+
+ upload = kgem_create_buffer(&sna->kgem,
+ bstride*bh,
+@@ -13736,6 +13930,7 @@ sna_poly_fill_rect_stippled_1_blt(DrawablePtr drawable,
+ return false;
+ _kgem_set_mode(&sna->kgem, KGEM_BLT);
+ }
++ kgem_bcs_set_tiling(&sna->kgem, NULL, bo);
+
+ assert(sna->kgem.mode == KGEM_BLT);
+ b = sna->kgem.batch + sna->kgem.nbatch;
+@@ -13797,6 +13992,7 @@ sna_poly_fill_rect_stippled_1_blt(DrawablePtr drawable,
+ return false;
+ _kgem_set_mode(&sna->kgem, KGEM_BLT);
+ }
++ kgem_bcs_set_tiling(&sna->kgem, NULL, bo);
+
+ upload = kgem_create_buffer(&sna->kgem,
+ bstride*bh,
+@@ -13927,6 +14123,7 @@ sna_poly_fill_rect_stippled_1_blt(DrawablePtr drawable,
+ return false;
+ _kgem_set_mode(&sna->kgem, KGEM_BLT);
+ }
++ kgem_bcs_set_tiling(&sna->kgem, NULL, bo);
+
+ assert(sna->kgem.mode == KGEM_BLT);
+ b = sna->kgem.batch + sna->kgem.nbatch;
+@@ -13987,6 +14184,7 @@ sna_poly_fill_rect_stippled_1_blt(DrawablePtr drawable,
+ return false;
+ _kgem_set_mode(&sna->kgem, KGEM_BLT);
+ }
++ kgem_bcs_set_tiling(&sna->kgem, NULL, bo);
+
+ upload = kgem_create_buffer(&sna->kgem,
+ bstride*bh,
+@@ -14064,7 +14262,7 @@ sna_poly_fill_rect_stippled_1_blt(DrawablePtr drawable,
+ }
+ }
+
+- sna->blt_state.fill_bo = 0;
++ blt_done(sna);
+ return true;
+ }
+
+@@ -14126,6 +14324,7 @@ sna_poly_fill_rect_stippled_n_box__imm(struct sna *sna,
+ return; /* XXX fallback? */
+ _kgem_set_mode(&sna->kgem, KGEM_BLT);
+ }
++ kgem_bcs_set_tiling(&sna->kgem, NULL, bo);
+
+ assert(sna->kgem.mode == KGEM_BLT);
+ b = sna->kgem.batch + sna->kgem.nbatch;
+@@ -14251,6 +14450,7 @@ sna_poly_fill_rect_stippled_n_box(struct sna *sna,
+ return; /* XXX fallback? */
+ _kgem_set_mode(&sna->kgem, KGEM_BLT);
+ }
++ kgem_bcs_set_tiling(&sna->kgem, NULL, bo);
+
+ assert(sna->kgem.mode == KGEM_BLT);
+ b = sna->kgem.batch + sna->kgem.nbatch;
+@@ -14414,6 +14614,7 @@ sna_poly_fill_rect_stippled_n_blt__imm(DrawablePtr drawable,
+ get_drawable_deltas(drawable, pixmap, &dx, &dy);
+ kgem_set_mode(&sna->kgem, KGEM_BLT, bo);
+ assert(kgem_bo_can_blt(&sna->kgem, bo));
++ kgem_bcs_set_tiling(&sna->kgem, NULL, bo);
+
+ br00 = XY_MONO_SRC_COPY_IMM | 3 << 20;
+ br13 = bo->pitch;
+@@ -14526,7 +14727,7 @@ sna_poly_fill_rect_stippled_n_blt__imm(DrawablePtr drawable,
+ }
+
+ assert_pixmap_damage(pixmap);
+- sna->blt_state.fill_bo = 0;
++ blt_done(sna);
+ return true;
+ }
+
+@@ -14559,6 +14760,7 @@ sna_poly_fill_rect_stippled_n_blt(DrawablePtr drawable,
+ get_drawable_deltas(drawable, pixmap, &dx, &dy);
+ kgem_set_mode(&sna->kgem, KGEM_BLT, bo);
+ assert(kgem_bo_can_blt(&sna->kgem, bo));
++ kgem_bcs_set_tiling(&sna->kgem, NULL, bo);
+
+ br00 = XY_MONO_SRC_COPY | 3 << 20;
+ br13 = bo->pitch;
+@@ -14673,7 +14875,7 @@ sna_poly_fill_rect_stippled_n_blt(DrawablePtr drawable,
+ assert_pixmap_damage(pixmap);
+ if (tile)
+ kgem_bo_destroy(&sna->kgem, tile);
+- sna->blt_state.fill_bo = 0;
++ blt_done(sna);
+ return true;
+ }
+
+@@ -15281,6 +15483,7 @@ sna_glyph_blt(DrawablePtr drawable, GCPtr gc,
+ }
+ _kgem_set_mode(&sna->kgem, KGEM_BLT);
+ }
++ kgem_bcs_set_tiling(&sna->kgem, NULL, bo);
+
+ DBG(("%s: glyph clip box (%d, %d), (%d, %d)\n",
+ __FUNCTION__,
+@@ -15368,6 +15571,7 @@ sna_glyph_blt(DrawablePtr drawable, GCPtr gc,
+ if (!kgem_check_batch(&sna->kgem, 3+len)) {
+ _kgem_submit(&sna->kgem);
+ _kgem_set_mode(&sna->kgem, KGEM_BLT);
++ kgem_bcs_set_tiling(&sna->kgem, NULL, bo);
+
+ DBG(("%s: new batch, glyph clip box (%d, %d), (%d, %d)\n",
+ __FUNCTION__,
+@@ -15479,7 +15683,7 @@ skip:
+ }
+
+ assert_pixmap_damage(pixmap);
+- sna->blt_state.fill_bo = 0;
++ blt_done(sna);
+ return true;
+ }
+
+@@ -16002,6 +16206,7 @@ sna_reversed_glyph_blt(DrawablePtr drawable, GCPtr gc,
+ }
+ _kgem_set_mode(&sna->kgem, KGEM_BLT);
+ }
++ kgem_bcs_set_tiling(&sna->kgem, NULL, bo);
+
+ unwind_batch = sna->kgem.nbatch;
+ unwind_reloc = sna->kgem.nreloc;
+@@ -16111,6 +16316,7 @@ sna_reversed_glyph_blt(DrawablePtr drawable, GCPtr gc,
+ if (!kgem_check_batch(&sna->kgem, 3+len)) {
+ _kgem_submit(&sna->kgem);
+ _kgem_set_mode(&sna->kgem, KGEM_BLT);
++ kgem_bcs_set_tiling(&sna->kgem, NULL, bo);
+
+ unwind_batch = sna->kgem.nbatch;
+ unwind_reloc = sna->kgem.nreloc;
+@@ -16229,7 +16435,7 @@ skip:
+ }
+
+ assert_pixmap_damage(pixmap);
+- sna->blt_state.fill_bo = 0;
++ blt_done(sna);
+ return true;
+ }
+
+@@ -16450,6 +16656,7 @@ sna_push_pixels_solid_blt(GCPtr gc,
+
+ kgem_set_mode(&sna->kgem, KGEM_BLT, bo);
+ assert(kgem_bo_can_blt(&sna->kgem, bo));
++ kgem_bcs_set_tiling(&sna->kgem, NULL, bo);
+
+ /* Region is pre-clipped and translated into pixmap space */
+ box = region_rects(region);
+@@ -16471,6 +16678,7 @@ sna_push_pixels_solid_blt(GCPtr gc,
+ return false;
+ _kgem_set_mode(&sna->kgem, KGEM_BLT);
+ }
++ kgem_bcs_set_tiling(&sna->kgem, NULL, bo);
+
+ upload = kgem_create_buffer(&sna->kgem,
+ bstride*bh,
+@@ -16564,7 +16772,7 @@ sna_push_pixels_solid_blt(GCPtr gc,
+ box++;
+ } while (--n);
+
+- sna->blt_state.fill_bo = 0;
++ blt_done(sna);
+ return true;
+ }
+
+@@ -16754,7 +16962,9 @@ static int sna_create_gc(GCPtr gc)
+
+ gc->freeCompClip = 0;
+ gc->pCompositeClip = 0;
++#if XORG_VERSION_CURRENT < XORG_VERSION_NUMERIC(1,19,99,1,0)
+ gc->pRotatedPixmap = 0;
++#endif
+
+ fb_gc(gc)->bpp = bits_per_pixel(gc->depth);
+
+@@ -16789,7 +16999,8 @@ sna_get_image__inplace(PixmapPtr pixmap,
+ break;
+ }
+
+- if (!kgem_bo_can_map__cpu(&sna->kgem, priv->gpu_bo, FORCE_FULL_SYNC))
++ if ((flags & MOVE_INPLACE_HINT) == 0 &&
++ !kgem_bo_can_map__cpu(&sna->kgem, priv->gpu_bo, FORCE_FULL_SYNC))
+ return false;
+
+ if (idle && __kgem_bo_is_busy(&sna->kgem, priv->gpu_bo))
+@@ -16801,11 +17012,19 @@ sna_get_image__inplace(PixmapPtr pixmap,
+ assert(sna_damage_contains_box(&priv->gpu_damage, &region->extents) == PIXMAN_REGION_IN);
+ assert(sna_damage_contains_box(&priv->cpu_damage, &region->extents) == PIXMAN_REGION_OUT);
+
+- src = kgem_bo_map__cpu(&sna->kgem, priv->gpu_bo);
+- if (src == NULL)
+- return false;
++ if (kgem_bo_can_map__cpu(&sna->kgem, priv->gpu_bo, FORCE_FULL_SYNC)) {
++ src = kgem_bo_map__cpu(&sna->kgem, priv->gpu_bo);
++ if (src == NULL)
++ return false;
+
+- kgem_bo_sync__cpu_full(&sna->kgem, priv->gpu_bo, FORCE_FULL_SYNC);
++ kgem_bo_sync__cpu_full(&sna->kgem, priv->gpu_bo, FORCE_FULL_SYNC);
++ } else {
++ src = kgem_bo_map__wc(&sna->kgem, priv->gpu_bo);
++ if (src == NULL)
++ return false;
++
++ kgem_bo_sync__gtt(&sna->kgem, priv->gpu_bo);
++ }
+
+ if (sigtrap_get())
+ return false;
+@@ -16833,12 +17052,11 @@ sna_get_image__inplace(PixmapPtr pixmap,
+ region->extents.x2 - region->extents.x1,
+ region->extents.y2 - region->extents.y1);
+ if (!priv->shm) {
+- assert(src == MAP(priv->gpu_bo->map__cpu));
+ pixmap->devPrivate.ptr = src;
+ pixmap->devKind = priv->gpu_bo->pitch;
+- priv->mapped = MAPPED_CPU;
++ priv->mapped = src == MAP(priv->gpu_bo->map__cpu) ? MAPPED_CPU : MAPPED_GTT;
+ assert_pixmap_map(pixmap, priv);
+- priv->cpu = true;
++ priv->cpu &= priv->mapped == MAPPED_CPU;
+ }
+ }
+
+@@ -16930,7 +17148,7 @@ sna_get_image__fast(PixmapPtr pixmap,
+ if (priv == NULL || priv->gpu_damage == NULL)
+ return false;
+
+- if (priv->clear) {
++ if (priv->clear && sigtrap_get() == 0) {
+ int w = region->extents.x2 - region->extents.x1;
+ int h = region->extents.y2 - region->extents.y1;
+ int pitch = PixmapBytePad(w, pixmap->drawable.depth);
+@@ -16939,6 +17157,7 @@ sna_get_image__fast(PixmapPtr pixmap,
+ __FUNCTION__, priv->clear_color));
+ assert(DAMAGE_IS_ALL(priv->gpu_damage));
+ assert(priv->cpu_damage == NULL);
++ sigtrap_assert_active();
+
+ if (priv->clear_color == 0 ||
+ pixmap->drawable.bitsPerPixel == 8 ||
+@@ -16955,6 +17174,7 @@ sna_get_image__fast(PixmapPtr pixmap,
+ priv->clear_color);
+ }
+
++ sigtrap_put();
+ return true;
+ }
+
+@@ -17001,8 +17221,7 @@ sna_get_image(DrawablePtr drawable,
+ if (ACCEL_GET_IMAGE &&
+ !FORCE_FALLBACK &&
+ format == ZPixmap &&
+- drawable->bitsPerPixel >= 8 &&
+- PM_IS_SOLID(drawable, mask)) {
++ drawable->bitsPerPixel >= 8) {
+ PixmapPtr pixmap = get_drawable_pixmap(drawable);
+ int16_t dx, dy;
+
+@@ -17014,7 +17233,7 @@ sna_get_image(DrawablePtr drawable,
+ region.data = NULL;
+
+ if (sna_get_image__fast(pixmap, &region, dst, flags))
+- return;
++ goto apply_planemask;
+
+ if (!sna_drawable_move_region_to_cpu(&pixmap->drawable,
+ &region, flags))
+@@ -17032,6 +17251,16 @@ sna_get_image(DrawablePtr drawable,
+ region.extents.x1, region.extents.y1, 0, 0, w, h);
+ sigtrap_put();
+ }
++
++apply_planemask:
++ if (!PM_IS_SOLID(drawable, mask)) {
++ FbStip pm = fbReplicatePixel(mask, drawable->bitsPerPixel);
++ FbStip *d = (FbStip *)dst;
++ int i, n = PixmapBytePad(w, drawable->depth) / sizeof(FbStip) * h;
++
++ for (i = 0; i < n; i++)
++ d[i] &= pm;
++ }
+ } else {
+ region.extents.x1 = x + drawable->x;
+ region.extents.y1 = y + drawable->y;
+@@ -17162,17 +17391,19 @@ void sna_accel_flush(struct sna *sna)
+ __sna_free_pixmap(sna, priv->pixmap, priv);
+ }
+ } else {
++ unsigned hints;
+ DBG(("%s: flushing DRI pixmap=%ld\n", __FUNCTION__,
+ priv->pixmap->drawable.serialNumber));
+ assert(priv->flush);
+- if (sna_pixmap_move_to_gpu(priv->pixmap,
+- MOVE_READ | __MOVE_FORCE)) {
+- if (priv->flush & IS_CLIPPED) {
++ hints = MOVE_READ | __MOVE_FORCE;
++ if (priv->flush & FLUSH_WRITE)
++ hints |= MOVE_WRITE;
++ if (sna_pixmap_move_to_gpu(priv->pixmap, hints)) {
++ if (priv->flush & FLUSH_WRITE) {
+ kgem_bo_unclean(&sna->kgem, priv->gpu_bo);
+ sna_damage_all(&priv->gpu_damage, priv->pixmap);
+ assert(priv->cpu_damage == NULL);
+- priv->clear = false;
+- priv->cpu = false;
++ assert(priv->clear == false);
+ }
+ }
+ }
+@@ -17184,10 +17415,46 @@ void sna_accel_flush(struct sna *sna)
+ }
+
+ static void
+-sna_accel_flush_callback(CallbackListPtr *list,
+- pointer user_data, pointer call_data)
++sna_shm_flush_callback(CallbackListPtr *list,
++ pointer user_data, pointer call_data)
+ {
+- sna_accel_flush(user_data);
++ struct sna *sna = user_data;
++
++ if (!sna->needs_shm_flush)
++ return;
++
++ sna_accel_flush(sna);
++ sna->needs_shm_flush = false;
++}
++
++static void
++sna_flush_callback(CallbackListPtr *list, pointer user_data, pointer call_data)
++{
++ struct sna *sna = user_data;
++
++ if (!sna->needs_dri_flush)
++ return;
++
++ sna_accel_flush(sna);
++ sna->needs_dri_flush = false;
++}
++
++static void
++sna_event_callback(CallbackListPtr *list, pointer user_data, pointer call_data)
++{
++ EventInfoRec *eventinfo = call_data;
++ struct sna *sna = user_data;
++ int i;
++
++ if (sna->needs_dri_flush)
++ return;
++
++ for (i = 0; i < eventinfo->count; i++) {
++ if (eventinfo->events[i].u.u.type == sna->damage_event) {
++ sna->needs_dri_flush = true;
++ return;
++ }
++ }
+ }
+
+ static struct sna_pixmap *sna_accel_scanout(struct sna *sna)
+@@ -17199,6 +17466,7 @@ static struct sna_pixmap *sna_accel_scanout(struct sna *sna)
+
+ assert(sna->vblank_interval);
+ assert(sna->front);
++ assert(!sna->mode.hidden);
+
+ priv = sna_pixmap(sna->front);
+ if (priv->gpu_bo == NULL)
+@@ -17217,7 +17485,7 @@ static void sna_accel_disarm_timer(struct sna *sna, int id)
+ static bool has_offload_slaves(struct sna *sna)
+ {
+ #if HAS_PIXMAP_SHARING
+- ScreenPtr screen = sna->scrn->pScreen;
++ ScreenPtr screen = to_screen_from_sna(sna);
+ PixmapDirtyUpdatePtr dirty;
+
+ xorg_list_for_each_entry(dirty, &screen->pixmap_dirty_list, ent) {
+@@ -17231,11 +17499,14 @@ static bool has_offload_slaves(struct sna *sna)
+
+ static bool has_shadow(struct sna *sna)
+ {
+- DamagePtr damage = sna->mode.shadow_damage;
++ DamagePtr damage;
+
+- if (damage == NULL)
++ if (!sna->mode.shadow_enabled)
+ return false;
+
++ damage = sna->mode.shadow_damage;
++ assert(damage);
++
+ DBG(("%s: has pending damage? %d, outstanding flips: %d\n",
+ __FUNCTION__,
+ RegionNotEmpty(DamageRegion(damage)),
+@@ -17365,9 +17636,8 @@ static bool sna_accel_do_expire(struct sna *sna)
+ static void sna_accel_post_damage(struct sna *sna)
+ {
+ #if HAS_PIXMAP_SHARING
+- ScreenPtr screen = sna->scrn->pScreen;
++ ScreenPtr screen = to_screen_from_sna(sna);
+ PixmapDirtyUpdatePtr dirty;
+- bool flush = false;
+
+ xorg_list_for_each_entry(dirty, &screen->pixmap_dirty_list, ent) {
+ RegionRec region, *damage;
+@@ -17376,8 +17646,6 @@ static void sna_accel_post_damage(struct sna *sna)
+ int16_t dx, dy;
+ int n;
+
+- assert(dirty->src == sna->front);
+-
+ damage = DamageRegion(dirty->damage);
+ if (RegionNil(damage))
+ continue;
+@@ -17477,7 +17745,14 @@ fallback:
+ box, n, COPY_LAST))
+ goto fallback;
+
+- flush = true;
++ /* Before signalling the slave via ProcessPending,
++ * ensure not only the batch is submitted as the
++ * slave may be using the Damage callback to perform
++ * its copy, but also that the memory must be coherent
++ * - we need to treat it as uncached for the PCI slave
++ * will bypass LLC.
++ */
++ kgem_bo_sync__gtt(&sna->kgem, __sna_pixmap_get_bo(dst));
+ }
+
+ DamageRegionProcessPending(&dirty->slave_dst->drawable);
+@@ -17485,8 +17760,6 @@ skip:
+ RegionUninit(&region);
+ DamageEmpty(dirty->damage);
+ }
+- if (flush)
+- kgem_submit(&sna->kgem);
+ #endif
+ }
+
+@@ -17689,6 +17962,7 @@ sna_set_screen_pixmap(PixmapPtr pixmap)
+ static Bool
+ sna_create_window(WindowPtr win)
+ {
++ DBG(("%s: window=%ld\n", __FUNCTION__, win->drawable.id));
+ sna_set_window_pixmap(win, win->drawable.pScreen->devPrivate);
+ return TRUE;
+ }
+@@ -17714,6 +17988,7 @@ sna_unmap_window(WindowPtr win)
+ static Bool
+ sna_destroy_window(WindowPtr win)
+ {
++ DBG(("%s: window=%ld\n", __FUNCTION__, win->drawable.id));
+ sna_video_destroy_window(win);
+ sna_dri2_destroy_window(win);
+ return TRUE;
+@@ -17790,20 +18065,34 @@ static bool sna_option_accel_none(struct sna *sna)
+ if (wedged(sna))
+ return true;
+
+- if (xf86ReturnOptValBool(sna->Options, OPTION_ACCEL_DISABLE, FALSE))
++ if (!xf86ReturnOptValBool(sna->Options, OPTION_ACCEL_ENABLE, TRUE))
+ return true;
+
++ if (sna->kgem.gen >= 0120)
++ return true;
++
++ if (!intel_option_cast_to_bool(sna->Options,
++ OPTION_ACCEL_METHOD,
++ !IS_DEFAULT_ACCEL_METHOD(NOACCEL)))
++ return false;
++
++#if XORG_VERSION_CURRENT >= XORG_VERSION_NUMERIC(1,7,99,901,0)
+ s = xf86GetOptValString(sna->Options, OPTION_ACCEL_METHOD);
+ if (s == NULL)
+ return IS_DEFAULT_ACCEL_METHOD(NOACCEL);
+
+ return strcasecmp(s, "none") == 0;
++#else
++ return IS_DEFAULT_ACCEL_METHOD(NOACCEL);
++#endif
+ }
+
+ static bool sna_option_accel_blt(struct sna *sna)
+ {
+ const char *s;
+
++ assert(sna->kgem.gen < 0120);
++
+ s = xf86GetOptValString(sna->Options, OPTION_ACCEL_METHOD);
+ if (s == NULL)
+ return false;
+@@ -17811,6 +18100,13 @@ static bool sna_option_accel_blt(struct sna *sna)
+ return strcasecmp(s, "blt") == 0;
+ }
+
++#if HAVE_NOTIFY_FD
++static void sna_accel_notify(int fd, int ready, void *data)
++{
++ sna_mode_wakeup(data);
++}
++#endif
++
+ bool sna_accel_init(ScreenPtr screen, struct sna *sna)
+ {
+ const char *backend;
+@@ -17822,7 +18118,7 @@ bool sna_accel_init(ScreenPtr screen, struct sna *sna)
+ list_init(&sna->flush_pixmaps);
+ list_init(&sna->active_pixmaps);
+
+- AddGeneralSocket(sna->kgem.fd);
++ SetNotifyFd(sna->kgem.fd, sna_accel_notify, X_NOTIFY_READ, sna);
+
+ #ifdef DEBUG_MEMORY
+ sna->timer_expire[DEBUG_MEMORY_TIMER] = GetTimeInMillis()+ 10 * 1000;
+@@ -17892,21 +18188,23 @@ bool sna_accel_init(ScreenPtr screen, struct sna *sna)
+ backend = "disabled";
+ sna->kgem.wedged = true;
+ sna_render_mark_wedged(sna);
+- } else if (sna_option_accel_blt(sna) || sna->info->gen >= 0110)
++ } else if (sna_option_accel_blt(sna))
+ (void)backend;
+- else if (sna->info->gen >= 0100)
++ else if (sna->kgem.gen >= 0110)
++ backend = gen9_render_init(sna, backend);
++ else if (sna->kgem.gen >= 0100)
+ backend = gen8_render_init(sna, backend);
+- else if (sna->info->gen >= 070)
++ else if (sna->kgem.gen >= 070)
+ backend = gen7_render_init(sna, backend);
+- else if (sna->info->gen >= 060)
++ else if (sna->kgem.gen >= 060)
+ backend = gen6_render_init(sna, backend);
+- else if (sna->info->gen >= 050)
++ else if (sna->kgem.gen >= 050)
+ backend = gen5_render_init(sna, backend);
+- else if (sna->info->gen >= 040)
++ else if (sna->kgem.gen >= 040)
+ backend = gen4_render_init(sna, backend);
+- else if (sna->info->gen >= 030)
++ else if (sna->kgem.gen >= 030)
+ backend = gen3_render_init(sna, backend);
+- else if (sna->info->gen >= 020)
++ else if (sna->kgem.gen >= 020)
+ backend = gen2_render_init(sna, backend);
+
+ DBG(("%s(backend=%s, prefer_gpu=%x)\n",
+@@ -17924,8 +18222,14 @@ bool sna_accel_init(ScreenPtr screen, struct sna *sna)
+
+ void sna_accel_create(struct sna *sna)
+ {
++ ExtensionEntry *damage;
++
+ DBG(("%s\n", __FUNCTION__));
+
++ damage = CheckExtension("DAMAGE");
++ if (damage)
++ sna->damage_event = damage->eventBase + XDamageNotify;
++
+ if (!sna_glyphs_create(sna))
+ goto fail;
+
+@@ -17943,27 +18247,59 @@ fail:
+ no_render_init(sna);
+ }
+
+-void sna_accel_watch_flush(struct sna *sna, int enable)
++static void sna_shm_watch_flush(struct sna *sna, int enable)
+ {
+ DBG(("%s: enable=%d\n", __FUNCTION__, enable));
+ assert(enable);
+
+- if (sna->watch_flush == 0) {
++ if (sna->watch_shm_flush == 0) {
++ DBG(("%s: installing shm watchers\n", __FUNCTION__));
++ assert(enable > 0);
++
++ if (!AddCallback(&FlushCallback, sna_shm_flush_callback, sna))
++ return;
++
++ sna->watch_shm_flush++;
++ }
++
++ sna->watch_shm_flush += enable;
++}
++
++void sna_watch_flush(struct sna *sna, int enable)
++{
++ DBG(("%s: enable=%d\n", __FUNCTION__, enable));
++ assert(enable);
++
++ if (sna->watch_dri_flush == 0) {
++ int err = 0;
++
+ DBG(("%s: installing watchers\n", __FUNCTION__));
+ assert(enable > 0);
+- if (!AddCallback(&FlushCallback, sna_accel_flush_callback, sna)) {
++
++ if (!sna->damage_event)
++ return;
++
++ if (!AddCallback(&EventCallback, sna_event_callback, sna))
++ err = 1;
++
++ if (!AddCallback(&FlushCallback, sna_flush_callback, sna))
++ err = 1;
++
++ if (err) {
+ xf86DrvMsg(sna->scrn->scrnIndex, X_Error,
+ "Failed to attach ourselves to the flush callbacks, expect missing synchronisation with DRI clients (e.g a compositor)\n");
+ }
+- sna->watch_flush++;
++
++ sna->watch_dri_flush++;
+ }
+
+- sna->watch_flush += enable;
++ sna->watch_dri_flush += enable;
+ }
+
+ void sna_accel_leave(struct sna *sna)
+ {
+ DBG(("%s\n", __FUNCTION__));
++ sna_scanout_flush(sna);
+
+ /* as root we always have permission to render */
+ if (geteuid() == 0)
+@@ -17997,13 +18333,15 @@ void sna_accel_close(struct sna *sna)
+
+ sna_pixmap_expire(sna);
+
+- DeleteCallback(&FlushCallback, sna_accel_flush_callback, sna);
+- RemoveGeneralSocket(sna->kgem.fd);
++ DeleteCallback(&FlushCallback, sna_shm_flush_callback, sna);
++ DeleteCallback(&FlushCallback, sna_flush_callback, sna);
++ DeleteCallback(&EventCallback, sna_event_callback, sna);
++ RemoveNotifyFd(sna->kgem.fd);
+
+ kgem_cleanup_cache(&sna->kgem);
+ }
+
+-void sna_accel_block_handler(struct sna *sna, struct timeval **tv)
++void sna_accel_block(struct sna *sna, struct timeval **tv)
+ {
+ sigtrap_assert_inactive();
+
+@@ -18044,10 +18382,17 @@ restart:
+ if (sna_accel_do_debug_memory(sna))
+ sna_accel_debug_memory(sna);
+
+- if (sna->watch_flush == 1) {
+- DBG(("%s: removing watchers\n", __FUNCTION__));
+- DeleteCallback(&FlushCallback, sna_accel_flush_callback, sna);
+- sna->watch_flush = 0;
++ if (sna->watch_shm_flush == 1) {
++ DBG(("%s: removing shm watchers\n", __FUNCTION__));
++ DeleteCallback(&FlushCallback, sna_shm_flush_callback, sna);
++ sna->watch_shm_flush = 0;
++ }
++
++ if (sna->watch_dri_flush == 1) {
++ DBG(("%s: removing dri watchers\n", __FUNCTION__));
++ DeleteCallback(&FlushCallback, sna_flush_callback, sna);
++ DeleteCallback(&EventCallback, sna_event_callback, sna);
++ sna->watch_dri_flush = 0;
+ }
+
+ if (sna->timer_active & 1) {
+@@ -18083,22 +18428,6 @@ set_tv:
+ }
+ }
+
+-void sna_accel_wakeup_handler(struct sna *sna)
+-{
+- DBG(("%s: nbatch=%d, need_retire=%d, need_purge=%d\n", __FUNCTION__,
+- sna->kgem.nbatch, sna->kgem.need_retire, sna->kgem.need_purge));
+-
+- if (!sna->kgem.nbatch)
+- return;
+-
+- if (kgem_is_idle(&sna->kgem)) {
+- DBG(("%s: GPU idle, flushing\n", __FUNCTION__));
+- _kgem_submit(&sna->kgem);
+- }
+-
+- sigtrap_assert_inactive();
+-}
+-
+ void sna_accel_free(struct sna *sna)
+ {
+ DBG(("%s\n", __FUNCTION__));
+diff --git a/src/sna/sna_acpi.c b/src/sna/sna_acpi.c
+index dcc0287b..643d04af 100644
+--- a/src/sna/sna_acpi.c
++++ b/src/sna/sna_acpi.c
+@@ -92,7 +92,7 @@ void _sna_acpi_wakeup(struct sna *sna)
+ DBG(("%s: error [%d], detaching from acpid\n", __FUNCTION__, n));
+
+ /* XXX reattach later? */
+- RemoveGeneralSocket(sna->acpi.fd);
++ RemoveNotifyFd(sna->acpi.fd);
+ sna_acpi_fini(sna);
+ return;
+ }
+@@ -136,6 +136,13 @@ void _sna_acpi_wakeup(struct sna *sna)
+ } while (n);
+ }
+
++#if HAVE_NOTIFY_FD
++static void sna_acpi_notify(int fd, int read, void *data)
++{
++ _sna_acpi_wakeup(data);
++}
++#endif
++
+ static int read_power_state(const char *path)
+ {
+ DIR *dir;
+@@ -200,7 +207,7 @@ void sna_acpi_init(struct sna *sna)
+
+ DBG(("%s: attaching to acpid\n", __FUNCTION__));
+
+- AddGeneralSocket(sna->acpi.fd);
++ SetNotifyFd(sna->acpi.fd, sna_acpi_notify, X_NOTIFY_READ, sna);
+ sna->acpi.remain = sizeof(sna->acpi.event) - 1;
+ sna->acpi.offset = 0;
+
+diff --git a/src/sna/sna_blt.c b/src/sna/sna_blt.c
+index de8f6ec3..ddd2586d 100644
+--- a/src/sna/sna_blt.c
++++ b/src/sna/sna_blt.c
+@@ -86,6 +86,11 @@ static const uint8_t fill_ROP[] = {
+ ROP_1
+ };
+
++static void sig_done(struct sna *sna, const struct sna_composite_op *op)
++{
++ sigtrap_put();
++}
++
+ static void nop_done(struct sna *sna, const struct sna_composite_op *op)
+ {
+ assert(sna->kgem.nbatch <= KGEM_BATCH_SIZE(&sna->kgem));
+@@ -129,7 +134,6 @@ static bool sna_blt_fill_init(struct sna *sna,
+ struct kgem *kgem = &sna->kgem;
+
+ assert(kgem_bo_can_blt (kgem, bo));
+- assert(bo->tiling != I915_TILING_Y);
+ blt->bo[0] = bo;
+
+ blt->br13 = bo->pitch;
+@@ -183,6 +187,7 @@ static bool sna_blt_fill_init(struct sna *sna,
+ return false;
+ _kgem_set_mode(kgem, KGEM_BLT);
+ }
++ kgem_bcs_set_tiling(kgem, NULL, bo);
+
+ assert(sna->kgem.mode == KGEM_BLT);
+ b = kgem->batch + kgem->nbatch;
+@@ -237,17 +242,13 @@ static bool sna_blt_fill_init(struct sna *sna,
+ return true;
+ }
+
+-noinline static void sna_blt_fill_begin(struct sna *sna,
+- const struct sna_blt_state *blt)
++noinline static void __sna_blt_fill_begin(struct sna *sna,
++ const struct sna_blt_state *blt)
+ {
+ struct kgem *kgem = &sna->kgem;
+ uint32_t *b;
+
+- if (kgem->nreloc) {
+- _kgem_submit(kgem);
+- _kgem_set_mode(kgem, KGEM_BLT);
+- assert(kgem->nbatch == 0);
+- }
++ kgem_bcs_set_tiling(&sna->kgem, NULL, blt->bo[0]);
+
+ assert(kgem->mode == KGEM_BLT);
+ b = kgem->batch + kgem->nbatch;
+@@ -293,6 +294,21 @@ noinline static void sna_blt_fill_begin(struct sna *sna,
+ }
+ }
+
++inline static void sna_blt_fill_begin(struct sna *sna,
++ const struct sna_blt_state *blt)
++{
++ struct kgem *kgem = &sna->kgem;
++
++ if (kgem->nreloc) {
++ _kgem_submit(kgem);
++ _kgem_set_mode(kgem, KGEM_BLT);
++ kgem_bcs_set_tiling(kgem, NULL, blt->bo[0]);
++ assert(kgem->nbatch == 0);
++ }
++
++ __sna_blt_fill_begin(sna, blt);
++}
++
+ inline static void sna_blt_fill_one(struct sna *sna,
+ const struct sna_blt_state *blt,
+ int16_t x, int16_t y,
+@@ -330,8 +346,8 @@ static bool sna_blt_copy_init(struct sna *sna,
+ {
+ struct kgem *kgem = &sna->kgem;
+
+- assert(kgem_bo_can_blt (kgem, src));
+- assert(kgem_bo_can_blt (kgem, dst));
++ assert(kgem_bo_can_blt(kgem, src));
++ assert(kgem_bo_can_blt(kgem, dst));
+
+ blt->bo[0] = src;
+ blt->bo[1] = dst;
+@@ -370,6 +386,7 @@ static bool sna_blt_copy_init(struct sna *sna,
+ return false;
+ _kgem_set_mode(kgem, KGEM_BLT);
+ }
++ kgem_bcs_set_tiling(&sna->kgem, src, dst);
+
+ sna->blt_state.fill_bo = 0;
+ return true;
+@@ -424,6 +441,7 @@ static bool sna_blt_alpha_fixup_init(struct sna *sna,
+ return false;
+ _kgem_set_mode(kgem, KGEM_BLT);
+ }
++ kgem_bcs_set_tiling(&sna->kgem, src, dst);
+
+ sna->blt_state.fill_bo = 0;
+ return true;
+@@ -454,6 +472,7 @@ static void sna_blt_alpha_fixup_one(struct sna *sna,
+ !kgem_check_reloc(kgem, 2)) {
+ _kgem_submit(kgem);
+ _kgem_set_mode(kgem, KGEM_BLT);
++ kgem_bcs_set_tiling(&sna->kgem, blt->bo[0], blt->bo[1]);
+ }
+
+ assert(sna->kgem.mode == KGEM_BLT);
+@@ -582,6 +601,7 @@ static void sna_blt_copy_one(struct sna *sna,
+ !kgem_check_reloc(kgem, 2)) {
+ _kgem_submit(kgem);
+ _kgem_set_mode(kgem, KGEM_BLT);
++ kgem_bcs_set_tiling(&sna->kgem, blt->bo[0], blt->bo[1]);
+ }
+
+ assert(sna->kgem.mode == KGEM_BLT);
+@@ -912,8 +932,27 @@ sna_composite_mask_is_opaque(PicturePtr mask)
+ return is_solid(mask) && is_white(mask);
+ else if (!PICT_FORMAT_A(mask->format))
+ return true;
+- else
+- return is_solid(mask) && is_opaque_solid(mask);
++ else if (mask->pSourcePict) {
++ PictSolidFill *fill = (PictSolidFill *) mask->pSourcePict;
++ return (fill->color >> 24) == 0xff;
++ } else {
++ struct sna_pixmap *priv;
++ assert(mask->pDrawable);
++
++ if (mask->pDrawable->width == 1 &&
++ mask->pDrawable->height == 1 &&
++ mask->repeat)
++ return pixel_is_opaque(get_pixel(mask), mask->format);
++
++ if (mask->transform)
++ return false;
++
++ priv = sna_pixmap_from_drawable(mask->pDrawable);
++ if (priv == NULL || !priv->clear)
++ return false;
++
++ return pixel_is_opaque(priv->clear_color, mask->format);
++ }
+ }
+
+ fastcall
+@@ -971,6 +1010,7 @@ static void blt_composite_fill__cpu(struct sna *sna,
+
+ assert(op->dst.pixmap->devPrivate.ptr);
+ assert(op->dst.pixmap->devKind);
++ sigtrap_assert_active();
+ pixman_fill(op->dst.pixmap->devPrivate.ptr,
+ op->dst.pixmap->devKind / sizeof(uint32_t),
+ op->dst.pixmap->drawable.bitsPerPixel,
+@@ -990,6 +1030,7 @@ blt_composite_fill_box_no_offset__cpu(struct sna *sna,
+
+ assert(op->dst.pixmap->devPrivate.ptr);
+ assert(op->dst.pixmap->devKind);
++ sigtrap_assert_active();
+ pixman_fill(op->dst.pixmap->devPrivate.ptr,
+ op->dst.pixmap->devKind / sizeof(uint32_t),
+ op->dst.pixmap->drawable.bitsPerPixel,
+@@ -1010,6 +1051,7 @@ blt_composite_fill_boxes_no_offset__cpu(struct sna *sna,
+
+ assert(op->dst.pixmap->devPrivate.ptr);
+ assert(op->dst.pixmap->devKind);
++ sigtrap_assert_active();
+ pixman_fill(op->dst.pixmap->devPrivate.ptr,
+ op->dst.pixmap->devKind / sizeof(uint32_t),
+ op->dst.pixmap->drawable.bitsPerPixel,
+@@ -1031,6 +1073,7 @@ blt_composite_fill_box__cpu(struct sna *sna,
+
+ assert(op->dst.pixmap->devPrivate.ptr);
+ assert(op->dst.pixmap->devKind);
++ sigtrap_assert_active();
+ pixman_fill(op->dst.pixmap->devPrivate.ptr,
+ op->dst.pixmap->devKind / sizeof(uint32_t),
+ op->dst.pixmap->drawable.bitsPerPixel,
+@@ -1052,6 +1095,7 @@ blt_composite_fill_boxes__cpu(struct sna *sna,
+
+ assert(op->dst.pixmap->devPrivate.ptr);
+ assert(op->dst.pixmap->devKind);
++ sigtrap_assert_active();
+ pixman_fill(op->dst.pixmap->devPrivate.ptr,
+ op->dst.pixmap->devKind / sizeof(uint32_t),
+ op->dst.pixmap->drawable.bitsPerPixel,
+@@ -1159,12 +1203,15 @@ static inline void _sna_blt_maybe_clear(const struct sna_composite_op *op, const
+ box->y2 - box->y1 >= op->dst.height) {
+ struct sna_pixmap *priv = sna_pixmap(op->dst.pixmap);
+ if (op->dst.bo == priv->gpu_bo) {
++ sna_damage_all(&priv->gpu_damage, op->dst.pixmap);
++ sna_damage_destroy(&priv->cpu_damage);
+ priv->clear = true;
+ priv->clear_color = op->u.blt.pixel;
+ DBG(("%s: pixmap=%ld marking clear [%08x]\n",
+ __FUNCTION__,
+ op->dst.pixmap->drawable.serialNumber,
+ op->u.blt.pixel));
++ ((struct sna_composite_op *)op)->damage = NULL;
+ }
+ }
+ }
+@@ -1404,6 +1451,7 @@ begin_blt(struct sna *sna,
+ return false;
+
+ _kgem_set_mode(&sna->kgem, KGEM_BLT);
++ kgem_bcs_set_tiling(&sna->kgem, NULL, op->dst.bo);
+ }
+
+ return true;
+@@ -1429,6 +1477,7 @@ prepare_blt_clear(struct sna *sna,
+ DBG(("%s\n", __FUNCTION__));
+
+ if (op->dst.bo == NULL) {
++ op->u.blt.pixel = 0;
+ op->blt = blt_composite_fill__cpu;
+ if (op->dst.x|op->dst.y) {
+ op->box = blt_composite_fill_box__cpu;
+@@ -1439,9 +1488,8 @@ prepare_blt_clear(struct sna *sna,
+ op->boxes = blt_composite_fill_boxes_no_offset__cpu;
+ op->thread_boxes = blt_composite_fill_boxes_no_offset__cpu;
+ }
+- op->done = nop_done;
+- op->u.blt.pixel = 0;
+- return true;
++ op->done = sig_done;
++ return sigtrap_get() == 0;
+ }
+
+ op->blt = blt_composite_fill;
+@@ -1484,8 +1532,8 @@ prepare_blt_fill(struct sna *sna,
+ op->boxes = blt_composite_fill_boxes_no_offset__cpu;
+ op->thread_boxes = blt_composite_fill_boxes_no_offset__cpu;
+ }
+- op->done = nop_done;
+- return true;
++ op->done = sig_done;
++ return sigtrap_get() == 0;
+ }
+
+ op->blt = blt_composite_fill;
+@@ -1668,6 +1716,7 @@ static void blt_composite_copy_boxes__thread(struct sna *sna,
+
+ _kgem_submit(kgem);
+ _kgem_set_mode(kgem, KGEM_BLT);
++ kgem_bcs_set_tiling(&sna->kgem, src_bo, dst_bo);
+ } while (1);
+ } else {
+ do {
+@@ -1724,6 +1773,7 @@ static void blt_composite_copy_boxes__thread(struct sna *sna,
+
+ _kgem_submit(kgem);
+ _kgem_set_mode(kgem, KGEM_BLT);
++ kgem_bcs_set_tiling(&sna->kgem, src_bo, dst_bo);
+ } while (1);
+ }
+ sna_vertex_unlock(&sna->render);
+@@ -1806,6 +1856,7 @@ static void blt_composite_copy_boxes__thread64(struct sna *sna,
+
+ _kgem_submit(kgem);
+ _kgem_set_mode(kgem, KGEM_BLT);
++ kgem_bcs_set_tiling(&sna->kgem, src_bo, dst_bo);
+ } while (1);
+ } else {
+ do {
+@@ -1864,6 +1915,7 @@ static void blt_composite_copy_boxes__thread64(struct sna *sna,
+
+ _kgem_submit(kgem);
+ _kgem_set_mode(kgem, KGEM_BLT);
++ kgem_bcs_set_tiling(&sna->kgem, src_bo, dst_bo);
+ } while (1);
+ }
+ sna_vertex_unlock(&sna->render);
+@@ -1973,6 +2025,7 @@ prepare_blt_copy(struct sna *sna,
+ }
+ _kgem_set_mode(&sna->kgem, KGEM_BLT);
+ }
++ kgem_bcs_set_tiling(&sna->kgem, bo, op->dst.bo);
+
+ DBG(("%s\n", __FUNCTION__));
+
+@@ -2396,6 +2449,9 @@ prepare_blt_put(struct sna *sna,
+ op->box = blt_put_composite_box;
+ op->boxes = blt_put_composite_boxes;
+ }
++
++ op->done = nop_done;
++ return true;
+ } else {
+ if (alpha_fixup) {
+ op->u.blt.pixel = alpha_fixup;
+@@ -2407,10 +2463,10 @@ prepare_blt_put(struct sna *sna,
+ op->box = blt_put_composite_box__cpu;
+ op->boxes = blt_put_composite_boxes__cpu;
+ }
+- }
+- op->done = nop_done;
+
+- return true;
++ op->done = sig_done;
++ return sigtrap_get() == 0;
++ }
+ }
+
+ static bool
+@@ -2544,6 +2600,7 @@ sna_blt_composite(struct sna *sna,
+ clear:
+ if (was_clear && sna_pixmap(tmp->dst.pixmap)->clear_color == 0) {
+ sna_pixmap(tmp->dst.pixmap)->clear = true;
++nop:
+ return prepare_blt_nop(sna, tmp);
+ }
+
+@@ -2559,6 +2616,7 @@ clear:
+ }
+ tmp->dst.bo = sna_drawable_use_bo(dst->pDrawable, hint,
+ &dst_box, &tmp->damage);
++ assert(!tmp->damage || !DAMAGE_IS_ALL(*tmp->damage));
+ if (tmp->dst.bo) {
+ if (!kgem_bo_can_blt(&sna->kgem, tmp->dst.bo)) {
+ DBG(("%s: can not blit to dst, tiling? %d, pitch? %d\n",
+@@ -2567,6 +2625,8 @@ clear:
+ }
+ if (hint & REPLACES)
+ kgem_bo_undo(&sna->kgem, tmp->dst.bo);
++ if (flags & COMPOSITE_UPLOAD)
++ return false;
+ } else {
+ RegionRec region;
+
+@@ -2590,32 +2650,40 @@ clear:
+ }
+ if (op == PictOpOver && is_opaque_solid(src))
+ op = PictOpSrc;
+- if (op == PictOpAdd && is_white(src))
++ if (op == PictOpAdd &&
++ PICT_FORMAT_RGB(src->format) == PICT_FORMAT_RGB(dst->format) &&
++ is_white(src))
+ op = PictOpSrc;
+ if (was_clear && (op == PictOpAdd || op == PictOpOver)) {
+ if (sna_pixmap(tmp->dst.pixmap)->clear_color == 0)
+ op = PictOpSrc;
+ if (op == PictOpOver) {
++ unsigned dst_color = solid_color(dst->format, sna_pixmap(tmp->dst.pixmap)->clear_color);
+ color = over(get_solid_color(src, PICT_a8r8g8b8),
+- color_convert(sna_pixmap(tmp->dst.pixmap)->clear_color,
+- dst->format, PICT_a8r8g8b8));
++ dst_color);
+ op = PictOpSrc;
+ DBG(("%s: precomputing solid OVER (%08x, %08x) -> %08x\n",
+ __FUNCTION__, get_solid_color(src, PICT_a8r8g8b8),
+- color_convert(sna_pixmap(tmp->dst.pixmap)->clear_color,
+- dst->format, PICT_a8r8g8b8),
++ solid_color(dst->format, sna_pixmap(tmp->dst.pixmap)->clear_color),
+ color));
++ if (color == dst_color)
++ goto nop;
++ else
++ goto fill;
+ }
+ if (op == PictOpAdd) {
++ unsigned dst_color = solid_color(dst->format, sna_pixmap(tmp->dst.pixmap)->clear_color);
+ color = add(get_solid_color(src, PICT_a8r8g8b8),
+- color_convert(sna_pixmap(tmp->dst.pixmap)->clear_color,
+- dst->format, PICT_a8r8g8b8));
++ dst_color);
+ op = PictOpSrc;
+ DBG(("%s: precomputing solid ADD (%08x, %08x) -> %08x\n",
+ __FUNCTION__, get_solid_color(src, PICT_a8r8g8b8),
+- color_convert(sna_pixmap(tmp->dst.pixmap)->clear_color,
+- dst->format, PICT_a8r8g8b8),
++ solid_color(dst->format, sna_pixmap(tmp->dst.pixmap)->clear_color),
+ color));
++ if (color == dst_color)
++ goto nop;
++ else
++ goto fill;
+ }
+ }
+ if (op == PictOpOutReverse && is_opaque_solid(src))
+@@ -2649,6 +2717,7 @@ fill:
+ }
+ tmp->dst.bo = sna_drawable_use_bo(dst->pDrawable, hint,
+ &dst_box, &tmp->damage);
++ assert(!tmp->damage || !DAMAGE_IS_ALL(*tmp->damage));
+ if (tmp->dst.bo) {
+ if (!kgem_bo_can_blt(&sna->kgem, tmp->dst.bo)) {
+ DBG(("%s: can not blit to dst, tiling? %d, pitch? %d\n",
+@@ -2657,6 +2726,8 @@ fill:
+ }
+ if (hint & REPLACES)
+ kgem_bo_undo(&sna->kgem, tmp->dst.bo);
++ if (flags & COMPOSITE_UPLOAD)
++ return false;
+ } else {
+ RegionRec region;
+
+@@ -2720,8 +2791,8 @@ fill:
+ if (is_clear(src_pixmap)) {
+ if (src->repeat ||
+ (x >= 0 && y >= 0 &&
+- x + width < src_pixmap->drawable.width &&
+- y + height < src_pixmap->drawable.height)) {
++ x + width <= src_pixmap->drawable.width &&
++ y + height <= src_pixmap->drawable.height)) {
+ color = color_convert(sna_pixmap(src_pixmap)->clear_color,
+ src->format, tmp->dst.format);
+ goto fill;
+@@ -2795,7 +2866,7 @@ fill:
+ if (src_pixmap->drawable.width <= sna->render.max_3d_size &&
+ src_pixmap->drawable.height <= sna->render.max_3d_size &&
+ bo->pitch <= sna->render.max_3d_pitch &&
+- (flags & COMPOSITE_FALLBACK) == 0)
++ (flags & (COMPOSITE_UPLOAD | COMPOSITE_FALLBACK)) == 0)
+ {
+ return false;
+ }
+@@ -2817,6 +2888,7 @@ fill:
+ }
+ tmp->dst.bo = sna_drawable_use_bo(dst->pDrawable, hint,
+ &dst_box, &tmp->damage);
++ assert(!tmp->damage || !DAMAGE_IS_ALL(*tmp->damage));
+
+ if (tmp->dst.bo && hint & REPLACES) {
+ struct sna_pixmap *priv = sna_pixmap(tmp->dst.pixmap);
+@@ -2846,7 +2918,7 @@ fallback:
+ DBG(("%s: fallback -- unaccelerated upload\n",
+ __FUNCTION__));
+ goto fallback;
+- } else {
++ } else if ((flags & COMPOSITE_UPLOAD) == 0) {
+ ret = prepare_blt_copy(sna, tmp, bo, alpha_fixup);
+ if (!ret)
+ goto fallback;
+@@ -3023,6 +3095,7 @@ sna_blt_composite__convert(struct sna *sna,
+ }
+ _kgem_set_mode(&sna->kgem, KGEM_BLT);
+ }
++ kgem_bcs_set_tiling(&sna->kgem, tmp->src.bo, tmp->dst.bo);
+
+ if (alpha_fixup) {
+ tmp->blt = blt_composite_copy_with_alpha;
+@@ -3062,7 +3135,7 @@ static void sna_blt_fill_op_blt(struct sna *sna,
+ if (sna->blt_state.fill_bo != op->base.u.blt.bo[0]->unique_id) {
+ const struct sna_blt_state *blt = &op->base.u.blt;
+
+- sna_blt_fill_begin(sna, blt);
++ __sna_blt_fill_begin(sna, blt);
+
+ sna->blt_state.fill_bo = blt->bo[0]->unique_id;
+ sna->blt_state.fill_pixel = blt->pixel;
+@@ -3079,7 +3152,7 @@ fastcall static void sna_blt_fill_op_box(struct sna *sna,
+ if (sna->blt_state.fill_bo != op->base.u.blt.bo[0]->unique_id) {
+ const struct sna_blt_state *blt = &op->base.u.blt;
+
+- sna_blt_fill_begin(sna, blt);
++ __sna_blt_fill_begin(sna, blt);
+
+ sna->blt_state.fill_bo = blt->bo[0]->unique_id;
+ sna->blt_state.fill_pixel = blt->pixel;
+@@ -3097,7 +3170,7 @@ fastcall static void sna_blt_fill_op_boxes(struct sna *sna,
+ if (sna->blt_state.fill_bo != op->base.u.blt.bo[0]->unique_id) {
+ const struct sna_blt_state *blt = &op->base.u.blt;
+
+- sna_blt_fill_begin(sna, blt);
++ __sna_blt_fill_begin(sna, blt);
+
+ sna->blt_state.fill_bo = blt->bo[0]->unique_id;
+ sna->blt_state.fill_pixel = blt->pixel;
+@@ -3132,7 +3205,7 @@ fastcall static void sna_blt_fill_op_points(struct sna *sna,
+ DBG(("%s: %08x x %d\n", __FUNCTION__, blt->pixel, n));
+
+ if (sna->blt_state.fill_bo != op->base.u.blt.bo[0]->unique_id) {
+- sna_blt_fill_begin(sna, blt);
++ __sna_blt_fill_begin(sna, blt);
+
+ sna->blt_state.fill_bo = blt->bo[0]->unique_id;
+ sna->blt_state.fill_pixel = blt->pixel;
+@@ -3162,65 +3235,15 @@ fastcall static void sna_blt_fill_op_points(struct sna *sna,
+ assert(kgem->nbatch < kgem->surface);
+
+ if ((dx|dy) == 0) {
+- while (n_this_time >= 8) {
+- *((uint64_t *)b + 0) = pt_add(cmd, p+0, 0, 0);
+- *((uint64_t *)b + 1) = pt_add(cmd, p+1, 0, 0);
+- *((uint64_t *)b + 2) = pt_add(cmd, p+2, 0, 0);
+- *((uint64_t *)b + 3) = pt_add(cmd, p+3, 0, 0);
+- *((uint64_t *)b + 4) = pt_add(cmd, p+4, 0, 0);
+- *((uint64_t *)b + 5) = pt_add(cmd, p+5, 0, 0);
+- *((uint64_t *)b + 6) = pt_add(cmd, p+6, 0, 0);
+- *((uint64_t *)b + 7) = pt_add(cmd, p+7, 0, 0);
+- b += 16;
+- n_this_time -= 8;
+- p += 8;
+- }
+- if (n_this_time & 4) {
+- *((uint64_t *)b + 0) = pt_add(cmd, p+0, 0, 0);
+- *((uint64_t *)b + 1) = pt_add(cmd, p+1, 0, 0);
+- *((uint64_t *)b + 2) = pt_add(cmd, p+2, 0, 0);
+- *((uint64_t *)b + 3) = pt_add(cmd, p+3, 0, 0);
+- b += 8;
+- p += 4;
+- }
+- if (n_this_time & 2) {
+- *((uint64_t *)b + 0) = pt_add(cmd, p+0, 0, 0);
+- *((uint64_t *)b + 1) = pt_add(cmd, p+1, 0, 0);
+- b += 4;
+- p += 2;
+- }
+- if (n_this_time & 1)
+- *((uint64_t *)b + 0) = pt_add(cmd, p++, 0, 0);
++ do {
++ *(uint64_t *)b = pt_add(cmd, p++, 0, 0);
++ b += 2;
++ } while (--n_this_time);
+ } else {
+- while (n_this_time >= 8) {
+- *((uint64_t *)b + 0) = pt_add(cmd, p+0, dx, dy);
+- *((uint64_t *)b + 1) = pt_add(cmd, p+1, dx, dy);
+- *((uint64_t *)b + 2) = pt_add(cmd, p+2, dx, dy);
+- *((uint64_t *)b + 3) = pt_add(cmd, p+3, dx, dy);
+- *((uint64_t *)b + 4) = pt_add(cmd, p+4, dx, dy);
+- *((uint64_t *)b + 5) = pt_add(cmd, p+5, dx, dy);
+- *((uint64_t *)b + 6) = pt_add(cmd, p+6, dx, dy);
+- *((uint64_t *)b + 7) = pt_add(cmd, p+7, dx, dy);
+- b += 16;
+- n_this_time -= 8;
+- p += 8;
+- }
+- if (n_this_time & 4) {
+- *((uint64_t *)b + 0) = pt_add(cmd, p+0, dx, dy);
+- *((uint64_t *)b + 1) = pt_add(cmd, p+1, dx, dy);
+- *((uint64_t *)b + 2) = pt_add(cmd, p+2, dx, dy);
+- *((uint64_t *)b + 3) = pt_add(cmd, p+3, dx, dy);
+- b += 8;
+- p += 8;
+- }
+- if (n_this_time & 2) {
+- *((uint64_t *)b + 0) = pt_add(cmd, p+0, dx, dy);
+- *((uint64_t *)b + 1) = pt_add(cmd, p+1, dx, dy);
+- b += 4;
+- p += 2;
+- }
+- if (n_this_time & 1)
+- *((uint64_t *)b + 0) = pt_add(cmd, p++, dx, dy);
++ do {
++ *(uint64_t *)b = pt_add(cmd, p++, dx, dy);
++ b += 2;
++ } while (--n_this_time);
+ }
+
+ if (!n)
+@@ -3414,6 +3437,7 @@ static bool sna_blt_fill_box(struct sna *sna, uint8_t alu,
+
+ _kgem_set_mode(kgem, KGEM_BLT);
+ }
++ kgem_bcs_set_tiling(&sna->kgem, NULL, bo);
+
+ assert(kgem_check_batch(kgem, 6));
+ assert(kgem_check_reloc(kgem, 1));
+@@ -3520,6 +3544,8 @@ bool sna_blt_fill_boxes(struct sna *sna, uint8_t alu,
+ _kgem_set_mode(kgem, KGEM_BLT);
+ }
+
++ kgem_bcs_set_tiling(&sna->kgem, NULL, bo);
++
+ assert(sna->kgem.mode == KGEM_BLT);
+ b = kgem->batch + kgem->nbatch;
+ if (kgem->gen >= 0100) {
+@@ -3608,6 +3634,7 @@ bool sna_blt_fill_boxes(struct sna *sna, uint8_t alu,
+
+ _kgem_submit(kgem);
+ _kgem_set_mode(kgem, KGEM_BLT);
++ kgem_bcs_set_tiling(&sna->kgem, NULL, bo);
+
+ assert(sna->kgem.mode == KGEM_BLT);
+ b = kgem->batch + kgem->nbatch;
+@@ -3754,6 +3781,7 @@ bool sna_blt_copy_boxes(struct sna *sna, uint8_t alu,
+ }
+ _kgem_set_mode(kgem, KGEM_BLT);
+ }
++ kgem_bcs_set_tiling(&sna->kgem, src_bo, dst_bo);
+
+ if ((dst_dx | dst_dy) == 0) {
+ if (kgem->gen >= 0100) {
+@@ -3814,6 +3842,7 @@ bool sna_blt_copy_boxes(struct sna *sna, uint8_t alu,
+
+ _kgem_submit(kgem);
+ _kgem_set_mode(kgem, KGEM_BLT);
++ kgem_bcs_set_tiling(&sna->kgem, src_bo, dst_bo);
+ } while (1);
+ } else {
+ uint64_t hdr = (uint64_t)br13 << 32 | cmd | 6;
+@@ -3871,6 +3900,7 @@ bool sna_blt_copy_boxes(struct sna *sna, uint8_t alu,
+
+ _kgem_submit(kgem);
+ _kgem_set_mode(kgem, KGEM_BLT);
++ kgem_bcs_set_tiling(&sna->kgem, src_bo, dst_bo);
+ } while (1);
+ }
+ } else {
+@@ -3932,6 +3962,7 @@ bool sna_blt_copy_boxes(struct sna *sna, uint8_t alu,
+
+ _kgem_submit(kgem);
+ _kgem_set_mode(kgem, KGEM_BLT);
++ kgem_bcs_set_tiling(&sna->kgem, src_bo, dst_bo);
+ } while (1);
+ } else {
+ cmd |= 6;
+@@ -3989,6 +4020,7 @@ bool sna_blt_copy_boxes(struct sna *sna, uint8_t alu,
+
+ _kgem_submit(kgem);
+ _kgem_set_mode(kgem, KGEM_BLT);
++ kgem_bcs_set_tiling(&sna->kgem, src_bo, dst_bo);
+ } while (1);
+ }
+ }
+@@ -4095,6 +4127,7 @@ bool sna_blt_copy_boxes__with_alpha(struct sna *sna, uint8_t alu,
+ !kgem_check_reloc(kgem, 2)) {
+ _kgem_submit(kgem);
+ _kgem_set_mode(kgem, KGEM_BLT);
++ kgem_bcs_set_tiling(&sna->kgem, src_bo, dst_bo);
+ }
+
+ assert(sna->kgem.mode == KGEM_BLT);
+@@ -4190,6 +4223,7 @@ bool sna_blt_copy_boxes_fallback(struct sna *sna, uint8_t alu,
+ DBG(("%s: dst == src\n", __FUNCTION__));
+
+ if (src_bo->tiling == I915_TILING_Y &&
++ !sna->kgem.can_blt_y &&
+ kgem_bo_blt_pitch_is_ok(&sna->kgem, src_bo)) {
+ struct kgem_bo *bo;
+
+@@ -4237,6 +4271,7 @@ bool sna_blt_copy_boxes_fallback(struct sna *sna, uint8_t alu,
+ }
+ } else {
+ if (src_bo->tiling == I915_TILING_Y &&
++ !sna->kgem.can_blt_y &&
+ kgem_bo_blt_pitch_is_ok(&sna->kgem, src_bo)) {
+ DBG(("%s: src is y-tiled\n", __FUNCTION__));
+ if (src->type != DRAWABLE_PIXMAP)
+@@ -4251,6 +4286,7 @@ bool sna_blt_copy_boxes_fallback(struct sna *sna, uint8_t alu,
+ }
+
+ if (dst_bo->tiling == I915_TILING_Y &&
++ !sna->kgem.can_blt_y &&
+ kgem_bo_blt_pitch_is_ok(&sna->kgem, dst_bo)) {
+ DBG(("%s: dst is y-tiled\n", __FUNCTION__));
+ if (dst->type != DRAWABLE_PIXMAP)
+diff --git a/src/sna/sna_composite.c b/src/sna/sna_composite.c
+index f01f020e..1da8c291 100644
+--- a/src/sna/sna_composite.c
++++ b/src/sna/sna_composite.c
+@@ -452,6 +452,8 @@ static void apply_damage(struct sna_composite_op *op, RegionPtr region)
+ op->damage = NULL;
+ } else
+ sna_damage_add(op->damage, region);
++
++ assert(!op->damage || !DAMAGE_IS_ALL(*op->damage));
+ }
+
+ static inline bool use_cpu(PixmapPtr pixmap, struct sna_pixmap *priv,
+@@ -653,8 +655,9 @@ sna_composite(CARD8 op,
+ RegionRec region;
+ int dx, dy;
+
+- DBG(("%s(%d src=%ld+(%d, %d), mask=%ld+(%d, %d), dst=%ld+(%d, %d)+(%d, %d), size=(%d, %d)\n",
+- __FUNCTION__, op,
++ DBG(("%s(pixmap=%ld, op=%d, src=%ld+(%d, %d), mask=%ld+(%d, %d), dst=%ld+(%d, %d)+(%d, %d), size=(%d, %d)\n",
++ __FUNCTION__,
++ pixmap->drawable.serialNumber, op,
+ get_picture_id(src), src_x, src_y,
+ get_picture_id(mask), mask_x, mask_y,
+ get_picture_id(dst), dst_x, dst_y,
+@@ -673,13 +676,6 @@ sna_composite(CARD8 op,
+ src = sna->clear;
+ }
+
+- if (mask && sna_composite_mask_is_opaque(mask)) {
+- DBG(("%s: removing opaque %smask\n",
+- __FUNCTION__,
+- mask->componentAlpha && PICT_FORMAT_RGB(mask->format) ? "CA " : ""));
+- mask = NULL;
+- }
+-
+ if (!sna_compute_composite_region(&region,
+ src, mask, dst,
+ src_x, src_y,
+@@ -688,6 +684,13 @@ sna_composite(CARD8 op,
+ width, height))
+ return;
+
++ if (mask && sna_composite_mask_is_opaque(mask)) {
++ DBG(("%s: removing opaque %smask\n",
++ __FUNCTION__,
++ mask->componentAlpha && PICT_FORMAT_RGB(mask->format) ? "CA " : ""));
++ mask = NULL;
++ }
++
+ if (NO_COMPOSITE)
+ goto fallback;
+
+@@ -756,6 +759,7 @@ sna_composite(CARD8 op,
+ DBG(("%s: fallback due unhandled composite op\n", __FUNCTION__));
+ goto fallback;
+ }
++ assert(!tmp.damage || !DAMAGE_IS_ALL(*tmp.damage));
+
+ if (region.data == NULL)
+ tmp.box(sna, &tmp, &region.extents);
+@@ -797,8 +801,10 @@ sna_composite_rectangles(CARD8 op,
+ int i, num_boxes;
+ unsigned hint;
+
+- DBG(("%s(op=%d, %08x x %d [(%d, %d)x(%d, %d) ...])\n",
+- __FUNCTION__, op,
++ DBG(("%s(pixmap=%ld, op=%d, %08x x %d [(%d, %d)x(%d, %d) ...])\n",
++ __FUNCTION__,
++ get_drawable_pixmap(dst->pDrawable)->drawable.serialNumber,
++ op,
+ (color->alpha >> 8 << 24) |
+ (color->red >> 8 << 16) |
+ (color->green >> 8 << 8) |
+@@ -814,38 +820,40 @@ sna_composite_rectangles(CARD8 op,
+ return;
+ }
+
+- if ((color->red|color->green|color->blue|color->alpha) <= 0x00ff) {
+- switch (op) {
+- case PictOpOver:
+- case PictOpOutReverse:
+- case PictOpAdd:
+- return;
+- case PictOpInReverse:
+- case PictOpSrc:
+- op = PictOpClear;
+- break;
+- case PictOpAtopReverse:
+- op = PictOpOut;
+- break;
+- case PictOpXor:
+- op = PictOpOverReverse;
+- break;
+- }
+- }
+ if (color->alpha <= 0x00ff) {
+- switch (op) {
+- case PictOpOver:
+- case PictOpOutReverse:
+- return;
+- case PictOpInReverse:
+- op = PictOpClear;
+- break;
+- case PictOpAtopReverse:
+- op = PictOpOut;
+- break;
+- case PictOpXor:
+- op = PictOpOverReverse;
+- break;
++ if (PICT_FORMAT_TYPE(dst->format) == PICT_TYPE_A ||
++ (color->red|color->green|color->blue) <= 0x00ff) {
++ switch (op) {
++ case PictOpOver:
++ case PictOpOutReverse:
++ case PictOpAdd:
++ return;
++ case PictOpInReverse:
++ case PictOpSrc:
++ op = PictOpClear;
++ break;
++ case PictOpAtopReverse:
++ op = PictOpOut;
++ break;
++ case PictOpXor:
++ op = PictOpOverReverse;
++ break;
++ }
++ } else {
++ switch (op) {
++ case PictOpOver:
++ case PictOpOutReverse:
++ return;
++ case PictOpInReverse:
++ op = PictOpClear;
++ break;
++ case PictOpAtopReverse:
++ op = PictOpOut;
++ break;
++ case PictOpXor:
++ op = PictOpOverReverse;
++ break;
++ }
+ }
+ } else if (color->alpha >= 0xff00) {
+ switch (op) {
+@@ -863,11 +871,16 @@ sna_composite_rectangles(CARD8 op,
+ case PictOpXor:
+ op = PictOpOut;
+ break;
++ case PictOpAdd:
++ if (PICT_FORMAT_TYPE(dst->format) == PICT_TYPE_A ||
++ (color->red&color->green&color->blue) >= 0xff00)
++ op = PictOpSrc;
++ break;
+ }
+ }
+
+ /* Avoid reducing overlapping translucent rectangles */
+- if (op == PictOpOver &&
++ if ((op == PictOpOver || op == PictOpAdd) &&
+ num_rects == 1 &&
+ sna_drawable_is_clear(dst->pDrawable))
+ op = PictOpSrc;
+@@ -979,6 +992,9 @@ sna_composite_rectangles(CARD8 op,
+ bool ok;
+
+ if (op == PictOpClear) {
++ if (priv->clear_color == 0)
++ goto done;
++
+ ok = sna_get_pixel_from_rgba(&pixel,
+ 0, 0, 0, 0,
+ dst->format);
+@@ -990,8 +1006,11 @@ sna_composite_rectangles(CARD8 op,
+ color->alpha,
+ dst->format);
+ }
+- if (ok && priv->clear_color == pixel)
++ if (ok && priv->clear_color == pixel) {
++ DBG(("%s: matches current clear, skipping\n",
++ __FUNCTION__));
+ goto done;
++ }
+ }
+
+ if (region.data == NULL) {
+diff --git a/src/sna/sna_damage.h b/src/sna/sna_damage.h
+index 272e83bc..d5c727ee 100644
+--- a/src/sna/sna_damage.h
++++ b/src/sna/sna_damage.h
+@@ -267,7 +267,7 @@ int _sna_damage_get_boxes(struct sna_damage *damage, const BoxRec **boxes);
+ static inline int
+ sna_damage_get_boxes(struct sna_damage *damage, const BoxRec **boxes)
+ {
+- assert(damage);
++ assert(DAMAGE_PTR(damage));
+
+ if (DAMAGE_IS_ALL(damage)) {
+ *boxes = &DAMAGE_PTR(damage)->extents;
+@@ -322,7 +322,8 @@ static inline void sna_damage_destroy(struct sna_damage **damage)
+ if (*damage == NULL)
+ return;
+
+- __sna_damage_destroy(DAMAGE_PTR(*damage));
++ if (DAMAGE_PTR(*damage))
++ __sna_damage_destroy(DAMAGE_PTR(*damage));
+ *damage = NULL;
+ }
+
+diff --git a/src/sna/sna_display.c b/src/sna/sna_display.c
+index 4b218b70..9b77550e 100644
+--- a/src/sna/sna_display.c
++++ b/src/sna/sna_display.c
+@@ -39,6 +39,25 @@
+ #include <errno.h>
+ #include <poll.h>
+ #include <ctype.h>
++#include <dirent.h>
++
++#if HAVE_ALLOCA_H
++#include <alloca.h>
++#elif defined __GNUC__
++#define alloca __builtin_alloca
++#elif defined _AIX
++#define alloca __alloca
++#elif defined _MSC_VER
++#include <malloc.h>
++#define alloca _alloca
++#else
++void *alloca(size_t);
++#endif
++
++#define _PARSE_EDID_
++/* Jump through a few hoops in order to fixup EDIDs */
++#undef VERSION
++#undef REVISION
+
+ #include "sna.h"
+ #include "sna_reg.h"
+@@ -72,6 +91,10 @@
+ #include <memcheck.h>
+ #endif
+
++#define FAIL_CURSOR_IOCTL 0
++
++#define COLDPLUG_DELAY_MS 2000
++
+ /* Minor discrepancy between 32-bit/64-bit ABI in old kernels */
+ union compat_mode_get_connector{
+ struct drm_mode_get_connector conn;
+@@ -88,6 +111,8 @@ union compat_mode_get_connector{
+ #define DEFAULT_DPI 96
+ #endif
+
++#define OUTPUT_STATUS_CACHE_MS 15000
++
+ #define DRM_MODE_PAGE_FLIP_ASYNC 0x02
+
+ #define DRM_CLIENT_CAP_UNIVERSAL_PLANES 2
+@@ -106,33 +131,87 @@ struct local_mode_obj_get_properties {
+ };
+ #define LOCAL_MODE_OBJECT_PLANE 0xeeeeeeee
+
+-#if 0
++struct local_mode_set_plane {
++ uint32_t plane_id;
++ uint32_t crtc_id;
++ uint32_t fb_id; /* fb object contains surface format type */
++ uint32_t flags;
++
++ /* Signed dest location allows it to be partially off screen */
++ int32_t crtc_x, crtc_y;
++ uint32_t crtc_w, crtc_h;
++
++ /* Source values are 16.16 fixed point */
++ uint32_t src_x, src_y;
++ uint32_t src_h, src_w;
++};
++#define LOCAL_IOCTL_MODE_SETPLANE DRM_IOWR(0xB7, struct local_mode_set_plane)
++
++struct local_mode_get_plane {
++ uint32_t plane_id;
++
++ uint32_t crtc_id;
++ uint32_t fb_id;
++
++ uint32_t possible_crtcs;
++ uint32_t gamma_size;
++
++ uint32_t count_format_types;
++ uint64_t format_type_ptr;
++};
++#define LOCAL_IOCTL_MODE_GETPLANE DRM_IOWR(0xb6, struct local_mode_get_plane)
++
++struct local_mode_get_plane_res {
++ uint64_t plane_id_ptr;
++ uint64_t count_planes;
++};
++#define LOCAL_IOCTL_MODE_GETPLANERESOURCES DRM_IOWR(0xb5, struct local_mode_get_plane_res)
++
++#if 1
+ #define __DBG DBG
+ #else
+ #define __DBG(x)
+ #endif
+
++#define DBG_NATIVE_ROTATION ~0 /* minimum RR_Rotate_0 */
++
+ extern XF86ConfigPtr xf86configptr;
+
++struct sna_cursor {
++ struct sna_cursor *next;
++ uint32_t *image;
++ bool transformed;
++ Rotation rotation;
++ int ref;
++ int size;
++ int last_width;
++ int last_height;
++ unsigned handle;
++ unsigned serial;
++ unsigned alloc;
++};
++
+ struct sna_crtc {
++ unsigned long flags;
++ uint32_t id;
+ xf86CrtcPtr base;
+ struct drm_mode_modeinfo kmode;
+- int dpms_mode;
+ PixmapPtr slave_pixmap;
+ DamagePtr slave_damage;
+- struct kgem_bo *bo, *shadow_bo, *client_bo;
++ struct kgem_bo *bo, *shadow_bo, *client_bo, *cache_bo;
+ struct sna_cursor *cursor;
+ unsigned int last_cursor_size;
+ uint32_t offset;
+ bool shadow;
+ bool fallback_shadow;
+ bool transform;
++ bool cursor_transform;
++ bool hwcursor;
+ bool flip_pending;
+- uint8_t id;
+- uint8_t pipe;
+
+- RegionRec client_damage; /* XXX overlap with shadow damage? */
++ struct pict_f_transform cursor_to_fb, fb_to_cursor;
+
++ RegionRec crtc_damage;
+ uint16_t shadow_bo_width, shadow_bo_height;
+
+ uint32_t rotation;
+@@ -143,7 +222,9 @@ struct sna_crtc {
+ uint32_t supported;
+ uint32_t current;
+ } rotation;
+- } primary, sprite;
++ struct list link;
++ } primary;
++ struct list sprites;
+
+ uint32_t mode_serial, flip_serial;
+
+@@ -173,21 +254,33 @@ struct sna_output {
+
+ unsigned int is_panel : 1;
+ unsigned int add_default_modes : 1;
++ int connector_type;
++ int connector_type_id;
++
++ uint32_t link_status_idx;
+
+ uint32_t edid_idx;
+ uint32_t edid_blob_id;
+ uint32_t edid_len;
+ void *edid_raw;
++ xf86MonPtr fake_edid_mon;
++ void *fake_edid_raw;
+
+ bool has_panel_limits;
+ int panel_hdisplay;
+ int panel_vdisplay;
+
+ uint32_t dpms_id;
+- int dpms_mode;
++ uint8_t dpms_mode;
+ struct backlight backlight;
+ int backlight_active_level;
+
++ uint32_t last_detect;
++ uint32_t status;
++ unsigned int hotplug_count;
++ bool update_properties;
++ bool reprobe;
++
+ int num_modes;
+ struct drm_mode_modeinfo *modes;
+
+@@ -218,13 +311,91 @@ enum { /* XXX copied from hw/xfree86/modes/xf86Crtc.c */
+ OPTION_DEFAULT_MODES,
+ };
+
++static void __sna_output_dpms(xf86OutputPtr output, int dpms, int fixup);
+ static void sna_crtc_disable_cursor(struct sna *sna, struct sna_crtc *crtc);
++static bool sna_crtc_flip(struct sna *sna, struct sna_crtc *crtc,
++ struct kgem_bo *bo, int x, int y);
+
+ static bool is_zaphod(ScrnInfoPtr scrn)
+ {
+ return xf86IsEntityShared(scrn->entityList[0]);
+ }
+
++static bool
++sna_zaphod_match(struct sna *sna, const char *output)
++{
++ const char *s, *colon;
++ char t[20];
++ unsigned int i = 0;
++
++ s = xf86GetOptValString(sna->Options, OPTION_ZAPHOD);
++ if (s == NULL)
++ return false;
++
++ colon = strchr(s, ':');
++ if (colon) /* Skip over the ZaphodPipes */
++ s = colon + 1;
++
++ do {
++ /* match any outputs in a comma list, stopping at whitespace */
++ switch (*s) {
++ case '\0':
++ t[i] = '\0';
++ return strcmp(t, output) == 0;
++
++ case ',':
++ t[i] ='\0';
++ if (strcmp(t, output) == 0)
++ return TRUE;
++ i = 0;
++ break;
++
++ case ' ':
++ case '\t':
++ case '\n':
++ case '\r':
++ break;
++
++ default:
++ t[i++] = *s;
++ break;
++ }
++
++ s++;
++ } while (i < sizeof(t));
++
++ return false;
++}
++
++static unsigned
++get_zaphod_crtcs(struct sna *sna)
++{
++ const char *str, *colon;
++ unsigned crtcs = 0;
++
++ str = xf86GetOptValString(sna->Options, OPTION_ZAPHOD);
++ if (str == NULL || (colon = strchr(str, ':')) == NULL) {
++ DBG(("%s: no zaphod pipes, using screen number: %x\n",
++ __FUNCTION__,
++ sna->scrn->confScreen->device->screen));
++ return 1 << sna->scrn->confScreen->device->screen;
++ }
++
++ DBG(("%s: ZaphodHeads='%s'\n", __FUNCTION__, str));
++ while (str < colon) {
++ char *end;
++ unsigned crtc = strtoul(str, &end, 0);
++ if (end == str)
++ break;
++ DBG(("%s: adding CRTC %d to zaphod pipes\n",
++ __FUNCTION__, crtc));
++ crtcs |= 1 << crtc;
++ str = end + 1;
++ }
++ DBG(("%s: ZaphodPipes=%x\n", __FUNCTION__, crtcs));
++ return crtcs;
++}
++
+ inline static unsigned count_to_mask(int x)
+ {
+ return (1 << x) - 1;
+@@ -247,6 +418,21 @@ static inline struct sna_crtc *to_sna_crtc(xf86CrtcPtr crtc)
+ return crtc->driver_private;
+ }
+
++static inline unsigned __sna_crtc_pipe(struct sna_crtc *crtc)
++{
++ return crtc->flags >> 8 & 0xff;
++}
++
++static inline unsigned __sna_crtc_id(struct sna_crtc *crtc)
++{
++ return crtc->id;
++}
++
++uint32_t sna_crtc_id(xf86CrtcPtr crtc)
++{
++ return __sna_crtc_id(to_sna_crtc(crtc));
++}
++
+ static inline bool event_pending(int fd)
+ {
+ struct pollfd pfd;
+@@ -268,29 +454,37 @@ static inline uint32_t fb_id(struct kgem_bo *bo)
+ return bo->delta;
+ }
+
+-uint32_t sna_crtc_id(xf86CrtcPtr crtc)
++unsigned sna_crtc_count_sprites(xf86CrtcPtr crtc)
+ {
+- if (to_sna_crtc(crtc) == NULL)
+- return 0;
+- return to_sna_crtc(crtc)->id;
+-}
++ struct plane *sprite;
++ unsigned count;
+
+-int sna_crtc_to_pipe(xf86CrtcPtr crtc)
+-{
+- assert(to_sna_crtc(crtc));
+- return to_sna_crtc(crtc)->pipe;
++ count = 0;
++ list_for_each_entry(sprite, &to_sna_crtc(crtc)->sprites, link)
++ count++;
++
++ return count;
+ }
+
+-uint32_t sna_crtc_to_sprite(xf86CrtcPtr crtc)
++static struct plane *lookup_sprite(struct sna_crtc *crtc, unsigned idx)
+ {
+- assert(to_sna_crtc(crtc));
+- return to_sna_crtc(crtc)->sprite.id;
++ struct plane *sprite;
++
++ list_for_each_entry(sprite, &crtc->sprites, link)
++ if (idx-- == 0)
++ return sprite;
++
++ return NULL;
+ }
+
+-bool sna_crtc_is_on(xf86CrtcPtr crtc)
++uint32_t sna_crtc_to_sprite(xf86CrtcPtr crtc, unsigned idx)
+ {
++ struct plane *sprite;
++
+ assert(to_sna_crtc(crtc));
+- return to_sna_crtc(crtc)->bo != NULL;
++
++ sprite = lookup_sprite(to_sna_crtc(crtc), idx);
++ return sprite ? sprite->id : 0;
+ }
+
+ bool sna_crtc_is_transformed(xf86CrtcPtr crtc)
+@@ -299,34 +493,48 @@ bool sna_crtc_is_transformed(xf86CrtcPtr crtc)
+ return to_sna_crtc(crtc)->transform;
+ }
+
+-static inline uint64_t msc64(struct sna_crtc *sna_crtc, uint32_t seq)
++static inline bool msc64(struct sna_crtc *sna_crtc, uint32_t seq, uint64_t *msc)
+ {
++ bool record = true;
+ if (seq < sna_crtc->last_seq) {
+ if (sna_crtc->last_seq - seq > 0x40000000) {
+ sna_crtc->wrap_seq++;
+ DBG(("%s: pipe=%d wrapped; was %u, now %u, wraps=%u\n",
+- __FUNCTION__, sna_crtc->pipe,
++ __FUNCTION__, __sna_crtc_pipe(sna_crtc),
+ sna_crtc->last_seq, seq, sna_crtc->wrap_seq));
+- } else {
+- ERR(("%s: pipe=%d msc went backwards; was %u, now %u\n",
+- __FUNCTION__, sna_crtc->pipe, sna_crtc->last_seq, seq));
+- seq = sna_crtc->last_seq;
++ } else {
++ DBG(("%s: pipe=%d msc went backwards; was %u, now %u; ignoring for last_swap\n",
++ __FUNCTION__, __sna_crtc_pipe(sna_crtc), sna_crtc->last_seq, seq));
++
++ record = false;
+ }
+ }
+- sna_crtc->last_seq = seq;
+- return (uint64_t)sna_crtc->wrap_seq << 32 | seq;
++ *msc = (uint64_t)sna_crtc->wrap_seq << 32 | seq;
++ return record;
+ }
+
+ uint64_t sna_crtc_record_swap(xf86CrtcPtr crtc,
+ int tv_sec, int tv_usec, unsigned seq)
+ {
+ struct sna_crtc *sna_crtc = to_sna_crtc(crtc);
++ uint64_t msc;
++
+ assert(sna_crtc);
+- DBG(("%s: recording last swap on pipe=%d, frame %d, time %d.%06d\n",
+- __FUNCTION__, sna_crtc->pipe, seq, tv_sec, tv_usec));
+- sna_crtc->swap.tv_sec = tv_sec;
+- sna_crtc->swap.tv_usec = tv_usec;
+- return sna_crtc->swap.msc = msc64(sna_crtc, seq);
++
++ if (msc64(sna_crtc, seq, &msc)) {
++ DBG(("%s: recording last swap on pipe=%d, frame %d [msc=%08lld], time %d.%06d\n",
++ __FUNCTION__, __sna_crtc_pipe(sna_crtc), seq, (long long)msc,
++ tv_sec, tv_usec));
++ sna_crtc->swap.tv_sec = tv_sec;
++ sna_crtc->swap.tv_usec = tv_usec;
++ sna_crtc->swap.msc = msc;
++ } else {
++ DBG(("%s: swap event on pipe=%d, frame %d [msc=%08lld], time %d.%06d\n",
++ __FUNCTION__, __sna_crtc_pipe(sna_crtc), seq, (long long)msc,
++ tv_sec, tv_usec));
++ }
++
++ return msc;
+ }
+
+ const struct ust_msc *sna_crtc_last_swap(xf86CrtcPtr crtc)
+@@ -342,15 +550,6 @@ const struct ust_msc *sna_crtc_last_swap(xf86CrtcPtr crtc)
+ }
+ }
+
+-xf86CrtcPtr sna_mode_first_crtc(struct sna *sna)
+-{
+- xf86CrtcConfigPtr config = XF86_CRTC_CONFIG_PTR(sna->scrn);
+- if (sna->mode.num_real_crtc)
+- return config->crtc[0];
+- else
+- return NULL;
+-}
+-
+ #ifndef NDEBUG
+ static void gem_close(int fd, uint32_t handle);
+ static void assert_scanout(struct kgem *kgem, struct kgem_bo *bo,
+@@ -372,12 +571,24 @@ static void assert_scanout(struct kgem *kgem, struct kgem_bo *bo,
+ #define assert_scanout(k, b, w, h)
+ #endif
+
++static void assert_crtc_fb(struct sna *sna, struct sna_crtc *crtc)
++{
++#ifndef NDEBUG
++ struct drm_mode_crtc mode = { .crtc_id = __sna_crtc_id(crtc) };
++ drmIoctl(sna->kgem.fd, DRM_IOCTL_MODE_GETCRTC, &mode);
++ assert(mode.fb_id == fb_id(crtc->bo));
++#endif
++}
++
+ static unsigned get_fb(struct sna *sna, struct kgem_bo *bo,
+ int width, int height)
+ {
+ ScrnInfoPtr scrn = sna->scrn;
+ struct drm_mode_fb_cmd arg;
+
++ if (!kgem_bo_is_fenced(&sna->kgem, bo))
++ return 0;
++
+ assert(bo->refcnt);
+ assert(bo->proxy == NULL);
+ assert(!bo->snoop);
+@@ -393,8 +604,9 @@ static unsigned get_fb(struct sna *sna, struct kgem_bo *bo,
+ DBG(("%s: create fb %dx%d@%d/%d\n",
+ __FUNCTION__, width, height, scrn->depth, scrn->bitsPerPixel));
+
+- assert(bo->tiling != I915_TILING_Y);
++ assert(bo->tiling != I915_TILING_Y || sna->kgem.can_scanout_y);
+ assert((bo->pitch & 63) == 0);
++ assert(scrn->vtSema); /* must be master */
+
+ VG_CLEAR(arg);
+ arg.width = width;
+@@ -404,21 +616,83 @@ static unsigned get_fb(struct sna *sna, struct kgem_bo *bo,
+ arg.depth = scrn->depth;
+ arg.handle = bo->handle;
+
+- assert(sna->scrn->vtSema); /* must be master */
+ if (drmIoctl(sna->kgem.fd, DRM_IOCTL_MODE_ADDFB, &arg)) {
+- xf86DrvMsg(scrn->scrnIndex, X_ERROR,
+- "%s: failed to add fb: %dx%d depth=%d, bpp=%d, pitch=%d: %d\n",
+- __FUNCTION__, width, height,
+- scrn->depth, scrn->bitsPerPixel, bo->pitch, errno);
+- return 0;
++ /* Try again with the fancy version */
++ struct local_mode_fb_cmd2 {
++ uint32_t fb_id;
++ uint32_t width, height;
++ uint32_t pixel_format;
++ uint32_t flags;
++
++ uint32_t handles[4];
++ uint32_t pitches[4]; /* pitch for each plane */
++ uint32_t offsets[4]; /* offset of each plane */
++ uint64_t modifiers[4];
++ } f;
++#define LOCAL_IOCTL_MODE_ADDFB2 DRM_IOWR(0xb8, struct local_mode_fb_cmd2)
++ memset(&f, 0, sizeof(f));
++ f.width = width;
++ f.height = height;
++ /* XXX interlaced */
++ f.flags = 1 << 1; /* +modifiers */
++ f.handles[0] = bo->handle;
++ f.pitches[0] = bo->pitch;
++
++ switch (bo->tiling) {
++ case I915_TILING_NONE:
++ break;
++ case I915_TILING_X:
++ /* I915_FORMAT_MOD_X_TILED */
++ f.modifiers[0] = (uint64_t)1 << 56 | 1;
++ break;
++ case I915_TILING_Y:
++ /* I915_FORMAT_MOD_X_TILED */
++ f.modifiers[0] = (uint64_t)1 << 56 | 2;
++ break;
++ }
++
++#define fourcc(a,b,c,d) ((a) | (b) << 8 | (c) << 16 | (d) << 24)
++ switch (scrn->depth) {
++ default:
++ ERR(("%s: unhandled screen format, depth=%d\n",
++ __FUNCTION__, scrn->depth));
++ goto fail;
++ case 8:
++ f.pixel_format = fourcc('C', '8', ' ', ' ');
++ break;
++ case 15:
++ f.pixel_format = fourcc('X', 'R', '1', '5');
++ break;
++ case 16:
++ f.pixel_format = fourcc('R', 'G', '1', '6');
++ break;
++ case 24:
++ f.pixel_format = fourcc('X', 'R', '2', '4');
++ break;
++ case 30:
++ f.pixel_format = fourcc('X', 'R', '3', '0');
++ break;
++ }
++#undef fourcc
++
++ if (drmIoctl(sna->kgem.fd, LOCAL_IOCTL_MODE_ADDFB2, &f)) {
++fail:
++ xf86DrvMsg(scrn->scrnIndex, X_ERROR,
++ "%s: failed to add fb: %dx%d depth=%d, bpp=%d, pitch=%d: %d\n",
++ __FUNCTION__, width, height,
++ scrn->depth, scrn->bitsPerPixel, bo->pitch, errno);
++ return 0;
++ }
++
++ arg.fb_id = f.fb_id;
+ }
+ assert(arg.fb_id != 0);
+-
++ bo->delta = arg.fb_id;
+ DBG(("%s: attached fb=%d to handle=%d\n",
+- __FUNCTION__, arg.fb_id, arg.handle));
++ __FUNCTION__, bo->delta, arg.handle));
+
+ bo->scanout = true;
+- return bo->delta = arg.fb_id;
++ return bo->delta;
+ }
+
+ static uint32_t gem_create(int fd, int size)
+@@ -438,6 +712,7 @@ static uint32_t gem_create(int fd, int size)
+ static void *gem_mmap(int fd, int handle, int size)
+ {
+ struct drm_i915_gem_mmap_gtt mmap_arg;
++ struct drm_i915_gem_set_domain set_domain;
+ void *ptr;
+
+ VG_CLEAR(mmap_arg);
+@@ -449,6 +724,15 @@ static void *gem_mmap(int fd, int handle, int size)
+ if (ptr == MAP_FAILED)
+ return NULL;
+
++ VG_CLEAR(set_domain);
++ set_domain.handle = handle;
++ set_domain.read_domains = I915_GEM_DOMAIN_GTT;
++ set_domain.write_domain = I915_GEM_DOMAIN_GTT;
++ if (drmIoctl(fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &set_domain)) {
++ munmap(ptr, size);
++ return NULL;
++ }
++
+ return ptr;
+ }
+
+@@ -497,8 +781,6 @@ sna_backlight_uevent(int fd, void *closure)
+ if (sna_output->dpms_mode != DPMSModeOn)
+ continue;
+
+- assert(output->randr_output);
+-
+ val = backlight_get(&sna_output->backlight);
+ if (val < 0)
+ continue;
+@@ -523,6 +805,7 @@ sna_backlight_uevent(int fd, void *closure)
+ TRUE, FALSE);
+ }
+ }
++ DBG(("%s: complete\n", __FUNCTION__));
+ }
+
+ static void sna_backlight_pre_init(struct sna *sna)
+@@ -570,6 +853,7 @@ static void sna_backlight_drain_uevents(struct sna *sna)
+ if (sna->mode.backlight_monitor == NULL)
+ return;
+
++ DBG(("%s()\n", __FUNCTION__));
+ sna_backlight_uevent(udev_monitor_get_fd(sna->mode.backlight_monitor),
+ sna);
+ }
+@@ -632,9 +916,22 @@ sna_output_backlight_set(struct sna_output *sna_output, int level)
+ return ret;
+ }
+
++static bool
++has_native_backlight(struct sna_output *sna_output)
++{
++ return sna_output->backlight.type == BL_RAW;
++}
++
+ static void
+ sna_output_backlight_off(struct sna_output *sna_output)
+ {
++ /* Trust the kernel to turn the native backlight off. However, we
++ * do explicitly turn the backlight back on (when we wake the output)
++ * just in case a third party turns it off!
++ */
++ if (has_native_backlight(sna_output))
++ return;
++
+ DBG(("%s(%s)\n", __FUNCTION__, sna_output->base->name));
+ backlight_off(&sna_output->backlight);
+ sna_output_backlight_set(sna_output, 0);
+@@ -674,7 +971,7 @@ has_user_backlight_override(xf86OutputPtr output)
+ if (*str == '\0')
+ return (char *)str;
+
+- if (backlight_exists(str) == BL_NONE) {
++ if (!backlight_exists(str)) {
+ xf86DrvMsg(output->scrn->scrnIndex, X_ERROR,
+ "Unrecognised backlight control interface '%s'\n",
+ str);
+@@ -684,6 +981,93 @@ has_user_backlight_override(xf86OutputPtr output)
+ return strdup(str);
+ }
+
++static int get_device_minor(int fd)
++{
++ struct stat st;
++
++ if (fstat(fd, &st) || !S_ISCHR(st.st_mode))
++ return -1;
++
++ return st.st_rdev & 0x63;
++}
++
++static const char * const sysfs_connector_types[] = {
++ /* DRM_MODE_CONNECTOR_Unknown */ "Unknown",
++ /* DRM_MODE_CONNECTOR_VGA */ "VGA",
++ /* DRM_MODE_CONNECTOR_DVII */ "DVI-I",
++ /* DRM_MODE_CONNECTOR_DVID */ "DVI-D",
++ /* DRM_MODE_CONNECTOR_DVIA */ "DVI-A",
++ /* DRM_MODE_CONNECTOR_Composite */ "Composite",
++ /* DRM_MODE_CONNECTOR_SVIDEO */ "SVIDEO",
++ /* DRM_MODE_CONNECTOR_LVDS */ "LVDS",
++ /* DRM_MODE_CONNECTOR_Component */ "Component",
++ /* DRM_MODE_CONNECTOR_9PinDIN */ "DIN",
++ /* DRM_MODE_CONNECTOR_DisplayPort */ "DP",
++ /* DRM_MODE_CONNECTOR_HDMIA */ "HDMI-A",
++ /* DRM_MODE_CONNECTOR_HDMIB */ "HDMI-B",
++ /* DRM_MODE_CONNECTOR_TV */ "TV",
++ /* DRM_MODE_CONNECTOR_eDP */ "eDP",
++ /* DRM_MODE_CONNECTOR_VIRTUAL */ "Virtual",
++ /* DRM_MODE_CONNECTOR_DSI */ "DSI",
++ /* DRM_MODE_CONNECTOR_DPI */ "DPI"
++};
++
++static char *has_connector_backlight(xf86OutputPtr output)
++{
++ struct sna_output *sna_output = output->driver_private;
++ struct sna *sna = to_sna(output->scrn);
++ char path[1024];
++ DIR *dir;
++ struct dirent *de;
++ int minor, len;
++ char *str = NULL;
++
++ if (sna_output->connector_type >= ARRAY_SIZE(sysfs_connector_types))
++ return NULL;
++
++ minor = get_device_minor(sna->kgem.fd);
++ if (minor < 0)
++ return NULL;
++
++ len = snprintf(path, sizeof(path),
++ "/sys/class/drm/card%d-%s-%d",
++ minor,
++ sysfs_connector_types[sna_output->connector_type],
++ sna_output->connector_type_id);
++ DBG(("%s: lookup %s\n", __FUNCTION__, path));
++
++ dir = opendir(path);
++ if (dir == NULL)
++ return NULL;
++
++ while ((de = readdir(dir))) {
++ struct stat st;
++
++ if (*de->d_name == '.')
++ continue;
++
++ snprintf(path + len, sizeof(path) - len,
++ "/%s", de->d_name);
++
++ if (stat(path, &st))
++ continue;
++
++ if (!S_ISDIR(st.st_mode))
++ continue;
++
++ DBG(("%s: testing %s as backlight\n",
++ __FUNCTION__, de->d_name));
++
++ if (backlight_exists(de->d_name)) {
++ str = strdup(de->d_name); /* leak! */
++ break;
++ }
++ }
++
++ closedir(dir);
++ return str;
++}
++
+ static void
+ sna_output_backlight_init(xf86OutputPtr output)
+ {
+@@ -696,11 +1080,20 @@ sna_output_backlight_init(xf86OutputPtr output)
+ return;
+ #endif
+
+- from = X_CONFIG;
+- best_iface = has_user_backlight_override(output);
++ if (sna_output->is_panel) {
++ from = X_CONFIG;
++ best_iface = has_user_backlight_override(output);
++ if (best_iface)
++ goto done;
++ }
++
++ best_iface = has_connector_backlight(output);
+ if (best_iface)
+ goto done;
+
++ if (!sna_output->is_panel)
++ return;
++
+ /* XXX detect right backlight for multi-GPU/panels */
+ from = X_PROBED;
+ pci = xf86GetPciInfoForEntity(to_sna(output->scrn)->pEnt->index);
+@@ -728,6 +1121,38 @@ done:
+ sna_output->backlight.iface, best_iface, output->name);
+ }
+
++#if ABI_VIDEODRV_VERSION >= SET_ABI_VERSION(22, 0)
++static inline int sigio_block(void)
++{
++ return 0;
++}
++static inline void sigio_unblock(int was_blocked)
++{
++ (void)was_blocked;
++}
++#elif XORG_VERSION_CURRENT >= XORG_VERSION_NUMERIC(1,12,99,901,0)
++static inline int sigio_block(void)
++{
++ OsBlockSIGIO();
++ return 0;
++}
++static inline void sigio_unblock(int was_blocked)
++{
++ OsReleaseSIGIO();
++ (void)was_blocked;
++}
++#else
++#include <xf86_OSproc.h>
++static inline int sigio_block(void)
++{
++ return xf86BlockSIGIO();
++}
++static inline void sigio_unblock(int was_blocked)
++{
++ xf86UnblockSIGIO(was_blocked);
++}
++#endif
++
+ static char *canonical_kmode_name(const struct drm_mode_modeinfo *kmode)
+ {
+ char tmp[32], *buf;
+@@ -781,6 +1206,7 @@ mode_from_kmode(ScrnInfoPtr scrn,
+ mode->VTotal = kmode->vtotal;
+ mode->VScan = kmode->vscan;
+
++ mode->VRefresh = kmode->vrefresh;
+ mode->Flags = kmode->flags;
+ mode->name = get_kmode_name(kmode);
+
+@@ -814,6 +1240,7 @@ mode_to_kmode(struct drm_mode_modeinfo *kmode, DisplayModePtr mode)
+ kmode->vtotal = mode->VTotal;
+ kmode->vscan = mode->VScan;
+
++ kmode->vrefresh = mode->VRefresh;
+ kmode->flags = mode->Flags;
+ if (mode->name)
+ strncpy(kmode->name, mode->name, DRM_DISPLAY_MODE_LEN);
+@@ -824,11 +1251,12 @@ static void
+ sna_crtc_force_outputs_on(xf86CrtcPtr crtc)
+ {
+ xf86CrtcConfigPtr config = XF86_CRTC_CONFIG_PTR(crtc->scrn);
++ /* All attached outputs are valid, so update our timestamps */
++ unsigned now = GetTimeInMillis();
+ int i;
+
+ assert(to_sna_crtc(crtc));
+- DBG(("%s(pipe=%d), currently? %d\n", __FUNCTION__,
+- to_sna_crtc(crtc)->pipe, to_sna_crtc(crtc)->dpms_mode));
++ DBG(("%s(pipe=%d)\n", __FUNCTION__, sna_crtc_pipe(crtc)));
+
+ /* DPMS handling by the kernel is inconsistent, so after setting a
+ * mode on an output presume that we intend for it to be on, or that
+@@ -843,10 +1271,11 @@ sna_crtc_force_outputs_on(xf86CrtcPtr crtc)
+ if (output->crtc != crtc)
+ continue;
+
+- output->funcs->dpms(output, DPMSModeOn);
++ __sna_output_dpms(output, DPMSModeOn, false);
++ if (to_sna_output(output)->last_detect)
++ to_sna_output(output)->last_detect = now;
+ }
+
+- to_sna_crtc(crtc)->dpms_mode = DPMSModeOn;
+ #if XF86_CRTC_VERSION >= 3
+ crtc->active = TRUE;
+ #endif
+@@ -859,8 +1288,7 @@ sna_crtc_force_outputs_off(xf86CrtcPtr crtc)
+ int i;
+
+ assert(to_sna_crtc(crtc));
+- DBG(("%s(pipe=%d), currently? %d\n", __FUNCTION__,
+- to_sna_crtc(crtc)->pipe, to_sna_crtc(crtc)->dpms_mode));
++ DBG(("%s(pipe=%d)\n", __FUNCTION__, sna_crtc_pipe(crtc)));
+
+ /* DPMS handling by the kernel is inconsistent, so after setting a
+ * mode on an output presume that we intend for it to be on, or that
+@@ -875,35 +1303,47 @@ sna_crtc_force_outputs_off(xf86CrtcPtr crtc)
+ if (output->crtc != crtc)
+ continue;
+
+- output->funcs->dpms(output, DPMSModeOff);
++ __sna_output_dpms(output, DPMSModeOff, false);
+ }
+-
+- to_sna_crtc(crtc)->dpms_mode = DPMSModeOff;
+ }
+
+ static unsigned
+-rotation_reduce(struct plane *p, unsigned rotation)
++rotation_reflect(unsigned rotation)
+ {
+- unsigned unsupported_rotations = rotation & ~p->rotation.supported;
++ unsigned other_bits;
+
+- if (unsupported_rotations == 0)
+- return rotation;
++ /* paranoia for future extensions */
++ other_bits = rotation & ~RR_Rotate_All;
+
+-#define RR_Reflect_XY (RR_Reflect_X | RR_Reflect_Y)
++ /* flip the reflection to compensate for reflecting the rotation */
++ other_bits ^= RR_Reflect_X | RR_Reflect_Y;
+
+- if ((unsupported_rotations & RR_Reflect_XY) == RR_Reflect_XY &&
+- p->rotation.supported& RR_Rotate_180) {
+- rotation &= ~RR_Reflect_XY;
+- rotation ^= RR_Rotate_180;
+- }
++ /* Reflect the screen by rotating the rotation bit,
++ * which has to have at least RR_Rotate_0 set. This allows
++ * us to reflect any of the rotation bits, not just 0.
++ */
++ rotation &= RR_Rotate_All;
++ assert(rotation);
++ rotation <<= 2; /* RR_Rotate_0 -> RR_Rotate_180 etc */
++ rotation |= rotation >> 4; /* RR_Rotate_270' to RR_Rotate_90 */
++ rotation &= RR_Rotate_All;
++ assert(rotation);
+
+- if ((unsupported_rotations & RR_Rotate_180) &&
+- (p->rotation.supported& RR_Reflect_XY) == RR_Reflect_XY) {
+- rotation ^= RR_Reflect_XY;
+- rotation &= ~RR_Rotate_180;
++ return rotation | other_bits;
++}
++
++static unsigned
++rotation_reduce(struct plane *p, unsigned rotation)
++{
++ /* If unsupported try exchanging rotation for a reflection */
++ if (rotation & ~p->rotation.supported) {
++ unsigned new_rotation = rotation_reflect(rotation);
++ if ((new_rotation & p->rotation.supported) == new_rotation)
++ rotation = new_rotation;
+ }
+
+-#undef RR_Reflect_XY
++ /* Only one rotation bit should be set */
++ assert(is_power_of_two(rotation & RR_Rotate_All));
+
+ return rotation;
+ }
+@@ -923,7 +1363,7 @@ rotation_set(struct sna *sna, struct plane *p, uint32_t desired)
+ if (desired == p->rotation.current)
+ return true;
+
+- if ((desired & p->rotation.supported) == 0) {
++ if ((desired & p->rotation.supported) != desired) {
+ errno = EINVAL;
+ return false;
+ }
+@@ -956,20 +1396,105 @@ rotation_reset(struct plane *p)
+ p->rotation.current = 0;
+ }
+
+-bool sna_crtc_set_sprite_rotation(xf86CrtcPtr crtc, uint32_t rotation)
++bool sna_crtc_set_sprite_rotation(xf86CrtcPtr crtc,
++ unsigned idx,
++ uint32_t rotation)
+ {
++ struct plane *sprite;
+ assert(to_sna_crtc(crtc));
++
++ sprite = lookup_sprite(to_sna_crtc(crtc), idx);
++ if (!sprite)
++ return false;
++
+ DBG(("%s: CRTC:%d [pipe=%d], sprite=%u set-rotation=%x\n",
+ __FUNCTION__,
+- to_sna_crtc(crtc)->id, to_sna_crtc(crtc)->pipe, to_sna_crtc(crtc)->sprite.id,
+- rotation));
++ sna_crtc_id(crtc), sna_crtc_pipe(crtc),
++ sprite->id, rotation));
+
+- return rotation_set(to_sna(crtc->scrn),
+- &to_sna_crtc(crtc)->sprite,
+- rotation_reduce(&to_sna_crtc(crtc)->sprite, rotation));
++ return rotation_set(to_sna(crtc->scrn), sprite,
++ rotation_reduce(sprite, rotation));
+ }
+
+-static bool
++#if HAS_DEBUG_FULL
++#if !HAS_DEBUG_FULL
++#define LogF ErrorF
++#endif
++struct kmsg {
++ int fd;
++ int saved_loglevel;
++};
++
++static int kmsg_get_debug(void)
++{
++ FILE *file;
++ int v = -1;
++
++ file = fopen("/sys/module/drm/parameters/debug", "r");
++ if (file) {
++ fscanf(file, "%d", &v);
++ fclose(file);
++ }
++
++ return v;
++}
++
++static void kmsg_set_debug(int v)
++{
++ FILE *file;
++
++ file = fopen("/sys/module/drm/parameters/debug", "w");
++ if (file) {
++ fprintf(file, "%d\n", v);
++ fclose(file);
++ }
++}
++
++static void kmsg_open(struct kmsg *k)
++{
++ k->saved_loglevel = kmsg_get_debug();
++ if (k->saved_loglevel != -1)
++ kmsg_set_debug(0xff);
++
++ k->fd = open("/dev/kmsg", O_RDONLY | O_NONBLOCK);
++ if (k->fd != -1)
++ lseek(k->fd, 0, SEEK_END);
++}
++
++static void kmsg_close(struct kmsg *k, int dump)
++{
++ FILE *file;
++
++ file = NULL;
++ if (k->fd != -1 && dump)
++ file = fdopen(k->fd, "r");
++ if (file) {
++ size_t len = 0;
++ char *line = NULL;
++
++ while (getline(&line, &len, file) != -1) {
++ char *start = strchr(line, ';');
++ if (start)
++ LogF("KMSG: %s", start + 1);
++ }
++
++ free(line);
++ fclose(file);
++ }
++
++ if (k->fd != -1)
++ close(k->fd);
++
++ if (k->saved_loglevel != -1)
++ kmsg_set_debug(k->saved_loglevel);
++}
++#else
++struct kmsg { int unused; };
++static void kmsg_open(struct kmsg *k) {}
++static void kmsg_close(struct kmsg *k, int dump) {}
++#endif
++
++static int
+ sna_crtc_apply(xf86CrtcPtr crtc)
+ {
+ struct sna *sna = to_sna(crtc->scrn);
+@@ -978,26 +1503,39 @@ sna_crtc_apply(xf86CrtcPtr crtc)
+ struct drm_mode_crtc arg;
+ uint32_t output_ids[32];
+ int output_count = 0;
+- int i;
++ int sigio, i;
++ struct kmsg kmsg;
++ int ret = EINVAL;
+
+- DBG(("%s CRTC:%d [pipe=%d], handle=%d\n", __FUNCTION__, sna_crtc->id, sna_crtc->pipe, sna_crtc->bo->handle));
++ DBG(("%s CRTC:%d [pipe=%d], handle=%d\n", __FUNCTION__,
++ __sna_crtc_id(sna_crtc), __sna_crtc_pipe(sna_crtc),
++ sna_crtc->bo->handle));
+ if (!sna_crtc->kmode.clock) {
+ ERR(("%s(CRTC:%d [pipe=%d]): attempted to set an invalid mode\n",
+- __FUNCTION__, sna_crtc->id, sna_crtc->pipe));
+- return false;
++ __FUNCTION__, __sna_crtc_id(sna_crtc), __sna_crtc_pipe(sna_crtc)));
++ return EINVAL;
+ }
+
++ kmsg_open(&kmsg);
++ sigio = sigio_block();
++
+ assert(sna->mode.num_real_output < ARRAY_SIZE(output_ids));
+ sna_crtc_disable_cursor(sna, sna_crtc);
+
+ if (!rotation_set(sna, &sna_crtc->primary, sna_crtc->rotation)) {
++ memset(&arg, 0, sizeof(arg));
++ arg.crtc_id = __sna_crtc_id(sna_crtc);
++ (void)drmIoctl(sna->kgem.fd, DRM_IOCTL_MODE_SETCRTC, &arg);
++ }
++
++ if (!rotation_set(sna, &sna_crtc->primary, sna_crtc->rotation)) {
+ ERR(("%s: set-primary-rotation failed (rotation-id=%d, rotation=%d) on CRTC:%d [pipe=%d], errno=%d\n",
+- __FUNCTION__, sna_crtc->primary.rotation.prop, sna_crtc->rotation, sna_crtc->id, sna_crtc->pipe, errno));
++ __FUNCTION__, sna_crtc->primary.rotation.prop, sna_crtc->rotation, __sna_crtc_id(sna_crtc), __sna_crtc_pipe(sna_crtc), errno));
+ sna_crtc->primary.rotation.supported &= ~sna_crtc->rotation;
+- return false;
++ goto unblock;
+ }
+ DBG(("%s: CRTC:%d [pipe=%d] primary rotation set to %x\n",
+- __FUNCTION__, sna_crtc->id, sna_crtc->pipe, sna_crtc->rotation));
++ __FUNCTION__, __sna_crtc_id(sna_crtc), __sna_crtc_pipe(sna_crtc), sna_crtc->rotation));
+
+ for (i = 0; i < sna->mode.num_real_output; i++) {
+ xf86OutputPtr output = config->output[i];
+@@ -1008,7 +1546,7 @@ sna_crtc_apply(xf86CrtcPtr crtc)
+ * and we lose track of the user settings.
+ */
+ if (output->crtc == NULL)
+- output->funcs->dpms(output, DPMSModeOff);
++ __sna_output_dpms(output, DPMSModeOff, false);
+
+ if (output->crtc != crtc)
+ continue;
+@@ -1022,29 +1560,27 @@ sna_crtc_apply(xf86CrtcPtr crtc)
+
+ DBG(("%s: attaching output '%s' %d [%d] to crtc:%d (pipe %d) (possible crtc:%x, possible clones:%x)\n",
+ __FUNCTION__, output->name, i, to_connector_id(output),
+- sna_crtc->id, sna_crtc->pipe,
++ __sna_crtc_id(sna_crtc), __sna_crtc_pipe(sna_crtc),
+ (uint32_t)output->possible_crtcs,
+ (uint32_t)output->possible_clones));
+
+- assert(output->possible_crtcs & (1 << sna_crtc->pipe) ||
++ assert(output->possible_crtcs & (1 << __sna_crtc_pipe(sna_crtc)) ||
+ is_zaphod(crtc->scrn));
+
+ output_ids[output_count] = to_connector_id(output);
+ if (++output_count == ARRAY_SIZE(output_ids)) {
+ DBG(("%s: too many outputs (%d) for me!\n",
+ __FUNCTION__, output_count));
+- errno = EINVAL;
+- return false;
++ goto unblock;
+ }
+ }
+ if (output_count == 0) {
+ DBG(("%s: no outputs\n", __FUNCTION__));
+- errno = EINVAL;
+- return false;
++ goto unblock;
+ }
+
+ VG_CLEAR(arg);
+- arg.crtc_id = sna_crtc->id;
++ arg.crtc_id = __sna_crtc_id(sna_crtc);
+ arg.fb_id = fb_id(sna_crtc->bo);
+ if (sna_crtc->transform || sna_crtc->slave_pixmap) {
+ arg.x = 0;
+@@ -1061,7 +1597,7 @@ sna_crtc_apply(xf86CrtcPtr crtc)
+ arg.mode_valid = 1;
+
+ DBG(("%s: applying crtc [%d, pipe=%d] mode=%dx%d+%d+%d@%d, fb=%d%s%s update to %d outputs [%d...]\n",
+- __FUNCTION__, sna_crtc->id, sna_crtc->pipe,
++ __FUNCTION__, __sna_crtc_id(sna_crtc), __sna_crtc_pipe(sna_crtc),
+ arg.mode.hdisplay,
+ arg.mode.vdisplay,
+ arg.x, arg.y,
+@@ -1071,12 +1607,19 @@ sna_crtc_apply(xf86CrtcPtr crtc)
+ sna_crtc->transform ? " [transformed]" : "",
+ output_count, output_count ? output_ids[0] : 0));
+
+- if (drmIoctl(sna->kgem.fd, DRM_IOCTL_MODE_SETCRTC, &arg))
+- return false;
++ ret = 0;
++ if (unlikely(drmIoctl(sna->kgem.fd, DRM_IOCTL_MODE_SETCRTC, &arg))) {
++ ret = errno;
++ goto unblock;
++ }
+
+ sna_crtc->mode_serial++;
+ sna_crtc_force_outputs_on(crtc);
+- return true;
++
++unblock:
++ sigio_unblock(sigio);
++ kmsg_close(&kmsg, ret);
++ return ret;
+ }
+
+ static bool overlap(const BoxRec *a, const BoxRec *b)
+@@ -1094,26 +1637,73 @@ static bool overlap(const BoxRec *a, const BoxRec *b)
+ return true;
+ }
+
++static void defer_event(struct sna *sna, struct drm_event *base)
++{
++ if (sna->mode.shadow_nevent == sna->mode.shadow_size) {
++ int size = sna->mode.shadow_size * 2;
++ void *ptr;
++
++ ptr = realloc(sna->mode.shadow_events,
++ sizeof(struct drm_event_vblank)*size);
++ if (!ptr)
++ return;
++
++ sna->mode.shadow_events = ptr;
++ sna->mode.shadow_size = size;
++ }
++
++ memcpy(&sna->mode.shadow_events[sna->mode.shadow_nevent++],
++ base, sizeof(struct drm_event_vblank));
++ DBG(("%s: deferring event count=%d\n",
++ __func__, sna->mode.shadow_nevent));
++}
++
++static void flush_events(struct sna *sna)
++{
++ int n;
++
++ if (!sna->mode.shadow_nevent)
++ return;
++
++ DBG(("%s: flushing %d events=%d\n", __func__, sna->mode.shadow_nevent));
++
++ for (n = 0; n < sna->mode.shadow_nevent; n++) {
++ struct drm_event_vblank *vb = &sna->mode.shadow_events[n];
++
++ if ((uintptr_t)(vb->user_data) & 2)
++ sna_present_vblank_handler(vb);
++ else
++ sna_dri2_vblank_handler(vb);
++ }
++
++ sna->mode.shadow_nevent = 0;
++}
++
++
+ static bool wait_for_shadow(struct sna *sna,
+ struct sna_pixmap *priv,
+ unsigned flags)
+ {
+ PixmapPtr pixmap = priv->pixmap;
+- DamagePtr damage;
+ struct kgem_bo *bo, *tmp;
+ int flip_active;
+ bool ret = true;
+
+- DBG(("%s: flags=%x, flips=%d, handle=%d, shadow=%d\n",
+- __FUNCTION__, flags, sna->mode.flip_active,
++ DBG(("%s: enabled? %d waiting? %d, flags=%x, flips=%d, pixmap=%ld [front?=%d], handle=%d, shadow=%d\n",
++ __FUNCTION__, sna->mode.shadow_enabled, sna->mode.shadow_wait,
++ flags, sna->mode.flip_active,
++ pixmap->drawable.serialNumber, pixmap == sna->front,
+ priv->gpu_bo->handle, sna->mode.shadow->handle));
+
+ assert(priv->move_to_gpu_data == sna);
+ assert(sna->mode.shadow != priv->gpu_bo);
+
+- if (flags == 0 || pixmap != sna->front || !sna->mode.shadow_damage)
++ if (flags == 0 || pixmap != sna->front || !sna->mode.shadow_enabled)
+ goto done;
+
++ assert(sna->mode.shadow_damage);
++ assert(!sna->mode.shadow_wait);
++
+ if ((flags & MOVE_WRITE) == 0) {
+ if ((flags & __MOVE_SCANOUT) == 0) {
+ struct sna_crtc *crtc;
+@@ -1154,9 +1744,7 @@ static bool wait_for_shadow(struct sna *sna,
+ }
+
+ assert(sna->mode.shadow_active);
+-
+- damage = sna->mode.shadow_damage;
+- sna->mode.shadow_damage = NULL;
++ sna->mode.shadow_wait = true;
+
+ flip_active = sna->mode.flip_active;
+ if (flip_active) {
+@@ -1208,6 +1796,8 @@ static bool wait_for_shadow(struct sna *sna,
+ bo = sna->mode.shadow;
+ }
+ }
++ assert(sna->mode.shadow_wait);
++ sna->mode.shadow_wait = false;
+
+ if (bo->refcnt > 1) {
+ bo = kgem_create_2d(&sna->kgem,
+@@ -1230,8 +1820,6 @@ static bool wait_for_shadow(struct sna *sna,
+ bo = sna->mode.shadow;
+ }
+
+- sna->mode.shadow_damage = damage;
+-
+ RegionSubtract(&sna->mode.shadow_region,
+ &sna->mode.shadow_region,
+ &sna->mode.shadow_cancel);
+@@ -1269,6 +1857,7 @@ static bool wait_for_shadow(struct sna *sna,
+ RegionSubtract(&sna->mode.shadow_region, &sna->mode.shadow_region, &region);
+ }
+
++ crtc->client_bo->active_scanout--;
+ kgem_bo_destroy(&sna->kgem, crtc->client_bo);
+ crtc->client_bo = NULL;
+ list_del(&crtc->shadow_link);
+@@ -1281,12 +1870,13 @@ static bool wait_for_shadow(struct sna *sna,
+ sna->mode.shadow_region.extents.y1,
+ sna->mode.shadow_region.extents.x2,
+ sna->mode.shadow_region.extents.y2));
+- ret = sna->render.copy_boxes(sna, GXcopy,
+- &pixmap->drawable, priv->gpu_bo, 0, 0,
+- &pixmap->drawable, bo, 0, 0,
+- region_rects(&sna->mode.shadow_region),
+- region_num_rects(&sna->mode.shadow_region),
+- 0);
++ if (!sna->render.copy_boxes(sna, GXcopy,
++ &pixmap->drawable, priv->gpu_bo, 0, 0,
++ &pixmap->drawable, bo, 0, 0,
++ region_rects(&sna->mode.shadow_region),
++ region_num_rects(&sna->mode.shadow_region),
++ 0))
++ ERR(("%s: copy failed\n", __FUNCTION__));
+ }
+
+ if (priv->cow)
+@@ -1295,11 +1885,13 @@ static bool wait_for_shadow(struct sna *sna,
+ sna_pixmap_unmap(pixmap, priv);
+
+ DBG(("%s: setting front pixmap to handle=%d\n", __FUNCTION__, bo->handle));
++ sna->mode.shadow->active_scanout--;
+ tmp = priv->gpu_bo;
+ priv->gpu_bo = bo;
+ if (bo != sna->mode.shadow)
+ kgem_bo_destroy(&sna->kgem, sna->mode.shadow);
+ sna->mode.shadow = tmp;
++ sna->mode.shadow->active_scanout++;
+
+ sna_dri2_pixmap_update_bo(sna, pixmap, bo);
+
+@@ -1311,6 +1903,9 @@ done:
+ priv->move_to_gpu_data = NULL;
+ priv->move_to_gpu = NULL;
+
++ assert(!sna->mode.shadow_wait);
++ flush_events(sna);
++
+ return ret;
+ }
+
+@@ -1358,22 +1953,43 @@ bool sna_pixmap_discard_shadow_damage(struct sna_pixmap *priv,
+ return RegionNil(&sna->mode.shadow_region);
+ }
+
++static void sna_mode_damage(DamagePtr damage, RegionPtr region, void *closure)
++{
++ struct sna *sna = closure;
++
++ if (sna->mode.rr_active)
++ return;
++
++ /* Throw away the rectangles if the region grows too big */
++ region = DamageRegion(damage);
++ if (region->data) {
++ RegionRec dup;
++
++ dup = *region;
++ RegionUninit(&dup);
++
++ region->data = NULL;
++ }
++}
++
+ static bool sna_mode_enable_shadow(struct sna *sna)
+ {
+- ScreenPtr screen = sna->scrn->pScreen;
++ ScreenPtr screen = to_screen_from_sna(sna);
+
+ DBG(("%s\n", __FUNCTION__));
+ assert(sna->mode.shadow == NULL);
+ assert(sna->mode.shadow_damage == NULL);
+ assert(sna->mode.shadow_active == 0);
++ assert(!sna->mode.shadow_enabled);
+
+- sna->mode.shadow_damage = DamageCreate(NULL, NULL,
+- DamageReportNone, TRUE,
+- screen, screen);
++ sna->mode.shadow_damage = DamageCreate(sna_mode_damage, NULL,
++ DamageReportRawRegion,
++ TRUE, screen, sna);
+ if (!sna->mode.shadow_damage)
+ return false;
+
+ DamageRegister(&sna->front->drawable, sna->mode.shadow_damage);
++ sna->mode.shadow_enabled = true;
+ return true;
+ }
+
+@@ -1381,8 +1997,10 @@ static void sna_mode_disable_shadow(struct sna *sna)
+ {
+ struct sna_pixmap *priv;
+
+- if (!sna->mode.shadow_damage)
++ if (!sna->mode.shadow_damage) {
++ assert(!sna->mode.shadow_enabled);
+ return;
++ }
+
+ DBG(("%s\n", __FUNCTION__));
+
+@@ -1393,8 +2011,10 @@ static void sna_mode_disable_shadow(struct sna *sna)
+ DamageUnregister(&sna->front->drawable, sna->mode.shadow_damage);
+ DamageDestroy(sna->mode.shadow_damage);
+ sna->mode.shadow_damage = NULL;
++ sna->mode.shadow_enabled = false;
+
+ if (sna->mode.shadow) {
++ sna->mode.shadow->active_scanout--;
+ kgem_bo_destroy(&sna->kgem, sna->mode.shadow);
+ sna->mode.shadow = NULL;
+ }
+@@ -1413,7 +2033,7 @@ static void sna_crtc_slave_damage(DamagePtr damage, RegionPtr region, void *clos
+ __FUNCTION__,
+ region->extents.x1, region->extents.y1, region->extents.x2, region->extents.y2,
+ region_num_rects(region),
+- crtc->pipe, crtc->base->x, crtc->base->y));
++ __sna_crtc_pipe(crtc), crtc->base->x, crtc->base->y));
+
+ assert(crtc->slave_damage == damage);
+ assert(sna->mode.shadow_damage);
+@@ -1431,7 +2051,7 @@ static bool sna_crtc_enable_shadow(struct sna *sna, struct sna_crtc *crtc)
+ return true;
+ }
+
+- DBG(("%s: enabling for crtc %d\n", __FUNCTION__, crtc->id));
++ DBG(("%s: enabling for crtc %d\n", __FUNCTION__, __sna_crtc_id(crtc)));
+
+ if (!sna->mode.shadow_active) {
+ if (!sna_mode_enable_shadow(sna))
+@@ -1443,9 +2063,12 @@ static bool sna_crtc_enable_shadow(struct sna *sna, struct sna_crtc *crtc)
+ if (crtc->slave_pixmap) {
+ assert(crtc->slave_damage == NULL);
+
++ DBG(("%s: enabling PRIME slave tracking on CRTC %d [pipe=%d], pixmap=%ld\n",
++ __FUNCTION__, __sna_crtc_id(crtc), __sna_crtc_pipe(crtc), crtc->slave_pixmap->drawable.serialNumber));
+ crtc->slave_damage = DamageCreate(sna_crtc_slave_damage, NULL,
+ DamageReportRawRegion, TRUE,
+- sna->scrn->pScreen, crtc);
++ to_screen_from_sna(sna),
++ crtc);
+ if (crtc->slave_damage == NULL) {
+ if (!--sna->mode.shadow_active)
+ sna_mode_disable_shadow(sna);
+@@ -1465,6 +2088,9 @@ static void sna_crtc_disable_override(struct sna *sna, struct sna_crtc *crtc)
+ if (crtc->client_bo == NULL)
+ return;
+
++ assert(crtc->client_bo->refcnt >= crtc->client_bo->active_scanout);
++ crtc->client_bo->active_scanout--;
++
+ if (!crtc->transform) {
+ DrawableRec tmp;
+
+@@ -1489,7 +2115,7 @@ static void sna_crtc_disable_shadow(struct sna *sna, struct sna_crtc *crtc)
+ if (!crtc->shadow)
+ return;
+
+- DBG(("%s: disabling for crtc %d\n", __FUNCTION__, crtc->id));
++ DBG(("%s: disabling for crtc %d\n", __FUNCTION__, __sna_crtc_id(crtc)));
+ assert(sna->mode.shadow_active > 0);
+
+ if (crtc->slave_damage) {
+@@ -1517,14 +2143,24 @@ __sna_crtc_disable(struct sna *sna, struct sna_crtc *sna_crtc)
+ sna_crtc_disable_shadow(sna, sna_crtc);
+
+ if (sna_crtc->bo) {
++ DBG(("%s: releasing handle=%d from scanout, active=%d\n",
++ __FUNCTION__,sna_crtc->bo->handle, sna_crtc->bo->active_scanout-1));
++ assert(sna_crtc->flags & CRTC_ON);
+ assert(sna_crtc->bo->active_scanout);
+ assert(sna_crtc->bo->refcnt >= sna_crtc->bo->active_scanout);
+ sna_crtc->bo->active_scanout--;
+ kgem_bo_destroy(&sna->kgem, sna_crtc->bo);
+ sna_crtc->bo = NULL;
++ sna_crtc->flags &= ~CRTC_ON;
+
+- assert(sna->mode.front_active);
+- sna->mode.front_active--;
++ if (sna->mode.hidden) {
++ sna->mode.hidden--;
++ assert(sna->mode.hidden);
++ assert(sna->mode.front_active == 0);
++ } else {
++ assert(sna->mode.front_active);
++ sna->mode.front_active--;
++ }
+ sna->mode.dirty = true;
+ }
+
+@@ -1532,13 +2168,19 @@ __sna_crtc_disable(struct sna *sna, struct sna_crtc *sna_crtc)
+ kgem_bo_destroy(&sna->kgem, sna_crtc->shadow_bo);
+ sna_crtc->shadow_bo = NULL;
+ }
+- sna_crtc->transform = false;
++ if (sna_crtc->transform) {
++ assert(sna->mode.rr_active);
++ sna->mode.rr_active--;
++ sna_crtc->transform = false;
++ }
+
++ sna_crtc->cursor_transform = false;
++ sna_crtc->hwcursor = true;
+ assert(!sna_crtc->shadow);
+ }
+
+ static void
+-sna_crtc_disable(xf86CrtcPtr crtc)
++sna_crtc_disable(xf86CrtcPtr crtc, bool force)
+ {
+ struct sna *sna = to_sna(crtc->scrn);
+ struct sna_crtc *sna_crtc = to_sna_crtc(crtc);
+@@ -1547,14 +2189,16 @@ sna_crtc_disable(xf86CrtcPtr crtc)
+ if (sna_crtc == NULL)
+ return;
+
+- DBG(("%s: disabling crtc [%d, pipe=%d]\n", __FUNCTION__,
+- sna_crtc->id, sna_crtc->pipe));
++ if (!force && sna_crtc->bo == NULL)
++ return;
++
++ DBG(("%s: disabling crtc [%d, pipe=%d], force?=%d\n", __FUNCTION__,
++ __sna_crtc_id(sna_crtc), __sna_crtc_pipe(sna_crtc), force));
+
+ sna_crtc_force_outputs_off(crtc);
+- assert(sna_crtc->dpms_mode == DPMSModeOff);
+
+ memset(&arg, 0, sizeof(arg));
+- arg.crtc_id = sna_crtc->id;
++ arg.crtc_id = __sna_crtc_id(sna_crtc);
+ (void)drmIoctl(sna->kgem.fd, DRM_IOCTL_MODE_SETCRTC, &arg);
+
+ __sna_crtc_disable(sna, sna_crtc);
+@@ -1574,19 +2218,19 @@ static void update_flush_interval(struct sna *sna)
+
+ if (!crtc->enabled) {
+ DBG(("%s: CRTC:%d (pipe %d) disabled\n",
+- __FUNCTION__,i, to_sna_crtc(crtc)->pipe));
++ __FUNCTION__,i, sna_crtc_pipe(crtc)));
+ assert(to_sna_crtc(crtc)->bo == NULL);
+ continue;
+ }
+
+- if (to_sna_crtc(crtc)->dpms_mode != DPMSModeOn) {
++ if (to_sna_crtc(crtc)->bo == NULL) {
+ DBG(("%s: CRTC:%d (pipe %d) turned off\n",
+- __FUNCTION__,i, to_sna_crtc(crtc)->pipe));
++ __FUNCTION__,i, sna_crtc_pipe(crtc)));
+ continue;
+ }
+
+ DBG(("%s: CRTC:%d (pipe %d) vrefresh=%f\n",
+- __FUNCTION__, i, to_sna_crtc(crtc)->pipe,
++ __FUNCTION__, i, sna_crtc_pipe(crtc),
+ xf86ModeVRefresh(&crtc->mode)));
+ max_vrefresh = max(max_vrefresh, xf86ModeVRefresh(&crtc->mode));
+ }
+@@ -1642,7 +2286,7 @@ void sna_copy_fbcon(struct sna *sna)
+ int dx, dy;
+ int i;
+
+- if (wedged(sna))
++ if (wedged(sna) || isGPU(sna->scrn))
+ return;
+
+ DBG(("%s\n", __FUNCTION__));
+@@ -1662,7 +2306,7 @@ void sna_copy_fbcon(struct sna *sna)
+ assert(crtc != NULL);
+
+ VG_CLEAR(mode);
+- mode.crtc_id = crtc->id;
++ mode.crtc_id = __sna_crtc_id(crtc);
+ if (drmIoctl(sna->kgem.fd, DRM_IOCTL_MODE_GETCRTC, &mode))
+ continue;
+ if (!mode.fb_id)
+@@ -1726,7 +2370,7 @@ void sna_copy_fbcon(struct sna *sna)
+ kgem_bo_destroy(&sna->kgem, bo);
+
+ #if ABI_VIDEODRV_VERSION >= SET_ABI_VERSION(10, 0)
+- sna->scrn->pScreen->canDoBGNoneRoot = ok;
++ to_screen_from_sna(sna)->canDoBGNoneRoot = ok;
+ #endif
+ }
+
+@@ -1736,7 +2380,6 @@ static bool use_shadow(struct sna *sna, xf86CrtcPtr crtc)
+ PictTransform crtc_to_fb;
+ struct pict_f_transform f_crtc_to_fb, f_fb_to_crtc;
+ unsigned pitch_limit;
+- struct sna_pixmap *priv;
+ BoxRec b;
+
+ assert(sna->scrn->virtualX && sna->scrn->virtualY);
+@@ -1765,27 +2408,31 @@ static bool use_shadow(struct sna *sna, xf86CrtcPtr crtc)
+ return true;
+ }
+
+- priv = sna_pixmap_force_to_gpu(sna->front, MOVE_READ | __MOVE_SCANOUT);
+- if (priv == NULL)
+- return true; /* maybe we can create a bo for the scanout? */
+-
+- if (sna->kgem.gen == 071)
+- pitch_limit = priv->gpu_bo->tiling ? 16 * 1024 : 32 * 1024;
+- else if ((sna->kgem.gen >> 3) > 4)
+- pitch_limit = 32 * 1024;
+- else if ((sna->kgem.gen >> 3) == 4)
+- pitch_limit = priv->gpu_bo->tiling ? 16 * 1024 : 32 * 1024;
+- else if ((sna->kgem.gen >> 3) == 3)
+- pitch_limit = priv->gpu_bo->tiling ? 8 * 1024 : 16 * 1024;
+- else
+- pitch_limit = 8 * 1024;
+- DBG(("%s: gpu bo handle=%d tiling=%d pitch=%d, limit=%d\n", __FUNCTION__, priv->gpu_bo->handle, priv->gpu_bo->tiling, priv->gpu_bo->pitch, pitch_limit));
+- if (priv->gpu_bo->pitch > pitch_limit)
+- return true;
++ if (!isGPU(sna->scrn)) {
++ struct sna_pixmap *priv;
+
+- if (priv->gpu_bo->tiling && sna->flags & SNA_LINEAR_FB) {
+- DBG(("%s: gpu bo is tiled, need linear, forcing shadow\n", __FUNCTION__));
+- return true;
++ priv = sna_pixmap_force_to_gpu(sna->front, MOVE_READ | __MOVE_SCANOUT);
++ if (priv == NULL)
++ return true; /* maybe we can create a bo for the scanout? */
++
++ if (sna->kgem.gen == 071)
++ pitch_limit = priv->gpu_bo->tiling ? 16 * 1024 : 32 * 1024;
++ else if ((sna->kgem.gen >> 3) > 4)
++ pitch_limit = 32 * 1024;
++ else if ((sna->kgem.gen >> 3) == 4)
++ pitch_limit = priv->gpu_bo->tiling ? 16 * 1024 : 32 * 1024;
++ else if ((sna->kgem.gen >> 3) == 3)
++ pitch_limit = priv->gpu_bo->tiling ? 8 * 1024 : 16 * 1024;
++ else
++ pitch_limit = 8 * 1024;
++ DBG(("%s: gpu bo handle=%d tiling=%d pitch=%d, limit=%d\n", __FUNCTION__, priv->gpu_bo->handle, priv->gpu_bo->tiling, priv->gpu_bo->pitch, pitch_limit));
++ if (priv->gpu_bo->pitch > pitch_limit)
++ return true;
++
++ if (priv->gpu_bo->tiling && sna->flags & SNA_LINEAR_FB) {
++ DBG(("%s: gpu bo is tiled, need linear, forcing shadow\n", __FUNCTION__));
++ return true;
++ }
+ }
+
+ transform = NULL;
+@@ -1800,9 +2447,9 @@ static bool use_shadow(struct sna *sna, xf86CrtcPtr crtc)
+ bool needs_transform = true;
+ unsigned rotation = rotation_reduce(&to_sna_crtc(crtc)->primary, crtc->rotation);
+ DBG(("%s: natively supported rotation? rotation=%x & supported=%x == %d\n",
+- __FUNCTION__, crtc->rotation, to_sna_crtc(crtc)->primary.rotation.supported,
+- !!(crtc->rotation & to_sna_crtc(crtc)->primary.rotation.supported)));
+- if (to_sna_crtc(crtc)->primary.rotation.supported & rotation)
++ __FUNCTION__, rotation, to_sna_crtc(crtc)->primary.rotation.supported,
++ rotation == (rotation & to_sna_crtc(crtc)->primary.rotation.supported)));
++ if ((to_sna_crtc(crtc)->primary.rotation.supported & rotation) == rotation)
+ needs_transform = RRTransformCompute(crtc->x, crtc->y,
+ crtc->mode.HDisplay, crtc->mode.VDisplay,
+ RR_Rotate_0, transform,
+@@ -1839,6 +2486,7 @@ static void set_shadow(struct sna *sna, RegionPtr region)
+
+ assert(priv->gpu_bo);
+ assert(sna->mode.shadow);
++ assert(sna->mode.shadow->active_scanout);
+
+ DBG(("%s: waiting for region %dx[(%d, %d), (%d, %d)], front handle=%d, shadow handle=%d\n",
+ __FUNCTION__,
+@@ -1912,6 +2560,28 @@ get_scanout_bo(struct sna *sna, PixmapPtr pixmap)
+ return priv->gpu_bo;
+ }
+
++static void shadow_clear(struct sna *sna,
++ PixmapPtr front, struct kgem_bo *bo,
++ xf86CrtcPtr crtc)
++{
++ bool ok = false;
++ if (!wedged(sna))
++ ok = sna->render.fill_one(sna, front, bo, 0,
++ 0, 0, crtc->mode.HDisplay, crtc->mode.VDisplay,
++ GXclear);
++ if (!ok) {
++ void *ptr = kgem_bo_map__gtt(&sna->kgem, bo);
++ if (ptr)
++ memset(ptr, 0, bo->pitch * crtc->mode.HDisplay);
++ }
++ sna->mode.shadow_dirty = true;
++}
++
++static bool rr_active(xf86CrtcPtr crtc)
++{
++ return crtc->transformPresent || crtc->rotation != RR_Rotate_0;
++}
++
+ static struct kgem_bo *sna_crtc_attach(xf86CrtcPtr crtc)
+ {
+ struct sna_crtc *sna_crtc = to_sna_crtc(crtc);
+@@ -1919,10 +2589,15 @@ static struct kgem_bo *sna_crtc_attach(xf86CrtcPtr crtc)
+ struct sna *sna = to_sna(scrn);
+ struct kgem_bo *bo;
+
+- sna_crtc->transform = false;
++ if (sna_crtc->transform) {
++ assert(sna->mode.rr_active);
++ sna_crtc->transform = false;
++ sna->mode.rr_active--;
++ }
+ sna_crtc->rotation = RR_Rotate_0;
+
+ if (use_shadow(sna, crtc)) {
++ PixmapPtr front;
+ unsigned long tiled_limit;
+ int tiling;
+
+@@ -1949,6 +2624,10 @@ force_shadow:
+ }
+
+ tiling = I915_TILING_X;
++ if (crtc->rotation & (RR_Rotate_90 | RR_Rotate_270) &&
++ sna->kgem.can_scanout_y)
++ tiling = I915_TILING_Y;
++
+ if (sna->kgem.gen == 071)
+ tiled_limit = 16 * 1024 * 8;
+ else if ((sna->kgem.gen >> 3) > 4)
+@@ -1977,8 +2656,8 @@ force_shadow:
+ return NULL;
+ }
+
+- if (__sna_pixmap_get_bo(sna->front) && !crtc->transformPresent) {
+- DrawableRec tmp;
++ front = sna_crtc->slave_pixmap ?: sna->front;
++ if (__sna_pixmap_get_bo(front) && !rr_active(crtc)) {
+ BoxRec b;
+
+ b.x1 = crtc->x;
+@@ -1986,28 +2665,48 @@ force_shadow:
+ b.x2 = crtc->x + crtc->mode.HDisplay;
+ b.y2 = crtc->y + crtc->mode.VDisplay;
+
+- DBG(("%s: copying onto shadow CRTC: (%d, %d)x(%d, %d), handle=%d\n",
+- __FUNCTION__,
+- b.x1, b.y1,
+- b.x2, b.y2,
+- bo->handle));
+-
+- tmp.width = crtc->mode.HDisplay;
+- tmp.height = crtc->mode.VDisplay;
+- tmp.depth = sna->front->drawable.depth;
+- tmp.bitsPerPixel = sna->front->drawable.bitsPerPixel;
+-
+- (void)sna->render.copy_boxes(sna, GXcopy,
+- &sna->front->drawable, __sna_pixmap_get_bo(sna->front), 0, 0,
+- &tmp, bo, -b.x1, -b.y1,
+- &b, 1, 0);
+- }
++ if (b.x1 < 0)
++ b.x1 = 0;
++ if (b.y1 < 0)
++ b.y1 = 0;
++ if (b.x2 > scrn->virtualX)
++ b.x2 = scrn->virtualX;
++ if (b.y2 > scrn->virtualY)
++ b.y2 = scrn->virtualY;
++ if (b.x2 - b.x1 < crtc->mode.HDisplay ||
++ b.y2 - b.y1 < crtc->mode.VDisplay)
++ shadow_clear(sna, front, bo, crtc);
++
++ if (b.y2 > b.y1 && b.x2 > b.x1) {
++ DrawableRec tmp;
++
++ DBG(("%s: copying onto shadow CRTC: (%d, %d)x(%d, %d) [fb=%dx%d], handle=%d\n",
++ __FUNCTION__,
++ b.x1, b.y1,
++ b.x2-b.x1, b.y2-b.y1,
++ scrn->virtualX, scrn->virtualY,
++ bo->handle));
++
++ tmp.width = crtc->mode.HDisplay;
++ tmp.height = crtc->mode.VDisplay;
++ tmp.depth = front->drawable.depth;
++ tmp.bitsPerPixel = front->drawable.bitsPerPixel;
++
++ if (!sna->render.copy_boxes(sna, GXcopy,
++ &front->drawable, __sna_pixmap_get_bo(front), 0, 0,
++ &tmp, bo, -crtc->x, -crtc->y,
++ &b, 1, COPY_LAST))
++ shadow_clear(sna, front, bo, crtc);
++ }
++ } else
++ shadow_clear(sna, front, bo, crtc);
+
+ sna_crtc->shadow_bo_width = crtc->mode.HDisplay;
+ sna_crtc->shadow_bo_height = crtc->mode.VDisplay;
+ sna_crtc->shadow_bo = bo;
+ out_shadow:
+ sna_crtc->transform = true;
++ sna->mode.rr_active++;
+ return kgem_bo_reference(bo);
+ } else {
+ if (sna_crtc->shadow_bo) {
+@@ -2048,26 +2747,26 @@ out_shadow:
+ }
+
+ if (sna->flags & SNA_TEAR_FREE) {
++ RegionRec region;
++
+ assert(sna_crtc->slave_pixmap == NULL);
+
+ DBG(("%s: enabling TearFree shadow\n", __FUNCTION__));
++ region.extents.x1 = 0;
++ region.extents.y1 = 0;
++ region.extents.x2 = sna->scrn->virtualX;
++ region.extents.y2 = sna->scrn->virtualY;
++ region.data = NULL;
++
+ if (!sna_crtc_enable_shadow(sna, sna_crtc)) {
+ DBG(("%s: failed to enable crtc shadow\n", __FUNCTION__));
+ return NULL;
+ }
+
+- if (sna->mode.shadow == NULL && !wedged(sna)) {
+- RegionRec region;
++ if (sna->mode.shadow == NULL) {
+ struct kgem_bo *shadow;
+
+ DBG(("%s: creating TearFree shadow bo\n", __FUNCTION__));
+-
+- region.extents.x1 = 0;
+- region.extents.y1 = 0;
+- region.extents.x2 = sna->scrn->virtualX;
+- region.extents.y2 = sna->scrn->virtualY;
+- region.data = NULL;
+-
+ shadow = kgem_create_2d(&sna->kgem,
+ region.extents.x2,
+ region.extents.y2,
+@@ -2093,9 +2792,12 @@ out_shadow:
+ goto force_shadow;
+ }
+
++ assert(__sna_pixmap_get_bo(sna->front) == NULL ||
++ __sna_pixmap_get_bo(sna->front)->pitch == shadow->pitch);
+ sna->mode.shadow = shadow;
+- set_shadow(sna, &region);
++ sna->mode.shadow->active_scanout++;
+ }
++ set_shadow(sna, &region);
+
+ sna_crtc_disable_override(sna, sna_crtc);
+ } else
+@@ -2107,6 +2809,37 @@ out_shadow:
+ }
+ }
+
++#define SCALING_EPSILON (1./256)
++
++static bool
++is_affine(const struct pixman_f_transform *t)
++{
++ return (fabs(t->m[2][0]) < SCALING_EPSILON &&
++ fabs(t->m[2][1]) < SCALING_EPSILON);
++}
++
++static double determinant(const struct pixman_f_transform *t)
++{
++ return t->m[0][0]*t->m[1][1] - t->m[1][0]*t->m[0][1];
++}
++
++static bool
++affine_is_pixel_exact(const struct pixman_f_transform *t)
++{
++ double det = t->m[2][2] * determinant(t);
++ if (fabs (det * det - 1.0) < SCALING_EPSILON) {
++ if (fabs(t->m[0][1]) < SCALING_EPSILON &&
++ fabs(t->m[1][0]) < SCALING_EPSILON)
++ return true;
++
++ if (fabs(t->m[0][0]) < SCALING_EPSILON &&
++ fabs(t->m[1][1]) < SCALING_EPSILON)
++ return true;
++ }
++
++ return false;
++}
++
+ static void sna_crtc_randr(xf86CrtcPtr crtc)
+ {
+ struct sna_crtc *sna_crtc = to_sna_crtc(crtc);
+@@ -2152,6 +2885,25 @@ static void sna_crtc_randr(xf86CrtcPtr crtc)
+ } else
+ crtc->transform_in_use = sna_crtc->rotation != RR_Rotate_0;
+
++ /* Recompute the cursor after a potential change in transform */
++ if (sna_crtc->cursor) {
++ assert(sna_crtc->cursor->ref > 0);
++ sna_crtc->cursor->ref--;
++ sna_crtc->cursor = NULL;
++ }
++
++ if (needs_transform) {
++ sna_crtc->hwcursor = is_affine(&f_fb_to_crtc);
++ sna_crtc->cursor_transform =
++ sna_crtc->hwcursor &&
++ !affine_is_pixel_exact(&f_fb_to_crtc);
++ } else {
++ sna_crtc->hwcursor = true;
++ sna_crtc->cursor_transform = false;
++ }
++ DBG(("%s: hwcursor?=%d, cursor_transform?=%d\n",
++ __FUNCTION__, sna_crtc->hwcursor, sna_crtc->cursor_transform));
++
+ crtc->crtc_to_framebuffer = crtc_to_fb;
+ crtc->f_crtc_to_framebuffer = f_crtc_to_fb;
+ crtc->f_framebuffer_to_crtc = f_fb_to_crtc;
+@@ -2184,7 +2936,7 @@ static void sna_crtc_randr(xf86CrtcPtr crtc)
+ static void
+ sna_crtc_damage(xf86CrtcPtr crtc)
+ {
+- ScreenPtr screen = crtc->scrn->pScreen;
++ ScreenPtr screen = xf86ScrnToScreen(crtc->scrn);
+ struct sna *sna = to_sna(crtc->scrn);
+ RegionRec region, *damage;
+
+@@ -2200,15 +2952,21 @@ sna_crtc_damage(xf86CrtcPtr crtc)
+ if (region.extents.y2 > screen->height)
+ region.extents.y2 = screen->height;
+
++ if (region.extents.x2 <= region.extents.x1 ||
++ region.extents.y2 <= region.extents.y1) {
++ DBG(("%s: crtc not damaged, all-clipped\n", __FUNCTION__));
++ return;
++ }
++
+ DBG(("%s: marking crtc %d as completely damaged (%d, %d), (%d, %d)\n",
+- __FUNCTION__, to_sna_crtc(crtc)->id,
++ __FUNCTION__, sna_crtc_id(crtc),
+ region.extents.x1, region.extents.y1,
+ region.extents.x2, region.extents.y2));
+- to_sna_crtc(crtc)->client_damage = region;
+
+ assert(sna->mode.shadow_damage && sna->mode.shadow_active);
+ damage = DamageRegion(sna->mode.shadow_damage);
+ RegionUnion(damage, damage, &region);
++ to_sna_crtc(crtc)->crtc_damage = region;
+
+ DBG(("%s: damage now %dx[(%d, %d), (%d, %d)]\n",
+ __FUNCTION__,
+@@ -2260,6 +3018,21 @@ static const char *reflection_to_str(Rotation rotation)
+ }
+ }
+
++static void reprobe_connectors(xf86CrtcPtr crtc)
++{
++ xf86CrtcConfigPtr config = XF86_CRTC_CONFIG_PTR(crtc->scrn);
++ struct sna *sna = to_sna(crtc->scrn);
++ int i;
++
++ for (i = 0; i < sna->mode.num_real_output; i++) {
++ xf86OutputPtr output = config->output[i];
++ if (output->crtc == crtc)
++ to_sna_output(output)->reprobe = true;
++ }
++
++ sna_mode_discover(sna, true);
++}
++
+ static Bool
+ __sna_crtc_set_mode(xf86CrtcPtr crtc)
+ {
+@@ -2268,11 +3041,19 @@ __sna_crtc_set_mode(xf86CrtcPtr crtc)
+ struct kgem_bo *saved_bo, *bo;
+ uint32_t saved_offset;
+ bool saved_transform;
++ bool saved_hwcursor;
++ bool saved_cursor_transform;
++ int ret;
+
+- DBG(("%s\n", __FUNCTION__));
++ DBG(("%s: CRTC=%d, pipe=%d, hidden?=%d\n", __FUNCTION__,
++ __sna_crtc_id(sna_crtc), __sna_crtc_pipe(sna_crtc), sna->mode.hidden));
++ if (sna->mode.hidden)
++ return TRUE;
+
+ saved_bo = sna_crtc->bo;
+ saved_transform = sna_crtc->transform;
++ saved_cursor_transform = sna_crtc->cursor_transform;
++ saved_hwcursor = sna_crtc->hwcursor;
+ saved_offset = sna_crtc->offset;
+
+ sna_crtc->fallback_shadow = false;
+@@ -2285,26 +3066,31 @@ retry: /* Attach per-crtc pixmap or direct */
+ }
+
+ /* Prevent recursion when enabling outputs during execbuffer */
+- if (bo->exec && RQ(bo->rq)->bo == NULL)
++ if (bo->exec && RQ(bo->rq)->bo == NULL) {
+ _kgem_submit(&sna->kgem);
++ __kgem_bo_clear_dirty(bo);
++ }
+
+ sna_crtc->bo = bo;
+- if (!sna_crtc_apply(crtc)) {
+- int err = errno;
+-
++ ret = sna_crtc_apply(crtc);
++ if (ret) {
+ kgem_bo_destroy(&sna->kgem, bo);
+
+- if (!sna_crtc->shadow) {
++ if (!sna_crtc->fallback_shadow) {
+ sna_crtc->fallback_shadow = true;
+ goto retry;
+ }
+
+ xf86DrvMsg(crtc->scrn->scrnIndex, X_ERROR,
+- "failed to set mode: %s [%d]\n", strerror(err), err);
++ "failed to set mode: %s [%d]\n", strerror(ret), ret);
+ goto error;
+ }
+
++ sna_crtc->flags |= CRTC_ON;
+ bo->active_scanout++;
++ DBG(("%s: marking handle=%d as active=%d (removing %d from scanout, active=%d)\n",
++ __FUNCTION__, bo->handle, bo->active_scanout,
++ saved_bo ? saved_bo->handle : 0, saved_bo ? saved_bo->active_scanout - 1: -1));
+ if (saved_bo) {
+ assert(saved_bo->active_scanout);
+ assert(saved_bo->refcnt >= saved_bo->active_scanout);
+@@ -2315,17 +3101,34 @@ retry: /* Attach per-crtc pixmap or direct */
+ sna_crtc_randr(crtc);
+ if (sna_crtc->transform)
+ sna_crtc_damage(crtc);
++ if (sna_crtc->cursor && /* Reload cursor if RandR maybe changed */
++ (!sna_crtc->hwcursor ||
++ saved_cursor_transform || sna_crtc->cursor_transform ||
++ sna_crtc->cursor->rotation != crtc->rotation))
++ sna_crtc_disable_cursor(sna, sna_crtc);
++
++ assert(!sna->mode.hidden);
+ sna->mode.front_active += saved_bo == NULL;
+ sna->mode.dirty = true;
+- DBG(("%s: front_active=%d\n", __FUNCTION__, sna->mode.front_active));
++ DBG(("%s: handle=%d, scanout_active=%d, front_active=%d\n",
++ __FUNCTION__, bo->handle, bo->active_scanout, sna->mode.front_active));
+
+ return TRUE;
+
+ error:
+ sna_crtc->offset = saved_offset;
++ if (sna_crtc->transform) {
++ assert(sna->mode.rr_active);
++ sna->mode.rr_active--;
++ }
++ if (saved_transform)
++ sna->mode.rr_active++;
+ sna_crtc->transform = saved_transform;
++ sna_crtc->cursor_transform = saved_cursor_transform;
++ sna_crtc->hwcursor = saved_hwcursor;
+ sna_crtc->bo = saved_bo;
+- sna_mode_discover(sna);
++
++ reprobe_connectors(crtc);
+ return FALSE;
+ }
+
+@@ -2346,14 +3149,14 @@ sna_crtc_set_mode_major(xf86CrtcPtr crtc, DisplayModePtr mode,
+ xf86DrvMsg(crtc->scrn->scrnIndex, X_INFO,
+ "switch to mode %dx%d@%.1f on %s using pipe %d, position (%d, %d), rotation %s, reflection %s\n",
+ mode->HDisplay, mode->VDisplay, xf86ModeVRefresh(mode),
+- outputs_for_crtc(crtc, outputs, sizeof(outputs)), sna_crtc->pipe,
++ outputs_for_crtc(crtc, outputs, sizeof(outputs)), __sna_crtc_pipe(sna_crtc),
+ x, y, rotation_to_str(rotation), reflection_to_str(rotation));
+
+ assert(mode->HDisplay <= sna->mode.max_crtc_width &&
+ mode->VDisplay <= sna->mode.max_crtc_height);
+
+ #if HAS_GAMMA
+- drmModeCrtcSetGamma(sna->kgem.fd, sna_crtc->id,
++ drmModeCrtcSetGamma(sna->kgem.fd, __sna_crtc_id(sna_crtc),
+ crtc->gamma_size,
+ crtc->gamma_red,
+ crtc->gamma_green,
+@@ -2372,17 +3175,10 @@ sna_crtc_set_mode_major(xf86CrtcPtr crtc, DisplayModePtr mode,
+ static void
+ sna_crtc_dpms(xf86CrtcPtr crtc, int mode)
+ {
+- struct sna_crtc *priv = to_sna_crtc(crtc);
+-
+ DBG(("%s(pipe %d, dpms mode -> %d):= active=%d\n",
+- __FUNCTION__, priv->pipe, mode, mode == DPMSModeOn));
+- if (priv->dpms_mode == mode)
+- return;
+-
+- assert(priv);
+- priv->dpms_mode = mode;
++ __FUNCTION__, sna_crtc_pipe(crtc), mode, mode == DPMSModeOn));
+
+- if (mode == DPMSModeOn && crtc->enabled && priv->bo == NULL) {
++ if (mode == DPMSModeOn && crtc->enabled) {
+ if (__sna_crtc_set_mode(crtc))
+ update_flush_interval(to_sna(crtc->scrn));
+ else
+@@ -2390,7 +3186,7 @@ sna_crtc_dpms(xf86CrtcPtr crtc, int mode)
+ }
+
+ if (mode != DPMSModeOn)
+- sna_crtc_disable(crtc);
++ sna_crtc_disable(crtc, false);
+ }
+
+ void sna_mode_adjust_frame(struct sna *sna, int x, int y)
+@@ -2426,7 +3222,7 @@ sna_crtc_gamma_set(xf86CrtcPtr crtc,
+ {
+ assert(to_sna_crtc(crtc));
+ drmModeCrtcSetGamma(to_sna(crtc->scrn)->kgem.fd,
+- to_sna_crtc(crtc)->id,
++ sna_crtc_id(crtc),
+ size, red, green, blue);
+ }
+
+@@ -2434,10 +3230,14 @@ static void
+ sna_crtc_destroy(xf86CrtcPtr crtc)
+ {
+ struct sna_crtc *sna_crtc = to_sna_crtc(crtc);
++ struct plane *sprite, *sn;
+
+ if (sna_crtc == NULL)
+ return;
+
++ list_for_each_entry_safe(sprite, sn, &sna_crtc->sprites, link)
++ free(sprite);
++
+ free(sna_crtc);
+ crtc->driver_private = NULL;
+ }
+@@ -2455,7 +3255,7 @@ sna_crtc_set_scanout_pixmap(xf86CrtcPtr crtc, PixmapPtr pixmap)
+ return TRUE;
+
+ DBG(("%s: CRTC:%d, pipe=%d setting scanout pixmap=%ld\n",
+- __FUNCTION__, sna_crtc->id, sna_crtc->pipe,
++ __FUNCTION__, __sna_crtc_id(sna_crtc), __sna_crtc_pipe(sna_crtc),
+ pixmap ? pixmap->drawable.serialNumber : 0));
+
+ /* Disable first so that we can unregister the damage tracking */
+@@ -2576,6 +3376,10 @@ static int plane_details(struct sna *sna, struct plane *p)
+ }
+ }
+
++ p->rotation.supported &= DBG_NATIVE_ROTATION;
++ if (!xf86ReturnOptValBool(sna->Options, OPTION_ROTATION, TRUE))
++ p->rotation.supported = RR_Rotate_0;
++
+ if (props != (uint32_t *)stack_props)
+ free(props);
+
+@@ -2583,20 +3387,26 @@ static int plane_details(struct sna *sna, struct plane *p)
+ return type;
+ }
+
++static void add_sprite_plane(struct sna_crtc *crtc,
++ struct plane *details)
++{
++ struct plane *sprite = malloc(sizeof(*sprite));
++ if (!sprite)
++ return;
++
++ memcpy(sprite, details, sizeof(*sprite));
++ list_add(&sprite->link, &crtc->sprites);
++}
++
+ static void
+ sna_crtc_find_planes(struct sna *sna, struct sna_crtc *crtc)
+ {
+ #define LOCAL_IOCTL_SET_CAP DRM_IOWR(0x0d, struct local_set_cap)
+-#define LOCAL_IOCTL_MODE_GETPLANERESOURCES DRM_IOWR(0xb5, struct local_mode_get_plane_res)
+-#define LOCAL_IOCTL_MODE_GETPLANE DRM_IOWR(0xb6, struct local_mode_get_plane)
+ struct local_set_cap {
+ uint64_t name;
+ uint64_t value;
+ } cap;
+- struct local_mode_get_plane_res {
+- uint64_t plane_id_ptr;
+- uint64_t count_planes;
+- } r;
++ struct local_mode_get_plane_res r;
+ uint32_t stack_planes[32];
+ uint32_t *planes = stack_planes;
+ int i;
+@@ -2629,18 +3439,7 @@ sna_crtc_find_planes(struct sna *sna, struct sna_crtc *crtc)
+ VG(VALGRIND_MAKE_MEM_DEFINED(planes, sizeof(uint32_t)*r.count_planes));
+
+ for (i = 0; i < r.count_planes; i++) {
+- struct local_mode_get_plane {
+- uint32_t plane_id;
+-
+- uint32_t crtc_id;
+- uint32_t fb_id;
+-
+- uint32_t possible_crtcs;
+- uint32_t gamma_size;
+-
+- uint32_t count_format_types;
+- uint64_t format_type_ptr;
+- } p;
++ struct local_mode_get_plane p;
+ struct plane details;
+
+ VG_CLEAR(p);
+@@ -2649,11 +3448,11 @@ sna_crtc_find_planes(struct sna *sna, struct sna_crtc *crtc)
+ if (drmIoctl(sna->kgem.fd, LOCAL_IOCTL_MODE_GETPLANE, &p))
+ continue;
+
+- if ((p.possible_crtcs & (1 << crtc->pipe)) == 0)
++ if ((p.possible_crtcs & (1 << __sna_crtc_pipe(crtc))) == 0)
+ continue;
+
+ DBG(("%s: plane %d is attached to our pipe=%d\n",
+- __FUNCTION__, planes[i], crtc->pipe));
++ __FUNCTION__, planes[i], __sna_crtc_pipe(crtc)));
+
+ details.id = p.plane_id;
+ details.rotation.prop = 0;
+@@ -2672,8 +3471,7 @@ sna_crtc_find_planes(struct sna *sna, struct sna_crtc *crtc)
+ break;
+
+ case DRM_PLANE_TYPE_OVERLAY:
+- if (crtc->sprite.id == 0)
+- crtc->sprite = details;
++ add_sprite_plane(crtc, &details);
+ break;
+ }
+ }
+@@ -2688,7 +3486,6 @@ sna_crtc_init__rotation(struct sna *sna, struct sna_crtc *crtc)
+ crtc->rotation = RR_Rotate_0;
+ crtc->primary.rotation.supported = RR_Rotate_0;
+ crtc->primary.rotation.current = RR_Rotate_0;
+- crtc->sprite.rotation = crtc->primary.rotation;
+ }
+
+ static void
+@@ -2698,55 +3495,55 @@ sna_crtc_init__cursor(struct sna *sna, struct sna_crtc *crtc)
+
+ VG_CLEAR(arg);
+ arg.flags = DRM_MODE_CURSOR_BO;
+- arg.crtc_id = crtc->id;
++ arg.crtc_id = __sna_crtc_id(crtc);
+ arg.width = arg.height = 0;
+ arg.handle = 0;
+
+ (void)drmIoctl(sna->kgem.fd, DRM_IOCTL_MODE_CURSOR, &arg);
++ crtc->hwcursor = true;
+ }
+
+ static bool
+-sna_crtc_add(ScrnInfoPtr scrn, int id)
++sna_crtc_add(ScrnInfoPtr scrn, unsigned id)
+ {
+ struct sna *sna = to_sna(scrn);
+ xf86CrtcPtr crtc;
+ struct sna_crtc *sna_crtc;
+ struct drm_i915_get_pipe_from_crtc_id get_pipe;
+
+- DBG(("%s(%d)\n", __FUNCTION__, id));
++ DBG(("%s(%d): is-zaphod? %d\n", __FUNCTION__, id, is_zaphod(scrn)));
+
+ sna_crtc = calloc(sizeof(struct sna_crtc), 1);
+ if (sna_crtc == NULL)
+ return false;
+
+ sna_crtc->id = id;
+- sna_crtc->dpms_mode = -1;
+
+ VG_CLEAR(get_pipe);
+ get_pipe.pipe = 0;
+- get_pipe.crtc_id = sna_crtc->id;
++ get_pipe.crtc_id = id;
+ if (drmIoctl(sna->kgem.fd,
+ DRM_IOCTL_I915_GET_PIPE_FROM_CRTC_ID,
+ &get_pipe)) {
+ free(sna_crtc);
+ return false;
+ }
+- sna_crtc->pipe = get_pipe.pipe;
++ assert((unsigned)get_pipe.pipe < 256);
++ sna_crtc->flags |= get_pipe.pipe << 8;
+
+ if (is_zaphod(scrn) &&
+- scrn->confScreen->device->screen != sna_crtc->pipe) {
++ (get_zaphod_crtcs(sna) & (1 << get_pipe.pipe)) == 0) {
+ free(sna_crtc);
+ return true;
+ }
+
++ list_init(&sna_crtc->sprites);
+ sna_crtc_init__rotation(sna, sna_crtc);
+-
+ sna_crtc_find_planes(sna, sna_crtc);
+
+- DBG(("%s: CRTC:%d [pipe=%d], primary id=%x: supported-rotations=%x, current-rotation=%x, sprite id=%x: supported-rotations=%x, current-rotation=%x\n",
+- __FUNCTION__, sna_crtc->id, sna_crtc->pipe,
+- sna_crtc->primary.id, sna_crtc->primary.rotation.supported, sna_crtc->primary.rotation.current,
+- sna_crtc->sprite.id, sna_crtc->sprite.rotation.supported, sna_crtc->sprite.rotation.current));
++ DBG(("%s: CRTC:%d [pipe=%d], primary id=%x: supported-rotations=%x, current-rotation=%x\n",
++ __FUNCTION__, id, get_pipe.pipe,
++ sna_crtc->primary.id, sna_crtc->primary.rotation.supported, sna_crtc->primary.rotation.current));
+
+ list_init(&sna_crtc->shadow_link);
+
+@@ -2761,7 +3558,7 @@ sna_crtc_add(ScrnInfoPtr scrn, int id)
+ crtc->driver_private = sna_crtc;
+ sna_crtc->base = crtc;
+ DBG(("%s: attached crtc[%d] pipe=%d\n",
+- __FUNCTION__, id, sna_crtc->pipe));
++ __FUNCTION__, id, __sna_crtc_pipe(sna_crtc)));
+
+ return true;
+ }
+@@ -2798,20 +3595,56 @@ find_property(struct sna *sna, struct sna_output *output, const char *name)
+ return -1;
+ }
+
++static void update_properties(struct sna *sna, struct sna_output *output)
++{
++ union compat_mode_get_connector compat_conn;
++ struct drm_mode_modeinfo dummy;
++
++ VG_CLEAR(compat_conn);
++
++ compat_conn.conn.connector_id = output->id;
++ compat_conn.conn.count_props = output->num_props;
++ compat_conn.conn.props_ptr = (uintptr_t)output->prop_ids;
++ compat_conn.conn.prop_values_ptr = (uintptr_t)output->prop_values;
++ compat_conn.conn.count_modes = 1; /* skip detect */
++ compat_conn.conn.modes_ptr = (uintptr_t)&dummy;
++ compat_conn.conn.count_encoders = 0;
++
++ (void)drmIoctl(sna->kgem.fd,
++ DRM_IOCTL_MODE_GETCONNECTOR,
++ &compat_conn.conn);
++
++ assert(compat_conn.conn.count_props == output->num_props);
++ output->update_properties = false;
++}
++
+ static xf86OutputStatus
+ sna_output_detect(xf86OutputPtr output)
+ {
+ struct sna *sna = to_sna(output->scrn);
+ struct sna_output *sna_output = output->driver_private;
+ union compat_mode_get_connector compat_conn;
++ uint32_t now;
+
+ DBG(("%s(%s:%d)\n", __FUNCTION__, output->name, sna_output->id));
++ sna_output->update_properties = false;
+
+ if (!sna_output->id) {
+ DBG(("%s(%s) hiding due to lost connection\n", __FUNCTION__, output->name));
+ return XF86OutputStatusDisconnected;
+ }
+
++ /* Cache detections for 15s or hotplug event */
++ now = GetTimeInMillis();
++ if (sna_output->last_detect != 0 &&
++ (int32_t)(now - sna_output->last_detect) <= OUTPUT_STATUS_CACHE_MS) {
++ DBG(("%s(%s) reporting cached status (since %dms): %d\n",
++ __FUNCTION__, output->name, now - sna_output->last_detect,
++ sna_output->status));
++ sna_output->update_properties = true;
++ return sna_output->status;
++ }
++
+ VG_CLEAR(compat_conn);
+ compat_conn.conn.connector_id = sna_output->id;
+ sna_output->num_modes = compat_conn.conn.count_modes = 0; /* reprobe */
+@@ -2854,15 +3687,23 @@ sna_output_detect(xf86OutputPtr output)
+ DBG(("%s(%s): found %d modes, connection status=%d\n",
+ __FUNCTION__, output->name, sna_output->num_modes, compat_conn.conn.connection));
+
++ sna_output->reprobe = false;
++ sna_output->last_detect = now;
+ switch (compat_conn.conn.connection) {
+ case DRM_MODE_CONNECTED:
+- return XF86OutputStatusConnected;
++ sna_output->status = XF86OutputStatusConnected;
++ output->mm_width = compat_conn.conn.mm_width;
++ output->mm_height = compat_conn.conn.mm_height;
++ break;
+ case DRM_MODE_DISCONNECTED:
+- return XF86OutputStatusDisconnected;
++ sna_output->status = XF86OutputStatusDisconnected;
++ break;
+ default:
+ case DRM_MODE_UNKNOWNCONNECTION:
+- return XF86OutputStatusUnknown;
++ sna_output->status = XF86OutputStatusUnknown;
++ break;
+ }
++ return sna_output->status;
+ }
+
+ static Bool
+@@ -2895,6 +3736,27 @@ sna_output_mode_valid(xf86OutputPtr output, DisplayModePtr mode)
+ return MODE_OK;
+ }
+
++static void sna_output_set_parsed_edid(xf86OutputPtr output, xf86MonPtr mon)
++{
++ unsigned conn_mm_width, conn_mm_height;
++
++ /* We set the output size based on values from the kernel */
++ conn_mm_width = output->mm_width;
++ conn_mm_height = output->mm_height;
++
++ xf86OutputSetEDID(output, mon);
++
++ if (output->mm_width != conn_mm_width || output->mm_height != conn_mm_height) {
++ DBG(("%s)%s): kernel and Xorg disagree over physical size: kernel=%dx%dmm, Xorg=%dx%dmm\n",
++ __FUNCTION__, output->name,
++ conn_mm_width, conn_mm_height,
++ output->mm_width, output->mm_height));
++ }
++
++ output->mm_width = conn_mm_width;
++ output->mm_height = conn_mm_height;
++}
++
+ static void
+ sna_output_attach_edid(xf86OutputPtr output)
+ {
+@@ -2907,6 +3769,13 @@ sna_output_attach_edid(xf86OutputPtr output)
+ if (sna_output->edid_idx == -1)
+ return;
+
++ /* Always refresh the blob as the kernel may randomly update the
++ * id even if the contents of the blob doesn't change, and a
++ * request for the stale id will return nothing.
++ */
++ if (sna_output->update_properties)
++ update_properties(sna, sna_output);
++
+ raw = sna_output->edid_raw;
+ blob.length = sna_output->edid_len;
+
+@@ -2917,8 +3786,12 @@ sna_output_attach_edid(xf86OutputPtr output)
+ old = NULL;
+
+ blob.blob_id = sna_output->prop_values[sna_output->edid_idx];
+- DBG(("%s: attaching EDID id=%d, current=%d\n",
+- __FUNCTION__, blob.blob_id, sna_output->edid_blob_id));
++ if (!blob.blob_id)
++ goto done;
++
++ DBG(("%s(%s): attaching EDID id=%d, current=%d\n",
++ __FUNCTION__, output->name,
++ blob.blob_id, sna_output->edid_blob_id));
+ if (blob.blob_id == sna_output->edid_blob_id && 0) { /* sigh */
+ if (output->MonInfo) {
+ /* XXX the property keeps on disappearing... */
+@@ -2936,26 +3809,45 @@ sna_output_attach_edid(xf86OutputPtr output)
+ }
+
+ blob.data = (uintptr_t)raw;
+- if (drmIoctl(sna->kgem.fd, DRM_IOCTL_MODE_GETPROPBLOB, &blob))
+- goto done;
++ do {
++ while (drmIoctl(sna->kgem.fd, DRM_IOCTL_MODE_GETPROPBLOB, &blob)) {
++ update_properties(sna, sna_output);
++ if (blob.blob_id == sna_output->prop_values[sna_output->edid_idx]) {
++ DBG(("%s(%s): failed to read blob, reusing previous\n",
++ __FUNCTION__, output->name));
++ goto done;
++ }
++ blob.blob_id = sna_output->prop_values[sna_output->edid_idx];
++ }
+
+- DBG(("%s: retrieving blob id=%d, length=%d\n",
+- __FUNCTION__, blob.blob_id, blob.length));
++ DBG(("%s(%s): retrieving blob id=%d, length=%d\n",
++ __FUNCTION__, output->name, blob.blob_id, blob.length));
+
+- if (blob.length > sna_output->edid_len) {
+- raw = realloc(raw, blob.length);
+- if (raw == NULL)
++ if (blob.length < 128)
+ goto done;
+
+- VG(memset(raw, 0, blob.length));
+- blob.data = (uintptr_t)raw;
+- if (drmIoctl(sna->kgem.fd, DRM_IOCTL_MODE_GETPROPBLOB, &blob))
+- goto done;
++ if (blob.length > sna_output->edid_len) {
++ raw = realloc(raw, blob.length);
++ if (raw == NULL)
++ goto done;
++
++ VG(memset(raw, 0, blob.length));
++ blob.data = (uintptr_t)raw;
++ }
++ } while (blob.length != sna_output->edid_len &&
++ drmIoctl(sna->kgem.fd, DRM_IOCTL_MODE_GETPROPBLOB, &blob));
++
++ if (blob.length & 127) {
++ /* Truncated EDID! Make sure no one reads too far */
++ *SECTION(NO_EDID, (uint8_t*)raw) = blob.length/128 - 1;
++ blob.length &= -128;
+ }
+
+ if (old &&
+ blob.length == sna_output->edid_len &&
+ memcmp(old, raw, blob.length) == 0) {
++ DBG(("%s(%s): EDID + MonInfo is unchanged\n",
++ __FUNCTION__, output->name));
+ assert(sna_output->edid_raw == raw);
+ sna_output->edid_blob_id = blob.blob_id;
+ RRChangeOutputProperty(output->randr_output,
+@@ -2974,31 +3866,186 @@ skip_read:
+ mon->flags |= MONITOR_EDID_COMPLETE_RAWDATA;
+ }
+
+-done:
+- xf86OutputSetEDID(output, mon);
+- if (raw) {
+- sna_output->edid_raw = raw;
+- sna_output->edid_len = blob.length;
+- sna_output->edid_blob_id = blob.blob_id;
++done:
++ sna_output_set_parsed_edid(output, mon);
++ if (raw) {
++ sna_output->edid_raw = raw;
++ sna_output->edid_len = blob.length;
++ sna_output->edid_blob_id = blob.blob_id;
++ }
++}
++
++static void
++sna_output_attach_tile(xf86OutputPtr output)
++{
++#if XF86_OUTPUT_VERSION >= 3
++ struct sna *sna = to_sna(output->scrn);
++ struct sna_output *sna_output = output->driver_private;
++ struct drm_mode_get_blob blob;
++ struct xf86CrtcTileInfo tile_info, *set = NULL;
++ char *tile;
++ int id;
++
++ id = find_property(sna, sna_output, "TILE");
++ DBG(("%s: found? TILE=%d\n", __FUNCTION__, id));
++ if (id == -1)
++ goto out;
++
++ if (sna_output->update_properties)
++ update_properties(sna, sna_output);
++
++ VG_CLEAR(blob);
++ blob.blob_id = sna_output->prop_values[id];
++ blob.length = 0;
++ if (drmIoctl(sna->kgem.fd, DRM_IOCTL_MODE_GETPROPBLOB, &blob))
++ goto out;
++
++ do {
++ id = blob.length;
++ tile = alloca(id + 1);
++ blob.data = (uintptr_t)tile;
++ VG(memset(tile, 0, id));
++ DBG(("%s: reading %d bytes for TILE blob\n", __FUNCTION__, id));
++ if (drmIoctl(sna->kgem.fd, DRM_IOCTL_MODE_GETPROPBLOB, &blob))
++ goto out;
++ } while (id != blob.length);
++
++ tile[blob.length] = '\0'; /* paranoia */
++ DBG(("%s: TILE='%s'\n", __FUNCTION__, tile));
++ if (xf86OutputParseKMSTile(tile, blob.length, &tile_info))
++ set = &tile_info;
++out:
++ xf86OutputSetTile(output, set);
++#endif
++}
++
++static bool duplicate_mode(DisplayModePtr modes, DisplayModePtr m)
++{
++ if (m == NULL)
++ return false;
++
++ while (modes) {
++ if (xf86ModesEqual(modes, m))
++ return true;
++
++ modes = modes->next;
++ }
++
++ return false;
++}
++
++static struct pixel_count {
++ int16_t width, height;
++} common_16_9[] = {
++ { 640, 360 },
++ { 720, 405 },
++ { 864, 486 },
++ { 960, 540 },
++ { 1024, 576 },
++ { 1280, 720 },
++ { 1366, 768 },
++ { 1600, 900 },
++ { 1920, 1080 },
++ { 2048, 1152 },
++ { 2560, 1440 },
++ { 2880, 1620 },
++ { 3200, 1800 },
++ { 3840, 2160 },
++ { 4096, 2304 },
++ { 5120, 2880 },
++ { 7680, 4320 },
++ { 15360, 8640 },
++}, common_16_10[] = {
++ { 1280, 800 },
++ { 1400, 900 },
++ { 1680, 1050 },
++ { 1920, 1200 },
++ { 2560, 1600 },
++};
++
++static DisplayModePtr
++default_modes(DisplayModePtr preferred)
++{
++ DisplayModePtr modes;
++ int n;
++
++#if XORG_VERSION_CURRENT >= XORG_VERSION_NUMERIC(1,6,99,900,0)
++ modes = xf86GetDefaultModes();
++#else
++ modes = xf86GetDefaultModes(0, 0);
++#endif
++
++ /* XXX O(n^2) mode list generation :( */
++
++#if XORG_VERSION_CURRENT >= XORG_VERSION_NUMERIC(1,4,99,901,0)
++ if (preferred) {
++ DisplayModePtr m;
++
++ /* Add a half-resolution mode useful for large panels */
++ m = xf86GTFMode(preferred->HDisplay/2,
++ preferred->VDisplay/2,
++ xf86ModeVRefresh(preferred),
++ FALSE, FALSE);
++ if (!duplicate_mode(modes, m))
++ modes = xf86ModesAdd(modes, m);
++ else
++ free(m);
++
++ if (preferred->VDisplay * 16 > preferred->HDisplay*9 - preferred->HDisplay/32 &&
++ preferred->VDisplay * 16 < preferred->HDisplay*9 + preferred->HDisplay/32) {
++ DBG(("Adding 16:9 modes -- %d < %d > %d\n",
++ preferred->HDisplay*9 - preferred->HDisplay/32,
++ preferred->VDisplay * 16,
++ preferred->HDisplay*9 + preferred->HDisplay/32));
++ for (n = 0; n < ARRAY_SIZE(common_16_9); n++) {
++ if (preferred->HDisplay <= common_16_9[n].width ||
++ preferred->VDisplay <= common_16_9[n].height)
++ break;
++
++ m = xf86GTFMode(common_16_9[n].width,
++ common_16_9[n].height,
++ xf86ModeVRefresh(preferred),
++ FALSE, FALSE);
++ if (!duplicate_mode(modes, m))
++ modes = xf86ModesAdd(modes, m);
++ else
++ free(m);
++ }
++ }
++
++ if (preferred->VDisplay * 16 > preferred->HDisplay*10 - preferred->HDisplay/32 &&
++ preferred->VDisplay * 16 < preferred->HDisplay*10 + preferred->HDisplay/32) {
++ DBG(("Adding 16:10 modes -- %d < %d > %d\n",
++ preferred->HDisplay*10 - preferred->HDisplay/32,
++ preferred->VDisplay * 16,
++ preferred->HDisplay*10 + preferred->HDisplay/32));
++ for (n = 0; n < ARRAY_SIZE(common_16_10); n++) {
++ if (preferred->HDisplay <= common_16_10[n].width ||
++ preferred->VDisplay <= common_16_10[n].height)
++ break;
++
++ m = xf86GTFMode(common_16_10[n].width,
++ common_16_10[n].height,
++ xf86ModeVRefresh(preferred),
++ FALSE, FALSE);
++ if (!duplicate_mode(modes, m))
++ modes = xf86ModesAdd(modes, m);
++ else
++ free(m);
++ }
++ }
+ }
+-}
+-
+-static DisplayModePtr
+-default_modes(void)
+-{
+-#if XORG_VERSION_CURRENT >= XORG_VERSION_NUMERIC(1,6,99,900,0)
+- return xf86GetDefaultModes();
+-#else
+- return xf86GetDefaultModes(0, 0);
+ #endif
++
++ return modes;
+ }
+
+ static DisplayModePtr
+-sna_output_panel_edid(xf86OutputPtr output, DisplayModePtr modes)
++sna_output_add_default_modes(xf86OutputPtr output, DisplayModePtr modes)
+ {
+ xf86MonPtr mon = output->MonInfo;
+ DisplayModePtr i, m, preferred = NULL;
+- int max_x = 0, max_y = 0;
++ int max_x = 0, max_y = 0, max_clock = 0;
+ float max_vrefresh = 0.0;
+
+ if (mon && GTF_SUPPORTED(mon->features.msc))
+@@ -3009,16 +4056,17 @@ sna_output_panel_edid(xf86OutputPtr output, DisplayModePtr modes)
+ preferred = m;
+ max_x = max(max_x, m->HDisplay);
+ max_y = max(max_y, m->VDisplay);
++ max_clock = max(max_clock, m->Clock);
+ max_vrefresh = max(max_vrefresh, xf86ModeVRefresh(m));
+ }
+-
+- max_vrefresh = max(max_vrefresh, 60.0);
+ max_vrefresh *= (1 + SYNC_TOLERANCE);
+
+- m = default_modes();
++ m = default_modes(preferred);
+ xf86ValidateModesSize(output->scrn, m, max_x, max_y, 0);
+
+ for (i = m; i; i = i->next) {
++ if (i->Clock > max_clock)
++ i->status = MODE_CLOCK_HIGH;
+ if (xf86ModeVRefresh(i) > max_vrefresh)
+ i->status = MODE_VSYNC;
+ if (preferred &&
+@@ -3034,28 +4082,47 @@ sna_output_panel_edid(xf86OutputPtr output, DisplayModePtr modes)
+ }
+
+ static DisplayModePtr
++sna_output_override_edid(xf86OutputPtr output)
++{
++ struct sna_output *sna_output = output->driver_private;
++
++ if (sna_output->fake_edid_mon == NULL)
++ return NULL;
++
++ xf86OutputSetEDID(output, sna_output->fake_edid_mon);
++ return xf86DDCGetModes(output->scrn->scrnIndex,
++ sna_output->fake_edid_mon);
++}
++
++static DisplayModePtr
+ sna_output_get_modes(xf86OutputPtr output)
+ {
+ struct sna_output *sna_output = output->driver_private;
+- DisplayModePtr Modes = NULL, current = NULL;
++ DisplayModePtr Modes, current;
+ int i;
+
+ DBG(("%s(%s:%d)\n", __FUNCTION__, output->name, sna_output->id));
+ assert(sna_output->id);
+
++ Modes = sna_output_override_edid(output);
++ if (Modes)
++ return Modes;
++
+ sna_output_attach_edid(output);
++ sna_output_attach_tile(output);
+
+- if (output->crtc) {
++ current = NULL;
++ if (output->crtc && !sna_output->hotplug_count) {
+ struct drm_mode_crtc mode;
+
+ VG_CLEAR(mode);
+ assert(to_sna_crtc(output->crtc));
+- mode.crtc_id = to_sna_crtc(output->crtc)->id;
++ mode.crtc_id = sna_crtc_id(output->crtc);
+
+ if (drmIoctl(to_sna(output->scrn)->kgem.fd, DRM_IOCTL_MODE_GETCRTC, &mode) == 0) {
+ DBG(("%s: CRTC:%d, pipe=%d: has mode?=%d\n", __FUNCTION__,
+- to_sna_crtc(output->crtc)->id,
+- to_sna_crtc(output->crtc)->pipe,
++ sna_crtc_id(output->crtc),
++ sna_crtc_pipe(output->crtc),
+ mode.mode_valid && mode.mode.clock));
+
+ if (mode.mode_valid && mode.mode.clock) {
+@@ -3117,7 +4184,7 @@ sna_output_get_modes(xf86OutputPtr output)
+ }
+
+ if (sna_output->add_default_modes)
+- Modes = sna_output_panel_edid(output, Modes);
++ Modes = sna_output_add_default_modes(output, Modes);
+
+ return Modes;
+ }
+@@ -3132,6 +4199,8 @@ sna_output_destroy(xf86OutputPtr output)
+ return;
+
+ free(sna_output->edid_raw);
++ free(sna_output->fake_edid_raw);
++
+ for (i = 0; i < sna_output->num_props; i++) {
+ if (sna_output->props[i].kprop == NULL)
+ continue;
+@@ -3155,7 +4224,7 @@ sna_output_destroy(xf86OutputPtr output)
+ }
+
+ static void
+-sna_output_dpms(xf86OutputPtr output, int dpms)
++__sna_output_dpms(xf86OutputPtr output, int dpms, int fixup)
+ {
+ struct sna *sna = to_sna(output->scrn);
+ struct sna_output *sna_output = output->driver_private;
+@@ -3182,8 +4251,9 @@ sna_output_dpms(xf86OutputPtr output, int dpms)
+ if (sna_output->backlight.iface && dpms != DPMSModeOn) {
+ if (old_dpms == DPMSModeOn) {
+ sna_output->backlight_active_level = sna_output_backlight_get(output);
+- DBG(("%s: saving current backlight %d\n",
+- __FUNCTION__, sna_output->backlight_active_level));
++ DBG(("%s(%s:%d): saving current backlight %d\n",
++ __FUNCTION__, output->name, sna_output->id,
++ sna_output->backlight_active_level));
+ }
+ sna_output->dpms_mode = dpms;
+ sna_output_backlight_off(sna_output);
+@@ -3193,18 +4263,31 @@ sna_output_dpms(xf86OutputPtr output, int dpms)
+ drmModeConnectorSetProperty(sna->kgem.fd,
+ sna_output->id,
+ sna_output->dpms_id,
+- dpms))
+- dpms = old_dpms;
++ dpms)) {
++ DBG(("%s(%s:%d): failed to set DPMS to %d (fixup? %d)\n",
++ __FUNCTION__, output->name, sna_output->id, dpms, fixup));
++ if (fixup && dpms != DPMSModeOn) {
++ sna_crtc_disable(output->crtc, false);
++ return;
++ }
++ }
+
+ if (sna_output->backlight.iface && dpms == DPMSModeOn) {
+- DBG(("%s: restoring previous backlight %d\n",
+- __FUNCTION__, sna_output->backlight_active_level));
++ DBG(("%s(%d:%d: restoring previous backlight %d\n",
++ __FUNCTION__, output->name, sna_output->id,
++ sna_output->backlight_active_level));
+ sna_output_backlight_on(sna_output);
+ }
+
+ sna_output->dpms_mode = dpms;
+ }
+
++static void
++sna_output_dpms(xf86OutputPtr output, int dpms)
++{
++ __sna_output_dpms(output, dpms, true);
++}
++
+ static bool
+ sna_property_ignore(drmModePropertyPtr prop)
+ {
+@@ -3239,14 +4322,14 @@ sna_output_create_ranged_atom(xf86OutputPtr output, Atom *atom,
+ err = RRConfigureOutputProperty(output->randr_output, *atom, FALSE,
+ TRUE, immutable, 2, atom_range);
+ if (err != 0)
+- xf86DrvMsg(output->scrn->scrnIndex, X_ERROR,
++ xf86DrvMsg(output->scrn->scrnIndex, X_WARNING,
+ "RRConfigureOutputProperty error, %d\n", err);
+
+ err = RRChangeOutputProperty(output->randr_output, *atom, XA_INTEGER,
+ 32, PropModeReplace, 1, &value,
+ FALSE, FALSE);
+ if (err != 0)
+- xf86DrvMsg(output->scrn->scrnIndex, X_ERROR,
++ xf86DrvMsg(output->scrn->scrnIndex, X_WARNING,
+ "RRChangeOutputProperty error, %d\n", err);
+ }
+
+@@ -3303,7 +4386,7 @@ sna_output_create_resources(xf86OutputPtr output)
+ p->kprop->flags & DRM_MODE_PROP_IMMUTABLE ? TRUE : FALSE,
+ p->num_atoms - 1, (INT32 *)&p->atoms[1]);
+ if (err != 0) {
+- xf86DrvMsg(output->scrn->scrnIndex, X_ERROR,
++ xf86DrvMsg(output->scrn->scrnIndex, X_WARNING,
+ "RRConfigureOutputProperty error, %d\n", err);
+ }
+
+@@ -3315,7 +4398,7 @@ sna_output_create_resources(xf86OutputPtr output)
+ XA_ATOM, 32, PropModeReplace, 1, &p->atoms[j+1],
+ FALSE, FALSE);
+ if (err != 0) {
+- xf86DrvMsg(output->scrn->scrnIndex, X_ERROR,
++ xf86DrvMsg(output->scrn->scrnIndex, X_WARNING,
+ "RRChangeOutputProperty error, %d\n", err);
+ }
+ }
+@@ -3385,18 +4468,19 @@ sna_output_set_property(xf86OutputPtr output, Atom property,
+ if (value->type != XA_INTEGER || value->format != 32 ||
+ value->size != 1)
+ return FALSE;
+- val = *(uint32_t *)value->data;
+
++ val = *(uint32_t *)value->data;
+ drmModeConnectorSetProperty(sna->kgem.fd, sna_output->id,
+ p->kprop->prop_id, (uint64_t)val);
+ return TRUE;
+ } else if (p->kprop->flags & DRM_MODE_PROP_ENUM) {
+- Atom atom;
+- const char *name;
+- int j;
++ Atom atom;
++ const char *name;
++ int j;
+
+ if (value->type != XA_ATOM || value->format != 32 || value->size != 1)
+ return FALSE;
++
+ memcpy(&atom, value->data, 4);
+ name = NameForAtom(atom);
+ if (name == NULL)
+@@ -3425,7 +4509,7 @@ static Bool
+ sna_output_get_property(xf86OutputPtr output, Atom property)
+ {
+ struct sna_output *sna_output = output->driver_private;
+- int err;
++ int err, i, j;
+
+ if (property == backlight_atom || property == backlight_deprecated_atom) {
+ INT32 val;
+@@ -3449,7 +4533,7 @@ sna_output_get_property(xf86OutputPtr output, Atom property)
+ XA_INTEGER, 32, PropModeReplace, 1, &val,
+ FALSE, FALSE);
+ if (err != 0) {
+- xf86DrvMsg(output->scrn->scrnIndex, X_ERROR,
++ xf86DrvMsg(output->scrn->scrnIndex, X_WARNING,
+ "RRChangeOutputProperty error, %d\n", err);
+ return FALSE;
+ }
+@@ -3457,6 +4541,40 @@ sna_output_get_property(xf86OutputPtr output, Atom property)
+ return TRUE;
+ }
+
++ for (i = 0; i < sna_output->num_props; i++) {
++ struct sna_property *p = &sna_output->props[i];
++
++ if (p->atoms == NULL || p->atoms[0] != property)
++ continue;
++
++ if (sna_output->update_properties && output->scrn->vtSema)
++ update_properties(to_sna(output->scrn), sna_output);
++
++ err = 0;
++ if (p->kprop->flags & DRM_MODE_PROP_RANGE) {
++ err = RRChangeOutputProperty(output->randr_output,
++ property, XA_INTEGER, 32,
++ PropModeReplace, 1,
++ &sna_output->prop_values[i],
++ FALSE, FALSE);
++ } else if (p->kprop->flags & DRM_MODE_PROP_ENUM) {
++ for (j = 0; j < p->kprop->count_enums; j++) {
++ if (p->kprop->enums[j].value == sna_output->prop_values[i])
++ break;
++ }
++ err = RRChangeOutputProperty(output->randr_output,
++ property, XA_ATOM, 32,
++ PropModeReplace, 1,
++ &p->atoms[j+1],
++ FALSE, FALSE);
++ }
++
++ if (err != 0)
++ xf86DrvMsg(output->scrn->scrnIndex, X_WARNING,
++ "RRChangeOutputProperty error, %d\n", err);
++ return TRUE;
++ }
++
+ return FALSE;
+ }
+
+@@ -3500,47 +4618,11 @@ static const char * const output_names[] = {
+ /* DRM_MODE_CONNECTOR_TV */ "TV",
+ /* DRM_MODE_CONNECTOR_eDP */ "eDP",
+ /* DRM_MODE_CONNECTOR_VIRTUAL */ "Virtual",
+- /* DRM_MODE_CONNECTOR_DSI */ "DSI"
++ /* DRM_MODE_CONNECTOR_DSI */ "DSI",
++ /* DRM_MODE_CONNECTOR_DPI */ "DPI"
+ };
+
+ static bool
+-sna_zaphod_match(const char *s, const char *output)
+-{
+- char t[20];
+- unsigned int i = 0;
+-
+- do {
+- /* match any outputs in a comma list, stopping at whitespace */
+- switch (*s) {
+- case '\0':
+- t[i] = '\0';
+- return strcmp(t, output) == 0;
+-
+- case ',':
+- t[i] ='\0';
+- if (strcmp(t, output) == 0)
+- return TRUE;
+- i = 0;
+- break;
+-
+- case ' ':
+- case '\t':
+- case '\n':
+- case '\r':
+- break;
+-
+- default:
+- t[i++] = *s;
+- break;
+- }
+-
+- s++;
+- } while (i < sizeof(t));
+-
+- return false;
+-}
+-
+-static bool
+ output_ignored(ScrnInfoPtr scrn, const char *name)
+ {
+ char monitor_name[64];
+@@ -3572,14 +4654,21 @@ gather_encoders(struct sna *sna, uint32_t id, int count,
+ struct drm_mode_get_encoder enc;
+ uint32_t *ids = NULL;
+
++ DBG(("%s(%d): expected count=%d\n", __FUNCTION__, id, count));
++
+ VG_CLEAR(compat_conn);
++ VG_CLEAR(enc);
+ memset(out, 0, sizeof(*out));
+
+ do {
+- free(ids);
+- ids = malloc(sizeof(*ids) * count);
+- if (ids == 0)
++ uint32_t *nids;
++
++ nids = realloc(ids, sizeof(*ids) * count);
++ if (nids == NULL) {
++ free(ids);
+ return false;
++ }
++ ids = nids;
+
+ compat_conn.conn.connector_id = id;
+ compat_conn.conn.count_props = 0;
+@@ -3593,12 +4682,14 @@ gather_encoders(struct sna *sna, uint32_t id, int count,
+ compat_conn.conn.count_encoders = count = 0;
+ }
+
++ VG(VALGRIND_MAKE_MEM_DEFINED(ids, sizeof(uint32_t)*compat_conn.conn.count_encoders));
+ if (count == compat_conn.conn.count_encoders)
+ break;
+
+ count = compat_conn.conn.count_encoders;
+ } while (1);
+
++ DBG(("%s(%d): gathering %d encoders\n", __FUNCTION__, id, count));
+ for (count = 0; count < compat_conn.conn.count_encoders; count++) {
+ enc.encoder_id = ids[count];
+ if (drmIoctl(sna->kgem.fd, DRM_IOCTL_MODE_GETENCODER, &enc)) {
+@@ -3606,6 +4697,8 @@ gather_encoders(struct sna *sna, uint32_t id, int count,
+ count = 0;
+ break;
+ }
++ DBG(("%s(%d): encoder=%d, possible_crtcs=%x, possible_clones=%x\n",
++ __FUNCTION__, id, enc.encoder_id, enc.possible_crtcs, enc.possible_clones));
+ out->possible_crtcs |= enc.possible_crtcs;
+ out->possible_clones |= enc.possible_clones;
+
+@@ -3731,6 +4824,116 @@ static int name_from_path(struct sna *sna,
+ return 0;
+ }
+
++static char *fake_edid_name(xf86OutputPtr output)
++{
++ struct sna *sna = to_sna(output->scrn);
++ const char *str, *colon;
++
++#if XORG_VERSION_CURRENT >= XORG_VERSION_NUMERIC(1,7,99,901,0)
++ str = xf86GetOptValString(sna->Options, OPTION_EDID);
++#else
++ str = NULL;
++#endif
++ if (str == NULL)
++ return NULL;
++
++ do {
++ colon = strchr(str, ':');
++ if (colon == NULL)
++ return NULL;
++
++ if (strncmp(str, output->name, colon-str) == 0 &&
++ output->name[colon-str] == '\0') {
++ char *path;
++ int len;
++
++ str = colon + 1;
++ colon = strchr(str, ',');
++ if (colon)
++ len = colon - str;
++ else
++ len = strlen(str);
++
++ path = malloc(len + 1);
++ if (path == NULL)
++ return NULL;
++
++ memcpy(path, str, len);
++ path[len] = '\0';
++ return path;
++ }
++
++ str = strchr(colon + 1, ',');
++ if (str == NULL)
++ return NULL;
++
++ str++;
++ } while (1);
++}
++
++static void
++sna_output_load_fake_edid(xf86OutputPtr output)
++{
++ struct sna_output *sna_output = output->driver_private;
++ const char *filename;
++ FILE *file;
++ void *raw;
++ int size;
++ xf86MonPtr mon;
++
++ filename = fake_edid_name(output);
++ if (filename == NULL)
++ return;
++
++ file = fopen(filename, "rb");
++ if (file == NULL)
++ goto err;
++
++ fseek(file, 0, SEEK_END);
++ size = ftell(file);
++ if (size % 128) {
++ fclose(file);
++ goto err;
++ }
++
++ raw = malloc(size);
++ if (raw == NULL) {
++ fclose(file);
++ free(raw);
++ goto err;
++ }
++
++ fseek(file, 0, SEEK_SET);
++ if (fread(raw, size, 1, file) != 1) {
++ fclose(file);
++ free(raw);
++ goto err;
++ }
++ fclose(file);
++
++ mon = xf86InterpretEDID(output->scrn->scrnIndex, raw);
++ if (mon == NULL) {
++ free(raw);
++ goto err;
++ }
++
++ if (mon && size > 128)
++ mon->flags |= MONITOR_EDID_COMPLETE_RAWDATA;
++
++ sna_output->fake_edid_mon = mon;
++ sna_output->fake_edid_raw = raw;
++
++ xf86DrvMsg(output->scrn->scrnIndex, X_CONFIG,
++ "Loading EDID from \"%s\" for output %s\n",
++ filename, output->name);
++ return;
++
++err:
++ xf86DrvMsg(output->scrn->scrnIndex, X_ERROR,
++ "Could not read EDID file \"%s\" for output %s\n",
++ filename, output->name);
++}
++
+ static int
+ sna_output_add(struct sna *sna, unsigned id, unsigned serial)
+ {
+@@ -3765,6 +4968,7 @@ sna_output_add(struct sna *sna, unsigned id, unsigned serial)
+ return -1;
+ }
+ assert(compat_conn.conn.connector_id == id);
++ DBG(("%s(%d): has %d associated encoders\n", __FUNCTION__, id, compat_conn.conn.count_encoders));
+
+ if (compat_conn.conn.connector_type < ARRAY_SIZE(output_names))
+ output_name = output_names[compat_conn.conn.connector_type];
+@@ -3813,34 +5017,43 @@ sna_output_add(struct sna *sna, unsigned id, unsigned serial)
+ }
+
+ if (is_zaphod(scrn)) {
+- const char *str;
++ unsigned zaphod_crtcs;
+
+- str = xf86GetOptValString(sna->Options, OPTION_ZAPHOD);
+- if (str && !sna_zaphod_match(str, name)) {
+- DBG(("%s: zaphod mismatch, want %s, have %s\n", __FUNCTION__, str, name));
++ if (!sna_zaphod_match(sna, name)) {
++ DBG(("%s: zaphod mismatch, want %s, have %s\n",
++ __FUNCTION__,
++ xf86GetOptValString(sna->Options, OPTION_ZAPHOD) ?: "???",
++ name));
+ return 0;
+ }
+
+- if ((possible_crtcs & (1 << scrn->confScreen->device->screen)) == 0) {
+- if (str) {
+- xf86DrvMsg(scrn->scrnIndex, X_ERROR,
+- "%s is an invalid output for screen (pipe) %d\n",
+- name, scrn->confScreen->device->screen);
+- return -1;
+- } else
+- return 0;
++ zaphod_crtcs = get_zaphod_crtcs(sna);
++ possible_crtcs &= zaphod_crtcs;
++ if (possible_crtcs == 0) {
++ xf86DrvMsg(scrn->scrnIndex, X_ERROR,
++ "%s is an invalid output for screen %d\n",
++ name, scrn->confScreen->device->screen);
++ return -1;
+ }
+
+- possible_crtcs = 1;
++ possible_crtcs >>= ffs(zaphod_crtcs) - 1;
+ }
+
+ sna_output = calloc(sizeof(struct sna_output), 1);
+ if (!sna_output)
+ return -1;
+
++ sna_output->connector_type = compat_conn.conn.connector_type;
++ sna_output->connector_type_id = compat_conn.conn.connector_type_id;
+ sna_output->num_props = compat_conn.conn.count_props;
+ sna_output->prop_ids = malloc(sizeof(uint32_t)*compat_conn.conn.count_props);
+ sna_output->prop_values = malloc(sizeof(uint64_t)*compat_conn.conn.count_props);
++ if (sna_output->prop_ids == NULL || sna_output->prop_values == NULL) {
++ free(sna_output->prop_ids);
++ free(sna_output->prop_values);
++ free(sna_output);
++ return -1;
++ }
+
+ compat_conn.conn.count_encoders = 0;
+
+@@ -3865,16 +5078,16 @@ sna_output_add(struct sna *sna, unsigned id, unsigned serial)
+ /* Construct name from topology, and recheck if output is acceptable */
+ path = name_from_path(sna, sna_output, name);
+ if (path) {
+- const char *str;
+-
+ if (output_ignored(scrn, name)) {
+ len = 0;
+ goto skip;
+ }
+
+- str = xf86GetOptValString(sna->Options, OPTION_ZAPHOD);
+- if (str && !sna_zaphod_match(str, name)) {
+- DBG(("%s: zaphod mismatch, want %s, have %s\n", __FUNCTION__, str, name));
++ if (is_zaphod(scrn) && !sna_zaphod_match(sna, name)) {
++ DBG(("%s: zaphod mismatch, want %s, have %s\n",
++ __FUNCTION__,
++ xf86GetOptValString(sna->Options, OPTION_ZAPHOD) ?: "???",
++ name));
+ len = 0;
+ goto skip;
+ }
+@@ -3889,7 +5102,6 @@ sna_output_add(struct sna *sna, unsigned id, unsigned serial)
+ if (strcmp(output->name, name) == 0) {
+ assert(output->scrn == scrn);
+ assert(output->funcs == &sna_output_funcs);
+- assert(to_sna_output(output)->id == 0);
+ sna_output_destroy(output);
+ goto reset;
+ }
+@@ -3935,6 +5147,8 @@ reset:
+ sna_output->id = compat_conn.conn.connector_id;
+ sna_output->is_panel = is_panel(compat_conn.conn.connector_type);
+ sna_output->edid_idx = find_property(sna, sna_output, "EDID");
++ sna_output->link_status_idx =
++ find_property(sna, sna_output, "link-status");
+ if (find_property(sna, sna_output, "scaling mode") != -1)
+ sna_output->add_default_modes =
+ xf86ReturnOptValBool(output->options, OPTION_DEFAULT_MODES, TRUE);
+@@ -3945,10 +5159,8 @@ reset:
+ sna_output->dpms_mode = sna_output->prop_values[i];
+ DBG(("%s: found 'DPMS' (idx=%d, id=%d), initial value=%d\n",
+ __FUNCTION__, i, sna_output->dpms_id, sna_output->dpms_mode));
+- } else {
+- sna_output->dpms_id = -1;
++ } else
+ sna_output->dpms_mode = DPMSModeOff;
+- }
+
+ sna_output->possible_encoders = possible_encoders;
+ sna_output->attached_encoders = attached_encoders;
+@@ -3963,12 +5175,13 @@ reset:
+ sna_output->base = output;
+
+ backlight_init(&sna_output->backlight);
+- if (sna_output->is_panel)
+- sna_output_backlight_init(output);
++ sna_output_backlight_init(output);
+
+ output->possible_crtcs = possible_crtcs & count_to_mask(sna->mode.num_real_crtc);
+ output->interlaceAllowed = TRUE;
+
++ sna_output_load_fake_edid(output);
++
+ if (serial) {
+ if (output->randr_output == NULL) {
+ output->randr_output = RROutputCreate(xf86ScrnToScreen(scrn), name, len, output);
+@@ -3976,6 +5189,7 @@ reset:
+ goto cleanup;
+ }
+
++ RROutputChanged(output->randr_output, TRUE);
+ sna_output_create_resources(output);
+ RRPostPendingProperties(output->randr_output);
+
+@@ -4009,38 +5223,6 @@ skip:
+ return len;
+ }
+
+-static void sna_output_del(xf86OutputPtr output)
+-{
+- ScrnInfoPtr scrn = output->scrn;
+- xf86CrtcConfigPtr config = XF86_CRTC_CONFIG_PTR(scrn);
+- int i;
+-
+- DBG(("%s(%s)\n", __FUNCTION__, output->name));
+- assert(to_sna_output(output));
+-
+- RROutputDestroy(output->randr_output);
+- sna_output_destroy(output);
+-
+- while (output->probed_modes)
+- xf86DeleteMode(&output->probed_modes, output->probed_modes);
+-
+- free(output);
+-
+- for (i = 0; i < config->num_output; i++)
+- if (config->output[i] == output)
+- break;
+- assert(i < to_sna(scrn)->mode.num_real_output);
+- DBG(("%s: removing output #%d of %d\n",
+- __FUNCTION__, i, to_sna(scrn)->mode.num_real_output));
+-
+- for (; i < config->num_output; i++) {
+- config->output[i] = config->output[i+1];
+- config->output[i]->possible_clones >>= 1;
+- }
+- config->num_output--;
+- to_sna(scrn)->mode.num_real_output--;
+-}
+-
+ static int output_rank(const void *A, const void *B)
+ {
+ const xf86OutputPtr *a = A;
+@@ -4058,6 +5240,7 @@ static void sort_config_outputs(struct sna *sna)
+ {
+ xf86CrtcConfigPtr config = XF86_CRTC_CONFIG_PTR(sna->scrn);
+ qsort(config->output, sna->mode.num_real_output, sizeof(*config->output), output_rank);
++ config->compat_output = 0; /* make sure it is a sane value */
+ sna_mode_compute_possible_outputs(sna);
+ }
+
+@@ -4080,11 +5263,15 @@ static bool disable_unused_crtc(struct sna *sna)
+ bool update = false;
+ int o, c;
+
++ DBG(("%s\n", __FUNCTION__));
++
+ for (c = 0; c < sna->mode.num_real_crtc; c++) {
+ xf86CrtcPtr crtc = config->crtc[c];
+
+- if (!crtc->enabled)
++ if (!crtc->enabled) {
++ sna_crtc_disable(crtc, false);
+ continue;
++ }
+
+ for (o = 0; o < sna->mode.num_real_output; o++) {
+ xf86OutputPtr output = config->output[o];
+@@ -4094,7 +5281,7 @@ static bool disable_unused_crtc(struct sna *sna)
+
+ if (o == sna->mode.num_real_output) {
+ DBG(("%s: CRTC:%d was enabled with no outputs\n",
+- __FUNCTION__, to_sna_crtc(crtc)->id));
++ __FUNCTION__, sna_crtc_id(crtc)));
+ crtc->enabled = false;
+ update = true;
+ }
+@@ -4108,17 +5295,145 @@ static bool disable_unused_crtc(struct sna *sna)
+ return update;
+ }
+
+-void sna_mode_discover(struct sna *sna)
++bool sna_mode_find_hotplug_connector(struct sna *sna, unsigned id)
++{
++ xf86CrtcConfigPtr config = XF86_CRTC_CONFIG_PTR(sna->scrn);
++ int i;
++
++ for (i = 0; i < sna->mode.num_real_output; i++) {
++ struct sna_output *output = to_sna_output(config->output[i]);
++ if (output->id == id) {
++ output->reprobe = true;
++ return true;
++ }
++ }
++
++ return false;
++}
++
++static bool
++output_retrain_link(struct sna *sna, struct sna_output *output)
++{
++ struct sna_crtc *crtc = to_sna_crtc(output->base->crtc);
++ int crtc_x = crtc->offset & 0xffff;
++ int crtc_y = crtc->offset >> 16;
++
++ return sna_crtc_flip(sna, crtc, crtc->bo, crtc_x, crtc_y);
++}
++
++static bool
++output_check_link(struct sna *sna, struct sna_output *output)
++{
++ uint64_t link_status;
++
++ if (!output->base->crtc)
++ return true;
++
++ if (output->link_status_idx == -1)
++ return true;
++
++#define LINK_STATUS_GOOD 0
++ link_status = output->prop_values[output->link_status_idx];
++ DBG(("%s: link_status=%d\n", __FUNCTION__, link_status));
++ if (link_status == LINK_STATUS_GOOD)
++ return true;
++
++ /* Perform a modeset as required for "link-status" = BAD */
++ if (!output_retrain_link(sna, output))
++ return false;
++
++ /* Query the "link-status" again to confirm the modeset */
++ update_properties(sna, output);
++
++ link_status = output->prop_values[output->link_status_idx];
++ DBG(("%s: link_status=%d after modeset\n", __FUNCTION__, link_status));
++ return link_status == LINK_STATUS_GOOD;
++}
++
++static bool
++output_check_status(struct sna *sna, struct sna_output *output)
++{
++ union compat_mode_get_connector compat_conn;
++ struct drm_mode_modeinfo dummy;
++ struct drm_mode_get_blob blob;
++ xf86OutputStatus status;
++ char *edid;
++
++ VG_CLEAR(compat_conn);
++
++ compat_conn.conn.connection = -1;
++ compat_conn.conn.connector_id = output->id;
++ compat_conn.conn.count_modes = 1; /* skip detect */
++ compat_conn.conn.modes_ptr = (uintptr_t)&dummy;
++ compat_conn.conn.count_encoders = 0;
++ compat_conn.conn.props_ptr = (uintptr_t)output->prop_ids;
++ compat_conn.conn.prop_values_ptr = (uintptr_t)output->prop_values;
++ compat_conn.conn.count_props = output->num_props;
++
++ if (drmIoctl(sna->kgem.fd,
++ DRM_IOCTL_MODE_GETCONNECTOR,
++ &compat_conn.conn) == 0)
++ output->update_properties = false;
++
++ if (!output_check_link(sna, output))
++ return false;
++
++ if (output->reprobe)
++ return false;
++
++ switch (compat_conn.conn.connection) {
++ case DRM_MODE_CONNECTED:
++ status = XF86OutputStatusConnected;
++ break;
++ case DRM_MODE_DISCONNECTED:
++ status = XF86OutputStatusDisconnected;
++ break;
++ default:
++ case DRM_MODE_UNKNOWNCONNECTION:
++ status = XF86OutputStatusUnknown;
++ break;
++ }
++ if (output->status != status)
++ return false;
++
++ if (status != XF86OutputStatusConnected)
++ return true;
++
++ if (output->num_modes != compat_conn.conn.count_modes)
++ return false;
++
++ if (output->edid_len == 0)
++ return false;
++
++ edid = alloca(output->edid_len);
++
++ VG_CLEAR(blob);
++ blob.blob_id = output->prop_values[output->edid_idx];
++ blob.length = output->edid_len;
++ blob.data = (uintptr_t)edid;
++ if (drmIoctl(sna->kgem.fd, DRM_IOCTL_MODE_GETPROPBLOB, &blob))
++ return false;
++
++ if (blob.length != output->edid_len)
++ return false;
++
++ return memcmp(edid, output->edid_raw, output->edid_len) == 0;
++}
++
++void sna_mode_discover(struct sna *sna, bool tell)
+ {
+ ScreenPtr screen = xf86ScrnToScreen(sna->scrn);
+ xf86CrtcConfigPtr config = XF86_CRTC_CONFIG_PTR(sna->scrn);
++ bool force = sna->flags & SNA_REPROBE;
+ struct drm_mode_card_res res;
+- uint32_t connectors[32];
++ uint32_t connectors[32], now;
+ unsigned changed = 0;
+ unsigned serial;
+ int i, j;
+
+ DBG(("%s()\n", __FUNCTION__));
++ sna->flags &= ~SNA_REPROBE;
++
+ VG_CLEAR(connectors);
+
+ memset(&res, 0, sizeof(res));
+@@ -4128,10 +5443,11 @@ void sna_mode_discover(struct sna *sna)
+ if (drmIoctl(sna->kgem.fd, DRM_IOCTL_MODE_GETRESOURCES, &res))
+ return;
+
+- DBG(("%s: now %d (was %d) connectors\n", __FUNCTION__,
+- res.count_connectors, sna->mode.num_real_output));
++ DBG(("%s: now %d (was %d) connectors, %d encoders, %d crtc\n", __FUNCTION__,
++ res.count_connectors, sna->mode.num_real_output,
++ res.count_encoders, res.count_crtcs));
+ if (res.count_connectors > 32)
+- return;
++ res.count_connectors = 32;
+
+ assert(sna->mode.num_real_crtc == res.count_crtcs || is_zaphod(sna->scrn));
+ assert(sna->mode.max_crtc_width == res.max_width);
+@@ -4142,6 +5458,11 @@ void sna_mode_discover(struct sna *sna)
+ if (serial == 0)
+ serial = ++sna->mode.serial;
+
++ if (force) {
++ changed = 4;
++ now = 0;
++ } else
++ now = GetTimeInMillis();
+ for (i = 0; i < res.count_connectors; i++) {
+ DBG(("%s: connector[%d] = %d\n", __FUNCTION__, i, connectors[i]));
+ for (j = 0; j < sna->mode.num_real_output; j++) {
+@@ -4161,32 +5482,42 @@ void sna_mode_discover(struct sna *sna)
+
+ for (i = 0; i < sna->mode.num_real_output; i++) {
+ xf86OutputPtr output = config->output[i];
++ struct sna_output *sna_output = to_sna_output(output);
+
+- if (to_sna_output(output)->id == 0)
++ if (sna_output->id == 0)
+ continue;
+
+- if (to_sna_output(output)->serial == serial)
++ if (sna_output->serial == serial) {
++ if (output_check_status(sna, sna_output)) {
++ DBG(("%s: output %s (id=%d), retained state\n",
++ __FUNCTION__, output->name, sna_output->id));
++ sna_output->last_detect = now;
++ } else {
++ DBG(("%s: output %s (id=%d), changed state, reprobing\n",
++ __FUNCTION__, output->name, sna_output->id));
++ sna_output->hotplug_count++;
++ sna_output->last_detect = 0;
++ changed |= 4;
++ }
+ continue;
++ }
+
+ DBG(("%s: removing output %s (id=%d), serial=%u [now %u]\n",
+- __FUNCTION__, output->name, to_sna_output(output)->id,
+- to_sna_output(output)->serial, serial));
++ __FUNCTION__, output->name, sna_output->id,
++ sna_output->serial, serial));
+
+ xf86DrvMsg(sna->scrn->scrnIndex, X_INFO,
+- "%s output %s\n",
+- sna->flags & SNA_REMOVE_OUTPUTS ? "Removed" : "Disabled",
++ "Disabled output %s\n",
+ output->name);
+- if (sna->flags & SNA_REMOVE_OUTPUTS) {
+- sna_output_del(output);
+- i--;
+- } else {
+- to_sna_output(output)->id = 0;
+- output->crtc = NULL;
+- }
++ sna_output->id = 0;
++ sna_output->last_detect = 0;
++ output->crtc = NULL;
++ RROutputChanged(output->randr_output, TRUE);
+ changed |= 2;
+ }
+
+- if (changed) {
++ /* Have the list of available outputs been updated? */
++ if (changed & 3) {
+ DBG(("%s: outputs changed, broadcasting\n", __FUNCTION__));
+
+ sna_mode_set_primary(sna);
+@@ -4200,6 +5531,51 @@ void sna_mode_discover(struct sna *sna)
+
+ xf86RandR12TellChanged(screen);
+ }
++
++ /* If anything has changed, refresh the RandR information.
++ * Note this could recurse once from udevless RRGetInfo() probes,
++ * but only once.
++ */
++ if (changed && tell)
++ RRGetInfo(screen, TRUE);
++}
++
++/* Since we only probe the current mode on startup, we may not have the full
++ * list of modes available until the user explicitly requests them. Fake a
++ * hotplug event after a second after starting to fill in any missing modes.
++ */
++static CARD32 sna_mode_coldplug(OsTimerPtr timer, CARD32 now, void *data)
++{
++ struct sna *sna = data;
++ ScreenPtr screen = xf86ScrnToScreen(sna->scrn);
++ xf86CrtcConfigPtr config = XF86_CRTC_CONFIG_PTR(sna->scrn);
++ bool reprobe = false;
++ int i;
++
++ DBG(("%s()\n", __FUNCTION__));
++
++ for (i = 0; i < sna->mode.num_real_output; i++) {
++ xf86OutputPtr output = config->output[i];
++ struct sna_output *sna_output = to_sna_output(output);
++
++ if (sna_output->id == 0)
++ continue;
++ if (sna_output->last_detect)
++ continue;
++ if (output->status == XF86OutputStatusDisconnected)
++ continue;
++
++ DBG(("%s: output %s connected, needs reprobe\n",
++ __FUNCTION__, output->name));
++ reprobe = true;
++ }
++
++ if (reprobe) {
++ RRGetInfo(screen, TRUE);
++ RRTellChanged(screen);
++ }
++ free(timer);
++ return 0;
+ }
+
+ static void copy_front(struct sna *sna, PixmapPtr old, PixmapPtr new)
+@@ -4208,7 +5584,7 @@ static void copy_front(struct sna *sna, PixmapPtr old, PixmapPtr new)
+
+ DBG(("%s\n", __FUNCTION__));
+
+- if (wedged(sna))
++ if (wedged(sna) || isGPU(sna->scrn))
+ return;
+
+ old_priv = sna_pixmap_force_to_gpu(old, MOVE_READ);
+@@ -4220,12 +5596,19 @@ static void copy_front(struct sna *sna, PixmapPtr old, PixmapPtr new)
+ return;
+
+ if (old_priv->clear) {
+- (void)sna->render.fill_one(sna, new, new_priv->gpu_bo,
+- old_priv->clear_color,
+- 0, 0,
+- new->drawable.width,
+- new->drawable.height,
+- GXcopy);
++ bool ok = false;
++ if (!wedged(sna))
++ ok = sna->render.fill_one(sna, new, new_priv->gpu_bo,
++ old_priv->clear_color,
++ 0, 0,
++ new->drawable.width,
++ new->drawable.height,
++ GXcopy);
++ if (!ok) {
++ void *ptr = kgem_bo_map__gtt(&sna->kgem, new_priv->gpu_bo);
++ if (ptr)
++ memset(ptr, 0, new_priv->gpu_bo->pitch*new->drawable.height);
++ }
+ new_priv->clear = true;
+ new_priv->clear_color = old_priv->clear_color;
+ } else {
+@@ -4281,11 +5664,18 @@ static void copy_front(struct sna *sna, PixmapPtr old, PixmapPtr new)
+ __FUNCTION__, box.x2, box.y2, sx, sy, dx, dy));
+
+ if (box.x2 != new->drawable.width || box.y2 != new->drawable.height) {
+- (void)sna->render.fill_one(sna, new, new_priv->gpu_bo, 0,
+- 0, 0,
+- new->drawable.width,
+- new->drawable.height,
+- GXclear);
++ bool ok = false;
++ if (!wedged(sna))
++ ok = sna->render.fill_one(sna, new, new_priv->gpu_bo, 0,
++ 0, 0,
++ new->drawable.width,
++ new->drawable.height,
++ GXclear);
++ if (!ok) {
++ void *ptr = kgem_bo_map__gtt(&sna->kgem, new_priv->gpu_bo);
++ if (ptr)
++ memset(ptr, 0, new_priv->gpu_bo->pitch*new->drawable.height);
++ }
+ }
+ (void)sna->render.copy_boxes(sna, GXcopy,
+ &old->drawable, old_priv->gpu_bo, sx, sy,
+@@ -4302,7 +5692,7 @@ sna_mode_resize(ScrnInfoPtr scrn, int width, int height)
+ {
+ xf86CrtcConfigPtr config = XF86_CRTC_CONFIG_PTR(scrn);
+ struct sna *sna = to_sna(scrn);
+- ScreenPtr screen = scrn->pScreen;
++ ScreenPtr screen = xf86ScrnToScreen(scrn);
+ PixmapPtr new_front;
+ int i;
+
+@@ -4337,9 +5727,20 @@ sna_mode_resize(ScrnInfoPtr scrn, int width, int height)
+ for (i = 0; i < sna->mode.num_real_crtc; i++)
+ sna_crtc_disable_shadow(sna, to_sna_crtc(config->crtc[i]));
+ assert(sna->mode.shadow_active == 0);
++ assert(!sna->mode.shadow_enabled);
+ assert(sna->mode.shadow_damage == NULL);
+ assert(sna->mode.shadow == NULL);
+
++ /* Flush pending shadow updates */
++ if (sna->mode.flip_active) {
++ DBG(("%s: waiting for %d outstanding TearFree flips\n",
++ __FUNCTION__, sna->mode.flip_active));
++ while (sna->mode.flip_active && sna_mode_wait_for_event(sna))
++ sna_mode_wakeup(sna);
++ }
++
++ /* Cancel a pending [un]flip (as the pixmaps no longer match) */
++ sna_present_cancel_flip(sna);
+ copy_front(sna, sna->front, new_front);
+
+ screen->SetScreenPixmap(new_front);
+@@ -4351,14 +5752,6 @@ sna_mode_resize(ScrnInfoPtr scrn, int width, int height)
+ scrn->virtualY = height;
+ scrn->displayWidth = width;
+
+- /* Flush pending shadow updates */
+- if (sna->mode.flip_active) {
+- DBG(("%s: waiting for %d outstanding TearFree flips\n",
+- __FUNCTION__, sna->mode.flip_active));
+- while (sna->mode.flip_active && sna_mode_wait_for_event(sna))
+- sna_mode_wakeup(sna);
+- }
+-
+ /* Only update the CRTCs if we are in control */
+ if (!scrn->vtSema)
+ return TRUE;
+@@ -4371,7 +5764,7 @@ sna_mode_resize(ScrnInfoPtr scrn, int width, int height)
+ continue;
+
+ if (!__sna_crtc_set_mode(crtc))
+- sna_crtc_disable(crtc);
++ sna_crtc_disable(crtc, false);
+ }
+
+ sna_mode_wakeup(sna);
+@@ -4381,19 +5774,6 @@ sna_mode_resize(ScrnInfoPtr scrn, int width, int height)
+ }
+
+ /* cursor handling */
+-struct sna_cursor {
+- struct sna_cursor *next;
+- uint32_t *image;
+- Rotation rotation;
+- int ref;
+- int size;
+- int last_width;
+- int last_height;
+- unsigned handle;
+- unsigned serial;
+- unsigned alloc;
+-};
+-
+ static void
+ rotate_coord(Rotation rotation, int size,
+ int x_dst, int y_dst,
+@@ -4429,36 +5809,6 @@ rotate_coord(Rotation rotation, int size,
+ *y_src = y_dst;
+ }
+
+-static void
+-rotate_coord_back(Rotation rotation, int size, int *x, int *y)
+-{
+- int t;
+-
+- if (rotation & RR_Reflect_X)
+- *x = size - *x - 1;
+- if (rotation & RR_Reflect_Y)
+- *y = size - *y - 1;
+-
+- switch (rotation & 0xf) {
+- case RR_Rotate_0:
+- break;
+- case RR_Rotate_90:
+- t = *x;
+- *x = *y;
+- *y = size - t - 1;
+- break;
+- case RR_Rotate_180:
+- *x = size - *x - 1;
+- *y = size - *y - 1;
+- break;
+- case RR_Rotate_270:
+- t = *x;
+- *x = size - *y - 1;
+- *y = t;
+- break;
+- }
+-}
+-
+ static struct sna_cursor *__sna_create_cursor(struct sna *sna, int size)
+ {
+ struct sna_cursor *c;
+@@ -4519,6 +5869,17 @@ static uint32_t *get_cursor_argb(CursorPtr c)
+ #endif
+ }
+
++static int __cursor_size(int width, int height)
++{
++ int i, size;
++
++ i = MAX(width, height);
++ for (size = 64; size < i; size <<= 1)
++ ;
++
++ return size;
++}
++
+ static struct sna_cursor *__sna_get_cursor(struct sna *sna, xf86CrtcPtr crtc)
+ {
+ struct sna_cursor *cursor;
+@@ -4526,6 +5887,7 @@ static struct sna_cursor *__sna_get_cursor(struct sna *sna, xf86CrtcPtr crtc)
+ const uint32_t *argb;
+ uint32_t *image;
+ int width, height, pitch, size, x, y;
++ bool transformed;
+ Rotation rotation;
+
+ assert(sna->cursor.ref);
+@@ -4537,8 +5899,8 @@ static struct sna_cursor *__sna_get_cursor(struct sna *sna, xf86CrtcPtr crtc)
+ cursor ? cursor->serial : 0,
+ sna->cursor.serial));
+ if (cursor && cursor->serial == sna->cursor.serial) {
+- assert(cursor->size == sna->cursor.size);
+- assert(cursor->rotation == crtc->transform_in_use ? crtc->rotation : RR_Rotate_0);
++ assert(cursor->size == sna->cursor.size || cursor->transformed);
++ assert(cursor->rotation == (!to_sna_crtc(crtc)->cursor_transform && crtc->transform_in_use) ? crtc->rotation : RR_Rotate_0);
+ assert(cursor->ref);
+ return cursor;
+ }
+@@ -4550,22 +5912,81 @@ static struct sna_cursor *__sna_get_cursor(struct sna *sna, xf86CrtcPtr crtc)
+ sna->cursor.serial,
+ get_cursor_argb(sna->cursor.ref) != NULL));
+
+- rotation = crtc->transform_in_use ? crtc->rotation : RR_Rotate_0;
++ transformed = to_sna_crtc(crtc)->cursor_transform;
++ rotation = (!transformed && crtc->transform_in_use) ? crtc->rotation : RR_Rotate_0;
++
++ if (transformed) {
++ struct pixman_box16 box;
++
++ box.x1 = box.y1 = 0;
++ box.x2 = sna->cursor.ref->bits->width;
++ box.y2 = sna->cursor.ref->bits->height;
+
+- if (sna->cursor.use_gtt) { /* Don't allow phys cursor sharing */
++ pixman_f_transform_bounds(&crtc->f_crtc_to_framebuffer, &box);
++ size = __cursor_size(box.x2 - box.x1, box.y2 - box.y1);
++ __DBG(("%s: transformed cursor %dx%d -> %dx%d\n",
++ __FUNCTION__ ,
++ sna->cursor.ref->bits->width,
++ sna->cursor.ref->bits->height,
++ box.x2 - box.x1, box.y2 - box.y1));
++ } else
++ size = sna->cursor.size;
++
++ if (crtc->transform_in_use) {
++ RRTransformPtr T = NULL;
++ struct pixman_vector v;
++
++ if (crtc->transformPresent) {
++ T = &crtc->transform;
++
++ /* Cancel any translation from this affine
++ * transformation. We just want to rotate and scale
++ * the cursor image.
++ */
++ v.vector[0] = 0;
++ v.vector[1] = 0;
++ v.vector[2] = pixman_fixed_1;
++ pixman_transform_point(&crtc->transform.transform, &v);
++ }
++
++ RRTransformCompute(0, 0, size, size, crtc->rotation, T, NULL,
++ &to_sna_crtc(crtc)->cursor_to_fb,
++ &to_sna_crtc(crtc)->fb_to_cursor);
++ if (T)
++ pixman_f_transform_translate(
++ &to_sna_crtc(crtc)->cursor_to_fb,
++ &to_sna_crtc(crtc)->fb_to_cursor,
++ -pixman_fixed_to_double(v.vector[0]),
++ -pixman_fixed_to_double(v.vector[1]));
++
++ __DBG(("%s: cursor_to_fb [%f %f %f, %f %f %f, %f %f %f]\n",
++ __FUNCTION__,
++ to_sna_crtc(crtc)->cursor_to_fb.m[0][0],
++ to_sna_crtc(crtc)->cursor_to_fb.m[0][1],
++ to_sna_crtc(crtc)->cursor_to_fb.m[0][2],
++ to_sna_crtc(crtc)->cursor_to_fb.m[1][0],
++ to_sna_crtc(crtc)->cursor_to_fb.m[1][1],
++ to_sna_crtc(crtc)->cursor_to_fb.m[1][2],
++ to_sna_crtc(crtc)->cursor_to_fb.m[2][0],
++ to_sna_crtc(crtc)->cursor_to_fb.m[2][1],
++ to_sna_crtc(crtc)->cursor_to_fb.m[2][2]));
++ }
++
++ /* Don't allow phys cursor sharing */
++ if (sna->cursor.use_gtt && !transformed) {
+ for (cursor = sna->cursor.cursors; cursor; cursor = cursor->next) {
+- if (cursor->serial == sna->cursor.serial && cursor->rotation == rotation) {
++ if (cursor->serial == sna->cursor.serial &&
++ cursor->rotation == rotation &&
++ !cursor->transformed) {
+ __DBG(("%s: reusing handle=%d, serial=%d, rotation=%d, size=%d\n",
+ __FUNCTION__, cursor->handle, cursor->serial, cursor->rotation, cursor->size));
+ assert(cursor->size == sna->cursor.size);
+ return cursor;
+ }
+ }
+-
+- cursor = to_sna_crtc(crtc)->cursor;
+ }
+
+- size = sna->cursor.size;
++ cursor = to_sna_crtc(crtc)->cursor;
+ if (cursor && cursor->alloc < 4*size*size)
+ cursor = NULL;
+
+@@ -4577,7 +5998,7 @@ static struct sna_cursor *__sna_get_cursor(struct sna *sna, xf86CrtcPtr crtc)
+ }
+ }
+
+- width = sna->cursor.ref->bits->width;
++ width = sna->cursor.ref->bits->width;
+ height = sna->cursor.ref->bits->height;
+ source = sna->cursor.ref->bits->source;
+ mask = sna->cursor.ref->bits->mask;
+@@ -4585,7 +6006,7 @@ static struct sna_cursor *__sna_get_cursor(struct sna *sna, xf86CrtcPtr crtc)
+ pitch = BitmapBytePad(width);
+
+ image = cursor->image;
+- if (image == NULL) {
++ if (image == NULL || transformed) {
+ image = sna->cursor.scratch;
+ cursor->last_width = cursor->last_height = size;
+ }
+@@ -4616,6 +6037,21 @@ static struct sna_cursor *__sna_get_cursor(struct sna *sna, xf86CrtcPtr crtc)
+ mask += pitch;
+ source += pitch;
+ }
++ if (transformed) {
++ __DBG(("%s: Applying affine BLT to bitmap\n", __FUNCTION__));
++ affine_blt(image, cursor->image, 32,
++ 0, 0, width, height, size * 4,
++ 0, 0, size, size, size * 4,
++ &to_sna_crtc(crtc)->cursor_to_fb);
++ image = cursor->image;
++ }
++ } else if (transformed) {
++ __DBG(("%s: Applying affine BLT to ARGB\n", __FUNCTION__));
++ affine_blt(argb, cursor->image, 32,
++ 0, 0, width, height, width * 4,
++ 0, 0, size, size, size * 4,
++ &to_sna_crtc(crtc)->cursor_to_fb);
++ image = cursor->image;
+ } else
+ memcpy_blt(argb, image, 32,
+ width * 4, size * 4,
+@@ -4662,9 +6098,16 @@ static struct sna_cursor *__sna_get_cursor(struct sna *sna, xf86CrtcPtr crtc)
+
+ cursor->size = size;
+ cursor->rotation = rotation;
++ cursor->transformed = transformed;
+ cursor->serial = sna->cursor.serial;
+- cursor->last_width = width;
+- cursor->last_height = height;
++ if (transformed) {
++ /* mark the transformed rectangle as dirty, not input */
++ cursor->last_width = size;
++ cursor->last_height = size;
++ } else {
++ cursor->last_width = width;
++ cursor->last_height = height;
++ }
+ return cursor;
+ }
+
+@@ -4674,40 +6117,55 @@ sna_realize_cursor(xf86CursorInfoPtr info, CursorPtr cursor)
+ return NULL;
+ }
+
+-#if XORG_VERSION_CURRENT >= XORG_VERSION_NUMERIC(1,12,99,901,0)
+-static inline int sigio_block(void)
+-{
+- OsBlockSIGIO();
+- return 0;
+-}
+-static inline void sigio_unblock(int was_blocked)
++static void enable_fb_access(ScrnInfoPtr scrn, int state)
+ {
+- OsReleaseSIGIO();
+- (void)was_blocked;
+-}
++ scrn->EnableDisableFBAccess(
++#ifdef XF86_HAS_SCRN_CONV
++ scrn,
+ #else
+-#include <xf86_OSproc.h>
+-static inline int sigio_block(void)
++ scrn->scrnIndex,
++#endif
++ state);
++}
++
++
++static void __restore_swcursor(ScrnInfoPtr scrn)
+ {
+- return xf86BlockSIGIO();
++ DBG(("%s: attempting to restore SW cursor\n", __FUNCTION__));
++ enable_fb_access(scrn, FALSE);
++ enable_fb_access(scrn, TRUE);
++
++ RemoveBlockAndWakeupHandlers((void *)__restore_swcursor,
++ (void *)NoopDDA,
++ scrn);
+ }
+-static inline void sigio_unblock(int was_blocked)
++
++static void restore_swcursor(struct sna *sna)
+ {
+- xf86UnblockSIGIO(was_blocked);
++ sna->cursor.info->HideCursor(sna->scrn);
++
++ /* XXX Force the cursor to be restored (avoiding recursion) */
++ FreeCursor(sna->cursor.ref, None);
++ sna->cursor.ref = NULL;
++
++ RegisterBlockAndWakeupHandlers((void *)__restore_swcursor,
++ (void *)NoopDDA,
++ sna->scrn);
+ }
+-#endif
+
+ static void
+ sna_show_cursors(ScrnInfoPtr scrn)
+ {
+ xf86CrtcConfigPtr xf86_config = XF86_CRTC_CONFIG_PTR(scrn);
+ struct sna *sna = to_sna(scrn);
++ struct kmsg kmsg;
+ int sigio, c;
+
+ DBG(("%s: cursor?=%d\n", __FUNCTION__, sna->cursor.ref != NULL));
+ if (sna->cursor.ref == NULL)
+ return;
+
++ kmsg_open(&kmsg);
+ sigio = sigio_block();
+ for (c = 0; c < sna->mode.num_real_crtc; c++) {
+ xf86CrtcPtr crtc = xf86_config->crtc[c];
+@@ -4721,7 +6179,7 @@ sna_show_cursors(ScrnInfoPtr scrn)
+
+ if (!crtc->cursor_in_range) {
+ DBG(("%s: skipping cursor outside CRTC (pipe=%d)\n",
+- __FUNCTION__, sna_crtc->pipe));
++ __FUNCTION__, sna_crtc_pipe(crtc)));
+ continue;
+ }
+
+@@ -4729,20 +6187,21 @@ sna_show_cursors(ScrnInfoPtr scrn)
+ if (cursor == NULL ||
+ (sna_crtc->cursor == cursor && sna_crtc->last_cursor_size == cursor->size)) {
+ DBG(("%s: skipping cursor already show on CRTC (pipe=%d)\n",
+- __FUNCTION__, sna_crtc->pipe));
++ __FUNCTION__, sna_crtc_pipe(crtc)));
+ continue;
+ }
+
+ DBG(("%s: CRTC pipe=%d, handle->%d\n", __FUNCTION__,
+- sna_crtc->pipe, cursor->handle));
++ sna_crtc_pipe(crtc), cursor->handle));
+
+ VG_CLEAR(arg);
+ arg.flags = DRM_MODE_CURSOR_BO;
+- arg.crtc_id = sna_crtc->id;
++ arg.crtc_id = __sna_crtc_id(sna_crtc);
+ arg.width = arg.height = cursor->size;
+ arg.handle = cursor->handle;
+
+- if (drmIoctl(sna->kgem.fd, DRM_IOCTL_MODE_CURSOR, &arg) == 0) {
++ if (!FAIL_CURSOR_IOCTL &&
++ drmIoctl(sna->kgem.fd, DRM_IOCTL_MODE_CURSOR, &arg) == 0) {
+ if (sna_crtc->cursor) {
+ assert(sna_crtc->cursor->ref > 0);
+ sna_crtc->cursor->ref--;
+@@ -4750,10 +6209,18 @@ sna_show_cursors(ScrnInfoPtr scrn)
+ cursor->ref++;
+ sna_crtc->cursor = cursor;
+ sna_crtc->last_cursor_size = cursor->size;
++ } else {
++ ERR(("%s: failed to show cursor on CRTC:%d [pipe=%d], disabling hwcursor: errno=%d\n",
++ __FUNCTION__, sna_crtc_id(crtc), sna_crtc_pipe(crtc), errno));
++ sna->cursor.disable = true;
+ }
+ }
+ sigio_unblock(sigio);
+ sna->cursor.active = true;
++ kmsg_close(&kmsg, sna->cursor.disable);
++
++ if (unlikely(sna->cursor.disable))
++ restore_swcursor(sna);
+ }
+
+ static void
+@@ -4789,24 +6256,45 @@ static void
+ sna_crtc_disable_cursor(struct sna *sna, struct sna_crtc *crtc)
+ {
+ struct drm_mode_cursor arg;
++ int sigio;
+
+ if (!crtc->cursor)
+ return;
+
+- DBG(("%s: CRTC:%d, handle=%d\n", __FUNCTION__, crtc->id, crtc->cursor->handle));
+- assert(crtc->cursor->ref);
++ sigio = sigio_block();
++ if (crtc->cursor) {
++ DBG(("%s: CRTC:%d, handle=%d\n", __FUNCTION__, __sna_crtc_id(crtc), crtc->cursor->handle));
++ assert(crtc->cursor->ref > 0);
++ crtc->cursor->ref--;
++ crtc->cursor = NULL;
++ crtc->last_cursor_size = 0;
+
+- VG_CLEAR(arg);
+- arg.flags = DRM_MODE_CURSOR_BO;
+- arg.crtc_id = crtc->id;
+- arg.width = arg.height = 0;
+- arg.handle = 0;
++ VG_CLEAR(arg);
++ arg.flags = DRM_MODE_CURSOR_BO;
++ arg.crtc_id = __sna_crtc_id(crtc);
++ arg.width = arg.height = 0;
++ arg.handle = 0;
+
+- (void)drmIoctl(sna->kgem.fd, DRM_IOCTL_MODE_CURSOR, &arg);
+- assert(crtc->cursor->ref > 0);
+- crtc->cursor->ref--;
+- crtc->cursor = NULL;
+- crtc->last_cursor_size = 0;
++ (void)drmIoctl(sna->kgem.fd, DRM_IOCTL_MODE_CURSOR, &arg);
++ }
++ sigio_unblock(sigio);
++}
++
++static void
++sna_disable_cursors(ScrnInfoPtr scrn)
++{
++ xf86CrtcConfigPtr xf86_config = XF86_CRTC_CONFIG_PTR(scrn);
++ struct sna *sna = to_sna(scrn);
++ int sigio, c;
++
++ DBG(("%s\n", __FUNCTION__));
++
++ sigio = sigio_block();
++ for (c = 0; c < sna->mode.num_real_crtc; c++) {
++ assert(to_sna_crtc(xf86_config->crtc[c]));
++ sna_crtc_disable_cursor(sna, to_sna_crtc(xf86_config->crtc[c]));
++ }
++ sigio_unblock(sigio);
+ }
+
+ static void
+@@ -4852,6 +6340,7 @@ sna_set_cursor_position(ScrnInfoPtr scrn, int x, int y)
+ {
+ xf86CrtcConfigPtr xf86_config = XF86_CRTC_CONFIG_PTR(scrn);
+ struct sna *sna = to_sna(scrn);
++ struct kmsg kmsg;
+ int sigio, c;
+
+ __DBG(("%s(%d, %d), cursor? %d\n", __FUNCTION__,
+@@ -4859,6 +6348,7 @@ sna_set_cursor_position(ScrnInfoPtr scrn, int x, int y)
+ if (sna->cursor.ref == NULL)
+ return;
+
++ kmsg_open(&kmsg);
+ sigio = sigio_block();
+ sna->cursor.last_x = x;
+ sna->cursor.last_y = y;
+@@ -4876,27 +6366,37 @@ sna_set_cursor_position(ScrnInfoPtr scrn, int x, int y)
+
+ VG_CLEAR(arg);
+ arg.flags = 0;
+- arg.crtc_id = sna_crtc->id;
++ arg.crtc_id = __sna_crtc_id(sna_crtc);
+ arg.handle = 0;
+
+ if (sna_crtc->bo == NULL)
+ goto disable;
+
++ cursor = __sna_get_cursor(sna, crtc);
++ if (cursor == NULL)
++ cursor = sna_crtc->cursor;
++ if (cursor == NULL) {
++ __DBG(("%s: failed to grab cursor, disabling\n", __FUNCTION__));
++ goto disable;
++ }
++
+ if (crtc->transform_in_use) {
+ int xhot = sna->cursor.ref->bits->xhot;
+ int yhot = sna->cursor.ref->bits->yhot;
+- struct pict_f_vector v;
++ struct pict_f_vector v, hot;
+
+- v.v[0] = (x + xhot) + 0.5;
+- v.v[1] = (y + yhot) + 0.5;
+- v.v[2] = 1;
++ v.v[0] = x + xhot + .5;
++ v.v[1] = y + yhot + .5;
++ v.v[2] = 1.;
+ pixman_f_transform_point(&crtc->f_framebuffer_to_crtc, &v);
+
+- rotate_coord_back(crtc->rotation, sna->cursor.size, &xhot, &yhot);
++ hot.v[0] = xhot;
++ hot.v[1] = yhot;
++ hot.v[2] = 1.;
++ pixman_f_transform_point(&sna_crtc->fb_to_cursor, &hot);
+
+- /* cursor will have 0.5 added to it already so floor is sufficent */
+- arg.x = floor(v.v[0]) - xhot;
+- arg.y = floor(v.v[1]) - yhot;
++ arg.x = floor(v.v[0] - hot.v[0]);
++ arg.y = floor(v.v[1] - hot.v[1]);
+ } else {
+ arg.x = x - crtc->x;
+ arg.y = y - crtc->y;
+@@ -4904,15 +6404,6 @@ sna_set_cursor_position(ScrnInfoPtr scrn, int x, int y)
+
+ if (arg.x < crtc->mode.HDisplay && arg.x > -sna->cursor.size &&
+ arg.y < crtc->mode.VDisplay && arg.y > -sna->cursor.size) {
+- cursor = __sna_get_cursor(sna, crtc);
+- if (cursor == NULL)
+- cursor = sna_crtc->cursor;
+- if (cursor == NULL) {
+- __DBG(("%s: failed to grab cursor, disabling\n",
+- __FUNCTION__));
+- goto disable;
+- }
+-
+ if (sna_crtc->cursor != cursor || sna_crtc->last_cursor_size != cursor->size) {
+ arg.flags |= DRM_MODE_CURSOR_BO;
+ arg.handle = cursor->handle;
+@@ -4932,10 +6423,13 @@ disable:
+ }
+
+ __DBG(("%s: CRTC:%d (%d, %d), handle=%d, flags=%x (old cursor handle=%d), move? %d, update handle? %d\n",
+- __FUNCTION__, sna_crtc->id, arg.x, arg.y, arg.handle, arg.flags, sna_crtc->cursor ? sna_crtc->cursor->handle : 0,
++ __FUNCTION__, __sna_crtc_id(sna_crtc), arg.x, arg.y, arg.handle, arg.flags, sna_crtc->cursor ? sna_crtc->cursor->handle : 0,
+ arg.flags & DRM_MODE_CURSOR_MOVE, arg.flags & DRM_MODE_CURSOR_BO));
+
+- if (arg.flags &&
++ if (arg.flags == 0)
++ continue;
++
++ if (!FAIL_CURSOR_IOCTL &&
+ drmIoctl(sna->kgem.fd, DRM_IOCTL_MODE_CURSOR, &arg) == 0) {
+ if (arg.flags & DRM_MODE_CURSOR_BO) {
+ if (sna_crtc->cursor) {
+@@ -4949,9 +6443,21 @@ disable:
+ } else
+ sna_crtc->last_cursor_size = 0;
+ }
++ } else {
++ ERR(("%s: failed to update cursor on CRTC:%d [pipe=%d], disabling hwcursor: errno=%d\n",
++ __FUNCTION__, sna_crtc_id(crtc), sna_crtc_pipe(crtc), errno));
++ /* XXX How to force switch back to SW cursor?
++ * Right now we just want until the next cursor image
++ * change, which is fairly frequent.
++ */
++ sna->cursor.disable = true;
+ }
+ }
+ sigio_unblock(sigio);
++ kmsg_close(&kmsg, sna->cursor.disable);
++
++ if (unlikely(sna->cursor.disable))
++ restore_swcursor(sna);
+ }
+
+ #if XORG_VERSION_CURRENT >= XORG_VERSION_NUMERIC(1,15,99,902,2)
+@@ -4978,17 +6484,6 @@ sna_load_cursor_image(ScrnInfoPtr scrn, unsigned char *src)
+ {
+ }
+
+-static int __cursor_size(CursorPtr cursor)
+-{
+- int i, size;
+-
+- i = MAX(cursor->bits->width, cursor->bits->height);
+- for (size = 64; size < i; size <<= 1)
+- ;
+-
+- return size;
+-}
+-
+ static bool
+ sna_cursor_preallocate(struct sna *sna)
+ {
+@@ -5006,6 +6501,50 @@ sna_cursor_preallocate(struct sna *sna)
+ return true;
+ }
+
++static bool
++transformable_cursor(struct sna *sna, CursorPtr cursor)
++{
++ xf86CrtcConfigPtr config = XF86_CRTC_CONFIG_PTR(sna->scrn);
++ int i;
++
++ for (i = 0; i < sna->mode.num_real_crtc; i++) {
++ xf86CrtcPtr crtc = config->crtc[i];
++ struct pixman_box16 box;
++ int size;
++
++ if (!to_sna_crtc(crtc)->hwcursor) {
++ DBG(("%s: hwcursor disabled on CRTC:%d [pipe=%d]\n",
++ __FUNCTION__, sna_crtc_id(crtc), sna_crtc_pipe(crtc)));
++ return false;
++ }
++
++ if (!sna->cursor.use_gtt || !sna->cursor.scratch) {
++ DBG(("%s: unable to use GTT curosor access [%d] or no scratch [%d]\n",
++ __FUNCTION__, sna->cursor.use_gtt, sna->cursor.scratch));
++ return false;
++ }
++
++ box.x1 = box.y1 = 0;
++ box.x2 = cursor->bits->width;
++ box.y2 = cursor->bits->height;
++
++ if (!pixman_f_transform_bounds(&crtc->f_crtc_to_framebuffer,
++ &box)) {
++ DBG(("%s: unable to transform bounds\n", __FUNCTION__));
++ return false;
++ }
++
++ size = __cursor_size(box.x2 - box.x1, box.y2 - box.y1);
++ if (size > sna->cursor.max_size) {
++ DBG(("%s: transformed cursor size=%d too large, max=%d\n",
++ __FUNCTION__, size, sna->cursor.max_size));
++ return false;
++ }
++ }
++
++ return true;
++}
++
+ static Bool
+ sna_use_hw_cursor(ScreenPtr screen, CursorPtr cursor)
+ {
+@@ -5014,6 +6553,9 @@ sna_use_hw_cursor(ScreenPtr screen, CursorPtr cursor)
+ DBG(("%s (%dx%d)?\n", __FUNCTION__,
+ cursor->bits->width, cursor->bits->height));
+
++ if (sna->cursor.disable)
++ return FALSE;
++
+ /* cursors are invariant */
+ if (cursor == sna->cursor.ref)
+ return TRUE;
+@@ -5023,12 +6565,24 @@ sna_use_hw_cursor(ScreenPtr screen, CursorPtr cursor)
+ sna->cursor.ref = NULL;
+ }
+
+- sna->cursor.size = __cursor_size(cursor);
+- if (sna->cursor.size > sna->cursor.max_size)
++ sna->cursor.size =
++ __cursor_size(cursor->bits->width, cursor->bits->height);
++ if (sna->cursor.size > sna->cursor.max_size) {
++ DBG(("%s: cursor size=%d too large, max %d: using sw cursor\n",
++ __FUNCTION__, sna->cursor.size, sna->cursor.max_size));
+ return FALSE;
++ }
++
++ if (sna->mode.rr_active && !transformable_cursor(sna, cursor)) {
++ DBG(("%s: RandR active [%d] and non-transformable cursor: using sw cursor\n",
++ __FUNCTION__, sna->mode.rr_active));
++ return FALSE;
++ }
+
+- if (!sna_cursor_preallocate(sna))
++ if (!sna_cursor_preallocate(sna)) {
++ DBG(("%s: cursor preallocation failed: using sw cursor\n", __FUNCTION__));
+ return FALSE;
++ }
+
+ sna->cursor.ref = cursor;
+ cursor->refcnt++;
+@@ -5056,8 +6610,12 @@ sna_cursor_pre_init(struct sna *sna)
+ return;
+
+ #define LOCAL_IOCTL_GET_CAP DRM_IOWR(0x0c, struct local_get_cap)
+-#define DRM_CAP_CURSOR_WIDTH 8
+-#define DRM_CAP_CURSOR_HEIGHT 9
++#ifndef DRM_CAP_CURSOR_WIDTH
++#define DRM_CAP_CURSOR_WIDTH 0x8
++#endif
++#ifndef DRM_CAP_CURSOR_HEIGHT
++#define DRM_CAP_CURSOR_HEIGHT 0x9
++#endif
+
+ #define I915_PARAM_HAS_COHERENT_PHYS_GTT 29
+
+@@ -5087,11 +6645,9 @@ sna_cursor_pre_init(struct sna *sna)
+ DBG(("%s: cursor updates use_gtt?=%d\n",
+ __FUNCTION__, sna->cursor.use_gtt));
+
+- if (!sna->cursor.use_gtt) {
+- sna->cursor.scratch = malloc(sna->cursor.max_size * sna->cursor.max_size * 4);
+- if (!sna->cursor.scratch)
+- sna->cursor.max_size = 0;
+- }
++ sna->cursor.scratch = malloc(sna->cursor.max_size * sna->cursor.max_size * 4);
++ if (!sna->cursor.scratch && !sna->cursor.use_gtt)
++ sna->cursor.max_size = 0;
+
+ sna->cursor.num_stash = -sna->mode.num_real_crtc;
+
+@@ -5193,7 +6749,7 @@ sna_crtc_flip(struct sna *sna, struct sna_crtc *crtc, struct kgem_bo *bo, int x,
+ int output_count = 0;
+ int i;
+
+- DBG(("%s CRTC:%d [pipe=%d], handle=%d\n", __FUNCTION__, crtc->id, crtc->pipe, bo->handle));
++ DBG(("%s CRTC:%d [pipe=%d], handle=%d\n", __FUNCTION__, __sna_crtc_id(crtc), __sna_crtc_pipe(crtc), bo->handle));
+
+ assert(sna->mode.num_real_output < ARRAY_SIZE(output_ids));
+ assert(crtc->bo);
+@@ -5207,11 +6763,11 @@ sna_crtc_flip(struct sna *sna, struct sna_crtc *crtc, struct kgem_bo *bo, int x,
+
+ DBG(("%s: attaching output '%s' %d [%d] to crtc:%d (pipe %d) (possible crtc:%x, possible clones:%x)\n",
+ __FUNCTION__, output->name, i, to_connector_id(output),
+- crtc->id, crtc->pipe,
++ __sna_crtc_id(crtc), __sna_crtc_pipe(crtc),
+ (uint32_t)output->possible_crtcs,
+ (uint32_t)output->possible_clones));
+
+- assert(output->possible_crtcs & (1 << crtc->pipe) ||
++ assert(output->possible_crtcs & (1 << __sna_crtc_pipe(crtc)) ||
+ is_zaphod(sna->scrn));
+
+ output_ids[output_count] = to_connector_id(output);
+@@ -5221,7 +6777,7 @@ sna_crtc_flip(struct sna *sna, struct sna_crtc *crtc, struct kgem_bo *bo, int x,
+ assert(output_count);
+
+ VG_CLEAR(arg);
+- arg.crtc_id = crtc->id;
++ arg.crtc_id = __sna_crtc_id(crtc);
+ arg.fb_id = fb_id(bo);
+ assert(arg.fb_id);
+ arg.x = x;
+@@ -5231,20 +6787,74 @@ sna_crtc_flip(struct sna *sna, struct sna_crtc *crtc, struct kgem_bo *bo, int x,
+ arg.mode = crtc->kmode;
+ arg.mode_valid = 1;
+
+- DBG(("%s: applying crtc [%d, pipe=%d] mode=%dx%d+%d+%d@%d, fb=%d across %d outputs [%d...]\n",
+- __FUNCTION__, crtc->id, crtc->pipe,
+- arg.mode.hdisplay,
+- arg.mode.vdisplay,
+- arg.x, arg.y,
+- arg.mode.clock,
+- arg.fb_id,
+- output_count, output_count ? output_ids[0] : 0));
++ DBG(("%s: applying crtc [%d, pipe=%d] mode=%dx%d+%d+%d@%d, fb=%d across %d outputs [%d...]\n",
++ __FUNCTION__, __sna_crtc_id(crtc), __sna_crtc_pipe(crtc),
++ arg.mode.hdisplay,
++ arg.mode.vdisplay,
++ arg.x, arg.y,
++ arg.mode.clock,
++ arg.fb_id,
++ output_count, output_count ? output_ids[0] : 0));
++
++ if (drmIoctl(sna->kgem.fd, DRM_IOCTL_MODE_SETCRTC, &arg))
++ return false;
++
++ crtc->offset = y << 16 | x;
++ __kgem_bo_clear_dirty(bo);
++ return true;
++}
++
++static void sna_mode_restore(struct sna *sna)
++{
++ xf86CrtcConfigPtr config = XF86_CRTC_CONFIG_PTR(sna->scrn);
++ int error = 0;
++ int i;
++
++ assert(!sna->mode.hidden);
++
++ for (i = 0; i < sna->mode.num_real_crtc; i++) {
++ xf86CrtcPtr crtc = config->crtc[i];
++
++ assert(to_sna_crtc(crtc) != NULL);
++ if (to_sna_crtc(crtc)->bo == NULL)
++ continue;
++
++ assert(crtc->enabled);
++ if (!__sna_crtc_set_mode(crtc)) {
++ sna_crtc_disable(crtc, false);
++ error++;
++ }
++ }
++ sna_mode_wakeup(sna);
++ while (sna->mode.flip_active && sna_mode_wakeup(sna))
++ ;
++ update_flush_interval(sna);
++ sna_cursors_reload(sna);
++ sna->mode.dirty = false;
++
++ if (error)
++ xf86DrvMsg(sna->scrn->scrnIndex, X_ERROR,
++ "Failed to restore display configuration\n");
++}
++
++bool sna_needs_page_flip(struct sna *sna, struct kgem_bo *bo)
++{
++ xf86CrtcConfigPtr config = XF86_CRTC_CONFIG_PTR(sna->scrn);
++ int i;
++
++ for (i = 0; i < sna->mode.num_real_crtc; i++) {
++ struct sna_crtc *crtc = config->crtc[i]->driver_private;
++
++ if (crtc->bo == NULL)
++ continue;
+
+- if (drmIoctl(sna->kgem.fd, DRM_IOCTL_MODE_SETCRTC, &arg))
+- return false;
++ if (crtc->bo == bo)
++ continue;
+
+- crtc->offset = y << 16 | x;
+- return true;
++ return true;
++ }
++
++ return false;
+ }
+
+ int
+@@ -5256,6 +6866,7 @@ sna_page_flip(struct sna *sna,
+ xf86CrtcConfigPtr config = XF86_CRTC_CONFIG_PTR(sna->scrn);
+ const int width = sna->scrn->virtualX;
+ const int height = sna->scrn->virtualY;
++ int sigio;
+ int count = 0;
+ int i;
+
+@@ -5263,23 +6874,26 @@ sna_page_flip(struct sna *sna,
+ assert(bo->refcnt);
+
+ assert((sna->flags & SNA_IS_HOSTED) == 0);
+- assert((sna->flags & SNA_TEAR_FREE) == 0);
+ assert(sna->mode.flip_active == 0);
+ assert(sna->mode.front_active);
++ assert(!sna->mode.hidden);
+ assert(sna->scrn->vtSema);
+
+ if ((sna->flags & (data ? SNA_HAS_FLIP : SNA_HAS_ASYNC_FLIP)) == 0)
+ return 0;
+
+ kgem_bo_submit(&sna->kgem, bo);
++ __kgem_bo_clear_dirty(bo);
+
++ sigio = sigio_block();
+ for (i = 0; i < sna->mode.num_real_crtc; i++) {
+ struct sna_crtc *crtc = config->crtc[i]->driver_private;
+ struct drm_mode_crtc_page_flip arg;
+ uint32_t crtc_offset;
++ int fixup;
+
+ DBG(("%s: crtc %d id=%d, pipe=%d active? %d\n",
+- __FUNCTION__, i, crtc->id, crtc->pipe, crtc->bo != NULL));
++ __FUNCTION__, i, __sna_crtc_id(crtc), __sna_crtc_pipe(crtc), crtc->bo != NULL));
+ if (crtc->bo == NULL)
+ continue;
+ assert(!crtc->transform);
+@@ -5288,13 +6902,18 @@ sna_page_flip(struct sna *sna,
+ assert(crtc->bo->refcnt >= crtc->bo->active_scanout);
+ assert(crtc->flip_bo == NULL);
+
+- arg.crtc_id = crtc->id;
++ assert_crtc_fb(sna, crtc);
++ if (data == NULL && crtc->bo == bo)
++ goto next_crtc;
++
++ arg.crtc_id = __sna_crtc_id(crtc);
+ arg.fb_id = get_fb(sna, bo, width, height);
+ if (arg.fb_id == 0) {
+ assert(count == 0);
+- return 0;
++ break;
+ }
+
++ fixup = 0;
+ crtc_offset = crtc->base->y << 16 | crtc->base->x;
+
+ if (bo->pitch != crtc->bo->pitch || crtc_offset != crtc->offset) {
+@@ -5303,7 +6922,12 @@ sna_page_flip(struct sna *sna,
+ bo->pitch, crtc->bo->pitch,
+ crtc_offset, crtc->offset));
+ fixup_flip:
++ fixup = 1;
+ if (crtc->bo != bo && sna_crtc_flip(sna, crtc, bo, crtc->base->x, crtc->base->y)) {
++update_scanout:
++ DBG(("%s: removing handle=%d [active_scanout=%d] from scanout, installing handle=%d [active_scanout=%d]\n",
++ __FUNCTION__, crtc->bo->handle, crtc->bo->active_scanout,
++ bo->handle, bo->active_scanout));
+ assert(crtc->bo->active_scanout);
+ assert(crtc->bo->refcnt >= crtc->bo->active_scanout);
+ crtc->bo->active_scanout--;
+@@ -5321,15 +6945,8 @@ fixup_flip:
+ goto next_crtc;
+
+ /* queue a flip in order to send the event */
+- } else {
+- if (count && !xf86SetDesiredModes(sna->scrn)) {
+- xf86DrvMsg(sna->scrn->scrnIndex, X_ERROR,
+- "failed to restore display configuration\n");
+- for (; i < sna->mode.num_real_crtc; i++)
+- sna_crtc_disable(config->crtc[i]);
+- }
+- return 0;
+- }
++ } else
++ goto error;
+ }
+
+ /* Only the reference crtc will finally deliver its page flip
+@@ -5346,7 +6963,7 @@ fixup_flip:
+
+ retry_flip:
+ DBG(("%s: crtc %d id=%d, pipe=%d --> fb %d\n",
+- __FUNCTION__, i, crtc->id, crtc->pipe, arg.fb_id));
++ __FUNCTION__, i, __sna_crtc_id(crtc), __sna_crtc_pipe(crtc), arg.fb_id));
+ if (drmIoctl(sna->kgem.fd, DRM_IOCTL_MODE_PAGE_FLIP, &arg)) {
+ ERR(("%s: pageflip failed with err=%d\n", __FUNCTION__, errno));
+
+@@ -5354,7 +6971,7 @@ retry_flip:
+ struct drm_mode_crtc mode;
+
+ memset(&mode, 0, sizeof(mode));
+- mode.crtc_id = crtc->id;
++ mode.crtc_id = __sna_crtc_id(crtc);
+ drmIoctl(sna->kgem.fd, DRM_IOCTL_MODE_GETCRTC, &mode);
+
+ DBG(("%s: crtc=%d, valid?=%d, fb attached?=%d, expected=%d\n",
+@@ -5366,7 +6983,7 @@ retry_flip:
+ goto fixup_flip;
+
+ if (count == 0)
+- return 0;
++ break;
+
+ DBG(("%s: throttling on busy flip / waiting for kernel to catch up\n", __FUNCTION__));
+ drmIoctl(sna->kgem.fd, DRM_IOCTL_I915_GEM_THROTTLE, 0);
+@@ -5375,15 +6992,25 @@ retry_flip:
+ goto retry_flip;
+ }
+
++ if (!fixup)
++ goto fixup_flip;
++
++error:
+ xf86DrvMsg(sna->scrn->scrnIndex, X_ERROR,
+- "page flipping failed, on CRTC:%d (pipe=%d), disabling %s page flips\n",
+- crtc->id, crtc->pipe, data ? "synchronous": "asynchronous");
++ "page flipping failed, on CRTC:%d (pipe=%d), disabling %s page flips\n",
++ __sna_crtc_id(crtc), __sna_crtc_pipe(crtc), data ? "synchronous": "asynchronous");
++
++ if (count || crtc->bo == bo)
++ sna_mode_restore(sna);
++
+ sna->flags &= ~(data ? SNA_HAS_FLIP : SNA_HAS_ASYNC_FLIP);
+- goto fixup_flip;
++ count = 0;
++ break;
+ }
+
+ if (data) {
+ assert(crtc->flip_bo == NULL);
++ assert(handler);
+ crtc->flip_handler = handler;
+ crtc->flip_data = data;
+ crtc->flip_bo = kgem_bo_reference(bo);
+@@ -5391,11 +7018,15 @@ retry_flip:
+ crtc->flip_serial = crtc->mode_serial;
+ crtc->flip_pending = true;
+ sna->mode.flip_active++;
+- }
+
++ DBG(("%s: recording flip on CRTC:%d handle=%d, active_scanout=%d, serial=%d\n",
++ __FUNCTION__, __sna_crtc_id(crtc), crtc->flip_bo->handle, crtc->flip_bo->active_scanout, crtc->flip_serial));
++ } else
++ goto update_scanout;
+ next_crtc:
+ count++;
+ }
++ sigio_unblock(sigio);
+
+ DBG(("%s: page flipped %d crtcs\n", __FUNCTION__, count));
+ return count;
+@@ -5471,7 +7102,7 @@ static void crtc_init_gamma(xf86CrtcPtr crtc)
+
+ assert(sna_crtc);
+
+- lut.crtc_id = sna_crtc->id;
++ lut.crtc_id = __sna_crtc_id(sna_crtc);
+ lut.gamma_size = 256;
+ lut.red = (uintptr_t)(gamma);
+ lut.green = (uintptr_t)(gamma + 256);
+@@ -5485,7 +7116,7 @@ static void crtc_init_gamma(xf86CrtcPtr crtc)
+ }
+
+ DBG(("%s: CRTC:%d, pipe=%d: gamma set?=%d\n",
+- __FUNCTION__, sna_crtc->id, sna_crtc->pipe,
++ __FUNCTION__, __sna_crtc_id(sna_crtc), __sna_crtc_pipe(sna_crtc),
+ gamma_set));
+ if (!gamma_set) {
+ int i;
+@@ -5502,6 +7133,7 @@ static void crtc_init_gamma(xf86CrtcPtr crtc)
+ crtc->gamma_red = gamma;
+ crtc->gamma_green = gamma + 256;
+ crtc->gamma_blue = gamma + 2*256;
++ crtc->gamma_size = 256;
+ }
+ }
+ }
+@@ -5528,6 +7160,7 @@ static bool sna_probe_initial_configuration(struct sna *sna)
+ {
+ ScrnInfoPtr scrn = sna->scrn;
+ xf86CrtcConfigPtr config = XF86_CRTC_CONFIG_PTR(scrn);
++ int crtc_active, crtc_enabled;
+ int width, height;
+ int i, j;
+
+@@ -5565,6 +7198,7 @@ static bool sna_probe_initial_configuration(struct sna *sna)
+ }
+
+ /* Copy the existing modes on each CRTCs */
++ crtc_active = crtc_enabled = 0;
+ for (i = 0; i < sna->mode.num_real_crtc; i++) {
+ xf86CrtcPtr crtc = config->crtc[i];
+ struct sna_crtc *sna_crtc = to_sna_crtc(crtc);
+@@ -5577,12 +7211,12 @@ static bool sna_probe_initial_configuration(struct sna *sna)
+
+ /* Retrieve the current mode */
+ VG_CLEAR(mode);
+- mode.crtc_id = sna_crtc->id;
++ mode.crtc_id = __sna_crtc_id(sna_crtc);
+ if (drmIoctl(sna->kgem.fd, DRM_IOCTL_MODE_GETCRTC, &mode))
+ continue;
+
+ DBG(("%s: CRTC:%d, pipe=%d: has mode?=%d\n", __FUNCTION__,
+- sna_crtc->id, sna_crtc->pipe,
++ __sna_crtc_id(sna_crtc), __sna_crtc_pipe(sna_crtc),
+ mode.mode_valid && mode.mode.clock));
+
+ if (!mode.mode_valid || mode.mode.clock == 0)
+@@ -5593,6 +7227,7 @@ static bool sna_probe_initial_configuration(struct sna *sna)
+ crtc->desiredX = mode.x;
+ crtc->desiredY = mode.y;
+ crtc->desiredTransformPresent = FALSE;
++ crtc_active++;
+ }
+
+ /* Reconstruct outputs pointing to active CRTC */
+@@ -5604,6 +7239,7 @@ static bool sna_probe_initial_configuration(struct sna *sna)
+
+ crtc_id = (uintptr_t)output->crtc;
+ output->crtc = NULL;
++ output->status = XF86OutputStatusUnknown;
+ if (sna->flags & SNA_IS_SLAVED)
+ continue;
+
+@@ -5623,7 +7259,7 @@ static bool sna_probe_initial_configuration(struct sna *sna)
+ xf86CrtcPtr crtc = config->crtc[j];
+
+ assert(to_sna_crtc(crtc));
+- if (to_sna_crtc(crtc)->id != crtc_id)
++ if (sna_crtc_id(crtc) != crtc_id)
+ continue;
+
+ if (crtc->desiredMode.status == MODE_OK) {
+@@ -5641,18 +7277,30 @@ static bool sna_probe_initial_configuration(struct sna *sna)
+ "Output %s using initial mode %s on pipe %d\n",
+ output->name,
+ crtc->desiredMode.name,
+- to_sna_crtc(crtc)->pipe);
++ sna_crtc_pipe(crtc));
+
+ output->crtc = crtc;
++ output->status = XF86OutputStatusConnected;
+ crtc->enabled = TRUE;
++ crtc_enabled++;
++
++ output_set_gamma(output, crtc);
++
++ if (output->conf_monitor) {
++ output->mm_width = output->conf_monitor->mon_width;
++ output->mm_height = output->conf_monitor->mon_height;
++ }
++
++#if 0
++ sna_output_attach_edid(output);
++ sna_output_attach_tile(output);
++#endif
+
+ if (output->mm_width == 0 || output->mm_height == 0) {
+ output->mm_height = (crtc->desiredMode.VDisplay * 254) / (10*DEFAULT_DPI);
+ output->mm_width = (crtc->desiredMode.HDisplay * 254) / (10*DEFAULT_DPI);
+ }
+
+- output_set_gamma(output, crtc);
+-
+ M = calloc(1, sizeof(DisplayModeRec));
+ if (M) {
+ *M = crtc->desiredMode;
+@@ -5673,6 +7321,12 @@ static bool sna_probe_initial_configuration(struct sna *sna)
+ }
+ }
+
++ if (crtc_active != crtc_enabled) {
++ DBG(("%s: only enabled %d out of %d active CRTC, forcing a reconfigure\n",
++ __FUNCTION__, crtc_enabled, crtc_active));
++ return false;
++ }
++
+ width = height = 0;
+ for (i = 0; i < sna->mode.num_real_crtc; i++) {
+ xf86CrtcPtr crtc = config->crtc[i];
+@@ -5707,8 +7361,8 @@ static bool sna_probe_initial_configuration(struct sna *sna)
+ if (sna_output->num_modes == 0)
+ continue;
+
+- width = sna_output->modes[0].hdisplay;
+- height= sna_output->modes[0].vdisplay;
++ width = sna_output->modes[0].hdisplay;
++ height = sna_output->modes[0].vdisplay;
+
+ DBG(("%s: panel '%s' is %dx%d\n",
+ __FUNCTION__, output->name, width, height));
+@@ -5788,7 +7442,7 @@ probe_capabilities(struct sna *sna)
+ sna->flags &= ~(SNA_HAS_FLIP | SNA_HAS_ASYNC_FLIP);
+ if (has_flip(sna))
+ sna->flags |= SNA_HAS_FLIP;
+- if (has_flip__async(sna))
++ if (has_flip__async(sna) && (sna->flags & SNA_TEAR_FREE) == 0)
+ sna->flags |= SNA_HAS_ASYNC_FLIP;
+ DBG(("%s: page flips? %s, async? %s\n", __FUNCTION__,
+ sna->flags & SNA_HAS_FLIP ? "enabled" : "disabled",
+@@ -5813,12 +7467,25 @@ sna_crtc_config_notify(ScreenPtr screen)
+ return;
+ }
+
++ /* Flush any events completed by the modeset */
++ sna_mode_wakeup(sna);
++
+ update_flush_interval(sna);
++ sna->cursor.disable = false; /* Reset HW cursor until the next fail */
+ sna_cursors_reload(sna);
+
+ probe_capabilities(sna);
+ sna_present_update(sna);
+
++ /* Allow TearFree to come back on when everything is off */
++ if (!sna->mode.front_active && sna->flags & SNA_WANT_TEAR_FREE) {
++ if ((sna->flags & SNA_TEAR_FREE) == 0)
++ DBG(("%s: enable TearFree next modeset\n",
++ __FUNCTION__));
++
++ sna->flags |= SNA_TEAR_FREE;
++ }
++
+ sna->mode.dirty = false;
+ }
+
+@@ -5840,6 +7507,7 @@ bool sna_mode_pre_init(ScrnInfoPtr scrn, struct sna *sna)
+ }
+
+ probe_capabilities(sna);
++ sna->mode.hidden = 1;
+
+ if (!xf86GetOptValInteger(sna->Options, OPTION_VIRTUAL, &num_fake))
+ num_fake = 1;
+@@ -5855,6 +7523,9 @@ bool sna_mode_pre_init(ScrnInfoPtr scrn, struct sna *sna)
+ if (res) {
+ xf86CrtcConfigPtr xf86_config;
+
++ DBG(("%s: found %d CRTC, %d encoders, %d connectors\n",
++ __FUNCTION__, res->count_crtcs, res->count_encoders, res->count_connectors));
++
+ assert(res->count_crtcs);
+ assert(res->count_connectors);
+
+@@ -5862,6 +7533,7 @@ bool sna_mode_pre_init(ScrnInfoPtr scrn, struct sna *sna)
+
+ xf86_config = XF86_CRTC_CONFIG_PTR(scrn);
+ xf86_config->xf86_crtc_notify = sna_crtc_config_notify;
++ xf86_config->compat_output = 0;
+
+ for (i = 0; i < res->count_crtcs; i++)
+ if (!sna_crtc_add(scrn, res->crtcs[i]))
+@@ -5900,6 +7572,11 @@ bool sna_mode_pre_init(ScrnInfoPtr scrn, struct sna *sna)
+ if (!sna_mode_fake_init(sna, num_fake))
+ return false;
+
++ sna->mode.shadow_size = 256;
++ sna->mode.shadow_events = malloc(sna->mode.shadow_size * sizeof(struct drm_event_vblank));
++ if (!sna->mode.shadow_events)
++ return false;
++
+ if (!sna_probe_initial_configuration(sna)) {
+ xf86CrtcConfigPtr config = XF86_CRTC_CONFIG_PTR(scrn);
+
+@@ -5912,6 +7589,7 @@ bool sna_mode_pre_init(ScrnInfoPtr scrn, struct sna *sna)
+ }
+ }
+ sort_config_outputs(sna);
++ TimerSet(NULL, 0, COLDPLUG_DELAY_MS, sna_mode_coldplug, sna);
+
+ sna_setup_provider(scrn);
+ return scrn->modes != NULL;
+@@ -5921,18 +7599,58 @@ bool
+ sna_mode_wants_tear_free(struct sna *sna)
+ {
+ xf86CrtcConfigPtr config = XF86_CRTC_CONFIG_PTR(sna->scrn);
++ bool found = false;
++ FILE *file;
+ int i;
+
++ file = fopen("/sys/module/i915/parameters/enable_fbc", "r");
++ if (file) {
++ int fbc_enabled = 0;
++ int value;
++
++ if (fscanf(file, "%d", &value) == 1)
++ fbc_enabled = value > 0;
++ fclose(file);
++
++ DBG(("%s: module parameter 'enable_fbc' enabled? %d\n",
++ __FUNCTION__, fbc_enabled));
++
++ if (fbc_enabled)
++ return true;
++ }
++
+ for (i = 0; i < sna->mode.num_real_output; i++) {
+ struct sna_output *output = to_sna_output(config->output[i]);
+ int id = find_property(sna, output, "Panel Self-Refresh");
+- if (id !=-1 && output->prop_values[id] != -1) {
++ if (id == -1)
++ continue;
++
++ found = true;
++ if (output->prop_values[id] != -1) {
+ DBG(("%s: Panel Self-Refresh detected on %s\n",
+ __FUNCTION__, config->output[i]->name));
+ return true;
+ }
+ }
+
++ if (!found) {
++ file = fopen("/sys/module/i915/parameters/enable_psr", "r");
++ if (file) {
++ int psr_enabled = 0;
++ int value;
++
++ if (fscanf(file, "%d", &value) == 1)
++ psr_enabled = value > 0;
++ fclose(file);
++
++ DBG(("%s: module parameter 'enable_psr' enabled? %d\n",
++ __FUNCTION__, psr_enabled));
++
++ if (psr_enabled)
++ return true;
++ }
++ }
++
+ return false;
+ }
+
+@@ -5955,7 +7673,7 @@ sna_mode_set_primary(struct sna *sna)
+
+ DBG(("%s: setting PrimaryOutput %s\n", __FUNCTION__, output->name));
+ rr->primaryOutput = output->randr_output;
+- RROutputChanged(rr->primaryOutput, 0);
++ RROutputChanged(rr->primaryOutput, FALSE);
+ rr->layoutChanged = TRUE;
+ break;
+ }
+@@ -5974,12 +7692,9 @@ sna_mode_disable(struct sna *sna)
+ if (!sna->scrn->vtSema)
+ return false;
+
+- /* XXX we will cause previously hidden cursors to be reshown, but
+- * this should be a rare fixup case for severe fragmentation.
+- */
+- sna_hide_cursors(sna->scrn);
++ sna_disable_cursors(sna->scrn);
+ for (i = 0; i < sna->mode.num_real_crtc; i++)
+- sna_crtc_disable(config->crtc[i]);
++ sna_crtc_disable(config->crtc[i], false);
+ assert(sna->mode.front_active == 0);
+
+ sna_mode_wakeup(sna);
+@@ -6001,6 +7716,11 @@ sna_mode_enable(struct sna *sna)
+ if (!sna->scrn->vtSema)
+ return;
+
++ if (sna->mode.hidden) {
++ DBG(("%s: hidden outputs\n", __FUNCTION__));
++ return;
++ }
++
+ for (i = 0; i < sna->mode.num_real_crtc; i++) {
+ xf86CrtcPtr crtc = config->crtc[i];
+
+@@ -6016,13 +7736,30 @@ sna_mode_enable(struct sna *sna)
+ }
+
+ update_flush_interval(sna);
+- sna_show_cursors(sna->scrn);
++ sna_cursors_reload(sna);
+ sna->mode.dirty = false;
+ }
+
++static void sna_randr_close(struct sna *sna)
++{
++ xf86CrtcConfigPtr config = XF86_CRTC_CONFIG_PTR(sna->scrn);
++ int n;
++
++ /* The RR structs are freed early during CloseScreen as they
++ * are tracked as Resources. However, we may be tempted to
++ * access them during shutdown so decouple them now.
++ */
++ for (n = 0; n < config->num_output; n++)
++ config->output[n]->randr_output = NULL;
++
++ for (n = 0; n < config->num_crtc; n++)
++ config->crtc[n]->randr_crtc = NULL;
++}
++
+ void
+ sna_mode_close(struct sna *sna)
+ {
++ sna_randr_close(sna);
+ sna_mode_wakeup(sna);
+
+ if (sna->flags & SNA_IS_HOSTED)
+@@ -6077,15 +7814,22 @@ xf86CrtcPtr
+ sna_covering_crtc(struct sna *sna, const BoxRec *box, xf86CrtcPtr desired)
+ {
+ xf86CrtcConfigPtr config = XF86_CRTC_CONFIG_PTR(sna->scrn);
+- xf86CrtcPtr best_crtc;
+- int best_coverage, c;
++ xf86CrtcPtr best_crtc = NULL;
++ int best_coverage = -1, c;
+
+ if (sna->flags & SNA_IS_HOSTED)
+ return NULL;
+
+ /* If we do not own the VT, we do not own the CRTC either */
+- if (!sna->scrn->vtSema)
++ if (!sna->scrn->vtSema) {
++ DBG(("%s: none, VT switched\n", __FUNCTION__));
++ return NULL;
++ }
++
++ if (sna->mode.hidden) {
++ DBG(("%s: none, hidden outputs\n", __FUNCTION__));
+ return NULL;
++ }
+
+ DBG(("%s for box=(%d, %d), (%d, %d)\n",
+ __FUNCTION__, box->x1, box->y1, box->x2, box->y2));
+@@ -6107,10 +7851,10 @@ sna_covering_crtc(struct sna *sna, const BoxRec *box, xf86CrtcPtr desired)
+ cover_box.x2, cover_box.y2));
+ return desired;
+ }
++ best_crtc = desired;
++ best_coverage = 0;
+ }
+
+- best_crtc = NULL;
+- best_coverage = 0;
+ for (c = 0; c < sna->mode.num_real_crtc; c++) {
+ xf86CrtcPtr crtc = config->crtc[c];
+ BoxRec cover_box;
+@@ -6156,6 +7900,38 @@ sna_covering_crtc(struct sna *sna, const BoxRec *box, xf86CrtcPtr desired)
+ return best_crtc;
+ }
+
++static xf86CrtcPtr first_active_crtc(struct sna *sna)
++{
++ xf86CrtcConfigPtr config = XF86_CRTC_CONFIG_PTR(sna->scrn);
++ int n;
++
++ for (n = 0; n < sna->mode.num_real_crtc; n++) {
++ xf86CrtcPtr crtc = config->crtc[n];
++ if (to_sna_crtc(crtc)->bo)
++ return crtc;
++ }
++
++ /* No active, use the first as a placeholder */
++ if (sna->mode.num_real_crtc)
++ return config->crtc[0];
++
++ return NULL;
++}
++
++xf86CrtcPtr sna_primary_crtc(struct sna *sna)
++{
++ rrScrPrivPtr rr = rrGetScrPriv(xf86ScrnToScreen(sna->scrn));
++ if (rr && rr->primaryOutput) {
++ xf86OutputPtr output = rr->primaryOutput->devPrivate;
++ if (output->crtc &&
++ output->scrn == sna->scrn &&
++ to_sna_crtc(output->crtc))
++ return output->crtc;
++ }
++
++ return first_active_crtc(sna);
++}
++
+ #define MI_LOAD_REGISTER_IMM (0x22<<23)
+
+ static bool sna_emit_wait_for_scanline_hsw(struct sna *sna,
+@@ -6433,7 +8209,7 @@ sna_wait_for_scanline(struct sna *sna,
+ y2 /= 2;
+ }
+
+- pipe = sna_crtc_to_pipe(crtc);
++ pipe = sna_crtc_pipe(crtc);
+ DBG(("%s: pipe=%d, y1=%d, y2=%d, full_height?=%d\n",
+ __FUNCTION__, pipe, y1, y2, full_height));
+
+@@ -6457,19 +8233,101 @@ sna_wait_for_scanline(struct sna *sna,
+ return ret;
+ }
+
++static bool sna_mode_shutdown_crtc(xf86CrtcPtr crtc)
++{
++ struct sna *sna = to_sna(crtc->scrn);
++ xf86CrtcConfigPtr config = XF86_CRTC_CONFIG_PTR(crtc->scrn);
++ bool disabled = false;
++ int o;
++
++ xf86DrvMsg(crtc->scrn->scrnIndex, X_ERROR,
++ "%s: invalid state found on pipe %d, disabling CRTC:%d\n",
++ __FUNCTION__,
++ __sna_crtc_pipe(to_sna_crtc(crtc)),
++ __sna_crtc_id(to_sna_crtc(crtc)));
++ sna_crtc_disable(crtc, true);
++#if XF86_CRTC_VERSION >= 3
++ crtc->active = FALSE;
++#endif
++ if (crtc->enabled) {
++ crtc->enabled = FALSE;
++ disabled = true;
++ }
++
++ for (o = 0; o < sna->mode.num_real_output; o++) {
++ xf86OutputPtr output = config->output[o];
++
++ if (output->crtc != crtc)
++ continue;
++
++ output->funcs->dpms(output, DPMSModeOff);
++ output->crtc = NULL;
++ }
++
++ return disabled;
++}
++
++static bool
++sna_mode_disable_secondary_planes(struct sna *sna)
++{
++ xf86CrtcConfigPtr config = XF86_CRTC_CONFIG_PTR(sna->scrn);
++ bool disabled = false;
++ int c;
++
++ /* Disable all secondary planes on our CRTCs, just in case
++ * other userspace left garbage in them.
++ */
++ for (c = 0; c < sna->mode.num_real_crtc; c++) {
++ xf86CrtcPtr crtc = config->crtc[c];
++ struct sna_crtc *sna_crtc = to_sna_crtc(crtc);
++ struct plane *plane;
++
++ list_for_each_entry(plane, &sna_crtc->sprites, link) {
++ struct local_mode_get_plane p;
++ struct local_mode_set_plane s;
++
++ VG_CLEAR(p);
++ p.plane_id = plane->id;
++ p.count_format_types = 0;
++ if (drmIoctl(sna->kgem.fd,
++ LOCAL_IOCTL_MODE_GETPLANE,
++ &p))
++ continue;
++
++ if (p.fb_id == 0 || p.crtc_id == 0)
++ continue;
++
++ memset(&s, 0, sizeof(s));
++ s.plane_id = p.plane_id;
++ s.crtc_id = p.crtc_id;
++ if (drmIoctl(sna->kgem.fd,
++ LOCAL_IOCTL_MODE_SETPLANE,
++ &s))
++ disabled |= sna_mode_shutdown_crtc(crtc);
++ }
++ }
++
++ return disabled;
++}
++
+ void sna_mode_check(struct sna *sna)
+ {
+ xf86CrtcConfigPtr config = XF86_CRTC_CONFIG_PTR(sna->scrn);
+- int i;
++ bool disabled;
++ int c, o;
+
+ if (sna->flags & SNA_IS_HOSTED)
+ return;
+
+- DBG(("%s\n", __FUNCTION__));
++ DBG(("%s: hidden?=%d\n", __FUNCTION__, sna->mode.hidden));
++ if (sna->mode.hidden)
++ return;
++
++ disabled = sna_mode_disable_secondary_planes(sna);
+
+ /* Validate CRTC attachments and force consistency upon the kernel */
+- for (i = 0; i < sna->mode.num_real_crtc; i++) {
+- xf86CrtcPtr crtc = config->crtc[i];
++ for (c = 0; c < sna->mode.num_real_crtc; c++) {
++ xf86CrtcPtr crtc = config->crtc[c];
+ struct sna_crtc *sna_crtc = to_sna_crtc(crtc);
+ struct drm_mode_crtc mode;
+ uint32_t expected[2];
+@@ -6483,7 +8341,7 @@ void sna_mode_check(struct sna *sna)
+ expected[1] = sna_crtc->flip_bo ? fb_id(sna_crtc->flip_bo) : -1;
+
+ VG_CLEAR(mode);
+- mode.crtc_id = sna_crtc->id;
++ mode.crtc_id = __sna_crtc_id(sna_crtc);
+ if (drmIoctl(sna->kgem.fd, DRM_IOCTL_MODE_GETCRTC, &mode))
+ continue;
+
+@@ -6492,16 +8350,12 @@ void sna_mode_check(struct sna *sna)
+ mode.crtc_id, mode.mode_valid,
+ mode.fb_id, expected[0], expected[1]));
+
+- if (mode.fb_id != expected[0] && mode.fb_id != expected[1]) {
+- xf86DrvMsg(crtc->scrn->scrnIndex, X_ERROR,
+- "%s: invalid state found on pipe %d, disabling CRTC:%d\n",
+- __FUNCTION__, sna_crtc->pipe, sna_crtc->id);
+- sna_crtc_disable(crtc);
+- }
++ if (mode.fb_id != expected[0] && mode.fb_id != expected[1])
++ disabled |= sna_mode_shutdown_crtc(crtc);
+ }
+
+- for (i = 0; i < config->num_output; i++) {
+- xf86OutputPtr output = config->output[i];
++ for (o = 0; o < config->num_output; o++) {
++ xf86OutputPtr output = config->output[o];
+ struct sna_output *sna_output;
+
+ if (output->crtc)
+@@ -6515,26 +8369,16 @@ void sna_mode_check(struct sna *sna)
+ }
+
+ update_flush_interval(sna);
++
++ if (disabled)
++ xf86RandR12TellChanged(xf86ScrnToScreen(sna->scrn));
+ }
+
+ static bool
+ sna_crtc_hide_planes(struct sna *sna, struct sna_crtc *crtc)
+ {
+-#define LOCAL_IOCTL_MODE_SETPLANE DRM_IOWR(0xB7, struct local_mode_set_plane)
+- struct local_mode_set_plane {
+- uint32_t plane_id;
+- uint32_t crtc_id;
+- uint32_t fb_id; /* fb object contains surface format type */
+- uint32_t flags;
+-
+- /* Signed dest location allows it to be partially off screen */
+- int32_t crtc_x, crtc_y;
+- uint32_t crtc_w, crtc_h;
+-
+- /* Source values are 16.16 fixed point */
+- uint32_t src_x, src_y;
+- uint32_t src_h, src_w;
+- } s;
++ struct local_mode_set_plane s;
++ struct plane *plane;
+
+ if (crtc->primary.id == 0)
+ return false;
+@@ -6544,8 +8388,10 @@ sna_crtc_hide_planes(struct sna *sna, struct sna_crtc *crtc)
+ if (drmIoctl(sna->kgem.fd, LOCAL_IOCTL_MODE_SETPLANE, &s))
+ return false;
+
+- s.plane_id = crtc->sprite.id;
+- (void)drmIoctl(sna->kgem.fd, LOCAL_IOCTL_MODE_SETPLANE, &s);
++ list_for_each_entry(plane, &crtc->sprites, link) {
++ s.plane_id = plane->id;
++ (void)drmIoctl(sna->kgem.fd, LOCAL_IOCTL_MODE_SETPLANE, &s);
++ }
+
+ __sna_crtc_disable(sna, crtc);
+ return true;
+@@ -6561,21 +8407,22 @@ void sna_mode_reset(struct sna *sna)
+
+ DBG(("%s\n", __FUNCTION__));
+
+- sna_hide_cursors(sna->scrn);
++ sna_disable_cursors(sna->scrn);
+ for (i = 0; i < sna->mode.num_real_crtc; i++)
+ if (!sna_crtc_hide_planes(sna, to_sna_crtc(config->crtc[i])))
+- sna_crtc_disable(config->crtc[i]);
++ sna_crtc_disable(config->crtc[i], true);
+ assert(sna->mode.front_active == 0);
+
+ for (i = 0; i < sna->mode.num_real_crtc; i++) {
+ struct sna_crtc *sna_crtc = to_sna_crtc(config->crtc[i]);
++ struct plane *plane;
+
+ assert(sna_crtc != NULL);
+- sna_crtc->dpms_mode = -1;
+
+ /* Force the rotation property to be reset on next use */
+ rotation_reset(&sna_crtc->primary);
+- rotation_reset(&sna_crtc->sprite);
++ list_for_each_entry(plane, &sna_crtc->sprites, link)
++ rotation_reset(plane);
+ }
+
+ /* VT switching, likely to be fbcon so make the backlight usable */
+@@ -6641,9 +8488,10 @@ sna_crtc_redisplay__fallback(xf86CrtcPtr crtc, RegionPtr region, struct kgem_bo
+ {
+ int16_t sx, sy;
+ struct sna *sna = to_sna(crtc->scrn);
+- ScreenPtr screen = sna->scrn->pScreen;
++ ScreenPtr screen = xf86ScrnToScreen(crtc->scrn);
+ DrawablePtr draw = crtc_source(crtc, &sx, &sy);
+ PictFormatPtr format;
++ PictTransform T;
+ PicturePtr src, dst;
+ PixmapPtr pixmap;
+ int depth, error;
+@@ -6664,6 +8512,14 @@ sna_crtc_redisplay__fallback(xf86CrtcPtr crtc, RegionPtr region, struct kgem_bo
+ __FUNCTION__, format->format, depth, draw->bitsPerPixel,
+ bo->pitch, crtc->mode.HDisplay, crtc->mode.VDisplay));
+
++ if (sx | sy)
++ RegionTranslate(region, sx, sy);
++ error = !sna_drawable_move_region_to_cpu(draw, region, MOVE_READ);
++ if (sx | sy)
++ RegionTranslate(region, -sx, -sy);
++ if (error)
++ return;
++
+ ptr = kgem_bo_map__gtt(&sna->kgem, bo);
+ if (ptr == NULL)
+ return;
+@@ -6683,9 +8539,37 @@ sna_crtc_redisplay__fallback(xf86CrtcPtr crtc, RegionPtr region, struct kgem_bo
+ if (!src)
+ goto free_pixmap;
+
+- error = SetPictureTransform(src, &crtc->crtc_to_framebuffer);
+- if (error)
+- goto free_src;
++ pixman_transform_init_translate(&T, sx << 16, sy << 16);
++ pixman_transform_multiply(&T, &T, &crtc->crtc_to_framebuffer);
++ if (!sna_transform_is_integer_translation(&T, &sx, &sy)) {
++#define f2d(x) (((double)(x))/65536.)
++ DBG(("%s: transform=[[%f %f %f], [%f %f %f], [%f %f %f]] (raw [[%x %x %x], [%x %x %x], [%x %x %x]])\n",
++ __FUNCTION__,
++ f2d(T.matrix[0][0]),
++ f2d(T.matrix[0][1]),
++ f2d(T.matrix[0][2]),
++ f2d(T.matrix[1][0]),
++ f2d(T.matrix[1][1]),
++ f2d(T.matrix[1][2]),
++ f2d(T.matrix[2][0]),
++ f2d(T.matrix[2][1]),
++ f2d(T.matrix[2][2]),
++ T.matrix[0][0],
++ T.matrix[0][1],
++ T.matrix[0][2],
++ T.matrix[1][0],
++ T.matrix[1][1],
++ T.matrix[1][2],
++ T.matrix[2][0],
++ T.matrix[2][1],
++ T.matrix[2][2]));
++#undef f2d
++
++ error = SetPictureTransform(src, &T);
++ if (error)
++ goto free_src;
++ sx = sy = 0;
++ }
+
+ if (crtc->filter && crtc->transform_in_use)
+ SetPicturePictFilter(src, crtc->filter,
+@@ -6733,10 +8617,11 @@ sna_crtc_redisplay__composite(xf86CrtcPtr crtc, RegionPtr region, struct kgem_bo
+ {
+ int16_t sx, sy;
+ struct sna *sna = to_sna(crtc->scrn);
+- ScreenPtr screen = crtc->scrn->pScreen;
++ ScreenPtr screen = xf86ScrnToScreen(crtc->scrn);
+ DrawablePtr draw = crtc_source(crtc, &sx, &sy);
+ struct sna_composite_op tmp;
+ PictFormatPtr format;
++ PictTransform T;
+ PicturePtr src, dst;
+ PixmapPtr pixmap;
+ const BoxRec *b;
+@@ -6777,9 +8662,14 @@ sna_crtc_redisplay__composite(xf86CrtcPtr crtc, RegionPtr region, struct kgem_bo
+ if (!src)
+ goto free_pixmap;
+
+- error = SetPictureTransform(src, &crtc->crtc_to_framebuffer);
+- if (error)
+- goto free_src;
++ pixman_transform_init_translate(&T, sx << 16, sy << 16);
++ pixman_transform_multiply(&T, &T, &crtc->crtc_to_framebuffer);
++ if (!sna_transform_is_integer_translation(&T, &sx, &sy)) {
++ error = SetPictureTransform(src, &T);
++ if (error)
++ goto free_src;
++ sx = sy = 0;
++ }
+
+ if (crtc->filter && crtc->transform_in_use)
+ SetPicturePictFilter(src, crtc->filter,
+@@ -6793,36 +8683,38 @@ sna_crtc_redisplay__composite(xf86CrtcPtr crtc, RegionPtr region, struct kgem_bo
+ ValidatePicture(src);
+ ValidatePicture(dst);
+
+- if (!sna->render.composite(sna,
+- PictOpSrc, src, NULL, dst,
+- sx, sy,
+- 0, 0,
+- 0, 0,
+- crtc->mode.HDisplay, crtc->mode.VDisplay,
+- COMPOSITE_PARTIAL, memset(&tmp, 0, sizeof(tmp)))) {
+- DBG(("%s: unsupported operation!\n", __FUNCTION__));
+- sna_crtc_redisplay__fallback(crtc, region, bo);
+- goto free_dst;
+- }
+-
++ /* Composite each box individually as if we are dealing with a rotation
++ * on a large display, we may have to perform intermediate copies. We
++ * can then minimise the overdraw by looking at individual boxes rather
++ * than the bbox.
++ */
+ n = region_num_rects(region);
+ b = region_rects(region);
+ do {
+- BoxRec box;
+-
+- box = *b++;
++ BoxRec box = *b;
+ transformed_box(&box, crtc);
+
+ DBG(("%s: (%d, %d)x(%d, %d) -> (%d, %d), (%d, %d)\n",
+ __FUNCTION__,
+- b[-1].x1, b[-1].y1, b[-1].x2-b[-1].x1, b[-1].y2-b[-1].y1,
++ b->x1, b->y1, b->x2-b->x1, b->y2-b->y1,
+ box.x1, box.y1, box.x2, box.y2));
+
+- tmp.box(sna, &tmp, &box);
+- } while (--n);
+- tmp.done(sna, &tmp);
++ if (!sna->render.composite(sna,
++ PictOpSrc, src, NULL, dst,
++ sx + box.x1, sy + box.y1,
++ 0, 0,
++ box.x1, box.y1,
++ box.x2 - box.x1, box.y2 - box.y1,
++ 0, memset(&tmp, 0, sizeof(tmp)))) {
++ DBG(("%s: unsupported operation!\n", __FUNCTION__));
++ sna_crtc_redisplay__fallback(crtc, region, bo);
++ break;
++ } else {
++ tmp.box(sna, &tmp, &box);
++ tmp.done(sna, &tmp);
++ }
++ } while (b++, --n);
+
+-free_dst:
+ FreePicture(dst, None);
+ free_src:
+ FreePicture(src, None);
+@@ -6839,7 +8731,7 @@ sna_crtc_redisplay(xf86CrtcPtr crtc, RegionPtr region, struct kgem_bo *bo)
+ struct sna_pixmap *priv = sna_pixmap((PixmapPtr)draw);
+
+ DBG(("%s: crtc %d [pipe=%d], damage (%d, %d), (%d, %d) x %d\n",
+- __FUNCTION__, to_sna_crtc(crtc)->id, to_sna_crtc(crtc)->pipe,
++ __FUNCTION__, sna_crtc_id(crtc), sna_crtc_pipe(crtc),
+ region->extents.x1, region->extents.y1,
+ region->extents.x2, region->extents.y2,
+ region_num_rects(region)));
+@@ -6898,7 +8790,10 @@ sna_crtc_redisplay(xf86CrtcPtr crtc, RegionPtr region, struct kgem_bo *bo)
+ static void shadow_flip_handler(struct drm_event_vblank *e,
+ void *data)
+ {
+- sna_mode_redisplay(data);
++ struct sna *sna = data;
++
++ if (!sna->mode.shadow_wait)
++ sna_mode_redisplay(sna);
+ }
+
+ void sna_shadow_set_crtc(struct sna *sna,
+@@ -6908,18 +8803,23 @@ void sna_shadow_set_crtc(struct sna *sna,
+ struct sna_crtc *sna_crtc = to_sna_crtc(crtc);
+ struct sna_pixmap *priv;
+
++ assert(sna_crtc);
+ DBG(("%s: setting shadow override for CRTC:%d to handle=%d\n",
+- __FUNCTION__, sna_crtc->id, bo->handle));
++ __FUNCTION__, __sna_crtc_id(sna_crtc), bo->handle));
+
+ assert(sna->flags & SNA_TEAR_FREE);
+- assert(sna_crtc);
+ assert(!sna_crtc->transform);
+
+ if (sna_crtc->client_bo != bo) {
+- if (sna_crtc->client_bo)
++ if (sna_crtc->client_bo) {
++ assert(sna_crtc->client_bo->refcnt >= sna_crtc->client_bo->active_scanout);
++ sna_crtc->client_bo->active_scanout--;
+ kgem_bo_destroy(&sna->kgem, sna_crtc->client_bo);
++ }
+
+ sna_crtc->client_bo = kgem_bo_reference(bo);
++ sna_crtc->client_bo->active_scanout++;
++ assert(sna_crtc->client_bo->refcnt >= sna_crtc->client_bo->active_scanout);
+ sna_crtc_damage(crtc);
+ }
+
+@@ -6969,11 +8869,13 @@ void sna_shadow_unset_crtc(struct sna *sna,
+ struct sna_crtc *sna_crtc = to_sna_crtc(crtc);
+
+ DBG(("%s: clearin shadow override for CRTC:%d\n",
+- __FUNCTION__, sna_crtc->id));
++ __FUNCTION__, __sna_crtc_id(sna_crtc)));
+
+ if (sna_crtc->client_bo == NULL)
+ return;
+
++ assert(sna_crtc->client_bo->refcnt >= sna_crtc->client_bo->active_scanout);
++ sna_crtc->client_bo->active_scanout--;
+ kgem_bo_destroy(&sna->kgem, sna_crtc->client_bo);
+ sna_crtc->client_bo = NULL;
+ list_del(&sna_crtc->shadow_link);
+@@ -6982,15 +8884,57 @@ void sna_shadow_unset_crtc(struct sna *sna,
+ sna_crtc_damage(crtc);
+ }
+
++static bool move_crtc_to_gpu(struct sna *sna)
++{
++ xf86CrtcConfigPtr config = XF86_CRTC_CONFIG_PTR(sna->scrn);
++ int i;
++
++ for (i = 0; i < sna->mode.num_real_crtc; i++) {
++ struct sna_crtc *crtc = to_sna_crtc(config->crtc[i]);
++ unsigned hint;
++
++ assert(crtc);
++
++ if (crtc->bo == NULL)
++ continue;
++
++ if (crtc->slave_pixmap)
++ continue;
++
++ if (crtc->client_bo)
++ continue;
++
++ if (crtc->shadow_bo)
++ continue;
++
++ hint = MOVE_READ | MOVE_ASYNC_HINT | __MOVE_SCANOUT;
++ if (sna->flags & SNA_TEAR_FREE)
++ hint |= __MOVE_FORCE;
++
++ DBG(("%s: CRTC %d [pipe=%d] requires frontbuffer\n",
++ __FUNCTION__, __sna_crtc_id(crtc), __sna_crtc_pipe(crtc)));
++ return sna_pixmap_move_to_gpu(sna->front, hint);
++ }
++
++ return true;
++}
++
+ void sna_mode_redisplay(struct sna *sna)
+ {
+ xf86CrtcConfigPtr config = XF86_CRTC_CONFIG_PTR(sna->scrn);
+ RegionPtr region;
+ int i;
+
+- if (!sna->mode.shadow_damage)
++ if (sna->mode.hidden) {
++ DBG(("%s: hidden outputs, skipping\n", __FUNCTION__));
++ return;
++ }
++
++ if (!sna->mode.shadow_enabled)
+ return;
+
++ assert(sna->mode.shadow_damage);
++
+ DBG(("%s: posting shadow damage? %d (flips pending? %d, mode reconfiguration pending? %d)\n",
+ __FUNCTION__,
+ !RegionNil(DamageRegion(sna->mode.shadow_damage)),
+@@ -7012,21 +8956,23 @@ void sna_mode_redisplay(struct sna *sna)
+ region->extents.x2, region->extents.y2));
+
+ if (sna->mode.flip_active) {
+- DamagePtr damage;
+-
+- damage = sna->mode.shadow_damage;
+- sna->mode.shadow_damage = NULL;
++ DBG(("%s: checking for %d outstanding flip completions\n",
++ __FUNCTION__, sna->mode.flip_active));
+
++ sna->mode.dirty = true;
+ while (sna->mode.flip_active && sna_mode_wakeup(sna))
+ ;
++ sna->mode.dirty = false;
+
+- sna->mode.shadow_damage = damage;
++ DBG(("%s: now %d outstanding flip completions (enabled? %d)\n",
++ __FUNCTION__,
++ sna->mode.flip_active,
++ sna->mode.shadow_enabled));
++ if (sna->mode.flip_active || !sna->mode.shadow_enabled)
++ return;
+ }
+
+- if (sna->mode.flip_active)
+- return;
+-
+- if (wedged(sna) || !sna_pixmap_move_to_gpu(sna->front, MOVE_READ | MOVE_ASYNC_HINT | __MOVE_SCANOUT)) {
++ if (!move_crtc_to_gpu(sna)) {
+ DBG(("%s: forcing scanout update using the CPU\n", __FUNCTION__));
+ if (!sna_pixmap_move_to_cpu(sna->front, MOVE_READ))
+ return;
+@@ -7047,90 +8993,14 @@ void sna_mode_redisplay(struct sna *sna)
+ damage.data = NULL;
+ RegionIntersect(&damage, &damage, region);
+ if (!box_empty(&damage.extents)) {
+- struct kgem_bo *bo = NULL;
+-
+ DBG(("%s: fallback intersects pipe=%d [(%d, %d), (%d, %d)]\n",
+- __FUNCTION__, sna_crtc->pipe,
++ __FUNCTION__, __sna_crtc_pipe(sna_crtc),
+ damage.extents.x1, damage.extents.y1,
+ damage.extents.x2, damage.extents.y2));
+
+- if (sna->flags & SNA_TEAR_FREE) {
+- RegionRec new_damage;
+-
+- RegionNull(&new_damage);
+- RegionCopy(&new_damage, &damage);
+-
+- bo = sna_crtc->client_bo;
+- if (bo == NULL) {
+- damage.extents = crtc->bounds;
+- damage.data = NULL;
+- bo = kgem_create_2d(&sna->kgem,
+- crtc->mode.HDisplay,
+- crtc->mode.VDisplay,
+- crtc->scrn->bitsPerPixel,
+- sna_crtc->bo->tiling,
+- CREATE_SCANOUT);
+- } else
+- RegionUnion(&damage, &damage, &sna_crtc->client_damage);
+-
+- DBG(("%s: TearFree fallback, shadow handle=%d, crtc handle=%d\n", __FUNCTION__, bo->handle, sna_crtc->bo->handle));
+-
+- sna_crtc->client_damage = new_damage;
+- }
+-
+- if (bo == NULL)
+- bo = sna_crtc->bo;
+- sna_crtc_redisplay__fallback(crtc, &damage, bo);
+-
+- if (bo != sna_crtc->bo) {
+- struct drm_mode_crtc_page_flip arg;
+-
+- arg.crtc_id = sna_crtc->id;
+- arg.fb_id = get_fb(sna, bo,
+- crtc->mode.HDisplay,
+- crtc->mode.VDisplay);
+-
+- arg.user_data = (uintptr_t)sna_crtc;
+- arg.flags = DRM_MODE_PAGE_FLIP_EVENT;
+- arg.reserved = 0;
+-
+- if (drmIoctl(sna->kgem.fd, DRM_IOCTL_MODE_PAGE_FLIP, &arg)) {
+- if (sna_crtc_flip(sna, sna_crtc, bo, 0, 0)) {
+- assert(sna_crtc->bo->active_scanout);
+- assert(sna_crtc->bo->refcnt >= sna_crtc->bo->active_scanout);
+- sna_crtc->bo->active_scanout--;
+- kgem_bo_destroy(&sna->kgem, sna_crtc->bo);
+-
+- sna_crtc->bo = bo;
+- sna_crtc->bo->active_scanout++;
+- sna_crtc->client_bo = NULL;
+- } else {
+- DBG(("%s: flip [fb=%d] on crtc %d [%d, pipe=%d] failed - %d\n",
+- __FUNCTION__, arg.fb_id, i, sna_crtc->id, sna_crtc->pipe, errno));
+- xf86DrvMsg(sna->scrn->scrnIndex, X_ERROR,
+- "Page flipping failed, disabling TearFree\n");
+- sna->flags &= ~SNA_TEAR_FREE;
+-
+- damage.extents = crtc->bounds;
+- damage.data = NULL;
+- sna_crtc_redisplay__fallback(crtc, &damage, sna_crtc->bo);
+-
+- kgem_bo_destroy(&sna->kgem, bo);
+- sna_crtc->client_bo = NULL;
+- }
+- } else {
+- sna->mode.flip_active++;
+-
+- assert(sna_crtc->flip_bo == NULL);
+- sna_crtc->flip_handler = shadow_flip_handler;
+- sna_crtc->flip_data = sna;
+- sna_crtc->flip_bo = bo;
+- sna_crtc->flip_bo->active_scanout++;
+- sna_crtc->flip_serial = sna_crtc->mode_serial;
+-
+- sna_crtc->client_bo = kgem_bo_reference(sna_crtc->bo);
+- }
+- }
++ sna_crtc_redisplay__fallback(crtc,
++ &damage,
++ sna_crtc->bo);
+ }
+ RegionUninit(&damage);
+
+@@ -7171,6 +9041,7 @@ void sna_mode_redisplay(struct sna *sna)
+ xf86CrtcPtr crtc = config->crtc[i];
+ struct sna_crtc *sna_crtc = to_sna_crtc(crtc);
+ RegionRec damage;
++ int sigio;
+
+ assert(sna_crtc != NULL);
+ DBG(("%s: crtc[%d] transformed? %d\n",
+@@ -7192,30 +9063,38 @@ void sna_mode_redisplay(struct sna *sna)
+ region_num_rects(&damage),
+ damage.extents.x1, damage.extents.y1,
+ damage.extents.x2, damage.extents.y2));
++ sigio = sigio_block();
+ if (!box_empty(&damage.extents)) {
+ if (sna->flags & SNA_TEAR_FREE) {
++ RegionRec new_damage;
+ struct drm_mode_crtc_page_flip arg;
+ struct kgem_bo *bo;
+
+- RegionUninit(&damage);
+- damage.extents = crtc->bounds;
+- damage.data = NULL;
++ RegionNull(&new_damage);
++ RegionCopy(&new_damage, &damage);
+
+- bo = sna_crtc->client_bo;
+- if (bo == NULL)
++ bo = sna_crtc->cache_bo;
++ if (bo == NULL) {
++ damage.extents = crtc->bounds;
++ damage.data = NULL;
+ bo = kgem_create_2d(&sna->kgem,
+ crtc->mode.HDisplay,
+ crtc->mode.VDisplay,
+ crtc->scrn->bitsPerPixel,
+ sna_crtc->bo->tiling,
+ CREATE_SCANOUT);
+- if (bo == NULL)
+- goto disable1;
++ if (bo == NULL)
++ continue;
++ } else
++ RegionUnion(&damage, &damage, &sna_crtc->crtc_damage);
++ sna_crtc->crtc_damage = new_damage;
+
+ sna_crtc_redisplay(crtc, &damage, bo);
+ kgem_bo_submit(&sna->kgem, bo);
++ __kgem_bo_clear_dirty(bo);
+
+- arg.crtc_id = sna_crtc->id;
++ assert_crtc_fb(sna, sna_crtc);
++ arg.crtc_id = __sna_crtc_id(sna_crtc);
+ arg.fb_id = get_fb(sna, bo,
+ crtc->mode.HDisplay,
+ crtc->mode.VDisplay);
+@@ -7228,6 +9107,9 @@ void sna_mode_redisplay(struct sna *sna)
+
+ if (drmIoctl(sna->kgem.fd, DRM_IOCTL_MODE_PAGE_FLIP, &arg)) {
+ if (sna_crtc_flip(sna, sna_crtc, bo, 0, 0)) {
++ DBG(("%s: removing handle=%d [active_scanout=%d] from scanout, installing handle=%d [active_scanout=%d]\n",
++ __FUNCTION__, sna_crtc->bo->handle, sna_crtc->bo->active_scanout - 1,
++ bo->handle, bo->active_scanout));
+ assert(sna_crtc->bo->active_scanout);
+ assert(sna_crtc->bo->refcnt >= sna_crtc->bo->active_scanout);
+ sna_crtc->bo->active_scanout--;
+@@ -7235,13 +9117,12 @@ void sna_mode_redisplay(struct sna *sna)
+
+ sna_crtc->bo = kgem_bo_reference(bo);
+ sna_crtc->bo->active_scanout++;
+- sna_crtc->client_bo = kgem_bo_reference(bo);
+ } else {
+ BoxRec box;
+ DrawableRec tmp;
+
+ DBG(("%s: flip [fb=%d] on crtc %d [%d, pipe=%d] failed - %d\n",
+- __FUNCTION__, arg.fb_id, i, sna_crtc->id, sna_crtc->pipe, errno));
++ __FUNCTION__, arg.fb_id, i, __sna_crtc_id(sna_crtc), __sna_crtc_pipe(sna_crtc), errno));
+ xf86DrvMsg(sna->scrn->scrnIndex, X_ERROR,
+ "Page flipping failed, disabling TearFree\n");
+ sna->flags &= ~SNA_TEAR_FREE;
+@@ -7260,13 +9141,13 @@ disable1:
+ &box, 1, COPY_LAST)) {
+ xf86DrvMsg(crtc->scrn->scrnIndex, X_ERROR,
+ "%s: page flipping failed, disabling CRTC:%d (pipe=%d)\n",
+- __FUNCTION__, sna_crtc->id, sna_crtc->pipe);
+- sna_crtc_disable(crtc);
++ __FUNCTION__, __sna_crtc_id(sna_crtc), __sna_crtc_pipe(sna_crtc));
++ sna_crtc_disable(crtc, false);
+ }
+-
+- kgem_bo_destroy(&sna->kgem, bo);
+- sna_crtc->client_bo = NULL;
+ }
++
++ kgem_bo_destroy(&sna->kgem, bo);
++ sna_crtc->cache_bo = NULL;
+ continue;
+ }
+ sna->mode.flip_active++;
+@@ -7279,13 +9160,20 @@ disable1:
+ sna_crtc->flip_serial = sna_crtc->mode_serial;
+ sna_crtc->flip_pending = true;
+
+- sna_crtc->client_bo = kgem_bo_reference(sna_crtc->bo);
++ if (sna_crtc->bo != sna->mode.shadow) {
++ assert_scanout(&sna->kgem, sna_crtc->bo,
++ crtc->mode.HDisplay, crtc->mode.VDisplay);
++ sna_crtc->cache_bo = kgem_bo_reference(sna_crtc->bo);
++ }
++ DBG(("%s: recording flip on CRTC:%d handle=%d, active_scanout=%d, serial=%d\n",
++ __FUNCTION__, __sna_crtc_id(sna_crtc), sna_crtc->flip_bo->handle, sna_crtc->flip_bo->active_scanout, sna_crtc->flip_serial));
+ } else {
+ sna_crtc_redisplay(crtc, &damage, sna_crtc->bo);
+ kgem_scanout_flush(&sna->kgem, sna_crtc->bo);
+ }
+ }
+ RegionUninit(&damage);
++ sigio_unblock(sigio);
+
+ if (sna_crtc->slave_damage)
+ DamageEmpty(sna_crtc->slave_damage);
+@@ -7296,6 +9184,7 @@ disable1:
+ struct kgem_bo *old = sna->mode.shadow;
+ struct drm_mode_crtc_page_flip arg;
+ uint32_t fb = 0;
++ int sigio;
+
+ DBG(("%s: flipping TearFree outputs, current scanout handle=%d [active?=%d], new handle=%d [active=%d]\n",
+ __FUNCTION__, old->handle, old->active_scanout, new->handle, new->active_scanout));
+@@ -7307,7 +9196,9 @@ disable1:
+ arg.reserved = 0;
+
+ kgem_bo_submit(&sna->kgem, new);
++ __kgem_bo_clear_dirty(new);
+
++ sigio = sigio_block();
+ for (i = 0; i < sna->mode.num_real_crtc; i++) {
+ struct sna_crtc *crtc = config->crtc[i]->driver_private;
+ struct kgem_bo *flip_bo;
+@@ -7315,20 +9206,20 @@ disable1:
+
+ assert(crtc != NULL);
+ DBG(("%s: crtc %d [%d, pipe=%d] active? %d, transformed? %d\n",
+- __FUNCTION__, i, crtc->id, crtc->pipe, crtc->bo ? crtc->bo->handle : 0, crtc->transform));
++ __FUNCTION__, i, __sna_crtc_id(crtc), __sna_crtc_pipe(crtc), crtc->bo ? crtc->bo->handle : 0, crtc->transform));
+ if (crtc->bo == NULL || crtc->transform)
+ continue;
+
+ assert(config->crtc[i]->enabled);
+- assert(crtc->dpms_mode <= DPMSModeOn);
+ assert(crtc->flip_bo == NULL);
++ assert_crtc_fb(sna, crtc);
+
+- arg.crtc_id = crtc->id;
++ arg.crtc_id = __sna_crtc_id(crtc);
+ arg.user_data = (uintptr_t)crtc;
+
+ if (crtc->client_bo) {
+ DBG(("%s: apply shadow override bo for CRTC:%d on pipe=%d, handle=%d\n",
+- __FUNCTION__, crtc->id, crtc->pipe, crtc->client_bo->handle));
++ __FUNCTION__, __sna_crtc_id(crtc), __sna_crtc_pipe(crtc), crtc->client_bo->handle));
+ arg.fb_id = get_fb(sna, crtc->client_bo,
+ crtc->base->mode.HDisplay,
+ crtc->base->mode.VDisplay);
+@@ -7356,6 +9247,7 @@ fixup_shadow:
+ }
+ }
+
++ sigio_unblock(sigio);
+ return;
+ }
+
+@@ -7365,8 +9257,12 @@ fixup_shadow:
+ y = crtc->base->y;
+ }
+
+- if (crtc->bo == flip_bo)
++ if (crtc->bo == flip_bo) {
++ assert(crtc->bo->refcnt >= crtc->bo->active_scanout);
++ DBG(("%s: flip handle=%d is already on the CRTC\n",
++ __FUNCTION__, flip_bo->handle));
+ continue;
++ }
+
+ if (flip_bo->pitch != crtc->bo->pitch || (y << 16 | x) != crtc->offset) {
+ DBG(("%s: changing pitch (new %d =?= old %d) or offset (new %x =?= old %x)\n",
+@@ -7375,6 +9271,9 @@ fixup_shadow:
+ y << 16 | x, crtc->offset));
+ fixup_flip:
+ if (sna_crtc_flip(sna, crtc, flip_bo, x, y)) {
++ DBG(("%s: removing handle=%d [active_scanout=%d] from scanout, installing handle=%d [active_scanout=%d]\n",
++ __FUNCTION__, crtc->bo->handle, crtc->bo->active_scanout-1,
++ flip_bo->handle, flip_bo->active_scanout));
+ assert(flip_bo != crtc->bo);
+ assert(crtc->bo->active_scanout);
+ assert(crtc->bo->refcnt >= crtc->bo->active_scanout);
+@@ -7389,9 +9288,11 @@ fixup_flip:
+ crtc->bo = kgem_bo_reference(flip_bo);
+ crtc->bo->active_scanout++;
+ } else {
+- xf86DrvMsg(sna->scrn->scrnIndex, X_ERROR,
+- "Failed to prepare CRTC for page flipping, disabling TearFree\n");
+- sna->flags &= ~SNA_TEAR_FREE;
++ if (sna->flags & SNA_TEAR_FREE) {
++ xf86DrvMsg(sna->scrn->scrnIndex, X_ERROR,
++ "Failed to prepare CRTC for page flipping, disabling TearFree\n");
++ sna->flags &= ~SNA_TEAR_FREE;
++ }
+
+ if (sna->mode.flip_active == 0) {
+ DBG(("%s: abandoning flip attempt\n", __FUNCTION__));
+@@ -7400,15 +9301,15 @@ fixup_flip:
+
+ xf86DrvMsg(sna->scrn->scrnIndex, X_ERROR,
+ "%s: page flipping failed, disabling CRTC:%d (pipe=%d)\n",
+- __FUNCTION__, crtc->id, crtc->pipe);
+- sna_crtc_disable(crtc->base);
++ __FUNCTION__, __sna_crtc_id(crtc), __sna_crtc_pipe(crtc));
++ sna_crtc_disable(crtc->base, false);
+ }
+ continue;
+ }
+
+ if (drmIoctl(sna->kgem.fd, DRM_IOCTL_MODE_PAGE_FLIP, &arg)) {
+ ERR(("%s: flip [fb=%d] on crtc %d [%d, pipe=%d] failed - %d\n",
+- __FUNCTION__, arg.fb_id, i, crtc->id, crtc->pipe, errno));
++ __FUNCTION__, arg.fb_id, i, __sna_crtc_id(crtc), __sna_crtc_pipe(crtc), errno));
+ goto fixup_flip;
+ }
+ sna->mode.flip_active++;
+@@ -7421,6 +9322,9 @@ fixup_flip:
+ crtc->flip_serial = crtc->mode_serial;
+ crtc->flip_pending = true;
+
++ DBG(("%s: recording flip on CRTC:%d handle=%d, active_scanout=%d, serial=%d\n",
++ __FUNCTION__, __sna_crtc_id(crtc), crtc->flip_bo->handle, crtc->flip_bo->active_scanout, crtc->flip_serial));
++
+ {
+ struct drm_i915_gem_busy busy = { flip_bo->handle };
+ if (drmIoctl(sna->kgem.fd, DRM_IOCTL_I915_GEM_BUSY, &busy) == 0) {
+@@ -7435,6 +9339,7 @@ fixup_flip:
+ }
+ }
+ }
++ sigio_unblock(sigio);
+
+ DBG(("%s: flipped %d outputs, shadow active? %d\n",
+ __FUNCTION__,
+@@ -7486,7 +9391,9 @@ again:
+ struct drm_event *e = (struct drm_event *)&buffer[i];
+ switch (e->type) {
+ case DRM_EVENT_VBLANK:
+- if (((uintptr_t)((struct drm_event_vblank *)e)->user_data) & 2)
++ if (sna->mode.shadow_wait)
++ defer_event(sna, e);
++ else if (((uintptr_t)((struct drm_event_vblank *)e)->user_data) & 2)
+ sna_present_vblank_handler((struct drm_event_vblank *)e);
+ else
+ sna_dri2_vblank_handler((struct drm_event_vblank *)e);
+@@ -7495,13 +9402,19 @@ again:
+ {
+ struct drm_event_vblank *vbl = (struct drm_event_vblank *)e;
+ struct sna_crtc *crtc = (void *)(uintptr_t)vbl->user_data;
++ uint64_t msc;
+
+ /* Beware Zaphod! */
+ sna = to_sna(crtc->base->scrn);
+
+- crtc->swap.tv_sec = vbl->tv_sec;
+- crtc->swap.tv_usec = vbl->tv_usec;
+- crtc->swap.msc = msc64(crtc, vbl->sequence);
++ if (msc64(crtc, vbl->sequence, &msc)) {
++ DBG(("%s: recording last swap on pipe=%d, frame %d [%08llx], time %d.%06d\n",
++ __FUNCTION__, __sna_crtc_pipe(crtc), vbl->sequence, (long long)msc, vbl->tv_sec, vbl->tv_usec));
++ crtc->swap.tv_sec = vbl->tv_sec;
++ crtc->swap.tv_usec = vbl->tv_usec;
++ crtc->swap.msc = msc;
++ }
++ assert(crtc->flip_pending);
+ crtc->flip_pending = false;
+
+ assert(crtc->flip_bo);
+@@ -7509,10 +9422,12 @@ again:
+ assert(crtc->flip_bo->refcnt >= crtc->flip_bo->active_scanout);
+
+ if (crtc->flip_serial == crtc->mode_serial) {
+- DBG(("%s: removing handle=%d from scanout, installing handle=%d\n",
+- __FUNCTION__, crtc->bo->handle, crtc->flip_bo->handle));
++ DBG(("%s: removing handle=%d [active_scanout=%d] from scanout, installing handle=%d [active_scanout=%d]\n",
++ __FUNCTION__, crtc->bo->handle, crtc->bo->active_scanout - 1,
++ crtc->flip_bo->handle, crtc->flip_bo->active_scanout));
+ assert(crtc->bo->active_scanout);
+ assert(crtc->bo->refcnt >= crtc->bo->active_scanout);
++
+ crtc->bo->active_scanout--;
+ kgem_bo_destroy(&sna->kgem, crtc->bo);
+
+@@ -7523,6 +9438,8 @@ again:
+
+ crtc->bo = crtc->flip_bo;
+ crtc->flip_bo = NULL;
++
++ assert_crtc_fb(sna, crtc);
+ } else {
+ crtc->flip_bo->active_scanout--;
+ kgem_bo_destroy(&sna->kgem, crtc->flip_bo);
+@@ -7531,8 +9448,10 @@ again:
+
+ DBG(("%s: flip complete, pending? %d\n", __FUNCTION__, sna->mode.flip_active));
+ assert(sna->mode.flip_active);
+- if (--sna->mode.flip_active == 0)
++ if (--sna->mode.flip_active == 0) {
++ assert(crtc->flip_handler);
+ crtc->flip_handler(vbl, crtc->flip_data);
++ }
+ }
+ break;
+ default:
+diff --git a/src/sna/sna_display_fake.c b/src/sna/sna_display_fake.c
+index 4d74c38d..fa26bda1 100644
+--- a/src/sna/sna_display_fake.c
++++ b/src/sna/sna_display_fake.c
+@@ -96,12 +96,6 @@ sna_crtc_set_mode_major(xf86CrtcPtr crtc, DisplayModePtr mode,
+ }
+
+ static void
+-sna_crtc_gamma_set(xf86CrtcPtr crtc,
+- CARD16 *red, CARD16 *green, CARD16 *blue, int size)
+-{
+-}
+-
+-static void
+ sna_crtc_destroy(xf86CrtcPtr crtc)
+ {
+ }
+@@ -109,7 +103,6 @@ sna_crtc_destroy(xf86CrtcPtr crtc)
+ static const xf86CrtcFuncsRec sna_crtc_funcs = {
+ .dpms = sna_crtc_dpms,
+ .set_mode_major = sna_crtc_set_mode_major,
+- .gamma_set = sna_crtc_gamma_set,
+ .destroy = sna_crtc_destroy,
+ };
+
+@@ -192,7 +185,7 @@ static const xf86OutputFuncsRec sna_output_funcs = {
+ static Bool
+ sna_mode_resize(ScrnInfoPtr scrn, int width, int height)
+ {
+- ScreenPtr screen = scrn->pScreen;
++ ScreenPtr screen = xf86ScrnToScreen(scrn);
+ PixmapPtr new_front;
+
+ DBG(("%s (%d, %d) -> (%d, %d)\n", __FUNCTION__,
+@@ -262,6 +255,7 @@ static bool add_fake_output(struct sna *sna, bool late)
+ output->mm_height = 0;
+ output->interlaceAllowed = FALSE;
+ output->subpixel_order = SubPixelNone;
++ output->status = XF86OutputStatusDisconnected;
+
+ output->possible_crtcs = ~((1 << sna->mode.num_real_crtc) - 1);
+ output->possible_clones = ~((1 << sna->mode.num_real_output) - 1);
+@@ -297,6 +291,8 @@ static bool add_fake_output(struct sna *sna, bool late)
+
+ RRCrtcSetRotations(crtc->randr_crtc,
+ RR_Rotate_All | RR_Reflect_All);
++ if (!RRCrtcGammaSetSize(crtc->randr_crtc, 256))
++ goto err;
+ }
+
+ sna->mode.num_fake++;
+@@ -312,13 +308,16 @@ err:
+ continue;
+
+ xf86OutputDestroy(output);
++ i--;
+ }
+
+ for (i = 0; i < xf86_config->num_crtc; i++) {
+ crtc = xf86_config->crtc[i];
+ if (crtc->driver_private)
+ continue;
++
+ xf86CrtcDestroy(crtc);
++ i--;
+ }
+ sna->mode.num_fake = -1;
+ return false;
+diff --git a/src/sna/sna_dri2.c b/src/sna/sna_dri2.c
+index e5c4d53e..d89525cc 100644
+--- a/src/sna/sna_dri2.c
++++ b/src/sna/sna_dri2.c
+@@ -82,12 +82,23 @@ get_private(void *buffer)
+ return (struct sna_dri2_private *)((DRI2Buffer2Ptr)buffer+1);
+ }
+
++pure static inline DRI2BufferPtr sna_pixmap_get_buffer(PixmapPtr pixmap)
++{
++ assert(pixmap->refcnt);
++ return ((void **)__get_private(pixmap, sna_pixmap_key))[2];
++}
++
++static inline void sna_pixmap_set_buffer(PixmapPtr pixmap, void *ptr)
++{
++ assert(pixmap->refcnt);
++ ((void **)__get_private(pixmap, sna_pixmap_key))[2] = ptr;
++}
++
+ #if DRI2INFOREC_VERSION >= 4
+ enum event_type {
+ WAITMSC = 0,
+ SWAP,
+- SWAP_WAIT,
+- SWAP_THROTTLE,
++ SWAP_COMPLETE,
+ FLIP,
+ FLIP_THROTTLE,
+ FLIP_COMPLETE,
+@@ -98,6 +109,7 @@ struct dri_bo {
+ struct list link;
+ struct kgem_bo *bo;
+ uint32_t name;
++ unsigned flags;
+ };
+
+ struct sna_dri2_event {
+@@ -108,6 +120,8 @@ struct sna_dri2_event {
+ xf86CrtcPtr crtc;
+ int pipe;
+ bool queued;
++ bool sync;
++ bool chained;
+
+ /* for swaps & flips only */
+ DRI2SwapEventPtr event_complete;
+@@ -116,35 +130,146 @@ struct sna_dri2_event {
+ DRI2BufferPtr back;
+ struct kgem_bo *bo;
+
++ struct copy {
++ struct kgem_bo *bo;
++ unsigned flags;
++ uint32_t name;
++ uint32_t size;
++ } pending;
++
+ struct sna_dri2_event *chain;
+
+- struct list cache;
+ struct list link;
+
+- int mode;
++ int flip_continue;
++ int keepalive;
++ int signal;
+ };
+
++#if DRI2INFOREC_VERSION < 10
++#undef USE_ASYNC_SWAP
++#endif
++
++#if USE_ASYNC_SWAP
++#define KEEPALIVE 8 /* wait ~100ms before discarding swap caches */
++#define APPLY_DAMAGE 0
++#else
++#define USE_ASYNC_SWAP 0
++#define KEEPALIVE 1
++#define APPLY_DAMAGE 1
++#endif
++
+ static void sna_dri2_flip_event(struct sna_dri2_event *flip);
++inline static DRI2BufferPtr dri2_window_get_front(WindowPtr win);
++
++static struct kgem_bo *
++__sna_dri2_copy_region(struct sna *sna, DrawablePtr draw, RegionPtr region,
++ DRI2BufferPtr src, DRI2BufferPtr dst,
++ unsigned flags);
++
++inline static void
++__sna_dri2_copy_event(struct sna_dri2_event *info, unsigned flags)
++{
++ DBG(("%s: flags = %x\n", __FUNCTION__, flags));
++ assert(info->front != info->back);
++ info->bo = __sna_dri2_copy_region(info->sna, info->draw, NULL,
++ info->back, info->front,
++ flags);
++ info->front->flags = info->back->flags;
++}
++
++static int front_pitch(DrawablePtr draw)
++{
++ DRI2BufferPtr buffer;
++
++ buffer = NULL;
++ if (draw->type != DRAWABLE_PIXMAP)
++ buffer = dri2_window_get_front((WindowPtr)draw);
++ if (buffer == NULL)
++ buffer = sna_pixmap_get_buffer(get_drawable_pixmap(draw));
++
++ return buffer ? buffer->pitch : 0;
++}
++
++struct dri2_window {
++ DRI2BufferPtr front;
++ struct sna_dri2_event *chain;
++ xf86CrtcPtr crtc;
++ int64_t msc_delta;
++ struct list cache;
++ uint32_t cache_size;
++ int scanout;
++};
++
++static struct dri2_window *dri2_window(WindowPtr win)
++{
++ assert(win->drawable.type != DRAWABLE_PIXMAP);
++ return ((void **)__get_private(win, sna_window_key))[1];
++}
++
++static bool use_scanout(struct sna *sna,
++ DrawablePtr draw,
++ struct dri2_window *priv)
++{
++ if (priv->front)
++ return true;
++
++ if (priv->scanout < 0)
++ priv->scanout =
++ (sna->flags & (SNA_LINEAR_FB | SNA_NO_WAIT | SNA_NO_FLIP)) == 0 &&
++ draw->width == sna->front->drawable.width &&
++ draw->height == sna->front->drawable.height &&
++ draw->bitsPerPixel == sna->front->drawable.bitsPerPixel;
++
++ return priv->scanout;
++}
+
+ static void
+ sna_dri2_get_back(struct sna *sna,
+ DrawablePtr draw,
+- DRI2BufferPtr back,
+- struct sna_dri2_event *info)
++ DRI2BufferPtr back)
+ {
++ struct dri2_window *priv = dri2_window((WindowPtr)draw);
++ uint32_t size;
+ struct kgem_bo *bo;
++ struct dri_bo *c;
+ uint32_t name;
++ int flags;
+ bool reuse;
+
+- DBG(("%s: draw size=%dx%d, buffer size=%dx%d\n",
++ DBG(("%s: draw size=%dx%d, back buffer handle=%d size=%dx%d, is-scanout? %d, active?=%d, pitch=%d, front pitch=%d\n",
+ __FUNCTION__, draw->width, draw->height,
+- get_private(back)->size & 0xffff, get_private(back)->size >> 16));
+- reuse = (draw->height << 16 | draw->width) == get_private(back)->size;
++ get_private(back)->bo->handle,
++ get_private(back)->size & 0xffff, get_private(back)->size >> 16,
++ get_private(back)->bo->scanout,
++ get_private(back)->bo->active_scanout,
++ back->pitch, front_pitch(draw)));
++ assert(priv);
++
++ size = draw->height << 16 | draw->width;
++ if (size != priv->cache_size) {
++ while (!list_is_empty(&priv->cache)) {
++ c = list_first_entry(&priv->cache, struct dri_bo, link);
++ list_del(&c->link);
++
++ DBG(("%s: releasing cached handle=%d\n", __FUNCTION__, c->bo ? c->bo->handle : 0));
++ assert(c->bo);
++ kgem_bo_destroy(&sna->kgem, c->bo);
++
++ free(c);
++ }
++ priv->cache_size = size;
++ }
++
++ reuse = size == get_private(back)->size;
++ if (reuse)
++ reuse = get_private(back)->bo->scanout == use_scanout(sna, draw, priv);
++ DBG(("%s: reuse backbuffer? %d\n", __FUNCTION__, reuse));
+ if (reuse) {
+ bo = get_private(back)->bo;
+ assert(bo->refcnt);
+- DBG(("%s: back buffer handle=%d, scanout?=%d, refcnt=%d\n",
+- __FUNCTION__, bo->handle, bo->active_scanout, get_private(back)->refcnt));
++ DBG(("%s: back buffer handle=%d, active?=%d, refcnt=%d\n",
++ __FUNCTION__, bo->handle, bo->active_scanout, get_private(back)->refcnt));
+ if (bo->active_scanout == 0) {
+ DBG(("%s: reuse unattached back\n", __FUNCTION__));
+ get_private(back)->stale = false;
+@@ -153,24 +278,37 @@ sna_dri2_get_back(struct sna *sna,
+ }
+
+ bo = NULL;
+- if (info) {
+- struct dri_bo *c;
+- list_for_each_entry(c, &info->cache, link) {
+- if (c->bo && c->bo->scanout == 0) {
+- bo = c->bo;
+- name = c->name;
+- DBG(("%s: reuse cache handle=%d\n", __FUNCTION__, bo->handle));
+- list_move_tail(&c->link, &info->cache);
+- c->bo = NULL;
++ list_for_each_entry(c, &priv->cache, link) {
++ DBG(("%s: cache: handle=%d, active=%d\n",
++ __FUNCTION__, c->bo ? c->bo->handle : 0, c->bo ? c->bo->active_scanout : -1));
++ assert(c->bo);
++ if (c->bo->active_scanout == 0) {
++ _list_del(&c->link);
++ if (c->bo == NULL) {
++ free(c);
++ goto out;
+ }
++ bo = c->bo;
++ name = c->name;
++ flags = c->flags;
++ DBG(("%s: reuse cache handle=%d, name=%d, flags=%d\n", __FUNCTION__, bo->handle, name, flags));
++ c->bo = NULL;
++ break;
+ }
+ }
+ if (bo == NULL) {
+ DBG(("%s: allocating new backbuffer\n", __FUNCTION__));
++ flags = CREATE_EXACT;
++
++ if (use_scanout(sna, draw, priv)) {
++ DBG(("%s: requesting scanout compatible back\n", __FUNCTION__));
++ flags |= CREATE_SCANOUT;
++ }
++
+ bo = kgem_create_2d(&sna->kgem,
+ draw->width, draw->height, draw->bitsPerPixel,
+ get_private(back)->bo->tiling,
+- get_private(back)->bo->scanout ? CREATE_SCANOUT : 0);
++ flags);
+ if (bo == NULL)
+ return;
+
+@@ -179,30 +317,42 @@ sna_dri2_get_back(struct sna *sna,
+ kgem_bo_destroy(&sna->kgem, bo);
+ return;
+ }
++
++ flags = 0;
++ if (USE_ASYNC_SWAP && back->flags) {
++ BoxRec box;
++
++ box.x1 = 0;
++ box.y1 = 0;
++ box.x2 = draw->width;
++ box.y2 = draw->height;
++
++ DBG(("%s: filling new buffer with old back\n", __FUNCTION__));
++ if (sna->render.copy_boxes(sna, GXcopy,
++ draw, get_private(back)->bo, 0, 0,
++ draw, bo, 0, 0,
++ &box, 1, COPY_LAST | COPY_DRI))
++ flags = back->flags;
++ }
+ }
+ assert(bo->active_scanout == 0);
+
+- if (info && reuse) {
+- bool found = false;
+- struct dri_bo *c;
+-
+- list_for_each_entry_reverse(c, &info->cache, link) {
+- if (c->bo == NULL) {
+- found = true;
+- _list_del(&c->link);
+- break;
+- }
+- }
+- if (!found)
++ if (reuse && get_private(back)->bo->refcnt == 1 + get_private(back)->bo->active_scanout) {
++ if (&c->link == &priv->cache)
+ c = malloc(sizeof(*c));
+ if (c != NULL) {
+ c->bo = ref(get_private(back)->bo);
+ c->name = back->name;
+- list_add(&c->link, &info->cache);
+- DBG(("%s: cacheing handle=%d (name=%d)\n", __FUNCTION__, c->bo->handle, c->name));
++ c->flags = back->flags;
++ list_add(&c->link, &priv->cache);
++ DBG(("%s: caching handle=%d (name=%d, flags=%d, active_scanout=%d)\n", __FUNCTION__, c->bo->handle, c->name, c->flags, c->bo->active_scanout));
+ }
++ } else {
++ if (&c->link != &priv->cache)
++ free(c);
+ }
+
++ assert(bo->active_scanout == 0);
+ assert(bo != get_private(back)->bo);
+ kgem_bo_destroy(&sna->kgem, get_private(back)->bo);
+
+@@ -210,21 +360,13 @@ sna_dri2_get_back(struct sna *sna,
+ get_private(back)->size = draw->height << 16 | draw->width;
+ back->pitch = bo->pitch;
+ back->name = name;
++ back->flags = flags;
+
+- get_private(back)->stale = false;
+-}
+-
+-struct dri2_window {
+- DRI2BufferPtr front;
+- struct sna_dri2_event *chain;
+- xf86CrtcPtr crtc;
+- int64_t msc_delta;
+-};
++ assert(back->pitch);
++ assert(back->name);
+
+-static struct dri2_window *dri2_window(WindowPtr win)
+-{
+- assert(win->drawable.type != DRAWABLE_PIXMAP);
+- return ((void **)__get_private(win, sna_window_key))[1];
++out:
++ get_private(back)->stale = false;
+ }
+
+ static struct sna_dri2_event *
+@@ -232,21 +374,25 @@ dri2_chain(DrawablePtr d)
+ {
+ struct dri2_window *priv = dri2_window((WindowPtr)d);
+ assert(priv != NULL);
++ assert(priv->chain == NULL || priv->chain->chained);
+ return priv->chain;
+ }
+ inline static DRI2BufferPtr dri2_window_get_front(WindowPtr win)
+ {
+ struct dri2_window *priv = dri2_window(win);
++ assert(priv->front == NULL || get_private(priv->front)->bo->active_scanout);
+ return priv ? priv->front : NULL;
+ }
+ #else
+ inline static void *dri2_window_get_front(WindowPtr win) { return NULL; }
++#define APPLY_DAMAGE 1
+ #endif
+
+ #if DRI2INFOREC_VERSION < 6
+
+ #define xorg_can_triple_buffer() 0
+ #define swap_limit(d, l) false
++#define mark_stale(b)
+
+ #else
+
+@@ -273,6 +419,8 @@ mark_stale(DRI2BufferPtr back)
+ * stale frame. (This is mostly useful for tracking down
+ * driver bugs!)
+ */
++ DBG(("%s(handle=%d) => %d\n", __FUNCTION__,
++ get_private(back)->bo->handle, xorg_can_triple_buffer()));
+ get_private(back)->stale = xorg_can_triple_buffer();
+ }
+
+@@ -286,21 +434,29 @@ sna_dri2_swap_limit_validate(DrawablePtr draw, int swap_limit)
+ static void
+ sna_dri2_reuse_buffer(DrawablePtr draw, DRI2BufferPtr buffer)
+ {
++ struct sna *sna = to_sna_from_drawable(draw);
++
+ DBG(("%s: reusing buffer pixmap=%ld, attachment=%d, handle=%d, name=%d\n",
+ __FUNCTION__, get_drawable_pixmap(draw)->drawable.serialNumber,
+ buffer->attachment, get_private(buffer)->bo->handle, buffer->name));
+ assert(get_private(buffer)->refcnt);
+- assert(get_private(buffer)->bo->refcnt > get_private(buffer)->bo->active_scanout);
++ assert(get_private(buffer)->bo->refcnt >= get_private(buffer)->bo->active_scanout);
++ assert(kgem_bo_flink(&sna->kgem, get_private(buffer)->bo) == buffer->name);
+
+ if (buffer->attachment == DRI2BufferBackLeft &&
+ draw->type != DRAWABLE_PIXMAP) {
+- DBG(("%s: replacing back buffer\n", __FUNCTION__));
+- sna_dri2_get_back(to_sna_from_drawable(draw), draw, buffer, dri2_chain(draw));
++ DBG(("%s: replacing back buffer on window %ld\n", __FUNCTION__, draw->id));
++ sna_dri2_get_back(sna, draw, buffer);
+
+- assert(kgem_bo_flink(&to_sna_from_drawable(draw)->kgem, get_private(buffer)->bo) == buffer->name);
+ assert(get_private(buffer)->bo->refcnt);
+ assert(get_private(buffer)->bo->active_scanout == 0);
++ assert(kgem_bo_flink(&sna->kgem, get_private(buffer)->bo) == buffer->name);
++ DBG(("%s: reusing back buffer handle=%d, name=%d, pitch=%d, age=%d\n",
++ __FUNCTION__, get_private(buffer)->bo->handle,
++ buffer->name, buffer->pitch, buffer->flags));
+ }
++
++ kgem_bo_submit(&sna->kgem, get_private(buffer)->bo);
+ }
+
+ static bool swap_limit(DrawablePtr draw, int limit)
+@@ -314,11 +470,6 @@ static bool swap_limit(DrawablePtr draw, int limit)
+ }
+ #endif
+
+-#if DRI2INFOREC_VERSION < 10
+-#undef USE_ASYNC_SWAP
+-#define USE_ASYNC_SWAP 0
+-#endif
+-
+ #define COLOR_PREFER_TILING_Y 0
+
+ /* Prefer to enable TILING_Y if this buffer will never be a
+@@ -328,6 +479,9 @@ static uint32_t color_tiling(struct sna *sna, DrawablePtr draw)
+ {
+ uint32_t tiling;
+
++ if (!sna->kgem.can_fence)
++ return I915_TILING_NONE;
++
+ if (COLOR_PREFER_TILING_Y &&
+ (draw->width != sna->front->drawable.width ||
+ draw->height != sna->front->drawable.height))
+@@ -355,7 +509,6 @@ static struct kgem_bo *sna_pixmap_set_dri(struct sna *sna,
+ PixmapPtr pixmap)
+ {
+ struct sna_pixmap *priv;
+- int tiling;
+
+ DBG(("%s: attaching DRI client to pixmap=%ld\n",
+ __FUNCTION__, pixmap->drawable.serialNumber));
+@@ -373,31 +526,29 @@ static struct kgem_bo *sna_pixmap_set_dri(struct sna *sna,
+ return NULL;
+ }
+
+- assert(priv->flush == false);
++ assert(priv->flush == false || priv->pinned & PIN_DRI3);
++ assert(priv->gpu_bo->flush == false || priv->pinned & PIN_DRI3);
+ assert(priv->cpu_damage == NULL);
+ assert(priv->gpu_bo);
+ assert(priv->gpu_bo->proxy == NULL);
+- assert(priv->gpu_bo->flush == false);
+-
+- tiling = color_tiling(sna, &pixmap->drawable);
+- if (tiling < 0)
+- tiling = -tiling;
+- if (priv->gpu_bo->tiling != tiling)
+- sna_pixmap_change_tiling(pixmap, tiling);
+
+- return priv->gpu_bo;
+-}
++ if (!kgem_bo_is_fenced(&sna->kgem, priv->gpu_bo)) {
++ if (priv->gpu_bo->tiling &&
++ !sna_pixmap_change_tiling(pixmap, I915_TILING_NONE)) {
++ DBG(("%s: failed to discard tiling (%d) for DRI2 protocol\n", __FUNCTION__, priv->gpu_bo->tiling));
++ return NULL;
++ }
++ } else {
++ int tiling = color_tiling(sna, &pixmap->drawable);
++ if (tiling < 0)
++ tiling = -tiling;
++ if (priv->gpu_bo->tiling < tiling && !priv->gpu_bo->scanout)
++ sna_pixmap_change_tiling(pixmap, tiling);
++ }
+
+-pure static inline void *sna_pixmap_get_buffer(PixmapPtr pixmap)
+-{
+- assert(pixmap->refcnt);
+- return ((void **)__get_private(pixmap, sna_pixmap_key))[2];
+-}
++ priv->gpu_bo->active_scanout++;
+
+-static inline void sna_pixmap_set_buffer(PixmapPtr pixmap, void *ptr)
+-{
+- assert(pixmap->refcnt);
+- ((void **)__get_private(pixmap, sna_pixmap_key))[2] = ptr;
++ return priv->gpu_bo;
+ }
+
+ void
+@@ -422,13 +573,18 @@ sna_dri2_pixmap_update_bo(struct sna *sna, PixmapPtr pixmap, struct kgem_bo *bo)
+ if (private->bo == bo)
+ return;
+
++ assert(private->bo->active_scanout > 0);
++ private->bo->active_scanout--;
++
+ DBG(("%s: dropping flush hint from handle=%d\n", __FUNCTION__, private->bo->handle));
+ private->bo->flush = false;
+ kgem_bo_destroy(&sna->kgem, private->bo);
+
++
+ buffer->name = kgem_bo_flink(&sna->kgem, bo);
+ buffer->pitch = bo->pitch;
+ private->bo = ref(bo);
++ bo->active_scanout++;
+
+ DBG(("%s: adding flush hint to handle=%d\n", __FUNCTION__, bo->handle));
+ bo->flush = true;
+@@ -449,9 +605,9 @@ sna_dri2_create_buffer(DrawablePtr draw,
+ struct sna_dri2_private *private;
+ PixmapPtr pixmap;
+ struct kgem_bo *bo;
+- unsigned flags = 0;
++ unsigned bpp = format ?: draw->bitsPerPixel;
++ unsigned flags = CREATE_EXACT;
+ uint32_t size;
+- int bpp;
+
+ DBG(("%s pixmap=%ld, (attachment=%d, format=%d, drawable=%dx%d), window?=%d\n",
+ __FUNCTION__,
+@@ -468,11 +624,11 @@ sna_dri2_create_buffer(DrawablePtr draw,
+ if (draw->type != DRAWABLE_PIXMAP)
+ buffer = dri2_window_get_front((WindowPtr)draw);
+ if (buffer == NULL)
+- buffer = sna_pixmap_get_buffer(pixmap);
++ buffer = (DRI2Buffer2Ptr)sna_pixmap_get_buffer(pixmap);
+ if (buffer) {
+ private = get_private(buffer);
+
+- DBG(("%s: reusing front buffer attachment, win=%lu %dx%d, pixmap=%ld [%ld] %dx%d, handle=%d, name=%d\n",
++ DBG(("%s: reusing front buffer attachment, win=%lu %dx%d, pixmap=%ld [%ld] %dx%d, handle=%d, name=%d, active_scanout=%d\n",
+ __FUNCTION__,
+ draw->type != DRAWABLE_PIXMAP ? (long)draw->id : (long)0,
+ draw->width, draw->height,
+@@ -480,12 +636,22 @@ sna_dri2_create_buffer(DrawablePtr draw,
+ private->pixmap->drawable.serialNumber,
+ pixmap->drawable.width,
+ pixmap->drawable.height,
+- private->bo->handle, buffer->name));
++ private->bo->handle, buffer->name,
++ private->bo->active_scanout));
+
++ assert(buffer->attachment == DRI2BufferFrontLeft);
+ assert(private->pixmap == pixmap);
+ assert(sna_pixmap(pixmap)->flush);
+ assert(sna_pixmap(pixmap)->pinned & PIN_DRI2);
+ assert(kgem_bo_flink(&sna->kgem, private->bo) == buffer->name);
++ assert(private->bo->pitch == buffer->pitch);
++ assert(private->bo->active_scanout);
++
++ sna_pixmap_move_to_gpu(pixmap,
++ MOVE_READ |
++ __MOVE_FORCE |
++ __MOVE_DRI);
++ kgem_bo_submit(&sna->kgem, private->bo);
+
+ private->refcnt++;
+ return buffer;
+@@ -498,7 +664,6 @@ sna_dri2_create_buffer(DrawablePtr draw,
+ assert(sna_pixmap(pixmap) != NULL);
+
+ bo = ref(bo);
+- bpp = pixmap->drawable.bitsPerPixel;
+ if (pixmap == sna->front && !(sna->flags & SNA_LINEAR_FB))
+ flags |= CREATE_SCANOUT;
+ DBG(("%s: attaching to front buffer %dx%d [%p:%d], scanout? %d\n",
+@@ -506,6 +671,7 @@ sna_dri2_create_buffer(DrawablePtr draw,
+ pixmap->drawable.width, pixmap->drawable.height,
+ pixmap, pixmap->refcnt, flags & CREATE_SCANOUT));
+ size = (uint32_t)pixmap->drawable.height << 16 | pixmap->drawable.width;
++ bpp = pixmap->drawable.bitsPerPixel;
+ break;
+
+ case DRI2BufferBackLeft:
+@@ -514,6 +680,7 @@ sna_dri2_create_buffer(DrawablePtr draw,
+ flags |= CREATE_SCANOUT;
+ if (draw->width == sna->front->drawable.width &&
+ draw->height == sna->front->drawable.height &&
++ draw->bitsPerPixel == bpp &&
+ (sna->flags & (SNA_LINEAR_FB | SNA_NO_WAIT | SNA_NO_FLIP)) == 0)
+ flags |= CREATE_SCANOUT;
+ }
+@@ -521,7 +688,6 @@ sna_dri2_create_buffer(DrawablePtr draw,
+ case DRI2BufferFrontRight:
+ case DRI2BufferFakeFrontLeft:
+ case DRI2BufferFakeFrontRight:
+- bpp = draw->bitsPerPixel;
+ DBG(("%s: creating back buffer %dx%d, suitable for scanout? %d\n",
+ __FUNCTION__,
+ draw->width, draw->height,
+@@ -530,7 +696,7 @@ sna_dri2_create_buffer(DrawablePtr draw,
+ bo = kgem_create_2d(&sna->kgem,
+ draw->width,
+ draw->height,
+- draw->bitsPerPixel,
++ bpp,
+ color_tiling(sna, draw),
+ flags);
+ break;
+@@ -558,7 +724,6 @@ sna_dri2_create_buffer(DrawablePtr draw,
+ * not understand W tiling and the GTT is incapable of
+ * W fencing.
+ */
+- bpp = format ? format : draw->bitsPerPixel;
+ bpp *= 2;
+ bo = kgem_create_2d(&sna->kgem,
+ ALIGN(draw->width, 64),
+@@ -570,7 +735,6 @@ sna_dri2_create_buffer(DrawablePtr draw,
+ case DRI2BufferDepthStencil:
+ case DRI2BufferHiz:
+ case DRI2BufferAccum:
+- bpp = format ? format : draw->bitsPerPixel,
+ bo = kgem_create_2d(&sna->kgem,
+ draw->width, draw->height, bpp,
+ other_tiling(sna, draw),
+@@ -614,7 +778,7 @@ sna_dri2_create_buffer(DrawablePtr draw,
+ pixmap->refcnt++;
+
+ priv = sna_pixmap(pixmap);
+- assert(priv->flush == false);
++ assert(priv->flush == false || priv->pinned & PIN_DRI3);
+ assert((priv->pinned & PIN_DRI2) == 0);
+
+ /* Don't allow this named buffer to be replaced */
+@@ -630,17 +794,17 @@ sna_dri2_create_buffer(DrawablePtr draw,
+ if (priv->gpu_bo->exec)
+ sna->kgem.flush = 1;
+
+- priv->flush |= 1;
++ priv->flush |= FLUSH_READ;
+ if (draw->type == DRAWABLE_PIXMAP) {
+ /* DRI2 renders directly into GLXPixmaps, treat as hostile */
+ kgem_bo_unclean(&sna->kgem, priv->gpu_bo);
+ sna_damage_all(&priv->gpu_damage, pixmap);
+ priv->clear = false;
+ priv->cpu = false;
+- priv->flush |= 2;
++ priv->flush |= FLUSH_WRITE;
+ }
+
+- sna_accel_watch_flush(sna, 1);
++ sna_watch_flush(sna, 1);
+ }
+
+ return buffer;
+@@ -651,16 +815,80 @@ err:
+ return NULL;
+ }
+
+-static void _sna_dri2_destroy_buffer(struct sna *sna, DRI2Buffer2Ptr buffer)
++static void
++sna_dri2_cache_bo(struct sna *sna,
++ DrawablePtr draw,
++ struct kgem_bo *bo,
++ uint32_t name,
++ uint32_t size,
++ uint32_t flags)
++{
++ struct dri_bo *c;
++
++ DBG(("%s(handle=%d, name=%d)\n", __FUNCTION__, bo->handle, name));
++
++ if (draw == NULL) {
++ DBG(("%s: no draw, releasing handle=%d\n",
++ __FUNCTION__, bo->handle));
++ goto err;
++ }
++
++ if (draw->type == DRAWABLE_PIXMAP) {
++ DBG(("%s: not a window, releasing handle=%d\n",
++ __FUNCTION__, bo->handle));
++ goto err;
++ }
++
++ if (bo->refcnt > 1 + bo->active_scanout) {
++ DBG(("%s: multiple references [%d], releasing handle\n",
++ __FUNCTION__, bo->refcnt, bo->handle));
++ goto err;
++ }
++
++ if ((draw->height << 16 | draw->width) != size) {
++ DBG(("%s: wrong size [%dx%d], releasing handle\n",
++ __FUNCTION__,
++ size & 0xffff, size >> 16,
++ bo->handle));
++ goto err;
++ }
++
++ if (bo->scanout && front_pitch(draw) != bo->pitch) {
++ DBG(("%s: scanout with pitch change [%d != %d], releasing handle\n",
++ __FUNCTION__, bo->pitch, front_pitch(draw), bo->handle));
++ goto err;
++ }
++
++ c = malloc(sizeof(*c));
++ if (!c)
++ goto err;
++
++ DBG(("%s: caching handle=%d (name=%d, flags=%d, active_scanout=%d)\n", __FUNCTION__, bo->handle, name, flags, bo->active_scanout));
++
++ c->bo = bo;
++ c->name = name;
++ c->flags = flags;
++ list_add(&c->link, &dri2_window((WindowPtr)draw)->cache);
++ return;
++
++err:
++ kgem_bo_destroy(&sna->kgem, bo);
++}
++
++static void _sna_dri2_destroy_buffer(struct sna *sna,
++ DrawablePtr draw,
++ DRI2Buffer2Ptr buffer)
+ {
+ struct sna_dri2_private *private = get_private(buffer);
+
+ if (buffer == NULL)
+ return;
+
+- DBG(("%s: %p [handle=%d] -- refcnt=%d, pixmap=%ld\n",
++ DBG(("%s: %p [handle=%d] -- refcnt=%d, draw=%ld, pixmap=%ld, proxy?=%d\n",
+ __FUNCTION__, buffer, private->bo->handle, private->refcnt,
+- private->pixmap ? private->pixmap->drawable.serialNumber : 0));
++ draw ? draw->id : 0,
++ private->pixmap ? private->pixmap->drawable.serialNumber : 0,
++ private->proxy != NULL));
+ assert(private->refcnt > 0);
+ if (--private->refcnt)
+ return;
+@@ -669,7 +897,10 @@ static void _sna_dri2_destroy_buffer(struct sna *sna, DRI2Buffer2Ptr buffer)
+
+ if (private->proxy) {
+ DBG(("%s: destroying proxy\n", __FUNCTION__));
+- _sna_dri2_destroy_buffer(sna, private->proxy);
++ assert(private->bo->active_scanout > 0);
++ private->bo->active_scanout--;
++
++ _sna_dri2_destroy_buffer(sna, draw, private->proxy);
+ private->pixmap = NULL;
+ }
+
+@@ -683,6 +914,11 @@ static void _sna_dri2_destroy_buffer(struct sna *sna, DRI2Buffer2Ptr buffer)
+ assert(priv->pinned & PIN_DRI2);
+ assert(priv->flush);
+
++ DBG(("%s: removing active_scanout=%d from pixmap handle=%d\n",
++ __FUNCTION__, priv->gpu_bo->active_scanout, priv->gpu_bo->handle));
++ assert(priv->gpu_bo->active_scanout > 0);
++ priv->gpu_bo->active_scanout--;
++
+ /* Undo the DRI markings on this pixmap */
+ DBG(("%s: releasing last DRI pixmap=%ld, scanout?=%d\n",
+ __FUNCTION__,
+@@ -692,28 +928,34 @@ static void _sna_dri2_destroy_buffer(struct sna *sna, DRI2Buffer2Ptr buffer)
+ list_del(&priv->flush_list);
+
+ DBG(("%s: dropping flush hint from handle=%d\n", __FUNCTION__, private->bo->handle));
+- priv->gpu_bo->flush = false;
+ priv->pinned &= ~PIN_DRI2;
+
+- priv->flush = false;
+- sna_accel_watch_flush(sna, -1);
++ if ((priv->pinned & PIN_DRI3) == 0) {
++ priv->gpu_bo->flush = false;
++ priv->flush = false;
++ }
++ sna_watch_flush(sna, -1);
+
+ sna_pixmap_set_buffer(pixmap, NULL);
+ pixmap->drawable.pScreen->DestroyPixmap(pixmap);
+ }
+- assert(private->bo->flush == false);
+
+- kgem_bo_destroy(&sna->kgem, private->bo);
++ sna_dri2_cache_bo(sna, draw,
++ private->bo,
++ buffer->name,
++ private->size,
++ buffer->flags);
+ free(buffer);
+ }
+
+ static void sna_dri2_destroy_buffer(DrawablePtr draw, DRI2Buffer2Ptr buffer)
+ {
+- _sna_dri2_destroy_buffer(to_sna_from_drawable(draw), buffer);
++ _sna_dri2_destroy_buffer(to_sna_from_drawable(draw), draw, buffer);
+ }
+
+ static DRI2BufferPtr sna_dri2_reference_buffer(DRI2BufferPtr buffer)
+ {
++ assert(get_private(buffer)->refcnt > 0);
+ get_private(buffer)->refcnt++;
+ return buffer;
+ }
+@@ -746,10 +988,9 @@ static void set_bo(PixmapPtr pixmap, struct kgem_bo *bo)
+ {
+ struct sna *sna = to_sna_from_pixmap(pixmap);
+ struct sna_pixmap *priv = sna_pixmap(pixmap);
+- RegionRec region;
+
+- DBG(("%s: pixmap=%ld, handle=%d\n",
+- __FUNCTION__, pixmap->drawable.serialNumber, bo->handle));
++ DBG(("%s: pixmap=%ld, handle=%d (old handle=%d)\n",
++ __FUNCTION__, pixmap->drawable.serialNumber, bo->handle, priv->gpu_bo->handle));
+
+ assert(pixmap->drawable.width * pixmap->drawable.bitsPerPixel <= 8*bo->pitch);
+ assert(pixmap->drawable.height * bo->pitch <= kgem_bo_size(bo));
+@@ -758,21 +999,34 @@ static void set_bo(PixmapPtr pixmap, struct kgem_bo *bo)
+ assert((priv->pinned & (PIN_PRIME | PIN_DRI3)) == 0);
+ assert(priv->flush);
+
+- /* Post damage on the new front buffer so that listeners, such
+- * as DisplayLink know take a copy and shove it over the USB,
+- * also for software cursors and the like.
+- */
+- region.extents.x1 = region.extents.y1 = 0;
+- region.extents.x2 = pixmap->drawable.width;
+- region.extents.y2 = pixmap->drawable.height;
+- region.data = NULL;
+- DamageRegionAppend(&pixmap->drawable, &region);
++ if (APPLY_DAMAGE) {
++ RegionRec region;
++
++ /* Post damage on the new front buffer so that listeners, such
++ * as DisplayLink know take a copy and shove it over the USB,
++ * also for software cursors and the like.
++ */
++ region.extents.x1 = region.extents.y1 = 0;
++ region.extents.x2 = pixmap->drawable.width;
++ region.extents.y2 = pixmap->drawable.height;
++ region.data = NULL;
++
++ /*
++ * Eeek, beware the sw cursor copying to the old bo
++ * causing recursion and mayhem.
++ */
++ DBG(("%s: marking whole pixmap as damaged\n", __FUNCTION__));
++ sna->ignore_copy_area = sna->flags & SNA_TEAR_FREE;
++ DamageRegionAppend(&pixmap->drawable, &region);
++ }
+
+ damage(pixmap, priv, NULL);
+
+ assert(bo->refcnt);
+- if (priv->move_to_gpu)
++ if (priv->move_to_gpu) {
++ DBG(("%s: applying final/discard move-to-gpu\n", __FUNCTION__));
+ priv->move_to_gpu(sna, priv, 0);
++ }
+ if (priv->gpu_bo != bo) {
+ DBG(("%s: dropping flush hint from handle=%d\n", __FUNCTION__, priv->gpu_bo->handle));
+ priv->gpu_bo->flush = false;
+@@ -792,8 +1046,27 @@ static void set_bo(PixmapPtr pixmap, struct kgem_bo *bo)
+ bo->domain = DOMAIN_NONE;
+ assert(bo->flush);
+
+- DamageRegionProcessPending(&pixmap->drawable);
++ if (APPLY_DAMAGE) {
++ sna->ignore_copy_area = false;
++ DamageRegionProcessPending(&pixmap->drawable);
++ }
++}
++
++#if defined(__GNUC__)
++#define popcount(x) __builtin_popcount(x)
++#else
++static int popcount(unsigned int x)
++{
++ int count = 0;
++
++ while (x) {
++ count += x&1;
++ x >>= 1;
++ }
++
++ return count;
+ }
++#endif
+
+ static void sna_dri2_select_mode(struct sna *sna, struct kgem_bo *dst, struct kgem_bo *src, bool sync)
+ {
+@@ -823,6 +1096,12 @@ static void sna_dri2_select_mode(struct sna *sna, struct kgem_bo *dst, struct kg
+ return;
+ }
+
++ if (sna->render_state.gt < 2 && sna->kgem.has_semaphores) {
++ DBG(("%s: small GT [%d], not forcing selection\n",
++ __FUNCTION__, sna->render_state.gt));
++ return;
++ }
++
+ VG_CLEAR(busy);
+ busy.handle = src->handle;
+ if (drmIoctl(sna->kgem.fd, DRM_IOCTL_I915_GEM_BUSY, &busy))
+@@ -860,9 +1139,11 @@ static void sna_dri2_select_mode(struct sna *sna, struct kgem_bo *dst, struct kg
+ * the cost of the query.
+ */
+ mode = KGEM_RENDER;
+- if (busy.busy & (0xfffe << 16))
++ if ((busy.busy & 0xffff) == I915_EXEC_BLT)
+ mode = KGEM_BLT;
+- kgem_bo_mark_busy(&sna->kgem, busy.handle == src->handle ? src : dst, mode);
++ kgem_bo_mark_busy(&sna->kgem,
++ busy.handle == src->handle ? src : dst,
++ mode);
+ _kgem_set_mode(&sna->kgem, mode);
+ }
+
+@@ -871,10 +1152,13 @@ static bool is_front(int attachment)
+ return attachment == DRI2BufferFrontLeft;
+ }
+
++#define DRI2_SYNC 0x1
++#define DRI2_DAMAGE 0x2
++#define DRI2_BO 0x4
+ static struct kgem_bo *
+ __sna_dri2_copy_region(struct sna *sna, DrawablePtr draw, RegionPtr region,
+ DRI2BufferPtr src, DRI2BufferPtr dst,
+- bool sync)
++ unsigned flags)
+ {
+ PixmapPtr pixmap = get_drawable_pixmap(draw);
+ DrawableRec scratch, *src_draw = &pixmap->drawable, *dst_draw = &pixmap->drawable;
+@@ -886,7 +1170,7 @@ __sna_dri2_copy_region(struct sna *sna, DrawablePtr draw, RegionPtr region,
+ struct kgem_bo *dst_bo;
+ const BoxRec *boxes;
+ int16_t dx, dy, sx, sy;
+- unsigned flags;
++ unsigned hint;
+ int n;
+
+ /* To hide a stale DRI2Buffer, one may choose to substitute
+@@ -962,8 +1246,9 @@ __sna_dri2_copy_region(struct sna *sna, DrawablePtr draw, RegionPtr region,
+ }
+ }
+ } else
+- sync = false;
++ flags &= ~DRI2_SYNC;
+
++ scratch.pScreen = draw->pScreen;
+ scratch.x = scratch.y = 0;
+ scratch.width = scratch.height = 0;
+ scratch.depth = draw->depth;
+@@ -971,6 +1256,7 @@ __sna_dri2_copy_region(struct sna *sna, DrawablePtr draw, RegionPtr region,
+
+ src_bo = src_priv->bo;
+ assert(src_bo->refcnt);
++ kgem_bo_unclean(&sna->kgem, src_bo);
+ if (is_front(src->attachment)) {
+ struct sna_pixmap *priv;
+
+@@ -987,11 +1273,12 @@ __sna_dri2_copy_region(struct sna *sna, DrawablePtr draw, RegionPtr region,
+ scratch.height = src_priv->size >> 16;
+ src_draw = &scratch;
+
+- DBG(("%s: source size %dx%d, region size %dx%d\n",
++ DBG(("%s: source size %dx%d, region size %dx%d, src offset %dx%d\n",
+ __FUNCTION__,
+ scratch.width, scratch.height,
+ clip.extents.x2 - clip.extents.x1,
+- clip.extents.y2 - clip.extents.y1));
++ clip.extents.y2 - clip.extents.y1,
++ -sx, -sy));
+
+ source.extents.x1 = -sx;
+ source.extents.y1 = -sy;
+@@ -1002,6 +1289,10 @@ __sna_dri2_copy_region(struct sna *sna, DrawablePtr draw, RegionPtr region,
+ assert(region == NULL || region == &clip);
+ pixman_region_intersect(&clip, &clip, &source);
+
++ if (!pixman_region_not_empty(&clip)) {
++ DBG(("%s: region doesn't overlap pixmap\n", __FUNCTION__));
++ return NULL;
++ }
+ }
+
+ dst_bo = dst_priv->bo;
+@@ -1013,12 +1304,12 @@ __sna_dri2_copy_region(struct sna *sna, DrawablePtr draw, RegionPtr region,
+ /* Preserve the CRTC shadow overrides */
+ sna_shadow_steal_crtcs(sna, &shadow);
+
+- flags = MOVE_WRITE | __MOVE_FORCE;
++ hint = MOVE_WRITE | __MOVE_FORCE;
+ if (clip.data)
+- flags |= MOVE_READ;
++ hint |= MOVE_READ;
+
+ assert(region == NULL || region == &clip);
+- priv = sna_pixmap_move_area_to_gpu(pixmap, &clip.extents, flags);
++ priv = sna_pixmap_move_area_to_gpu(pixmap, &clip.extents, hint);
+ if (priv) {
+ damage(pixmap, priv, region);
+ dst_bo = priv->gpu_bo;
+@@ -1050,20 +1341,20 @@ __sna_dri2_copy_region(struct sna *sna, DrawablePtr draw, RegionPtr region,
+ assert(region == NULL || region == &clip);
+ pixman_region_intersect(&clip, &clip, &target);
+
+- sync = false;
++ flags &= ~DRI2_SYNC;
+ }
+
+ if (!wedged(sna)) {
+ xf86CrtcPtr crtc;
+
+ crtc = NULL;
+- if (sync && sna_pixmap_is_scanout(sna, pixmap))
++ if (flags & DRI2_SYNC && sna_pixmap_is_scanout(sna, pixmap))
+ crtc = sna_covering_crtc(sna, &clip.extents, NULL);
+ sna_dri2_select_mode(sna, dst_bo, src_bo, crtc != NULL);
+
+- sync = (crtc != NULL&&
+- sna_wait_for_scanline(sna, pixmap, crtc,
+- &clip.extents));
++ if (crtc == NULL ||
++ !sna_wait_for_scanline(sna, pixmap, crtc, &clip.extents))
++ flags &= ~DRI2_SYNC;
+ }
+
+ if (region) {
+@@ -1075,8 +1366,11 @@ __sna_dri2_copy_region(struct sna *sna, DrawablePtr draw, RegionPtr region,
+ boxes = &clip.extents;
+ n = 1;
+ }
+- DamageRegionAppend(&pixmap->drawable, region);
+-
++ if (APPLY_DAMAGE || flags & DRI2_DAMAGE) {
++ DBG(("%s: marking region as damaged\n", __FUNCTION__));
++ sna->ignore_copy_area = sna->flags & SNA_TEAR_FREE;
++ DamageRegionAppend(&pixmap->drawable, region);
++ }
+
+ DBG(("%s: copying [(%d, %d), (%d, %d)]x%d src=(%d, %d), dst=(%d, %d)\n",
+ __FUNCTION__,
+@@ -1084,29 +1378,36 @@ __sna_dri2_copy_region(struct sna *sna, DrawablePtr draw, RegionPtr region,
+ boxes[0].x2, boxes[0].y2,
+ n, sx, sy, dx, dy));
+
+- flags = COPY_LAST;
+- if (sync)
+- flags |= COPY_SYNC;
++ hint = COPY_LAST | COPY_DRI;
++ if (flags & DRI2_SYNC)
++ hint |= COPY_SYNC;
+ if (!sna->render.copy_boxes(sna, GXcopy,
+ src_draw, src_bo, sx, sy,
+ dst_draw, dst_bo, dx, dy,
+- boxes, n, flags))
++ boxes, n, hint))
+ memcpy_copy_boxes(sna, GXcopy,
+ src_draw, src_bo, sx, sy,
+ dst_draw, dst_bo, dx, dy,
+- boxes, n, flags);
+-
+- DBG(("%s: flushing? %d\n", __FUNCTION__, sync));
+- if (sync) { /* STAT! */
+- struct kgem_request *rq = sna->kgem.next_request;
+- kgem_submit(&sna->kgem);
+- if (rq->bo) {
+- bo = ref(rq->bo);
+- DBG(("%s: recording sync fence handle=%d\n", __FUNCTION__, bo->handle));
++ boxes, n, hint);
++
++ sna->needs_dri_flush = true;
++ if (flags & (DRI2_SYNC | DRI2_BO)) { /* STAT! */
++ struct kgem_request *rq = RQ(dst_bo->rq);
++ if (rq && rq != (void *)&sna->kgem) {
++ if (rq->bo == NULL)
++ kgem_submit(&sna->kgem);
++ if (rq->bo) { /* Becareful in case the gpu is wedged */
++ bo = ref(rq->bo);
++ DBG(("%s: recording sync fence handle=%d\n",
++ __FUNCTION__, bo->handle));
++ }
+ }
+ }
+
+- DamageRegionProcessPending(&pixmap->drawable);
++ if (APPLY_DAMAGE || flags & DRI2_DAMAGE) {
++ sna->ignore_copy_area = false;
++ DamageRegionProcessPending(&pixmap->drawable);
++ }
+
+ if (clip.data)
+ pixman_region_fini(&clip);
+@@ -1142,6 +1443,8 @@ sna_dri2_copy_region(DrawablePtr draw,
+ assert(get_private(src)->refcnt);
+ assert(get_private(dst)->refcnt);
+
++ assert(get_private(src)->bo != get_private(dst)->bo);
++
+ assert(get_private(src)->bo->refcnt);
+ assert(get_private(dst)->bo->refcnt);
+
+@@ -1151,7 +1454,7 @@ sna_dri2_copy_region(DrawablePtr draw,
+ region->extents.x2, region->extents.y2,
+ region_num_rects(region)));
+
+- __sna_dri2_copy_region(sna, draw, region, src, dst, false);
++ __sna_dri2_copy_region(sna, draw, region, src, dst, DRI2_DAMAGE);
+ }
+
+ inline static uint32_t pipe_select(int pipe)
+@@ -1161,6 +1464,7 @@ inline static uint32_t pipe_select(int pipe)
+ * we can safely ignore the capability check - if we have more
+ * than two pipes, we can assume that they are fully supported.
+ */
++ assert(pipe < _DRM_VBLANK_HIGH_CRTC_MASK);
+ if (pipe > 1)
+ return pipe << DRM_VBLANK_HIGH_CRTC_SHIFT;
+ else if (pipe > 0)
+@@ -1169,15 +1473,53 @@ inline static uint32_t pipe_select(int pipe)
+ return 0;
+ }
+
+-static inline int sna_wait_vblank(struct sna *sna, union drm_wait_vblank *vbl, int pipe)
++static inline bool sna_next_vblank(struct sna_dri2_event *info)
+ {
+- DBG(("%s(pipe=%d, waiting until seq=%u%s)\n",
+- __FUNCTION__, pipe, vbl->request.sequence,
+- vbl->request.type & DRM_VBLANK_RELATIVE ? " [relative]" : ""));
+- assert(pipe != -1);
++ union drm_wait_vblank vbl;
+
+- vbl->request.type |= pipe_select(pipe);
+- return drmIoctl(sna->kgem.fd, DRM_IOCTL_WAIT_VBLANK, vbl);
++ DBG(("%s(pipe=%d, waiting until next vblank)\n",
++ __FUNCTION__, info->pipe));
++ assert(info->pipe != -1);
++
++ VG_CLEAR(vbl);
++ vbl.request.type =
++ DRM_VBLANK_RELATIVE |
++ DRM_VBLANK_EVENT |
++ pipe_select(info->pipe);
++ vbl.request.sequence = 1;
++ vbl.request.signal = (uintptr_t)info;
++
++ assert(!info->queued);
++ if (drmIoctl(info->sna->kgem.fd, DRM_IOCTL_WAIT_VBLANK, &vbl))
++ return false;
++
++ info->queued = true;
++ return true;
++}
++
++static inline bool sna_wait_vblank(struct sna_dri2_event *info,
++ unsigned seq)
++{
++ union drm_wait_vblank vbl;
++
++ DBG(("%s(pipe=%d, waiting until vblank %u)\n",
++ __FUNCTION__, info->pipe, seq));
++ assert(info->pipe != -1);
++
++ VG_CLEAR(vbl);
++ vbl.request.type =
++ DRM_VBLANK_ABSOLUTE |
++ DRM_VBLANK_EVENT |
++ pipe_select(info->pipe);
++ vbl.request.sequence = seq;
++ vbl.request.signal = (uintptr_t)info;
++
++ assert(!info->queued);
++ if (drmIoctl(info->sna->kgem.fd, DRM_IOCTL_WAIT_VBLANK, &vbl))
++ return false;
++
++ info->queued = true;
++ return true;
+ }
+
+ #if DRI2INFOREC_VERSION >= 4
+@@ -1195,6 +1537,7 @@ draw_current_msc(DrawablePtr draw, xf86CrtcPtr crtc, uint64_t msc)
+ {
+ struct dri2_window *priv;
+
++ assert(draw);
+ if (draw->type != DRAWABLE_WINDOW)
+ return msc;
+
+@@ -1206,6 +1549,9 @@ draw_current_msc(DrawablePtr draw, xf86CrtcPtr crtc, uint64_t msc)
+ priv->crtc = crtc;
+ priv->msc_delta = 0;
+ priv->chain = NULL;
++ priv->scanout = -1;
++ priv->cache_size = 0;
++ list_init(&priv->cache);
+ dri2_window_attach((WindowPtr)draw, priv);
+ }
+ } else {
+@@ -1214,8 +1560,8 @@ draw_current_msc(DrawablePtr draw, xf86CrtcPtr crtc, uint64_t msc)
+ const struct ust_msc *this = sna_crtc_last_swap(crtc);
+ DBG(("%s: Window transferring from pipe=%d [msc=%llu] to pipe=%d [msc=%llu], delta now %lld\n",
+ __FUNCTION__,
+- sna_crtc_to_pipe(priv->crtc), (long long)last->msc,
+- sna_crtc_to_pipe(crtc), (long long)this->msc,
++ sna_crtc_pipe(priv->crtc), (long long)last->msc,
++ sna_crtc_pipe(crtc), (long long)this->msc,
+ (long long)(priv->msc_delta + this->msc - last->msc)));
+ priv->msc_delta += this->msc - last->msc;
+ priv->crtc = crtc;
+@@ -1248,57 +1594,119 @@ sna_dri2_get_crtc(DrawablePtr draw)
+ NULL);
+ }
+
+-static void
+-sna_dri2_remove_event(WindowPtr win, struct sna_dri2_event *info)
++static void frame_swap_complete(struct sna_dri2_event *frame, int type)
+ {
+- struct dri2_window *priv;
+- struct sna_dri2_event *chain;
+-
+- assert(win->drawable.type == DRAWABLE_WINDOW);
+- DBG(("%s: remove[%p] from window %ld, active? %d\n",
+- __FUNCTION__, info, (long)win->drawable.id, info->draw != NULL));
++ const struct ust_msc *swap;
+
+- priv = dri2_window(win);
+- assert(priv);
+- assert(priv->chain != NULL);
++ assert(frame->signal);
++ frame->signal = false;
+
+- if (priv->chain == info) {
+- priv->chain = info->chain;
++ if (frame->client == NULL) {
++ DBG(("%s: client already gone\n", __FUNCTION__));
+ return;
+ }
+
+- chain = priv->chain;
+- while (chain->chain != info)
+- chain = chain->chain;
+- assert(chain != info);
+- assert(info->chain != chain);
+- chain->chain = info->chain;
++ assert(frame->draw);
++
++ swap = sna_crtc_last_swap(frame->crtc);
++ DBG(("%s(type=%d): draw=%ld, pipe=%d, frame=%lld [msc=%lld], tv=%d.%06d\n",
++ __FUNCTION__, type, (long)frame->draw->id, frame->pipe,
++ (long long)swap->msc,
++ (long long)draw_current_msc(frame->draw, frame->crtc, swap->msc),
++ swap->tv_sec, swap->tv_usec));
++
++ DRI2SwapComplete(frame->client, frame->draw,
++ draw_current_msc(frame->draw, frame->crtc, swap->msc),
++ swap->tv_sec, swap->tv_usec,
++ type, frame->event_complete, frame->event_data);
+ }
+
+-static void
+-sna_dri2_event_free(struct sna_dri2_event *info)
++static void fake_swap_complete(struct sna *sna, ClientPtr client,
++ DrawablePtr draw, xf86CrtcPtr crtc,
++ int type, DRI2SwapEventPtr func, void *data)
+ {
+- DrawablePtr draw = info->draw;
++ const struct ust_msc *swap;
+
+- DBG(("%s(draw?=%d)\n", __FUNCTION__, draw != NULL));
+- if (draw && draw->type == DRAWABLE_WINDOW)
+- sna_dri2_remove_event((WindowPtr)draw, info);
++ assert(draw);
+
+- _sna_dri2_destroy_buffer(info->sna, info->front);
+- _sna_dri2_destroy_buffer(info->sna, info->back);
++ if (crtc == NULL)
++ crtc = sna_primary_crtc(sna);
+
+- while (!list_is_empty(&info->cache)) {
+- struct dri_bo *c;
++ swap = sna_crtc_last_swap(crtc);
++ DBG(("%s(type=%d): draw=%ld, pipe=%d, frame=%lld [msc %lld], tv=%d.%06d\n",
++ __FUNCTION__, type, (long)draw->id, crtc ? sna_crtc_pipe(crtc) : -1,
++ (long long)swap->msc,
++ (long long)draw_current_msc(draw, crtc, swap->msc),
++ swap->tv_sec, swap->tv_usec));
+
+- c = list_first_entry(&info->cache, struct dri_bo, link);
+- list_del(&c->link);
++ DRI2SwapComplete(client, draw,
++ draw_current_msc(draw, crtc, swap->msc),
++ swap->tv_sec, swap->tv_usec,
++ type, func, data);
++}
+
+- DBG(("%s: releasing cached handle=%d\n", __FUNCTION__, c->bo ? c->bo->handle : 0));
+- if (c->bo)
+- kgem_bo_destroy(&info->sna->kgem, c->bo);
++static void
++sna_dri2_remove_event(struct sna_dri2_event *info)
++{
++ WindowPtr win = (WindowPtr)info->draw;
++ struct dri2_window *priv;
+
+- free(c);
++ assert(win->drawable.type == DRAWABLE_WINDOW);
++ DBG(("%s: remove[%p] from window %ld, active? %d\n",
++ __FUNCTION__, info, (long)win->drawable.id, info->draw != NULL));
++ assert(!info->signal);
++
++ priv = dri2_window(win);
++ assert(priv);
++ assert(priv->chain != NULL);
++ assert(info->chained);
++ info->chained = false;
++
++ if (priv->chain != info) {
++ struct sna_dri2_event *chain = priv->chain;
++ while (chain->chain != info) {
++ assert(chain->chained);
++ chain = chain->chain;
++ }
++ assert(chain != info);
++ assert(info->chain != chain);
++ chain->chain = info->chain;
++ return;
++ }
++
++ priv->chain = info->chain;
++ if (priv->chain == NULL) {
++ struct dri_bo *c, *tmp;
++
++ c = list_entry(priv->cache.next->next, struct dri_bo, link);
++ list_for_each_entry_safe_from(c, tmp, &priv->cache, link) {
++ list_del(&c->link);
++
++ DBG(("%s: releasing cached handle=%d\n", __FUNCTION__, c->bo ? c->bo->handle : 0));
++ assert(c->bo);
++ kgem_bo_destroy(&info->sna->kgem, c->bo);
++ free(c);
++ }
+ }
++}
++
++static void
++sna_dri2_event_free(struct sna_dri2_event *info)
++{
++ DBG(("%s(draw?=%d)\n", __FUNCTION__, info->draw != NULL));
++ assert(!info->queued);
++ assert(!info->signal);
++ assert(info->pending.bo == NULL);
++
++ if (info->sna->dri2.flip_pending == info)
++ info->sna->dri2.flip_pending = NULL;
++ assert(info->sna->dri2.flip_pending != info);
++ if (info->chained)
++ sna_dri2_remove_event(info);
++
++ assert((info->front == NULL && info->back == NULL) || info->front != info->back);
++ _sna_dri2_destroy_buffer(info->sna, info->draw, info->front);
++ _sna_dri2_destroy_buffer(info->sna, info->draw, info->back);
+
+ if (info->bo) {
+ DBG(("%s: releasing batch handle=%d\n", __FUNCTION__, info->bo->handle));
+@@ -1331,15 +1739,26 @@ sna_dri2_client_gone(CallbackListPtr *list, void *closure, void *data)
+
+ event = list_first_entry(&priv->events, struct sna_dri2_event, link);
+ assert(event->client == client);
++ list_del(&event->link);
++ event->signal = false;
+
+- if (event->queued) {
+- if (event->draw)
+- sna_dri2_remove_event((WindowPtr)event->draw,
+- event);
+- event->client = NULL;
+- event->draw = NULL;
+- list_del(&event->link);
+- } else
++ if (event->pending.bo) {
++ assert(event->pending.bo->active_scanout > 0);
++ event->pending.bo->active_scanout--;
++
++ kgem_bo_destroy(&sna->kgem, event->pending.bo);
++ event->pending.bo = NULL;
++ }
++
++ if (event->chained)
++ sna_dri2_remove_event(event);
++
++ event->client = NULL;
++ event->draw = NULL;
++ event->keepalive = 1;
++ assert(!event->signal);
++
++ if (!event->queued)
+ sna_dri2_event_free(event);
+ }
+
+@@ -1365,11 +1784,15 @@ static bool add_event_to_client(struct sna_dri2_event *info, struct sna *sna, Cl
+ }
+
+ static struct sna_dri2_event *
+-sna_dri2_add_event(struct sna *sna, DrawablePtr draw, ClientPtr client)
++sna_dri2_add_event(struct sna *sna,
++ DrawablePtr draw,
++ ClientPtr client,
++ xf86CrtcPtr crtc)
+ {
+ struct dri2_window *priv;
+ struct sna_dri2_event *info, *chain;
+
++ assert(draw != NULL);
+ assert(draw->type == DRAWABLE_WINDOW);
+ DBG(("%s: adding event to window %ld)\n",
+ __FUNCTION__, (long)draw->id));
+@@ -1382,11 +1805,11 @@ sna_dri2_add_event(struct sna *sna, DrawablePtr draw, ClientPtr client)
+ if (info == NULL)
+ return NULL;
+
+- list_init(&info->cache);
+ info->sna = sna;
+ info->draw = draw;
+- info->crtc = priv->crtc;
+- info->pipe = sna_crtc_to_pipe(priv->crtc);
++ info->crtc = crtc;
++ info->pipe = sna_crtc_pipe(crtc);
++ info->keepalive = 1;
+
+ if (!add_event_to_client(info, sna, client)) {
+ free(info);
+@@ -1394,6 +1817,7 @@ sna_dri2_add_event(struct sna *sna, DrawablePtr draw, ClientPtr client)
+ }
+
+ assert(priv->chain != info);
++ info->chained = true;
+
+ if (priv->chain == NULL) {
+ priv->chain = info;
+@@ -1409,6 +1833,66 @@ sna_dri2_add_event(struct sna *sna, DrawablePtr draw, ClientPtr client)
+ return info;
+ }
+
++static void decouple_window(WindowPtr win,
++ struct dri2_window *priv,
++ struct sna *sna,
++ bool signal)
++{
++ if (priv->front) {
++ DBG(("%s: decouple private front\n", __FUNCTION__));
++ assert(priv->crtc);
++ sna_shadow_unset_crtc(sna, priv->crtc);
++
++ _sna_dri2_destroy_buffer(sna, NULL, priv->front);
++ priv->front = NULL;
++ }
++
++ if (priv->chain) {
++ struct sna_dri2_event *info, *chain;
++
++ DBG(("%s: freeing chain\n", __FUNCTION__));
++
++ chain = priv->chain;
++ while ((info = chain)) {
++ DBG(("%s: freeing event, pending signal? %d, pending swap? handle=%d\n",
++ __FUNCTION__, info->signal,
++ info->pending.bo ? info->pending.bo->handle : 0));
++ assert(info->draw == &win->drawable);
++
++ if (info->pending.bo) {
++ if (signal) {
++ bool was_signalling = info->signal;
++ info->signal = true;
++ frame_swap_complete(info, DRI2_EXCHANGE_COMPLETE);
++ info->signal = was_signalling;
++ }
++ assert(info->pending.bo->active_scanout > 0);
++ info->pending.bo->active_scanout--;
++
++ kgem_bo_destroy(&sna->kgem, info->pending.bo);
++ info->pending.bo = NULL;
++ }
++
++ if (info->signal && signal)
++ frame_swap_complete(info, DRI2_EXCHANGE_COMPLETE);
++ info->signal = false;
++ info->draw = NULL;
++ info->keepalive = 1;
++ assert(!info->signal);
++ list_del(&info->link);
++
++ chain = info->chain;
++ info->chain = NULL;
++ info->chained = false;
++
++ if (!info->queued)
++ sna_dri2_event_free(info);
++ }
++
++ priv->chain = NULL;
++ }
++}
++
+ void sna_dri2_decouple_window(WindowPtr win)
+ {
+ struct dri2_window *priv;
+@@ -1418,50 +1902,34 @@ void sna_dri2_decouple_window(WindowPtr win)
+ return;
+
+ DBG(("%s: window=%ld\n", __FUNCTION__, win->drawable.id));
++ decouple_window(win, priv, to_sna_from_drawable(&win->drawable), true);
+
+- if (priv->front) {
+- struct sna *sna = to_sna_from_drawable(&win->drawable);
+- assert(priv->crtc);
+- sna_shadow_unset_crtc(sna, priv->crtc);
+- _sna_dri2_destroy_buffer(sna, priv->front);
+- priv->front = NULL;
+- }
++ priv->scanout = -1;
+ }
+
+ void sna_dri2_destroy_window(WindowPtr win)
+ {
+ struct dri2_window *priv;
++ struct sna *sna;
+
+ priv = dri2_window(win);
+ if (priv == NULL)
+ return;
+
+ DBG(("%s: window=%ld\n", __FUNCTION__, win->drawable.id));
++ sna = to_sna_from_drawable(&win->drawable);
++ decouple_window(win, priv, sna, false);
+
+- if (priv->front) {
+- struct sna *sna = to_sna_from_drawable(&win->drawable);
+- assert(priv->crtc);
+- sna_shadow_unset_crtc(sna, priv->crtc);
+- _sna_dri2_destroy_buffer(sna, priv->front);
+- }
+-
+- if (priv->chain) {
+- struct sna_dri2_event *info, *chain;
+-
+- DBG(("%s: freeing chain\n", __FUNCTION__));
+-
+- chain = priv->chain;
+- while ((info = chain)) {
+- info->draw = NULL;
+- info->client = NULL;
+- list_del(&info->link);
++ while (!list_is_empty(&priv->cache)) {
++ struct dri_bo *c;
+
+- chain = info->chain;
+- info->chain = NULL;
++ c = list_first_entry(&priv->cache, struct dri_bo, link);
++ list_del(&c->link);
+
+- if (!info->queued)
+- sna_dri2_event_free(info);
+- }
++ DBG(("%s: releasing cached handle=%d\n", __FUNCTION__, c->bo ? c->bo->handle : 0));
++ assert(c->bo);
++ kgem_bo_destroy(&sna->kgem, c->bo);
++ free(c);
+ }
+
+ free(priv);
+@@ -1479,19 +1947,30 @@ sna_dri2_flip(struct sna_dri2_event *info)
+ {
+ struct kgem_bo *bo = get_private(info->back)->bo;
+ struct kgem_bo *tmp_bo;
+- uint32_t tmp_name;
++ uint32_t tmp_name, tmp_flags;
+ int tmp_pitch;
+
+ DBG(("%s(type=%d)\n", __FUNCTION__, info->type));
+
+ assert(sna_pixmap_get_buffer(info->sna->front) == info->front);
+ assert(get_drawable_pixmap(info->draw)->drawable.height * bo->pitch <= kgem_bo_size(bo));
++ assert(get_private(info->front)->size == get_private(info->back)->size);
+ assert(bo->refcnt);
+
++ if (info->sna->mode.flip_active) {
++ DBG(("%s: %d flips still active, aborting\n",
++ __FUNCTION__, info->sna->mode.flip_active));
++ return false;
++ }
++
++ assert(!info->queued);
+ if (!sna_page_flip(info->sna, bo, sna_dri2_flip_handler,
+ info->type == FLIP_ASYNC ? NULL : info))
+ return false;
+
++ DBG(("%s: queued flip=%p\n", __FUNCTION__, info->type == FLIP_ASYNC ? NULL : info));
++ assert(info->signal || info->type != FLIP_THROTTLE);
++
+ assert(info->sna->dri2.flip_pending == NULL ||
+ info->sna->dri2.flip_pending == info);
+ if (info->type != FLIP_ASYNC)
+@@ -1505,13 +1984,21 @@ sna_dri2_flip(struct sna_dri2_event *info)
+ tmp_bo = get_private(info->front)->bo;
+ tmp_name = info->front->name;
+ tmp_pitch = info->front->pitch;
++ tmp_flags = info->front->flags;
++
++ assert(tmp_bo->active_scanout > 0);
++ tmp_bo->active_scanout--;
+
+ set_bo(info->sna->front, bo);
+
++ info->front->flags = info->back->flags;
+ info->front->name = info->back->name;
+ info->front->pitch = info->back->pitch;
+ get_private(info->front)->bo = bo;
++ bo->active_scanout++;
++ assert(bo->active_scanout <= bo->refcnt);
+
++ info->back->flags = tmp_flags;
+ info->back->name = tmp_name;
+ info->back->pitch = tmp_pitch;
+ get_private(info->back)->bo = tmp_bo;
+@@ -1521,6 +2008,7 @@ sna_dri2_flip(struct sna_dri2_event *info)
+ assert(get_private(info->back)->bo->refcnt);
+ assert(get_private(info->front)->bo != get_private(info->back)->bo);
+
++ info->keepalive = KEEPALIVE;
+ info->queued = true;
+ return true;
+ }
+@@ -1549,15 +2037,16 @@ can_flip(struct sna * sna,
+ }
+
+ assert(sna->scrn->vtSema);
++ assert(!sna->mode.hidden);
+
+ if ((sna->flags & (SNA_HAS_FLIP | SNA_HAS_ASYNC_FLIP)) == 0) {
+ DBG(("%s: no, pageflips disabled\n", __FUNCTION__));
+ return false;
+ }
+
+- if (front->format != back->format) {
++ if (front->cpp != back->cpp) {
+ DBG(("%s: no, format mismatch, front = %d, back = %d\n",
+- __FUNCTION__, front->format, back->format));
++ __FUNCTION__, front->cpp, back->cpp));
+ return false;
+ }
+
+@@ -1567,7 +2056,7 @@ can_flip(struct sna * sna,
+ }
+
+ if (!sna_crtc_is_on(crtc)) {
+- DBG(("%s: ref-pipe=%d is disabled\n", __FUNCTION__, sna_crtc_to_pipe(crtc)));
++ DBG(("%s: ref-pipe=%d is disabled\n", __FUNCTION__, sna_crtc_pipe(crtc)));
+ return false;
+ }
+
+@@ -1581,7 +2070,7 @@ can_flip(struct sna * sna,
+ if (sna_pixmap_get_buffer(pixmap) != front) {
+ DBG(("%s: no, DRI2 drawable is no longer attached (old name=%d, new name=%d) to pixmap=%ld\n",
+ __FUNCTION__, front->name,
+- sna_pixmap_get_buffer(pixmap) ? ((DRI2BufferPtr)sna_pixmap_get_buffer(pixmap))->name : 0,
++ sna_pixmap_get_buffer(pixmap) ? sna_pixmap_get_buffer(pixmap)->name : 0,
+ pixmap->drawable.serialNumber));
+ return false;
+ }
+@@ -1661,7 +2150,6 @@ can_flip(struct sna * sna,
+ }
+
+ DBG(("%s: yes, pixmap=%ld\n", __FUNCTION__, pixmap->drawable.serialNumber));
+- assert(dri2_window(win)->front == NULL);
+ return true;
+ }
+
+@@ -1680,9 +2168,9 @@ can_xchg(struct sna *sna,
+ if (draw->type == DRAWABLE_PIXMAP)
+ return false;
+
+- if (front->format != back->format) {
++ if (front->cpp != back->cpp) {
+ DBG(("%s: no, format mismatch, front = %d, back = %d\n",
+- __FUNCTION__, front->format, back->format));
++ __FUNCTION__, front->cpp, back->cpp));
+ return false;
+ }
+
+@@ -1714,6 +2202,8 @@ can_xchg(struct sna *sna,
+ return false;
+ }
+
++ DBG(("%s: back size=%x, front size=%x\n",
++ __FUNCTION__, get_private(back)->size, get_private(front)->size));
+ if (get_private(back)->size != get_private(front)->size) {
+ DBG(("%s: no, back buffer %dx%d does not match front buffer %dx%d\n",
+ __FUNCTION__,
+@@ -1766,9 +2256,9 @@ overlaps_other_crtc(struct sna *sna, xf86CrtcPtr desired)
+ static bool
+ can_xchg_crtc(struct sna *sna,
+ DrawablePtr draw,
++ xf86CrtcPtr crtc,
+ DRI2BufferPtr front,
+- DRI2BufferPtr back,
+- xf86CrtcPtr crtc)
++ DRI2BufferPtr back)
+ {
+ WindowPtr win = (WindowPtr)draw;
+ PixmapPtr pixmap;
+@@ -1785,9 +2275,9 @@ can_xchg_crtc(struct sna *sna,
+ if (draw->type == DRAWABLE_PIXMAP)
+ return false;
+
+- if (front->format != back->format) {
++ if (front->cpp != back->cpp) {
+ DBG(("%s: no, format mismatch, front = %d, back = %d\n",
+- __FUNCTION__, front->format, back->format));
++ __FUNCTION__, front->cpp, back->cpp));
+ return false;
+ }
+
+@@ -1866,20 +2356,21 @@ sna_dri2_xchg(DrawablePtr draw, DRI2BufferPtr front, DRI2BufferPtr back)
+
+ back_bo = get_private(back)->bo;
+ front_bo = get_private(front)->bo;
+- assert(front_bo != back_bo);
+
+- DBG(("%s: win=%ld, exchange front=%d/%d and back=%d/%d, pixmap=%ld %dx%d\n",
++ DBG(("%s: win=%ld, exchange front=%d/%d,ref=%d and back=%d/%d,ref=%d, pixmap=%ld %dx%d\n",
+ __FUNCTION__, win->drawable.id,
+- front_bo->handle, front->name,
+- back_bo->handle, back->name,
++ front_bo->handle, front->name, get_private(front)->refcnt,
++ back_bo->handle, back->name, get_private(back)->refcnt,
+ pixmap->drawable.serialNumber,
+ pixmap->drawable.width,
+ pixmap->drawable.height));
+
+- DBG(("%s: back_bo pitch=%d, size=%d, ref=%d, active_scanout?=%d\n",
+- __FUNCTION__, back_bo->pitch, kgem_bo_size(back_bo), back_bo->refcnt, back_bo->active_scanout));
+- DBG(("%s: front_bo pitch=%d, size=%d, ref=%d, active_scanout?=%d\n",
+- __FUNCTION__, front_bo->pitch, kgem_bo_size(front_bo), front_bo->refcnt, front_bo->active_scanout));
++ DBG(("%s: back_bo handle=%d, pitch=%d, size=%d, ref=%d, active_scanout?=%d\n",
++ __FUNCTION__, back_bo->handle, back_bo->pitch, kgem_bo_size(back_bo), back_bo->refcnt, back_bo->active_scanout));
++ DBG(("%s: front_bo handle=%d, pitch=%d, size=%d, ref=%d, active_scanout?=%d\n",
++ __FUNCTION__, front_bo->handle, front_bo->pitch, kgem_bo_size(front_bo), front_bo->refcnt, front_bo->active_scanout));
++
++ assert(front_bo != back_bo);
+ assert(front_bo->refcnt);
+ assert(back_bo->refcnt);
+
+@@ -1894,6 +2385,11 @@ sna_dri2_xchg(DrawablePtr draw, DRI2BufferPtr front, DRI2BufferPtr back)
+ get_private(back)->bo = front_bo;
+ mark_stale(back);
+
++ assert(front_bo->active_scanout > 0);
++ front_bo->active_scanout--;
++ back_bo->active_scanout++;
++ assert(back_bo->active_scanout <= back_bo->refcnt);
++
+ tmp = front->name;
+ front->name = back->name;
+ back->name = tmp;
+@@ -1902,17 +2398,23 @@ sna_dri2_xchg(DrawablePtr draw, DRI2BufferPtr front, DRI2BufferPtr back)
+ front->pitch = back->pitch;
+ back->pitch = tmp;
+
++ tmp = front->flags;
++ front->flags = back->flags;
++ back->flags = tmp;
++
+ assert(front_bo->refcnt);
+ assert(back_bo->refcnt);
+
++ assert(front_bo->pitch == get_private(front)->bo->pitch);
++ assert(back_bo->pitch == get_private(back)->bo->pitch);
++
+ assert(get_private(front)->bo == sna_pixmap(pixmap)->gpu_bo);
+ }
+
+ static void sna_dri2_xchg_crtc(struct sna *sna, DrawablePtr draw, xf86CrtcPtr crtc, DRI2BufferPtr front, DRI2BufferPtr back)
+ {
+ WindowPtr win = (WindowPtr)draw;
+- DRI2Buffer2Ptr tmp;
+- struct kgem_bo *bo;
++ struct dri2_window *priv = dri2_window(win);
+
+ DBG(("%s: exchange front=%d/%d and back=%d/%d, win id=%lu, pixmap=%ld %dx%d\n",
+ __FUNCTION__,
+@@ -1922,162 +2424,130 @@ static void sna_dri2_xchg_crtc(struct sna *sna, DrawablePtr draw, xf86CrtcPtr cr
+ get_window_pixmap(win)->drawable.serialNumber,
+ get_window_pixmap(win)->drawable.width,
+ get_window_pixmap(win)->drawable.height));
++ assert(can_xchg_crtc(sna, draw, crtc, front, back));
+
+- DamageRegionAppend(&win->drawable, &win->clipList);
++ if (APPLY_DAMAGE) {
++ DBG(("%s: marking drawable as damaged\n", __FUNCTION__));
++ sna->ignore_copy_area = sna->flags & SNA_TEAR_FREE;
++ DamageRegionAppend(&win->drawable, &win->clipList);
++ }
+ sna_shadow_set_crtc(sna, crtc, get_private(back)->bo);
+- DamageRegionProcessPending(&win->drawable);
++ if (APPLY_DAMAGE) {
++ sna->ignore_copy_area = false;
++ DamageRegionProcessPending(&win->drawable);
++ }
+
+- assert(dri2_window(win)->front == NULL);
++ if (priv->front == NULL) {
++ DRI2Buffer2Ptr tmp;
+
+- tmp = calloc(1, sizeof(*tmp) + sizeof(struct sna_dri2_private));
+- if (tmp == NULL) {
+- back->attachment = -1;
+- if (get_private(back)->proxy == NULL) {
+- get_private(back)->pixmap = get_window_pixmap(win);
+- get_private(back)->proxy = sna_dri2_reference_buffer(sna_pixmap_get_buffer(get_private(back)->pixmap));
++ tmp = calloc(1, sizeof(*tmp) + sizeof(struct sna_dri2_private));
++ if (tmp == NULL) {
++ sna_shadow_unset_crtc(sna, crtc);
++ return;
+ }
+- dri2_window(win)->front = sna_dri2_reference_buffer(back);
+- return;
+- }
+
+- *tmp = *back;
+- tmp->attachment = DRI2BufferFrontLeft;
+- tmp->driverPrivate = tmp + 1;
+- get_private(tmp)->refcnt = 1;
+- get_private(tmp)->bo = get_private(back)->bo;
+- get_private(tmp)->size = get_private(back)->size;
+- get_private(tmp)->pixmap = get_window_pixmap(win);
+- get_private(tmp)->proxy = sna_dri2_reference_buffer(sna_pixmap_get_buffer(get_private(tmp)->pixmap));
+- dri2_window(win)->front = tmp;
+-
+- DBG(("%s: allocating new backbuffer\n", __FUNCTION__));
+- back->name = 0;
+- bo = kgem_create_2d(&sna->kgem,
+- draw->width, draw->height, draw->bitsPerPixel,
+- get_private(back)->bo->tiling,
+- CREATE_SCANOUT);
+- if (bo != NULL) {
+- get_private(back)->bo = bo;
+- back->pitch = bo->pitch;
+- back->name = kgem_bo_flink(&sna->kgem, bo);
+- }
+- if (back->name == 0) {
+- if (bo != NULL)
+- kgem_bo_destroy(&sna->kgem, bo);
+- get_private(back)->bo = NULL;
+- back->attachment = -1;
++ tmp->attachment = DRI2BufferFrontLeft;
++ tmp->driverPrivate = tmp + 1;
++ tmp->cpp = back->cpp;
++ tmp->format = back->format;
++
++ get_private(tmp)->refcnt = 1;
++ get_private(tmp)->bo = kgem_create_2d(&sna->kgem,
++ draw->width, draw->height, draw->bitsPerPixel,
++ get_private(back)->bo->tiling,
++ CREATE_SCANOUT | CREATE_EXACT);
++ if (get_private(tmp)->bo != NULL) {
++ tmp->pitch = get_private(tmp)->bo->pitch;
++ tmp->name = kgem_bo_flink(&sna->kgem, get_private(tmp)->bo);
++ }
++ if (tmp->name == 0) {
++ if (get_private(tmp)->bo != NULL)
++ kgem_bo_destroy(&sna->kgem, get_private(tmp)->bo);
++ sna_shadow_unset_crtc(sna, crtc);
++ return;
++ }
++ get_private(tmp)->size = get_private(back)->size;
++ get_private(tmp)->pixmap = get_private(front)->pixmap;
++ get_private(tmp)->proxy = sna_dri2_reference_buffer(front);
++ get_private(tmp)->bo->active_scanout++;
++
++ priv->front = front = tmp;
+ }
+-}
++ assert(front == priv->front);
+
+-static void frame_swap_complete(struct sna_dri2_event *frame, int type)
+-{
+- const struct ust_msc *swap;
++ {
++ struct kgem_bo *front_bo = get_private(front)->bo;
++ struct kgem_bo *back_bo = get_private(back)->bo;
++ unsigned tmp;
+
+- if (frame->draw == NULL)
+- return;
++ assert(front_bo->refcnt);
++ assert(back_bo->refcnt);
+
+- assert(frame->client);
++ get_private(back)->bo = front_bo;
++ get_private(front)->bo = back_bo;
++ mark_stale(back);
+
+- swap = sna_crtc_last_swap(frame->crtc);
+- DBG(("%s(type=%d): draw=%ld, pipe=%d, frame=%lld [msc=%lld], tv=%d.%06d\n",
+- __FUNCTION__, type, (long)frame->draw, frame->pipe,
+- (long long)swap->msc,
+- (long long)draw_current_msc(frame->draw, frame->crtc, swap->msc),
+- swap->tv_sec, swap->tv_usec));
++ assert(front_bo->active_scanout > 0);
++ front_bo->active_scanout--;
++ back_bo->active_scanout++;
++ assert(back_bo->active_scanout <= back_bo->refcnt);
+
+- DRI2SwapComplete(frame->client, frame->draw,
+- draw_current_msc(frame->draw, frame->crtc, swap->msc),
+- swap->tv_sec, swap->tv_usec,
+- type, frame->event_complete, frame->event_data);
+-}
++ tmp = front->name;
++ front->name = back->name;
++ back->name = tmp;
+
+-static void fake_swap_complete(struct sna *sna, ClientPtr client,
+- DrawablePtr draw, xf86CrtcPtr crtc,
+- int type, DRI2SwapEventPtr func, void *data)
+-{
+- const struct ust_msc *swap;
+-
+- swap = sna_crtc_last_swap(crtc);
+- DBG(("%s(type=%d): draw=%ld, pipe=%d, frame=%lld [msc %lld], tv=%d.%06d\n",
+- __FUNCTION__, type, (long)draw->id, crtc ? sna_crtc_to_pipe(crtc) : -1,
+- (long long)swap->msc,
+- (long long)draw_current_msc(draw, crtc, swap->msc),
+- swap->tv_sec, swap->tv_usec));
++ tmp = front->pitch;
++ front->pitch = back->pitch;
++ back->pitch = tmp;
+
+- DRI2SwapComplete(client, draw,
+- draw_current_msc(draw, crtc, swap->msc),
+- swap->tv_sec, swap->tv_usec,
+- type, func, data);
++ tmp = front->flags;
++ front->flags = back->flags;
++ back->flags = tmp;
++ }
+ }
+
+ static void chain_swap(struct sna_dri2_event *chain)
+ {
+- union drm_wait_vblank vbl;
++ DBG(("%s: draw=%ld, queued?=%d, type=%d\n",
++ __FUNCTION__, (long)chain->draw->id, chain->queued, chain->type));
++
++ if (chain->queued) /* too early! */
++ return;
+
+ if (chain->draw == NULL) {
+ sna_dri2_event_free(chain);
+ return;
+ }
+
+- if (chain->queued) /* too early! */
+- return;
+-
+ assert(chain == dri2_chain(chain->draw));
+- DBG(("%s: chaining draw=%ld, type=%d\n",
+- __FUNCTION__, (long)chain->draw->id, chain->type));
+- chain->queued = true;
++ assert(chain->signal);
+
+ switch (chain->type) {
+- case SWAP_THROTTLE:
++ case SWAP_COMPLETE:
+ DBG(("%s: emitting chained vsync'ed blit\n", __FUNCTION__));
+- if (chain->sna->mode.shadow &&
+- !chain->sna->mode.shadow_damage) {
+- /* recursed from wait_for_shadow(), simply requeue */
+- DBG(("%s -- recursed from wait_for_shadow(), requeuing\n", __FUNCTION__));
+- VG_CLEAR(vbl);
+- vbl.request.type =
+- DRM_VBLANK_RELATIVE |
+- DRM_VBLANK_EVENT;
+- vbl.request.sequence = 1;
+- vbl.request.signal = (uintptr_t)chain;
+-
+- if (!sna_wait_vblank(chain->sna, &vbl, chain->pipe))
+- return;
+-
+- DBG(("%s -- requeue failed, errno=%d\n", __FUNCTION__, errno));
+- }
+-
+ if (can_xchg(chain->sna, chain->draw, chain->front, chain->back)) {
+ sna_dri2_xchg(chain->draw, chain->front, chain->back);
+- } else if (can_xchg_crtc(chain->sna, chain->draw, chain->front, chain->back, chain->crtc)) {
+- sna_dri2_xchg_crtc(chain->sna, chain->draw, chain->crtc, chain->front, chain->back);
++ } else if (can_xchg_crtc(chain->sna, chain->draw, chain->crtc,
++ chain->front, chain->back)) {
++ sna_dri2_xchg_crtc(chain->sna, chain->draw, chain->crtc,
++ chain->front, chain->back);
+ } else {
+- assert(chain->queued);
+- chain->bo = __sna_dri2_copy_region(chain->sna, chain->draw, NULL,
+- chain->back, chain->front,
+- true);
++ __sna_dri2_copy_event(chain, chain->sync | DRI2_BO);
+ }
++ assert(get_private(chain->back)->bo != get_private(chain->front)->bo);
+ case SWAP:
+ break;
+ default:
+ return;
+ }
+
+- VG_CLEAR(vbl);
+- vbl.request.type =
+- DRM_VBLANK_RELATIVE |
+- DRM_VBLANK_EVENT;
+- vbl.request.sequence = 1;
+- vbl.request.signal = (uintptr_t)chain;
+- if (sna_wait_vblank(chain->sna, &vbl, chain->pipe)) {
++ if ((chain->type == SWAP_COMPLETE &&
++ !swap_limit(chain->draw, 2 + !chain->sync) &&
++ !chain->sync) ||
++ !sna_next_vblank(chain)) {
+ DBG(("%s: vblank wait failed, unblocking client\n", __FUNCTION__));
+ frame_swap_complete(chain, DRI2_BLIT_COMPLETE);
+ sna_dri2_event_free(chain);
+- } else {
+- if (chain->type == SWAP_THROTTLE && !swap_limit(chain->draw, 2)) {
+- DBG(("%s: fake triple buffering, unblocking client\n", __FUNCTION__));
+- frame_swap_complete(chain, DRI2_BLIT_COMPLETE);
+- }
+ }
+ }
+
+@@ -2086,40 +2556,27 @@ static inline bool rq_is_busy(struct kgem *kgem, struct kgem_bo *bo)
+ if (bo == NULL)
+ return false;
+
+- DBG(("%s: handle=%d, domain: %d exec? %d, rq? %d\n", __FUNCTION__,
+- bo->handle, bo->domain, bo->exec != NULL, bo->rq != NULL));
+- assert(bo->refcnt);
+-
+- if (bo->exec)
+- return true;
+-
+- if (bo->rq == NULL)
+- return false;
+-
+- return __kgem_busy(kgem, bo->handle);
++ return __kgem_bo_is_busy(kgem, bo);
+ }
+
+-static bool sna_dri2_blit_complete(struct sna *sna,
+- struct sna_dri2_event *info)
++static bool sna_dri2_blit_complete(struct sna_dri2_event *info)
+ {
+- if (rq_is_busy(&sna->kgem, info->bo)) {
+- union drm_wait_vblank vbl;
++ if (!info->bo)
++ return true;
+
++ if (__kgem_bo_is_busy(&info->sna->kgem, info->bo)) {
+ DBG(("%s: vsync'ed blit is still busy, postponing\n",
+ __FUNCTION__));
+-
+- VG_CLEAR(vbl);
+- vbl.request.type =
+- DRM_VBLANK_RELATIVE |
+- DRM_VBLANK_EVENT;
+- vbl.request.sequence = 1;
+- vbl.request.signal = (uintptr_t)info;
+- assert(info->queued);
+- if (!sna_wait_vblank(sna, &vbl, info->pipe))
++ if (sna_next_vblank(info))
+ return false;
++
++ kgem_bo_sync__gtt(&info->sna->kgem, info->bo);
+ }
+
+ DBG(("%s: blit finished\n", __FUNCTION__));
++ kgem_bo_destroy(&info->sna->kgem, info->bo);
++ info->bo = NULL;
++
+ return true;
+ }
+
+@@ -2128,11 +2585,12 @@ void sna_dri2_vblank_handler(struct drm_event_vblank *event)
+ struct sna_dri2_event *info = (void *)(uintptr_t)event->user_data;
+ struct sna *sna = info->sna;
+ DrawablePtr draw;
+- union drm_wait_vblank vbl;
+ uint64_t msc;
+
+- DBG(("%s(type=%d, sequence=%d)\n", __FUNCTION__, info->type, event->sequence));
++ DBG(("%s(type=%d, sequence=%d, draw=%ld)\n", __FUNCTION__, info->type, event->sequence, info->draw ? info->draw->serialNumber : 0));
+ assert(info->queued);
++ info->queued = false;
++
+ msc = sna_crtc_record_event(info->crtc, event);
+
+ draw = info->draw;
+@@ -2141,68 +2599,120 @@ void sna_dri2_vblank_handler(struct drm_event_vblank *event)
+ goto done;
+ }
+
++ assert((info->front == NULL && info->back == NULL) || info->front != info->back);
+ switch (info->type) {
+ case FLIP:
+ /* If we can still flip... */
++ assert(info->signal);
+ if (can_flip(sna, draw, info->front, info->back, info->crtc) &&
+ sna_dri2_flip(info))
+ return;
+
+ /* else fall through to blit */
+ case SWAP:
+- assert(info->queued);
+- if (sna->mode.shadow && !sna->mode.shadow_damage) {
+- /* recursed from wait_for_shadow(), simply requeue */
+- DBG(("%s -- recursed from wait_for_shadow(), requeuing\n", __FUNCTION__));
+-
+- } else if (can_xchg(info->sna, draw, info->front, info->back)) {
++ assert(info->signal);
++ if (can_xchg(info->sna, draw, info->front, info->back)) {
+ sna_dri2_xchg(draw, info->front, info->back);
+- info->type = SWAP_WAIT;
+- } else if (can_xchg_crtc(sna, draw, info->front, info->back, info->crtc)) {
+- sna_dri2_xchg_crtc(sna, draw, info->crtc, info->front, info->back);
+- info->type = SWAP_WAIT;
++ info->type = SWAP_COMPLETE;
++ } else if (can_xchg_crtc(sna, draw, info->crtc,
++ info->front, info->back)) {
++ sna_dri2_xchg_crtc(sna, draw, info->crtc,
++ info->front, info->back);
++ info->type = SWAP_COMPLETE;
+ } else {
+- assert(info->queued);
+- info->bo = __sna_dri2_copy_region(sna, draw, NULL,
+- info->back, info->front, true);
+- info->type = SWAP_WAIT;
++ __sna_dri2_copy_event(info, DRI2_BO | DRI2_SYNC);
++ info->type = SWAP_COMPLETE;
+ }
+
+- VG_CLEAR(vbl);
+- vbl.request.type =
+- DRM_VBLANK_RELATIVE |
+- DRM_VBLANK_EVENT;
+- vbl.request.sequence = 1;
+- vbl.request.signal = (uintptr_t)info;
+-
+- assert(info->queued);
+- if (!sna_wait_vblank(sna, &vbl, info->pipe))
++ if (sna_next_vblank(info))
+ return;
+
+ DBG(("%s -- requeue failed, errno=%d\n", __FUNCTION__, errno));
++ assert(info->pending.bo == NULL);
++ assert(info->keepalive == 1);
+ /* fall through to SwapComplete */
+- case SWAP_WAIT:
+- if (!sna_dri2_blit_complete(sna, info))
+- return;
+-
+- DBG(("%s: swap complete, unblocking client (frame=%d, tv=%d.%06d)\n", __FUNCTION__,
+- event->sequence, event->tv_sec, event->tv_usec));
+- frame_swap_complete(info, DRI2_BLIT_COMPLETE);
+- break;
+-
+- case SWAP_THROTTLE:
++ case SWAP_COMPLETE:
+ DBG(("%s: %d complete, frame=%d tv=%d.%06d\n",
+ __FUNCTION__, info->type,
+ event->sequence, event->tv_sec, event->tv_usec));
+
+- if (xorg_can_triple_buffer()) {
+- if (!sna_dri2_blit_complete(sna, info))
++ if (info->signal) {
++ if (!sna_dri2_blit_complete(info))
+ return;
+
+ DBG(("%s: triple buffer swap complete, unblocking client (frame=%d, tv=%d.%06d)\n", __FUNCTION__,
+ event->sequence, event->tv_sec, event->tv_usec));
+ frame_swap_complete(info, DRI2_BLIT_COMPLETE);
+ }
++
++ if (info->pending.bo) {
++ struct copy current_back;
++
++ DBG(("%s: swapping back handle=%d [name=%d, active=%d] for pending handle=%d [name=%d, active=%d], front handle=%d [name=%d, active=%d]\n",
++ __FUNCTION__,
++ get_private(info->back)->bo->handle, info->back->name, get_private(info->back)->bo->active_scanout,
++ info->pending.bo->handle, info->pending.name, info->pending.bo->active_scanout,
++ get_private(info->front)->bo->handle, info->front->name, get_private(info->front)->bo->active_scanout));
++
++ assert(info->pending.bo->active_scanout > 0);
++ info->pending.bo->active_scanout--;
++
++ current_back.bo = get_private(info->back)->bo;
++ current_back.size = get_private(info->back)->size;
++ current_back.name = info->back->name;
++ current_back.flags = info->back->flags;
++
++ get_private(info->back)->bo = info->pending.bo;
++ get_private(info->back)->size = info->pending.size;
++ info->back->name = info->pending.name;
++ info->back->pitch = info->pending.bo->pitch;
++ info->back->flags = info->pending.flags;
++ info->pending.bo = NULL;
++
++ assert(get_private(info->back)->bo != get_private(info->front)->bo);
++
++ if (can_xchg(info->sna, info->draw, info->front, info->back))
++ sna_dri2_xchg(info->draw, info->front, info->back);
++ else if (can_xchg_crtc(info->sna, info->draw, info->crtc,
++ info->front, info->back))
++ sna_dri2_xchg_crtc(info->sna, info->draw, info->crtc,
++ info->front, info->back);
++ else
++ __sna_dri2_copy_event(info, info->sync | DRI2_BO);
++
++ sna_dri2_cache_bo(info->sna, info->draw,
++ get_private(info->back)->bo,
++ info->back->name,
++ get_private(info->back)->size,
++ info->back->flags);
++
++ get_private(info->back)->bo = current_back.bo;
++ get_private(info->back)->size = current_back.size;
++ info->back->name = current_back.name;
++ info->back->pitch = current_back.bo->pitch;
++ info->back->flags = current_back.flags;
++
++ DBG(("%s: restored current back handle=%d [name=%d, active=%d], active=%d], front handle=%d [name=%d, active=%d]\n",
++ __FUNCTION__,
++ get_private(info->back)->bo->handle, info->back->name, get_private(info->back)->bo->active_scanout,
++ get_private(info->front)->bo->handle, info->front->name, get_private(info->front)->bo->active_scanout));
++
++ assert(info->draw);
++ assert(!info->signal);
++ info->keepalive++;
++ info->signal = true;
++ }
++
++ if (--info->keepalive) {
++ if (sna_next_vblank(info))
++ return;
++
++ if (info->signal) {
++ DBG(("%s: triple buffer swap complete, unblocking client (frame=%d, tv=%d.%06d)\n", __FUNCTION__,
++ event->sequence, event->tv_sec, event->tv_usec));
++ frame_swap_complete(info, DRI2_BLIT_COMPLETE);
++ }
++ }
+ break;
+
+ case WAITMSC:
+@@ -2218,11 +2728,11 @@ void sna_dri2_vblank_handler(struct drm_event_vblank *event)
+ }
+
+ if (info->chain) {
++ DBG(("%s: continuing chain\n", __FUNCTION__));
+ assert(info->chain != info);
+ assert(info->draw == draw);
+- sna_dri2_remove_event((WindowPtr)draw, info);
++ sna_dri2_remove_event(info);
+ chain_swap(info->chain);
+- info->draw = NULL;
+ }
+
+ done:
+@@ -2230,101 +2740,148 @@ done:
+ DBG(("%s complete\n", __FUNCTION__));
+ }
+
+-static bool
++static void
+ sna_dri2_immediate_blit(struct sna *sna,
+ struct sna_dri2_event *info,
+- bool sync, bool event)
++ bool sync)
+ {
+- DrawablePtr draw = info->draw;
+- bool ret = false;
++ struct sna_dri2_event *chain = dri2_chain(info->draw);
+
+ if (sna->flags & SNA_NO_WAIT)
+ sync = false;
+
+- DBG(("%s: emitting immediate blit, throttling client, synced? %d, chained? %d, send-event? %d\n",
+- __FUNCTION__, sync, dri2_chain(draw) != info,
+- event));
++ DBG(("%s: emitting immediate blit, throttling client, synced? %d, chained? %d, pipe %d\n",
++ __FUNCTION__, sync, chain != info, info->pipe));
++ assert(chain);
+
+- info->type = SWAP_THROTTLE;
+- if (!sync || dri2_chain(draw) == info) {
+- DBG(("%s: no pending blit, starting chain\n",
+- __FUNCTION__));
++ info->type = SWAP_COMPLETE;
++ info->sync = sync;
++ info->keepalive = KEEPALIVE;
+
+- info->queued = true;
+- info->bo = __sna_dri2_copy_region(sna, draw, NULL,
+- info->back,
+- info->front,
+- sync);
+- if (event) {
+- if (sync) {
+- union drm_wait_vblank vbl;
+-
+- VG_CLEAR(vbl);
+- vbl.request.type =
+- DRM_VBLANK_RELATIVE |
+- DRM_VBLANK_EVENT;
+- vbl.request.sequence = 1;
+- vbl.request.signal = (uintptr_t)info;
+- ret = !sna_wait_vblank(sna, &vbl, info->pipe);
+- if (ret)
+- event = !swap_limit(draw, 2);
+- }
+- if (event) {
+- DBG(("%s: fake triple buffering, unblocking client\n", __FUNCTION__));
+- frame_swap_complete(info, DRI2_BLIT_COMPLETE);
+- }
++ if (chain == info) {
++ DBG(("%s: no pending blit, starting chain\n", __FUNCTION__));
++
++ assert(info->front != info->back);
++ if (can_xchg(info->sna, info->draw, info->front, info->back)) {
++ sna_dri2_xchg(info->draw, info->front, info->back);
++ } else if (can_xchg_crtc(info->sna, info->draw, info->crtc,
++ info->front, info->back)) {
++ sna_dri2_xchg_crtc(info->sna, info->draw, info->crtc,
++ info->front, info->back);
++ } else
++ __sna_dri2_copy_event(info, sync | DRI2_BO);
++
++ assert(info->signal);
++
++ if ((!swap_limit(info->draw, 2 + !sync) && !sync) ||
++ !sna_next_vblank(info)) {
++ DBG(("%s: fake triple buffering, unblocking client\n", __FUNCTION__));
++ frame_swap_complete(info, DRI2_BLIT_COMPLETE);
++ sna_dri2_event_free(info);
++ }
++ return;
++ }
++
++ DBG(("%s: current event front=%d [name=%d, active?=%d], back=%d [name=%d, active?=%d]\n", __FUNCTION__,
++ get_private(chain->front)->bo->handle, chain->front->name, get_private(chain->front)->bo->active_scanout,
++ get_private(chain->back)->bo->handle, chain->back->name, get_private(chain->back)->bo->active_scanout));
++
++ if (chain->type == SWAP_COMPLETE && chain->front == info->front) {
++ assert(chain->draw == info->draw);
++ assert(chain->client == info->client);
++ assert(chain->event_complete == info->event_complete);
++ assert(chain->event_data == info->event_data);
++ assert(chain->queued);
++
++ if ((!sync || !chain->sync) && chain->pending.bo) {
++ bool signal = chain->signal;
++
++ DBG(("%s: swap elision, unblocking client\n", __FUNCTION__));
++ assert(chain->draw);
++ chain->signal = true;
++ frame_swap_complete(chain, DRI2_EXCHANGE_COMPLETE);
++ chain->signal = signal;
++
++ assert(chain->pending.bo->active_scanout > 0);
++ chain->pending.bo->active_scanout--;
++
++ sna_dri2_cache_bo(chain->sna, chain->draw,
++ chain->pending.bo,
++ chain->pending.name,
++ chain->pending.size,
++ chain->pending.flags);
++ chain->pending.bo = NULL;
++ }
++
++ if (chain->pending.bo == NULL && swap_limit(info->draw, 2 + !sync)) {
++ DBG(("%s: setting handle=%d as pending blit (current event front=%d, back=%d)\n", __FUNCTION__,
++ get_private(info->back)->bo->handle,
++ get_private(chain->front)->bo->handle,
++ get_private(chain->back)->bo->handle));
++ chain->pending.bo = ref(get_private(info->back)->bo);
++ chain->pending.size = get_private(info->back)->size;
++ chain->pending.name = info->back->name;
++ chain->pending.flags = info->back->flags;
++ chain->sync = sync;
++ info->signal = false; /* transfer signal to pending */
++
++ /* Prevent us from handing it back on next GetBuffers */
++ chain->pending.bo->active_scanout++;
++
++ sna_dri2_event_free(info);
++ return;
+ }
+- } else {
+- DBG(("%s: pending blit, chained\n", __FUNCTION__));
+- ret = true;
+ }
+
+- DBG(("%s: continue? %d\n", __FUNCTION__, ret));
+- return ret;
++ DBG(("%s: pending blit, chained\n", __FUNCTION__));
+ }
+
+ static bool
+ sna_dri2_flip_continue(struct sna_dri2_event *info)
+ {
+- DBG(("%s(mode=%d)\n", __FUNCTION__, info->mode));
++ struct kgem_bo *bo = get_private(info->front)->bo;
+
+- if (info->mode > 0){
+- struct kgem_bo *bo = get_private(info->front)->bo;
++ DBG(("%s(mode=%d)\n", __FUNCTION__, info->flip_continue));
++ assert(info->flip_continue > 0);
++ info->type = info->flip_continue;
++ info->flip_continue = 0;
+
+- info->type = info->mode;
++ assert(!info->signal);
++ info->signal = info->type == FLIP_THROTTLE && info->draw;
+
+- if (bo != sna_pixmap(info->sna->front)->gpu_bo)
+- return false;
++ if (info->sna->mode.front_active == 0)
++ return false;
+
+- if (!sna_page_flip(info->sna, bo, sna_dri2_flip_handler, info))
+- return false;
++ if (bo != sna_pixmap(info->sna->front)->gpu_bo)
++ return false;
+
+- assert(info->sna->dri2.flip_pending == NULL ||
+- info->sna->dri2.flip_pending == info);
+- info->sna->dri2.flip_pending = info;
+- assert(info->queued);
+- } else {
+- info->type = -info->mode;
++ assert(!info->queued);
++ if (!sna_page_flip(info->sna, bo, sna_dri2_flip_handler, info))
++ return false;
+
+- if (!info->draw)
+- return false;
++ DBG(("%s: queued flip=%p\n", __FUNCTION__, info));
++ assert(info->sna->dri2.flip_pending == NULL ||
++ info->sna->dri2.flip_pending == info);
++ info->sna->dri2.flip_pending = info;
++ info->queued = true;
+
+- if (!can_flip(info->sna, info->draw, info->front, info->back, info->crtc))
+- return false;
++ return true;
++}
+
+- assert(sna_pixmap_get_buffer(get_drawable_pixmap(info->draw)) == info->front);
+- if (!sna_dri2_flip(info))
+- return false;
++static bool
++sna_dri2_flip_keepalive(struct sna_dri2_event *info)
++{
++ DBG(("%s(keepalive?=%d)\n", __FUNCTION__, info->keepalive-1));
++ assert(info->keepalive > 0);
++ if (!--info->keepalive)
++ return false;
+
+- if (!xorg_can_triple_buffer()) {
+- sna_dri2_get_back(info->sna, info->draw, info->back, info);
+- DBG(("%s: fake triple buffering, unblocking client\n", __FUNCTION__));
+- frame_swap_complete(info, DRI2_FLIP_COMPLETE);
+- }
+- }
++ if (info->draw == NULL)
++ return false;
+
+- info->mode = 0;
+- return true;
++ DBG(("%s: marking next flip as complete\n", __FUNCTION__));
++ info->flip_continue = FLIP_COMPLETE;
++ return sna_dri2_flip_continue(info);
+ }
+
+ static void chain_flip(struct sna *sna)
+@@ -2332,8 +2889,8 @@ static void chain_flip(struct sna *sna)
+ struct sna_dri2_event *chain = sna->dri2.flip_pending;
+
+ assert(chain->type == FLIP);
+- DBG(("%s: chaining type=%d, cancelled?=%d\n",
+- __FUNCTION__, chain->type, chain->draw == NULL));
++ DBG(("%s: chaining type=%d, cancelled?=%d window=%ld\n",
++ __FUNCTION__, chain->type, chain->draw == NULL, chain->draw ? chain->draw->id : 0));
+
+ sna->dri2.flip_pending = NULL;
+ if (chain->draw == NULL) {
+@@ -2343,31 +2900,18 @@ static void chain_flip(struct sna *sna)
+
+ assert(chain == dri2_chain(chain->draw));
+ assert(!chain->queued);
+- chain->queued = true;
+
+ if (can_flip(sna, chain->draw, chain->front, chain->back, chain->crtc) &&
+ sna_dri2_flip(chain)) {
+ DBG(("%s: performing chained flip\n", __FUNCTION__));
+ } else {
+ DBG(("%s: emitting chained vsync'ed blit\n", __FUNCTION__));
+- chain->bo = __sna_dri2_copy_region(sna, chain->draw, NULL,
+- chain->back, chain->front,
+- true);
++ __sna_dri2_copy_event(chain, DRI2_SYNC);
+
+ if (xorg_can_triple_buffer()) {
+- union drm_wait_vblank vbl;
+-
+- VG_CLEAR(vbl);
+-
+- chain->type = SWAP_WAIT;
+- vbl.request.type =
+- DRM_VBLANK_RELATIVE |
+- DRM_VBLANK_EVENT;
+- vbl.request.sequence = 1;
+- vbl.request.signal = (uintptr_t)chain;
+-
+- assert(chain->queued);
+- if (!sna_wait_vblank(sna, &vbl, chain->pipe))
++ chain->type = SWAP_COMPLETE;
++ assert(chain->signal);
++ if (sna_next_vblank(chain))
+ return;
+ }
+
+@@ -2381,8 +2925,10 @@ static void sna_dri2_flip_event(struct sna_dri2_event *flip)
+ {
+ struct sna *sna = flip->sna;
+
+- DBG(("%s(pipe=%d, event=%d)\n", __FUNCTION__, flip->pipe, flip->type));
+- assert(flip->queued);
++ DBG(("%s flip=%p (pipe=%d, event=%d, queued?=%d)\n", __FUNCTION__, flip, flip->pipe, flip->type, flip->queued));
++ if (!flip->queued) /* pageflip died whilst being queued */
++ return;
++ flip->queued = false;
+
+ if (sna->dri2.flip_pending == flip)
+ sna->dri2.flip_pending = NULL;
+@@ -2390,8 +2936,10 @@ static void sna_dri2_flip_event(struct sna_dri2_event *flip)
+ /* We assume our flips arrive in order, so we don't check the frame */
+ switch (flip->type) {
+ case FLIP:
+- DBG(("%s: swap complete, unblocking client\n", __FUNCTION__));
+- frame_swap_complete(flip, DRI2_FLIP_COMPLETE);
++ if (flip->signal) {
++ DBG(("%s: swap complete, unblocking client\n", __FUNCTION__));
++ frame_swap_complete(flip, DRI2_FLIP_COMPLETE);
++ }
+ sna_dri2_event_free(flip);
+
+ if (sna->dri2.flip_pending)
+@@ -2399,27 +2947,35 @@ static void sna_dri2_flip_event(struct sna_dri2_event *flip)
+ break;
+
+ case FLIP_THROTTLE:
+- DBG(("%s: triple buffer swap complete, unblocking client\n", __FUNCTION__));
+- frame_swap_complete(flip, DRI2_FLIP_COMPLETE);
++ if (flip->signal) {
++ DBG(("%s: triple buffer swap complete, unblocking client\n", __FUNCTION__));
++ frame_swap_complete(flip, DRI2_FLIP_COMPLETE);
++ }
+ case FLIP_COMPLETE:
++ assert(!flip->signal);
+ if (sna->dri2.flip_pending) {
++ DBG(("%s: pending flip\n", __FUNCTION__));
+ sna_dri2_event_free(flip);
+ chain_flip(sna);
+- } else if (!flip->mode) {
++ } else if (!flip->flip_continue) {
+ DBG(("%s: flip chain complete\n", __FUNCTION__));
++ if (!sna_dri2_flip_keepalive(flip)) {
++ if (flip->chain) {
++ sna_dri2_remove_event(flip);
++ chain_swap(flip->chain);
++ }
+
+- if (flip->chain) {
+- sna_dri2_remove_event((WindowPtr)flip->draw,
+- flip);
+- chain_swap(flip->chain);
+- flip->draw = NULL;
++ sna_dri2_event_free(flip);
+ }
+-
+- sna_dri2_event_free(flip);
+ } else if (!sna_dri2_flip_continue(flip)) {
+ DBG(("%s: no longer able to flip\n", __FUNCTION__));
+- if (flip->draw == NULL || !sna_dri2_immediate_blit(sna, flip, false, flip->mode < 0))
+- sna_dri2_event_free(flip);
++ if (flip->draw != NULL)
++ __sna_dri2_copy_event(flip, 0);
++ if (flip->signal) {
++ DBG(("%s: fake triple buffering, unblocking client\n", __FUNCTION__));
++ frame_swap_complete(flip, DRI2_BLIT_COMPLETE);
++ }
++ sna_dri2_event_free(flip);
+ }
+ break;
+
+@@ -2433,17 +2989,27 @@ static void sna_dri2_flip_event(struct sna_dri2_event *flip)
+ }
+ }
+
++static int
++sna_query_vblank(struct sna *sna, xf86CrtcPtr crtc, union drm_wait_vblank *vbl)
++{
++ VG_CLEAR(*vbl);
++ vbl->request.type =
++ _DRM_VBLANK_RELATIVE | pipe_select(sna_crtc_pipe(crtc));
++ vbl->request.sequence = 0;
++
++ return drmIoctl(sna->kgem.fd, DRM_IOCTL_WAIT_VBLANK, vbl);
++}
++
+ static uint64_t
+ get_current_msc(struct sna *sna, DrawablePtr draw, xf86CrtcPtr crtc)
+ {
+ union drm_wait_vblank vbl;
+- uint64_t ret = -1;
++ uint64_t ret;
+
+- VG_CLEAR(vbl);
+- vbl.request.type = _DRM_VBLANK_RELATIVE;
+- vbl.request.sequence = 0;
+- if (sna_wait_vblank(sna, &vbl, sna_crtc_to_pipe(crtc)) == 0)
++ if (sna_query_vblank(sna, crtc, &vbl) == 0)
+ ret = sna_crtc_record_vblank(crtc, &vbl);
++ else
++ ret = sna_crtc_last_swap(crtc)->msc;
+
+ return draw_current_msc(draw, crtc, ret);
+ }
+@@ -2494,12 +3060,18 @@ static int use_triple_buffer(struct sna *sna, ClientPtr client, bool async)
+ }
+
+ static bool immediate_swap(struct sna *sna,
+- uint64_t target_msc,
+- uint64_t divisor,
+ DrawablePtr draw,
+ xf86CrtcPtr crtc,
++ uint64_t *target_msc,
++ uint64_t divisor,
++ uint64_t remainder,
+ uint64_t *current_msc)
+ {
++ /*
++ * If divisor is zero, or current_msc is smaller than target_msc
++ * we just need to make sure target_msc passes before initiating
++ * the swap.
++ */
+ if (divisor == 0) {
+ *current_msc = -1;
+
+@@ -2508,72 +3080,97 @@ static bool immediate_swap(struct sna *sna,
+ return true;
+ }
+
+- if (target_msc)
++ if (*target_msc)
+ *current_msc = get_current_msc(sna, draw, crtc);
+
+ DBG(("%s: current_msc=%ld, target_msc=%ld -- %s\n",
+- __FUNCTION__, (long)*current_msc, (long)target_msc,
+- (*current_msc >= target_msc - 1) ? "yes" : "no"));
+- return *current_msc >= target_msc - 1;
++ __FUNCTION__, (long)*current_msc, (long)*target_msc,
++ (*current_msc >= *target_msc - 1) ? "yes" : "no"));
++ return *current_msc >= *target_msc - 1;
+ }
+
+ DBG(("%s: explicit waits requests, divisor=%ld\n",
+ __FUNCTION__, (long)divisor));
+ *current_msc = get_current_msc(sna, draw, crtc);
+- return false;
++ if (*current_msc >= *target_msc) {
++ DBG(("%s: missed target, queueing event for next: current=%lld, target=%lld, divisor=%lld, remainder=%lld\n",
++ __FUNCTION__,
++ (long long)*current_msc,
++ (long long)*target_msc,
++ (long long)divisor,
++ (long long)remainder));
++
++ *target_msc = *current_msc + remainder - *current_msc % divisor;
++ if (*target_msc <= *current_msc)
++ *target_msc += divisor;
++ }
++
++ DBG(("%s: target_msc=%lld, current_msc=%lld, immediate?=%d\n",
++ __FUNCTION__, (long long)*target_msc, (long long)*current_msc,
++ *current_msc >= *target_msc - 1));
++ return *current_msc >= *target_msc - 1;
+ }
+
+ static bool
+ sna_dri2_schedule_flip(ClientPtr client, DrawablePtr draw, xf86CrtcPtr crtc,
+ DRI2BufferPtr front, DRI2BufferPtr back,
+- CARD64 *target_msc, CARD64 divisor, CARD64 remainder,
++ bool immediate, CARD64 *target_msc, CARD64 current_msc,
+ DRI2SwapEventPtr func, void *data)
+ {
+ struct sna *sna = to_sna_from_drawable(draw);
+ struct sna_dri2_event *info;
+- uint64_t current_msc;
+-
+- if (immediate_swap(sna, *target_msc, divisor, draw, crtc, &current_msc)) {
+- int type;
+
++ if (immediate) {
++ bool signal = false;
+ info = sna->dri2.flip_pending;
+ DBG(("%s: performing immediate swap on pipe %d, pending? %d, mode: %d, continuation? %d\n",
+- __FUNCTION__, sna_crtc_to_pipe(crtc),
+- info != NULL, info ? info->mode : 0,
++ __FUNCTION__, sna_crtc_pipe(crtc),
++ info != NULL, info ? info->flip_continue : 0,
+ info && info->draw == draw));
+
+ if (info && info->draw == draw) {
+ assert(info->type != FLIP);
+- assert(info->front == front);
++ assert(info->queued);
++ assert(info->front != info->back);
++ if (info->front != front) {
++ assert(info->front != NULL);
++ _sna_dri2_destroy_buffer(sna, draw, info->front);
++ info->front = sna_dri2_reference_buffer(front);
++ }
+ if (info->back != back) {
+- _sna_dri2_destroy_buffer(sna, info->back);
++ assert(info->back != NULL);
++ _sna_dri2_destroy_buffer(sna, draw, info->back);
+ info->back = sna_dri2_reference_buffer(back);
+ }
+- if (info->mode || current_msc >= *target_msc) {
+- DBG(("%s: executing xchg of pending flip\n",
+- __FUNCTION__));
+- sna_dri2_xchg(draw, front, back);
+- info->mode = type = FLIP_COMPLETE;
+- goto new_back;
+- } else {
++ assert(info->front != info->back);
++ DBG(("%s: executing xchg of pending flip: flip_continue=%d, keepalive=%d, chain?=%d\n", __FUNCTION__, info->flip_continue, info->keepalive, current_msc < *target_msc));
++ sna_dri2_xchg(draw, front, back);
++ info->keepalive = KEEPALIVE;
++ if (xorg_can_triple_buffer() &&
++ current_msc < *target_msc) {
+ DBG(("%s: chaining flip\n", __FUNCTION__));
+- type = FLIP_THROTTLE;
+- if (xorg_can_triple_buffer())
+- info->mode = -type;
+- else
+- info->mode = -FLIP_COMPLETE;
++ info->flip_continue = FLIP_THROTTLE;
+ goto out;
++ } else {
++ info->flip_continue = FLIP_COMPLETE;
++ signal = info->signal;
++ assert(info->draw);
++ info->signal = true;
++ goto new_back;
+ }
+ }
+
+- info = sna_dri2_add_event(sna, draw, client);
++ info = sna_dri2_add_event(sna, draw, client, crtc);
+ if (info == NULL)
+ return false;
+
+ assert(info->crtc == crtc);
+ info->event_complete = func;
+ info->event_data = data;
++ assert(info->draw);
++ info->signal = true;
+
++ assert(front != back);
+ info->front = sna_dri2_reference_buffer(front);
+ info->back = sna_dri2_reference_buffer(back);
+
+@@ -2584,26 +3181,33 @@ sna_dri2_schedule_flip(ClientPtr client, DrawablePtr draw, xf86CrtcPtr crtc,
+ */
+ DBG(("%s: queueing flip after pending completion\n",
+ __FUNCTION__));
+- info->type = type = FLIP;
++ info->type = FLIP;
+ sna->dri2.flip_pending = info;
+- assert(info->queued);
+ current_msc++;
++ } else if (sna->mode.flip_active) {
++ DBG(("%s: %d outstanding flips from old client, queueing\n",
++ __FUNCTION__, sna->mode.flip_active));
++ goto queue;
+ } else {
+- info->type = type = use_triple_buffer(sna, client, *target_msc == 0);
++ info->type = use_triple_buffer(sna, client, *target_msc == 0);
+ if (!sna_dri2_flip(info)) {
+ DBG(("%s: flip failed, falling back\n", __FUNCTION__));
++ info->signal = false;
+ sna_dri2_event_free(info);
+ return false;
+ }
++ assert(get_private(info->front)->bo->active_scanout);
+ }
+
+- swap_limit(draw, 1 + (type == FLIP_THROTTLE));
+- if (type >= FLIP_COMPLETE) {
++ swap_limit(draw, 1 + (info->type == FLIP_THROTTLE));
++ if (info->type >= FLIP_COMPLETE) {
+ new_back:
+ if (!xorg_can_triple_buffer())
+- sna_dri2_get_back(sna, draw, back, info);
++ sna_dri2_get_back(sna, draw, back);
+ DBG(("%s: fake triple buffering, unblocking client\n", __FUNCTION__));
+ frame_swap_complete(info, DRI2_EXCHANGE_COMPLETE);
++ assert(info->draw);
++ info->signal = signal;
+ if (info->type == FLIP_ASYNC)
+ sna_dri2_event_free(info);
+ }
+@@ -2613,57 +3217,34 @@ out:
+ return true;
+ }
+
+- info = sna_dri2_add_event(sna, draw, client);
++queue:
++ if (KEEPALIVE > 1 && sna->dri2.flip_pending) {
++ info = sna->dri2.flip_pending;
++ info->keepalive = 1;
++ }
++
++ info = sna_dri2_add_event(sna, draw, client, crtc);
+ if (info == NULL)
+ return false;
+
+ assert(info->crtc == crtc);
+ info->event_complete = func;
+ info->event_data = data;
++ assert(info->draw);
++ info->signal = true;
+ info->type = FLIP;
+
++ assert(front != back);
+ info->front = sna_dri2_reference_buffer(front);
+ info->back = sna_dri2_reference_buffer(back);
+
+- /*
+- * If divisor is zero, or current_msc is smaller than target_msc
+- * we just need to make sure target_msc passes before initiating
+- * the swap.
+- */
+- if (divisor && current_msc >= *target_msc) {
+- DBG(("%s: missed target, queueing event for next: current=%lld, target=%lld, divisor=%lld, remainder=%lld\n",
+- __FUNCTION__,
+- (long long)current_msc,
+- (long long)*target_msc,
+- (long long)divisor,
+- (long long)remainder));
+-
+- *target_msc = current_msc + remainder - current_msc % divisor;
+- if (*target_msc <= current_msc)
+- *target_msc += divisor;
+- }
+-
+- if (*target_msc <= current_msc + 1) {
+- if (!sna_dri2_flip(info)) {
+- sna_dri2_event_free(info);
+- return false;
+- }
++ if (*target_msc <= current_msc + 1 && sna_dri2_flip(info)) {
+ *target_msc = current_msc + 1;
+ } else {
+- union drm_wait_vblank vbl;
+-
+- VG_CLEAR(vbl);
+-
+- vbl.request.type =
+- DRM_VBLANK_ABSOLUTE |
+- DRM_VBLANK_EVENT;
+-
+ /* Account for 1 frame extra pageflip delay */
+- vbl.reply.sequence = draw_target_seq(draw, *target_msc - 1);
+- vbl.request.signal = (uintptr_t)info;
+-
+- info->queued = true;
+- if (sna_wait_vblank(sna, &vbl, info->pipe)) {
++ if (!sna_wait_vblank(info,
++ draw_target_seq(draw, *target_msc - 1))) {
++ info->signal = false;
+ sna_dri2_event_free(info);
+ return false;
+ }
+@@ -2674,128 +3255,6 @@ out:
+ return true;
+ }
+
+-static bool
+-sna_dri2_schedule_xchg(ClientPtr client, DrawablePtr draw, xf86CrtcPtr crtc,
+- DRI2BufferPtr front, DRI2BufferPtr back,
+- CARD64 *target_msc, CARD64 divisor, CARD64 remainder,
+- DRI2SwapEventPtr func, void *data)
+-{
+- struct sna *sna = to_sna_from_drawable(draw);
+- uint64_t current_msc;
+- bool sync, event;
+-
+- if (!immediate_swap(sna, *target_msc, divisor, draw, crtc, &current_msc))
+- return false;
+-
+- sync = current_msc < *target_msc;
+- event = dri2_chain(draw) == NULL;
+- if (!sync || event) {
+- DBG(("%s: performing immediate xchg on pipe %d\n",
+- __FUNCTION__, sna_crtc_to_pipe(crtc)));
+- sna_dri2_xchg(draw, front, back);
+- }
+- if (sync) {
+- struct sna_dri2_event *info;
+-
+- info = sna_dri2_add_event(sna, draw, client);
+- if (!info)
+- goto complete;
+-
+- info->event_complete = func;
+- info->event_data = data;
+-
+- info->front = sna_dri2_reference_buffer(front);
+- info->back = sna_dri2_reference_buffer(back);
+- info->type = SWAP_THROTTLE;
+-
+- if (event) {
+- union drm_wait_vblank vbl;
+-
+- VG_CLEAR(vbl);
+- vbl.request.type =
+- DRM_VBLANK_RELATIVE |
+- DRM_VBLANK_EVENT;
+- vbl.request.sequence = 1;
+- vbl.request.signal = (uintptr_t)info;
+-
+- info->queued = true;
+- if (sna_wait_vblank(sna, &vbl, info->pipe)) {
+- sna_dri2_event_free(info);
+- goto complete;
+- }
+-
+- swap_limit(draw, 2);
+- }
+- } else {
+-complete:
+- fake_swap_complete(sna, client, draw, crtc, DRI2_EXCHANGE_COMPLETE, func, data);
+- }
+-
+- *target_msc = current_msc + 1;
+- return true;
+-}
+-
+-static bool
+-sna_dri2_schedule_xchg_crtc(ClientPtr client, DrawablePtr draw, xf86CrtcPtr crtc,
+- DRI2BufferPtr front, DRI2BufferPtr back,
+- CARD64 *target_msc, CARD64 divisor, CARD64 remainder,
+- DRI2SwapEventPtr func, void *data)
+-{
+- struct sna *sna = to_sna_from_drawable(draw);
+- uint64_t current_msc;
+- bool sync, event;
+-
+- if (!immediate_swap(sna, *target_msc, divisor, draw, crtc, &current_msc))
+- return false;
+-
+- sync = current_msc < *target_msc;
+- event = dri2_chain(draw) == NULL;
+- if (!sync || event) {
+- DBG(("%s: performing immediate xchg only on pipe %d\n",
+- __FUNCTION__, sna_crtc_to_pipe(crtc)));
+- sna_dri2_xchg_crtc(sna, draw, crtc, front, back);
+- }
+- if (sync) {
+- struct sna_dri2_event *info;
+-
+- info = sna_dri2_add_event(sna, draw, client);
+- if (!info)
+- goto complete;
+-
+- info->event_complete = func;
+- info->event_data = data;
+-
+- info->front = sna_dri2_reference_buffer(front);
+- info->back = sna_dri2_reference_buffer(back);
+- info->type = SWAP_THROTTLE;
+-
+- if (event) {
+- union drm_wait_vblank vbl;
+-
+- VG_CLEAR(vbl);
+- vbl.request.type =
+- DRM_VBLANK_RELATIVE |
+- DRM_VBLANK_EVENT;
+- vbl.request.sequence = 1;
+- vbl.request.signal = (uintptr_t)info;
+-
+- info->queued = true;
+- if (sna_wait_vblank(sna, &vbl, info->pipe)) {
+- sna_dri2_event_free(info);
+- goto complete;
+- }
+-
+- swap_limit(draw, 2);
+- }
+- } else {
+-complete:
+- fake_swap_complete(sna, client, draw, crtc, DRI2_EXCHANGE_COMPLETE, func, data);
+- }
+-
+- *target_msc = current_msc + 1;
+- return true;
+-}
+-
+ static bool has_pending_events(struct sna *sna)
+ {
+ struct pollfd pfd;
+@@ -2830,11 +3289,11 @@ sna_dri2_schedule_swap(ClientPtr client, DrawablePtr draw, DRI2BufferPtr front,
+ CARD64 remainder, DRI2SwapEventPtr func, void *data)
+ {
+ struct sna *sna = to_sna_from_drawable(draw);
+- union drm_wait_vblank vbl;
+ xf86CrtcPtr crtc = NULL;
+ struct sna_dri2_event *info = NULL;
+ int type = DRI2_EXCHANGE_COMPLETE;
+ CARD64 current_msc;
++ bool immediate;
+
+ DBG(("%s: draw=%lu %dx%d, pixmap=%ld %dx%d, back=%u (refs=%d/%d, flush=%d) , front=%u (refs=%d/%d, flush=%d)\n",
+ __FUNCTION__,
+@@ -2860,6 +3319,7 @@ sna_dri2_schedule_swap(ClientPtr client, DrawablePtr draw, DRI2BufferPtr front,
+ assert(get_private(front)->refcnt);
+ assert(get_private(back)->refcnt);
+
++ assert(get_private(back)->bo != get_private(front)->bo);
+ assert(get_private(front)->bo->refcnt);
+ assert(get_private(back)->bo->refcnt);
+
+@@ -2876,17 +3336,17 @@ sna_dri2_schedule_swap(ClientPtr client, DrawablePtr draw, DRI2BufferPtr front,
+ goto skip;
+ }
+
+- assert(sna_pixmap_from_drawable(draw)->flush);
+-
+ if (draw->type != DRAWABLE_PIXMAP) {
+ WindowPtr win = (WindowPtr)draw;
+ struct dri2_window *priv = dri2_window(win);
++
+ if (priv->front) {
+- assert(front == priv->front);
+- assert(get_private(priv->front)->refcnt > 1);
+- get_private(priv->front)->refcnt--;
+- priv->front = NULL;
++ front = priv->front;
++ assert(front->attachment == DRI2BufferFrontLeft);
++ assert(get_private(front)->refcnt);
++ assert(get_private(front)->pixmap == get_drawable_pixmap(draw));
+ }
++
+ if (win->clipList.extents.x2 <= win->clipList.extents.x1 ||
+ win->clipList.extents.y2 <= win->clipList.extents.y1) {
+ DBG(("%s: window clipped (%d, %d), (%d, %d)\n",
+@@ -2899,6 +3359,10 @@ sna_dri2_schedule_swap(ClientPtr client, DrawablePtr draw, DRI2BufferPtr front,
+ }
+ }
+
++ DBG(("%s: using front handle=%d, active_scanout?=%d, flush?=%d\n", __FUNCTION__, get_private(front)->bo->handle, get_private(front)->bo->active_scanout, sna_pixmap_from_drawable(draw)->flush));
++ assert(get_private(front)->bo->active_scanout);
++ assert(sna_pixmap_from_drawable(draw)->flush);
++
+ /* Drawable not displayed... just complete the swap */
+ if ((sna->flags & SNA_NO_WAIT) == 0)
+ crtc = sna_dri2_get_crtc(draw);
+@@ -2914,109 +3378,112 @@ sna_dri2_schedule_swap(ClientPtr client, DrawablePtr draw, DRI2BufferPtr front,
+ sna_mode_wakeup(sna);
+ }
+
+- if (can_xchg(sna, draw, front, back) &&
+- sna_dri2_schedule_xchg(client, draw, crtc, front, back,
++ immediate = immediate_swap(sna, draw, crtc,
+ target_msc, divisor, remainder,
+- func, data))
+- return TRUE;
+-
+- if (can_xchg_crtc(sna, draw, front, back, crtc) &&
+- sna_dri2_schedule_xchg_crtc(client, draw, crtc, front, back,
+- target_msc, divisor, remainder,
+- func, data))
+- return TRUE;
++ &current_msc);
+
+ if (can_flip(sna, draw, front, back, crtc) &&
+ sna_dri2_schedule_flip(client, draw, crtc, front, back,
+- target_msc, divisor, remainder,
++ immediate, target_msc, current_msc,
+ func, data))
+ return TRUE;
+
+- VG_CLEAR(vbl);
+-
+- info = sna_dri2_add_event(sna, draw, client);
++ info = sna_dri2_add_event(sna, draw, client, crtc);
+ if (!info)
+ goto blit;
+
+ assert(info->crtc == crtc);
+ info->event_complete = func;
+ info->event_data = data;
++ assert(info->draw);
++ info->signal = true;
+
++ assert(front != back);
+ info->front = sna_dri2_reference_buffer(front);
+ info->back = sna_dri2_reference_buffer(back);
+
+- if (immediate_swap(sna, *target_msc, divisor, draw, crtc, &current_msc)) {
++ if (immediate) {
+ bool sync = current_msc < *target_msc;
+- if (!sna_dri2_immediate_blit(sna, info, sync, true))
+- sna_dri2_event_free(info);
++ sna_dri2_immediate_blit(sna, info, sync);
+ *target_msc = current_msc + sync;
++ DBG(("%s: reported target_msc=%llu\n",
++ __FUNCTION__, *target_msc));
+ return TRUE;
+ }
+
+- vbl.request.type =
+- DRM_VBLANK_ABSOLUTE |
+- DRM_VBLANK_EVENT;
+- vbl.request.signal = (uintptr_t)info;
+-
+- /*
+- * If divisor is zero, or current_msc is smaller than target_msc
+- * we just need to make sure target_msc passes before initiating
+- * the swap.
+- */
+ info->type = SWAP;
+- info->queued = true;
+- if (divisor && current_msc >= *target_msc) {
+- DBG(("%s: missed target, queueing event for next: current=%lld, target=%lld, divisor=%lld, remainder=%lld\n",
+- __FUNCTION__,
+- (long long)current_msc,
+- (long long)*target_msc,
+- (long long)divisor,
+- (long long)remainder));
+-
+- *target_msc = current_msc + remainder - current_msc % divisor;
+- if (*target_msc <= current_msc)
+- *target_msc += divisor;
+- }
+- vbl.request.sequence = draw_target_seq(draw, *target_msc - 1);
+ if (*target_msc <= current_msc + 1) {
+ DBG(("%s: performing blit before queueing\n", __FUNCTION__));
+- assert(info->queued);
+- info->bo = __sna_dri2_copy_region(sna, draw, NULL,
+- back, front,
+- true);
+- info->type = SWAP_WAIT;
+-
+- vbl.request.type =
+- DRM_VBLANK_RELATIVE |
+- DRM_VBLANK_EVENT;
+- vbl.request.sequence = 1;
++ __sna_dri2_copy_event(info, DRI2_SYNC);
++ info->type = SWAP_COMPLETE;
++ if (!sna_next_vblank(info))
++ goto fake;
++
++ DBG(("%s: reported target_msc=%llu\n",
++ __FUNCTION__, *target_msc));
+ *target_msc = current_msc + 1;
+- }
++ swap_limit(draw, 2);
++ } else {
++ if (!sna_wait_vblank(info,
++ draw_target_seq(draw, *target_msc - 1)))
++ goto blit;
+
+- assert(info->queued);
+- if (sna_wait_vblank(sna, &vbl, info->pipe))
+- goto blit;
++ DBG(("%s: reported target_msc=%llu (in)\n",
++ __FUNCTION__, *target_msc));
++ swap_limit(draw, 1);
++ }
+
+- DBG(("%s: reported target_msc=%llu\n", __FUNCTION__, *target_msc));
+- swap_limit(draw, 1 + (info->type == SWAP_WAIT));
+ return TRUE;
+
+ blit:
+ DBG(("%s -- blit\n", __FUNCTION__));
+- if (info)
+- sna_dri2_event_free(info);
+ if (can_xchg(sna, draw, front, back)) {
+ sna_dri2_xchg(draw, front, back);
+ } else {
+- __sna_dri2_copy_region(sna, draw, NULL, back, front, false);
++ __sna_dri2_copy_region(sna, draw, NULL, back, front, 0);
++ front->flags = back->flags;
+ type = DRI2_BLIT_COMPLETE;
+ }
++ if (draw->type == DRAWABLE_PIXMAP)
++ goto fake;
+ skip:
+ DBG(("%s: unable to show frame, unblocking client\n", __FUNCTION__));
+- if (crtc == NULL)
+- crtc = sna_mode_first_crtc(sna);
+- fake_swap_complete(sna, client, draw, crtc, type, func, data);
+- *target_msc = 0; /* offscreen, so zero out target vblank count */
++ if (crtc == NULL && (sna->flags & SNA_NO_WAIT) == 0)
++ crtc = sna_primary_crtc(sna);
++ if (crtc && sna_crtc_is_on(crtc)) {
++ if (info == NULL)
++ info = sna_dri2_add_event(sna, draw, client, crtc);
++ if (info != dri2_chain(draw))
++ goto fake;
++
++ assert(info->crtc == crtc);
++
++ info->type = SWAP_COMPLETE;
++ info->event_complete = func;
++ info->event_data = data;
++ assert(info->draw);
++ info->signal = true;
++
++ if (info->front == NULL)
++ info->front = sna_dri2_reference_buffer(front);
++ if (info->back == NULL)
++ info->back = sna_dri2_reference_buffer(back);
++
++ if (!sna_next_vblank(info))
++ goto fake;
++
++ swap_limit(draw, 1);
++ } else {
++fake:
++ /* XXX Use a Timer to throttle the client? */
++ fake_swap_complete(sna, client, draw, crtc, type, func, data);
++ if (info) {
++ assert(info->draw);
++ info->signal = false;
++ sna_dri2_event_free(info);
++ }
++ }
++ DBG(("%s: reported target_msc=%llu (in)\n", __FUNCTION__, *target_msc));
+ return TRUE;
+ }
+
+@@ -3030,27 +3497,25 @@ sna_dri2_get_msc(DrawablePtr draw, CARD64 *ust, CARD64 *msc)
+ struct sna *sna = to_sna_from_drawable(draw);
+ xf86CrtcPtr crtc = sna_dri2_get_crtc(draw);
+ const struct ust_msc *swap;
++ union drm_wait_vblank vbl;
+
+ DBG(("%s(draw=%ld, pipe=%d)\n", __FUNCTION__, draw->id,
+- crtc ? sna_crtc_to_pipe(crtc) : -1));
++ crtc ? sna_crtc_pipe(crtc) : -1));
+
+- if (crtc != NULL) {
+- union drm_wait_vblank vbl;
++ /* Drawable not displayed, make up a *monotonic* value */
++ if (crtc == NULL)
++ crtc = sna_primary_crtc(sna);
++ if (crtc == NULL)
++ return FALSE;
+
+- VG_CLEAR(vbl);
+- vbl.request.type = _DRM_VBLANK_RELATIVE;
+- vbl.request.sequence = 0;
+- if (sna_wait_vblank(sna, &vbl, sna_crtc_to_pipe(crtc)) == 0)
+- sna_crtc_record_vblank(crtc, &vbl);
+- } else
+- /* Drawable not displayed, make up a *monotonic* value */
+- crtc = sna_mode_first_crtc(sna);
++ if (sna_query_vblank(sna, crtc, &vbl) == 0)
++ sna_crtc_record_vblank(crtc, &vbl);
+
+ swap = sna_crtc_last_swap(crtc);
+ *msc = draw_current_msc(draw, crtc, swap->msc);
+ *ust = ust64(swap->tv_sec, swap->tv_usec);
+- DBG(("%s: msc=%llu, ust=%llu\n", __FUNCTION__,
+- (long long)*msc, (long long)*ust));
++ DBG(("%s: msc=%llu [raw=%llu], ust=%llu\n", __FUNCTION__,
++ (long long)*msc, swap->msc, (long long)*ust));
+ return TRUE;
+ }
+
+@@ -3068,32 +3533,22 @@ sna_dri2_schedule_wait_msc(ClientPtr client, DrawablePtr draw, CARD64 target_msc
+ struct sna_dri2_event *info = NULL;
+ xf86CrtcPtr crtc;
+ CARD64 current_msc;
+- union drm_wait_vblank vbl;
+ const struct ust_msc *swap;
+- int pipe;
+
+ crtc = sna_dri2_get_crtc(draw);
+ DBG(("%s(pipe=%d, target_msc=%llu, divisor=%llu, rem=%llu)\n",
+- __FUNCTION__, crtc ? sna_crtc_to_pipe(crtc) : -1,
++ __FUNCTION__, crtc ? sna_crtc_pipe(crtc) : -1,
+ (long long)target_msc,
+ (long long)divisor,
+ (long long)remainder));
+
+ /* Drawable not visible, return immediately */
+ if (crtc == NULL)
+- goto out_complete;
+-
+- pipe = sna_crtc_to_pipe(crtc);
+-
+- VG_CLEAR(vbl);
+-
+- /* Get current count */
+- vbl.request.type = _DRM_VBLANK_RELATIVE;
+- vbl.request.sequence = 0;
+- if (sna_wait_vblank(sna, &vbl, pipe))
+- goto out_complete;
++ crtc = sna_primary_crtc(sna);
++ if (crtc == NULL)
++ return FALSE;
+
+- current_msc = draw_current_msc(draw, crtc, sna_crtc_record_vblank(crtc, &vbl));
++ current_msc = get_current_msc(sna, draw, crtc);
+
+ /* If target_msc already reached or passed, set it to
+ * current_msc to ensure we return a reasonable value back
+@@ -3104,15 +3559,13 @@ sna_dri2_schedule_wait_msc(ClientPtr client, DrawablePtr draw, CARD64 target_msc
+ if (divisor == 0 && current_msc >= target_msc)
+ goto out_complete;
+
+- info = sna_dri2_add_event(sna, draw, client);
++ info = sna_dri2_add_event(sna, draw, client, crtc);
+ if (!info)
+ goto out_complete;
+
+ assert(info->crtc == crtc);
+ info->type = WAITMSC;
+
+- vbl.request.signal = (uintptr_t)info;
+- vbl.request.type = DRM_VBLANK_ABSOLUTE | DRM_VBLANK_EVENT;
+ /*
+ * If divisor is zero, or current_msc is smaller than target_msc,
+ * we just need to make sure target_msc passes before waking up the
+@@ -3129,10 +3582,8 @@ sna_dri2_schedule_wait_msc(ClientPtr client, DrawablePtr draw, CARD64 target_msc
+ if (target_msc <= current_msc)
+ target_msc += divisor;
+ }
+- vbl.request.sequence = draw_target_seq(draw, target_msc);
+
+- info->queued = true;
+- if (sna_wait_vblank(sna, &vbl, pipe))
++ if (!sna_wait_vblank(info, draw_target_seq(draw, target_msc)))
+ goto out_free_info;
+
+ DRI2BlockClient(client, draw);
+@@ -3141,8 +3592,6 @@ sna_dri2_schedule_wait_msc(ClientPtr client, DrawablePtr draw, CARD64 target_msc
+ out_free_info:
+ sna_dri2_event_free(info);
+ out_complete:
+- if (crtc == NULL)
+- crtc = sna_mode_first_crtc(sna);
+ swap = sna_crtc_last_swap(crtc);
+ DRI2WaitMSCComplete(client, draw,
+ draw_current_msc(draw, crtc, swap->msc),
+@@ -3231,9 +3680,18 @@ static bool is_level(const char **str)
+ return false;
+ }
+
++static const char *options_get_dri(struct sna *sna)
++{
++#if XORG_VERSION_CURRENT >= XORG_VERSION_NUMERIC(1,7,99,901,0)
++ return xf86GetOptValString(sna->Options, OPTION_DRI);
++#else
++ return NULL;
++#endif
++}
++
+ static const char *dri_driver_name(struct sna *sna)
+ {
+- const char *s = xf86GetOptValString(sna->Options, OPTION_DRI);
++ const char *s = options_get_dri(sna);
+
+ if (is_level(&s)) {
+ if (sna->kgem.gen < 030)
+@@ -3259,7 +3717,7 @@ bool sna_dri2_open(struct sna *sna, ScreenPtr screen)
+
+ if (wedged(sna)) {
+ xf86DrvMsg(sna->scrn->scrnIndex, X_WARNING,
+- "loading DRI2 whilst the GPU is wedged.\n");
++ "loading DRI2 whilst acceleration is disabled.\n");
+ }
+
+ if (xf86LoaderCheckSymbol("DRI2Version"))
+@@ -3274,7 +3732,7 @@ bool sna_dri2_open(struct sna *sna, ScreenPtr screen)
+ memset(&info, '\0', sizeof(info));
+ info.fd = sna->kgem.fd;
+ info.driverName = dri_driver_name(sna);
+- info.deviceName = intel_get_client_name(sna->dev);
++ info.deviceName = intel_get_master_name(sna->dev);
+
+ DBG(("%s: loading dri driver '%s' [gen=%d] for device '%s'\n",
+ __FUNCTION__, info.driverName, sna->kgem.gen, info.deviceName));
+@@ -3299,11 +3757,12 @@ bool sna_dri2_open(struct sna *sna, ScreenPtr screen)
+ info.numDrivers = 2;
+ info.driverNames = driverNames;
+ driverNames[0] = info.driverName;
+- driverNames[1] = info.driverName;
++ driverNames[1] = "va_gl";
+ #endif
+
+ #if DRI2INFOREC_VERSION >= 6
+ if (xorg_can_triple_buffer()) {
++ DBG(("%s: enabling Xorg triple buffering\n", __FUNCTION__));
+ info.version = 6;
+ info.SwapLimitValidate = sna_dri2_swap_limit_validate;
+ info.ReuseBufferNotify = sna_dri2_reuse_buffer;
+@@ -3311,8 +3770,10 @@ bool sna_dri2_open(struct sna *sna, ScreenPtr screen)
+ #endif
+
+ #if USE_ASYNC_SWAP
++ DBG(("%s: enabled async swap and buffer age\n", __FUNCTION__));
+ info.version = 10;
+ info.scheduleSwap0 = 1;
++ info.bufferAge = 1;
+ #endif
+
+ return DRI2ScreenInit(screen, &info);
+diff --git a/src/sna/sna_dri3.c b/src/sna/sna_dri3.c
+index f586e242..ce4970ae 100644
+--- a/src/sna/sna_dri3.c
++++ b/src/sna/sna_dri3.c
+@@ -55,11 +55,14 @@ static inline void mark_dri3_pixmap(struct sna *sna, struct sna_pixmap *priv, st
+ if (bo->exec)
+ sna->kgem.flush = 1;
+ if (bo == priv->gpu_bo)
+- priv->flush |= 3;
++ priv->flush |= FLUSH_READ | FLUSH_WRITE;
+ else
+ priv->shm = true;
+
+- sna_accel_watch_flush(sna, 1);
++ sna_watch_flush(sna, 1);
++
++ kgem_bo_submit(&sna->kgem, bo);
++ kgem_bo_unclean(&sna->kgem, bo);
+ }
+
+ static void sna_sync_flush(struct sna *sna, struct sna_pixmap *priv)
+@@ -270,6 +273,8 @@ static PixmapPtr sna_dri3_pixmap_from_fd(ScreenPtr screen,
+ priv->ptr = MAKE_STATIC_PTR(pixmap->devPrivate.ptr);
+ } else {
+ assert(priv->gpu_bo == bo);
++ priv->create = kgem_can_create_2d(&sna->kgem,
++ width, height, depth);
+ priv->pinned |= PIN_DRI3;
+ }
+ list_add(&priv->cow_list, &sna->dri3.pixmaps);
+@@ -325,6 +330,15 @@ static int sna_dri3_fd_from_pixmap(ScreenPtr screen,
+ return -1;
+ }
+
++ if (bo->tiling && !sna->kgem.can_fence) {
++ if (!sna_pixmap_change_tiling(pixmap, I915_TILING_NONE)) {
++ DBG(("%s: unable to discard GPU tiling (%d) for DRI3 protocol\n",
++ __FUNCTION__, bo->tiling));
++ return -1;
++ }
++ bo = priv->gpu_bo;
++ }
++
+ fd = kgem_bo_export_to_prime(&sna->kgem, bo);
+ if (fd == -1) {
+ DBG(("%s: exporting handle=%d to fd failed\n", __FUNCTION__, bo->handle));
+diff --git a/src/sna/sna_driver.c b/src/sna/sna_driver.c
+index 8a3599c7..1b4015de 100644
+--- a/src/sna/sna_driver.c
++++ b/src/sna/sna_driver.c
+@@ -57,6 +57,13 @@ USE OR OTHER DEALINGS IN THE SOFTWARE.
+ #include <mi.h>
+ #include <micmap.h>
+
++#if defined(HAVE_X11_EXTENSIONS_DPMSCONST_H)
++#include <X11/extensions/dpmsconst.h>
++#else
++#define DPMSModeOn 0
++#define DPMSModeOff 3
++#endif
++
+ #include <sys/ioctl.h>
+ #include <sys/fcntl.h>
+ #include <sys/poll.h>
+@@ -69,6 +76,8 @@ USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+ #if HAVE_DOT_GIT
+ #include "git_version.h"
++#else
++#define git_version "not compiled from git"
+ #endif
+
+ #ifdef TEARFREE
+@@ -185,12 +194,12 @@ sna_set_fallback_mode(ScrnInfoPtr scrn)
+
+ xf86DisableUnusedFunctions(scrn);
+ #ifdef RANDR_12_INTERFACE
+- if (get_root_window(scrn->pScreen))
+- xf86RandR12TellChanged(scrn->pScreen);
++ if (get_root_window(xf86ScrnToScreen(scrn)))
++ xf86RandR12TellChanged(xf86ScrnToScreen(scrn));
+ #endif
+ }
+
+-static Bool sna_set_desired_mode(struct sna *sna)
++static void sna_set_desired_mode(struct sna *sna)
+ {
+ ScrnInfoPtr scrn = sna->scrn;
+
+@@ -203,7 +212,6 @@ static Bool sna_set_desired_mode(struct sna *sna)
+ }
+
+ sna_mode_check(sna);
+- return TRUE;
+ }
+
+ /**
+@@ -222,7 +230,7 @@ static Bool sna_create_screen_resources(ScreenPtr screen)
+ screen->width, screen->height, screen->rootDepth));
+
+ assert(sna->scrn == xf86ScreenToScrn(screen));
+- assert(sna->scrn->pScreen == screen);
++ assert(to_screen_from_sna(sna) == screen);
+
+ /* free the data used during miInitScreen */
+ free(screen->devPrivate);
+@@ -273,33 +281,89 @@ static Bool sna_create_screen_resources(ScreenPtr screen)
+ if (serverGeneration == 1 && (sna->flags & SNA_IS_HOSTED) == 0)
+ sna_copy_fbcon(sna);
+
+- (void)sna_set_desired_mode(sna);
++ sna_set_desired_mode(sna);
+ }
+
+ return TRUE;
+ }
+
+-static Bool sna_save_screen(ScreenPtr screen, int mode)
++static void sna_dpms_set(ScrnInfoPtr scrn, int mode, int flags)
+ {
+- ScrnInfoPtr scrn = xf86ScreenToScrn(screen);
++ xf86CrtcConfigPtr config = XF86_CRTC_CONFIG_PTR(scrn);
++ struct sna *sna = to_sna(scrn);
++ bool changed = false;
++ int i;
+
+- DBG(("%s(mode=%d)\n", __FUNCTION__, mode));
++ DBG(("%s(mode=%d, flags=%d), vtSema=%d => off?=%d\n",
++ __FUNCTION__, mode, flags, scrn->vtSema, mode!=DPMSModeOn));
+ if (!scrn->vtSema)
+- return FALSE;
++ return;
+
+- xf86SaveScreen(screen, mode);
+- sna_crtc_config_notify(screen);
+- return TRUE;
++ /* Opencoded version of xf86DPMSSet().
++ *
++ * The principle difference is to skip calling crtc->dpms() when
++ * turning off the display. This (on recent enough kernels at
++ * least) should be equivalent in power consumption, but require
++ * less work (hence quicker and less likely to fail) when switching
++ * back on.
++ */
++ if (mode != DPMSModeOn) {
++ if (sna->mode.hidden == 0 && !(sna->flags & SNA_NO_DPMS)) {
++ DBG(("%s: hiding %d outputs\n",
++ __FUNCTION__, config->num_output));
++ for (i = 0; i < config->num_output; i++) {
++ xf86OutputPtr output = config->output[i];
++ if (output->crtc != NULL)
++ output->funcs->dpms(output, mode);
++ }
++ sna->mode.hidden = sna->mode.front_active + 1;
++ sna->mode.front_active = 0;
++ changed = true;
++ }
++ } else {
++ /* Re-enable CRTC that have been forced off via other means */
++ if (sna->mode.hidden != 0) {
++ DBG(("%s: unhiding %d crtc, %d outputs\n",
++ __FUNCTION__, config->num_crtc, config->num_output));
++ sna->mode.front_active = sna->mode.hidden - 1;
++ sna->mode.hidden = 0;
++ for (i = 0; i < config->num_crtc; i++) {
++ xf86CrtcPtr crtc = config->crtc[i];
++ if (crtc->enabled)
++ crtc->funcs->dpms(crtc, mode);
++ }
++
++ for (i = 0; i < config->num_output; i++) {
++ xf86OutputPtr output = config->output[i];
++ if (output->crtc != NULL)
++ output->funcs->dpms(output, mode);
++ }
++ changed = true;
++ }
++ }
++
++ DBG(("%s: hiding outputs? %d, front active? %d, changed? %d\n",
++ __FUNCTION__, sna->mode.hidden, sna->mode.front_active, changed));
++
++ if (changed)
++ sna_crtc_config_notify(xf86ScrnToScreen(scrn));
+ }
+
+-static void sna_dpms_set(ScrnInfoPtr scrn, int mode, int flags)
++static Bool sna_save_screen(ScreenPtr screen, int mode)
+ {
+- DBG(("%s(mode=%d, flags=%d)\n", __FUNCTION__, mode));
+- if (!scrn->vtSema)
+- return;
++ ScrnInfoPtr scrn = xf86ScreenToScrn(screen);
++
++ DBG(("%s(mode=%d [unblank=%d])\n",
++ __FUNCTION__, mode, xf86IsUnblank(mode)));
+
+- xf86DPMSSet(scrn, mode, flags);
+- sna_crtc_config_notify(xf86ScrnToScreen(scrn));
++ /* We have to unroll xf86SaveScreen() here as it is called
++ * by DPMSSet() nullifying our special handling crtc->dpms()
++ * in sna_dpms_set().
++ */
++ sna_dpms_set(scrn,
++ xf86IsUnblank(mode) ? DPMSModeOn : DPMSModeOff,
++ 0);
++ return TRUE;
+ }
+
+ static void sna_selftest(void)
+@@ -330,107 +394,6 @@ static void sna_setup_capabilities(ScrnInfoPtr scrn, int fd)
+ #endif
+ }
+
+-static int
+-namecmp(const char *s1, const char *s2)
+-{
+- char c1, c2;
+-
+- if (!s1 || *s1 == 0) {
+- if (!s2 || *s2 == 0)
+- return 0;
+- else
+- return 1;
+- }
+-
+- while (*s1 == '_' || *s1 == ' ' || *s1 == '\t')
+- s1++;
+-
+- while (*s2 == '_' || *s2 == ' ' || *s2 == '\t')
+- s2++;
+-
+- c1 = isupper(*s1) ? tolower(*s1) : *s1;
+- c2 = isupper(*s2) ? tolower(*s2) : *s2;
+- while (c1 == c2) {
+- if (c1 == '\0')
+- return 0;
+-
+- s1++;
+- while (*s1 == '_' || *s1 == ' ' || *s1 == '\t')
+- s1++;
+-
+- s2++;
+- while (*s2 == '_' || *s2 == ' ' || *s2 == '\t')
+- s2++;
+-
+- c1 = isupper(*s1) ? tolower(*s1) : *s1;
+- c2 = isupper(*s2) ? tolower(*s2) : *s2;
+- }
+-
+- return c1 - c2;
+-}
+-
+-static Bool sna_option_cast_to_bool(struct sna *sna, int id, Bool val)
+-{
+- const char *str = xf86GetOptValString(sna->Options, id);
+-
+- if (str == NULL)
+- return val;
+-
+- if (*str == '\0')
+- return TRUE;
+-
+- if (namecmp(str, "1") == 0)
+- return TRUE;
+- if (namecmp(str, "on") == 0)
+- return TRUE;
+- if (namecmp(str, "true") == 0)
+- return TRUE;
+- if (namecmp(str, "yes") == 0)
+- return TRUE;
+-
+- if (namecmp(str, "0") == 0)
+- return FALSE;
+- if (namecmp(str, "off") == 0)
+- return FALSE;
+- if (namecmp(str, "false") == 0)
+- return FALSE;
+- if (namecmp(str, "no") == 0)
+- return FALSE;
+-
+- return val;
+-}
+-
+-static unsigned sna_option_cast_to_unsigned(struct sna *sna, int id, unsigned val)
+-{
+- const char *str = xf86GetOptValString(sna->Options, id);
+- unsigned v;
+-
+- if (str == NULL || *str == '\0')
+- return val;
+-
+- if (namecmp(str, "on") == 0)
+- return val;
+- if (namecmp(str, "true") == 0)
+- return val;
+- if (namecmp(str, "yes") == 0)
+- return val;
+-
+- if (namecmp(str, "0") == 0)
+- return 0;
+- if (namecmp(str, "off") == 0)
+- return 0;
+- if (namecmp(str, "false") == 0)
+- return 0;
+- if (namecmp(str, "no") == 0)
+- return 0;
+-
+- v = atoi(str);
+- if (v)
+- return v;
+-
+- return val;
+-}
+-
+ static Bool fb_supports_depth(int fd, int depth)
+ {
+ struct drm_i915_gem_create create;
+@@ -470,16 +433,24 @@ static void setup_dri(struct sna *sna)
+ unsigned level;
+
+ sna->dri2.available = false;
++ sna->dri2.enable = false;
+ sna->dri3.available = false;
++ sna->dri3.enable = false;
++ sna->dri3.override = false;
+
+- level = sna_option_cast_to_unsigned(sna, OPTION_DRI, ~0);
++ level = intel_option_cast_to_unsigned(sna->Options, OPTION_DRI, DEFAULT_DRI_LEVEL);
+ #if HAVE_DRI3
+- if (level >= 3)
+- sna->dri3.available = !!xf86LoadSubModule(sna->scrn, "dri3");
++ sna->dri3.available = !!xf86LoadSubModule(sna->scrn, "dri3");
++ sna->dri3.override =
++ !sna->dri3.available ||
++ xf86IsOptionSet(sna->Options, OPTION_DRI);
++ if (level >= 3 && sna->kgem.gen >= 040)
++ sna->dri3.enable = sna->dri3.available;
+ #endif
+ #if HAVE_DRI2
++ sna->dri2.available = !!xf86LoadSubModule(sna->scrn, "dri2");
+ if (level >= 2)
+- sna->dri2.available = !!xf86LoadSubModule(sna->scrn, "dri2");
++ sna->dri2.enable = sna->dri2.available;
+ #endif
+ }
+
+@@ -498,13 +469,13 @@ static bool enable_tear_free(struct sna *sna)
+ return ENABLE_TEAR_FREE;
+ }
+
+-static void setup_tear_free(struct sna *sna)
++static bool setup_tear_free(struct sna *sna)
+ {
+ MessageType from;
+ Bool enable;
+
+ if (sna->flags & SNA_LINEAR_FB)
+- return;
++ return false;
+
+ if ((sna->flags & SNA_HAS_FLIP) == 0) {
+ from = X_PROBED;
+@@ -518,11 +489,12 @@ static void setup_tear_free(struct sna *sna)
+ from = X_CONFIG;
+
+ if (enable)
+- sna->flags |= SNA_TEAR_FREE;
++ sna->flags |= SNA_WANT_TEAR_FREE | SNA_TEAR_FREE;
+
+ done:
+ xf86DrvMsg(sna->scrn->scrnIndex, from, "TearFree %sabled\n",
+ sna->flags & SNA_TEAR_FREE ? "en" : "dis");
++ return sna->flags & SNA_TEAR_FREE;
+ }
+
+ /**
+@@ -612,8 +584,10 @@ static Bool sna_pre_init(ScrnInfoPtr scrn, int probe)
+ }
+
+ intel_detect_chipset(scrn, sna->dev);
+- xf86DrvMsg(scrn->scrnIndex, X_PROBED, "CPU: %s\n",
+- sna_cpu_features_to_string(sna->cpu_features, buf));
++ xf86DrvMsg(scrn->scrnIndex, X_PROBED,
++ "CPU: %s; using a maximum of %d threads\n",
++ sna_cpu_features_to_string(sna->cpu_features, buf),
++ sna_use_threads(64*1024, 64*1024, 1));
+
+ if (!xf86SetDepthBpp(scrn, 24, 0, 0,
+ Support32bppFb |
+@@ -651,18 +625,11 @@ static Bool sna_pre_init(ScrnInfoPtr scrn, int probe)
+ kgem_init(&sna->kgem, fd,
+ xf86GetPciInfoForEntity(pEnt->index),
+ sna->info->gen);
+- if (xf86ReturnOptValBool(sna->Options, OPTION_ACCEL_DISABLE, FALSE) ||
+- !sna_option_cast_to_bool(sna, OPTION_ACCEL_METHOD, TRUE)) {
+- xf86DrvMsg(sna->scrn->scrnIndex, X_CONFIG,
+- "Disabling hardware acceleration.\n");
+- sna->kgem.wedged = true;
+- }
+
+ if (xf86ReturnOptValBool(sna->Options, OPTION_TILING_FB, FALSE))
+ sna->flags |= SNA_LINEAR_FB;
+-
+- if (xf86ReturnOptValBool(sna->Options, OPTION_DELETE_DP12, FALSE))
+- sna->flags |= SNA_REMOVE_OUTPUTS;
++ if (!sna->kgem.can_fence)
++ sna->flags |= SNA_LINEAR_FB;
+
+ if (!xf86ReturnOptValBool(sna->Options, OPTION_SWAPBUFFERS_WAIT, TRUE))
+ sna->flags |= SNA_NO_WAIT;
+@@ -695,7 +662,8 @@ static Bool sna_pre_init(ScrnInfoPtr scrn, int probe)
+ }
+ scrn->currentMode = scrn->modes;
+
+- setup_tear_free(sna);
++ if (!setup_tear_free(sna) && sna_mode_wants_tear_free(sna))
++ sna->kgem.needs_dirtyfb = sna->kgem.has_dirtyfb;
+
+ xf86SetGamma(scrn, zeros);
+ xf86SetDpi(scrn, 0, 0);
+@@ -721,11 +689,13 @@ cleanup:
+ return FALSE;
+ }
+
++#if !HAVE_NOTIFY_FD
+ static bool has_shadow(struct sna *sna)
+ {
+- if (!sna->mode.shadow_damage)
++ if (!sna->mode.shadow_enabled)
+ return false;
+
++ assert(sna->mode.shadow_damage);
+ if (RegionNil(DamageRegion(sna->mode.shadow_damage)))
+ return false;
+
+@@ -748,7 +718,7 @@ sna_block_handler(BLOCKHANDLER_ARGS_DECL)
+ sna->BlockHandler(BLOCKHANDLER_ARGS);
+
+ if (*tv == NULL || ((*tv)->tv_usec | (*tv)->tv_sec) || has_shadow(sna))
+- sna_accel_block_handler(sna, tv);
++ sna_accel_block(sna, tv);
+ }
+
+ static void
+@@ -770,52 +740,102 @@ sna_wakeup_handler(WAKEUPHANDLER_ARGS_DECL)
+
+ sna->WakeupHandler(WAKEUPHANDLER_ARGS);
+
+- sna_accel_wakeup_handler(sna);
+-
+ if (FD_ISSET(sna->kgem.fd, (fd_set*)read_mask)) {
+ sna_mode_wakeup(sna);
+ /* Clear the flag so that subsequent ZaphodHeads don't block */
+ FD_CLR(sna->kgem.fd, (fd_set*)read_mask);
+ }
+ }
++#else
++static void
++sna_block_handler(void *data, void *_timeout)
++{
++ struct sna *sna = data;
++ int *timeout = _timeout;
++ struct timeval tv, *tvp;
++
++ DBG(("%s (timeout=%d)\n", __FUNCTION__, *timeout));
++ if (*timeout == 0)
++ return;
++
++ if (*timeout < 0) {
++ tvp = NULL;
++ } else {
++ tv.tv_sec = *timeout / 1000;
++ tv.tv_usec = (*timeout % 1000) * 1000;
++ tvp = &tv;
++ }
++
++ sna_accel_block(sna, &tvp);
++ if (tvp)
++ *timeout = tvp->tv_sec * 1000 + tvp->tv_usec / 1000;
++}
++#endif
+
+ #if HAVE_UDEV
++#include <sys/stat.h>
++
+ static void
+ sna_handle_uevents(int fd, void *closure)
+ {
+ struct sna *sna = closure;
+- struct udev_device *dev;
+- const char *str;
+ struct stat s;
+- dev_t udev_devnum;
++ struct pollfd pfd;
++ bool hotplug = false;
+
+ DBG(("%s\n", __FUNCTION__));
+
+- dev = udev_monitor_receive_device(sna->uevent_monitor);
+- if (!dev)
+- return;
++ pfd.fd = udev_monitor_get_fd(sna->uevent_monitor);
++ pfd.events = POLLIN;
++
++ if (fstat(sna->kgem.fd, &s))
++ memset(&s, 0, sizeof(s));
++
++ while (poll(&pfd, 1, 0) > 0) {
++ struct udev_device *dev;
++ dev_t devnum;
++
++ dev = udev_monitor_receive_device(sna->uevent_monitor);
++ if (dev == NULL)
++ break;
++
++ devnum = udev_device_get_devnum(dev);
++ if (memcmp(&s.st_rdev, &devnum, sizeof(dev_t)) == 0) {
++ const char *str;
++
++ str = udev_device_get_property_value(dev, "HOTPLUG");
++ if (str && atoi(str) == 1) {
++ str = udev_device_get_property_value(dev, "CONNECTOR");
++ if (str) {
++ hotplug |= sna_mode_find_hotplug_connector(sna, atoi(str));
++ } else {
++ sna->flags |= SNA_REPROBE;
++ hotplug = true;
++ }
++ }
++ }
+
+- udev_devnum = udev_device_get_devnum(dev);
+- if (fstat(sna->kgem.fd, &s) || memcmp(&s.st_rdev, &udev_devnum, sizeof (dev_t))) {
+ udev_device_unref(dev);
+- return;
+ }
+
+- str = udev_device_get_property_value(dev, "HOTPLUG");
+- if (str && atoi(str) == 1) {
+- ScrnInfoPtr scrn = sna->scrn;
+-
+- DBG(("%s: hotplug event (vtSema?=%d)\n", __FUNCTION__, scrn->vtSema));
++ if (hotplug) {
++ DBG(("%s: hotplug event (vtSema?=%d)\n",
++ __FUNCTION__, sna->scrn->vtSema));
+
+- if (scrn->vtSema) {
+- sna_mode_discover(sna);
+- sna_mode_check(sna);
+- RRGetInfo(xf86ScrnToScreen(scrn), TRUE);
+- } else
++ if (sna->scrn->vtSema)
++ sna_mode_discover(sna, true);
++ else
+ sna->flags |= SNA_REPROBE;
+ }
++}
+
+- udev_device_unref(dev);
++static bool has_randr(void)
++{
++#if HAS_DIXREGISTERPRIVATEKEY
++ return dixPrivateKeyRegistered(rrPrivKey);
++#else
++ return *rrPrivKey;
++#endif
+ }
+
+ static void
+@@ -833,7 +853,7 @@ sna_uevent_init(struct sna *sna)
+ /* RandR will be disabled if Xinerama is active, and so generating
+ * RR hotplug events is then verboten.
+ */
+- if (!dixPrivateKeyRegistered(rrPrivKey))
++ if (!has_randr())
+ goto out;
+
+ u = NULL;
+@@ -861,7 +881,8 @@ sna_uevent_init(struct sna *sna)
+
+ sna->uevent_monitor = mon;
+ out:
+- xf86DrvMsg(sna->scrn->scrnIndex, from, "display hotplug detection %s\n",
++ xf86DrvMsg(sna->scrn->scrnIndex, from,
++ "Display hotplug detection %s\n",
+ sna->uevent_monitor ? "enabled" : "disabled");
+ return;
+
+@@ -874,17 +895,10 @@ err_dev:
+
+ static bool sna_uevent_poll(struct sna *sna)
+ {
+- struct pollfd pfd;
+-
+ if (sna->uevent_monitor == NULL)
+ return false;
+
+- pfd.fd = udev_monitor_get_fd(sna->uevent_monitor);
+- pfd.events = POLLIN;
+-
+- while (poll(&pfd, 1, 0) > 0)
+- sna_handle_uevents(pfd.fd, sna);
+-
++ sna_handle_uevents(udev_monitor_get_fd(sna->uevent_monitor), sna);
+ return true;
+ }
+
+@@ -918,8 +932,10 @@ sna_randr_getinfo(ScreenPtr screen, Rotation *rotations)
+ {
+ struct sna *sna = to_sna_from_screen(screen);
+
++ DBG(("%s()\n", __FUNCTION__));
++
+ if (!sna_uevent_poll(sna))
+- sna_mode_discover(sna);
++ sna_mode_discover(sna, false);
+
+ return sna->mode.rrGetInfo(screen, rotations);
+ }
+@@ -931,8 +947,8 @@ static void sna_leave_vt(VT_FUNC_ARGS_DECL)
+
+ DBG(("%s\n", __FUNCTION__));
+
+- sna_accel_leave(sna);
+ sna_mode_reset(sna);
++ sna_accel_leave(sna);
+
+ if (intel_put_master(sna->dev))
+ xf86DrvMsg(scrn->scrnIndex, X_WARNING,
+@@ -948,6 +964,12 @@ static Bool sna_early_close_screen(CLOSE_SCREEN_ARGS_DECL)
+
+ /* XXX Note that we will leak kernel resources if !vtSema */
+
++#if HAVE_NOTIFY_FD
++ RemoveBlockAndWakeupHandlers(sna_block_handler,
++ (ServerWakeupHandlerProcPtr)NoopDDA,
++ sna);
++#endif
++
+ sna_uevent_fini(sna);
+ sna_mode_close(sna);
+
+@@ -1047,12 +1069,13 @@ static void sna_dri_init(struct sna *sna, ScreenPtr screen)
+ {
+ char str[128] = "";
+
+- if (sna->dri2.available)
++ if (sna->dri2.enable)
+ sna->dri2.open = sna_dri2_open(sna, screen);
+ if (sna->dri2.open)
+ strcat(str, "DRI2 ");
+
+- if (sna->dri3.available)
++ /* Load DRI3 in case DRI2 doesn't work, e.g. vgaarb */
++ if (sna->dri3.enable || (!sna->dri2.open && !sna->dri3.override))
+ sna->dri3.open = sna_dri3_open(sna, screen);
+ if (sna->dri3.open)
+ strcat(str, "DRI3 ");
+@@ -1098,7 +1121,8 @@ sna_screen_init(SCREEN_INIT_ARGS_DECL)
+ DBG(("%s\n", __FUNCTION__));
+
+ assert(sna->scrn == scrn);
+- assert(scrn->pScreen == NULL); /* set afterwards */
++ assert(to_screen_from_sna(sna) == NULL || /* set afterwards */
++ to_screen_from_sna(sna) == screen);
+
+ assert(sna->freed_pixmap == NULL);
+
+@@ -1166,11 +1190,17 @@ sna_screen_init(SCREEN_INIT_ARGS_DECL)
+ * later memory should be bound when allocating, e.g rotate_mem */
+ scrn->vtSema = TRUE;
+
++#if !HAVE_NOTIFY_FD
+ sna->BlockHandler = screen->BlockHandler;
+ screen->BlockHandler = sna_block_handler;
+
+ sna->WakeupHandler = screen->WakeupHandler;
+ screen->WakeupHandler = sna_wakeup_handler;
++#else
++ RegisterBlockAndWakeupHandlers(sna_block_handler,
++ (ServerWakeupHandlerProcPtr)NoopDDA,
++ sna);
++#endif
+
+ screen->SaveScreen = sna_save_screen;
+ screen->CreateScreenResources = sna_create_screen_resources;
+@@ -1190,6 +1220,8 @@ sna_screen_init(SCREEN_INIT_ARGS_DECL)
+ CMAP_PALETTED_TRUECOLOR))
+ return FALSE;
+
++ if (!xf86CheckBoolOption(scrn->options, "dpms", TRUE))
++ sna->flags |= SNA_NO_DPMS;
+ xf86DPMSInit(screen, sna_dpms_set, 0);
+
+ sna_uevent_init(sna);
+@@ -1244,20 +1276,15 @@ static Bool sna_enter_vt(VT_FUNC_ARGS_DECL)
+ if (intel_get_master(sna->dev))
+ return FALSE;
+
++ sna_accel_enter(sna);
++
+ if (sna->flags & SNA_REPROBE) {
+- DBG(("%s: reporting deferred hotplug event\n",
+- __FUNCTION__));
+- sna_mode_discover(sna);
+- RRGetInfo(xf86ScrnToScreen(scrn), TRUE);
+- sna->flags &= ~SNA_REPROBE;
++ DBG(("%s: reporting deferred hotplug event\n", __FUNCTION__));
++ sna_mode_discover(sna, true);
+ }
+
+- if (!sna_set_desired_mode(sna)) {
+- intel_put_master(sna->dev);
+- return FALSE;
+- }
++ sna_set_desired_mode(sna);
+
+- sna_accel_enter(sna);
+ return TRUE;
+ }
+
+@@ -1379,6 +1406,9 @@ static void describe_sna(ScrnInfoPtr scrn)
+ xf86DrvMsg(scrn->scrnIndex, X_INFO,
+ "SNA compiled: %s\n", BUILDER_DESCRIPTION);
+ #endif
++#if HAS_DEBUG_FULL
++ ErrorF("SNA compiled with full debug logging; expect to run slowly\n");
++#endif
+ #if !NDEBUG
+ xf86DrvMsg(scrn->scrnIndex, X_INFO,
+ "SNA compiled with assertions enabled\n");
+@@ -1400,6 +1430,7 @@ static void describe_sna(ScrnInfoPtr scrn)
+ "SNA compiled for use with valgrind\n");
+ VALGRIND_PRINTF("SNA compiled for use with valgrind\n");
+ #endif
++ DBG(("xf86-video-intel version: %s\n", git_version));
+ DBG(("pixman version: %s\n", pixman_version_string()));
+ }
+
+diff --git a/src/sna/sna_glyphs.c b/src/sna/sna_glyphs.c
+index a5dfb06b..6ee40336 100644
+--- a/src/sna/sna_glyphs.c
++++ b/src/sna/sna_glyphs.c
+@@ -74,7 +74,7 @@
+ #define NO_GLYPHS_VIA_MASK 0
+ #define FORCE_SMALL_MASK 0 /* -1 = never, 1 = always */
+ #define NO_GLYPHS_SLOW 0
+-#define NO_DISCARD_MASK 0
++#define DISCARD_MASK 0 /* -1 = never, 1 = always */
+
+ #define CACHE_PICTURE_SIZE 1024
+ #define GLYPH_MIN_SIZE 8
+@@ -185,7 +185,7 @@ void sna_glyphs_close(struct sna *sna)
+ */
+ bool sna_glyphs_create(struct sna *sna)
+ {
+- ScreenPtr screen = sna->scrn->pScreen;
++ ScreenPtr screen = to_screen_from_sna(sna);
+ pixman_color_t white = { 0xffff, 0xffff, 0xffff, 0xffff };
+ unsigned int formats[] = {
+ PIXMAN_a8,
+@@ -1094,6 +1094,9 @@ sna_glyph_get_image(GlyphPtr g, ScreenPtr s)
+
+ static inline bool use_small_mask(struct sna *sna, int16_t width, int16_t height, int depth)
+ {
++ if (depth < 8)
++ return true;
++
+ if (FORCE_SMALL_MASK)
+ return FORCE_SMALL_MASK > 0;
+
+@@ -1156,12 +1159,6 @@ glyphs_via_mask(struct sna *sna,
+ src_x += box.x1 - list->xOff;
+ src_y += box.y1 - list->yOff;
+
+- if (format->depth < 8) {
+- format = PictureMatchFormat(screen, 8, PICT_a8);
+- if (!format)
+- return false;
+- }
+-
+ component_alpha = NeedsComponent(format->format);
+ if (use_small_mask(sna, width, height, format->depth)) {
+ pixman_image_t *mask_image;
+@@ -1179,7 +1176,7 @@ use_small_mask:
+ return false;
+
+ mask_image =
+- pixman_image_create_bits(format->depth << 24 | format->format,
++ pixman_image_create_bits(pixmap->drawable.bitsPerPixel << 24 | format->format,
+ width, height,
+ pixmap->devPrivate.ptr,
+ pixmap->devKind);
+@@ -1386,10 +1383,11 @@ next_image:
+ DBG(("%s: atlas format=%08x, mask format=%08x\n",
+ __FUNCTION__,
+ (int)p->atlas->format,
+- (int)(format->depth << 24 | format->format)));
++ (int)mask->format));
+
+ memset(&tmp, 0, sizeof(tmp));
+- if (p->atlas->format == (format->depth << 24 | format->format)) {
++ if (p->atlas->format == mask->format ||
++ alphaless(p->atlas->format) == mask->format) {
+ ok = sna->render.composite(sna, PictOpAdd,
+ p->atlas, NULL, mask,
+ 0, 0, 0, 0, 0, 0,
+@@ -1561,6 +1559,9 @@ skip_glyph:
+ }
+ }
+
++ assert(format);
++ DBG(("%s: format=%08d, depth=%d\n",
++ __FUNCTION__, format->format, format->depth));
+ out:
+ if (list_extents != stack_extents)
+ free(list_extents);
+@@ -1573,24 +1574,34 @@ static bool can_discard_mask(uint8_t op, PicturePtr src, PictFormatPtr mask,
+ PictFormatPtr g;
+ uint32_t color;
+
+- if (NO_DISCARD_MASK)
+- return false;
++ if (DISCARD_MASK)
++ return DISCARD_MASK > 0;
+
+ DBG(("%s: nlist=%d, mask=%08x, depth %d, op=%d (bounded? %d)\n",
+ __FUNCTION__, nlist,
+ mask ? (unsigned)mask->format : 0, mask ? mask->depth : 0,
+ op, op_is_bounded(op)));
+
+- if (nlist == 1 && list->len == 1)
+- return true;
++ if (nlist == 1 && list->len == 1) {
++ if (mask == list->format)
++ return true;
++
++ g = list->format;
++ goto skip;
++ }
+
+- if (!op_is_bounded(op))
++ if (!op_is_bounded(op)) {
++ DBG(("%s: unbounded op, not discarding\n", __FUNCTION__));
+ return false;
++ }
+
+ /* No glyphs overlap and we are not performing a mask conversion. */
+ g = glyphs_format(nlist, list, glyphs);
+- if (mask == g)
++ if (mask == g) {
++ DBG(("%s: mask matches glyphs format, no conversion, so discard mask\n",
++ __FUNCTION__));
+ return true;
++ }
+
+ DBG(("%s: preferred mask format %08x, depth %d\n",
+ __FUNCTION__, g ? (unsigned)g->format : 0, g ? g->depth : 0));
+@@ -1605,18 +1616,41 @@ static bool can_discard_mask(uint8_t op, PicturePtr src, PictFormatPtr mask,
+
+ list++;
+ }
++
++ if (!sna_picture_is_solid(src, &color))
++ return false;
++
++ return color >> 24 == 0xff;
+ } else {
+- if (PICT_FORMAT_A(mask->format) >= PICT_FORMAT_A(g->format))
++skip:
++ if (mask->format == g->format)
+ return true;
+
+- if (g->depth != 1)
+- return false;
+- }
++ if (mask->format == alphaless(g->format))
++ return true;
++
++ if (PICT_FORMAT_TYPE(g->format) == PICT_TYPE_A &&
++ PICT_FORMAT_TYPE(mask->format) != PICT_TYPE_A)
++ return true;
+
+- if (!sna_picture_is_solid(src, &color))
+ return false;
++ }
++}
+
+- return color >> 24 == 0xff;
++static uint32_t pixman_format(PictFormatPtr short_format)
++{
++ uint32_t bpp;
++
++ bpp = short_format->depth;
++ if (bpp <= 1)
++ bpp = 1;
++ else if (bpp <= 8)
++ bpp = 8;
++ else if (bpp <= 16)
++ bpp = 16;
++ else
++ bpp = 32;
++ return bpp << 24 | short_format->format;
+ }
+
+ static void
+@@ -1756,7 +1790,7 @@ next:
+ if (sigtrap_get() == 0) {
+ if (mask_format) {
+ pixman_composite_glyphs(op, src_image, dst_image,
+- mask_format->format | (mask_format->depth << 24),
++ pixman_format(mask_format),
+ src_x + src_dx + region.extents.x1 - dst_x,
+ src_y + src_dy + region.extents.y1 - dst_y,
+ region.extents.x1, region.extents.y1,
+@@ -1815,10 +1849,10 @@ out:
+ x, y,
+ mask_format->depth,
+ (long)mask_format->format,
+- (long)(mask_format->depth << 24 | mask_format->format),
++ (long)pixman_format(mask_format),
+ NeedsComponent(mask_format->format)));
+ mask_image =
+- pixman_image_create_bits(mask_format->depth << 24 | mask_format->format,
++ pixman_image_create_bits(pixman_format(mask_format),
+ region.extents.x2 - region.extents.x1,
+ region.extents.y2 - region.extents.y1,
+ NULL, 0);
+@@ -2086,12 +2120,6 @@ glyphs_via_image(struct sna *sna,
+ src_x += box.x1 - list->xOff;
+ src_y += box.y1 - list->yOff;
+
+- if (format->depth < 8) {
+- format = PictureMatchFormat(screen, 8, PICT_a8);
+- if (!format)
+- return false;
+- }
+-
+ DBG(("%s: small mask [format=%lx, depth=%d, size=%d], rendering glyphs to upload buffer\n",
+ __FUNCTION__, (unsigned long)format->format,
+ format->depth, (uint32_t)width*height*format->depth));
+@@ -2104,7 +2132,7 @@ glyphs_via_image(struct sna *sna,
+ return false;
+
+ mask_image =
+- pixman_image_create_bits(format->depth << 24 | format->format,
++ pixman_image_create_bits(pixmap->drawable.bitsPerPixel << 24 | format->format,
+ width, height,
+ pixmap->devPrivate.ptr,
+ pixmap->devKind);
+diff --git a/src/sna/sna_io.c b/src/sna/sna_io.c
+index d6aa1294..d32bd583 100644
+--- a/src/sna/sna_io.c
++++ b/src/sna/sna_io.c
+@@ -105,8 +105,10 @@ read_boxes_inplace__cpu(struct kgem *kgem,
+ if (!download_inplace__cpu(kgem, dst, bo, box, n))
+ return false;
+
++ if (bo->tiling == I915_TILING_Y)
++ return false;
++
+ assert(kgem_bo_can_map__cpu(kgem, bo, false));
+- assert(bo->tiling != I915_TILING_Y);
+
+ src = kgem_bo_map__cpu(kgem, bo);
+ if (src == NULL)
+@@ -281,6 +283,9 @@ fallback:
+ if (box[n].y2 > extents.y2)
+ extents.y2 = box[n].y2;
+ }
++ if (!can_blt && sna->render.max_3d_size == 0)
++ goto fallback;
++
+ if (kgem_bo_can_map(kgem, src_bo)) {
+ /* Is it worth detiling? */
+ if ((extents.y2 - extents.y1 - 1) * src_bo->pitch < 4096)
+@@ -477,6 +482,7 @@ fallback:
+ goto fallback;
+ _kgem_set_mode(kgem, KGEM_BLT);
+ }
++ kgem_bcs_set_tiling(&sna->kgem, src_bo, NULL);
+
+ tmp_nbox = nbox;
+ tmp_box = box;
+@@ -539,6 +545,7 @@ fallback:
+ break;
+
+ _kgem_set_mode(kgem, KGEM_BLT);
++ kgem_bcs_set_tiling(&sna->kgem, src_bo, NULL);
+ tmp_box += nbox_this_time;
+ } while (1);
+ } else {
+@@ -597,6 +604,7 @@ fallback:
+ break;
+
+ _kgem_set_mode(kgem, KGEM_BLT);
++ kgem_bcs_set_tiling(&sna->kgem, src_bo, NULL);
+ tmp_box += nbox_this_time;
+ } while (1);
+ }
+@@ -666,8 +674,10 @@ write_boxes_inplace__tiled(struct kgem *kgem,
+ {
+ uint8_t *dst;
+
++ if (bo->tiling == I915_TILING_Y)
++ return false;
++
+ assert(kgem->has_wc_mmap || kgem_bo_can_map__cpu(kgem, bo, true));
+- assert(bo->tiling != I915_TILING_Y);
+
+ if (kgem_bo_can_map__cpu(kgem, bo, true)) {
+ dst = kgem_bo_map__cpu(kgem, bo);
+@@ -778,6 +788,15 @@ static bool __upload_inplace(struct kgem *kgem,
+ if (FORCE_INPLACE)
+ return FORCE_INPLACE > 0;
+
++ if (bo->exec)
++ return false;
++
++ if (bo->flush)
++ return true;
++
++ if (kgem_bo_can_map__cpu(kgem, bo, true))
++ return true;
++
+ /* If we are writing through the GTT, check first if we might be
+ * able to almagamate a series of small writes into a single
+ * operation.
+@@ -849,6 +868,8 @@ bool sna_write_boxes(struct sna *sna, PixmapPtr dst,
+ if (box[n].y2 > extents.y2)
+ extents.y2 = box[n].y2;
+ }
++ if (!can_blt && sna->render.max_3d_size == 0)
++ goto fallback;
+
+ /* Try to avoid switching rings... */
+ if (!can_blt || kgem->ring == KGEM_RENDER ||
+@@ -1038,6 +1059,7 @@ tile:
+ goto fallback;
+ _kgem_set_mode(kgem, KGEM_BLT);
+ }
++ kgem_bcs_set_tiling(&sna->kgem, NULL, dst_bo);
+
+ if (kgem->gen >= 0100) {
+ cmd |= 8;
+@@ -1129,6 +1151,7 @@ tile:
+ if (nbox) {
+ _kgem_submit(kgem);
+ _kgem_set_mode(kgem, KGEM_BLT);
++ kgem_bcs_set_tiling(&sna->kgem, NULL, dst_bo);
+ }
+
+ kgem_bo_destroy(kgem, src_bo);
+@@ -1224,6 +1247,7 @@ tile:
+ if (nbox) {
+ _kgem_submit(kgem);
+ _kgem_set_mode(kgem, KGEM_BLT);
++ kgem_bcs_set_tiling(&sna->kgem, NULL, dst_bo);
+ }
+
+ kgem_bo_destroy(kgem, src_bo);
+@@ -1541,6 +1565,7 @@ tile:
+ goto fallback;
+ _kgem_set_mode(kgem, KGEM_BLT);
+ }
++ kgem_bcs_set_tiling(&sna->kgem, NULL, dst_bo);
+
+ if (sna->kgem.gen >= 0100) {
+ cmd |= 8;
+@@ -1636,6 +1661,7 @@ tile:
+ if (nbox) {
+ _kgem_submit(kgem);
+ _kgem_set_mode(kgem, KGEM_BLT);
++ kgem_bcs_set_tiling(&sna->kgem, NULL, dst_bo);
+ }
+
+ kgem_bo_destroy(kgem, src_bo);
+@@ -1732,6 +1758,7 @@ tile:
+ if (nbox) {
+ _kgem_submit(kgem);
+ _kgem_set_mode(kgem, KGEM_BLT);
++ kgem_bcs_set_tiling(&sna->kgem, NULL, dst_bo);
+ }
+
+ kgem_bo_destroy(kgem, src_bo);
+diff --git a/src/sna/sna_present.c b/src/sna/sna_present.c
+index 6dd6fe88..2796d972 100644
+--- a/src/sna/sna_present.c
++++ b/src/sna/sna_present.c
+@@ -27,6 +27,7 @@
+ #include <sys/types.h>
+ #include <fcntl.h>
+ #include <unistd.h>
++#include <sys/poll.h>
+ #include <errno.h>
+ #include <xf86drm.h>
+
+@@ -38,21 +39,73 @@
+ static present_screen_info_rec present_info;
+
+ struct sna_present_event {
+- uint64_t event_id;
+ xf86CrtcPtr crtc;
++ struct sna *sna;
++ struct list link;
++ uint64_t *event_id;
++ uint64_t target_msc;
++ int n_event_id;
++ bool queued;
+ };
+
++static void sna_present_unflip(ScreenPtr screen, uint64_t event_id);
++static bool sna_present_queue(struct sna_present_event *info,
++ uint64_t last_msc);
++
+ static inline struct sna_present_event *
+ to_present_event(uintptr_t data)
+ {
+ return (struct sna_present_event *)(data & ~3);
+ }
+
++static struct sna_present_event *info_alloc(struct sna *sna)
++{
++ struct sna_present_event *info;
++
++ info = sna->present.freed_info;
++ if (info) {
++ sna->present.freed_info = NULL;
++ return info;
++ }
++
++ return malloc(sizeof(struct sna_present_event) + sizeof(uint64_t));
++}
++
++static void info_free(struct sna_present_event *info)
++{
++ struct sna *sna = info->sna;
++
++ if (sna->present.freed_info)
++ free(sna->present.freed_info);
++
++ sna->present.freed_info = info;
++}
++
++static inline bool msc_before(uint64_t msc, uint64_t target)
++{
++ return (int64_t)(msc - target) < 0;
++}
++
+ #define MARK_PRESENT(x) ((void *)((uintptr_t)(x) | 2))
+
+-static int pipe_from_crtc(RRCrtcPtr crtc)
++static inline xf86CrtcPtr unmask_crtc(xf86CrtcPtr crtc)
++{
++ return (xf86CrtcPtr)((uintptr_t)crtc & ~1);
++}
++
++static inline xf86CrtcPtr mark_crtc(xf86CrtcPtr crtc)
++{
++ return (xf86CrtcPtr)((uintptr_t)crtc | 1);
++}
++
++static inline bool has_vblank(xf86CrtcPtr crtc)
++{
++ return (uintptr_t)crtc & 1;
++}
++
++static inline int pipe_from_crtc(RRCrtcPtr crtc)
+ {
+- return crtc ? sna_crtc_to_pipe(crtc->devPrivate) : -1;
++ return crtc ? sna_crtc_pipe(crtc->devPrivate) : -1;
+ }
+
+ static uint32_t pipe_select(int pipe)
+@@ -74,6 +127,215 @@ static inline int sna_wait_vblank(struct sna *sna, union drm_wait_vblank *vbl, i
+ return drmIoctl(sna->kgem.fd, DRM_IOCTL_WAIT_VBLANK, vbl);
+ }
+
++static uint64_t gettime_ust64(void)
++{
++ struct timespec tv;
++
++ if (clock_gettime(CLOCK_MONOTONIC, &tv))
++ return GetTimeInMicros();
++
++ return ust64(tv.tv_sec, tv.tv_nsec / 1000);
++}
++
++static void vblank_complete(struct sna_present_event *info,
++ uint64_t ust, uint64_t msc)
++{
++ int n;
++
++ if (msc_before(msc, info->target_msc)) {
++ DBG(("%s: event=%d too early, now %lld, expected %lld\n",
++ __FUNCTION__,
++ info->event_id[0],
++ (long long)msc, (long long)info->target_msc));
++ if (sna_present_queue(info, msc))
++ return;
++ }
++
++ DBG(("%s: %d events complete\n", __FUNCTION__, info->n_event_id));
++ for (n = 0; n < info->n_event_id; n++) {
++ DBG(("%s: pipe=%d tv=%d.%06d msc=%lld (target=%lld), event=%lld complete%s\n", __FUNCTION__,
++ sna_crtc_pipe(info->crtc),
++ (int)(ust / 1000000), (int)(ust % 1000000),
++ (long long)msc, (long long)info->target_msc,
++ (long long)info->event_id[n],
++ info->target_msc && msc == (uint32_t)info->target_msc ? "" : ": MISS"));
++ present_event_notify(info->event_id[n], ust, msc);
++ }
++ if (info->n_event_id > 1)
++ free(info->event_id);
++ list_del(&info->link);
++ info_free(info);
++}
++
++static uint32_t msc_to_delay(xf86CrtcPtr crtc, uint64_t target)
++{
++ const DisplayModeRec *mode = &crtc->desiredMode;
++ const struct ust_msc *swap = sna_crtc_last_swap(crtc);
++ int64_t delay, subframe;
++
++ assert(mode->Clock);
++
++ delay = target - swap->msc;
++ assert(delay >= 0);
++ if (delay > 1) { /* try to use the hw vblank for the last frame */
++ delay--;
++ subframe = 0;
++ } else {
++ subframe = gettime_ust64() - swap_ust(swap);
++ subframe += 500;
++ subframe /= 1000;
++ }
++ delay *= mode->VTotal * mode->HTotal / mode->Clock;
++ if (subframe < delay)
++ delay -= subframe;
++ else
++ delay = 0;
++
++ DBG(("%s: sleep %d frames, %llu ms\n", __FUNCTION__,
++ (int)(target - swap->msc), (long long)delay));
++ assert(delay >= 0);
++ return MIN(delay, INT32_MAX);
++}
++
++static CARD32 sna_fake_vblank_handler(OsTimerPtr timer, CARD32 now, void *data)
++{
++ struct sna_present_event *info = data;
++ union drm_wait_vblank vbl;
++ uint64_t msc, ust;
++
++ DBG(("%s(event=%lldx%d, now=%d)\n", __FUNCTION__, (long long)info->event_id[0], info->n_event_id, now));
++
++ VG_CLEAR(vbl);
++ vbl.request.type = DRM_VBLANK_RELATIVE;
++ vbl.request.sequence = 0;
++ if (sna_wait_vblank(info->sna, &vbl, sna_crtc_pipe(info->crtc)) == 0) {
++ ust = ust64(vbl.reply.tval_sec, vbl.reply.tval_usec);
++ msc = sna_crtc_record_vblank(info->crtc, &vbl);
++ DBG(("%s: event=%lld, target msc=%lld, now %lld\n",
++ __FUNCTION__, (long long)info->event_id[0], (long long)info->target_msc, (long long)msc));
++ if (msc_before(msc, info->target_msc)) {
++ int delta = info->target_msc - msc;
++ uint32_t delay;
++
++ DBG(("%s: too early, requeuing delta=%d\n", __FUNCTION__, delta));
++ assert(info->target_msc - msc < 1ull<<31);
++ if (delta <= 2) {
++ vbl.request.type = DRM_VBLANK_ABSOLUTE | DRM_VBLANK_EVENT;
++ vbl.request.sequence = info->target_msc;
++ vbl.request.signal = (uintptr_t)MARK_PRESENT(info);
++ if (sna_wait_vblank(info->sna, &vbl, sna_crtc_pipe(info->crtc)) == 0) {
++ DBG(("%s: scheduled new vblank event for %lld\n", __FUNCTION__, (long long)info->target_msc));
++ info->queued = true;
++ if (delta == 1) {
++ sna_crtc_set_vblank(info->crtc);
++ info->crtc = mark_crtc(info->crtc);
++ }
++ free(timer);
++ return 0;
++ }
++ }
++
++ delay = msc_to_delay(info->crtc, info->target_msc);
++ if (delay) {
++ DBG(("%s: requeueing timer for %dms delay\n", __FUNCTION__, delay));
++ return delay;
++ }
++
++ /* As a last resort use a blocking wait.
++ * Less than a millisecond for (hopefully) a rare case.
++ */
++ DBG(("%s: blocking wait!\n", __FUNCTION__));
++ vbl.request.type = DRM_VBLANK_ABSOLUTE;
++ vbl.request.sequence = info->target_msc;
++ if (sna_wait_vblank(info->sna, &vbl, sna_crtc_pipe(info->crtc)) == 0) {
++ ust = ust64(vbl.reply.tval_sec, vbl.reply.tval_usec);
++ msc = sna_crtc_record_vblank(info->crtc, &vbl);
++ } else {
++ DBG(("%s: blocking wait failed, fudging\n",
++ __FUNCTION__));
++ goto fixup;
++ }
++ }
++ } else {
++fixup:
++ ust = gettime_ust64();
++ msc = info->target_msc;
++ DBG(("%s: event=%lld, CRTC OFF, target msc=%lld, was %lld (off)\n",
++ __FUNCTION__, (long long)info->event_id[0], (long long)info->target_msc, (long long)sna_crtc_last_swap(info->crtc)->msc));
++ }
++
++ vblank_complete(info, ust, msc);
++ free(timer);
++ return 0;
++}
++
++static bool sna_fake_vblank(struct sna_present_event *info)
++{
++ const struct ust_msc *swap = sna_crtc_last_swap(info->crtc);
++ uint32_t delay;
++
++ if (msc_before(swap->msc, info->target_msc))
++ delay = msc_to_delay(info->crtc, info->target_msc);
++ else
++ delay = 0;
++
++ DBG(("%s(event=%lldx%d, target_msc=%lld, msc=%lld, delay=%ums)\n",
++ __FUNCTION__, (long long)info->event_id[0], info->n_event_id,
++ (long long)info->target_msc, (long long)swap->msc, delay));
++ if (delay == 0) {
++ uint64_t ust, msc;
++
++ if (msc_before(swap->msc, info->target_msc)) {
++ /* Fixup and pretend it completed immediately */
++ msc = info->target_msc;
++ ust = gettime_ust64();
++ } else {
++ msc = swap->msc;
++ ust = swap_ust(swap);
++ }
++
++ vblank_complete(info, ust, msc);
++ return true;
++ }
++
++ return TimerSet(NULL, 0, delay, sna_fake_vblank_handler, info);
++}
++
++static bool sna_present_queue(struct sna_present_event *info,
++ uint64_t last_msc)
++{
++ union drm_wait_vblank vbl;
++ int delta = info->target_msc - last_msc;
++
++ DBG(("%s: target msc=%llu, seq=%u (last_msc=%llu), delta=%d\n",
++ __FUNCTION__,
++ (long long)info->target_msc,
++ (unsigned)info->target_msc,
++ (long long)last_msc,
++ delta));
++ assert(info->target_msc - last_msc < 1ull<<31);
++ assert(delta >= 0);
++
++ VG_CLEAR(vbl);
++ vbl.request.type = DRM_VBLANK_ABSOLUTE | DRM_VBLANK_EVENT;
++ vbl.request.sequence = info->target_msc;
++ vbl.request.signal = (uintptr_t)MARK_PRESENT(info);
++ if (delta > 2 ||
++ sna_wait_vblank(info->sna, &vbl, sna_crtc_pipe(info->crtc))) {
++ DBG(("%s: vblank enqueue failed, faking delta=%d\n", __FUNCTION__, delta));
++ if (!sna_fake_vblank(info))
++ return false;
++ } else {
++ info->queued = true;
++ if (delta == 1) {
++ sna_crtc_set_vblank(info->crtc);
++ info->crtc = mark_crtc(info->crtc);
++ }
++ }
++
++ return true;
++}
++
+ static RRCrtcPtr
+ sna_present_get_crtc(WindowPtr window)
+ {
+@@ -81,7 +343,10 @@ sna_present_get_crtc(WindowPtr window)
+ BoxRec box;
+ xf86CrtcPtr crtc;
+
+- DBG(("%s\n", __FUNCTION__));
++ DBG(("%s: window=%ld (pixmap=%ld), box=(%d, %d)x(%d, %d)\n",
++ __FUNCTION__, window->drawable.id, get_window_pixmap(window)->drawable.serialNumber,
++ window->drawable.x, window->drawable.y,
++ window->drawable.width, window->drawable.height));
+
+ box.x1 = window->drawable.x;
+ box.y1 = window->drawable.y;
+@@ -99,26 +364,59 @@ static int
+ sna_present_get_ust_msc(RRCrtcPtr crtc, CARD64 *ust, CARD64 *msc)
+ {
+ struct sna *sna = to_sna_from_screen(crtc->pScreen);
+- int pipe = pipe_from_crtc(crtc);
+ union drm_wait_vblank vbl;
+
+- DBG(("%s(pipe=%d)\n", __FUNCTION__, pipe));
++ DBG(("%s(pipe=%d)\n", __FUNCTION__, sna_crtc_pipe(crtc->devPrivate)));
++ if (sna_crtc_has_vblank(crtc->devPrivate)) {
++ DBG(("%s: vblank active, reusing last swap msc/ust\n",
++ __FUNCTION__));
++ goto last;
++ }
+
+ VG_CLEAR(vbl);
+ vbl.request.type = DRM_VBLANK_RELATIVE;
+ vbl.request.sequence = 0;
+- if (sna_wait_vblank(sna, &vbl, pipe) == 0) {
++ if (sna_wait_vblank(sna, &vbl, sna_crtc_pipe(crtc->devPrivate)) == 0) {
++ struct sna_present_event *info;
++
+ *ust = ust64(vbl.reply.tval_sec, vbl.reply.tval_usec);
+ *msc = sna_crtc_record_vblank(crtc->devPrivate, &vbl);
++
++ info = info_alloc(sna);
++ if (info) {
++ info->crtc = crtc->devPrivate;
++ info->sna = sna;
++ info->target_msc = *msc + 1;
++ info->event_id = (uint64_t *)(info + 1);
++ info->n_event_id = 0;
++
++ vbl.request.type =
++ DRM_VBLANK_ABSOLUTE | DRM_VBLANK_EVENT;
++ vbl.request.sequence = info->target_msc;
++ vbl.request.signal = (uintptr_t)MARK_PRESENT(info);
++
++ if (sna_wait_vblank(info->sna, &vbl,
++ sna_crtc_pipe(info->crtc)) == 0) {
++ list_add(&info->link,
++ &sna->present.vblank_queue);
++ info->queued = true;
++ sna_crtc_set_vblank(info->crtc);
++ info->crtc = mark_crtc(info->crtc);
++ } else
++ info_free(info);
++ }
+ } else {
+- const struct ust_msc *swap = sna_crtc_last_swap(crtc->devPrivate);
+- *ust = ust64(swap->tv_sec, swap->tv_usec);
++ const struct ust_msc *swap;
++last:
++ swap = sna_crtc_last_swap(crtc->devPrivate);
++ *ust = swap_ust(swap);
+ *msc = swap->msc;
+ }
+
+- DBG(("%s: pipe=%d, tv=%d.%06d msc=%lld\n", __FUNCTION__, pipe,
++ DBG(("%s: pipe=%d, tv=%d.%06d seq=%d msc=%lld\n", __FUNCTION__,
++ sna_crtc_pipe(crtc->devPrivate),
+ (int)(*ust / 1000000), (int)(*ust % 1000000),
+- (long long)*msc));
++ vbl.reply.sequence, (long long)*msc));
+
+ return Success;
+ }
+@@ -127,43 +425,106 @@ void
+ sna_present_vblank_handler(struct drm_event_vblank *event)
+ {
+ struct sna_present_event *info = to_present_event(event->user_data);
++ uint64_t msc;
+
+- DBG(("%s: pipe=%d tv=%d.%06d msc=%d, event %lld complete\n", __FUNCTION__,
+- sna_crtc_to_pipe(info->crtc),
+- event->tv_sec, event->tv_usec, event->sequence,
+- (long long)info->event_id));
+- present_event_notify(info->event_id,
+- ust64(event->tv_sec, event->tv_usec),
+- sna_crtc_record_event(info->crtc, event));
+- free(info);
++ if (!info->queued) {
++ DBG(("%s: arrived unexpectedly early (not queued)\n", __FUNCTION__));
++ assert(!has_vblank(info->crtc));
++ return;
++ }
++
++ if (has_vblank(info->crtc)) {
++ DBG(("%s: clearing immediate flag\n", __FUNCTION__));
++ info->crtc = unmask_crtc(info->crtc);
++ sna_crtc_clear_vblank(info->crtc);
++ }
++
++ msc = sna_crtc_record_event(info->crtc, event);
++
++ if (info->sna->mode.shadow_wait) {
++ DBG(("%s: recursed from TearFree\n", __FUNCTION__));
++ if (TimerSet(NULL, 0, 1, sna_fake_vblank_handler, info))
++ return;
++ }
++
++ vblank_complete(info, ust64(event->tv_sec, event->tv_usec), msc);
+ }
+
+ static int
+ sna_present_queue_vblank(RRCrtcPtr crtc, uint64_t event_id, uint64_t msc)
+ {
+ struct sna *sna = to_sna_from_screen(crtc->pScreen);
+- struct sna_present_event *event;
+- union drm_wait_vblank vbl;
+-
+- DBG(("%s(pipe=%d, event=%lld, msc=%lld)\n",
+- __FUNCTION__, pipe_from_crtc(crtc),
+- (long long)event_id, (long long)msc));
++ struct sna_present_event *info, *tmp;
++ const struct ust_msc *swap;
+
+- event = malloc(sizeof(struct sna_present_event));
+- if (event == NULL)
++ if (!sna_crtc_is_on(crtc->devPrivate))
+ return BadAlloc;
+
+- event->event_id = event_id;
+- event->crtc = crtc->devPrivate;
++ swap = sna_crtc_last_swap(crtc->devPrivate);
++ DBG(("%s(pipe=%d, event=%lld, msc=%lld, last swap=%lld)\n",
++ __FUNCTION__, sna_crtc_pipe(crtc->devPrivate),
++ (long long)event_id, (long long)msc, (long long)swap->msc));
+
+- VG_CLEAR(vbl);
+- vbl.request.type = DRM_VBLANK_ABSOLUTE | DRM_VBLANK_EVENT;
+- vbl.request.sequence = msc;
+- vbl.request.signal = (uintptr_t)MARK_PRESENT(event);
+- if (sna_wait_vblank(sna, &vbl, sna_crtc_to_pipe(event->crtc))) {
+- DBG(("%s: vblank enqueue failed\n", __FUNCTION__));
+- free(event);
+- return BadMatch;
++ if (warn_unless((int64_t)(msc - swap->msc) >= 0)) {
++ DBG(("%s: pipe=%d tv=%d.%06d msc=%lld (target=%lld), event=%lld complete\n", __FUNCTION__,
++ sna_crtc_pipe(crtc->devPrivate),
++ swap->tv_sec, swap->tv_usec,
++ (long long)swap->msc, (long long)msc,
++ (long long)event_id));
++ present_event_notify(event_id, swap_ust(swap), swap->msc);
++ return Success;
++ }
++ if (warn_unless(msc - swap->msc < 1ull<<31))
++ return BadValue;
++
++ list_for_each_entry(tmp, &sna->present.vblank_queue, link) {
++ if (tmp->target_msc == msc &&
++ unmask_crtc(tmp->crtc) == crtc->devPrivate) {
++ uint64_t *events = tmp->event_id;
++
++ if (tmp->n_event_id &&
++ is_power_of_two(tmp->n_event_id)) {
++ events = malloc(2*sizeof(uint64_t)*tmp->n_event_id);
++ if (events == NULL)
++ return BadAlloc;
++
++ memcpy(events,
++ tmp->event_id,
++ tmp->n_event_id*sizeof(uint64_t));
++ if (tmp->n_event_id != 1)
++ free(tmp->event_id);
++ tmp->event_id = events;
++ }
++
++ DBG(("%s: appending event=%lld to vblank %lld x %d\n",
++ __FUNCTION__, (long long)event_id, (long long)msc, tmp->n_event_id+1));
++ events[tmp->n_event_id++] = event_id;
++ return Success;
++ }
++ if ((int64_t)(tmp->target_msc - msc) > 0) {
++ DBG(("%s: previous target_msc=%lld invalid for coalescing\n",
++ __FUNCTION__, (long long)tmp->target_msc));
++ break;
++ }
++ }
++
++ info = info_alloc(sna);
++ if (info == NULL)
++ return BadAlloc;
++
++ info->crtc = crtc->devPrivate;
++ info->sna = sna;
++ info->target_msc = msc;
++ info->event_id = (uint64_t *)(info + 1);
++ info->event_id[0] = event_id;
++ info->n_event_id = 1;
++ list_add_tail(&info->link, &tmp->link);
++ info->queued = false;
++
++ if (!sna_present_queue(info, swap->msc)) {
++ list_del(&info->link);
++ info_free(info);
++ return BadAlloc;
+ }
+
+ return Success;
+@@ -180,14 +541,6 @@ sna_present_abort_vblank(RRCrtcPtr crtc, uint64_t event_id, uint64_t msc)
+ static void
+ sna_present_flush(WindowPtr window)
+ {
+- PixmapPtr pixmap = get_window_pixmap(window);
+- struct sna_pixmap *priv;
+-
+- DBG(("%s(pixmap=%ld)\n", __FUNCTION__, pixmap->drawable.serialNumber));
+-
+- priv = sna_pixmap_move_to_gpu(pixmap, MOVE_READ | MOVE_ASYNC_HINT | __MOVE_FORCE);
+- if (priv && priv->gpu_bo)
+- kgem_scanout_flush(&to_sna_from_pixmap(pixmap)->kgem, priv->gpu_bo);
+ }
+
+ static bool
+@@ -201,8 +554,13 @@ check_flip__crtc(struct sna *sna,
+
+ assert(sna->scrn->vtSema);
+
+- if (sna->mode.shadow_active) {
+- DBG(("%s: shadow buffer active\n", __FUNCTION__));
++ if (!sna->mode.front_active) {
++ DBG(("%s: DPMS off, no flips\n", __FUNCTION__));
++ return FALSE;
++ }
++
++ if (sna->mode.rr_active) {
++ DBG(("%s: RandR transformation active\n", __FUNCTION__));
+ return false;
+ }
+
+@@ -224,6 +582,11 @@ sna_present_check_flip(RRCrtcPtr crtc,
+ pixmap->drawable.serialNumber,
+ sync_flip));
+
++ if (!sna->scrn->vtSema) {
++ DBG(("%s: VT switched away, no flips\n", __FUNCTION__));
++ return FALSE;
++ }
++
+ if (sna->flags & SNA_NO_FLIP) {
+ DBG(("%s: flips not suported\n", __FUNCTION__));
+ return FALSE;
+@@ -231,7 +594,7 @@ sna_present_check_flip(RRCrtcPtr crtc,
+
+ if (sync_flip) {
+ if ((sna->flags & SNA_HAS_FLIP) == 0) {
+- DBG(("%s: async flips not suported\n", __FUNCTION__));
++ DBG(("%s: sync flips not suported\n", __FUNCTION__));
+ return FALSE;
+ }
+ } else {
+@@ -257,24 +620,39 @@ sna_present_check_flip(RRCrtcPtr crtc,
+ return FALSE;
+ }
+
+- return TRUE;
+-}
+-
+-static uint64_t gettime_ust64(void)
+-{
+- struct timespec tv;
++ if (flip->pinned) {
++ assert(flip->gpu_bo);
++ if (sna->flags & SNA_LINEAR_FB) {
++ if (flip->gpu_bo->tiling != I915_TILING_NONE) {
++ DBG(("%s: pined bo, tilng=%d needs NONE\n",
++ __FUNCTION__, flip->gpu_bo->tiling));
++ return FALSE;
++ }
++ } else {
++ if (!sna->kgem.can_scanout_y &&
++ flip->gpu_bo->tiling == I915_TILING_Y) {
++ DBG(("%s: pined bo, tilng=%d and can't scanout Y\n",
++ __FUNCTION__, flip->gpu_bo->tiling));
++ return FALSE;
++ }
++ }
+
+- if (clock_gettime(CLOCK_MONOTONIC, &tv))
+- return 0;
++ if (flip->gpu_bo->pitch & 63) {
++ DBG(("%s: pined bo, bad pitch=%d\n",
++ __FUNCTION__, flip->gpu_bo->pitch));
++ return FALSE;
++ }
++ }
+
+- return ust64(tv.tv_sec, tv.tv_nsec / 1000);
++ return TRUE;
+ }
+
+ static Bool
+-page_flip__async(RRCrtcPtr crtc,
+- uint64_t event_id,
+- uint64_t target_msc,
+- struct kgem_bo *bo)
++flip__async(struct sna *sna,
++ RRCrtcPtr crtc,
++ uint64_t event_id,
++ uint64_t target_msc,
++ struct kgem_bo *bo)
+ {
+ DBG(("%s(pipe=%d, event=%lld, handle=%d)\n",
+ __FUNCTION__,
+@@ -282,17 +660,17 @@ page_flip__async(RRCrtcPtr crtc,
+ (long long)event_id,
+ bo->handle));
+
+- if (!sna_page_flip(to_sna_from_screen(crtc->pScreen), bo, NULL, NULL)) {
++ if (!sna_page_flip(sna, bo, NULL, NULL)) {
+ DBG(("%s: async pageflip failed\n", __FUNCTION__));
+ present_info.capabilities &= ~PresentCapabilityAsync;
+ return FALSE;
+ }
+
+- DBG(("%s: pipe=%d tv=%d.%06d msc=%d, event %lld complete\n", __FUNCTION__,
++ DBG(("%s: pipe=%d tv=%ld.%06d msc=%lld (target=%lld), event=%lld complete\n", __FUNCTION__,
+ pipe_from_crtc(crtc),
+- gettime_ust64() / 1000000, gettime_ust64() % 1000000,
+- sna_crtc_last_swap(crtc->devPrivate)->msc,
+- (long long)event_id));
++ (long)(gettime_ust64() / 1000000), (int)(gettime_ust64() % 1000000),
++ crtc ? (long long)sna_crtc_last_swap(crtc->devPrivate)->msc : 0LL,
++ (long long)target_msc, (long long)event_id));
+ present_event_notify(event_id, gettime_ust64(), target_msc);
+ return TRUE;
+ }
+@@ -303,7 +681,12 @@ present_flip_handler(struct drm_event_vblank *event, void *data)
+ struct sna_present_event *info = data;
+ struct ust_msc swap;
+
+- DBG(("%s(sequence=%d)\n", __FUNCTION__, event->sequence));
++ DBG(("%s(sequence=%d): event=%lld\n", __FUNCTION__, event->sequence, (long long)info->event_id[0]));
++ assert(info->n_event_id == 1);
++ if (!info->queued) {
++ DBG(("%s: arrived unexpectedly early (not queued)\n", __FUNCTION__));
++ return;
++ }
+
+ if (info->crtc == NULL) {
+ swap.tv_sec = event->tv_sec;
+@@ -312,22 +695,33 @@ present_flip_handler(struct drm_event_vblank *event, void *data)
+ } else
+ swap = *sna_crtc_last_swap(info->crtc);
+
+- DBG(("%s: pipe=%d, tv=%d.%06d msc %lld, event %lld complete\n", __FUNCTION__,
+- info->crtc ? sna_crtc_to_pipe(info->crtc) : -1,
++ DBG(("%s: pipe=%d, tv=%d.%06d msc=%lld (target %lld), event=%lld complete%s\n", __FUNCTION__,
++ info->crtc ? sna_crtc_pipe(info->crtc) : -1,
+ swap.tv_sec, swap.tv_usec, (long long)swap.msc,
+- (long long)info->event_id));
+- present_event_notify(info->event_id, ust64(swap.tv_sec, swap.tv_usec), swap.msc);
+- free(info);
++ (long long)info->target_msc,
++ (long long)info->event_id[0],
++ info->target_msc && info->target_msc == swap.msc ? "" : ": MISS"));
++ present_event_notify(info->event_id[0], swap_ust(&swap), swap.msc);
++ if (info->crtc)
++ sna_crtc_clear_vblank(info->crtc);
++
++ if (info->sna->present.unflip) {
++ DBG(("%s: executing queued unflip (event=%lld)\n", __FUNCTION__, (long long)info->sna->present.unflip));
++ sna_present_unflip(xf86ScrnToScreen(info->sna->scrn),
++ info->sna->present.unflip);
++ info->sna->present.unflip = 0;
++ }
++ info_free(info);
+ }
+
+ static Bool
+-page_flip(ScreenPtr screen,
+- RRCrtcPtr crtc,
+- uint64_t event_id,
+- struct kgem_bo *bo)
++flip(struct sna *sna,
++ RRCrtcPtr crtc,
++ uint64_t event_id,
++ uint64_t target_msc,
++ struct kgem_bo *bo)
+ {
+- struct sna *sna = to_sna_from_screen(screen);
+- struct sna_present_event *event;
++ struct sna_present_event *info;
+
+ DBG(("%s(pipe=%d, event=%lld, handle=%d)\n",
+ __FUNCTION__,
+@@ -335,18 +729,27 @@ page_flip(ScreenPtr screen,
+ (long long)event_id,
+ bo->handle));
+
+- event = malloc(sizeof(struct sna_present_event));
+- if (event == NULL)
++ info = info_alloc(sna);
++ if (info == NULL)
+ return FALSE;
+
+- event->event_id = event_id;
+- event->crtc = crtc ? crtc->devPrivate : NULL;
+- if (!sna_page_flip(sna, bo, present_flip_handler, event)) {
++ info->crtc = crtc ? crtc->devPrivate : NULL;
++ info->sna = sna;
++ info->event_id = (uint64_t *)(info + 1);
++ info->event_id[0] = event_id;
++ info->n_event_id = 1;
++ info->target_msc = target_msc;
++ info->queued = false;
++
++ if (!sna_page_flip(sna, bo, present_flip_handler, info)) {
+ DBG(("%s: pageflip failed\n", __FUNCTION__));
+- free(event);
++ info_free(info);
+ return FALSE;
+ }
+
++ info->queued = true;
++ if (info->crtc)
++ sna_crtc_set_vblank(info->crtc);
+ return TRUE;
+ }
+
+@@ -358,12 +761,48 @@ get_flip_bo(PixmapPtr pixmap)
+
+ DBG(("%s(pixmap=%ld)\n", __FUNCTION__, pixmap->drawable.serialNumber));
+
+- priv = sna_pixmap_move_to_gpu(pixmap, MOVE_READ | __MOVE_FORCE);
++ priv = sna_pixmap_move_to_gpu(pixmap, MOVE_READ | __MOVE_SCANOUT | __MOVE_FORCE);
+ if (priv == NULL) {
+ DBG(("%s: cannot force pixmap to the GPU\n", __FUNCTION__));
+ return NULL;
+ }
+
++ if (priv->gpu_bo->scanout)
++ return priv->gpu_bo;
++
++ if (sna->kgem.has_llc && !wedged(sna) && !priv->pinned) {
++ struct kgem_bo *bo;
++ uint32_t tiling;
++
++ tiling = I915_TILING_NONE;
++ if ((sna->flags & SNA_LINEAR_FB) == 0)
++ tiling = I915_TILING_X;
++
++ bo = kgem_create_2d(&sna->kgem,
++ pixmap->drawable.width,
++ pixmap->drawable.height,
++ pixmap->drawable.bitsPerPixel,
++ tiling, CREATE_SCANOUT | CREATE_CACHED);
++ if (bo) {
++ BoxRec box;
++
++ box.x1 = box.y1 = 0;
++ box.x2 = pixmap->drawable.width;
++ box.y2 = pixmap->drawable.height;
++
++ if (sna->render.copy_boxes(sna, GXcopy,
++ &pixmap->drawable, priv->gpu_bo, 0, 0,
++ &pixmap->drawable, bo, 0, 0,
++ &box, 1, 0)) {
++ sna_pixmap_unmap(pixmap, priv);
++ kgem_bo_destroy(&sna->kgem, priv->gpu_bo);
++
++ priv->gpu_bo = bo;
++ } else
++ kgem_bo_destroy(&sna->kgem, bo);
++ }
++ }
++
+ if (sna->flags & SNA_LINEAR_FB &&
+ priv->gpu_bo->tiling &&
+ !sna_pixmap_change_tiling(pixmap, I915_TILING_NONE)) {
+@@ -372,12 +811,17 @@ get_flip_bo(PixmapPtr pixmap)
+ }
+
+ if (priv->gpu_bo->tiling == I915_TILING_Y &&
++ !sna->kgem.can_scanout_y &&
+ !sna_pixmap_change_tiling(pixmap, I915_TILING_X)) {
+ DBG(("%s: invalid Y-tiling, cannot convert\n", __FUNCTION__));
+ return NULL;
+ }
+
+- priv->pinned |= PIN_SCANOUT;
++ if (priv->gpu_bo->pitch & 63) {
++ DBG(("%s: invalid pitch, no conversion\n", __FUNCTION__));
++ return NULL;
++ }
++
+ return priv->gpu_bo;
+ }
+
+@@ -388,6 +832,7 @@ sna_present_flip(RRCrtcPtr crtc,
+ PixmapPtr pixmap,
+ Bool sync_flip)
+ {
++ struct sna *sna = to_sna_from_pixmap(pixmap);
+ struct kgem_bo *bo;
+
+ DBG(("%s(pipe=%d, event=%lld, msc=%lld, pixmap=%ld, sync?=%d)\n",
+@@ -397,11 +842,32 @@ sna_present_flip(RRCrtcPtr crtc,
+ (long long)target_msc,
+ pixmap->drawable.serialNumber, sync_flip));
+
+- if (!check_flip__crtc(to_sna_from_pixmap(pixmap), crtc)) {
++ if (!check_flip__crtc(sna, crtc)) {
+ DBG(("%s: flip invalid for CRTC\n", __FUNCTION__));
+ return FALSE;
+ }
+
++ assert(sna->present.unflip == 0);
++
++ if (sna->flags & SNA_TEAR_FREE) {
++ DBG(("%s: disabling TearFree (was %s) in favour of Present flips\n",
++ __FUNCTION__, sna->mode.shadow_enabled ? "enabled" : "disabled"));
++ sna->mode.shadow_enabled = false;
++ }
++ assert(!sna->mode.shadow_enabled);
++
++ if (sna->mode.flip_active) {
++ struct pollfd pfd;
++
++ DBG(("%s: flips still pending, stalling\n", __FUNCTION__));
++ pfd.fd = sna->kgem.fd;
++ pfd.events = POLLIN;
++ while (poll(&pfd, 1, 0) == 1)
++ sna_mode_wakeup(sna);
++ if (sna->mode.flip_active)
++ return FALSE;
++ }
++
+ bo = get_flip_bo(pixmap);
+ if (bo == NULL) {
+ DBG(("%s: flip invalid bo\n", __FUNCTION__));
+@@ -409,9 +875,9 @@ sna_present_flip(RRCrtcPtr crtc,
+ }
+
+ if (sync_flip)
+- return page_flip(crtc->pScreen, crtc, event_id, bo);
++ return flip(sna, crtc, event_id, target_msc, bo);
+ else
+- return page_flip__async(crtc, event_id, target_msc, bo);
++ return flip__async(sna, crtc, event_id, target_msc, bo);
+ }
+
+ static void
+@@ -421,29 +887,70 @@ sna_present_unflip(ScreenPtr screen, uint64_t event_id)
+ struct kgem_bo *bo;
+
+ DBG(("%s(event=%lld)\n", __FUNCTION__, (long long)event_id));
+- if (sna->mode.front_active == 0 || sna->mode.shadow_active) {
++ if (sna->mode.front_active == 0 || sna->mode.rr_active) {
+ const struct ust_msc *swap;
+
+ DBG(("%s: no CRTC active, perform no-op flip\n", __FUNCTION__));
+
+ notify:
+- swap = sna_crtc_last_swap(sna_mode_first_crtc(sna));
+- DBG(("%s: pipe=%d, tv=%d.%06d msc %lld, event %lld complete\n", __FUNCTION__,
++ swap = sna_crtc_last_swap(sna_primary_crtc(sna));
++ DBG(("%s: pipe=%d, tv=%d.%06d msc=%lld, event=%lld complete\n", __FUNCTION__,
+ -1,
+ swap->tv_sec, swap->tv_usec, (long long)swap->msc,
+ (long long)event_id));
+- present_event_notify(event_id,
+- ust64(swap->tv_sec, swap->tv_usec),
+- swap->msc);
++ present_event_notify(event_id, swap_ust(swap), swap->msc);
++ return;
++ }
++
++ assert(!sna->mode.shadow_enabled);
++ if (sna->mode.flip_active) {
++ DBG(("%s: %d outstanding flips, queueing unflip\n", __FUNCTION__, sna->mode.flip_active));
++ assert(sna->present.unflip == 0);
++ sna->present.unflip = event_id;
+ return;
+ }
+
++ if (sna->flags & SNA_TEAR_FREE) {
++ DBG(("%s: %s TearFree after Present flips\n",
++ __FUNCTION__, sna->mode.shadow_damage != NULL ? "enabling" : "disabling"));
++ sna->mode.shadow_enabled = sna->mode.shadow_damage != NULL;
++ }
++
+ bo = get_flip_bo(screen->GetScreenPixmap(screen));
+- if (bo == NULL || !page_flip(screen, NULL, event_id, bo)) {
++ if (bo == NULL) {
++reset_mode:
+ DBG(("%s: failed, trying to restore original mode\n", __FUNCTION__));
+ xf86SetDesiredModes(sna->scrn);
+ goto notify;
+ }
++
++ /* Are we unflipping after a failure that left our ScreenP in place? */
++ if (!sna_needs_page_flip(sna, bo))
++ goto notify;
++
++ assert(sna_pixmap(screen->GetScreenPixmap(screen))->pinned & PIN_SCANOUT);
++
++ if (sna->flags & SNA_HAS_ASYNC_FLIP) {
++ DBG(("%s: trying async flip restore\n", __FUNCTION__));
++ if (flip__async(sna, NULL, event_id, 0, bo))
++ return;
++ }
++
++ if (!flip(sna, NULL, event_id, 0, bo))
++ goto reset_mode;
++}
++
++void sna_present_cancel_flip(struct sna *sna)
++{
++ if (sna->present.unflip) {
++ const struct ust_msc *swap;
++
++ swap = sna_crtc_last_swap(sna_primary_crtc(sna));
++ present_event_notify(sna->present.unflip,
++ swap_ust(swap), swap->msc);
++
++ sna->present.unflip = 0;
++ }
+ }
+
+ static present_screen_info_rec present_info = {
+@@ -463,10 +970,13 @@ static present_screen_info_rec present_info = {
+
+ bool sna_present_open(struct sna *sna, ScreenPtr screen)
+ {
++ DBG(("%s(num_crtc=%d)\n", __FUNCTION__, sna->mode.num_real_crtc));
++
+ if (sna->mode.num_real_crtc == 0)
+ return false;
+
+ sna_present_update(sna);
++ list_init(&sna->present.vblank_queue);
+
+ return present_screen_init(screen, &present_info);
+ }
+diff --git a/src/sna/sna_render.c b/src/sna/sna_render.c
+index 3fbb9ecb..3e935d57 100644
+--- a/src/sna/sna_render.c
++++ b/src/sna/sna_render.c
+@@ -54,7 +54,7 @@ sna_format_for_depth(int depth)
+ {
+ switch (depth) {
+ case 1: return PICT_a1;
+- case 4: return PICT_a4;
++ case 4: return PICT_x4a4;
+ case 8: return PICT_a8;
+ case 15: return PICT_x1r5g5b5;
+ case 16: return PICT_r5g6b5;
+@@ -272,18 +272,6 @@ no_render_context_switch(struct kgem *kgem,
+ }
+
+ static void
+-no_render_retire(struct kgem *kgem)
+-{
+- (void)kgem;
+-}
+-
+-static void
+-no_render_expire(struct kgem *kgem)
+-{
+- (void)kgem;
+-}
+-
+-static void
+ no_render_fini(struct sna *sna)
+ {
+ (void)sna;
+@@ -316,8 +304,6 @@ const char *no_render_init(struct sna *sna)
+ render->fini = no_render_fini;
+
+ sna->kgem.context_switch = no_render_context_switch;
+- sna->kgem.retire = no_render_retire;
+- sna->kgem.expire = no_render_expire;
+ if (sna->kgem.has_blt)
+ sna->kgem.ring = KGEM_BLT;
+
+@@ -407,10 +393,7 @@ use_cpu_bo(struct sna *sna, PixmapPtr pixmap, const BoxRec *box, bool blt)
+ }
+ }
+
+- if (priv->shm) {
+- assert(!priv->flush);
+- sna_add_flush_pixmap(sna, priv, priv->cpu_bo);
+- }
++ add_shm_flush(sna, priv);
+
+ DBG(("%s for box=(%d, %d), (%d, %d)\n",
+ __FUNCTION__, box->x1, box->y1, box->x2, box->y2));
+@@ -567,6 +550,7 @@ static struct kgem_bo *upload(struct sna *sna,
+ assert(priv->gpu_damage == NULL);
+ assert(priv->gpu_bo == NULL);
+ assert(bo->proxy != NULL);
++ sna_damage_all(&priv->cpu_damage, pixmap);
+ kgem_proxy_bo_attach(bo, &priv->gpu_bo);
+ }
+ }
+@@ -627,10 +611,7 @@ sna_render_pixmap_bo(struct sna *sna,
+ !priv->cpu_bo->snoop && priv->cpu_bo->pitch < 4096) {
+ DBG(("%s: CPU all damaged\n", __FUNCTION__));
+ channel->bo = priv->cpu_bo;
+- if (priv->shm) {
+- assert(!priv->flush);
+- sna_add_flush_pixmap(sna, priv, priv->cpu_bo);
+- }
++ add_shm_flush(sna, priv);
+ goto done;
+ }
+ }
+@@ -1275,6 +1256,7 @@ sna_render_picture_extract(struct sna *sna,
+ assert(priv->gpu_damage == NULL);
+ assert(priv->gpu_bo == NULL);
+ assert(bo->proxy != NULL);
++ sna_damage_all(&priv->cpu_damage, pixmap);
+ kgem_proxy_bo_attach(bo, &priv->gpu_bo);
+ }
+ }
+@@ -1338,6 +1320,8 @@ sna_render_picture_convolve(struct sna *sna,
+ */
+ DBG(("%s: origin=(%d,%d) kernel=%dx%d, size=%dx%d\n",
+ __FUNCTION__, x_off, y_off, cw, ch, w, h));
++ if (cw*ch > 32) /* too much loss of precision from quantization! */
++ return -1;
+
+ assert(picture->pDrawable);
+ assert(picture->filter == PictFilterConvolution);
+@@ -1388,9 +1372,9 @@ sna_render_picture_convolve(struct sna *sna,
+ alpha = CreateSolidPicture(0, &color, &error);
+ if (alpha) {
+ sna_composite(PictOpAdd, picture, alpha, tmp,
+- x, y,
++ x-(x_off+i), y-(y_off+j),
++ 0, 0,
+ 0, 0,
+- x_off+i, y_off+j,
+ w, h);
+ FreePicture(alpha, 0);
+ }
+@@ -2183,11 +2167,11 @@ copy_overlap(struct sna *sna, uint8_t alu,
+ ret = (sna->render.copy_boxes(sna, GXcopy,
+ draw, bo, src_dx, src_dy,
+ &tmp->drawable, tmp_bo, -extents->x1, -extents->y1,
+- box, n , 0) &&
++ box, n, 0) &&
+ sna->render.copy_boxes(sna, alu,
+ &tmp->drawable, tmp_bo, -extents->x1, -extents->y1,
+ draw, bo, dst_dx, dst_dy,
+- box, n , 0));
++ box, n, 0));
+
+ screen->DestroyPixmap(tmp);
+ return ret;
+@@ -2308,16 +2292,22 @@ static bool can_copy_cpu(struct sna *sna,
+ struct kgem_bo *src,
+ struct kgem_bo *dst)
+ {
+- if (src->tiling != dst->tiling)
+- return false;
++ DBG(("%s: tiling=%d:%d, pitch=%d:%d, can_map=%d:%d[%d]\n",
++ __FUNCTION__,
++ src->tiling, dst->tiling,
++ src->pitch, dst->pitch,
++ kgem_bo_can_map__cpu(&sna->kgem, src, false),
++ kgem_bo_can_map__cpu(&sna->kgem, dst, true),
++ sna->kgem.has_wc_mmap));
+
+- if (src->pitch != dst->pitch)
++ if (src->tiling != dst->tiling)
+ return false;
+
+ if (!kgem_bo_can_map__cpu(&sna->kgem, src, false))
+ return false;
+
+- if (!kgem_bo_can_map__cpu(&sna->kgem, dst, true))
++ if (!kgem_bo_can_map__cpu(&sna->kgem, dst, true) &&
++ !sna->kgem.has_wc_mmap)
+ return false;
+
+ DBG(("%s -- yes, src handle=%d, dst handle=%d\n", __FUNCTION__, src->handle, dst->handle));
+@@ -2330,31 +2320,62 @@ memcpy_copy_boxes(struct sna *sna, uint8_t op,
+ const DrawableRec *dst_draw, struct kgem_bo *dst_bo, int16_t dx, int16_t dy,
+ const BoxRec *box, int n, unsigned flags)
+ {
++ memcpy_box_func detile = NULL;
+ void *dst, *src;
+- bool clipped;
+
+ if (op != GXcopy)
+ return false;
+
+- clipped = (n > 1 ||
+- box->x1 + dx > 0 ||
+- box->y1 + dy > 0 ||
+- box->x2 + dx < dst_draw->width ||
+- box->y2 + dy < dst_draw->height);
++ if (src_draw->depth != dst_draw->depth)
++ return false;
+
+ dst = src = NULL;
+- if (!clipped && can_copy_cpu(sna, src_bo, dst_bo)) {
+- dst = kgem_bo_map__cpu(&sna->kgem, dst_bo);
++ if (can_copy_cpu(sna, src_bo, dst_bo)) {
++ if (src_bo->pitch != dst_bo->pitch ||
++ dx != sx || dy != sy || n > 1 ||
++ box->x1 + dx > 0 ||
++ box->y1 + dy > 0 ||
++ box->x2 + dx < dst_draw->width ||
++ box->y2 + dy < dst_draw->height) {
++ if (dx != sx) /* not implemented in memcpy yet */
++ goto use_gtt;
++
++ switch (dst_bo->tiling) {
++ default:
++ case I915_TILING_Y:
++ goto use_gtt;
++
++ case I915_TILING_X:
++ detile = sna->kgem.memcpy_between_tiled_x;
++ if (detile == NULL)
++ goto use_gtt;
++ break;
++
++ case I915_TILING_NONE:
++ break;
++ }
++ }
++
++ if (kgem_bo_can_map__cpu(&sna->kgem, dst_bo, true))
++ dst = kgem_bo_map__cpu(&sna->kgem, dst_bo);
++ else
++ dst = kgem_bo_map__wc(&sna->kgem, dst_bo);
+ src = kgem_bo_map__cpu(&sna->kgem, src_bo);
+ }
+
+ if (dst == NULL || src == NULL) {
++use_gtt:
+ dst = kgem_bo_map__gtt(&sna->kgem, dst_bo);
+ src = kgem_bo_map__gtt(&sna->kgem, src_bo);
+ if (dst == NULL || src == NULL)
+ return false;
++
++ detile = NULL;
+ } else {
+- kgem_bo_sync__cpu_full(&sna->kgem, dst_bo, true);
++ if (dst == dst_bo->map__wc)
++ kgem_bo_sync__gtt(&sna->kgem, dst_bo);
++ else
++ kgem_bo_sync__cpu_full(&sna->kgem, dst_bo, true);
+ kgem_bo_sync__cpu_full(&sna->kgem, src_bo, false);
+ }
+
+@@ -2362,7 +2383,16 @@ memcpy_copy_boxes(struct sna *sna, uint8_t op,
+ __FUNCTION__, sx, sy, dx, dy, n));
+
+ if (sigtrap_get() == 0) {
+- do {
++ if (detile) {
++ do {
++ detile(src, dst, dst_draw->bitsPerPixel,
++ src_bo->pitch, dst_bo->pitch,
++ box->x1 + sx, box->y1 + sy,
++ box->x1 + dx, box->y1 + dy,
++ box->x2 - box->x1, box->y2 - box->y1);
++ box++;
++ } while (--n);
++ } else do {
+ memcpy_blt(src, dst, dst_draw->bitsPerPixel,
+ src_bo->pitch, dst_bo->pitch,
+ box->x1 + sx, box->y1 + sy,
+@@ -2380,4 +2410,5 @@ void
+ sna_render_mark_wedged(struct sna *sna)
+ {
+ sna->render.copy_boxes = memcpy_copy_boxes;
++ sna->render.prefer_gpu = 0;
+ }
+diff --git a/src/sna/sna_render.h b/src/sna/sna_render.h
+index 6e1fa480..4ba345a7 100644
+--- a/src/sna/sna_render.h
++++ b/src/sna/sna_render.h
+@@ -148,6 +148,10 @@ struct sna_composite_op {
+ struct {
+ uint32_t flags;
+ } gen8;
++
++ struct {
++ uint32_t flags;
++ } gen9;
+ } u;
+
+ void *priv;
+@@ -238,8 +242,9 @@ struct sna_render {
+ int16_t w, int16_t h,
+ unsigned flags,
+ struct sna_composite_op *tmp);
+-#define COMPOSITE_PARTIAL 0x1
+-#define COMPOSITE_FALLBACK 0x80000000
++#define COMPOSITE_PARTIAL 0x1
++#define COMPOSITE_UPLOAD 0x40000000
++#define COMPOSITE_FALLBACK 0x80000000
+
+ bool (*check_composite_spans)(struct sna *sna, uint8_t op,
+ PicturePtr dst, PicturePtr src,
+@@ -286,6 +291,8 @@ struct sna_render {
+ #define COPY_LAST 0x1
+ #define COPY_SYNC 0x2
+ #define COPY_NO_OVERLAP 0x4
++#define COPY_SMALL 0x8
++#define COPY_DRI 0x10
+
+ bool (*copy)(struct sna *sna, uint8_t alu,
+ PixmapPtr src, struct kgem_bo *src_bo,
+@@ -481,6 +488,7 @@ enum {
+
+ GEN7_WM_KERNEL_VIDEO_PLANAR,
+ GEN7_WM_KERNEL_VIDEO_PACKED,
++ GEN7_WM_KERNEL_VIDEO_RGB,
+ GEN7_WM_KERNEL_COUNT
+ };
+
+@@ -533,12 +541,13 @@ enum {
+
+ GEN8_WM_KERNEL_VIDEO_PLANAR,
+ GEN8_WM_KERNEL_VIDEO_PACKED,
++ GEN8_WM_KERNEL_VIDEO_RGB,
+ GEN8_WM_KERNEL_COUNT
+ };
+
+ struct gen8_render_state {
+ unsigned gt;
+-
++ const struct gt_info *info;
+ struct kgem_bo *general_bo;
+
+ uint32_t vs_state;
+@@ -565,6 +574,58 @@ struct gen8_render_state {
+ bool emit_flush;
+ };
+
++enum {
++ GEN9_WM_KERNEL_NOMASK = 0,
++ GEN9_WM_KERNEL_NOMASK_P,
++
++ GEN9_WM_KERNEL_MASK,
++ GEN9_WM_KERNEL_MASK_P,
++
++ GEN9_WM_KERNEL_MASKCA,
++ GEN9_WM_KERNEL_MASKCA_P,
++
++ GEN9_WM_KERNEL_MASKSA,
++ GEN9_WM_KERNEL_MASKSA_P,
++
++ GEN9_WM_KERNEL_OPACITY,
++ GEN9_WM_KERNEL_OPACITY_P,
++
++ GEN9_WM_KERNEL_VIDEO_PLANAR,
++ GEN9_WM_KERNEL_VIDEO_PACKED,
++ GEN9_WM_KERNEL_VIDEO_RGB,
++ GEN9_WM_KERNEL_COUNT
++};
++
++struct gen9_render_state {
++ unsigned gt;
++ const struct gt_info *info;
++ struct kgem_bo *general_bo;
++
++ uint32_t vs_state;
++ uint32_t sf_state;
++ uint32_t sf_mask_state;
++ uint32_t wm_state;
++ uint32_t wm_kernel[GEN9_WM_KERNEL_COUNT][3];
++
++ uint32_t cc_blend;
++
++ uint32_t drawrect_offset;
++ uint32_t drawrect_limit;
++ uint32_t blend;
++ uint32_t samplers;
++ uint32_t kernel;
++
++ uint16_t num_sf_outputs;
++ uint16_t ve_id;
++ uint16_t last_primitive;
++ int16_t floats_per_vertex;
++ uint16_t surface_table;
++
++ bool needs_invariant;
++ bool emit_flush;
++ bool ve_dirty;
++};
++
+ struct sna_static_stream {
+ uint32_t size, used;
+ uint8_t *data;
+@@ -620,6 +681,7 @@ const char *gen5_render_init(struct sna *sna, const char *backend);
+ const char *gen6_render_init(struct sna *sna, const char *backend);
+ const char *gen7_render_init(struct sna *sna, const char *backend);
+ const char *gen8_render_init(struct sna *sna, const char *backend);
++const char *gen9_render_init(struct sna *sna, const char *backend);
+
+ void sna_render_mark_wedged(struct sna *sna);
+
+diff --git a/src/sna/sna_render_inline.h b/src/sna/sna_render_inline.h
+index 10fbbfe2..e162e37f 100644
+--- a/src/sna/sna_render_inline.h
++++ b/src/sna/sna_render_inline.h
+@@ -304,6 +304,12 @@ color_convert(uint32_t pixel,
+ return pixel;
+ }
+
++inline static uint32_t
++solid_color(uint32_t format, uint32_t pixel)
++{
++ return color_convert(pixel, format, PICT_a8r8g8b8);
++}
++
+ inline static bool dst_use_gpu(PixmapPtr pixmap)
+ {
+ struct sna_pixmap *priv = sna_pixmap(pixmap);
+diff --git a/src/sna/sna_tiling.c b/src/sna/sna_tiling.c
+index 308efc0a..8e2627f7 100644
+--- a/src/sna/sna_tiling.c
++++ b/src/sna/sna_tiling.c
+@@ -369,8 +369,7 @@ sna_tiling_composite_spans_boxes(struct sna *sna,
+ const BoxRec *box, int nbox, float opacity)
+ {
+ while (nbox--)
+- sna_tiling_composite_spans_box(sna, op->base.priv, box++, opacity);
+- (void)sna;
++ sna_tiling_composite_spans_box(sna, op, box++, opacity);
+ }
+
+ fastcall static void
+@@ -581,6 +580,7 @@ sna_tiling_composite_spans(uint32_t op,
+ tile->rects = tile->rects_embedded;
+ tile->rect_count = 0;
+ tile->rect_size = ARRAY_SIZE(tile->rects_embedded);
++ COMPILE_TIME_ASSERT(sizeof(tile->rects_embedded[0]) >= sizeof(struct sna_tile_span));
+
+ tmp->box = sna_tiling_composite_spans_box;
+ tmp->boxes = sna_tiling_composite_spans_boxes;
+diff --git a/src/sna/sna_trapezoids_boxes.c b/src/sna/sna_trapezoids_boxes.c
+index 9900e3f0..bbf83759 100644
+--- a/src/sna/sna_trapezoids_boxes.c
++++ b/src/sna/sna_trapezoids_boxes.c
+@@ -198,7 +198,7 @@ composite_aligned_boxes(struct sna *sna,
+ if (op == PictOpClear && sna->clear)
+ src = sna->clear;
+
+- DBG(("%s: clipped extents (%d, %d), (%d, %d); now offset by (%d, %d), orgin (%d, %d)\n",
++ DBG(("%s: clipped extents (%d, %d), (%d, %d); now offset by (%d, %d), origin (%d, %d)\n",
+ __FUNCTION__,
+ clip.extents.x1, clip.extents.y1,
+ clip.extents.x2, clip.extents.y2,
+@@ -592,6 +592,8 @@ lerp32_opacity(PixmapPtr scratch,
+ uint32_t *ptr;
+ int stride, i;
+
++ sigtrap_assert_active();
++
+ ptr = (uint32_t*)((uint8_t *)scratch->devPrivate.ptr + scratch->devKind * y);
+ ptr += x;
+ stride = scratch->devKind / 4;
+diff --git a/src/sna/sna_trapezoids_imprecise.c b/src/sna/sna_trapezoids_imprecise.c
+index 37def2f9..8bc7c8a8 100644
+--- a/src/sna/sna_trapezoids_imprecise.c
++++ b/src/sna/sna_trapezoids_imprecise.c
+@@ -962,6 +962,16 @@ tor_add_trapezoid(struct tor *tor,
+ const xTrapezoid *t,
+ int dx, int dy)
+ {
++ if (!xTrapezoidValid(t)) {
++ __DBG(("%s: skipping invalid trapezoid: top=%d, bottom=%d, left=(%d, %d), (%d, %d), right=(%d, %d), (%d, %d)\n",
++ __FUNCTION__,
++ t->top, t->bottom,
++ t->left.p1.x, t->left.p1.y,
++ t->left.p2.x, t->left.p2.y,
++ t->right.p1.x, t->right.p1.y,
++ t->right.p2.x, t->right.p2.y));
++ return;
++ }
+ polygon_add_edge(tor->polygon, t, &t->left, 1, dx, dy);
+ polygon_add_edge(tor->polygon, t, &t->right, -1, dx, dy);
+ }
+@@ -1687,31 +1697,27 @@ struct span_thread {
+ #define SPAN_THREAD_MAX_BOXES (8192/sizeof(struct sna_opacity_box))
+ struct span_thread_boxes {
+ const struct sna_composite_spans_op *op;
++ const BoxRec *clip_start, *clip_end;
+ int num_boxes;
+ struct sna_opacity_box boxes[SPAN_THREAD_MAX_BOXES];
+ };
+
+-static void span_thread_add_boxes(struct sna *sna, void *data,
+- const BoxRec *box, int count, float alpha)
++static void span_thread_add_box(struct sna *sna, void *data,
++ const BoxRec *box, float alpha)
+ {
+ struct span_thread_boxes *b = data;
+
+- __DBG(("%s: adding %d boxes with alpha=%f\n",
+- __FUNCTION__, count, alpha));
++ __DBG(("%s: adding box with alpha=%f\n", __FUNCTION__, alpha));
+
+- assert(count > 0 && count <= SPAN_THREAD_MAX_BOXES);
+- if (unlikely(b->num_boxes + count > SPAN_THREAD_MAX_BOXES)) {
+- DBG(("%s: flushing %d boxes, adding %d\n", __FUNCTION__, b->num_boxes, count));
+- assert(b->num_boxes <= SPAN_THREAD_MAX_BOXES);
++ if (unlikely(b->num_boxes == SPAN_THREAD_MAX_BOXES)) {
++ DBG(("%s: flushing %d boxes\n", __FUNCTION__, b->num_boxes));
+ b->op->thread_boxes(sna, b->op, b->boxes, b->num_boxes);
+ b->num_boxes = 0;
+ }
+
+- do {
+- b->boxes[b->num_boxes].box = *box++;
+- b->boxes[b->num_boxes].alpha = alpha;
+- b->num_boxes++;
+- } while (--count);
++ b->boxes[b->num_boxes].box = *box++;
++ b->boxes[b->num_boxes].alpha = alpha;
++ b->num_boxes++;
+ assert(b->num_boxes <= SPAN_THREAD_MAX_BOXES);
+ }
+
+@@ -1722,8 +1728,22 @@ span_thread_box(struct sna *sna,
+ const BoxRec *box,
+ int coverage)
+ {
++ struct span_thread_boxes *b = (struct span_thread_boxes *)op;
++
+ __DBG(("%s: %d -> %d @ %d\n", __FUNCTION__, box->x1, box->x2, coverage));
+- span_thread_add_boxes(sna, op, box, 1, AREA_TO_ALPHA(coverage));
++ if (b->num_boxes) {
++ struct sna_opacity_box *bb = &b->boxes[b->num_boxes-1];
++ if (bb->box.x1 == box->x1 &&
++ bb->box.x2 == box->x2 &&
++ bb->box.y2 == box->y1 &&
++ bb->alpha == AREA_TO_ALPHA(coverage)) {
++ bb->box.y2 = box->y2;
++ __DBG(("%s: contracted double row: %d -> %d\n", __func__, bb->box.y1, bb->box.y2));
++ return;
++ }
++ }
++
++ span_thread_add_box(sna, op, box, AREA_TO_ALPHA(coverage));
+ }
+
+ static void
+@@ -1733,20 +1753,28 @@ span_thread_clipped_box(struct sna *sna,
+ const BoxRec *box,
+ int coverage)
+ {
+- pixman_region16_t region;
++ struct span_thread_boxes *b = (struct span_thread_boxes *)op;
++ const BoxRec *c;
+
+ __DBG(("%s: %d -> %d @ %f\n", __FUNCTION__, box->x1, box->x2,
+ AREA_TO_ALPHA(coverage)));
+
+- pixman_region_init_rects(&region, box, 1);
+- RegionIntersect(&region, &region, clip);
+- if (region_num_rects(&region)) {
+- span_thread_add_boxes(sna, op,
+- region_rects(&region),
+- region_num_rects(&region),
+- AREA_TO_ALPHA(coverage));
++ b->clip_start =
++ find_clip_box_for_y(b->clip_start, b->clip_end, box->y1);
++
++ c = b->clip_start;
++ while (c != b->clip_end) {
++ BoxRec clipped;
++
++ if (box->y2 <= c->y1)
++ break;
++
++ clipped = *box;
++ if (!box_intersect(&clipped, c++))
++ continue;
++
++ span_thread_add_box(sna, op, &clipped, AREA_TO_ALPHA(coverage));
+ }
+- pixman_region_fini(&region);
+ }
+
+ static span_func_t
+@@ -1777,6 +1805,16 @@ thread_choose_span(struct sna_composite_spans_op *tmp,
+ return span;
+ }
+
++inline static void
++span_thread_boxes_init(struct span_thread_boxes *boxes,
++ const struct sna_composite_spans_op *op,
++ const RegionRec *clip)
++{
++ boxes->op = op;
++ region_get_boxes(clip, &boxes->clip_start, &boxes->clip_end);
++ boxes->num_boxes = 0;
++}
++
+ static void
+ span_thread(void *arg)
+ {
+@@ -1789,8 +1827,7 @@ span_thread(void *arg)
+ if (!tor_init(&tor, &thread->extents, 2*thread->ntrap))
+ return;
+
+- boxes.op = thread->op;
+- boxes.num_boxes = 0;
++ span_thread_boxes_init(&boxes, thread->op, thread->clip);
+
+ y1 = thread->extents.y1 - thread->draw_y;
+ y2 = thread->extents.y2 - thread->draw_y;
+@@ -2190,6 +2227,52 @@ static void _tor_blt_src(struct inplace *in, const BoxRec *box, uint8_t v)
+ } while (--h);
+ }
+
++struct clipped_span {
++ span_func_t span;
++ const BoxRec *clip_start, *clip_end;
++};
++
++static void
++tor_blt_clipped(struct sna *sna,
++ struct sna_composite_spans_op *op,
++ pixman_region16_t *clip,
++ const BoxRec *box,
++ int coverage)
++{
++ struct clipped_span *cs = (struct clipped_span *)clip;
++ const BoxRec *c;
++
++ cs->clip_start =
++ find_clip_box_for_y(cs->clip_start, cs->clip_end, box->y1);
++
++ c = cs->clip_start;
++ while (c != cs->clip_end) {
++ BoxRec clipped;
++
++ if (box->y2 <= c->y1)
++ break;
++
++ clipped = *box;
++ if (!box_intersect(&clipped, c++))
++ continue;
++
++ cs->span(sna, op, NULL, &clipped, coverage);
++ }
++}
++
++inline static span_func_t
++clipped_span(struct clipped_span *cs,
++ span_func_t span,
++ const RegionRec *clip)
++{
++ if (clip->data) {
++ cs->span = span;
++ region_get_boxes(clip, &cs->clip_start, &cs->clip_end);
++ span = tor_blt_clipped;
++ }
++ return span;
++}
++
+ static void
+ tor_blt_src(struct sna *sna,
+ struct sna_composite_spans_op *op,
+@@ -2203,25 +2286,6 @@ tor_blt_src(struct sna *sna,
+ }
+
+ static void
+-tor_blt_src_clipped(struct sna *sna,
+- struct sna_composite_spans_op *op,
+- pixman_region16_t *clip,
+- const BoxRec *box,
+- int coverage)
+-{
+- pixman_region16_t region;
+- int n;
+-
+- pixman_region_init_rects(&region, box, 1);
+- RegionIntersect(&region, &region, clip);
+- n = region_num_rects(&region);
+- box = region_rects(&region);
+- while (n--)
+- tor_blt_src(sna, op, NULL, box++, coverage);
+- pixman_region_fini(&region);
+-}
+-
+-static void
+ tor_blt_in(struct sna *sna,
+ struct sna_composite_spans_op *op,
+ pixman_region16_t *clip,
+@@ -2253,25 +2317,6 @@ tor_blt_in(struct sna *sna,
+ }
+
+ static void
+-tor_blt_in_clipped(struct sna *sna,
+- struct sna_composite_spans_op *op,
+- pixman_region16_t *clip,
+- const BoxRec *box,
+- int coverage)
+-{
+- pixman_region16_t region;
+- int n;
+-
+- pixman_region_init_rects(&region, box, 1);
+- RegionIntersect(&region, &region, clip);
+- n = region_num_rects(&region);
+- box = region_rects(&region);
+- while (n--)
+- tor_blt_in(sna, op, NULL, box++, coverage);
+- pixman_region_fini(&region);
+-}
+-
+-static void
+ tor_blt_add(struct sna *sna,
+ struct sna_composite_spans_op *op,
+ pixman_region16_t *clip,
+@@ -2310,25 +2355,6 @@ tor_blt_add(struct sna *sna,
+ }
+
+ static void
+-tor_blt_add_clipped(struct sna *sna,
+- struct sna_composite_spans_op *op,
+- pixman_region16_t *clip,
+- const BoxRec *box,
+- int coverage)
+-{
+- pixman_region16_t region;
+- int n;
+-
+- pixman_region_init_rects(&region, box, 1);
+- RegionIntersect(&region, &region, clip);
+- n = region_num_rects(&region);
+- box = region_rects(&region);
+- while (n--)
+- tor_blt_add(sna, op, NULL, box++, coverage);
+- pixman_region_fini(&region);
+-}
+-
+-static void
+ tor_blt_lerp32(struct sna *sna,
+ struct sna_composite_spans_op *op,
+ pixman_region16_t *clip,
+@@ -2343,6 +2369,7 @@ tor_blt_lerp32(struct sna *sna,
+ if (coverage == 0)
+ return;
+
++ sigtrap_assert_active();
+ ptr += box->y1 * stride + box->x1;
+
+ h = box->y2 - box->y1;
+@@ -2383,25 +2410,6 @@ tor_blt_lerp32(struct sna *sna,
+ }
+ }
+
+-static void
+-tor_blt_lerp32_clipped(struct sna *sna,
+- struct sna_composite_spans_op *op,
+- pixman_region16_t *clip,
+- const BoxRec *box,
+- int coverage)
+-{
+- pixman_region16_t region;
+- int n;
+-
+- pixman_region_init_rects(&region, box, 1);
+- RegionIntersect(&region, &region, clip);
+- n = region_num_rects(&region);
+- box = region_rects(&region);
+- while (n--)
+- tor_blt_lerp32(sna, op, NULL, box++, coverage);
+- pixman_region_fini(&region);
+-}
+-
+ struct pixman_inplace {
+ pixman_image_t *image, *source, *mask;
+ uint32_t color;
+@@ -2431,24 +2439,6 @@ pixmask_span_solid(struct sna *sna,
+ pi->dx + box->x1, pi->dy + box->y1,
+ box->x2 - box->x1, box->y2 - box->y1);
+ }
+-static void
+-pixmask_span_solid__clipped(struct sna *sna,
+- struct sna_composite_spans_op *op,
+- pixman_region16_t *clip,
+- const BoxRec *box,
+- int coverage)
+-{
+- pixman_region16_t region;
+- int n;
+-
+- pixman_region_init_rects(&region, box, 1);
+- RegionIntersect(&region, &region, clip);
+- n = region_num_rects(&region);
+- box = region_rects(&region);
+- while (n--)
+- pixmask_span_solid(sna, op, NULL, box++, coverage);
+- pixman_region_fini(&region);
+-}
+
+ static void
+ pixmask_span(struct sna *sna,
+@@ -2471,24 +2461,6 @@ pixmask_span(struct sna *sna,
+ pi->dx + box->x1, pi->dy + box->y1,
+ box->x2 - box->x1, box->y2 - box->y1);
+ }
+-static void
+-pixmask_span__clipped(struct sna *sna,
+- struct sna_composite_spans_op *op,
+- pixman_region16_t *clip,
+- const BoxRec *box,
+- int coverage)
+-{
+- pixman_region16_t region;
+- int n;
+-
+- pixman_region_init_rects(&region, box, 1);
+- RegionIntersect(&region, &region, clip);
+- n = region_num_rects(&region);
+- box = region_rects(&region);
+- while (n--)
+- pixmask_span(sna, op, NULL, box++, coverage);
+- pixman_region_fini(&region);
+-}
+
+ struct inplace_x8r8g8b8_thread {
+ xTrapezoid *traps;
+@@ -2507,6 +2479,7 @@ static void inplace_x8r8g8b8_thread(void *arg)
+ struct inplace_x8r8g8b8_thread *thread = arg;
+ struct tor tor;
+ span_func_t span;
++ struct clipped_span clipped;
+ RegionPtr clip;
+ int y1, y2, n;
+
+@@ -2537,12 +2510,11 @@ static void inplace_x8r8g8b8_thread(void *arg)
+ inplace.stride = pixmap->devKind;
+ inplace.color = thread->color;
+
+- if (clip->data)
+- span = tor_blt_lerp32_clipped;
+- else
+- span = tor_blt_lerp32;
++ span = clipped_span(&clipped, tor_blt_lerp32, clip);
+
+- tor_render(NULL, &tor, (void*)&inplace, clip, span, false);
++ tor_render(NULL, &tor,
++ (void*)&inplace, (void*)&clipped,
++ span, false);
+ } else if (thread->is_solid) {
+ struct pixman_inplace pi;
+
+@@ -2555,12 +2527,11 @@ static void inplace_x8r8g8b8_thread(void *arg)
+ 1, 1, pi.bits, 0);
+ pixman_image_set_repeat(pi.source, PIXMAN_REPEAT_NORMAL);
+
+- if (clip->data)
+- span = pixmask_span_solid__clipped;
+- else
+- span = pixmask_span_solid;
++ span = clipped_span(&clipped, pixmask_span_solid, clip);
+
+- tor_render(NULL, &tor, (void*)&pi, clip, span, false);
++ tor_render(NULL, &tor,
++ (void*)&pi, (void *)&clipped,
++ span, false);
+
+ pixman_image_unref(pi.source);
+ pixman_image_unref(pi.image);
+@@ -2579,12 +2550,11 @@ static void inplace_x8r8g8b8_thread(void *arg)
+ pi.bits = pixman_image_get_data(pi.mask);
+ pi.op = thread->op;
+
+- if (clip->data)
+- span = pixmask_span__clipped;
+- else
+- span = pixmask_span;
++ span = clipped_span(&clipped, pixmask_span, clip);
+
+- tor_render(NULL, &tor, (void*)&pi, clip, span, false);
++ tor_render(NULL, &tor,
++ (void*)&pi, (void *)&clipped,
++ span, false);
+
+ pixman_image_unref(pi.mask);
+ pixman_image_unref(pi.source);
+@@ -2698,6 +2668,7 @@ trapezoid_span_inplace__x8r8g8b8(CARD8 op,
+ if (num_threads == 1) {
+ struct tor tor;
+ span_func_t span;
++ struct clipped_span clipped;
+
+ if (!tor_init(&tor, &region.extents, 2*ntrap))
+ return true;
+@@ -2723,17 +2694,15 @@ trapezoid_span_inplace__x8r8g8b8(CARD8 op,
+ inplace.stride = pixmap->devKind;
+ inplace.color = color;
+
+- if (dst->pCompositeClip->data)
+- span = tor_blt_lerp32_clipped;
+- else
+- span = tor_blt_lerp32;
++ span = clipped_span(&clipped, tor_blt_lerp32, dst->pCompositeClip);
+
+ DBG(("%s: render inplace op=%d, color=%08x\n",
+ __FUNCTION__, op, color));
+
+ if (sigtrap_get() == 0) {
+- tor_render(NULL, &tor, (void*)&inplace,
+- dst->pCompositeClip, span, false);
++ tor_render(NULL, &tor,
++ (void*)&inplace, (void*)&clipped,
++ span, false);
+ sigtrap_put();
+ }
+ } else if (is_solid) {
+@@ -2748,15 +2717,12 @@ trapezoid_span_inplace__x8r8g8b8(CARD8 op,
+ 1, 1, pi.bits, 0);
+ pixman_image_set_repeat(pi.source, PIXMAN_REPEAT_NORMAL);
+
+- if (dst->pCompositeClip->data)
+- span = pixmask_span_solid__clipped;
+- else
+- span = pixmask_span_solid;
++ span = clipped_span(&clipped, pixmask_span_solid, dst->pCompositeClip);
+
+ if (sigtrap_get() == 0) {
+- tor_render(NULL, &tor, (void*)&pi,
+- dst->pCompositeClip, span,
+- false);
++ tor_render(NULL, &tor,
++ (void*)&pi, (void*)&clipped,
++ span, false);
+ sigtrap_put();
+ }
+
+@@ -2777,15 +2743,12 @@ trapezoid_span_inplace__x8r8g8b8(CARD8 op,
+ pi.bits = pixman_image_get_data(pi.mask);
+ pi.op = op;
+
+- if (dst->pCompositeClip->data)
+- span = pixmask_span__clipped;
+- else
+- span = pixmask_span;
++ span = clipped_span(&clipped, pixmask_span, dst->pCompositeClip);
+
+ if (sigtrap_get() == 0) {
+- tor_render(NULL, &tor, (void*)&pi,
+- dst->pCompositeClip, span,
+- false);
++ tor_render(NULL, &tor,
++ (void*)&pi, (void*)&clipped,
++ span, false);
+ sigtrap_put();
+ }
+
+@@ -2847,9 +2810,9 @@ trapezoid_span_inplace__x8r8g8b8(CARD8 op,
+
+ struct inplace_thread {
+ xTrapezoid *traps;
+- RegionPtr clip;
+ span_func_t span;
+ struct inplace inplace;
++ struct clipped_span clipped;
+ BoxRec extents;
+ int dx, dy;
+ int draw_x, draw_y;
+@@ -2874,8 +2837,9 @@ static void inplace_thread(void *arg)
+ tor_add_trapezoid(&tor, &thread->traps[n], thread->dx, thread->dy);
+ }
+
+- tor_render(NULL, &tor, (void*)&thread->inplace,
+- thread->clip, thread->span, thread->unbounded);
++ tor_render(NULL, &tor,
++ (void*)&thread->inplace, (void*)&thread->clipped,
++ thread->span, thread->unbounded);
+
+ tor_fini(&tor);
+ }
+@@ -2889,6 +2853,7 @@ imprecise_trapezoid_span_inplace(struct sna *sna,
+ bool fallback)
+ {
+ struct inplace inplace;
++ struct clipped_span clipped;
+ span_func_t span;
+ PixmapPtr pixmap;
+ struct sna_pixmap *priv;
+@@ -3005,21 +2970,12 @@ imprecise_trapezoid_span_inplace(struct sna *sna,
+ region.extents.x2, region.extents.y2));
+
+ if (op == PictOpSrc) {
+- if (dst->pCompositeClip->data)
+- span = tor_blt_src_clipped;
+- else
+- span = tor_blt_src;
++ span = tor_blt_src;
+ } else if (op == PictOpIn) {
+- if (dst->pCompositeClip->data)
+- span = tor_blt_in_clipped;
+- else
+- span = tor_blt_in;
++ span = tor_blt_in;
+ } else {
+ assert(op == PictOpAdd);
+- if (dst->pCompositeClip->data)
+- span = tor_blt_add_clipped;
+- else
+- span = tor_blt_add;
++ span = tor_blt_add;
+ }
+
+ DBG(("%s: move-to-cpu\n", __FUNCTION__));
+@@ -3037,6 +2993,8 @@ imprecise_trapezoid_span_inplace(struct sna *sna,
+ inplace.stride = pixmap->devKind;
+ inplace.opacity = color >> 24;
+
++ span = clipped_span(&clipped, span, dst->pCompositeClip);
++
+ num_threads = 1;
+ if ((flags & COMPOSITE_SPANS_RECTILINEAR) == 0)
+ num_threads = sna_use_threads(region.extents.x2 - region.extents.x1,
+@@ -3057,8 +3015,9 @@ imprecise_trapezoid_span_inplace(struct sna *sna,
+ }
+
+ if (sigtrap_get() == 0) {
+- tor_render(NULL, &tor, (void*)&inplace,
+- dst->pCompositeClip, span, unbounded);
++ tor_render(NULL, &tor,
++ (void*)&inplace, (void *)&clipped,
++ span, unbounded);
+ sigtrap_put();
+ }
+
+@@ -3075,8 +3034,8 @@ imprecise_trapezoid_span_inplace(struct sna *sna,
+ threads[0].traps = traps;
+ threads[0].ntrap = ntrap;
+ threads[0].inplace = inplace;
++ threads[0].clipped = clipped;
+ threads[0].extents = region.extents;
+- threads[0].clip = dst->pCompositeClip;
+ threads[0].span = span;
+ threads[0].unbounded = unbounded;
+ threads[0].dx = dx;
+@@ -3707,8 +3666,7 @@ tristrip_thread(void *arg)
+ if (!tor_init(&tor, &thread->extents, 2*thread->count))
+ return;
+
+- boxes.op = thread->op;
+- boxes.num_boxes = 0;
++ span_thread_boxes_init(&boxes, thread->op, thread->clip);
+
+ cw = 0; ccw = 1;
+ polygon_add_line(tor.polygon,
+@@ -3874,7 +3832,7 @@ imprecise_tristrip_span_converter(struct sna *sna,
+ break;
+ } while (1);
+ polygon_add_line(tor.polygon,
+- &points[cw], &points[2+ccw],
++ &points[cw], &points[ccw],
+ dx, dy);
+ assert(tor.polygon->num_edges <= 2*count);
+
+diff --git a/src/sna/sna_trapezoids_mono.c b/src/sna/sna_trapezoids_mono.c
+index 808703a9..07a7867d 100644
+--- a/src/sna/sna_trapezoids_mono.c
++++ b/src/sna/sna_trapezoids_mono.c
+@@ -72,13 +72,14 @@ struct mono {
+ struct sna *sna;
+ struct sna_composite_op op;
+ pixman_region16_t clip;
++ const BoxRec *clip_start, *clip_end;
+
+ fastcall void (*span)(struct mono *, int, int, BoxPtr);
+
+ struct mono_polygon polygon;
+ };
+
+-#define I(x) pixman_fixed_to_int ((x) + pixman_fixed_1_minus_e/2)
++#define I(x) pixman_fixed_to_int((x) + pixman_fixed_1_minus_e/2)
+
+ static struct quorem
+ floored_muldivrem(int32_t x, int32_t a, int32_t b)
+@@ -249,22 +250,22 @@ mono_add_line(struct mono *mono,
+
+ e->dxdy = floored_muldivrem(dx, pixman_fixed_1, dy);
+
+- e->x = floored_muldivrem((ytop - dst_y) * pixman_fixed_1 + pixman_fixed_1_minus_e/2 - p1->y,
++ e->x = floored_muldivrem((ytop - dst_y) * pixman_fixed_1 + pixman_fixed_1/2 - p1->y,
+ dx, dy);
+ e->x.quo += p1->x;
+ e->x.rem -= dy;
+
+ e->dy = dy;
+-
+- __DBG(("%s: initial x=%d [%d.%d/%d] + dxdy=%d.%d/%d\n",
+- __FUNCTION__,
+- I(e->x.quo), e->x.quo, e->x.rem, e->dy,
+- e->dxdy.quo, e->dxdy.rem, e->dy));
+ }
+ e->x.quo += dst_x*pixman_fixed_1;
++ __DBG(("%s: initial x=%d [%d.%d/%d] + dxdy=%d.%d/%d\n",
++ __FUNCTION__,
++ I(e->x.quo), e->x.quo, e->x.rem, e->dy,
++ e->dxdy.quo, e->dxdy.rem, e->dy));
+
+ {
+ struct mono_edge **ptail = &polygon->y_buckets[ytop - mono->clip.extents.y1];
++ assert(ytop - mono->clip.extents.y1 < mono->clip.extents.y2 - mono->clip.extents.y1);
+ if (*ptail)
+ (*ptail)->prev = e;
+ e->next = *ptail;
+@@ -368,6 +369,10 @@ static struct mono_edge *mono_filter(struct mono_edge *edges)
+ e->x.rem == n->x.rem &&
+ e->dxdy.quo == n->dxdy.quo &&
+ e->dxdy.rem == n->dxdy.rem) {
++ assert(e->dy == n->dy);
++ __DBG(("%s: discarding cancellation pair (%d.%d) + (%d.%d)\n",
++ __FUNCTION__, e->x.quo, e->x.rem, e->dxdy.quo, e->dxdy.rem));
++
+ if (e->prev)
+ e->prev->next = n->next;
+ else
+@@ -378,8 +383,11 @@ static struct mono_edge *mono_filter(struct mono_edge *edges)
+ break;
+
+ e = n->next;
+- } else
++ } else {
++ __DBG(("%s: adding edge (%d.%d) + (%d.%d)/%d, height=%d\n",
++ __FUNCTION__, n->x.quo, n->x.rem, n->dxdy.quo, n->dxdy.rem, n->dy, n->height_left));
+ e = n;
++ }
+ }
+
+ return edges;
+@@ -474,6 +482,34 @@ mono_span__fast(struct mono *c, int x1, int x2, BoxPtr box)
+ c->op.box(c->sna, &c->op, box);
+ }
+
++fastcall static void
++mono_span__clipped(struct mono *c, int x1, int x2, BoxPtr box)
++{
++ const BoxRec *b;
++
++ __DBG(("%s [%d, %d]\n", __FUNCTION__, x1, x2));
++
++ c->clip_start =
++ find_clip_box_for_y(c->clip_start, c->clip_end, box->y1);
++
++ b = c->clip_start;
++ while (b != c->clip_end) {
++ BoxRec clipped;
++
++ if (box->y2 <= b->y1)
++ break;
++
++ clipped.x1 = x1;
++ clipped.x2 = x2;
++ clipped.y1 = box->y1;
++ clipped.y2 = box->y2;
++ if (!box_intersect(&clipped, b++))
++ continue;
++
++ c->op.box(c->sna, &c->op, &clipped);
++ }
++}
++
+ struct mono_span_thread_boxes {
+ const struct sna_composite_op *op;
+ #define MONO_SPAN_MAX_BOXES (8192/sizeof(BoxRec))
+@@ -482,40 +518,45 @@ struct mono_span_thread_boxes {
+ };
+
+ inline static void
+-thread_mono_span_add_boxes(struct mono *c, const BoxRec *box, int count)
++thread_mono_span_add_box(struct mono *c, const BoxRec *box)
+ {
+ struct mono_span_thread_boxes *b = c->op.priv;
+
+- assert(count > 0 && count <= MONO_SPAN_MAX_BOXES);
+- if (unlikely(b->num_boxes + count > MONO_SPAN_MAX_BOXES)) {
++ if (unlikely(b->num_boxes == MONO_SPAN_MAX_BOXES)) {
+ b->op->thread_boxes(c->sna, b->op, b->boxes, b->num_boxes);
+ b->num_boxes = 0;
+ }
+
+- memcpy(b->boxes + b->num_boxes, box, count*sizeof(BoxRec));
+- b->num_boxes += count;
++ b->boxes[b->num_boxes++] = *box;
+ assert(b->num_boxes <= MONO_SPAN_MAX_BOXES);
+ }
+
+ fastcall static void
+ thread_mono_span_clipped(struct mono *c, int x1, int x2, BoxPtr box)
+ {
+- pixman_region16_t region;
++ const BoxRec *b;
+
+ __DBG(("%s [%d, %d]\n", __FUNCTION__, x1, x2));
+
+- box->x1 = x1;
+- box->x2 = x2;
++ c->clip_start =
++ find_clip_box_for_y(c->clip_start, c->clip_end, box->y1);
+
+- assert(c->clip.data);
++ b = c->clip_start;
++ while (b != c->clip_end) {
++ BoxRec clipped;
++
++ if (box->y2 <= b->y1)
++ break;
++
++ clipped.x1 = x1;
++ clipped.x2 = x2;
++ clipped.y1 = box->y1;
++ clipped.y2 = box->y2;
++ if (!box_intersect(&clipped, b++))
++ continue;
+
+- pixman_region_init_rects(&region, box, 1);
+- RegionIntersect(&region, &region, &c->clip);
+- if (region_num_rects(&region))
+- thread_mono_span_add_boxes(c,
+- region_rects(&region),
+- region_num_rects(&region));
+- pixman_region_fini(&region);
++ thread_mono_span_add_box(c, &clipped);
++ }
+ }
+
+ fastcall static void
+@@ -525,7 +566,7 @@ thread_mono_span(struct mono *c, int x1, int x2, BoxPtr box)
+
+ box->x1 = x1;
+ box->x2 = x2;
+- thread_mono_span_add_boxes(c, box, 1);
++ thread_mono_span_add_box(c, box);
+ }
+
+ inline static void
+@@ -537,6 +578,8 @@ mono_row(struct mono *c, int16_t y, int16_t h)
+ int winding = 0;
+ BoxRec box;
+
++ __DBG(("%s: y=%d, h=%d\n", __FUNCTION__, y, h));
++
+ DBG_MONO_EDGES(edge);
+ VALIDATE_MONO_EDGES(&c->head);
+
+@@ -547,6 +590,8 @@ mono_row(struct mono *c, int16_t y, int16_t h)
+ struct mono_edge *next = edge->next;
+ int16_t xend = I(edge->x.quo);
+
++ __DBG(("%s: adding edge dir=%d [winding=%d], x=%d [%d]\n",
++ __FUNCTION__, edge->dir, winding + edge->dir, xend, edge->x.quo));
+ if (--edge->height_left) {
+ if (edge->dy) {
+ edge->x.quo += edge->dxdy.quo;
+@@ -555,6 +600,8 @@ mono_row(struct mono *c, int16_t y, int16_t h)
+ ++edge->x.quo;
+ edge->x.rem -= edge->dy;
+ }
++ __DBG(("%s: stepped edge (%d.%d) + (%d.%d)/%d, height=%d, prev_x=%d\n",
++ __FUNCTION__, edge->x.quo, edge->x.rem, edge->dxdy.quo, edge->dxdy.rem, edge->dy, edge->height_left, edge->x.quo));
+ }
+
+ if (edge->x.quo < prev_x) {
+@@ -578,17 +625,22 @@ mono_row(struct mono *c, int16_t y, int16_t h)
+ winding += edge->dir;
+ if (winding == 0) {
+ assert(I(next->x.quo) >= xend);
+- if (I(next->x.quo) > xend + 1) {
++ if (I(next->x.quo) > xend) {
++ __DBG(("%s: end span: %d\n", __FUNCTION__, xend));
+ if (xstart < c->clip.extents.x1)
+ xstart = c->clip.extents.x1;
+ if (xend > c->clip.extents.x2)
+ xend = c->clip.extents.x2;
+- if (xend > xstart)
++ if (xend > xstart) {
++ __DBG(("%s: emit span [%d, %d]\n", __FUNCTION__, xstart, xend));
+ c->span(c, xstart, xend, &box);
++ }
+ xstart = INT16_MIN;
+ }
+- } else if (xstart == INT16_MIN)
++ } else if (xstart == INT16_MIN) {
++ __DBG(("%s: starting new span: %d\n", __FUNCTION__, xend));
+ xstart = xend;
++ }
+
+ edge = next;
+ }
+@@ -650,9 +702,14 @@ mono_render(struct mono *mono)
+ for (i = 0; i < h; i = j) {
+ j = i + 1;
+
++ __DBG(("%s: row=%d, new edges? %d\n", __FUNCTION__,
++ i, polygon->y_buckets[i] != NULL));
++
+ if (polygon->y_buckets[i])
+ mono_merge_edges(mono, polygon->y_buckets[i]);
+
++ __DBG(("%s: row=%d, vertical? %d\n", __FUNCTION__,
++ i, mono->is_vertical));
+ if (mono->is_vertical) {
+ struct mono_edge *e = mono->head.next;
+ int min_height = h - i;
+@@ -667,6 +724,7 @@ mono_render(struct mono *mono)
+ j++;
+ if (j != i + 1)
+ mono_step_edges(mono, j - (i + 1));
++ __DBG(("%s: %d vertical rows\n", __FUNCTION__, j-i));
+ }
+
+ mono_row(mono, i, j-i);
+@@ -717,6 +775,7 @@ mono_span_thread(void *arg)
+ if (RegionNil(&mono.clip))
+ return;
+ }
++ region_get_boxes(&mono.clip, &mono.clip_start, &mono.clip_end);
+
+ boxes.op = thread->op;
+ boxes.num_boxes = 0;
+@@ -891,9 +950,12 @@ mono_trapezoids_span_converter(struct sna *sna,
+
+ if (mono.clip.data == NULL && mono.op.damage == NULL)
+ mono.span = mono_span__fast;
++ else if (mono.clip.data != NULL && mono.op.damage == NULL)
++ mono.span = mono_span__clipped;
+ else
+ mono.span = mono_span;
+
++ region_get_boxes(&mono.clip, &mono.clip_start, &mono.clip_end);
+ mono_render(&mono);
+ mono.op.done(mono.sna, &mono.op);
+ mono_fini(&mono);
+@@ -939,6 +1001,7 @@ mono_trapezoids_span_converter(struct sna *sna,
+ mono.clip.extents.x2 - mono.clip.extents.x1,
+ mono.clip.extents.y2 - mono.clip.extents.y1,
+ COMPOSITE_PARTIAL, memset(&mono.op, 0, sizeof(mono.op)))) {
++ region_get_boxes(&mono.clip, &mono.clip_start, &mono.clip_end);
+ mono_render(&mono);
+ mono.op.done(mono.sna, &mono.op);
+ }
+@@ -974,6 +1037,7 @@ mono_inplace_fill_box(struct sna *sna,
+ box->x2 - box->x1,
+ box->y2 - box->y1,
+ fill->color));
++ sigtrap_assert_active();
+ pixman_fill(fill->data, fill->stride, fill->bpp,
+ box->x1, box->y1,
+ box->x2 - box->x1,
+@@ -995,6 +1059,7 @@ mono_inplace_fill_boxes(struct sna *sna,
+ box->x2 - box->x1,
+ box->y2 - box->y1,
+ fill->color));
++ sigtrap_assert_active();
+ pixman_fill(fill->data, fill->stride, fill->bpp,
+ box->x1, box->y1,
+ box->x2 - box->x1,
+@@ -1382,10 +1447,13 @@ mono_triangles_span_converter(struct sna *sna,
+ mono_render(&mono);
+ mono.op.done(mono.sna, &mono.op);
+ }
++ mono_fini(&mono);
+
+ if (!was_clear && !operator_is_bounded(op)) {
+ xPointFixed p1, p2;
+
++ DBG(("%s: performing unbounded clear\n", __FUNCTION__));
++
+ if (!mono_init(&mono, 2+3*count))
+ return false;
+
+@@ -1431,7 +1499,6 @@ mono_triangles_span_converter(struct sna *sna,
+ mono_fini(&mono);
+ }
+
+- mono_fini(&mono);
+ REGION_UNINIT(NULL, &mono.clip);
+ return true;
+ }
+diff --git a/src/sna/sna_trapezoids_precise.c b/src/sna/sna_trapezoids_precise.c
+index 9187ab48..242b4acb 100644
+--- a/src/sna/sna_trapezoids_precise.c
++++ b/src/sna/sna_trapezoids_precise.c
+@@ -1023,6 +1023,16 @@ tor_init(struct tor *converter, const BoxRec *box, int num_edges)
+ static void
+ tor_add_trapezoid(struct tor *tor, const xTrapezoid *t, int dx, int dy)
+ {
++ if (!xTrapezoidValid(t)) {
++ __DBG(("%s: skipping invalid trapezoid: top=%d, bottom=%d, left=(%d, %d), (%d, %d), right=(%d, %d), (%d, %d)\n",
++ __FUNCTION__,
++ t->top, t->bottom,
++ t->left.p1.x, t->left.p1.y,
++ t->left.p2.x, t->left.p2.y,
++ t->right.p1.x, t->right.p1.y,
++ t->right.p2.x, t->right.p2.y));
++ return;
++ }
+ polygon_add_edge(tor->polygon, t, &t->left, 1, dx, dy);
+ polygon_add_edge(tor->polygon, t, &t->right, -1, dx, dy);
+ }
+@@ -1635,31 +1645,27 @@ struct span_thread {
+ #define SPAN_THREAD_MAX_BOXES (8192/sizeof(struct sna_opacity_box))
+ struct span_thread_boxes {
+ const struct sna_composite_spans_op *op;
++ const BoxRec *clip_start, *clip_end;
+ int num_boxes;
+ struct sna_opacity_box boxes[SPAN_THREAD_MAX_BOXES];
+ };
+
+-static void span_thread_add_boxes(struct sna *sna, void *data,
+- const BoxRec *box, int count, float alpha)
++static void span_thread_add_box(struct sna *sna, void *data,
++ const BoxRec *box, float alpha)
+ {
+ struct span_thread_boxes *b = data;
+
+- __DBG(("%s: adding %d boxes with alpha=%f\n",
+- __FUNCTION__, count, alpha));
++ __DBG(("%s: adding box with alpha=%f\n", __FUNCTION__, alpha));
+
+- assert(count > 0 && count <= SPAN_THREAD_MAX_BOXES);
+- if (unlikely(b->num_boxes + count > SPAN_THREAD_MAX_BOXES)) {
+- DBG(("%s: flushing %d boxes, adding %d\n", __FUNCTION__, b->num_boxes, count));
+- assert(b->num_boxes <= SPAN_THREAD_MAX_BOXES);
++ if (unlikely(b->num_boxes == SPAN_THREAD_MAX_BOXES)) {
++ DBG(("%s: flushing %d boxes\n", __FUNCTION__, b->num_boxes));
+ b->op->thread_boxes(sna, b->op, b->boxes, b->num_boxes);
+ b->num_boxes = 0;
+ }
+
+- do {
+- b->boxes[b->num_boxes].box = *box++;
+- b->boxes[b->num_boxes].alpha = alpha;
+- b->num_boxes++;
+- } while (--count);
++ b->boxes[b->num_boxes].box = *box++;
++ b->boxes[b->num_boxes].alpha = alpha;
++ b->num_boxes++;
+ assert(b->num_boxes <= SPAN_THREAD_MAX_BOXES);
+ }
+
+@@ -1670,8 +1676,22 @@ span_thread_box(struct sna *sna,
+ const BoxRec *box,
+ int coverage)
+ {
++ struct span_thread_boxes *b = (struct span_thread_boxes *)op;
++
+ __DBG(("%s: %d -> %d @ %d\n", __FUNCTION__, box->x1, box->x2, coverage));
+- span_thread_add_boxes(sna, op, box, 1, AREA_TO_FLOAT(coverage));
++ if (b->num_boxes) {
++ struct sna_opacity_box *bb = &b->boxes[b->num_boxes-1];
++ if (bb->box.x1 == box->x1 &&
++ bb->box.x2 == box->x2 &&
++ bb->box.y2 == box->y1 &&
++ bb->alpha == AREA_TO_FLOAT(coverage)) {
++ bb->box.y2 = box->y2;
++ __DBG(("%s: contracted double row: %d -> %d\n", __func__, bb->box.y1, bb->box.y2));
++ return;
++ }
++ }
++
++ span_thread_add_box(sna, op, box, AREA_TO_FLOAT(coverage));
+ }
+
+ static void
+@@ -1681,20 +1701,28 @@ span_thread_clipped_box(struct sna *sna,
+ const BoxRec *box,
+ int coverage)
+ {
+- pixman_region16_t region;
++ struct span_thread_boxes *b = (struct span_thread_boxes *)op;
++ const BoxRec *c;
+
+ __DBG(("%s: %d -> %d @ %f\n", __FUNCTION__, box->x1, box->x2,
+ AREA_TO_FLOAT(coverage)));
+
+- pixman_region_init_rects(&region, box, 1);
+- RegionIntersect(&region, &region, clip);
+- if (region_num_rects(&region)) {
+- span_thread_add_boxes(sna, op,
+- region_rects(&region),
+- region_num_rects(&region),
+- AREA_TO_FLOAT(coverage));
++ b->clip_start =
++ find_clip_box_for_y(b->clip_start, b->clip_end, box->y1);
++
++ c = b->clip_start;
++ while (c != b->clip_end) {
++ BoxRec clipped;
++
++ if (box->y2 <= c->y1)
++ break;
++
++ clipped = *box;
++ if (!box_intersect(&clipped, c++))
++ continue;
++
++ span_thread_add_box(sna, op, &clipped, AREA_TO_FLOAT(coverage));
+ }
+- pixman_region_fini(&region);
+ }
+
+ static span_func_t
+@@ -1712,7 +1740,7 @@ thread_choose_span(struct sna_composite_spans_op *tmp,
+
+ assert(!is_mono(dst, maskFormat));
+ assert(tmp->thread_boxes);
+- DBG(("%s: clipped? %d\n", __FUNCTION__, clip->data != NULL));
++ DBG(("%s: clipped? %d x %d\n", __FUNCTION__, clip->data != NULL, region_num_rects(clip)));
+ if (clip->data)
+ span = span_thread_clipped_box;
+ else
+@@ -1721,6 +1749,17 @@ thread_choose_span(struct sna_composite_spans_op *tmp,
+ return span;
+ }
+
++inline static void
++span_thread_boxes_init(struct span_thread_boxes *boxes,
++ const struct sna_composite_spans_op *op,
++ const RegionRec *clip)
++{
++ boxes->op = op;
++ boxes->clip_start = region_rects(clip);
++ boxes->clip_end = boxes->clip_start + region_num_rects(clip);
++ boxes->num_boxes = 0;
++}
++
+ static void
+ span_thread(void *arg)
+ {
+@@ -1733,8 +1772,7 @@ span_thread(void *arg)
+ if (!tor_init(&tor, &thread->extents, 2*thread->ntrap))
+ return;
+
+- boxes.op = thread->op;
+- boxes.num_boxes = 0;
++ span_thread_boxes_init(&boxes, thread->op, thread->clip);
+
+ y1 = thread->extents.y1 - thread->draw_y;
+ y2 = thread->extents.y2 - thread->draw_y;
+@@ -2183,6 +2221,52 @@ static force_inline uint8_t coverage_opacity(int coverage, uint8_t opacity)
+ return opacity == 255 ? coverage : mul_8_8(coverage, opacity);
+ }
+
++struct clipped_span {
++ span_func_t span;
++ const BoxRec *clip_start, *clip_end;
++};
++
++static void
++tor_blt_clipped(struct sna *sna,
++ struct sna_composite_spans_op *op,
++ pixman_region16_t *clip,
++ const BoxRec *box,
++ int coverage)
++{
++ struct clipped_span *cs = (struct clipped_span *)clip;
++ const BoxRec *c;
++
++ cs->clip_start =
++ find_clip_box_for_y(cs->clip_start, cs->clip_end, box->y1);
++
++ c = cs->clip_start;
++ while (c != cs->clip_end) {
++ BoxRec clipped;
++
++ if (box->y2 <= c->y1)
++ break;
++
++ clipped = *box;
++ if (!box_intersect(&clipped, c++))
++ continue;
++
++ cs->span(sna, op, NULL, &clipped, coverage);
++ }
++}
++
++inline static span_func_t
++clipped_span(struct clipped_span *cs,
++ span_func_t span,
++ const RegionRec *clip)
++{
++ if (clip->data) {
++ cs->span = span;
++ region_get_boxes(clip, &cs->clip_start, &cs->clip_end);
++ span = tor_blt_clipped;
++ }
++ return span;
++}
++
+ static void _tor_blt_src(struct inplace *in, const BoxRec *box, uint8_t v)
+ {
+ uint8_t *ptr = in->ptr;
+@@ -2218,25 +2302,6 @@ tor_blt_src(struct sna *sna,
+ }
+
+ static void
+-tor_blt_src_clipped(struct sna *sna,
+- struct sna_composite_spans_op *op,
+- pixman_region16_t *clip,
+- const BoxRec *box,
+- int coverage)
+-{
+- pixman_region16_t region;
+- int n;
+-
+- pixman_region_init_rects(&region, box, 1);
+- RegionIntersect(&region, &region, clip);
+- n = region_num_rects(&region);
+- box = region_rects(&region);
+- while (n--)
+- tor_blt_src(sna, op, NULL, box++, coverage);
+- pixman_region_fini(&region);
+-}
+-
+-static void
+ tor_blt_in(struct sna *sna,
+ struct sna_composite_spans_op *op,
+ pixman_region16_t *clip,
+@@ -2268,25 +2333,6 @@ tor_blt_in(struct sna *sna,
+ }
+
+ static void
+-tor_blt_in_clipped(struct sna *sna,
+- struct sna_composite_spans_op *op,
+- pixman_region16_t *clip,
+- const BoxRec *box,
+- int coverage)
+-{
+- pixman_region16_t region;
+- int n;
+-
+- pixman_region_init_rects(&region, box, 1);
+- RegionIntersect(&region, &region, clip);
+- n = region_num_rects(&region);
+- box = region_rects(&region);
+- while (n--)
+- tor_blt_in(sna, op, NULL, box++, coverage);
+- pixman_region_fini(&region);
+-}
+-
+-static void
+ tor_blt_add(struct sna *sna,
+ struct sna_composite_spans_op *op,
+ pixman_region16_t *clip,
+@@ -2325,25 +2371,6 @@ tor_blt_add(struct sna *sna,
+ }
+
+ static void
+-tor_blt_add_clipped(struct sna *sna,
+- struct sna_composite_spans_op *op,
+- pixman_region16_t *clip,
+- const BoxRec *box,
+- int coverage)
+-{
+- pixman_region16_t region;
+- int n;
+-
+- pixman_region_init_rects(&region, box, 1);
+- RegionIntersect(&region, &region, clip);
+- n = region_num_rects(&region);
+- box = region_rects(&region);
+- while (n--)
+- tor_blt_add(sna, op, NULL, box++, coverage);
+- pixman_region_fini(&region);
+-}
+-
+-static void
+ tor_blt_lerp32(struct sna *sna,
+ struct sna_composite_spans_op *op,
+ pixman_region16_t *clip,
+@@ -2358,6 +2385,7 @@ tor_blt_lerp32(struct sna *sna,
+ if (coverage == 0)
+ return;
+
++ sigtrap_assert_active();
+ ptr += box->y1 * stride + box->x1;
+
+ h = box->y2 - box->y1;
+@@ -2396,25 +2424,6 @@ tor_blt_lerp32(struct sna *sna,
+ }
+ }
+
+-static void
+-tor_blt_lerp32_clipped(struct sna *sna,
+- struct sna_composite_spans_op *op,
+- pixman_region16_t *clip,
+- const BoxRec *box,
+- int coverage)
+-{
+- pixman_region16_t region;
+- int n;
+-
+- pixman_region_init_rects(&region, box, 1);
+- RegionIntersect(&region, &region, clip);
+- n = region_num_rects(&region);
+- box = region_rects(&region);
+- while (n--)
+- tor_blt_lerp32(sna, op, NULL, box++, coverage);
+- pixman_region_fini(&region);
+-}
+-
+ struct pixman_inplace {
+ pixman_image_t *image, *source, *mask;
+ uint32_t color;
+@@ -2442,24 +2451,6 @@ pixmask_span_solid(struct sna *sna,
+ pi->dx + box->x1, pi->dy + box->y1,
+ box->x2 - box->x1, box->y2 - box->y1);
+ }
+-static void
+-pixmask_span_solid__clipped(struct sna *sna,
+- struct sna_composite_spans_op *op,
+- pixman_region16_t *clip,
+- const BoxRec *box,
+- int coverage)
+-{
+- pixman_region16_t region;
+- int n;
+-
+- pixman_region_init_rects(&region, box, 1);
+- RegionIntersect(&region, &region, clip);
+- n = region_num_rects(&region);
+- box = region_rects(&region);
+- while (n--)
+- pixmask_span_solid(sna, op, NULL, box++, coverage);
+- pixman_region_fini(&region);
+-}
+
+ static void
+ pixmask_span(struct sna *sna,
+@@ -2480,24 +2471,6 @@ pixmask_span(struct sna *sna,
+ pi->dx + box->x1, pi->dy + box->y1,
+ box->x2 - box->x1, box->y2 - box->y1);
+ }
+-static void
+-pixmask_span__clipped(struct sna *sna,
+- struct sna_composite_spans_op *op,
+- pixman_region16_t *clip,
+- const BoxRec *box,
+- int coverage)
+-{
+- pixman_region16_t region;
+- int n;
+-
+- pixman_region_init_rects(&region, box, 1);
+- RegionIntersect(&region, &region, clip);
+- n = region_num_rects(&region);
+- box = region_rects(&region);
+- while (n--)
+- pixmask_span(sna, op, NULL, box++, coverage);
+- pixman_region_fini(&region);
+-}
+
+ struct inplace_x8r8g8b8_thread {
+ xTrapezoid *traps;
+@@ -2516,6 +2489,7 @@ static void inplace_x8r8g8b8_thread(void *arg)
+ struct inplace_x8r8g8b8_thread *thread = arg;
+ struct tor tor;
+ span_func_t span;
++ struct clipped_span clipped;
+ RegionPtr clip;
+ int y1, y2, n;
+
+@@ -2546,12 +2520,11 @@ static void inplace_x8r8g8b8_thread(void *arg)
+ inplace.stride = pixmap->devKind;
+ inplace.color = thread->color;
+
+- if (clip->data)
+- span = tor_blt_lerp32_clipped;
+- else
+- span = tor_blt_lerp32;
++ span = clipped_span(&clipped, tor_blt_lerp32, clip);
+
+- tor_render(NULL, &tor, (void*)&inplace, clip, span, false);
++ tor_render(NULL, &tor,
++ (void*)&inplace, (void *)&clipped,
++ span, false);
+ } else if (thread->is_solid) {
+ struct pixman_inplace pi;
+
+@@ -2564,10 +2537,7 @@ static void inplace_x8r8g8b8_thread(void *arg)
+ 1, 1, pi.bits, 0);
+ pixman_image_set_repeat(pi.source, PIXMAN_REPEAT_NORMAL);
+
+- if (clip->data)
+- span = pixmask_span_solid__clipped;
+- else
+- span = pixmask_span_solid;
++ span = clipped_span(&clipped, pixmask_span_solid, clip);
+
+ tor_render(NULL, &tor, (void*)&pi, clip, span, false);
+
+@@ -2588,12 +2558,11 @@ static void inplace_x8r8g8b8_thread(void *arg)
+ pi.bits = pixman_image_get_data(pi.mask);
+ pi.op = thread->op;
+
+- if (clip->data)
+- span = pixmask_span__clipped;
+- else
+- span = pixmask_span;
++ span = clipped_span(&clipped, pixmask_span, clip);
+
+- tor_render(NULL, &tor, (void*)&pi, clip, span, false);
++ tor_render(NULL, &tor,
++ (void*)&pi, (void *)&clipped,
++ span, false);
+
+ pixman_image_unref(pi.mask);
+ pixman_image_unref(pi.source);
+@@ -2712,6 +2681,7 @@ trapezoid_span_inplace__x8r8g8b8(CARD8 op,
+ if (num_threads == 1) {
+ struct tor tor;
+ span_func_t span;
++ struct clipped_span clipped;
+
+ if (!tor_init(&tor, &region.extents, 2*ntrap))
+ return true;
+@@ -2737,17 +2707,14 @@ trapezoid_span_inplace__x8r8g8b8(CARD8 op,
+ inplace.stride = pixmap->devKind;
+ inplace.color = color;
+
+- if (dst->pCompositeClip->data)
+- span = tor_blt_lerp32_clipped;
+- else
+- span = tor_blt_lerp32;
+-
++ span = clipped_span(&clipped, tor_blt_lerp32, dst->pCompositeClip);
+ DBG(("%s: render inplace op=%d, color=%08x\n",
+ __FUNCTION__, op, color));
+
+ if (sigtrap_get() == 0) {
+- tor_render(NULL, &tor, (void*)&inplace,
+- dst->pCompositeClip, span, false);
++ tor_render(NULL, &tor,
++ (void*)&inplace, (void*)&clipped,
++ span, false);
+ sigtrap_put();
+ }
+ } else if (is_solid) {
+@@ -2762,15 +2729,11 @@ trapezoid_span_inplace__x8r8g8b8(CARD8 op,
+ 1, 1, pi.bits, 0);
+ pixman_image_set_repeat(pi.source, PIXMAN_REPEAT_NORMAL);
+
+- if (dst->pCompositeClip->data)
+- span = pixmask_span_solid__clipped;
+- else
+- span = pixmask_span_solid;
+-
++ span = clipped_span(&clipped, pixmask_span_solid, dst->pCompositeClip);
+ if (sigtrap_get() == 0) {
+- tor_render(NULL, &tor, (void*)&pi,
+- dst->pCompositeClip, span,
+- false);
++ tor_render(NULL, &tor,
++ (void*)&pi, (void*)&clipped,
++ span, false);
+ sigtrap_put();
+ }
+
+@@ -2791,15 +2754,11 @@ trapezoid_span_inplace__x8r8g8b8(CARD8 op,
+ pi.bits = pixman_image_get_data(pi.mask);
+ pi.op = op;
+
+- if (dst->pCompositeClip->data)
+- span = pixmask_span__clipped;
+- else
+- span = pixmask_span;
+-
++ span = clipped_span(&clipped, pixmask_span, dst->pCompositeClip);
+ if (sigtrap_get() == 0) {
+- tor_render(NULL, &tor, (void*)&pi,
+- dst->pCompositeClip, span,
+- false);
++ tor_render(NULL, &tor,
++ (void*)&pi, (void *)&clipped,
++ span, false);
+ sigtrap_put();
+ }
+
+@@ -2861,9 +2820,9 @@ trapezoid_span_inplace__x8r8g8b8(CARD8 op,
+
+ struct inplace_thread {
+ xTrapezoid *traps;
+- RegionPtr clip;
+ span_func_t span;
+ struct inplace inplace;
++ struct clipped_span clipped;
+ BoxRec extents;
+ int dx, dy;
+ int draw_x, draw_y;
+@@ -2888,8 +2847,9 @@ static void inplace_thread(void *arg)
+ tor_add_trapezoid(&tor, &thread->traps[n], thread->dx, thread->dy);
+ }
+
+- tor_render(NULL, &tor, (void*)&thread->inplace,
+- thread->clip, thread->span, thread->unbounded);
++ tor_render(NULL, &tor,
++ (void*)&thread->inplace, (void*)&thread->clipped,
++ thread->span, thread->unbounded);
+
+ tor_fini(&tor);
+ }
+@@ -2903,6 +2863,7 @@ precise_trapezoid_span_inplace(struct sna *sna,
+ bool fallback)
+ {
+ struct inplace inplace;
++ struct clipped_span clipped;
+ span_func_t span;
+ PixmapPtr pixmap;
+ struct sna_pixmap *priv;
+@@ -3020,21 +2981,12 @@ precise_trapezoid_span_inplace(struct sna *sna,
+ dst->pCompositeClip->data != NULL));
+
+ if (op == PictOpSrc) {
+- if (dst->pCompositeClip->data)
+- span = tor_blt_src_clipped;
+- else
+- span = tor_blt_src;
++ span = tor_blt_src;
+ } else if (op == PictOpIn) {
+- if (dst->pCompositeClip->data)
+- span = tor_blt_in_clipped;
+- else
+- span = tor_blt_in;
++ span = tor_blt_in;
+ } else {
+ assert(op == PictOpAdd);
+- if (dst->pCompositeClip->data)
+- span = tor_blt_add_clipped;
+- else
+- span = tor_blt_add;
++ span = tor_blt_add;
+ }
+
+ DBG(("%s: move-to-cpu(dst)\n", __FUNCTION__));
+@@ -3052,6 +3004,8 @@ precise_trapezoid_span_inplace(struct sna *sna,
+ inplace.stride = pixmap->devKind;
+ inplace.opacity = color >> 24;
+
++ span = clipped_span(&clipped, span, dst->pCompositeClip);
++
+ num_threads = 1;
+ if (!NO_GPU_THREADS &&
+ (flags & COMPOSITE_SPANS_RECTILINEAR) == 0)
+@@ -3074,8 +3028,9 @@ precise_trapezoid_span_inplace(struct sna *sna,
+ }
+
+ if (sigtrap_get() == 0) {
+- tor_render(NULL, &tor, (void*)&inplace,
+- dst->pCompositeClip, span, unbounded);
++ tor_render(NULL, &tor,
++ (void*)&inplace, (void *)&clipped,
++ span, unbounded);
+ sigtrap_put();
+ }
+
+@@ -3093,7 +3048,7 @@ precise_trapezoid_span_inplace(struct sna *sna,
+ threads[0].ntrap = ntrap;
+ threads[0].inplace = inplace;
+ threads[0].extents = region.extents;
+- threads[0].clip = dst->pCompositeClip;
++ threads[0].clipped = clipped;
+ threads[0].span = span;
+ threads[0].unbounded = unbounded;
+ threads[0].dx = dx;
+@@ -3316,8 +3271,7 @@ tristrip_thread(void *arg)
+ if (!tor_init(&tor, &thread->extents, 2*thread->count))
+ return;
+
+- boxes.op = thread->op;
+- boxes.num_boxes = 0;
++ span_thread_boxes_init(&boxes, thread->op, thread->clip);
+
+ cw = 0; ccw = 1;
+ polygon_add_line(tor.polygon,
+diff --git a/src/sna/sna_video.c b/src/sna/sna_video.c
+index ed0e7b31..e2b11c31 100644
+--- a/src/sna/sna_video.c
++++ b/src/sna/sna_video.c
+@@ -591,6 +591,72 @@ use_gtt: /* copy data, must use GTT so that we keep the overlay uncached */
+ return true;
+ }
+
++void sna_video_fill_colorkey(struct sna_video *video,
++ const RegionRec *clip)
++{
++ struct sna *sna = video->sna;
++ PixmapPtr front = sna->front;
++ struct kgem_bo *bo = __sna_pixmap_get_bo(front);
++ uint8_t *dst, *tmp;
++ int w, width;
++
++ if (video->AlwaysOnTop || RegionEqual(&video->clip, (RegionPtr)clip))
++ return;
++
++ assert(bo);
++ if (!wedged(sna) &&
++ sna_blt_fill_boxes(sna, GXcopy, bo,
++ front->drawable.bitsPerPixel,
++ video->color_key,
++ region_rects(clip),
++ region_num_rects(clip))) {
++ RegionCopy(&video->clip, (RegionPtr)clip);
++ return;
++ }
++
++ dst = kgem_bo_map__gtt(&sna->kgem, bo);
++ if (dst == NULL)
++ return;
++
++ w = front->drawable.bitsPerPixel/8;
++ width = (clip->extents.x2 - clip->extents.x1) * w;
++ tmp = malloc(width);
++ if (tmp == NULL)
++ return;
++
++ memcpy(tmp, &video->color_key, w);
++ while (2 * w < width) {
++ memcpy(tmp + w, tmp, w);
++ w *= 2;
++ }
++ if (w < width)
++ memcpy(tmp + w, tmp, width - w);
++
++ if (sigtrap_get() == 0) {
++ const BoxRec *box = region_rects(clip);
++ int n = region_num_rects(clip);
++
++ w = front->drawable.bitsPerPixel/8;
++ do {
++ int y = box->y1;
++ uint8_t *row = dst + y*bo->pitch + w*box->x1;
++
++ width = (box->x2 - box->x1) * w;
++ while (y < box->y2) {
++ memcpy(row, tmp, width);
++ row += bo->pitch;
++ y++;
++ }
++ box++;
++ } while (--n);
++ sigtrap_put();
++
++ RegionCopy(&video->clip, (RegionPtr)clip);
++ }
++
++ free(tmp);
++}
++
+ XvAdaptorPtr sna_xv_adaptor_alloc(struct sna *sna)
+ {
+ XvAdaptorPtr new_adaptors;
+diff --git a/src/sna/sna_video.h b/src/sna/sna_video.h
+index f21605fc..39cb725f 100644
+--- a/src/sna/sna_video.h
++++ b/src/sna/sna_video.h
+@@ -72,6 +72,8 @@ THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ struct sna_video {
+ struct sna *sna;
+
++ int idx; /* XXX expose struct plane instead? */
++
+ int brightness;
+ int contrast;
+ int saturation;
+@@ -193,6 +195,9 @@ bool
+ sna_video_copy_data(struct sna_video *video,
+ struct sna_video_frame *frame,
+ const uint8_t *buf);
++void
++sna_video_fill_colorkey(struct sna_video *video,
++ const RegionRec *clip);
+
+ void sna_video_buffer_fini(struct sna_video *video);
+
+@@ -210,4 +215,26 @@ sna_window_set_port(WindowPtr window, XvPortPtr port)
+ ((void **)__get_private(window, sna_window_key))[2] = port;
+ }
+
++static inline int offset_and_clip(int x, int dx)
++{
++ x += dx;
++ if (x <= 0)
++ return 0;
++ if (x >= MAXSHORT)
++ return MAXSHORT;
++ return x;
++}
++
++static inline void init_video_region(RegionRec *region,
++ DrawablePtr draw,
++ int drw_x, int drw_y,
++ int drw_w, int drw_h)
++{
++ region->extents.x1 = offset_and_clip(draw->x, drw_x);
++ region->extents.y1 = offset_and_clip(draw->y, drw_y);
++ region->extents.x2 = offset_and_clip(draw->x, drw_x + drw_w);
++ region->extents.y2 = offset_and_clip(draw->y, drw_y + drw_h);
++ region->data = NULL;
++}
++
+ #endif /* SNA_VIDEO_H */
+diff --git a/src/sna/sna_video_overlay.c b/src/sna/sna_video_overlay.c
+index ac81f1a0..9bc5ce40 100644
+--- a/src/sna/sna_video_overlay.c
++++ b/src/sna/sna_video_overlay.c
+@@ -130,7 +130,7 @@ static int sna_video_overlay_stop(ddStopVideo_ARGS)
+
+ DBG(("%s()\n", __FUNCTION__));
+
+- REGION_EMPTY(scrn->pScreen, &video->clip);
++ REGION_EMPTY(to_screen_from_sna(sna), &video->clip);
+
+ request.flags = 0;
+ (void)drmIoctl(sna->kgem.fd,
+@@ -474,15 +474,13 @@ sna_video_overlay_put_image(ddPutImage_ARGS)
+ if (src_h >= (drw_h * 8))
+ drw_h = src_h / 7;
+
+- clip.extents.x1 = draw->x + drw_x;
+- clip.extents.y1 = draw->y + drw_y;
+- clip.extents.x2 = clip.extents.x1 + drw_w;
+- clip.extents.y2 = clip.extents.y1 + drw_h;
+- clip.data = NULL;
++ init_video_region(&clip, draw, drw_x, drw_y, drw_w, drw_h);
+
+ DBG(("%s: always_on_top=%d\n", __FUNCTION__, video->AlwaysOnTop));
+- if (!video->AlwaysOnTop)
++ if (!video->AlwaysOnTop) {
++ ValidateGC(draw, gc);
+ RegionIntersect(&clip, &clip, gc->pCompositeClip);
++ }
+ if (box_empty(&clip.extents))
+ goto invisible;
+
+@@ -551,15 +549,7 @@ sna_video_overlay_put_image(ddPutImage_ARGS)
+ ret = Success;
+ if (sna_video_overlay_show
+ (sna, video, &frame, crtc, &dstBox, src_w, src_h, drw_w, drw_h)) {
+- //xf86XVFillKeyHelperDrawable(draw, video->color_key, &clip);
+- if (!video->AlwaysOnTop && !RegionEqual(&video->clip, &clip) &&
+- sna_blt_fill_boxes(sna, GXcopy,
+- __sna_pixmap_get_bo(sna->front),
+- sna->front->drawable.bitsPerPixel,
+- video->color_key,
+- region_rects(&clip),
+- region_num_rects(&clip)))
+- RegionCopy(&video->clip, &clip);
++ sna_video_fill_colorkey(video, &clip);
+ sna_window_set_port((WindowPtr)draw, port);
+ } else {
+ DBG(("%s: failed to show video frame\n", __FUNCTION__));
+diff --git a/src/sna/sna_video_sprite.c b/src/sna/sna_video_sprite.c
+index 92230f97..69bfdfd2 100644
+--- a/src/sna/sna_video_sprite.c
++++ b/src/sna/sna_video_sprite.c
+@@ -47,6 +47,8 @@
+ #define DRM_FORMAT_YUYV fourcc_code('Y', 'U', 'Y', 'V') /* [31:0] Cr0:Y1:Cb0:Y0 8:8:8:8 little endian */
+ #define DRM_FORMAT_UYVY fourcc_code('U', 'Y', 'V', 'Y') /* [31:0] Y1:Cr0:Y0:Cb0 8:8:8:8 little endian */
+
++#define has_hw_scaling(sna) ((sna)->kgem.gen < 071)
++
+ #define LOCAL_IOCTL_MODE_SETPLANE DRM_IOWR(0xB7, struct local_mode_set_plane)
+ struct local_mode_set_plane {
+ uint32_t plane_id;
+@@ -81,19 +83,17 @@ static int sna_video_sprite_stop(ddStopVideo_ARGS)
+ xf86CrtcConfigPtr config = XF86_CRTC_CONFIG_PTR(video->sna->scrn);
+ int i;
+
+- for (i = 0; i < config->num_crtc; i++) {
++ for (i = 0; i < video->sna->mode.num_real_crtc; i++) {
+ xf86CrtcPtr crtc = config->crtc[i];
+ int pipe;
+
+- if (sna_crtc_id(crtc) == 0)
+- break;
+-
+- pipe = sna_crtc_to_pipe(crtc);
++ pipe = sna_crtc_pipe(crtc);
++ assert(pipe < ARRAY_SIZE(video->bo));
+ if (video->bo[pipe] == NULL)
+ continue;
+
+ memset(&s, 0, sizeof(s));
+- s.plane_id = sna_crtc_to_sprite(crtc);
++ s.plane_id = sna_crtc_to_sprite(crtc, video->idx);
+ if (drmIoctl(video->sna->kgem.fd, LOCAL_IOCTL_MODE_SETPLANE, &s))
+ xf86DrvMsg(video->sna->scrn->scrnIndex, X_ERROR,
+ "failed to disable plane\n");
+@@ -153,7 +153,7 @@ static int sna_video_sprite_best_size(ddQueryBestSize_ARGS)
+ struct sna_video *video = port->devPriv.ptr;
+ struct sna *sna = video->sna;
+
+- if (sna->kgem.gen >= 075) {
++ if (!has_hw_scaling(sna) && !sna->render.video) {
+ *p_w = vid_w;
+ *p_h = vid_h;
+ } else {
+@@ -221,12 +221,12 @@ sna_video_sprite_show(struct sna *sna,
+ BoxPtr dstBox)
+ {
+ struct local_mode_set_plane s;
+- int pipe = sna_crtc_to_pipe(crtc);
++ int pipe = sna_crtc_pipe(crtc);
+
+ /* XXX handle video spanning multiple CRTC */
+
+ VG_CLEAR(s);
+- s.plane_id = sna_crtc_to_sprite(crtc);
++ s.plane_id = sna_crtc_to_sprite(crtc, video->idx);
+
+ #define DRM_I915_SET_SPRITE_COLORKEY 0x2b
+ #define LOCAL_IOCTL_I915_SET_SPRITE_COLORKEY DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_SET_SPRITE_COLORKEY, struct local_intel_sprite_colorkey)
+@@ -263,9 +263,6 @@ sna_video_sprite_show(struct sna *sna,
+ video->color_key_changed &= ~(1 << pipe);
+ }
+
+- if (video->bo[pipe] == frame->bo)
+- return true;
+-
+ update_dst_box_to_crtc_coords(sna, crtc, dstBox);
+ if (frame->rotation & (RR_Rotate_90 | RR_Rotate_270)) {
+ int tmp = frame->width;
+@@ -283,15 +280,30 @@ sna_video_sprite_show(struct sna *sna,
+ uint32_t handles[4];
+ uint32_t pitches[4]; /* pitch for each plane */
+ uint32_t offsets[4]; /* offset of each plane */
++ uint64_t modifiers[4];
+ } f;
+ bool purged = true;
+
+ memset(&f, 0, sizeof(f));
+ f.width = frame->width;
+ f.height = frame->height;
++ f.flags = 1 << 1; /* +modifiers */
+ f.handles[0] = frame->bo->handle;
+ f.pitches[0] = frame->pitch[0];
+
++ switch (frame->bo->tiling) {
++ case I915_TILING_NONE:
++ break;
++ case I915_TILING_X:
++ /* I915_FORMAT_MOD_X_TILED */
++ f.modifiers[0] = (uint64_t)1 << 56 | 1;
++ break;
++ case I915_TILING_Y:
++ /* I915_FORMAT_MOD_X_TILED */
++ f.modifiers[0] = (uint64_t)1 << 56 | 2;
++ break;
++ }
++
+ switch (frame->id) {
+ case FOURCC_RGB565:
+ f.pixel_format = DRM_FORMAT_RGB565;
+@@ -360,7 +372,7 @@ sna_video_sprite_show(struct sna *sna,
+ return false;
+ }
+
+- frame->bo->domain = DOMAIN_NONE;
++ __kgem_bo_clear_dirty(frame->bo);
+
+ if (video->bo[pipe])
+ kgem_bo_destroy(&sna->kgem, video->bo[pipe]);
+@@ -374,17 +386,17 @@ static int sna_video_sprite_put_image(ddPutImage_ARGS)
+ struct sna *sna = video->sna;
+ xf86CrtcConfigPtr config = XF86_CRTC_CONFIG_PTR(sna->scrn);
+ RegionRec clip;
++ BoxRec draw_extents;
+ int ret, i;
+
+- clip.extents.x1 = draw->x + drw_x;
+- clip.extents.y1 = draw->y + drw_y;
+- clip.extents.x2 = clip.extents.x1 + drw_w;
+- clip.extents.y2 = clip.extents.y1 + drw_h;
+- clip.data = NULL;
++ init_video_region(&clip, draw, drw_x, drw_y, drw_w, drw_h);
++ draw_extents = clip.extents;
+
+ DBG(("%s: always_on_top=%d\n", __FUNCTION__, video->AlwaysOnTop));
+- if (!video->AlwaysOnTop)
++ if (!video->AlwaysOnTop) {
++ ValidateGC(draw, gc);
+ RegionIntersect(&clip, &clip, gc->pCompositeClip);
++ }
+
+ DBG(("%s: src=(%d, %d),(%d, %d), dst=(%d, %d),(%d, %d), id=%d, sizep=%dx%d, sync?=%d\n",
+ __FUNCTION__,
+@@ -402,19 +414,17 @@ static int sna_video_sprite_put_image(ddPutImage_ARGS)
+ goto err;
+ }
+
+- for (i = 0; i < config->num_crtc; i++) {
++ for (i = 0; i < video->sna->mode.num_real_crtc; i++) {
+ xf86CrtcPtr crtc = config->crtc[i];
+ struct sna_video_frame frame;
++ BoxRec dst = draw_extents;
+ int pipe;
+ INT32 x1, x2, y1, y2;
+- BoxRec dst;
+ RegionRec reg;
+ Rotation rotation;
++ bool cache_bo;
+
+- if (sna_crtc_id(crtc) == 0)
+- break;
+-
+- pipe = sna_crtc_to_pipe(crtc);
++ pipe = sna_crtc_pipe(crtc);
+
+ sna_video_frame_init(video, format->id, width, height, &frame);
+
+@@ -423,10 +433,11 @@ static int sna_video_sprite_put_image(ddPutImage_ARGS)
+ RegionIntersect(&reg, &reg, &clip);
+ if (RegionNil(&reg)) {
+ off:
++ assert(pipe < ARRAY_SIZE(video->bo));
+ if (video->bo[pipe]) {
+ struct local_mode_set_plane s;
+ memset(&s, 0, sizeof(s));
+- s.plane_id = sna_crtc_to_sprite(crtc);
++ s.plane_id = sna_crtc_to_sprite(crtc, video->idx);
+ if (drmIoctl(video->sna->kgem.fd, LOCAL_IOCTL_MODE_SETPLANE, &s))
+ xf86DrvMsg(video->sna->scrn->scrnIndex, X_ERROR,
+ "failed to disable plane\n");
+@@ -440,8 +451,6 @@ off:
+ y1 = src_y;
+ y2 = src_y + src_h;
+
+- dst = clip.extents;
+-
+ ret = xf86XVClipVideoHelper(&dst, &x1, &x2, &y1, &y2,
+ &reg, frame.width, frame.height);
+ RegionUninit(&reg);
+@@ -465,8 +474,8 @@ off:
+
+ /* if sprite can't handle rotation natively, store it for the copy func */
+ rotation = RR_Rotate_0;
+- if (!sna_crtc_set_sprite_rotation(crtc, crtc->rotation)) {
+- sna_crtc_set_sprite_rotation(crtc, RR_Rotate_0);
++ if (!sna_crtc_set_sprite_rotation(crtc, video->idx, crtc->rotation)) {
++ sna_crtc_set_sprite_rotation(crtc, video->idx, RR_Rotate_0);
+ rotation = crtc->rotation;
+ }
+ sna_video_frame_set_rotation(video, &frame, rotation);
+@@ -496,6 +505,8 @@ off:
+ frame.image.y1 = 0;
+ frame.image.x2 = frame.width;
+ frame.image.y2 = frame.height;
++
++ cache_bo = false;
+ } else {
+ frame.bo = sna_video_buffer(video, &frame);
+ if (frame.bo == NULL) {
+@@ -509,6 +520,60 @@ off:
+ ret = BadAlloc;
+ goto err;
+ }
++
++ cache_bo = true;
++ }
++
++ if (!has_hw_scaling(sna) && sna->render.video &&
++ !((frame.src.x2 - frame.src.x1) == (dst.x2 - dst.x1) &&
++ (frame.src.y2 - frame.src.y1) == (dst.y2 - dst.y1))) {
++ ScreenPtr screen = to_screen_from_sna(sna);
++ PixmapPtr scaled;
++ RegionRec r;
++
++ r.extents.x1 = r.extents.y1 = 0;
++ r.extents.x2 = dst.x2 - dst.x1;
++ r.extents.y2 = dst.y2 - dst.y1;
++ r.data = NULL;
++
++ DBG(("%s: scaling from (%d, %d) to (%d, %d)\n",
++ __FUNCTION__,
++ frame.src.x2 - frame.src.x1,
++ frame.src.y2 - frame.src.y1,
++ r.extents.x2, r.extents.y2));
++
++ scaled = screen->CreatePixmap(screen,
++ r.extents.x2,
++ r.extents.y2,
++ 24,
++ CREATE_PIXMAP_USAGE_SCRATCH);
++ if (scaled == NULL) {
++ ret = BadAlloc;
++ goto err;
++ }
++
++ if (!sna->render.video(sna, video, &frame, &r, scaled)) {
++ screen->DestroyPixmap(scaled);
++ ret = BadAlloc;
++ goto err;
++ }
++
++ if (cache_bo)
++ sna_video_buffer_fini(video);
++ else
++ kgem_bo_destroy(&sna->kgem, frame.bo);
++
++ frame.bo = kgem_bo_reference(__sna_pixmap_get_bo(scaled));
++ kgem_bo_submit(&sna->kgem, frame.bo);
++
++ frame.id = FOURCC_RGB888;
++ frame.src = frame.image = r.extents;
++ frame.width = frame.image.x2;
++ frame.height = frame.image.y2;
++ frame.pitch[0] = frame.bo->pitch;
++
++ screen->DestroyPixmap(scaled);
++ cache_bo = false;
+ }
+
+ ret = Success;
+@@ -517,24 +582,16 @@ off:
+ ret = BadAlloc;
+ }
+
+- frame.bo->domain = DOMAIN_NONE;
+- if (xvmc_passthrough(format->id))
+- kgem_bo_destroy(&sna->kgem, frame.bo);
+- else
++ if (cache_bo)
+ sna_video_buffer_fini(video);
++ else
++ kgem_bo_destroy(&sna->kgem, frame.bo);
+
+ if (ret != Success)
+ goto err;
+ }
+
+- if (!video->AlwaysOnTop && !RegionEqual(&video->clip, &clip) &&
+- sna_blt_fill_boxes(sna, GXcopy,
+- __sna_pixmap_get_bo(sna->front),
+- sna->front->drawable.bitsPerPixel,
+- video->color_key,
+- region_rects(&clip),
+- region_num_rects(&clip)))
+- RegionCopy(&video->clip, &clip);
++ sna_video_fill_colorkey(video, &clip);
+ sna_window_set_port((WindowPtr)draw, port);
+
+ return Success;
+@@ -606,25 +663,28 @@ static int sna_video_sprite_color_key(struct sna *sna)
+ return color_key & ((1 << scrn->depth) - 1);
+ }
+
+-static bool sna_video_has_sprites(struct sna *sna)
++static int sna_video_has_sprites(struct sna *sna)
+ {
+ xf86CrtcConfigPtr config = XF86_CRTC_CONFIG_PTR(sna->scrn);
++ unsigned min;
+ int i;
+
+ DBG(("%s: num_crtc=%d\n", __FUNCTION__, sna->mode.num_real_crtc));
+
+ if (sna->mode.num_real_crtc == 0)
+- return false;
++ return 0;
+
++ min = -1;
+ for (i = 0; i < sna->mode.num_real_crtc; i++) {
+- if (!sna_crtc_to_sprite(config->crtc[i])) {
+- DBG(("%s: no sprite found on pipe %d\n", __FUNCTION__, sna_crtc_to_pipe(config->crtc[i])));
+- return false;
+- }
++ unsigned count = sna_crtc_count_sprites(config->crtc[i]);
++ DBG(("%s: %d sprites found on pipe %d\n", __FUNCTION__,
++ count, sna_crtc_pipe(config->crtc[i])));
++ if (count < min)
++ min = count;
+ }
+
+- DBG(("%s: yes\n", __FUNCTION__));
+- return true;
++ DBG(("%s: min=%d\n", __FUNCTION__, min));
++ return min;
+ }
+
+ void sna_video_sprite_setup(struct sna *sna, ScreenPtr screen)
+@@ -632,16 +692,18 @@ void sna_video_sprite_setup(struct sna *sna, ScreenPtr screen)
+ XvAdaptorPtr adaptor;
+ struct sna_video *video;
+ XvPortPtr port;
++ int count, i;
+
+- if (!sna_video_has_sprites(sna))
++ count = sna_video_has_sprites(sna);
++ if (!count)
+ return;
+
+ adaptor = sna_xv_adaptor_alloc(sna);
+ if (!adaptor)
+ return;
+
+- video = calloc(1, sizeof(*video));
+- port = calloc(1, sizeof(*port));
++ video = calloc(count, sizeof(*video));
++ port = calloc(count, sizeof(*port));
+ if (video == NULL || port == NULL) {
+ free(video);
+ free(port);
+@@ -686,36 +748,43 @@ void sna_video_sprite_setup(struct sna *sna, ScreenPtr screen)
+ adaptor->ddPutImage = sna_video_sprite_put_image;
+ adaptor->ddQueryImageAttributes = sna_video_sprite_query;
+
+- adaptor->nPorts = 1;
++ adaptor->nPorts = count;
+ adaptor->pPorts = port;
+
+- adaptor->base_id = port->id = FakeClientID(0);
+- AddResource(port->id, XvGetRTPort(), port);
+- port->pAdaptor = adaptor;
+- port->pNotify = NULL;
+- port->pDraw = NULL;
+- port->client = NULL;
+- port->grab.client = NULL;
+- port->time = currentTime;
+- port->devPriv.ptr = video;
+-
+- video->sna = sna;
+- video->alignment = 64;
+- video->color_key = sna_video_sprite_color_key(sna);
+- video->color_key_changed = ~0;
+- video->has_color_key = true;
+- video->brightness = -19; /* (255/219) * -16 */
+- video->contrast = 75; /* 255/219 * 64 */
+- video->saturation = 146; /* 128/112 * 128 */
+- video->desired_crtc = NULL;
+- video->gamma5 = 0xc0c0c0;
+- video->gamma4 = 0x808080;
+- video->gamma3 = 0x404040;
+- video->gamma2 = 0x202020;
+- video->gamma1 = 0x101010;
+- video->gamma0 = 0x080808;
+- RegionNull(&video->clip);
+- video->SyncToVblank = 1;
++ for (i = 0; i < count; i++) {
++ port->id = FakeClientID(0);
++ AddResource(port->id, XvGetRTPort(), port);
++ port->pAdaptor = adaptor;
++ port->pNotify = NULL;
++ port->pDraw = NULL;
++ port->client = NULL;
++ port->grab.client = NULL;
++ port->time = currentTime;
++ port->devPriv.ptr = video;
++
++ video->sna = sna;
++ video->idx = i;
++ video->alignment = 64;
++ video->color_key = sna_video_sprite_color_key(sna);
++ video->color_key_changed = ~0;
++ video->has_color_key = true;
++ video->brightness = -19; /* (255/219) * -16 */
++ video->contrast = 75; /* 255/219 * 64 */
++ video->saturation = 146; /* 128/112 * 128 */
++ video->desired_crtc = NULL;
++ video->gamma5 = 0xc0c0c0;
++ video->gamma4 = 0x808080;
++ video->gamma3 = 0x404040;
++ video->gamma2 = 0x202020;
++ video->gamma1 = 0x101010;
++ video->gamma0 = 0x080808;
++ RegionNull(&video->clip);
++ video->SyncToVblank = 1;
++
++ port++;
++ video++;
++ }
++ adaptor->base_id = adaptor->pPorts[0].id;
+
+ xvColorKey = MAKE_ATOM("XV_COLORKEY");
+ xvAlwaysOnTop = MAKE_ATOM("XV_ALWAYS_ON_TOP");
+diff --git a/src/sna/sna_video_textured.c b/src/sna/sna_video_textured.c
+index 95011939..3cce5cf1 100644
+--- a/src/sna/sna_video_textured.c
++++ b/src/sna/sna_video_textured.c
+@@ -48,7 +48,12 @@ static const XvAttributeRec Attributes[] = {
+ //{XvSettable | XvGettable, 0, 255, (char *)"XV_CONTRAST"},
+ };
+
+-static const XvImageRec Images[] = {
++static const XvImageRec gen2_Images[] = {
++ XVIMAGE_YUY2,
++ XVIMAGE_UYVY,
++};
++
++static const XvImageRec gen3_Images[] = {
+ XVIMAGE_YUY2,
+ XVIMAGE_YV12,
+ XVIMAGE_I420,
+@@ -149,15 +154,16 @@ sna_video_textured_put_image(ddPutImage_ARGS)
+ BoxRec dstBox;
+ RegionRec clip;
+ xf86CrtcPtr crtc;
++ int16_t dx, dy;
+ bool flush = false;
+ bool ret;
+
+- clip.extents.x1 = draw->x + drw_x;
+- clip.extents.y1 = draw->y + drw_y;
+- clip.extents.x2 = clip.extents.x1 + drw_w;
+- clip.extents.y2 = clip.extents.y1 + drw_h;
+- clip.data = NULL;
++ if (wedged(sna))
++ return BadAlloc;
+
++ init_video_region(&clip, draw, drw_x, drw_y, drw_w, drw_h);
++
++ ValidateGC(draw, gc);
+ RegionIntersect(&clip, &clip, gc->pCompositeClip);
+ if (!RegionNotEmpty(&clip))
+ return Success;
+@@ -181,6 +187,9 @@ sna_video_textured_put_image(ddPutImage_ARGS)
+ &clip))
+ return Success;
+
++ if (get_drawable_deltas(draw, pixmap, &dx, &dy))
++ RegionTranslate(&clip, dx, dy);
++
+ flags = MOVE_WRITE | __MOVE_FORCE;
+ if (clip.data)
+ flags |= MOVE_READ;
+@@ -234,7 +243,7 @@ sna_video_textured_put_image(ddPutImage_ARGS)
+ DBG(("%s: failed to render video\n", __FUNCTION__));
+ ret = BadAlloc;
+ } else
+- DamageDamageRegion(draw, &clip);
++ DamageDamageRegion(&pixmap->drawable, &clip);
+
+ kgem_bo_destroy(&sna->kgem, frame.bo);
+
+@@ -316,7 +325,7 @@ void sna_video_textured_setup(struct sna *sna, ScreenPtr screen)
+
+ if (!sna->render.video) {
+ xf86DrvMsg(sna->scrn->scrnIndex, X_INFO,
+- "Textured video not supported on this hardware\n");
++ "Textured video not supported on this hardware or backend\n");
+ return;
+ }
+
+@@ -362,8 +371,13 @@ void sna_video_textured_setup(struct sna *sna, ScreenPtr screen)
+ ARRAY_SIZE(Formats));
+ adaptor->nAttributes = ARRAY_SIZE(Attributes);
+ adaptor->pAttributes = (XvAttributeRec *)Attributes;
+- adaptor->nImages = ARRAY_SIZE(Images);
+- adaptor->pImages = (XvImageRec *)Images;
++ if (sna->kgem.gen < 030) {
++ adaptor->nImages = ARRAY_SIZE(gen2_Images);
++ adaptor->pImages = (XvImageRec *)gen2_Images;
++ } else {
++ adaptor->nImages = ARRAY_SIZE(gen3_Images);
++ adaptor->pImages = (XvImageRec *)gen3_Images;
++ }
+ #if XORG_XV_VERSION < 2
+ adaptor->ddAllocatePort = sna_xv_alloc_port;
+ adaptor->ddFreePort = sna_xv_free_port;
+diff --git a/src/sna/xassert.h b/src/sna/xassert.h
+index 1bcfd080..e648e4bc 100644
+--- a/src/sna/xassert.h
++++ b/src/sna/xassert.h
+@@ -43,6 +43,28 @@
+ xorg_backtrace(); \
+ FatalError("%s:%d assertion '%s' failed\n", __func__, __LINE__, #E); \
+ } while (0)
++
++#define warn_unless(E) \
++({ \
++ bool fail = !(E); \
++ if (unlikely(fail)) { \
++ static int __warn_once__; \
++ if (!__warn_once__) { \
++ xorg_backtrace(); \
++ ErrorF("%s:%d assertion '%s' failed\n", __func__, __LINE__, #E); \
++ __warn_once__ = 1; \
++ } \
++ } \
++ unlikely(fail); \
++})
++
++#define dbg(EXPR) EXPR
++
++#else
++
++#define warn_unless(E) ({ bool fail = !(E); unlikely(fail); })
++#define dbg(EXPR)
++
+ #endif
+
+ #endif /* __XASSERT_H__ */
+diff --git a/src/uxa/i830_reg.h b/src/uxa/i830_reg.h
+index d8306bcd..ba39d82c 100644
+--- a/src/uxa/i830_reg.h
++++ b/src/uxa/i830_reg.h
+@@ -65,6 +65,12 @@
+ #define MI_LOAD_SCAN_LINES_DISPLAY_PIPEA (0)
+ #define MI_LOAD_SCAN_LINES_DISPLAY_PIPEB (0x1<<20)
+
++#define MI_LOAD_REGISTER_IMM (0x22<<23 | (3-2))
++
++#define BCS_SWCTRL 0x22200
++# define BCS_SWCTRL_SRC_Y (1 << 0)
++# define BCS_SWCTRL_DST_Y (1 << 1)
++
+ /* BLT commands */
+ #define COLOR_BLT_CMD ((2<<29)|(0x40<<22)|(0x3))
+ #define COLOR_BLT_WRITE_ALPHA (1<<21)
+diff --git a/src/uxa/i965_video.c b/src/uxa/i965_video.c
+index 68e6fd38..438ab909 100644
+--- a/src/uxa/i965_video.c
++++ b/src/uxa/i965_video.c
+@@ -37,7 +37,6 @@
+ #include "fourcc.h"
+
+ #include "intel.h"
+-#include "intel_xvmc.h"
+ #include "intel_uxa.h"
+ #include "i830_reg.h"
+ #include "i965_reg.h"
+diff --git a/src/uxa/intel.h b/src/uxa/intel.h
+index 1b7e5339..a5e77af4 100644
+--- a/src/uxa/intel.h
++++ b/src/uxa/intel.h
+@@ -121,7 +121,6 @@ typedef struct intel_screen_private {
+
+ void *modes;
+ drm_intel_bo *front_buffer, *back_buffer;
+- unsigned int back_name;
+ long front_pitch, front_tiling;
+
+ dri_bufmgr *bufmgr;
+@@ -169,6 +168,7 @@ typedef struct intel_screen_private {
+ const struct intel_device_info *info;
+
+ unsigned int BR[20];
++ unsigned int BR_tiling[2];
+
+ CloseScreenProcPtr CloseScreen;
+
+@@ -196,7 +196,9 @@ typedef struct intel_screen_private {
+
+ int colorKey;
+ XF86VideoAdaptorPtr adaptor;
++#if !HAVE_NOTIFY_FD
+ ScreenBlockHandlerProcPtr BlockHandler;
++#endif
+ Bool overlayOn;
+
+ struct {
+@@ -285,8 +287,6 @@ typedef struct intel_screen_private {
+ Bool has_kernel_flush;
+ Bool needs_flush;
+
+- struct _DRI2FrameEvent *pending_flip[MAX_PIPES];
+-
+ /* Broken-out options. */
+ OptionInfoPtr Options;
+
+@@ -368,6 +368,7 @@ typedef void (*intel_drm_abort_proc)(ScrnInfoPtr scrn,
+
+ extern uint32_t intel_drm_queue_alloc(ScrnInfoPtr scrn, xf86CrtcPtr crtc, void *data, intel_drm_handler_proc handler, intel_drm_abort_proc abort);
+ extern void intel_drm_abort(ScrnInfoPtr scrn, Bool (*match)(void *data, void *match_data), void *match_data);
++extern void intel_drm_abort_seq(ScrnInfoPtr scrn, uint32_t seq);
+
+ extern int intel_get_pipe_from_crtc_id(drm_intel_bufmgr *bufmgr, xf86CrtcPtr crtc);
+ extern int intel_crtc_id(xf86CrtcPtr crtc);
+@@ -408,7 +409,6 @@ typedef struct _DRI2FrameEvent {
+ ClientPtr client;
+ enum DRI2FrameEventType type;
+ int frame;
+- int pipe;
+
+ struct list drawable_resource, client_resource;
+
+@@ -418,7 +418,12 @@ typedef struct _DRI2FrameEvent {
+ DRI2BufferPtr front;
+ DRI2BufferPtr back;
+
+- struct _DRI2FrameEvent *chain;
++ /* current scanout for triple buffer */
++ int old_width;
++ int old_height;
++ int old_pitch;
++ int old_tiling;
++ dri_bo *old_buffer;
+ } DRI2FrameEventRec, *DRI2FrameEventPtr;
+
+ extern Bool intel_do_pageflip(intel_screen_private *intel,
+@@ -456,10 +461,6 @@ extern xf86CrtcPtr intel_covering_crtc(ScrnInfoPtr scrn, BoxPtr box,
+
+ Bool I830DRI2ScreenInit(ScreenPtr pScreen);
+ void I830DRI2CloseScreen(ScreenPtr pScreen);
+-void I830DRI2FrameEventHandler(unsigned int frame, unsigned int tv_sec,
+- unsigned int tv_usec, DRI2FrameEventPtr flip_info);
+-void I830DRI2FlipEventHandler(unsigned int frame, unsigned int tv_sec,
+- unsigned int tv_usec, DRI2FrameEventPtr flip_info);
+
+ /* intel_dri3.c */
+ Bool intel_dri3_screen_init(ScreenPtr screen);
+diff --git a/src/uxa/intel_batchbuffer.c b/src/uxa/intel_batchbuffer.c
+index a29e4434..114c6026 100644
+--- a/src/uxa/intel_batchbuffer.c
++++ b/src/uxa/intel_batchbuffer.c
+@@ -245,6 +245,17 @@ void intel_batch_submit(ScrnInfoPtr scrn)
+ if (intel->batch_used == 0)
+ return;
+
++ if (intel->current_batch == I915_EXEC_BLT &&
++ INTEL_INFO(intel)->gen >= 060) {
++ OUT_BATCH(MI_FLUSH_DW);
++ OUT_BATCH(0);
++ OUT_BATCH(0);
++ OUT_BATCH(0);
++ OUT_BATCH(MI_LOAD_REGISTER_IMM);
++ OUT_BATCH(BCS_SWCTRL);
++ OUT_BATCH((BCS_SWCTRL_DST_Y | BCS_SWCTRL_SRC_Y) << 16);
++ }
++
+ /* Mark the end of the batchbuffer. */
+ OUT_BATCH(MI_BATCH_BUFFER_END);
+ /* Emit a padding dword if we aren't going to be quad-word aligned. */
+diff --git a/src/uxa/intel_batchbuffer.h b/src/uxa/intel_batchbuffer.h
+index e5fb8d08..e71ffd19 100644
+--- a/src/uxa/intel_batchbuffer.h
++++ b/src/uxa/intel_batchbuffer.h
+@@ -30,7 +30,7 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ #ifndef _INTEL_BATCHBUFFER_H
+ #define _INTEL_BATCHBUFFER_H
+
+-#define BATCH_RESERVED 16
++#define BATCH_RESERVED 64
+
+
+ void intel_batch_init(ScrnInfoPtr scrn);
+@@ -202,6 +202,23 @@ do { \
+
+ #define BEGIN_BATCH(n) __BEGIN_BATCH(n,RENDER_BATCH)
+ #define BEGIN_BATCH_BLT(n) __BEGIN_BATCH(n,BLT_BATCH)
++#define BEGIN_BATCH_BLT_TILED(n) \
++do { \
++ if (INTEL_INFO(intel)->gen < 060) { \
++ __BEGIN_BATCH(n, BLT_BATCH); \
++ } else { \
++ __BEGIN_BATCH(n+7, BLT_BATCH); \
++ OUT_BATCH(MI_FLUSH_DW); \
++ OUT_BATCH(0); \
++ OUT_BATCH(0); \
++ OUT_BATCH(0); \
++ OUT_BATCH(MI_LOAD_REGISTER_IMM); \
++ OUT_BATCH(BCS_SWCTRL); \
++ OUT_BATCH((BCS_SWCTRL_DST_Y | BCS_SWCTRL_SRC_Y) << 16 | \
++ ((intel->BR_tiling[0] == I915_TILING_Y) ? BCS_SWCTRL_DST_Y : 0) | \
++ ((intel->BR_tiling[1] == I915_TILING_Y) ? BCS_SWCTRL_SRC_Y : 0)); \
++ } \
++} while (0)
+
+ #define ADVANCE_BATCH() do { \
+ if (intel->batch_emitting == 0) \
+diff --git a/src/uxa/intel_display.c b/src/uxa/intel_display.c
+index 7b4d4e0c..809cda1d 100644
+--- a/src/uxa/intel_display.c
++++ b/src/uxa/intel_display.c
+@@ -89,11 +89,11 @@ struct intel_mode {
+ struct list outputs;
+ struct list crtcs;
+
+- void *pageflip_data;
+- intel_pageflip_handler_proc pageflip_handler;
+- intel_pageflip_abort_proc pageflip_abort;
+-
+- Bool delete_dp_12_displays;
++ struct {
++ intel_pageflip_handler_proc handler;
++ intel_pageflip_abort_proc abort;
++ void *data;
++ } pageflip;
+ };
+
+ struct intel_pageflip {
+@@ -114,7 +114,6 @@ struct intel_crtc {
+ struct list link;
+ PixmapPtr scanout_pixmap;
+ uint32_t scanout_fb_id;
+- int32_t vblank_offset;
+ uint32_t msc_prev;
+ uint64_t msc_high;
+ };
+@@ -193,7 +192,7 @@ intel_output_backlight_init(xf86OutputPtr output)
+
+ str = xf86GetOptValString(intel->Options, OPTION_BACKLIGHT);
+ if (str != NULL) {
+- if (backlight_exists(str) != BL_NONE) {
++ if (backlight_exists(str)) {
+ intel_output->backlight_active_level =
+ backlight_open(&intel_output->backlight,
+ strdup(str));
+@@ -689,9 +688,11 @@ intel_set_scanout_pixmap(xf86CrtcPtr crtc, PixmapPtr ppix)
+ }
+
+ bo = intel_get_pixmap_bo(ppix);
+- if (intel->front_buffer) {
+- ErrorF("have front buffer\n");
+- }
++ if (!bo)
++ return FALSE;
++
++ if (intel->front_buffer)
++ return FALSE;
+
+ drm_intel_bo_disable_reuse(bo);
+
+@@ -867,6 +868,48 @@ intel_output_attach_edid(xf86OutputPtr output)
+ xf86OutputSetEDID(output, mon);
+ }
+
++static void
++intel_output_attach_tile(xf86OutputPtr output)
++{
++#if XF86_OUTPUT_VERSION >= 3
++ struct intel_output *intel_output = output->driver_private;
++ drmModeConnectorPtr koutput = intel_output->mode_output;
++ struct intel_mode *mode = intel_output->mode;
++ drmModePropertyBlobPtr blob = NULL;
++ struct xf86CrtcTileInfo tile_info, *set = NULL;
++ int i;
++
++ for (i = 0; koutput && i < koutput->count_props; i++) {
++ drmModePropertyPtr props;
++
++ props = drmModeGetProperty(mode->fd, koutput->props[i]);
++ if (!props)
++ continue;
++
++ if (!(props->flags & DRM_MODE_PROP_BLOB)) {
++ drmModeFreeProperty(props);
++ continue;
++ }
++
++ if (!strcmp(props->name, "TILE")) {
++ blob = drmModeGetPropertyBlob(mode->fd,
++ koutput->prop_values[i]);
++ }
++ drmModeFreeProperty(props);
++ }
++
++ if (blob) {
++ if (xf86OutputParseKMSTile(blob->data,
++ blob->length,
++ &tile_info))
++ set = &tile_info;
++ drmModeFreePropertyBlob(blob);
++ }
++
++ xf86OutputSetTile(output, set);
++#endif
++}
++
+ static DisplayModePtr
+ intel_output_panel_edid(xf86OutputPtr output, DisplayModePtr modes)
+ {
+@@ -922,6 +965,7 @@ intel_output_get_modes(xf86OutputPtr output)
+ int i;
+
+ intel_output_attach_edid(output);
++ intel_output_attach_tile(output);
+
+ if (!koutput)
+ return Modes;
+@@ -1492,6 +1536,7 @@ intel_output_init(ScrnInfoPtr scrn, struct intel_mode *mode, drmModeResPtr mode_
+ intel_output = output->driver_private;
+ intel_output->output_id = mode_res->connectors[num];
+ intel_output->mode_output = koutput;
++ RROutputChanged(output->randr_output, TRUE);
+ return;
+ }
+ }
+@@ -1650,9 +1695,6 @@ intel_pageflip_abort(ScrnInfoPtr scrn, xf86CrtcPtr crtc, void *data);
+ static void
+ intel_pageflip_complete(struct intel_mode *mode);
+
+-static void
+-intel_drm_abort_seq (ScrnInfoPtr scrn, uint32_t seq);
+-
+ Bool
+ intel_do_pageflip(intel_screen_private *intel,
+ dri_bo *new_front,
+@@ -1671,23 +1713,30 @@ intel_do_pageflip(intel_screen_private *intel,
+ uint32_t new_fb_id;
+ uint32_t flags;
+ uint32_t seq;
++ int err = 0;
+ int i;
+
+ /*
++ * We only have a single length queue in the kernel, so any
++ * attempts to schedule a second flip before processing the first
++ * is a bug. Punt it back to the caller.
++ */
++ if (mode->flip_count)
++ return FALSE;
++
++ /*
+ * Create a new handle for the back buffer
+ */
+ if (drmModeAddFB(mode->fd, scrn->virtualX, scrn->virtualY,
+ scrn->depth, scrn->bitsPerPixel, pitch,
+- new_front->handle, &new_fb_id))
++ new_front->handle, &new_fb_id)) {
++ err = errno;
+ goto error_out;
++ }
+
+ drm_intel_bo_disable_reuse(new_front);
+ intel_flush(intel);
+
+- mode->pageflip_data = pageflip_data;
+- mode->pageflip_handler = pageflip_handler;
+- mode->pageflip_abort = pageflip_abort;
+-
+ /*
+ * Queue flips on all enabled CRTCs
+ * Note that if/when we get per-CRTC buffers, we'll have to update this.
+@@ -1699,6 +1748,7 @@ intel_do_pageflip(intel_screen_private *intel,
+ */
+ mode->fe_msc = 0;
+ mode->fe_usec = 0;
++ memset(&mode->pageflip, 0, sizeof(mode->pageflip));
+
+ flags = DRM_MODE_PAGE_FLIP_EVENT;
+ if (async)
+@@ -1711,8 +1761,7 @@ intel_do_pageflip(intel_screen_private *intel,
+
+ flip = calloc(1, sizeof(struct intel_pageflip));
+ if (flip == NULL) {
+- xf86DrvMsg(scrn->scrnIndex, X_WARNING,
+- "flip queue: carrier alloc failed.\n");
++ err = errno;
+ goto error_undo;
+ }
+
+@@ -1724,33 +1773,30 @@ intel_do_pageflip(intel_screen_private *intel,
+
+ seq = intel_drm_queue_alloc(scrn, config->crtc[i], flip, intel_pageflip_handler, intel_pageflip_abort);
+ if (!seq) {
++ err = errno;
+ free(flip);
+ goto error_undo;
+ }
+
+-again:
++ mode->flip_count++;
++
+ if (drmModePageFlip(mode->fd,
+ crtc_id(crtc),
+ new_fb_id,
+ flags, (void *)(uintptr_t)seq)) {
+- if (intel_mode_read_drm_events(intel)) {
+- xf86DrvMsg(scrn->scrnIndex, X_WARNING,
+- "flip queue retry\n");
+- goto again;
+- }
+- xf86DrvMsg(scrn->scrnIndex, X_WARNING,
+- "flip queue failed: %s\n", strerror(errno));
+- if (seq)
+- intel_drm_abort_seq(scrn, seq);
+- free(flip);
++ err = errno;
++ intel_drm_abort_seq(scrn, seq);
+ goto error_undo;
+ }
+- mode->flip_count++;
+ }
+
+ mode->old_fb_id = mode->fb_id;
+ mode->fb_id = new_fb_id;
+
++ mode->pageflip.data = pageflip_data;
++ mode->pageflip.handler = pageflip_handler;
++ mode->pageflip.abort = pageflip_abort;
++
+ if (!mode->flip_count)
+ intel_pageflip_complete(mode);
+
+@@ -1765,7 +1811,7 @@ error_undo:
+
+ error_out:
+ xf86DrvMsg(scrn->scrnIndex, X_WARNING, "Page flip failed: %s\n",
+- strerror(errno));
++ strerror(err));
+
+ mode->flip_count = 0;
+ return FALSE;
+@@ -1839,7 +1885,7 @@ intel_drm_abort(ScrnInfoPtr scrn, Bool (*match)(void *data, void *match_data), v
+ /*
+ * Abort by drm queue sequence number
+ */
+-static void
++void
+ intel_drm_abort_seq(ScrnInfoPtr scrn, uint32_t seq)
+ {
+ struct intel_drm_queue *q;
+@@ -1911,7 +1957,6 @@ intel_sequence_to_crtc_msc(xf86CrtcPtr crtc, uint32_t sequence)
+ {
+ struct intel_crtc *intel_crtc = crtc->driver_private;
+
+- sequence += intel_crtc->vblank_offset;
+ if ((int32_t) (sequence - intel_crtc->msc_prev) < -0x40000000)
+ intel_crtc->msc_high += 0x100000000L;
+ intel_crtc->msc_prev = sequence;
+@@ -1935,37 +1980,10 @@ intel_get_crtc_msc_ust(ScrnInfoPtr scrn, xf86CrtcPtr crtc, uint64_t *msc, uint64
+ return 0;
+ }
+
+-/*
+- * Convert a 64-bit adjusted MSC value into a 32-bit kernel sequence number,
+- * removing the high 32 bits and subtracting out the vblank_offset term.
+- *
+- * This also updates the vblank_offset when it notices that the value should
+- * change.
+- */
+-
+-#define MAX_VBLANK_OFFSET 1000
+-
+ uint32_t
+ intel_crtc_msc_to_sequence(ScrnInfoPtr scrn, xf86CrtcPtr crtc, uint64_t expect)
+ {
+- struct intel_crtc *intel_crtc = crtc->driver_private;
+- uint64_t msc, ust;
+-
+- if (intel_get_crtc_msc_ust(scrn, crtc, &msc, &ust) == 0) {
+- int64_t diff = expect - msc;
+-
+- /* We're way off here, assume that the kernel has lost its mind
+- * and smack the vblank back to something sensible
+- */
+- if (diff < -MAX_VBLANK_OFFSET || diff > MAX_VBLANK_OFFSET) {
+- intel_crtc->vblank_offset += (int32_t) diff;
+- if (intel_crtc->vblank_offset > -MAX_VBLANK_OFFSET &&
+- intel_crtc->vblank_offset < MAX_VBLANK_OFFSET)
+- intel_crtc->vblank_offset = 0;
+- }
+- }
+-
+- return (uint32_t) (expect - intel_crtc->vblank_offset);
++ return (uint32_t)expect;
+ }
+
+ /*
+@@ -1998,14 +2016,13 @@ intel_drm_handler(int fd, uint32_t frame, uint32_t sec, uint32_t usec, void *use
+ static void
+ intel_pageflip_complete(struct intel_mode *mode)
+ {
+- /* Release framebuffer */
+- drmModeRmFB(mode->fd, mode->old_fb_id);
+-
+- if (!mode->pageflip_handler)
++ if (!mode->pageflip.handler)
+ return;
+
+- mode->pageflip_handler(mode->fe_msc, mode->fe_usec,
+- mode->pageflip_data);
++ /* Release framebuffer */
++ drmModeRmFB(mode->fd, mode->old_fb_id);
++ mode->pageflip.handler(mode->fe_msc, mode->fe_usec,
++ mode->pageflip.data);
+ }
+
+ /*
+@@ -2045,6 +2062,7 @@ intel_pageflip_handler(ScrnInfoPtr scrn, xf86CrtcPtr crtc,
+
+ if (!mode)
+ return;
++
+ intel_pageflip_complete(mode);
+ }
+
+@@ -2060,18 +2078,18 @@ intel_pageflip_abort(ScrnInfoPtr scrn, xf86CrtcPtr crtc, void *data)
+ if (!mode)
+ return;
+
+- /* Release framebuffer */
+- drmModeRmFB(mode->fd, mode->old_fb_id);
+-
+- if (!mode->pageflip_abort)
++ if (!mode->pageflip.abort)
+ return;
+
+- mode->pageflip_abort(mode->pageflip_data);
++ /* Release framebuffer */
++ drmModeRmFB(mode->fd, mode->old_fb_id);
++ mode->pageflip.abort(mode->pageflip.data);
+ }
+
+ /*
+ * Check for pending DRM events and process them.
+ */
++#if !HAVE_NOTIFY_FD
+ static void
+ drm_wakeup_handler(pointer data, int err, pointer p)
+ {
+@@ -2086,6 +2104,14 @@ drm_wakeup_handler(pointer data, int err, pointer p)
+ if (FD_ISSET(mode->fd, read_mask))
+ drmHandleEvent(mode->fd, &mode->event_context);
+ }
++#else
++static void
++drm_notify_fd(int fd, int ready, void *data)
++{
++ struct intel_mode *mode = data;
++ drmHandleEvent(mode->fd, &mode->event_context);
++}
++#endif
+
+ /*
+ * If there are any available, read drm_events
+@@ -2231,10 +2257,6 @@ Bool intel_mode_pre_init(ScrnInfoPtr scrn, int fd, int cpp)
+ intel->use_pageflipping = TRUE;
+ }
+
+- if (xf86ReturnOptValBool(intel->Options, OPTION_DELETE_DP12, FALSE)) {
+- mode->delete_dp_12_displays = TRUE;
+- }
+-
+ intel->modes = mode;
+ drmModeFreeResources(mode_res);
+ return TRUE;
+@@ -2250,9 +2272,11 @@ intel_mode_init(struct intel_screen_private *intel)
+ * registration within ScreenInit and not PreInit.
+ */
+ mode->flip_count = 0;
+- AddGeneralSocket(mode->fd);
++ SetNotifyFd(mode->fd, drm_notify_fd, X_NOTIFY_READ, mode);
++#if !HAVE_NOTIFY_FD
+ RegisterBlockAndWakeupHandlers((BlockHandlerProcPtr)NoopDDA,
+ drm_wakeup_handler, mode);
++#endif
+ }
+
+ void
+@@ -2276,9 +2300,11 @@ intel_mode_close(intel_screen_private *intel)
+
+ intel_drm_abort_scrn(intel->scrn);
+
++#if !HAVE_NOTIFY_FD
+ RemoveBlockAndWakeupHandlers((BlockHandlerProcPtr)NoopDDA,
+ drm_wakeup_handler, mode);
+- RemoveGeneralSocket(mode->fd);
++#endif
++ RemoveNotifyFd(mode->fd);
+ }
+
+ void
+@@ -2498,12 +2524,11 @@ intel_mode_hotplug(struct intel_screen_private *intel)
+ int i, j;
+ Bool found;
+ Bool changed = FALSE;
+- struct intel_mode *mode = intel->modes;
++
+ mode_res = drmModeGetResources(intel->drmSubFD);
+ if (!mode_res)
+ goto out;
+
+-restart_destroy:
+ for (i = 0; i < config->num_output; i++) {
+ xf86OutputPtr output = config->output[i];
+ struct intel_output *intel_output;
+@@ -2522,13 +2547,9 @@ restart_destroy:
+ drmModeFreeConnector(intel_output->mode_output);
+ intel_output->mode_output = NULL;
+ intel_output->output_id = -1;
++ RROutputChanged(output->randr_output, TRUE);
+
+ changed = TRUE;
+- if (mode->delete_dp_12_displays) {
+- RROutputDestroy(output->randr_output);
+- xf86OutputDestroy(output);
+- goto restart_destroy;
+- }
+ }
+
+ /* find new output ids we don't have outputs for */
+@@ -2552,10 +2573,8 @@ restart_destroy:
+ intel_output_init(scrn, intel->modes, mode_res, i, 1);
+ }
+
+- if (changed) {
+- RRSetChanged(xf86ScrnToScreen(scrn));
++ if (changed)
+ RRTellChanged(xf86ScrnToScreen(scrn));
+- }
+
+ drmModeFreeResources(mode_res);
+ out:
+diff --git a/src/uxa/intel_dri.c b/src/uxa/intel_dri.c
+index f61c6210..524826d2 100644
+--- a/src/uxa/intel_dri.c
++++ b/src/uxa/intel_dri.c
+@@ -81,6 +81,47 @@ static DevPrivateKeyRec i830_client_key;
+ static int i830_client_key;
+ #endif
+
++static void I830DRI2FlipEventHandler(unsigned int frame,
++ unsigned int tv_sec,
++ unsigned int tv_usec,
++ DRI2FrameEventPtr flip_info);
++
++static void I830DRI2FrameEventHandler(unsigned int frame,
++ unsigned int tv_sec,
++ unsigned int tv_usec,
++ DRI2FrameEventPtr swap_info);
++
++static void
++i830_dri2_del_frame_event(DRI2FrameEventPtr info);
++
++static uint32_t pipe_select(int pipe)
++{
++ if (pipe > 1)
++ return pipe << DRM_VBLANK_HIGH_CRTC_SHIFT;
++ else if (pipe > 0)
++ return DRM_VBLANK_SECONDARY;
++ else
++ return 0;
++}
++
++static void
++intel_dri2_vblank_handler(ScrnInfoPtr scrn,
++ xf86CrtcPtr crtc,
++ uint64_t msc,
++ uint64_t usec,
++ void *data)
++{
++ I830DRI2FrameEventHandler((uint32_t) msc, usec / 1000000, usec % 1000000, data);
++}
++
++static void
++intel_dri2_vblank_abort(ScrnInfoPtr scrn,
++ xf86CrtcPtr crtc,
++ void *data)
++{
++ i830_dri2_del_frame_event(data);
++}
++
+ static uint32_t pixmap_flink(PixmapPtr pixmap)
+ {
+ struct intel_uxa_pixmap *priv = intel_uxa_get_pixmap_private(pixmap);
+@@ -135,9 +176,6 @@ I830DRI2CreateBuffers(DrawablePtr drawable, unsigned int *attachments,
+ pixmap = NULL;
+ if (attachments[i] == DRI2BufferFrontLeft) {
+ pixmap = get_front_buffer(drawable);
+-
+- if (pixmap == NULL)
+- drawable = &(get_drawable_pixmap(drawable)->drawable);
+ } else if (attachments[i] == DRI2BufferStencil && pDepthPixmap) {
+ pixmap = pDepthPixmap;
+ pixmap->refcnt++;
+@@ -246,11 +284,8 @@ I830DRI2CreateBuffer(DrawablePtr drawable, unsigned int attachment,
+ }
+
+ pixmap = NULL;
+- if (attachment == DRI2BufferFrontLeft) {
++ if (attachment == DRI2BufferFrontLeft)
+ pixmap = get_front_buffer(drawable);
+- if (pixmap == NULL)
+- drawable = &(get_drawable_pixmap(drawable)->drawable);
+- }
+
+ if (pixmap == NULL) {
+ unsigned int hint = INTEL_CREATE_PIXMAP_DRI2;
+@@ -673,6 +708,20 @@ i830_dri2_del_frame_event(DRI2FrameEventPtr info)
+ if (info->back)
+ I830DRI2DestroyBuffer(NULL, info->back);
+
++ if (info->old_buffer) {
++ /* Check that the old buffer still matches the front buffer
++ * in case a mode change occurred before we woke up.
++ */
++ if (info->intel->back_buffer == NULL &&
++ info->old_width == info->intel->scrn->virtualX &&
++ info->old_height == info->intel->scrn->virtualY &&
++ info->old_pitch == info->intel->front_pitch &&
++ info->old_tiling == info->intel->front_tiling)
++ info->intel->back_buffer = info->old_buffer;
++ else
++ dri_bo_unreference(info->old_buffer);
++ }
++
+ free(info);
+ }
+
+@@ -708,16 +757,14 @@ static void
+ I830DRI2ExchangeBuffers(struct intel_screen_private *intel, DRI2BufferPtr front, DRI2BufferPtr back)
+ {
+ I830DRI2BufferPrivatePtr front_priv, back_priv;
+- int tmp;
+ struct intel_uxa_pixmap *new_front;
+
+ front_priv = front->driverPrivate;
+ back_priv = back->driverPrivate;
+
+ /* Swap BO names so DRI works */
+- tmp = front->name;
+ front->name = back->name;
+- back->name = tmp;
++ back->name = pixmap_flink(front_priv->pixmap);
+
+ /* Swap pixmap bos */
+ new_front = intel_exchange_pixmap_buffers(intel,
+@@ -753,87 +800,30 @@ I830DRI2FlipAbort(void *pageflip_data)
+ i830_dri2_del_frame_event(info);
+ }
+
+-/*
+- * Our internal swap routine takes care of actually exchanging, blitting, or
+- * flipping buffers as necessary.
+- */
+ static Bool
+-I830DRI2ScheduleFlip(struct intel_screen_private *intel,
+- DrawablePtr draw,
+- DRI2FrameEventPtr info)
++allocate_back_buffer(struct intel_screen_private *intel)
+ {
+- I830DRI2BufferPrivatePtr priv = info->back->driverPrivate;
+- drm_intel_bo *new_back, *old_back;
+- int tmp_name;
+-
+- if (!intel->use_triple_buffer) {
+- info->type = DRI2_SWAP;
+- if (!intel_do_pageflip(intel,
+- get_pixmap_bo(priv),
+- info->pipe, FALSE, info,
+- I830DRI2FlipComplete,
+- I830DRI2FlipAbort))
+- return FALSE;
+-
+- I830DRI2ExchangeBuffers(intel, info->front, info->back);
+- return TRUE;
+- }
++ drm_intel_bo *bo;
++ int pitch;
++ uint32_t tiling;
+
+- if (intel->pending_flip[info->pipe]) {
+- assert(intel->pending_flip[info->pipe]->chain == NULL);
+- intel->pending_flip[info->pipe]->chain = info;
++ if (intel->back_buffer)
+ return TRUE;
+- }
+
+- if (intel->back_buffer == NULL) {
+- new_back = drm_intel_bo_alloc(intel->bufmgr, "front buffer",
+- intel->front_buffer->size, 0);
+- if (new_back == NULL)
+- return FALSE;
+-
+- if (intel->front_tiling != I915_TILING_NONE) {
+- uint32_t tiling = intel->front_tiling;
+- drm_intel_bo_set_tiling(new_back, &tiling, intel->front_pitch);
+- if (tiling != intel->front_tiling) {
+- drm_intel_bo_unreference(new_back);
+- return FALSE;
+- }
+- }
+-
+- drm_intel_bo_disable_reuse(new_back);
+- dri_bo_flink(new_back, &intel->back_name);
+- } else {
+- new_back = intel->back_buffer;
+- intel->back_buffer = NULL;
+- }
++ bo = intel_allocate_framebuffer(intel->scrn,
++ intel->scrn->virtualX,
++ intel->scrn->virtualY,
++ intel->cpp,
++ &pitch, &tiling);
++ if (bo == NULL)
++ return FALSE;
+
+- old_back = get_pixmap_bo(priv);
+- if (!intel_do_pageflip(intel, old_back, info->pipe, FALSE, info, I830DRI2FlipComplete, I830DRI2FlipAbort)) {
+- intel->back_buffer = new_back;
++ if (pitch != intel->front_pitch || tiling != intel->front_tiling) {
++ drm_intel_bo_unreference(bo);
+ return FALSE;
+ }
+- info->type = DRI2_SWAP_CHAIN;
+- intel->pending_flip[info->pipe] = info;
+-
+- priv = info->front->driverPrivate;
+-
+- /* Exchange the current front-buffer with the fresh bo */
+-
+- intel->back_buffer = intel->front_buffer;
+- drm_intel_bo_reference(intel->back_buffer);
+- intel_set_pixmap_bo(priv->pixmap, new_back);
+- drm_intel_bo_unreference(new_back);
+-
+- tmp_name = info->front->name;
+- info->front->name = intel->back_name;
+- intel->back_name = tmp_name;
+
+- /* Then flip DRI2 pointers and update the screen pixmap */
+- I830DRI2ExchangeBuffers(intel, info->front, info->back);
+- DRI2SwapComplete(info->client, draw, 0, 0, 0,
+- DRI2_EXCHANGE_COMPLETE,
+- info->event_complete,
+- info->event_data);
++ intel->back_buffer = bo;
+ return TRUE;
+ }
+
+@@ -889,8 +879,88 @@ can_exchange(DrawablePtr drawable, DRI2BufferPtr front, DRI2BufferPtr back)
+ return TRUE;
+ }
+
+-void I830DRI2FrameEventHandler(unsigned int frame, unsigned int tv_sec,
+- unsigned int tv_usec, DRI2FrameEventPtr swap_info)
++static Bool
++queue_flip(struct intel_screen_private *intel,
++ DrawablePtr draw,
++ DRI2FrameEventPtr info)
++{
++ xf86CrtcPtr crtc = I830DRI2DrawableCrtc(draw);
++ I830DRI2BufferPrivatePtr priv = info->back->driverPrivate;
++ drm_intel_bo *old_back = get_pixmap_bo(priv);
++
++ if (crtc == NULL)
++ return FALSE;
++
++ if (!can_exchange(draw, info->front, info->back))
++ return FALSE;
++
++ if (!intel_do_pageflip(intel, old_back,
++ intel_crtc_to_pipe(crtc),
++ FALSE, info,
++ I830DRI2FlipComplete, I830DRI2FlipAbort))
++ return FALSE;
++
++#if DRI2INFOREC_VERSION >= 6
++ if (intel->use_triple_buffer && allocate_back_buffer(intel)) {
++ info->old_width = intel->scrn->virtualX;
++ info->old_height = intel->scrn->virtualY;
++ info->old_pitch = intel->front_pitch;
++ info->old_tiling = intel->front_tiling;
++ info->old_buffer = intel->front_buffer;
++ dri_bo_reference(info->old_buffer);
++
++ priv = info->front->driverPrivate;
++ intel_set_pixmap_bo(priv->pixmap, intel->back_buffer);
++
++ dri_bo_unreference(intel->back_buffer);
++ intel->back_buffer = NULL;
++
++ DRI2SwapLimit(draw, 2);
++ } else
++ DRI2SwapLimit(draw, 1);
++#endif
++
++ /* Then flip DRI2 pointers and update the screen pixmap */
++ I830DRI2ExchangeBuffers(intel, info->front, info->back);
++ return TRUE;
++}
++
++static Bool
++queue_swap(struct intel_screen_private *intel,
++ DrawablePtr draw,
++ DRI2FrameEventPtr info)
++{
++ xf86CrtcPtr crtc = I830DRI2DrawableCrtc(draw);
++ drmVBlank vbl;
++
++ if (crtc == NULL)
++ return FALSE;
++
++ vbl.request.type =
++ DRM_VBLANK_RELATIVE |
++ DRM_VBLANK_EVENT |
++ pipe_select(intel_crtc_to_pipe(crtc));
++ vbl.request.sequence = 1;
++ vbl.request.signal =
++ intel_drm_queue_alloc(intel->scrn, crtc, info,
++ intel_dri2_vblank_handler,
++ intel_dri2_vblank_abort);
++ if (vbl.request.signal == 0)
++ return FALSE;
++
++ info->type = DRI2_SWAP;
++ if (drmWaitVBlank(intel->drmSubFD, &vbl)) {
++ intel_drm_abort_seq(intel->scrn, vbl.request.signal);
++ return FALSE;
++ }
++
++ return TRUE;
++}
++
++static void I830DRI2FrameEventHandler(unsigned int frame,
++ unsigned int tv_sec,
++ unsigned int tv_usec,
++ DRI2FrameEventPtr swap_info)
+ {
+ intel_screen_private *intel = swap_info->intel;
+ DrawablePtr drawable;
+@@ -906,24 +976,22 @@ void I830DRI2FrameEventHandler(unsigned int frame, unsigned int tv_sec,
+ return;
+ }
+
+-
+ switch (swap_info->type) {
+ case DRI2_FLIP:
+ /* If we can still flip... */
+- if (can_exchange(drawable, swap_info->front, swap_info->back) &&
+- I830DRI2ScheduleFlip(intel, drawable, swap_info))
+- return;
+-
+- /* else fall through to exchange/blit */
+- case DRI2_SWAP: {
+- I830DRI2FallbackBlitSwap(drawable,
+- swap_info->front, swap_info->back);
+- DRI2SwapComplete(swap_info->client, drawable, frame, tv_sec, tv_usec,
+- DRI2_BLIT_COMPLETE,
+- swap_info->client ? swap_info->event_complete : NULL,
+- swap_info->event_data);
+- break;
+- }
++ if (!queue_flip(intel, drawable, swap_info) &&
++ !queue_swap(intel, drawable, swap_info)) {
++ case DRI2_SWAP:
++ I830DRI2FallbackBlitSwap(drawable,
++ swap_info->front, swap_info->back);
++ DRI2SwapComplete(swap_info->client, drawable, frame, tv_sec, tv_usec,
++ DRI2_BLIT_COMPLETE,
++ swap_info->client ? swap_info->event_complete : NULL,
++ swap_info->event_data);
++ break;
++ }
++ return;
++
+ case DRI2_WAITMSC:
+ if (swap_info->client)
+ DRI2WaitMSCComplete(swap_info->client, drawable,
+@@ -939,12 +1007,13 @@ void I830DRI2FrameEventHandler(unsigned int frame, unsigned int tv_sec,
+ i830_dri2_del_frame_event(swap_info);
+ }
+
+-void I830DRI2FlipEventHandler(unsigned int frame, unsigned int tv_sec,
+- unsigned int tv_usec, DRI2FrameEventPtr flip_info)
++static void I830DRI2FlipEventHandler(unsigned int frame,
++ unsigned int tv_sec,
++ unsigned int tv_usec,
++ DRI2FrameEventPtr flip_info)
+ {
+ struct intel_screen_private *intel = flip_info->intel;
+ DrawablePtr drawable;
+- DRI2FrameEventPtr chain;
+
+ drawable = NULL;
+ if (flip_info->drawable_id)
+@@ -954,6 +1023,7 @@ void I830DRI2FlipEventHandler(unsigned int frame, unsigned int tv_sec,
+
+ /* We assume our flips arrive in order, so we don't check the frame */
+ switch (flip_info->type) {
++ case DRI2_FLIP:
+ case DRI2_SWAP:
+ if (!drawable)
+ break;
+@@ -984,35 +1054,6 @@ void I830DRI2FlipEventHandler(unsigned int frame, unsigned int tv_sec,
+ flip_info->event_data);
+ break;
+
+- case DRI2_SWAP_CHAIN:
+- assert(intel->pending_flip[flip_info->pipe] == flip_info);
+- intel->pending_flip[flip_info->pipe] = NULL;
+-
+- chain = flip_info->chain;
+- if (chain) {
+- DrawablePtr chain_drawable = NULL;
+- if (chain->drawable_id)
+- dixLookupDrawable(&chain_drawable,
+- chain->drawable_id,
+- serverClient,
+- M_ANY, DixWriteAccess);
+- if (chain_drawable == NULL) {
+- i830_dri2_del_frame_event(chain);
+- } else if (!can_exchange(chain_drawable, chain->front, chain->back) ||
+- !I830DRI2ScheduleFlip(intel, chain_drawable, chain)) {
+- I830DRI2FallbackBlitSwap(chain_drawable,
+- chain->front,
+- chain->back);
+-
+- DRI2SwapComplete(chain->client, chain_drawable, frame, tv_sec, tv_usec,
+- DRI2_BLIT_COMPLETE,
+- chain->client ? chain->event_complete : NULL,
+- chain->event_data);
+- i830_dri2_del_frame_event(chain);
+- }
+- }
+- break;
+-
+ default:
+ xf86DrvMsg(intel->scrn->scrnIndex, X_WARNING,
+ "%s: unknown vblank event received\n", __func__);
+@@ -1023,38 +1064,6 @@ void I830DRI2FlipEventHandler(unsigned int frame, unsigned int tv_sec,
+ i830_dri2_del_frame_event(flip_info);
+ }
+
+-static uint32_t pipe_select(int pipe)
+-{
+- if (pipe > 1)
+- return pipe << DRM_VBLANK_HIGH_CRTC_SHIFT;
+- else if (pipe > 0)
+- return DRM_VBLANK_SECONDARY;
+- else
+- return 0;
+-}
+-
+-static void
+-intel_dri2_vblank_handler(ScrnInfoPtr scrn,
+- xf86CrtcPtr crtc,
+- uint64_t msc,
+- uint64_t usec,
+- void *data)
+-{
+- DRI2FrameEventPtr swap_info = data;
+-
+- I830DRI2FrameEventHandler((uint32_t) msc, usec / 1000000, usec % 1000000, swap_info);
+-}
+-
+-static void
+-intel_dri2_vblank_abort(ScrnInfoPtr scrn,
+- xf86CrtcPtr crtc,
+- void *data)
+-{
+- DRI2FrameEventPtr swap_info = data;
+-
+- i830_dri2_del_frame_event(swap_info);
+-}
+-
+ /*
+ * ScheduleSwap is responsible for requesting a DRM vblank event for the
+ * appropriate frame.
+@@ -1089,7 +1098,6 @@ I830DRI2ScheduleSwap(ClientPtr client, DrawablePtr draw, DRI2BufferPtr front,
+ int pipe = crtc ? intel_crtc_to_pipe(crtc) : -1;
+ int flip = 0;
+ DRI2FrameEventPtr swap_info = NULL;
+- enum DRI2FrameEventType swap_type = DRI2_SWAP;
+ uint64_t current_msc, current_ust;
+ uint64_t request_msc;
+ uint32_t seq;
+@@ -1109,7 +1117,7 @@ I830DRI2ScheduleSwap(ClientPtr client, DrawablePtr draw, DRI2BufferPtr front,
+ swap_info->event_data = data;
+ swap_info->front = front;
+ swap_info->back = back;
+- swap_info->pipe = pipe;
++ swap_info->type = DRI2_SWAP;
+
+ if (!i830_dri2_add_frame_event(swap_info)) {
+ free(swap_info);
+@@ -1124,20 +1132,27 @@ I830DRI2ScheduleSwap(ClientPtr client, DrawablePtr draw, DRI2BufferPtr front,
+ if (ret)
+ goto blit_fallback;
+
+- /* Flips need to be submitted one frame before */
++ /*
++ * If we can, schedule the flip directly from here rather
++ * than waiting for an event from the kernel for the current
++ * (or a past) MSC.
++ */
++ if (divisor == 0 &&
++ current_msc >= *target_msc &&
++ queue_flip(intel, draw, swap_info))
++ return TRUE;
++
+ if (can_exchange(draw, front, back)) {
+- swap_type = DRI2_FLIP;
+- flip = 1;
++ swap_info->type = DRI2_FLIP;
++ /* Flips need to be submitted one frame before */
++ if (*target_msc > 0)
++ --*target_msc;
++ flip = 1;
+ }
+
+- swap_info->type = swap_type;
+-
+- /* Correct target_msc by 'flip' if swap_type == DRI2_FLIP.
+- * Do it early, so handling of different timing constraints
+- * for divisor, remainder and msc vs. target_msc works.
+- */
+- if (*target_msc > 0)
+- *target_msc -= flip;
++#if DRI2INFOREC_VERSION >= 6
++ DRI2SwapLimit(draw, 1);
++#endif
+
+ /*
+ * If divisor is zero, or current_msc is smaller than target_msc
+@@ -1145,15 +1160,6 @@ I830DRI2ScheduleSwap(ClientPtr client, DrawablePtr draw, DRI2BufferPtr front,
+ * the swap.
+ */
+ if (divisor == 0 || current_msc < *target_msc) {
+- /*
+- * If we can, schedule the flip directly from here rather
+- * than waiting for an event from the kernel for the current
+- * (or a past) MSC.
+- */
+- if (flip && divisor == 0 && current_msc >= *target_msc &&
+- I830DRI2ScheduleFlip(intel, draw, swap_info))
+- return TRUE;
+-
+ vbl.request.type =
+ DRM_VBLANK_ABSOLUTE | DRM_VBLANK_EVENT | pipe_select(pipe);
+
+@@ -1168,7 +1174,7 @@ I830DRI2ScheduleSwap(ClientPtr client, DrawablePtr draw, DRI2BufferPtr front,
+ * current_msc to ensure we return a reasonable value back
+ * to the caller. This makes swap_interval logic more robust.
+ */
+- if (current_msc >= *target_msc)
++ if (current_msc > *target_msc)
+ *target_msc = current_msc;
+
+ seq = intel_drm_queue_alloc(scrn, crtc, swap_info, intel_dri2_vblank_handler, intel_dri2_vblank_abort);
+@@ -1183,6 +1189,8 @@ I830DRI2ScheduleSwap(ClientPtr client, DrawablePtr draw, DRI2BufferPtr front,
+ xf86DrvMsg(scrn->scrnIndex, X_WARNING,
+ "divisor 0 get vblank counter failed: %s\n",
+ strerror(errno));
++ intel_drm_abort_seq(intel->scrn, seq);
++ swap_info = NULL;
+ goto blit_fallback;
+ }
+
+@@ -1332,7 +1340,6 @@ I830DRI2ScheduleWaitMSC(ClientPtr client, DrawablePtr draw, CARD64 target_msc,
+
+ if (!i830_dri2_add_frame_event(wait_info)) {
+ free(wait_info);
+- wait_info = NULL;
+ goto out_complete;
+ }
+
+@@ -1374,7 +1381,8 @@ I830DRI2ScheduleWaitMSC(ClientPtr client, DrawablePtr draw, CARD64 target_msc,
+ strerror(errno));
+ limit--;
+ }
+- goto out_free;
++ intel_drm_abort_seq(intel->scrn, seq);
++ goto out_complete;
+ }
+
+ wait_info->frame = intel_sequence_to_crtc_msc(crtc, vbl.reply.sequence);
+@@ -1417,7 +1425,8 @@ I830DRI2ScheduleWaitMSC(ClientPtr client, DrawablePtr draw, CARD64 target_msc,
+ strerror(errno));
+ limit--;
+ }
+- goto out_free;
++ intel_drm_abort_seq(intel->scrn, seq);
++ goto out_complete;
+ }
+
+ wait_info->frame = intel_sequence_to_crtc_msc(crtc, vbl.reply.sequence);
+@@ -1440,13 +1449,92 @@ static int has_i830_dri(void)
+ return access(DRI_DRIVER_PATH "/i830_dri.so", R_OK) == 0;
+ }
+
+-static const char *dri_driver_name(intel_screen_private *intel)
++static int
++namecmp(const char *s1, const char *s2)
++{
++ char c1, c2;
++
++ if (!s1 || *s1 == 0) {
++ if (!s2 || *s2 == 0)
++ return 0;
++ else
++ return 1;
++ }
++
++ while (*s1 == '_' || *s1 == ' ' || *s1 == '\t')
++ s1++;
++
++ while (*s2 == '_' || *s2 == ' ' || *s2 == '\t')
++ s2++;
++
++ c1 = isupper(*s1) ? tolower(*s1) : *s1;
++ c2 = isupper(*s2) ? tolower(*s2) : *s2;
++ while (c1 == c2) {
++ if (c1 == '\0')
++ return 0;
++
++ s1++;
++ while (*s1 == '_' || *s1 == ' ' || *s1 == '\t')
++ s1++;
++
++ s2++;
++ while (*s2 == '_' || *s2 == ' ' || *s2 == '\t')
++ s2++;
++
++ c1 = isupper(*s1) ? tolower(*s1) : *s1;
++ c2 = isupper(*s2) ? tolower(*s2) : *s2;
++ }
++
++ return c1 - c2;
++}
++
++static Bool is_level(const char **str)
++{
++ const char *s = *str;
++ char *end;
++ unsigned val;
++
++ if (s == NULL || *s == '\0')
++ return TRUE;
++
++ if (namecmp(s, "on") == 0)
++ return TRUE;
++ if (namecmp(s, "true") == 0)
++ return TRUE;
++ if (namecmp(s, "yes") == 0)
++ return TRUE;
++
++ if (namecmp(s, "0") == 0)
++ return TRUE;
++ if (namecmp(s, "off") == 0)
++ return TRUE;
++ if (namecmp(s, "false") == 0)
++ return TRUE;
++ if (namecmp(s, "no") == 0)
++ return TRUE;
++
++ val = strtoul(s, &end, 0);
++ if (val && *end == '\0')
++ return TRUE;
++ if (val && *end == ':')
++ *str = end + 1;
++ return FALSE;
++}
++
++static const char *options_get_dri(intel_screen_private *intel)
+ {
+ #if XORG_VERSION_CURRENT >= XORG_VERSION_NUMERIC(1,7,99,901,0)
+- const char *s = xf86GetOptValString(intel->Options, OPTION_DRI);
+- Bool dummy;
++ return xf86GetOptValString(intel->Options, OPTION_DRI);
++#else
++ return NULL;
++#endif
++}
+
+- if (s == NULL || xf86getBoolValue(&dummy, s)) {
++static const char *dri_driver_name(intel_screen_private *intel)
++{
++ const char *s = options_get_dri(intel);
++
++ if (is_level(&s)) {
+ if (INTEL_INFO(intel)->gen < 030)
+ return has_i830_dri() ? "i830" : "i915";
+ else if (INTEL_INFO(intel)->gen < 040)
+@@ -1456,14 +1544,6 @@ static const char *dri_driver_name(intel_screen_private *intel)
+ }
+
+ return s;
+-#else
+- if (INTEL_INFO(intel)->gen < 030)
+- return has_i830_dri() ? "i830" : "i915";
+- else if (INTEL_INFO(intel)->gen < 040)
+- return "i915";
+- else
+- return "i965";
+-#endif
+ }
+
+ Bool I830DRI2ScreenInit(ScreenPtr screen)
+@@ -1544,7 +1624,7 @@ Bool I830DRI2ScreenInit(ScreenPtr screen)
+ info.numDrivers = 2;
+ info.driverNames = driverNames;
+ driverNames[0] = info.driverName;
+- driverNames[1] = info.driverName;
++ driverNames[1] = "va_gl";
+ #endif
+
+ return DRI2ScreenInit(screen, &info);
+diff --git a/src/uxa/intel_driver.c b/src/uxa/intel_driver.c
+index 2793da5d..3703c412 100644
+--- a/src/uxa/intel_driver.c
++++ b/src/uxa/intel_driver.c
+@@ -237,24 +237,17 @@ static Bool I830GetEarlyOptions(ScrnInfoPtr scrn)
+ return TRUE;
+ }
+
+-static Bool intel_option_cast_string_to_bool(intel_screen_private *intel,
+- int id, Bool val)
+-{
+-#if XORG_VERSION_CURRENT >= XORG_VERSION_NUMERIC(1,7,99,901,0)
+- xf86getBoolValue(&val, xf86GetOptValString(intel->Options, id));
+- return val;
+-#else
+- return val;
+-#endif
+-}
+-
+ static void intel_check_dri_option(ScrnInfoPtr scrn)
+ {
+ intel_screen_private *intel = intel_get_screen_private(scrn);
++ unsigned level;
+
+ intel->dri2 = intel->dri3 = DRI_NONE;
+- if (!intel_option_cast_string_to_bool(intel, OPTION_DRI, TRUE))
+- intel->dri2 = intel->dri3 = DRI_DISABLED;
++ level = intel_option_cast_to_unsigned(intel->Options, OPTION_DRI, DEFAULT_DRI_LEVEL);
++ if (level < 3 || INTEL_INFO(intel)->gen < 040)
++ intel->dri3 = DRI_DISABLED;
++ if (level < 2)
++ intel->dri2 = DRI_DISABLED;
+
+ if (scrn->depth != 16 && scrn->depth != 24 && scrn->depth != 30) {
+ xf86DrvMsg(scrn->scrnIndex, X_CONFIG,
+@@ -371,8 +364,8 @@ static Bool can_accelerate_blt(struct intel_screen_private *intel)
+ if (INTEL_INFO(intel)->gen == -1)
+ return FALSE;
+
+- if (xf86ReturnOptValBool(intel->Options, OPTION_ACCEL_DISABLE, FALSE) ||
+- !intel_option_cast_string_to_bool(intel, OPTION_ACCEL_METHOD, TRUE)) {
++ if (!xf86ReturnOptValBool(intel->Options, OPTION_ACCEL_ENABLE, TRUE) ||
++ !intel_option_cast_to_bool(intel->Options, OPTION_ACCEL_METHOD, TRUE)) {
+ xf86DrvMsg(intel->scrn->scrnIndex, X_CONFIG,
+ "Disabling hardware acceleration.\n");
+ return FALSE;
+@@ -659,8 +652,9 @@ redisplay_dirty(ScreenPtr screen, PixmapDirtyUpdatePtr dirty)
+ }
+
+ static void
+-intel_dirty_update(ScreenPtr screen)
++intel_dirty_update(intel_screen_private *intel)
+ {
++ ScreenPtr screen = xf86ScrnToScreen(intel->scrn);
+ RegionPtr region;
+ PixmapDirtyUpdatePtr ent;
+
+@@ -677,6 +671,7 @@ intel_dirty_update(ScreenPtr screen)
+ }
+ #endif
+
++#if !HAVE_NOTIFY_FD
+ static void
+ I830BlockHandler(BLOCKHANDLER_ARGS_DECL)
+ {
+@@ -694,9 +689,22 @@ I830BlockHandler(BLOCKHANDLER_ARGS_DECL)
+ intel_uxa_block_handler(intel);
+ intel_video_block_handler(intel);
+ #ifdef INTEL_PIXMAP_SHARING
+- intel_dirty_update(screen);
++ intel_dirty_update(intel);
+ #endif
+ }
++#else
++static void
++I830BlockHandler(void *data, void *timeout)
++{
++ intel_screen_private *intel = data;
++
++ intel_uxa_block_handler(intel);
++ intel_video_block_handler(intel);
++#ifdef INTEL_PIXMAP_SHARING
++ intel_dirty_update(intel);
++#endif
++}
++#endif
+
+ static Bool
+ intel_init_initial_framebuffer(ScrnInfoPtr scrn)
+@@ -735,6 +743,8 @@ intel_flush_callback(CallbackListPtr *list,
+ }
+
+ #if HAVE_UDEV
++#include <sys/stat.h>
++
+ static void
+ I830HandleUEvents(int fd, void *closure)
+ {
+@@ -771,6 +781,15 @@ I830HandleUEvents(int fd, void *closure)
+ udev_device_unref(dev);
+ }
+
++static int has_randr(void)
++{
++#if HAS_DIXREGISTERPRIVATEKEY
++ return dixPrivateKeyRegistered(rrPrivKey);
++#else
++ return *rrPrivKey;
++#endif
++}
++
+ static void
+ I830UeventInit(ScrnInfoPtr scrn)
+ {
+@@ -780,6 +799,10 @@ I830UeventInit(ScrnInfoPtr scrn)
+ Bool hotplug;
+ MessageType from = X_CONFIG;
+
++ /* Without RR, nothing we can do here */
++ if (!has_randr())
++ return;
++
+ if (!xf86GetOptValBool(intel->Options, OPTION_HOTPLUG, &hotplug)) {
+ from = X_DEFAULT;
+ hotplug = TRUE;
+@@ -939,8 +962,14 @@ I830ScreenInit(SCREEN_INIT_ARGS_DECL)
+ "Hardware cursor initialization failed\n");
+ }
+
++#if !HAVE_NOTIFY_FD
+ intel->BlockHandler = screen->BlockHandler;
+ screen->BlockHandler = I830BlockHandler;
++#else
++ RegisterBlockAndWakeupHandlers(I830BlockHandler,
++ (ServerWakeupHandlerProcPtr)NoopDDA,
++ intel);
++#endif
+
+ #ifdef INTEL_PIXMAP_SHARING
+ screen->StartPixmapTracking = PixmapStartDirtyTracking;
+@@ -1164,8 +1193,6 @@ static Bool I830CloseScreen(CLOSE_SCREEN_ARGS_DECL)
+
+ intel_sync_close(screen);
+
+- xf86GARTCloseScreen(scrn->scrnIndex);
+-
+ scrn->vtSema = FALSE;
+ return TRUE;
+ }
+diff --git a/src/uxa/intel_hwmc.c b/src/uxa/intel_hwmc.c
+index 829cb8e0..78540600 100644
+--- a/src/uxa/intel_hwmc.c
++++ b/src/uxa/intel_hwmc.c
+@@ -193,7 +193,7 @@ Bool intel_xvmc_adaptor_init(ScreenPtr pScreen)
+ intel_screen_private *intel = intel_get_screen_private(scrn);
+ struct pci_device *pci;
+ static XF86MCAdaptorRec *pAdapt;
+- char *name;
++ const char *name;
+ char buf[64];
+
+ if (!intel->XvMCEnabled)
+diff --git a/src/uxa/intel_memory.c b/src/uxa/intel_memory.c
+index 0c6cf30c..b2d7a367 100644
+--- a/src/uxa/intel_memory.c
++++ b/src/uxa/intel_memory.c
+@@ -42,7 +42,7 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ * This is the video memory allocator. Our memory allocation is different from
+ * other graphics chips, where you have a fixed amount of graphics memory
+ * available that you want to put to the best use. Instead, we have almost no
+- * memory pre-allocated, and we have to choose an appropriate amount of sytem
++ * memory pre-allocated, and we have to choose an appropriate amount of system
+ * memory to use.
+ *
+ * The allocations we might do:
+diff --git a/src/uxa/intel_present.c b/src/uxa/intel_present.c
+index d20043f3..ac028edd 100644
+--- a/src/uxa/intel_present.c
++++ b/src/uxa/intel_present.c
+@@ -244,6 +244,7 @@ intel_present_check_flip(RRCrtcPtr crtc,
+ ScrnInfoPtr scrn = xf86ScreenToScrn(screen);
+ intel_screen_private *intel = intel_get_screen_private(scrn);
+ dri_bo *bo;
++ uint32_t tiling, swizzle;
+
+ if (!scrn->vtSema)
+ return FALSE;
+@@ -266,6 +267,12 @@ intel_present_check_flip(RRCrtcPtr crtc,
+ if (!bo)
+ return FALSE;
+
++ if (drm_intel_bo_get_tiling(bo, &tiling, &swizzle))
++ return FALSE;
++
++ if (tiling == I915_TILING_Y)
++ return FALSE;
++
+ return TRUE;
+ }
+
+@@ -343,29 +350,33 @@ intel_present_unflip(ScreenPtr screen, uint64_t event_id)
+ {
+ ScrnInfoPtr scrn = xf86ScreenToScrn(screen);
+ intel_screen_private *intel = intel_get_screen_private(scrn);
+- struct intel_present_vblank_event *event;
+ PixmapPtr pixmap = screen->GetScreenPixmap(screen);
++ struct intel_present_vblank_event *event = NULL;
+ dri_bo *bo;
+- Bool ret;
+
+ if (!intel_present_check_flip(NULL, screen->root, pixmap, true))
+- return;
++ goto fail;
+
+ bo = intel_get_pixmap_bo(pixmap);
+ if (!bo)
+- return;
++ goto fail;
+
+ event = calloc(1, sizeof(struct intel_present_vblank_event));
+ if (!event)
+- return;
++ goto fail;
+
+ event->event_id = event_id;
+
+- ret = intel_do_pageflip(intel, bo, -1, FALSE, event, intel_present_flip_event, intel_present_flip_abort);
+- if (!ret) {
+- xf86DrvMsg(scrn->scrnIndex, X_ERROR,
+- "present unflip failed\n");
+- }
++ if (!intel_do_pageflip(intel, bo, -1, FALSE, event,
++ intel_present_flip_event,
++ intel_present_flip_abort))
++ goto fail;
++
++ return;
++fail:
++ xf86SetDesiredModes(scrn);
++ present_event_notify(event_id, 0, 0);
++ free(event);
+ }
+
+ static present_screen_info_rec intel_present_screen_info = {
+diff --git a/src/uxa/intel_uxa.c b/src/uxa/intel_uxa.c
+index 590ff5d1..ec32a723 100644
+--- a/src/uxa/intel_uxa.c
++++ b/src/uxa/intel_uxa.c
+@@ -176,6 +176,24 @@ intel_uxa_check_solid(DrawablePtr drawable, int alu, Pixel planemask)
+ return TRUE;
+ }
+
++static Bool
++intel_uxa_check_bo_tiling(intel_screen_private *intel,
++ PixmapPtr pixmap,
++ unsigned *tiling_out)
++{
++ struct intel_uxa_pixmap *priv;
++
++ priv = intel_uxa_get_pixmap_private(pixmap);
++ if (!priv)
++ return FALSE;
++
++ if (priv->tiling == I915_TILING_Y && INTEL_INFO(intel)->gen < 060)
++ return FALSE;
++
++ *tiling_out = priv->tiling;
++ return TRUE;
++}
++
+ /**
+ * Sets up hardware state for a series of solid fills.
+ */
+@@ -189,6 +207,9 @@ intel_uxa_prepare_solid(PixmapPtr pixmap, int alu, Pixel planemask, Pixel fg)
+ intel_uxa_get_pixmap_bo(pixmap),
+ };
+
++ if (!intel_uxa_check_bo_tiling(intel, pixmap, &intel->BR_tiling[0]))
++ return FALSE;
++
+ if (!intel_uxa_check_pitch_2d(pixmap))
+ return FALSE;
+
+@@ -236,7 +257,7 @@ static void intel_uxa_solid(PixmapPtr pixmap, int x1, int y1, int x2, int y2)
+
+ {
+ int len = INTEL_INFO(intel)->gen >= 0100 ? 7 : 6;
+- BEGIN_BATCH_BLT(len);
++ BEGIN_BATCH_BLT_TILED(len);
+
+ cmd = XY_COLOR_BLT_CMD | (len - 2);
+
+@@ -310,6 +331,10 @@ intel_uxa_prepare_copy(PixmapPtr source, PixmapPtr dest, int xdir,
+ intel_uxa_get_pixmap_bo(dest),
+ };
+
++ if (!intel_uxa_check_bo_tiling(intel, dest, &intel->BR_tiling[0]) ||
++ !intel_uxa_check_bo_tiling(intel, source, &intel->BR_tiling[1]))
++ return FALSE;
++
+ if (!intel_uxa_get_aperture_space(scrn, bo_table, ARRAY_SIZE(bo_table)))
+ return FALSE;
+
+@@ -375,7 +400,7 @@ intel_uxa_copy(PixmapPtr dest, int src_x1, int src_y1, int dst_x1,
+
+ {
+ int len = INTEL_INFO(intel)->gen >= 0100 ? 10 : 8;
+- BEGIN_BATCH_BLT(len);
++ BEGIN_BATCH_BLT_TILED(len);
+
+ cmd = XY_SRC_COPY_BLT_CMD | (len - 2);
+
+@@ -1068,7 +1093,7 @@ Bool intel_uxa_create_screen_resources(ScreenPtr screen)
+ ScrnInfoPtr scrn = xf86ScreenToScrn(screen);
+ PixmapPtr pixmap;
+ intel_screen_private *intel = intel_get_screen_private(scrn);
+- dri_bo *bo = intel->front_buffer;
++ dri_bo *bo = intel->front_buffer, *old_bo;
+ int old_width, old_height, old_pitch;
+
+ if (!uxa_resources_init(screen))
+@@ -1081,6 +1106,7 @@ Bool intel_uxa_create_screen_resources(ScreenPtr screen)
+ old_width = pixmap->drawable.width;
+ old_height = pixmap->drawable.height;
+ old_pitch = pixmap->devKind;
++ old_bo = intel_uxa_get_pixmap_bo(pixmap);
+
+ if (!screen->ModifyPixmapHeader(pixmap,
+ scrn->virtualX,
+@@ -1102,6 +1128,9 @@ Bool intel_uxa_create_screen_resources(ScreenPtr screen)
+ err:
+ screen->ModifyPixmapHeader(pixmap,
+ old_width, old_height, -1, -1, old_pitch, NULL);
++ if (old_bo)
++ intel_uxa_set_pixmap_bo(pixmap, old_bo);
++
+ return FALSE;
+ }
+
+diff --git a/test/Makefile.am b/test/Makefile.am
+index 66ed8ebb..12b5d5d8 100644
+--- a/test/Makefile.am
++++ b/test/Makefile.am
+@@ -5,6 +5,7 @@ stress_TESTS = \
+ basic-rectangle \
+ basic-string \
+ basic-copyarea \
++ basic-copyplane \
+ basic-copyarea-size \
+ basic-putimage \
+ basic-lines \
+@@ -12,8 +13,10 @@ stress_TESTS = \
+ DrawSegments \
+ cursor-test \
+ render-fill \
++ render-glyphs \
+ render-trapezoid \
+ render-trapezoid-image \
++ render-triangle \
+ render-fill-copy \
+ render-composite-solid \
+ render-composite-solid-mask \
+@@ -25,9 +28,16 @@ stress_TESTS = \
+ shm-test \
+ $(NULL)
+
++if X11_VM
++stress_TESTS += \
++ xvidmode \
++ $(NULL)
++endif
++
+ if DRI2
+ stress_TESTS += \
+ dri2-race \
++ dri2-speed \
+ dri2-swap \
+ dri2-test \
+ $(NULL)
+@@ -36,8 +46,11 @@ endif
+ if X11_DRI3
+ stress_TESTS += \
+ dri3-test \
++ present-race \
++ present-speed \
+ present-test \
+ $(NULL)
++present_speed_CFLAGS = ${AM_CFLAGS} -pthread
+ endif
+ check_PROGRAMS = $(stress_TESTS)
+
+diff --git a/test/basic-copyplane.c b/test/basic-copyplane.c
+new file mode 100644
+index 00000000..f049b82b
+--- /dev/null
++++ b/test/basic-copyplane.c
+@@ -0,0 +1,99 @@
++#include <stdint.h>
++#include <stdio.h>
++#include <stdlib.h>
++
++#include <X11/Xutil.h> /* for XDestroyImage */
++#include <pixman.h> /* for pixman blt functions */
++
++#include "test.h"
++
++static uint8_t clock_bits[] = {0x3C, 0x5E, 0xEF, 0xF7, 0x87, 0xFF, 0x7E, 0x3C};
++
++/* https://bugs.freedesktop.org/show_bug.cgi?id=91499 */
++static void draw_clock(struct test_display *t, Drawable d,
++ uint8_t alu, int x, int y, uint32_t fg, uint32_t bg)
++{
++ Pixmap pixmap;
++ XGCValues val;
++ GC gc;
++
++ val.graphics_exposures = 0;
++ val.function = alu;
++ val.foreground = fg;
++ val.background = fg;
++
++ gc = XCreateGC(t->dpy, d,
++ GCGraphicsExposures | GCForeground | GCBackground | GCFunction,
++ &val);
++ pixmap = XCreateBitmapFromData(t->dpy, d, (char *)clock_bits, 8, 8);
++
++ XCopyPlane(t->dpy, pixmap, d, gc, 0, 0, 8, 8, x, y, 1);
++
++ XFreePixmap(t->dpy, pixmap);
++ XFreeGC(t->dpy, gc);
++}
++
++static void clear(struct test_display *dpy, struct test_target *tt)
++{
++ XRenderColor render_color = {0};
++ XRenderFillRectangle(dpy->dpy, PictOpClear, tt->picture, &render_color,
++ 0, 0, tt->width, tt->height);
++}
++
++static void clock_tests(struct test *t, int reps, int sets, enum target target)
++{
++ struct test_target out, ref;
++ int r, s;
++
++ printf("Testing clock (%s): ", test_target_name(target));
++ fflush(stdout);
++
++ test_target_create_render(&t->out, target, &out);
++ clear(&t->out, &out);
++
++ test_target_create_render(&t->ref, target, &ref);
++ clear(&t->ref, &ref);
++
++ for (s = 0; s < sets; s++) {
++ for (r = 0; r < reps; r++) {
++ int x = rand() % (out.width - 8);
++ int y = rand() % (out.height - 8);
++ uint8_t alu = rand() % (GXset + 1);
++ uint32_t bg = rand();
++ uint32_t fg = rand();
++
++ draw_clock(&t->out, out.draw, alu, x, y, fg, bg);
++ draw_clock(&t->ref, ref.draw, alu, x, y, fg, bg);
++ }
++
++ test_compare(t,
++ out.draw, out.format,
++ ref.draw, ref.format,
++ 0, 0, out.width, out.height,
++ "");
++ }
++
++ printf("passed [%d iterations x %d]\n", reps, sets);
++
++ test_target_destroy_render(&t->out, &out);
++ test_target_destroy_render(&t->ref, &ref);
++}
++
++int main(int argc, char **argv)
++{
++ struct test test;
++ int i;
++
++ test_init(&test, argc, argv);
++
++ for (i = 0; i <= DEFAULT_ITERATIONS; i++) {
++ int reps = REPS(i), sets = SETS(i);
++ enum target t;
++
++ for (t = TARGET_FIRST; t <= TARGET_LAST; t++) {
++ clock_tests(&test, reps, sets, t);
++ }
++ }
++
++ return 0;
++}
+diff --git a/test/dri2-race.c b/test/dri2-race.c
+index 8862c84c..ece624f6 100644
+--- a/test/dri2-race.c
++++ b/test/dri2-race.c
+@@ -5,6 +5,11 @@
+ #include <X11/Xlib.h>
+ #include <X11/Xutil.h>
+ #include <X11/extensions/Xfixes.h>
++#include <X11/extensions/Xcomposite.h>
++#include <X11/Xlib-xcb.h>
++#include <xcb/xcb.h>
++#include <xcb/xcbext.h>
++#include <xcb/dri2.h>
+ #include <unistd.h>
+ #include <fcntl.h>
+ #include <string.h>
+@@ -12,11 +17,49 @@
+
+ #include <xf86drm.h>
+ #include <drm.h>
++#include <setjmp.h>
+
+ #include "dri2.h"
+
+ #define COUNT 60
+
++#define N_DIVISORS 3
++static const int divisors[N_DIVISORS] = { 0, 1, 16 };
++
++static jmp_buf error_handler[4];
++static int have_error_handler;
++
++#define error_get() \
++ setjmp(error_handler[have_error_handler++])
++
++#define error_put() \
++ have_error_handler--
++
++static int (*saved_io_error)(Display *dpy);
++
++static int io_error(Display *dpy)
++{
++ if (have_error_handler)
++ longjmp(error_handler[--have_error_handler], 0);
++
++ return saved_io_error(dpy);
++}
++
++static int x_error(Display *dpy, XErrorEvent *e)
++{
++ return Success;
++}
++
++static uint32_t upper_32_bits(uint64_t val)
++{
++ return val >> 32;
++}
++
++static uint32_t lower_32_bits(uint64_t val)
++{
++ return val & 0xffffffff;
++}
++
+ static int dri2_open(Display *dpy)
+ {
+ drm_auth_t auth;
+@@ -41,45 +84,701 @@ static int dri2_open(Display *dpy)
+ return fd;
+ }
+
+-static void run(Display *dpy, int width, int height,
+- unsigned int *attachments, int nattachments,
+- const char *name)
++static void swap_buffers(Display *dpy, Window win, int divisor,
++ unsigned int *attachments, int nattachments)
++{
++ xcb_connection_t *c = XGetXCBConnection(dpy);
++ unsigned int seq[2];
++
++ seq[0] = xcb_dri2_swap_buffers_unchecked(c, win,
++ 0, 0, 0, divisor, 0, 0).sequence;
++
++
++ seq[1] = xcb_dri2_get_buffers_unchecked(c, win,
++ nattachments, nattachments,
++ attachments).sequence;
++
++ xcb_flush(c);
++ xcb_discard_reply(c, seq[0]);
++ xcb_discard_reply(c, seq[1]);
++}
++
++#define COMPOSITE 1
++
++static int has_composite(Display *dpy)
++{
++ Display *dummy = NULL;
++ int event, error;
++ int major = -1, minor = -1;
++
++ if (dpy == NULL)
++ dummy = dpy = XOpenDisplay(NULL);
++
++ if (XCompositeQueryExtension(dpy, &event, &error))
++ XCompositeQueryVersion(dpy, &major, &minor);
++
++ if (dummy)
++ XCloseDisplay(dummy);
++
++ return major > 0 || minor >= 4;
++}
++
++static void race_window(Display *dpy, int width, int height,
++ unsigned int *attachments, int nattachments,
++ unsigned flags, const char *name)
+ {
+ Window win;
+ XSetWindowAttributes attr;
+- int count, loop;
++ int count, loop, n;
+ DRI2Buffer *buffers;
+
++ if (flags & COMPOSITE && !has_composite(dpy))
++ return;
++
++ printf("%s(%s)\n", __func__, name);
++
+ /* Be nasty and install a fullscreen window on top so that we
+ * can guarantee we do not get clipped by children.
+ */
+ attr.override_redirect = 1;
+- loop = 100;
+- do {
++ for (n = 0; n < N_DIVISORS; n++) {
++ loop = 256 >> ffs(divisors[n]);
++ printf("DRI2SwapBuffers(divisor=%d), loop=%d", divisors[n], loop);
++ fflush(stdout);
++ do {
++ win = XCreateWindow(dpy, DefaultRootWindow(dpy),
++ 0, 0, width, height, 0,
++ DefaultDepth(dpy, DefaultScreen(dpy)),
++ InputOutput,
++ DefaultVisual(dpy, DefaultScreen(dpy)),
++ CWOverrideRedirect, &attr);
++ if (flags & COMPOSITE)
++ XCompositeRedirectWindow(dpy, win, CompositeRedirectManual);
++ XMapWindow(dpy, win);
++
++ DRI2CreateDrawable(dpy, win);
++
++ buffers = DRI2GetBuffers(dpy, win, &width, &height,
++ attachments, nattachments, &count);
++ if (count != nattachments)
++ return;
++
++ free(buffers);
++ for (count = 0; count < loop; count++)
++ DRI2SwapBuffers(dpy, win, 0, divisors[n], count & (divisors[n]-1));
++ XDestroyWindow(dpy, win);
++ printf("."); fflush(stdout);
++ } while (--loop);
++ printf("*\n");
++ }
++
++ for (n = 0; n < N_DIVISORS; n++) {
++ loop = 256 >> ffs(divisors[n]);
++ printf("xcb_dri2_swap_buffers(divisor=%d), loops=%d", divisors[n], loop);
++ fflush(stdout);
++ do {
++ win = XCreateWindow(dpy, DefaultRootWindow(dpy),
++ 0, 0, width, height, 0,
++ DefaultDepth(dpy, DefaultScreen(dpy)),
++ InputOutput,
++ DefaultVisual(dpy, DefaultScreen(dpy)),
++ CWOverrideRedirect, &attr);
++ if (flags & COMPOSITE)
++ XCompositeRedirectWindow(dpy, win, CompositeRedirectManual);
++ XMapWindow(dpy, win);
++
++ DRI2CreateDrawable(dpy, win);
++
++ buffers = DRI2GetBuffers(dpy, win, &width, &height,
++ attachments, nattachments, &count);
++ if (count != nattachments)
++ return;
++
++ free(buffers);
++ for (count = 0; count < loop; count++)
++ swap_buffers(dpy, win, divisors[n], attachments, nattachments);
++ XDestroyWindow(dpy, win);
++ printf("."); fflush(stdout);
++ } while (--loop);
++ printf("*\n");
++ }
++
++ for (n = 0; n < N_DIVISORS; n++) {
++ loop = 256 >> ffs(divisors[n]);
++ printf("DRI2WaitMsc(divisor=%d), loop=%d", divisors[n], loop);
++ fflush(stdout);
++ do {
++ uint64_t ignore, msc;
++ xcb_connection_t *c = XGetXCBConnection(dpy);
++
++ win = XCreateWindow(dpy, DefaultRootWindow(dpy),
++ 0, 0, width, height, 0,
++ DefaultDepth(dpy, DefaultScreen(dpy)),
++ InputOutput,
++ DefaultVisual(dpy, DefaultScreen(dpy)),
++ CWOverrideRedirect, &attr);
++ if (flags & COMPOSITE)
++ XCompositeRedirectWindow(dpy, win, CompositeRedirectManual);
++ XMapWindow(dpy, win);
++
++ DRI2CreateDrawable(dpy, win);
++ DRI2GetMSC(dpy, win, &ignore, &msc, &ignore);
++ msc++;
++ for (count = 0; count < loop; count++) {
++ xcb_discard_reply(c,
++ xcb_dri2_wait_msc(c, win,
++ upper_32_bits(msc),
++ lower_32_bits(msc),
++ 0, 0, 0, 0).sequence);
++ msc += divisors[n];
++ }
++ XFlush(dpy);
++ XDestroyWindow(dpy, win);
++ printf("."); fflush(stdout);
++ } while (--loop);
++ printf("*\n");
++ }
++
++ XSync(dpy, 1);
++ sleep(2);
++ XSync(dpy, 1);
++}
++
++static int rand_size(int max)
++{
++ return 1 + (rand() % (max - 1));
++}
++
++static void race_resize(Display *dpy, int width, int height,
++ unsigned int *attachments, int nattachments,
++ unsigned flags, const char *name)
++{
++ Window win;
++ XSetWindowAttributes attr;
++ int count, loop, n;
++ DRI2Buffer *buffers;
++
++ if (flags & COMPOSITE && !has_composite(dpy))
++ return;
++
++ printf("%s(%s)\n", __func__, name);
++
++ attr.override_redirect = 1;
++ for (n = 0; n < N_DIVISORS; n++) {
++ win = XCreateWindow(dpy, DefaultRootWindow(dpy),
++ 0, 0, width, height, 0,
++ DefaultDepth(dpy, DefaultScreen(dpy)),
++ InputOutput,
++ DefaultVisual(dpy, DefaultScreen(dpy)),
++ CWOverrideRedirect, &attr);
++ if (flags & COMPOSITE)
++ XCompositeRedirectWindow(dpy, win, CompositeRedirectManual);
++ XMapWindow(dpy, win);
++
++ DRI2CreateDrawable(dpy, win);
++
++ loop = 256 >> ffs(divisors[n]);
++ printf("DRI2SwapBuffers(divisor=%d), loop=%d", divisors[n], loop);
++ fflush(stdout);
++ do {
++ int w, h;
++
++ buffers = DRI2GetBuffers(dpy, win, &w, &h,
++ attachments, nattachments, &count);
++ if (count != nattachments)
++ return;
++
++ free(buffers);
++ for (count = 0; count < loop; count++)
++ DRI2SwapBuffers(dpy, win, 0, divisors[n], count & (divisors[n]-1));
++ XResizeWindow(dpy, win, rand_size(width), rand_size(height));
++ printf("."); fflush(stdout);
++ } while (--loop);
++ XDestroyWindow(dpy, win);
++ XSync(dpy, True);
++ printf("*\n");
++ }
++
++ for (n = 0; n < N_DIVISORS; n++) {
+ win = XCreateWindow(dpy, DefaultRootWindow(dpy),
+ 0, 0, width, height, 0,
+ DefaultDepth(dpy, DefaultScreen(dpy)),
+ InputOutput,
+ DefaultVisual(dpy, DefaultScreen(dpy)),
+ CWOverrideRedirect, &attr);
++ if (flags & COMPOSITE)
++ XCompositeRedirectWindow(dpy, win, CompositeRedirectManual);
+ XMapWindow(dpy, win);
+
+ DRI2CreateDrawable(dpy, win);
+
+- buffers = DRI2GetBuffers(dpy, win, &width, &height,
+- attachments, nattachments, &count);
+- if (count != nattachments)
+- return;
++ loop = 256 >> ffs(divisors[n]);
++ printf("xcb_dri2_swap_buffers(divisor=%d), loops=%d", divisors[n], loop);
++ fflush(stdout);
++ do {
++ int w, h;
++
++ buffers = DRI2GetBuffers(dpy, win, &w, &h,
++ attachments, nattachments, &count);
++ if (count != nattachments)
++ return;
+
+- free(buffers);
+- for (count = 0; count < loop; count++)
+- DRI2SwapBuffers(dpy, win, 0, 0, 0);
++ free(buffers);
++ for (count = 0; count < loop; count++)
++ swap_buffers(dpy, win, divisors[n], attachments, nattachments);
++ XResizeWindow(dpy, win, rand_size(width), rand_size(height));
++ printf("."); fflush(stdout);
++ } while (--loop);
+ XDestroyWindow(dpy, win);
+- } while (--loop);
++ XSync(dpy, True);
++ printf("*\n");
++ }
++
++ for (n = 0; n < N_DIVISORS; n++) {
++ win = XCreateWindow(dpy, DefaultRootWindow(dpy),
++ 0, 0, width, height, 0,
++ DefaultDepth(dpy, DefaultScreen(dpy)),
++ InputOutput,
++ DefaultVisual(dpy, DefaultScreen(dpy)),
++ CWOverrideRedirect, &attr);
++ if (flags & COMPOSITE)
++ XCompositeRedirectWindow(dpy, win, CompositeRedirectManual);
++ XMapWindow(dpy, win);
++
++ DRI2CreateDrawable(dpy, win);
++
++ loop = 256 >> ffs(divisors[n]);
++ printf("DRI2WaitMsc(divisor=%d), loop=%d", divisors[n], loop);
++ fflush(stdout);
++ do {
++ uint64_t ignore, msc;
++ xcb_connection_t *c = XGetXCBConnection(dpy);
++
++ DRI2GetMSC(dpy, win, &ignore, &msc, &ignore);
++ msc++;
++ for (count = 0; count < loop; count++) {
++ xcb_discard_reply(c,
++ xcb_dri2_wait_msc(c, win,
++ upper_32_bits(msc),
++ lower_32_bits(msc),
++ 0, 0, 0, 0).sequence);
++ msc += divisors[n];
++ }
++ XFlush(dpy);
++ XResizeWindow(dpy, win, rand_size(width), rand_size(height));
++ printf("."); fflush(stdout);
++ } while (--loop);
++ XDestroyWindow(dpy, win);
++ XSync(dpy, True);
++ printf("*\n");
++ }
++
++ XSync(dpy, 1);
++ sleep(2);
++ XSync(dpy, 1);
++}
++
++static void race_manager(Display *dpy, int width, int height,
++ unsigned int *attachments, int nattachments,
++ unsigned flags, const char *name)
++{
++ Display *mgr = XOpenDisplay(NULL);
++ Window win;
++ XSetWindowAttributes attr;
++ int count, loop, n;
++ DRI2Buffer *buffers;
++
++ if (flags & COMPOSITE && !has_composite(dpy))
++ return;
++
++ printf("%s(%s)\n", __func__, name);
++
++ /* Be nasty and install a fullscreen window on top so that we
++ * can guarantee we do not get clipped by children.
++ */
++ attr.override_redirect = 1;
++ for (n = 0; n < N_DIVISORS; n++) {
++ printf("DRI2SwapBuffers(divisor=%d)", divisors[n]);
++ fflush(stdout);
++ loop = 256 >> ffs(divisors[n]);
++ do {
++ win = XCreateWindow(dpy, DefaultRootWindow(dpy),
++ 0, 0, width, height, 0,
++ DefaultDepth(dpy, DefaultScreen(dpy)),
++ InputOutput,
++ DefaultVisual(dpy, DefaultScreen(dpy)),
++ CWOverrideRedirect, &attr);
++ if (flags & COMPOSITE)
++ XCompositeRedirectWindow(dpy, win, CompositeRedirectManual);
++ XMapWindow(dpy, win);
++
++ DRI2CreateDrawable(dpy, win);
++
++ buffers = DRI2GetBuffers(dpy, win, &width, &height,
++ attachments, nattachments, &count);
++ if (count != nattachments)
++ return;
++
++ free(buffers);
++ for (count = 0; count < loop; count++)
++ DRI2SwapBuffers(dpy, win, 0, divisors[n], count & (divisors[n]-1));
++ XFlush(dpy);
++ XDestroyWindow(mgr, win);
++ XFlush(mgr);
++ printf("."); fflush(stdout);
++ } while (--loop);
++ printf("*\n");
++ }
++
++ for (n = 0; n < N_DIVISORS; n++) {
++ printf("xcb_dri2_swap_buffers(divisor=%d)", divisors[n]);
++ fflush(stdout);
++ loop = 256 >> ffs(divisors[n]);
++ do {
++ win = XCreateWindow(dpy, DefaultRootWindow(dpy),
++ 0, 0, width, height, 0,
++ DefaultDepth(dpy, DefaultScreen(dpy)),
++ InputOutput,
++ DefaultVisual(dpy, DefaultScreen(dpy)),
++ CWOverrideRedirect, &attr);
++ if (flags & COMPOSITE)
++ XCompositeRedirectWindow(dpy, win, CompositeRedirectManual);
++ XMapWindow(dpy, win);
++
++ DRI2CreateDrawable(dpy, win);
++
++ buffers = DRI2GetBuffers(dpy, win, &width, &height,
++ attachments, nattachments, &count);
++ if (count != nattachments)
++ return;
++
++ free(buffers);
++ for (count = 0; count < loop; count++)
++ swap_buffers(dpy, win, divisors[n], attachments, nattachments);
++ XFlush(dpy);
++ XDestroyWindow(mgr, win);
++ XFlush(mgr);
++ printf("."); fflush(stdout);
++ } while (--loop);
++ printf("*\n");
++ }
++
++ for (n = 0; n < N_DIVISORS; n++) {
++ printf("DRI2WaitMsc(divisor=%d)", divisors[n]);
++ fflush(stdout);
++ loop = 256 >> ffs(divisors[n]);
++ do {
++ uint64_t ignore, msc;
++ xcb_connection_t *c = XGetXCBConnection(dpy);
++
++ win = XCreateWindow(dpy, DefaultRootWindow(dpy),
++ 0, 0, width, height, 0,
++ DefaultDepth(dpy, DefaultScreen(dpy)),
++ InputOutput,
++ DefaultVisual(dpy, DefaultScreen(dpy)),
++ CWOverrideRedirect, &attr);
++ if (flags & COMPOSITE)
++ XCompositeRedirectWindow(dpy, win, CompositeRedirectManual);
++ XMapWindow(dpy, win);
++
++ DRI2CreateDrawable(dpy, win);
++ DRI2GetMSC(dpy, win, &ignore, &msc, &ignore);
++ msc++;
++ for (count = 0; count < loop; count++) {
++ xcb_discard_reply(c,
++ xcb_dri2_wait_msc(c, win,
++ upper_32_bits(msc),
++ lower_32_bits(msc),
++ 0, 0, 0, 0).sequence);
++ msc += divisors[n];
++ }
++ XFlush(dpy);
++ XDestroyWindow(mgr, win);
++ XFlush(mgr);
++ printf("."); fflush(stdout);
++ } while (--loop);
++ printf("*\n");
++ }
+
+ XSync(dpy, 1);
++ XSync(mgr, 1);
+ sleep(2);
+ XSync(dpy, 1);
++ XSync(mgr, 1);
++
++ XCloseDisplay(mgr);
++}
++
++static void race_close(int width, int height,
++ unsigned int *attachments, int nattachments,
++ unsigned flags, const char *name)
++{
++ XSetWindowAttributes attr;
++ int count, loop, n;
++
++ if (flags & COMPOSITE && !has_composite(NULL))
++ return;
++
++ printf("%s(%s)\n", __func__, name);
++
++ /* Be nasty and install a fullscreen window on top so that we
++ * can guarantee we do not get clipped by children.
++ */
++ attr.override_redirect = 1;
++ for (n = 0; n < N_DIVISORS; n++) {
++ printf("DRI2SwapBuffers(divisor=%d)", divisors[n]);
++ fflush(stdout);
++ loop = 256 >> ffs(divisors[n]);
++ do {
++ Display *dpy = XOpenDisplay(NULL);
++ Window win = XCreateWindow(dpy, DefaultRootWindow(dpy),
++ 0, 0, width, height, 0,
++ DefaultDepth(dpy, DefaultScreen(dpy)),
++ InputOutput,
++ DefaultVisual(dpy, DefaultScreen(dpy)),
++ CWOverrideRedirect, &attr);
++ if (flags & COMPOSITE)
++ XCompositeRedirectWindow(dpy, win, CompositeRedirectManual);
++ XMapWindow(dpy, win);
++
++ DRI2CreateDrawable(dpy, win);
++ free(DRI2GetBuffers(dpy, win, &width, &height,
++ attachments, nattachments, &count));
++ if (count != nattachments)
++ return;
++
++ for (count = 0; count < loop; count++)
++ DRI2SwapBuffers(dpy, win, 0, divisors[n], count & (divisors[n]-1));
++ XCloseDisplay(dpy);
++ printf("."); fflush(stdout);
++ } while (--loop);
++ printf("*\n");
++ }
++
++ for (n = 0; n < N_DIVISORS; n++) {
++ printf("xcb_dri2_swap_buffers(divisor=%d)", divisors[n]);
++ fflush(stdout);
++ loop = 256 >> ffs(divisors[n]);
++ do {
++ Display *dpy = XOpenDisplay(NULL);
++ Window win = XCreateWindow(dpy, DefaultRootWindow(dpy),
++ 0, 0, width, height, 0,
++ DefaultDepth(dpy, DefaultScreen(dpy)),
++ InputOutput,
++ DefaultVisual(dpy, DefaultScreen(dpy)),
++ CWOverrideRedirect, &attr);
++ if (flags & COMPOSITE)
++ XCompositeRedirectWindow(dpy, win, CompositeRedirectManual);
++ XMapWindow(dpy, win);
++
++ DRI2CreateDrawable(dpy, win);
++ free(DRI2GetBuffers(dpy, win, &width, &height,
++ attachments, nattachments, &count));
++ if (count != nattachments)
++ return;
++
++ for (count = 0; count < loop; count++)
++ swap_buffers(dpy, win, divisors[n], attachments, nattachments);
++ XCloseDisplay(dpy);
++ printf("."); fflush(stdout);
++ } while (--loop);
++ printf("*\n");
++ }
++
++ for (n = 0; n < N_DIVISORS; n++) {
++ printf("DRI2WaitMsc(divisor=%d)", divisors[n]);
++ fflush(stdout);
++ loop = 256 >> ffs(divisors[n]);
++ do {
++ uint64_t ignore, msc;
++ Display *dpy = XOpenDisplay(NULL);
++ xcb_connection_t *c = XGetXCBConnection(dpy);
++ Window win = XCreateWindow(dpy, DefaultRootWindow(dpy),
++ 0, 0, width, height, 0,
++ DefaultDepth(dpy, DefaultScreen(dpy)),
++ InputOutput,
++ DefaultVisual(dpy, DefaultScreen(dpy)),
++ CWOverrideRedirect, &attr);
++ if (flags & COMPOSITE)
++ XCompositeRedirectWindow(dpy, win, CompositeRedirectManual);
++ XMapWindow(dpy, win);
++
++ DRI2CreateDrawable(dpy, win);
++ DRI2GetMSC(dpy, win, &ignore, &msc, &ignore);
++ msc++;
++ for (count = 0; count < loop; count++) {
++ xcb_discard_reply(c,
++ xcb_dri2_wait_msc(c, win,
++ upper_32_bits(msc),
++ lower_32_bits(msc),
++ 0, 0, 0, 0).sequence);
++ msc += divisors[n];
++ }
++ XFlush(dpy);
++ XCloseDisplay(dpy);
++ printf("."); fflush(stdout);
++ } while (--loop);
++ printf("*\n");
++ }
++}
++
++static void race_client(int width, int height,
++ unsigned int *attachments, int nattachments,
++ unsigned flags, const char *name)
++{
++ Display *mgr = XOpenDisplay(NULL);
++ XSetWindowAttributes attr;
++ int count, loop, n;
++
++ if (flags & COMPOSITE && !has_composite(NULL))
++ return;
++
++ printf("%s(%s)\n", __func__, name);
++
++ /* Be nasty and install a fullscreen window on top so that we
++ * can guarantee we do not get clipped by children.
++ */
++ attr.override_redirect = 1;
++ for (n = 0; n < N_DIVISORS; n++) {
++ printf("DRI2SwapBuffers(divisor=%d)", divisors[n]);
++ fflush(stdout);
++ loop = 256 >> ffs(divisors[n]);
++ do {
++ Display *dpy = XOpenDisplay(NULL);
++ Window win;
++
++ if (error_get()) {
++ XCloseDisplay(dpy);
++ printf("+"); fflush(stdout);
++ continue;
++ }
++
++ win = XCreateWindow(dpy, DefaultRootWindow(dpy),
++ 0, 0, width, height, 0,
++ DefaultDepth(dpy, DefaultScreen(dpy)),
++ InputOutput,
++ DefaultVisual(dpy, DefaultScreen(dpy)),
++ CWOverrideRedirect, &attr);
++ if (flags & COMPOSITE)
++ XCompositeRedirectWindow(dpy, win, CompositeRedirectManual);
++ XMapWindow(dpy, win);
++
++ DRI2CreateDrawable(dpy, win);
++ free(DRI2GetBuffers(dpy, win, &width, &height,
++ attachments, nattachments, &count));
++ if (count == nattachments) {
++ for (count = 0; count < loop; count++)
++ DRI2SwapBuffers(dpy, win, 0, divisors[n], count & (divisors[n]-1));
++ }
++
++ XFlush(dpy);
++ XKillClient(mgr, win);
++ XFlush(mgr);
++
++ XCloseDisplay(dpy);
++ printf("."); fflush(stdout);
++
++ error_put();
++ } while (--loop);
++ printf("*\n");
++ }
++
++ for (n = 0; n < N_DIVISORS; n++) {
++ printf("xcb_dri2_swap_buffers(divisor=%d)", divisors[n]);
++ fflush(stdout);
++ loop = 256 >> ffs(divisors[n]);
++ do {
++ Display *dpy = XOpenDisplay(NULL);
++ Window win;
++
++ if (error_get()) {
++ XCloseDisplay(dpy);
++ printf("+"); fflush(stdout);
++ continue;
++ }
++
++ win = XCreateWindow(dpy, DefaultRootWindow(dpy),
++ 0, 0, width, height, 0,
++ DefaultDepth(dpy, DefaultScreen(dpy)),
++ InputOutput,
++ DefaultVisual(dpy, DefaultScreen(dpy)),
++ CWOverrideRedirect, &attr);
++ if (flags & COMPOSITE)
++ XCompositeRedirectWindow(dpy, win, CompositeRedirectManual);
++ XMapWindow(dpy, win);
++
++ DRI2CreateDrawable(dpy, win);
++ free(DRI2GetBuffers(dpy, win, &width, &height,
++ attachments, nattachments, &count));
++ if (count == nattachments) {
++ for (count = 0; count < loop; count++)
++ swap_buffers(dpy, win, divisors[n], attachments, nattachments);
++ }
++
++ XFlush(dpy);
++ XKillClient(mgr, win);
++ XFlush(mgr);
++
++ XCloseDisplay(dpy);
++ printf("."); fflush(stdout);
++
++ error_put();
++ } while (--loop);
++ printf("*\n");
++ }
++
++ for (n = 0; n < N_DIVISORS; n++) {
++ printf("DRI2WaitMsc(divisor=%d)", divisors[n]);
++ fflush(stdout);
++ loop = 256 >> ffs(divisors[n]);
++ do {
++ Display *dpy = XOpenDisplay(NULL);
++ uint64_t ignore, msc;
++ xcb_connection_t *c;
++ Window win;
++
++ if (error_get()) {
++ XCloseDisplay(dpy);
++ printf("+"); fflush(stdout);
++ continue;
++ }
++
++ win = XCreateWindow(dpy, DefaultRootWindow(dpy),
++ 0, 0, width, height, 0,
++ DefaultDepth(dpy, DefaultScreen(dpy)),
++ InputOutput,
++ DefaultVisual(dpy, DefaultScreen(dpy)),
++ CWOverrideRedirect, &attr);
++ if (flags & COMPOSITE)
++ XCompositeRedirectWindow(dpy, win, CompositeRedirectManual);
++ XMapWindow(dpy, win);
++
++ DRI2CreateDrawable(dpy, win);
++ DRI2GetMSC(dpy, win, &ignore, &msc, &ignore);
++ c = XGetXCBConnection(dpy);
++ msc++;
++ for (count = 0; count < loop; count++) {
++ xcb_discard_reply(c,
++ xcb_dri2_wait_msc(c, win,
++ upper_32_bits(msc),
++ lower_32_bits(msc),
++ 0, 0, 0, 0).sequence);
++ msc += divisors[n];
++ }
++
++ XFlush(dpy);
++ XKillClient(mgr, win);
++ XFlush(mgr);
++
++ XCloseDisplay(dpy);
++ printf("."); fflush(stdout);
++
++ error_put();
++ } while (--loop);
++ printf("*\n");
++ }
++
++ XCloseDisplay(mgr);
+ }
+
+ int main(void)
+@@ -91,7 +790,10 @@ int main(void)
+ DRI2BufferFrontLeft,
+ };
+
+- dpy = XOpenDisplay (NULL);
++ saved_io_error = XSetIOErrorHandler(io_error);
++ XSetErrorHandler(x_error);
++
++ dpy = XOpenDisplay(NULL);
+ if (dpy == NULL)
+ return 77;
+
+@@ -101,13 +803,52 @@ int main(void)
+
+ width = WidthOfScreen(DefaultScreenOfDisplay(dpy));
+ height = HeightOfScreen(DefaultScreenOfDisplay(dpy));
+- run(dpy, width, height, attachments, 1, "fullscreen");
+- run(dpy, width, height, attachments, 2, "fullscreen (with front)");
++ race_window(dpy, width, height, attachments, 1, 0, "fullscreen");
++ race_window(dpy, width, height, attachments, 1, COMPOSITE, "composite fullscreen");
++ race_window(dpy, width, height, attachments, 2, 0, "fullscreen (with front)");
++ race_window(dpy, width, height, attachments, 2, COMPOSITE, "composite fullscreen (with front)");
++
++ race_resize(dpy, width, height, attachments, 1, 0, "");
++ race_resize(dpy, width, height, attachments, 1, COMPOSITE, "composite");
++ race_resize(dpy, width, height, attachments, 2, 0, "with front");
++ race_resize(dpy, width, height, attachments, 2, COMPOSITE, "composite with front");
++
++ race_manager(dpy, width, height, attachments, 1, 0, "fullscreen");
++ race_manager(dpy, width, height, attachments, 1, COMPOSITE, "composite fullscreen");
++ race_manager(dpy, width, height, attachments, 2, 0, "fullscreen (with front)");
++ race_manager(dpy, width, height, attachments, 2, COMPOSITE, "composite fullscreen (with front)");
++
++ race_close(width, height, attachments, 1, 0, "fullscreen");
++ race_close(width, height, attachments, 1, COMPOSITE, "composite fullscreen");
++ race_close(width, height, attachments, 2, 0, "fullscreen (with front)");
++ race_close(width, height, attachments, 2, COMPOSITE, "composite fullscreen (with front)");
++
++ race_client(width, height, attachments, 1, 0, "fullscreen");
++ race_client(width, height, attachments, 1, COMPOSITE, "composite fullscreen");
++ race_client(width, height, attachments, 2, 0, "fullscreen (with front)");
++ race_client(width, height, attachments, 2, COMPOSITE, "composite fullscreen (with front)");
+
+ width /= 2;
+ height /= 2;
+- run(dpy, width, height, attachments, 1, "windowed");
+- run(dpy, width, height, attachments, 2, "windowed (with front)");
++ race_window(dpy, width, height, attachments, 1, 0, "windowed");
++ race_window(dpy, width, height, attachments, 1, COMPOSITE, "composite windowed");
++ race_window(dpy, width, height, attachments, 2, 0, "windowed (with front)");
++ race_window(dpy, width, height, attachments, 2, COMPOSITE, "composite windowed (with front)");
++
++ race_manager(dpy, width, height, attachments, 1, 0, "windowed");
++ race_manager(dpy, width, height, attachments, 1, COMPOSITE, "composite windowed");
++ race_manager(dpy, width, height, attachments, 2, 0, "windowed (with front)");
++ race_manager(dpy, width, height, attachments, 2, COMPOSITE, "composite windowed (with front)");
++
++ race_close(width, height, attachments, 1, 0, "windowed");
++ race_close(width, height, attachments, 1, COMPOSITE, "composite windowed");
++ race_close(width, height, attachments, 2, 0, "windowed (with front)");
++ race_close(width, height, attachments, 2, COMPOSITE, "composite windowed (with front)");
++
++ race_client(width, height, attachments, 1, 0, "windowed");
++ race_client(width, height, attachments, 1, COMPOSITE, "composite windowed");
++ race_client(width, height, attachments, 2, 0, "windowed (with front)");
++ race_client(width, height, attachments, 2, COMPOSITE, "composite windowed (with front)");
+
+ return 0;
+ }
+diff --git a/test/dri2-speed.c b/test/dri2-speed.c
+new file mode 100644
+index 00000000..87b9d0b6
+--- /dev/null
++++ b/test/dri2-speed.c
+@@ -0,0 +1,342 @@
++/*
++ * Copyright (c) 2015 Intel Corporation
++ *
++ * Permission is hereby granted, free of charge, to any person obtaining a
++ * copy of this software and associated documentation files (the "Software"),
++ * to deal in the Software without restriction, including without limitation
++ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
++ * and/or sell copies of the Software, and to permit persons to whom the
++ * Software is furnished to do so, subject to the following conditions:
++ *
++ * The above copyright notice and this permission notice (including the next
++ * paragraph) shall be included in all copies or substantial portions of the
++ * Software.
++ *
++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
++ * SOFTWARE.
++ *
++ */
++
++#ifdef HAVE_CONFIG_H
++#include "config.h"
++#endif
++
++#include <X11/Xlib.h>
++#include <X11/Xatom.h>
++#include <X11/Xlib-xcb.h>
++#include <X11/Xutil.h>
++#include <X11/Xlibint.h>
++#include <X11/extensions/dpms.h>
++#include <X11/extensions/randr.h>
++#include <X11/extensions/Xcomposite.h>
++#include <X11/extensions/Xdamage.h>
++#include <X11/extensions/Xrandr.h>
++#include <xcb/xcb.h>
++#include <xcb/dri2.h>
++#include <xf86drm.h>
++
++#include <stdio.h>
++#include <string.h>
++#include <fcntl.h>
++#include <unistd.h>
++#include <assert.h>
++#include <errno.h>
++#include <setjmp.h>
++#include <signal.h>
++
++#include "dri2.h"
++
++static int _x_error_occurred;
++
++static int
++_check_error_handler(Display *display,
++ XErrorEvent *event)
++{
++ printf("X11 error from display %s, serial=%ld, error=%d, req=%d.%d\n",
++ DisplayString(display),
++ event->serial,
++ event->error_code,
++ event->request_code,
++ event->minor_code);
++ _x_error_occurred++;
++ return False; /* ignored */
++}
++
++static double elapsed(const struct timespec *start,
++ const struct timespec *end)
++{
++ return 1e6*(end->tv_sec - start->tv_sec) + (end->tv_nsec - start->tv_nsec)/1000;
++}
++
++static void run(Display *dpy, Window win, const char *name)
++{
++ xcb_connection_t *c = XGetXCBConnection(dpy);
++ struct timespec start, end;
++ int n, completed = 0;
++
++ _x_error_occurred = 0;
++
++ clock_gettime(CLOCK_MONOTONIC, &start);
++ do {
++ for (n = 0; n < 1000; n++) {
++ unsigned int attachments[] = { DRI2BufferBackLeft };
++ unsigned int seq[2];
++
++ seq[0] = xcb_dri2_swap_buffers_unchecked(c, win,
++ 0, 0, 0, 0, 0, 0).sequence;
++
++
++ seq[1] = xcb_dri2_get_buffers_unchecked(c, win,
++ 1, 1, attachments).sequence;
++
++ xcb_flush(c);
++ xcb_discard_reply(c, seq[0]);
++ xcb_discard_reply(c, seq[1]);
++ completed++;
++ }
++ clock_gettime(CLOCK_MONOTONIC, &end);
++ } while (end.tv_sec < start.tv_sec + 10);
++
++ XSync(dpy, True);
++ if (_x_error_occurred)
++ abort();
++
++ printf("%s: Completed %d swaps in %.1fs, %.3fus each (%.1f FPS)\n",
++ name, completed, elapsed(&start, &end) / 1000000,
++ elapsed(&start, &end) / completed,
++ completed / (elapsed(&start, &end) / 1000000));
++}
++
++static inline XRRScreenResources *_XRRGetScreenResourcesCurrent(Display *dpy, Window window)
++{
++ XRRScreenResources *res;
++
++ res = XRRGetScreenResourcesCurrent(dpy, window);
++ if (res == NULL)
++ res = XRRGetScreenResources(dpy, window);
++
++ return res;
++}
++
++static XRRModeInfo *lookup_mode(XRRScreenResources *res, int id)
++{
++ int i;
++
++ for (i = 0; i < res->nmode; i++) {
++ if (res->modes[i].id == id)
++ return &res->modes[i];
++ }
++
++ return NULL;
++}
++
++static int dri2_open(Display *dpy)
++{
++ drm_auth_t auth;
++ char *driver, *device;
++ int fd;
++
++ if (!DRI2Connect(dpy, DefaultRootWindow(dpy), &driver, &device))
++ return -1;
++
++ printf ("Connecting to %s driver on %s\n", driver, device);
++
++ fd = open(device, O_RDWR);
++ if (fd < 0)
++ return -1;
++
++ if (drmIoctl(fd, DRM_IOCTL_GET_MAGIC, &auth))
++ return -1;
++
++ if (!DRI2Authenticate(dpy, DefaultRootWindow(dpy), auth.magic))
++ return -1;
++
++ return fd;
++}
++
++static void fullscreen(Display *dpy, Window win)
++{
++ Atom atom = XInternAtom(dpy, "_NET_WM_STATE_FULLSCREEN", False);
++ XChangeProperty(dpy, win,
++ XInternAtom(dpy, "_NET_WM_STATE", False),
++ XA_ATOM, 32, PropModeReplace,
++ (unsigned char *)&atom, 1);
++}
++
++static int has_composite(Display *dpy)
++{
++ int event, error;
++ int major, minor;
++
++ if (!XDamageQueryExtension (dpy, &event, &error))
++ return 0;
++
++ if (!XCompositeQueryExtension(dpy, &event, &error))
++ return 0;
++
++ XCompositeQueryVersion(dpy, &major, &minor);
++
++ return major > 0 || minor >= 4;
++}
++
++int main(void)
++{
++ Display *dpy;
++ Window root, win;
++ XRRScreenResources *res;
++ XRRCrtcInfo **original_crtc;
++ XSetWindowAttributes attr;
++ int i, j, fd;
++
++ attr.override_redirect = 1;
++
++ dpy = XOpenDisplay(NULL);
++ if (dpy == NULL)
++ return 77;
++
++ fd = dri2_open(dpy);
++ if (fd < 0)
++ return 77;
++
++ if (DPMSQueryExtension(dpy, &i, &i))
++ DPMSDisable(dpy);
++
++ root = DefaultRootWindow(dpy);
++
++ signal(SIGALRM, SIG_IGN);
++ XSetErrorHandler(_check_error_handler);
++
++ res = NULL;
++ if (XRRQueryVersion(dpy, &i, &i))
++ res = _XRRGetScreenResourcesCurrent(dpy, root);
++ if (res == NULL)
++ return 77;
++
++ original_crtc = malloc(sizeof(XRRCrtcInfo *)*res->ncrtc);
++ for (i = 0; i < res->ncrtc; i++)
++ original_crtc[i] = XRRGetCrtcInfo(dpy, res, res->crtcs[i]);
++
++ printf("noutput=%d, ncrtc=%d\n", res->noutput, res->ncrtc);
++ for (i = 0; i < res->ncrtc; i++)
++ XRRSetCrtcConfig(dpy, res, res->crtcs[i], CurrentTime,
++ 0, 0, None, RR_Rotate_0, NULL, 0);
++
++ DRI2CreateDrawable(dpy, root);
++ DRI2SwapInterval(dpy, root, 0);
++ run(dpy, root, "off");
++ XSync(dpy, True);
++
++ for (i = 0; i < res->noutput; i++) {
++ XRROutputInfo *output;
++ XRRModeInfo *mode;
++
++ output = XRRGetOutputInfo(dpy, res, res->outputs[i]);
++ if (output == NULL)
++ continue;
++
++ mode = NULL;
++ if (res->nmode)
++ mode = lookup_mode(res, output->modes[0]);
++
++ for (j = 0; mode && j < 2*output->ncrtc; j++) {
++ int c = j;
++ if (c >= output->ncrtc)
++ c = 2*output->ncrtc - j - 1;
++
++ printf("[%d, %d] -- OUTPUT:%ld, CRTC:%ld: %dx%d\n",
++ i, c, (long)res->outputs[i], (long)output->crtcs[c],
++ mode->width, mode->height);
++ XRRSetCrtcConfig(dpy, res, output->crtcs[c], CurrentTime,
++ 0, 0, output->modes[0], RR_Rotate_0, &res->outputs[i], 1);
++
++ run(dpy, root, "root");
++ XSync(dpy, True);
++
++ win = XCreateWindow(dpy, root,
++ 0, 0, mode->width, mode->height, 0,
++ DefaultDepth(dpy, DefaultScreen(dpy)),
++ InputOutput,
++ DefaultVisual(dpy, DefaultScreen(dpy)),
++ CWOverrideRedirect, &attr);
++ DRI2CreateDrawable(dpy, win);
++ DRI2SwapInterval(dpy, win, 0);
++ fullscreen(dpy, win);
++ XMapWindow(dpy, win);
++ run(dpy, win, "fullscreen");
++ XDestroyWindow(dpy, win);
++ XSync(dpy, True);
++
++ win = XCreateWindow(dpy, root,
++ 0, 0, mode->width, mode->height, 0,
++ DefaultDepth(dpy, DefaultScreen(dpy)),
++ InputOutput,
++ DefaultVisual(dpy, DefaultScreen(dpy)),
++ CWOverrideRedirect, &attr);
++ DRI2CreateDrawable(dpy, win);
++ DRI2SwapInterval(dpy, win, 0);
++ XMapWindow(dpy, win);
++ run(dpy, win, "windowed");
++ XDestroyWindow(dpy, win);
++ XSync(dpy, True);
++
++ if (has_composite(dpy)) {
++ Damage damage;
++
++ _x_error_occurred = 0;
++ win = XCreateWindow(dpy, root,
++ 0, 0, mode->width, mode->height, 0,
++ DefaultDepth(dpy, DefaultScreen(dpy)),
++ InputOutput,
++ DefaultVisual(dpy, DefaultScreen(dpy)),
++ CWOverrideRedirect, &attr);
++ XCompositeRedirectWindow(dpy, win, CompositeRedirectManual);
++ damage = XDamageCreate(dpy, win, XDamageReportRawRectangles);
++ DRI2CreateDrawable(dpy, win);
++ DRI2SwapInterval(dpy, win, 0);
++ XMapWindow(dpy, win);
++ XSync(dpy, True);
++ if (!_x_error_occurred)
++ run(dpy, win, "composited");
++ XDamageDestroy(dpy, damage);
++ XDestroyWindow(dpy, win);
++ XSync(dpy, True);
++ }
++
++ win = XCreateWindow(dpy, root,
++ 0, 0, mode->width/2, mode->height/2, 0,
++ DefaultDepth(dpy, DefaultScreen(dpy)),
++ InputOutput,
++ DefaultVisual(dpy, DefaultScreen(dpy)),
++ CWOverrideRedirect, &attr);
++ DRI2CreateDrawable(dpy, win);
++ DRI2SwapInterval(dpy, win, 0);
++ XMapWindow(dpy, win);
++ run(dpy, win, "half");
++ XDestroyWindow(dpy, win);
++ XSync(dpy, True);
++
++ XRRSetCrtcConfig(dpy, res, output->crtcs[c], CurrentTime,
++ 0, 0, None, RR_Rotate_0, NULL, 0);
++ }
++
++ XRRFreeOutputInfo(output);
++ }
++
++ for (i = 0; i < res->ncrtc; i++)
++ XRRSetCrtcConfig(dpy, res, res->crtcs[i], CurrentTime,
++ original_crtc[i]->x,
++ original_crtc[i]->y,
++ original_crtc[i]->mode,
++ original_crtc[i]->rotation,
++ original_crtc[i]->outputs,
++ original_crtc[i]->noutput);
++
++ if (DPMSQueryExtension(dpy, &i, &i))
++ DPMSEnable(dpy);
++ return 0;
++}
+diff --git a/test/dri2-test.c b/test/dri2-test.c
+index dd4179f3..bdf01f38 100644
+--- a/test/dri2-test.c
++++ b/test/dri2-test.c
+@@ -6,6 +6,10 @@
+ #include <X11/Xutil.h>
+ #include <X11/extensions/Xfixes.h>
+ #include <X11/extensions/Xrandr.h>
++#include <X11/Xlib-xcb.h>
++#include <xcb/xcb.h>
++#include <xcb/xcbext.h>
++#include <xcb/dri2.h>
+ #include <unistd.h>
+ #include <fcntl.h>
+ #include <string.h>
+@@ -18,6 +22,8 @@
+
+ #define COUNT 60
+
++static int prime[] = { 0, 1, 2, 3, 5, 7, 11, 13, 17, 19, 23, 27, 29, 31, 37, 41, 43, 47, 51, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131 };
++
+ static inline XRRScreenResources *_XRRGetScreenResourcesCurrent(Display *dpy, Window window)
+ {
+ XRRScreenResources *res;
+@@ -101,16 +107,41 @@ static uint64_t check_msc(Display *dpy, Window win, uint64_t last_msc)
+ return current_msc;
+ }
+
++static void wait_next_vblank(Display *dpy, Window win)
++{
++ uint64_t msc, ust, sbc;
++ DRI2WaitMSC(dpy, win, 0, 1, 0, &ust, &msc, &sbc);
++}
++
++static void swap_buffers(xcb_connection_t *c, Window win,
++ unsigned int *attachments, int nattachments)
++{
++ unsigned int seq[2];
++
++ seq[0] = xcb_dri2_swap_buffers_unchecked(c, win,
++ 0, 0, 0, 0, 0, 0).sequence;
++
++
++ seq[1] = xcb_dri2_get_buffers_unchecked(c, win,
++ nattachments, nattachments,
++ attachments).sequence;
++
++ xcb_flush(c);
++ xcb_discard_reply(c, seq[0]);
++ xcb_discard_reply(c, seq[1]);
++}
++
+ static void run(Display *dpy, int width, int height,
+ unsigned int *attachments, int nattachments,
+ const char *name)
+ {
++ xcb_connection_t *c = XGetXCBConnection(dpy);
+ Window win;
+ XSetWindowAttributes attr;
+- int count;
+ DRI2Buffer *buffers;
+ struct timespec start, end;
+- uint64_t msc;
++ uint64_t start_msc, end_msc;
++ int modulus, remainder, count;
+
+ /* Be nasty and install a fullscreen window on top so that we
+ * can guarantee we do not get clipped by children.
+@@ -125,42 +156,99 @@ static void run(Display *dpy, int width, int height,
+ XMapWindow(dpy, win);
+
+ DRI2CreateDrawable(dpy, win);
+- msc = check_msc(dpy, win, 0);
++ DRI2SwapInterval(dpy, win, 1);
++ start_msc = check_msc(dpy, win, 0);
+
+ buffers = DRI2GetBuffers(dpy, win, &width, &height,
+ attachments, nattachments, &count);
+ if (count != nattachments)
+ return;
+
+- msc = check_msc(dpy, win, msc);
++ swap_buffers(c, win, attachments, nattachments);
++ start_msc = check_msc(dpy, win, start_msc);
+ clock_gettime(CLOCK_MONOTONIC, &start);
+ for (count = 0; count < COUNT; count++)
+- DRI2SwapBuffers(dpy, win, 0, 0, 0);
+- msc = check_msc(dpy, win, msc);
++ swap_buffers(c, win, attachments, nattachments);
++ end_msc = check_msc(dpy, win, start_msc);
+ clock_gettime(CLOCK_MONOTONIC, &end);
+- printf("%d %s (%dx%d) swaps in %fs.\n",
+- count, name, width, height, elapsed(&start, &end));
++ printf("%d [%ld] %s (%dx%d) swaps in %fs.\n",
++ count, (long)(end_msc - start_msc),
++ name, width, height, elapsed(&start, &end));
+
+- msc = check_msc(dpy, win, msc);
++ swap_buffers(c, win, attachments, nattachments);
++ start_msc = check_msc(dpy, win, end_msc);
+ clock_gettime(CLOCK_MONOTONIC, &start);
+ for (count = 0; count < COUNT; count++)
+ dri2_copy_swap(dpy, win, width, height, nattachments == 2);
+- msc = check_msc(dpy, win, msc);
++ end_msc = check_msc(dpy, win, start_msc);
+ clock_gettime(CLOCK_MONOTONIC, &end);
+
+- printf("%d %s (%dx%d) blits in %fs.\n",
+- count, name, width, height, elapsed(&start, &end));
++ printf("%d [%ld] %s (%dx%d) blits in %fs.\n",
++ count, (long)(end_msc - start_msc),
++ name, width, height, elapsed(&start, &end));
+
+ DRI2SwapInterval(dpy, win, 0);
++ wait_next_vblank(dpy, win);
++
++ swap_buffers(c, win, attachments, nattachments);
++ start_msc = check_msc(dpy, win, end_msc);
++ clock_gettime(CLOCK_MONOTONIC, &start);
++ for (count = 0; count < COUNT; count++)
++ swap_buffers(c, win, attachments, nattachments);
++ end_msc = check_msc(dpy, win, start_msc);
++ clock_gettime(CLOCK_MONOTONIC, &end);
++ printf("%d [%ld] %s (%dx%d) vblank=0 swaps in %fs.\n",
++ count, (long)(end_msc - start_msc),
++ name, width, height, elapsed(&start, &end));
+
+- msc = check_msc(dpy, win, msc);
++ start_msc = check_msc(dpy, win, end_msc);
+ clock_gettime(CLOCK_MONOTONIC, &start);
+ for (count = 0; count < COUNT; count++)
+- DRI2SwapBuffers(dpy, win, 0, 0, 0);
+- msc = check_msc(dpy, win, msc);
++ wait_next_vblank(dpy, win);
++ end_msc = check_msc(dpy, win, start_msc);
+ clock_gettime(CLOCK_MONOTONIC, &end);
+- printf("%d %s (%dx%d) vblank=0 swaps in %fs.\n",
+- count, name, width, height, elapsed(&start, &end));
++ printf("%d [%ld] %s waits in %fs.\n",
++ count, (long)(end_msc - start_msc),
++ name, elapsed(&start, &end));
++
++ printf("Testing past & future waits\n");
++ for (modulus = 1; modulus <= 128; modulus <<= 1) {
++ for (count = 0; prime[count] < modulus; count++) {
++ uint64_t msc, ust, sbc;
++ uint64_t target;
++
++ remainder = prime[count];
++
++ DRI2WaitMSC(dpy, win, 0, 1, 0, &ust, &msc, &sbc);
++
++ target = msc + modulus + 1;
++ target &= -modulus;
++ target += remainder;
++
++ DRI2WaitMSC(dpy, win, target, modulus, remainder,
++ &ust, &msc, &sbc);
++ if (msc != target) {
++ printf("Missed future MSC (%d, %d): expected=%lld, found=%lld\n",
++ modulus, remainder,
++ (long long)target, (long long)msc);
++ }
++
++ target = msc;
++ target &= -modulus;
++ target += remainder;
++ if (target <= msc)
++ target += modulus;
++
++ DRI2WaitMSC(dpy, win, msc, modulus, remainder,
++ &ust, &msc, &sbc);
++
++ if (msc != target) {
++ printf("Missed past MSC (%d, %d): expected=%lld, found=%lld\n",
++ modulus, remainder,
++ (long long)target, (long long)msc);
++ }
++ }
++ }
+
+ XDestroyWindow(dpy, win);
+ free(buffers);
+diff --git a/test/dri3-test.c b/test/dri3-test.c
+index c66da313..78e105a8 100644
+--- a/test/dri3-test.c
++++ b/test/dri3-test.c
+@@ -93,14 +93,9 @@ static const struct pci_id_match ids[] = {
+ INTEL_IVB_D_IDS(070),
+ INTEL_IVB_M_IDS(070),
+
+- INTEL_HSW_D_IDS(075),
+- INTEL_HSW_M_IDS(075),
+-
+- INTEL_VLV_D_IDS(071),
+- INTEL_VLV_M_IDS(071),
+-
+- INTEL_BDW_D_IDS(0100),
+- INTEL_BDW_M_IDS(0100),
++ INTEL_HSW_IDS(075),
++ INTEL_VLV_IDS(071),
++ INTEL_BDW_IDS(0100),
+ };
+
+ static int i915_gen(int device)
+@@ -1020,6 +1015,67 @@ fail:
+ return 1;
+ }
+
++static int gem_set_tiling(int fd, uint32_t handle, int tiling, int stride)
++{
++ struct drm_i915_gem_set_tiling set_tiling;
++
++ set_tiling.handle = handle;
++ set_tiling.tiling_mode = tiling;
++ set_tiling.stride = stride;
++
++ return drmIoctl(fd, DRM_IOCTL_I915_GEM_SET_TILING, &set_tiling) == 0;
++}
++
++static int test_tiling(Display *dpy, int device)
++{
++ Window root = RootWindow(dpy, DefaultScreen(dpy));
++ const int tiling[] = { I915_TILING_NONE, I915_TILING_X, I915_TILING_Y };
++ int line = -1;
++ int t;
++
++ _x_error_occurred = 0;
++
++ for (t = 0; t < sizeof(tiling)/sizeof(tiling[0]); t++) {
++ uint32_t src;
++ int src_fd;
++ Pixmap src_pix;
++
++ src = gem_create(device, 4*4096);
++ if (!src) {
++ line = __LINE__;
++ goto fail;
++ }
++
++ gem_set_tiling(device, src, tiling[t], 512);
++
++ src_fd = gem_export(device, src);
++ if (src_fd < 0) {
++ line = __LINE__;
++ goto fail;
++ }
++
++ src_pix = dri3_create_pixmap(dpy, root,
++ 128, 32, 32,
++ src_fd, 32, 512, 4*4096);
++ XSync(dpy, True);
++ if (_x_error_occurred) {
++ line = __LINE__;
++ goto fail;
++ }
++ XFreePixmap(dpy, src_pix);
++ _x_error_occurred = 0;
++
++ close(src_fd);
++ gem_close(device, src);
++ }
++
++ return 0;
++
++fail:
++ printf("%s failed with tiling %d, line %d\n", __func__, tiling[t], line);
++ return 1;
++}
++
+ static int
+ _check_error_handler(Display *display,
+ XErrorEvent *event)
+@@ -1060,6 +1116,7 @@ int main(void)
+
+ error += test_bad_size(dpy, device);
+ error += test_bad_pitch(dpy, device);
++ error += test_tiling(dpy, device);
+
+ error += test_shm(dpy, device, 400, 300);
+ error += test_shm(dpy, device, 300, 400);
+diff --git a/test/dri3.c b/test/dri3.c
+index 45f3285c..e5644629 100644
+--- a/test/dri3.c
++++ b/test/dri3.c
+@@ -29,6 +29,7 @@
+ #include <xcb/dri3.h>
+ #include <xcb/sync.h>
+ #include <unistd.h>
++#include <stdlib.h>
+
+ #include "dri3.h"
+
+@@ -109,12 +110,45 @@ void dri3_fence_free(Display *dpy, struct dri3_fence *fence)
+ xcb_sync_destroy_fence(c, fence->xid);
+ }
+
++static void dri3_query_version(xcb_connection_t *c, int *major, int *minor)
++{
++ xcb_dri3_query_version_reply_t *reply;
++
++ reply = xcb_dri3_query_version_reply(c,
++ xcb_dri3_query_version(c,
++ XCB_DRI3_MAJOR_VERSION,
++ XCB_DRI3_MINOR_VERSION),
++ NULL);
++ if (reply != NULL) {
++ *major = reply->major_version;
++ *minor = reply->minor_version;
++ free(reply);
++ }
++}
++
++static int dri3_exists(xcb_connection_t *c)
++{
++ const xcb_query_extension_reply_t *ext;
++ int major, minor;
++
++ major = minor = -1;
++
++ ext = xcb_get_extension_data(c, &xcb_dri3_id);
++ if (ext != NULL && ext->present)
++ dri3_query_version(c, &major, &minor);
++
++ return major >= 0;
++}
++
+ int dri3_open__full(Display *dpy, Window root, unsigned provider)
+ {
+ xcb_connection_t *c = XGetXCBConnection(dpy);
+ xcb_dri3_open_cookie_t cookie;
+ xcb_dri3_open_reply_t *reply;
+
++ if (!dri3_exists(c))
++ return -1;
++
+ cookie = xcb_dri3_open(c, root, provider);
+ reply = xcb_dri3_open_reply(c, cookie, NULL);
+
+diff --git a/test/present-race.c b/test/present-race.c
+new file mode 100644
+index 00000000..b2b6aa2b
+--- /dev/null
++++ b/test/present-race.c
+@@ -0,0 +1,484 @@
++/*
++ * Copyright (c) 2014 Intel Corporation
++ *
++ * Permission is hereby granted, free of charge, to any person obtaining a
++ * copy of this software and associated documentation files (the "Software"),
++ * to deal in the Software without restriction, including without limitation
++ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
++ * and/or sell copies of the Software, and to permit persons to whom the
++ * Software is furnished to do so, subject to the following conditions:
++ *
++ * The above copyright notice and this permission notice (including the next
++ * paragraph) shall be included in all copies or substantial portions of the
++ * Software.
++ *
++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
++ * SOFTWARE.
++ *
++ */
++
++#ifdef HAVE_CONFIG_H
++#include "config.h"
++#endif
++
++#include <X11/Xlib.h>
++#include <X11/Xlib-xcb.h>
++#include <X11/xshmfence.h>
++#include <X11/Xutil.h>
++#include <X11/Xlibint.h>
++#include <X11/extensions/dpms.h>
++#include <X11/extensions/randr.h>
++#include <X11/extensions/Xcomposite.h>
++#include <X11/extensions/Xrandr.h>
++#include <X11/extensions/Xrender.h>
++#include <X11/extensions/XShm.h>
++#if HAVE_X11_EXTENSIONS_SHMPROTO_H
++#include <X11/extensions/shmproto.h>
++#elif HAVE_X11_EXTENSIONS_SHMSTR_H
++#include <X11/extensions/shmstr.h>
++#else
++#error Failed to find the right header for X11 MIT-SHM protocol definitions
++#endif
++#include <xcb/xcb.h>
++#include <xcb/present.h>
++#include <xcb/xfixes.h>
++#include <xcb/dri3.h>
++#include <xf86drm.h>
++#include <i915_drm.h>
++
++#include <stdio.h>
++#include <string.h>
++#include <fcntl.h>
++#include <unistd.h>
++#include <assert.h>
++#include <errno.h>
++#include <setjmp.h>
++#include <signal.h>
++
++#include <sys/mman.h>
++#include <sys/ipc.h>
++#include <sys/shm.h>
++#include <pciaccess.h>
++
++#include "dri3.h"
++
++static int _x_error_occurred;
++static uint32_t stamp;
++
++static int
++_check_error_handler(Display *display,
++ XErrorEvent *event)
++{
++ printf("X11 error from display %s, serial=%ld, error=%d, req=%d.%d\n",
++ DisplayString(display),
++ event->serial,
++ event->error_code,
++ event->request_code,
++ event->minor_code);
++ _x_error_occurred++;
++ return False; /* ignored */
++}
++
++static int has_composite(Display *dpy)
++{
++ int event, error;
++ int major, minor;
++
++ if (!XCompositeQueryExtension(dpy, &event, &error))
++ return 0;
++
++ XCompositeQueryVersion(dpy, &major, &minor);
++
++ return major > 0 || minor >= 4;
++}
++
++static void *setup_msc(Display *dpy, Window win)
++{
++ xcb_connection_t *c = XGetXCBConnection(dpy);
++ xcb_void_cookie_t cookie;
++ uint32_t id = xcb_generate_id(c);
++ xcb_generic_error_t *error;
++ void *q;
++
++ cookie = xcb_present_select_input_checked(c, id, win, XCB_PRESENT_EVENT_MASK_COMPLETE_NOTIFY);
++ q = xcb_register_for_special_xge(c, &xcb_present_id, id, &stamp);
++
++ error = xcb_request_check(c, cookie);
++ assert(error == NULL);
++
++ return q;
++}
++
++static void teardown_msc(Display *dpy, void *q)
++{
++ xcb_unregister_for_special_event(XGetXCBConnection(dpy), q);
++}
++
++static uint64_t wait_vblank(Display *dpy, Window win)
++{
++ xcb_connection_t *c = XGetXCBConnection(dpy);
++ static uint32_t serial = 1;
++ uint64_t msc = 0;
++ int complete = 0;
++ void *q;
++
++ if (win == 0)
++ win = DefaultRootWindow(dpy);
++
++ q = setup_msc(dpy, win);
++
++ xcb_present_notify_msc(c, win, serial ^ 0xdeadbeef, 0, 1, 0);
++ xcb_flush(c);
++
++ do {
++ xcb_present_complete_notify_event_t *ce;
++ xcb_generic_event_t *ev;
++
++ ev = xcb_wait_for_special_event(c, q);
++ if (ev == NULL)
++ break;
++
++ ce = (xcb_present_complete_notify_event_t *)ev;
++ if (ce->kind == XCB_PRESENT_COMPLETE_KIND_NOTIFY_MSC &&
++ ce->serial == (serial ^ 0xdeadbeef)) {
++ msc = ce->msc;
++ complete = 1;
++ }
++ free(ev);
++ } while (!complete);
++
++ if (++serial == 0)
++ serial = 1;
++
++ teardown_msc(dpy, q);
++
++ return msc;
++}
++
++static int test_basic(Display *dpy, int dummy)
++{
++ xcb_connection_t *c = XGetXCBConnection(dpy);
++ XSetWindowAttributes attr;
++ Visual *visual = DefaultVisual(dpy, DefaultScreen(dpy));
++ Pixmap pixmap;
++ struct dri3_fence fence;
++ Window root, win;
++ unsigned int width, height;
++ unsigned border, depth;
++ int x, y, ret = 1;
++ const char *phase;
++ uint64_t msc;
++
++ root = DefaultRootWindow(dpy);
++ XGetGeometry(dpy, root,
++ &win, &x, &y,
++ &width, &height, &border, &depth);
++
++ _x_error_occurred = 0;
++ attr.override_redirect = 1;
++ switch (dummy) {
++ case 0:
++ win = root;
++ phase = "root";
++ break;
++ case 1:
++ win = XCreateWindow(dpy, root,
++ 0, 0, width, height, 0, depth,
++ InputOutput, visual,
++ CWOverrideRedirect, &attr);
++ phase = "fullscreen";
++ break;
++ case 2:
++ width /= 2;
++ height /= 2;
++ win = XCreateWindow(dpy, root,
++ 0, 0, width, height, 0, depth,
++ InputOutput, visual,
++ CWOverrideRedirect, &attr);
++ phase = "window";
++ break;
++ case 3:
++ if (!has_composite(dpy))
++ return 0;
++
++ win = XCreateWindow(dpy, root,
++ 0, 0, width, height, 0,
++ DefaultDepth(dpy, DefaultScreen(dpy)),
++ InputOutput,
++ DefaultVisual(dpy, DefaultScreen(dpy)),
++ CWOverrideRedirect, &attr);
++ XCompositeRedirectWindow(dpy, win, CompositeRedirectManual);
++ phase = "composite";
++ break;
++
++ default:
++ phase = "broken";
++ win = root;
++ abort();
++ break;
++ }
++
++ XMapWindow(dpy, win);
++ XSync(dpy, True);
++ if (_x_error_occurred)
++ return 1;
++
++ if (dri3_create_fence(dpy, win, &fence))
++ return 0;
++
++ printf("%s: Testing basic flip: %dx%d\n", phase, width, height);
++ fflush(stdout);
++ _x_error_occurred = 0;
++
++ xshmfence_reset(fence.addr);
++ msc = wait_vblank(dpy, win);
++
++ pixmap = XCreatePixmap(dpy, win, width, height, depth);
++ xcb_present_pixmap(c, win, pixmap, 0,
++ 0, /* valid */
++ 0, /* update */
++ 0, /* x_off */
++ 0, /* y_off */
++ None,
++ None, /* wait fence */
++ fence.xid,
++ XCB_PRESENT_OPTION_NONE,
++ (msc + 64) & -64, /* target msc */
++ 64, /* divisor */
++ 32, /* remainder */
++ 0, NULL);
++ XFreePixmap(dpy, pixmap);
++
++ pixmap = XCreatePixmap(dpy, win, width, height, depth);
++ xcb_present_pixmap(c, win, pixmap, 0,
++ 0, /* valid */
++ 0, /* update */
++ 0, /* x_off */
++ 0, /* y_off */
++ None,
++ None, /* wait fence */
++ None, /* sync fence */
++ XCB_PRESENT_OPTION_NONE,
++ (msc + 64) & -64, /* target msc */
++ 64, /* divisor */
++ 48, /* remainder */
++ 0, NULL);
++ XFreePixmap(dpy, pixmap);
++ XDestroyWindow(dpy, win);
++ XFlush(dpy);
++
++ ret = !!xshmfence_await(fence.addr);
++ dri3_fence_free(dpy, &fence);
++
++ XSync(dpy, True);
++ ret += !!_x_error_occurred;
++
++ return ret;
++}
++
++static int test_race(Display *dpy, int dummy)
++{
++ Display *mgr = XOpenDisplay(NULL);
++ xcb_connection_t *c = XGetXCBConnection(dpy);
++ XSetWindowAttributes attr;
++ Visual *visual = DefaultVisual(dpy, DefaultScreen(dpy));
++ Pixmap pixmap;
++ struct dri3_fence fence;
++ Window root, win;
++ unsigned int width, height;
++ unsigned border, depth;
++ int x, y, ret = 1;
++ const char *phase;
++ uint64_t msc;
++
++ root = DefaultRootWindow(dpy);
++ XGetGeometry(dpy, root,
++ &win, &x, &y,
++ &width, &height, &border, &depth);
++
++ _x_error_occurred = 0;
++ attr.override_redirect = 1;
++ switch (dummy) {
++ case 0:
++ win = root;
++ phase = "root";
++ break;
++ case 1:
++ win = XCreateWindow(dpy, root,
++ 0, 0, width, height, 0, depth,
++ InputOutput, visual,
++ CWOverrideRedirect, &attr);
++ phase = "fullscreen";
++ break;
++ case 2:
++ width /= 2;
++ height /= 2;
++ win = XCreateWindow(dpy, root,
++ 0, 0, width, height, 0, depth,
++ InputOutput, visual,
++ CWOverrideRedirect, &attr);
++ phase = "window";
++ break;
++ case 3:
++ if (!has_composite(dpy))
++ return 0;
++
++ win = XCreateWindow(dpy, root,
++ 0, 0, width, height, 0,
++ DefaultDepth(dpy, DefaultScreen(dpy)),
++ InputOutput,
++ DefaultVisual(dpy, DefaultScreen(dpy)),
++ CWOverrideRedirect, &attr);
++ XCompositeRedirectWindow(dpy, win, CompositeRedirectManual);
++ phase = "composite";
++ break;
++
++ default:
++ phase = "broken";
++ win = root;
++ abort();
++ break;
++ }
++
++ XMapWindow(dpy, win);
++ XSync(dpy, True);
++ if (_x_error_occurred)
++ return 1;
++
++ if (dri3_create_fence(dpy, win, &fence))
++ return 0;
++
++ printf("%s: Testing race with manager: %dx%d\n", phase, width, height);
++ fflush(stdout);
++ _x_error_occurred = 0;
++
++ xshmfence_reset(fence.addr);
++ msc = wait_vblank(dpy, win);
++
++ pixmap = XCreatePixmap(dpy, win, width, height, depth);
++ xcb_present_pixmap(c, win, pixmap, 0,
++ 0, /* valid */
++ 0, /* update */
++ 0, /* x_off */
++ 0, /* y_off */
++ None,
++ None, /* wait fence */
++ fence.xid,
++ XCB_PRESENT_OPTION_NONE,
++ (msc + 64) & -64, /* target msc */
++ 64, /* divisor */
++ 32, /* remainder */
++ 0, NULL);
++ XFreePixmap(dpy, pixmap);
++
++ XFlush(dpy);
++ XDestroyWindow(mgr, win);
++ XFlush(mgr);
++
++ pixmap = XCreatePixmap(dpy, win, width, height, depth);
++ xcb_present_pixmap(c, win, pixmap, 0,
++ 0, /* valid */
++ 0, /* update */
++ 0, /* x_off */
++ 0, /* y_off */
++ None,
++ None, /* wait fence */
++ None, /* sync fence */
++ XCB_PRESENT_OPTION_NONE,
++ (msc + 64) & -64, /* target msc */
++ 64, /* divisor */
++ 48, /* remainder */
++ 0, NULL);
++ XFreePixmap(dpy, pixmap);
++ XFlush(dpy);
++
++ ret = !!xshmfence_await(fence.addr);
++ dri3_fence_free(dpy, &fence);
++
++ XSync(dpy, True);
++ ret += !!_x_error_occurred;
++
++ XCloseDisplay(mgr);
++
++ return ret;
++}
++
++static int has_present(Display *dpy)
++{
++ xcb_connection_t *c = XGetXCBConnection(dpy);
++ xcb_generic_error_t *error = NULL;
++ void *reply;
++
++ reply = xcb_xfixes_query_version_reply(c,
++ xcb_xfixes_query_version(c,
++ XCB_XFIXES_MAJOR_VERSION,
++ XCB_XFIXES_MINOR_VERSION),
++ &error);
++ free(reply);
++ free(error);
++ if (reply == NULL) {
++ fprintf(stderr, "XFixes not supported on %s\n", DisplayString(dpy));
++ return 0;
++ }
++
++ reply = xcb_dri3_query_version_reply(c,
++ xcb_dri3_query_version(c,
++ XCB_DRI3_MAJOR_VERSION,
++ XCB_DRI3_MINOR_VERSION),
++ &error);
++ free(reply);
++ free(error);
++ if (reply == NULL) {
++ fprintf(stderr, "DRI3 not supported on %s\n", DisplayString(dpy));
++ return 0;
++ }
++
++ reply = xcb_present_query_version_reply(c,
++ xcb_present_query_version(c,
++ XCB_PRESENT_MAJOR_VERSION,
++ XCB_PRESENT_MINOR_VERSION),
++ &error);
++
++ free(reply);
++ free(error);
++ if (reply == NULL) {
++ fprintf(stderr, "Present not supported on %s\n", DisplayString(dpy));
++ return 0;
++ }
++
++ return 1;
++}
++
++int main(void)
++{
++ Display *dpy;
++ int dummy;
++ int error = 0;
++
++ dpy = XOpenDisplay(NULL);
++ if (dpy == NULL)
++ return 77;
++
++ if (!has_present(dpy))
++ return 77;
++
++ if (DPMSQueryExtension(dpy, &dummy, &dummy))
++ DPMSDisable(dpy);
++
++ signal(SIGALRM, SIG_IGN);
++ XSetErrorHandler(_check_error_handler);
++
++ for (dummy = 0; dummy <= 3; dummy++) {
++ error += test_basic(dpy, dummy);
++ error += test_race(dpy, dummy);
++ }
++
++ if (DPMSQueryExtension(dpy, &dummy, &dummy))
++ DPMSEnable(dpy);
++ return !!error;
++}
+diff --git a/test/present-speed.c b/test/present-speed.c
+new file mode 100644
+index 00000000..eccde931
+--- /dev/null
++++ b/test/present-speed.c
+@@ -0,0 +1,1015 @@
++/*
++ * Copyright (c) 2015 Intel Corporation
++ *
++ * Permission is hereby granted, free of charge, to any person obtaining a
++ * copy of this software and associated documentation files (the "Software"),
++ * to deal in the Software without restriction, including without limitation
++ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
++ * and/or sell copies of the Software, and to permit persons to whom the
++ * Software is furnished to do so, subject to the following conditions:
++ *
++ * The above copyright notice and this permission notice (including the next
++ * paragraph) shall be included in all copies or substantial portions of the
++ * Software.
++ *
++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
++ * SOFTWARE.
++ *
++ */
++
++#ifdef HAVE_CONFIG_H
++#include "config.h"
++#endif
++
++#include <X11/Xlib.h>
++#include <X11/Xatom.h>
++#include <X11/Xlib-xcb.h>
++#include <X11/xshmfence.h>
++#include <X11/Xutil.h>
++#include <X11/Xlibint.h>
++#include <X11/extensions/Xcomposite.h>
++#include <X11/extensions/Xdamage.h>
++#include <X11/extensions/dpms.h>
++#include <X11/extensions/randr.h>
++#include <X11/extensions/Xrandr.h>
++#include <xcb/xcb.h>
++#include <xcb/present.h>
++#include <xcb/dri3.h>
++#include <xcb/xfixes.h>
++#include <xf86drm.h>
++#include <i915_drm.h>
++
++#include <stdio.h>
++#include <string.h>
++#include <fcntl.h>
++#include <unistd.h>
++#include <assert.h>
++#include <errno.h>
++#include <setjmp.h>
++#include <signal.h>
++#include <sys/wait.h>
++
++#include "dri3.h"
++
++static int _x_error_occurred;
++static uint32_t stamp;
++
++struct list {
++ struct list *next, *prev;
++};
++
++static void
++list_init(struct list *list)
++{
++ list->next = list->prev = list;
++}
++
++static inline void
++__list_add(struct list *entry,
++ struct list *prev,
++ struct list *next)
++{
++ next->prev = entry;
++ entry->next = next;
++ entry->prev = prev;
++ prev->next = entry;
++}
++
++static inline void
++list_add(struct list *entry, struct list *head)
++{
++ __list_add(entry, head, head->next);
++}
++
++static inline void
++__list_del(struct list *prev, struct list *next)
++{
++ next->prev = prev;
++ prev->next = next;
++}
++
++static inline void
++_list_del(struct list *entry)
++{
++ __list_del(entry->prev, entry->next);
++}
++
++static inline void
++list_move(struct list *list, struct list *head)
++{
++ if (list->prev != head) {
++ _list_del(list);
++ list_add(list, head);
++ }
++}
++
++#define __container_of(ptr, sample, member) \
++ (void *)((char *)(ptr) - ((char *)&(sample)->member - (char *)(sample)))
++
++#define list_for_each_entry(pos, head, member) \
++ for (pos = __container_of((head)->next, pos, member); \
++ &pos->member != (head); \
++ pos = __container_of(pos->member.next, pos, member))
++
++static int
++_check_error_handler(Display *display,
++ XErrorEvent *event)
++{
++ if (_x_error_occurred < 0)
++ return True;
++
++ printf("X11 error from display %s, serial=%ld, error=%d, req=%d.%d\n",
++ DisplayString(display),
++ event->serial,
++ event->error_code,
++ event->request_code,
++ event->minor_code);
++ _x_error_occurred++;
++ return False; /* ignored */
++}
++
++static double elapsed(const struct timespec *start,
++ const struct timespec *end)
++{
++ return 1e6*(end->tv_sec - start->tv_sec) + (end->tv_nsec - start->tv_nsec)/1000;
++}
++
++struct buffer {
++ struct list link;
++ Pixmap pixmap;
++ struct dri3_fence fence;
++ int fd;
++ int busy;
++ int id;
++};
++
++#define DRI3 1
++#define NOCOPY 2
++#define ASYNC 4
++static void run(Display *dpy, Window win, const char *name, unsigned options)
++{
++ xcb_connection_t *c = XGetXCBConnection(dpy);
++ struct timespec start, end;
++#define N_BACK 8
++ char test_name[128];
++ struct buffer buffer[N_BACK];
++ struct list mru;
++ Window root;
++ unsigned int width, height;
++ unsigned border, depth;
++ unsigned present_flags = 0;
++ xcb_xfixes_region_t update = 0;
++ int completed = 0;
++ int queued = 0;
++ uint32_t eid = 0;
++ void *Q = NULL;
++ int i, n;
++
++ list_init(&mru);
++
++ XGetGeometry(dpy, win,
++ &root, &i, &n, &width, &height, &border, &depth);
++
++ _x_error_occurred = 0;
++
++ for (n = 0; n < N_BACK; n++) {
++ buffer[n].pixmap = xcb_generate_id(c);
++ xcb_create_pixmap(c, depth, buffer[n].pixmap, win,
++ width, height);
++ buffer[n].fence.xid = 0;
++ buffer[n].fd = -1;
++ buffer[n].id = n;
++ if (options & DRI3) {
++ xcb_dri3_buffer_from_pixmap_reply_t *reply;
++ int *fds;
++
++ if (dri3_create_fence(dpy, win, &buffer[n].fence))
++ return;
++
++ reply = xcb_dri3_buffer_from_pixmap_reply (c,
++ xcb_dri3_buffer_from_pixmap(c, buffer[n].pixmap),
++ NULL);
++ if (reply == NULL)
++ return;
++
++ fds = xcb_dri3_buffer_from_pixmap_reply_fds (c, reply);
++ buffer[n].fd = fds[0];
++ free(reply);
++
++ /* start idle */
++ xshmfence_trigger(buffer[n].fence.addr);
++ }
++ buffer[n].busy = 0;
++ list_add(&buffer[n].link, &mru);
++ }
++ if (options & ASYNC)
++ present_flags |= XCB_PRESENT_OPTION_ASYNC;
++ if (options & NOCOPY) {
++ update = xcb_generate_id(c);
++ xcb_xfixes_create_region(c, update, 0, NULL);
++ present_flags |= XCB_PRESENT_OPTION_COPY;
++ }
++
++ if (!(options & DRI3)) {
++ eid = xcb_generate_id(c);
++ xcb_present_select_input(c, eid, win,
++ (options & NOCOPY ? 0 : XCB_PRESENT_EVENT_MASK_IDLE_NOTIFY) |
++ XCB_PRESENT_EVENT_MASK_COMPLETE_NOTIFY);
++ Q = xcb_register_for_special_xge(c, &xcb_present_id, eid, &stamp);
++ }
++
++ clock_gettime(CLOCK_MONOTONIC, &start);
++ do {
++ for (n = 0; n < 1000; n++) {
++ struct buffer *tmp, *b = NULL;
++retry:
++ list_for_each_entry(tmp, &mru, link) {
++ if (tmp->fence.xid)
++ tmp->busy = !xshmfence_query(tmp->fence.addr);
++ if (!tmp->busy) {
++ b = tmp;
++ break;
++ }
++ }
++ if (options & DRI3) {
++ if (b == NULL)
++ goto retry;
++
++ xshmfence_reset(b->fence.addr);
++ queued--;
++ completed++;
++ } else while (b == NULL) {
++ xcb_present_generic_event_t *ev;
++
++ ev = (xcb_present_generic_event_t *)
++ xcb_wait_for_special_event(c, Q);
++ if (ev == NULL)
++ abort();
++
++ do {
++ switch (ev->evtype) {
++ case XCB_PRESENT_COMPLETE_NOTIFY:
++ completed++;
++ queued--;
++ break;
++
++ case XCB_PRESENT_EVENT_IDLE_NOTIFY:
++ {
++ xcb_present_idle_notify_event_t *ie = (xcb_present_idle_notify_event_t *)ev;
++ assert(ie->serial < N_BACK);
++ buffer[ie->serial].busy = 0;
++ if (b == NULL)
++ b = &buffer[ie->serial];
++ break;
++ }
++ }
++ free(ev);
++ } while ((ev = (xcb_present_generic_event_t *)xcb_poll_for_special_event(c, Q)));
++ }
++
++ b->busy = (options & NOCOPY) == 0;
++ xcb_present_pixmap(c, win, b->pixmap, b->id,
++ 0, /* valid */
++ update, /* update */
++ 0, /* x_off */
++ 0, /* y_off */
++ None,
++ None, /* wait fence */
++ b->fence.xid,
++ present_flags,
++ 0, /* target msc */
++ 0, /* divisor */
++ 0, /* remainder */
++ 0, NULL);
++ list_move(&b->link, &mru);
++ queued++;
++ xcb_flush(c);
++ }
++ clock_gettime(CLOCK_MONOTONIC, &end);
++ } while (end.tv_sec < start.tv_sec + 10);
++
++ if (options & DRI3) {
++ struct buffer *b;
++ XID pixmap;
++
++ pixmap = xcb_generate_id(c);
++ xcb_create_pixmap(c, depth, pixmap, win, width, height);
++ xcb_present_pixmap(c, win, pixmap, 0xdeadbeef,
++ 0, /* valid */
++ None, /* update */
++ 0, /* x_off */
++ 0, /* y_off */
++ None,
++ None, /* wait fence */
++ None,
++ 0,
++ 0, /* target msc */
++ 0, /* divisor */
++ 0, /* remainder */
++ 0, NULL);
++ xcb_flush(c);
++
++ list_for_each_entry(b, &mru, link)
++ xshmfence_await(b->fence.addr);
++
++ xcb_free_pixmap(c, pixmap);
++ completed += queued;
++ } else while (queued) {
++ xcb_present_generic_event_t *ev;
++
++ ev = (xcb_present_generic_event_t *)
++ xcb_wait_for_special_event(c, Q);
++ if (ev == NULL)
++ abort();
++
++ do {
++ switch (ev->evtype) {
++ case XCB_PRESENT_COMPLETE_NOTIFY:
++ completed++;
++ queued--;
++ break;
++
++ case XCB_PRESENT_EVENT_IDLE_NOTIFY:
++ break;
++ }
++ free(ev);
++ } while ((ev = (xcb_present_generic_event_t *)xcb_poll_for_special_event(c, Q)));
++ }
++ clock_gettime(CLOCK_MONOTONIC, &end);
++
++ if (update)
++ xcb_xfixes_destroy_region(c, update);
++ for (n = 0; n < N_BACK; n++) {
++ if (buffer[n].fence.xid)
++ dri3_fence_free(dpy, &buffer[n].fence);
++ if (buffer[n].fd != -1)
++ close(buffer[n].fd);
++ xcb_free_pixmap(c, buffer[n].pixmap);
++ }
++
++ if (Q) {
++ xcb_discard_reply(c, xcb_present_select_input_checked(c, eid, win, 0).sequence);
++ XSync(dpy, True);
++ xcb_unregister_for_special_event(c, Q);
++ }
++
++ test_name[0] = '\0';
++ if (options) {
++ snprintf(test_name, sizeof(test_name), "(%s%s%s )",
++ options & NOCOPY ? " no-copy" : "",
++ options & DRI3 ? " dri3" : "",
++ options & ASYNC ? " async" : "");
++ }
++ printf("%s%s: Completed %d presents in %.1fs, %.3fus each (%.1f FPS)\n",
++ name, test_name,
++ completed, elapsed(&start, &end) / 1000000,
++ elapsed(&start, &end) / completed,
++ completed / (elapsed(&start, &end) / 1000000));
++}
++
++struct perpixel {
++ Window win;
++ struct buffer buffer[N_BACK];
++ struct list mru;
++ uint32_t eid;
++ void *Q;
++ int queued;
++};
++
++static void perpixel(Display *dpy,
++ int max_width, int max_height, unsigned options)
++{
++ //const int sz = max_width * max_height;
++ const int sz = 1048;
++ struct perpixel *pp;
++ xcb_connection_t *c = XGetXCBConnection(dpy);
++ struct timespec start, end;
++ char test_name[128];
++ unsigned present_flags = 0;
++ xcb_xfixes_region_t update = 0;
++ int completed = 0;
++ int i, n;
++
++ pp = calloc(sz, sizeof(*pp));
++ if (!pp)
++ return;
++
++ for (i = 0; i < sz; i++) {
++ XSetWindowAttributes attr = { .override_redirect = 1 };
++ int depth = DefaultDepth(dpy, DefaultScreen(dpy));
++ pp[i].win = XCreateWindow(dpy, DefaultRootWindow(dpy),
++ i % max_width, i / max_width, 1, 1, 0, depth,
++ InputOutput,
++ DefaultVisual(dpy, DefaultScreen(dpy)),
++ CWOverrideRedirect, &attr);
++ XMapWindow(dpy, pp[i].win);
++ list_init(&pp[i].mru);
++ for (n = 0; n < N_BACK; n++) {
++ pp[i].buffer[n].pixmap = xcb_generate_id(c);
++ xcb_create_pixmap(c, depth, pp[i].buffer[n].pixmap,
++ pp[i].win, 1, 1);
++ pp[i].buffer[n].fence.xid = 0;
++ pp[i].buffer[n].fd = -1;
++ pp[i].buffer[n].id = n;
++ if (options & DRI3) {
++ xcb_dri3_buffer_from_pixmap_reply_t *reply;
++ int *fds;
++
++ if (dri3_create_fence(dpy, pp[i].win, &pp[i].buffer[n].fence))
++ return;
++
++ reply = xcb_dri3_buffer_from_pixmap_reply(c,
++ xcb_dri3_buffer_from_pixmap(c, pp[i].buffer[n].pixmap),
++ NULL);
++ if (reply == NULL)
++ return;
++
++ fds = xcb_dri3_buffer_from_pixmap_reply_fds(c, reply);
++ pp[i].buffer[n].fd = fds[0];
++ free(reply);
++
++ /* start idle */
++ xshmfence_trigger(pp[i].buffer[n].fence.addr);
++ }
++ pp[i].buffer[n].busy = 0;
++ list_add(&pp[i].buffer[n].link, &pp[i].mru);
++ }
++
++ if (!(options & DRI3)) {
++ pp[i].eid = xcb_generate_id(c);
++ xcb_present_select_input(c, pp[i].eid, pp[i].win,
++ (options & NOCOPY ? 0 : XCB_PRESENT_EVENT_MASK_IDLE_NOTIFY) |
++ XCB_PRESENT_EVENT_MASK_COMPLETE_NOTIFY);
++ pp[i].Q = xcb_register_for_special_xge(c, &xcb_present_id, pp[i].eid, &stamp);
++ }
++ pp[i].queued = 0;
++ }
++
++ XSync(dpy, True);
++ _x_error_occurred = 0;
++
++ if (options & ASYNC)
++ present_flags |= XCB_PRESENT_OPTION_ASYNC;
++ if (options & NOCOPY) {
++ update = xcb_generate_id(c);
++ xcb_xfixes_create_region(c, update, 0, NULL);
++ present_flags |= XCB_PRESENT_OPTION_COPY;
++ }
++
++ clock_gettime(CLOCK_MONOTONIC, &start);
++ do {
++ for (i = 0; i < sz; i++) {
++ struct buffer *tmp, *b = NULL;
++retry:
++ list_for_each_entry(tmp, &pp[i].mru, link) {
++ if (tmp->fence.xid)
++ tmp->busy = !xshmfence_query(tmp->fence.addr);
++ if (!tmp->busy) {
++ b = tmp;
++ break;
++ }
++ }
++ if (options & DRI3) {
++ if (b == NULL)
++ goto retry;
++
++ xshmfence_reset(b->fence.addr);
++ pp[i].queued--;
++ completed++;
++ } else while (b == NULL) {
++ xcb_present_generic_event_t *ev;
++
++ ev = (xcb_present_generic_event_t *)
++ xcb_wait_for_special_event(c, pp[i].Q);
++ if (ev == NULL)
++ abort();
++
++ do {
++ switch (ev->evtype) {
++ case XCB_PRESENT_COMPLETE_NOTIFY:
++ completed++;
++ pp[i].queued--;
++ break;
++
++ case XCB_PRESENT_EVENT_IDLE_NOTIFY:
++ {
++ xcb_present_idle_notify_event_t *ie = (xcb_present_idle_notify_event_t *)ev;
++ assert(ie->serial < N_BACK);
++ pp[i].buffer[ie->serial].busy = 0;
++ if (b == NULL)
++ b = &pp[i].buffer[ie->serial];
++ break;
++ }
++ }
++ free(ev);
++ } while ((ev = (xcb_present_generic_event_t *)xcb_poll_for_special_event(c, pp[i].Q)));
++ }
++
++ b->busy = (options & NOCOPY) == 0;
++ xcb_present_pixmap(c, pp[i].win, b->pixmap, b->id,
++ 0, /* valid */
++ update, /* update */
++ 0, /* x_off */
++ 0, /* y_off */
++ None,
++ None, /* wait fence */
++ b->fence.xid,
++ present_flags,
++ 0, /* target msc */
++ 0, /* divisor */
++ 0, /* remainder */
++ 0, NULL);
++ list_move(&b->link, &pp[i].mru);
++ pp[i].queued++;
++ }
++ xcb_flush(c);
++ clock_gettime(CLOCK_MONOTONIC, &end);
++ } while (end.tv_sec < start.tv_sec + 10);
++
++ for (i = 0; i < sz; i++) {
++ if (options & DRI3) {
++ int depth = DefaultDepth(dpy, DefaultScreen(dpy));
++ struct buffer *b;
++ XID pixmap;
++
++ pixmap = xcb_generate_id(c);
++ xcb_create_pixmap(c, depth, pixmap, pp[i].win, 1, 1);
++ xcb_present_pixmap(c, pp[i].win, pixmap, 0xdeadbeef,
++ 0, /* valid */
++ None, /* update */
++ 0, /* x_off */
++ 0, /* y_off */
++ None,
++ None, /* wait fence */
++ None,
++ 0,
++ 0, /* target msc */
++ 0, /* divisor */
++ 0, /* remainder */
++ 0, NULL);
++ xcb_flush(c);
++
++ list_for_each_entry(b, &pp[i].mru, link)
++ xshmfence_await(b->fence.addr);
++
++ xcb_free_pixmap(c, pixmap);
++ completed += pp[i].queued;
++ } else while (pp[i].queued) {
++ xcb_present_generic_event_t *ev;
++
++ ev = (xcb_present_generic_event_t *)
++ xcb_wait_for_special_event(c, pp[i].Q);
++ if (ev == NULL)
++ abort();
++
++ do {
++ switch (ev->evtype) {
++ case XCB_PRESENT_COMPLETE_NOTIFY:
++ completed++;
++ pp[i].queued--;
++ break;
++
++ case XCB_PRESENT_EVENT_IDLE_NOTIFY:
++ break;
++ }
++ free(ev);
++ } while ((ev = (xcb_present_generic_event_t *)xcb_poll_for_special_event(c, pp[i].Q)));
++ }
++ }
++ clock_gettime(CLOCK_MONOTONIC, &end);
++
++ if (update)
++ xcb_xfixes_destroy_region(c, update);
++
++ for (i = 0; i < sz; i++) {
++ for (n = 0; n < N_BACK; n++) {
++ if (pp[i].buffer[n].fence.xid)
++ dri3_fence_free(dpy, &pp[i].buffer[n].fence);
++ if (pp[i].buffer[n].fd != -1)
++ close(pp[i].buffer[n].fd);
++ xcb_free_pixmap(c, pp[i].buffer[n].pixmap);
++ }
++
++ if (pp[i].Q) {
++ xcb_discard_reply(c, xcb_present_select_input_checked(c, pp[i].eid, pp[i].win, 0).sequence);
++ XSync(dpy, True);
++ xcb_unregister_for_special_event(c, pp[i].Q);
++ }
++
++ XDestroyWindow(dpy, pp[i].win);
++ }
++ free(pp);
++
++ test_name[0] = '\0';
++ if (options) {
++ snprintf(test_name, sizeof(test_name), "(%s%s%s )",
++ options & NOCOPY ? " no-copy" : "",
++ options & DRI3 ? " dri3" : "",
++ options & ASYNC ? " async" : "");
++ }
++ printf("%s%s: Completed %d presents in %.1fs, %.3fus each (%.1f FPS)\n",
++ __func__, test_name,
++ completed, elapsed(&start, &end) / 1000000,
++ elapsed(&start, &end) / completed,
++ completed / (elapsed(&start, &end) / 1000000));
++}
++
++static int isqrt(int x)
++{
++ int i;
++
++ for (i = 2; i*i < x; i++)
++ ;
++ return i;
++}
++
++struct sibling {
++ pthread_t thread;
++ Display *dpy;
++ int x, y;
++ int width, height;
++ unsigned options;
++};
++
++static void *sibling(void *arg)
++{
++ struct sibling *s = arg;
++ XSetWindowAttributes attr = { .override_redirect = 1 };
++ Window win = XCreateWindow(s->dpy, DefaultRootWindow(s->dpy),
++ s->x, s->y, s->width, s->height, 0,
++ DefaultDepth(s->dpy, DefaultScreen(s->dpy)),
++ InputOutput,
++ DefaultVisual(s->dpy, DefaultScreen(s->dpy)),
++ CWOverrideRedirect, &attr);
++ XMapWindow(s->dpy, win);
++ run(s->dpy, win, "sibling", s->options);
++ return NULL;
++}
++
++static void siblings(Display *dpy,
++ int max_width, int max_height, int ncpus, unsigned options)
++{
++ int sq_ncpus = isqrt(ncpus);
++ int width = max_width / sq_ncpus;
++ int height = max_height/ sq_ncpus;
++ struct sibling s[ncpus];
++ int child;
++
++ if (ncpus <= 1)
++ return;
++
++ for (child = 0; child < ncpus; child++) {
++ s[child].dpy = dpy;
++ s[child].x = (child % sq_ncpus) * width;
++ s[child].y = (child / sq_ncpus) * height;
++ s[child].width = width;
++ s[child].height = height;
++ s[child].options = options;
++ pthread_create(&s[child].thread, NULL, sibling, &s[child]);
++ }
++
++ for (child = 0; child < ncpus; child++)
++ pthread_join(s[child].thread, NULL);
++}
++
++static void cousins(int max_width, int max_height, int ncpus, unsigned options)
++{
++ int sq_ncpus = isqrt(ncpus);
++ int width = max_width / sq_ncpus;
++ int height = max_height/ sq_ncpus;
++ int child;
++
++ if (ncpus <= 1)
++ return;
++
++ for (child = 0; child < ncpus; child++) {
++ for (; fork() == 0; exit(0)) {
++ int x = (child % sq_ncpus) * width;
++ int y = (child / sq_ncpus) * height;
++ XSetWindowAttributes attr = { .override_redirect = 1 };
++ Display *dpy = XOpenDisplay(NULL);
++ Window win = XCreateWindow(dpy, DefaultRootWindow(dpy),
++ x, y, width, height, 0,
++ DefaultDepth(dpy, DefaultScreen(dpy)),
++ InputOutput,
++ DefaultVisual(dpy, DefaultScreen(dpy)),
++ CWOverrideRedirect, &attr);
++ XMapWindow(dpy, win);
++ run(dpy, win, "cousin", options);
++ }
++ }
++
++ while (child) {
++ int status = -1;
++ pid_t pid = wait(&status);
++ if (pid == -1)
++ continue;
++ child--;
++ }
++}
++
++static int has_present(Display *dpy)
++{
++ xcb_connection_t *c = XGetXCBConnection(dpy);
++ xcb_generic_error_t *error = NULL;
++ void *reply;
++
++ reply = xcb_present_query_version_reply(c,
++ xcb_present_query_version(c,
++ XCB_PRESENT_MAJOR_VERSION,
++ XCB_PRESENT_MINOR_VERSION),
++ &error);
++
++ free(reply);
++ free(error);
++ if (reply == NULL) {
++ fprintf(stderr, "Present not supported on %s\n", DisplayString(dpy));
++ return 0;
++ }
++
++ return 1;
++}
++
++static int has_composite(Display *dpy)
++{
++ int event, error;
++ int major, minor;
++
++ if (!XDamageQueryExtension (dpy, &event, &error))
++ return 0;
++
++ if (!XCompositeQueryExtension(dpy, &event, &error))
++ return 0;
++
++ XCompositeQueryVersion(dpy, &major, &minor);
++
++ return major > 0 || minor >= 4;
++}
++
++static int dri3_query_version(Display *dpy, int *major, int *minor)
++{
++ xcb_connection_t *c = XGetXCBConnection(dpy);
++ xcb_dri3_query_version_reply_t *reply;
++ xcb_generic_error_t *error;
++
++ *major = *minor = -1;
++
++ reply = xcb_dri3_query_version_reply(c,
++ xcb_dri3_query_version(c,
++ XCB_DRI3_MAJOR_VERSION,
++ XCB_DRI3_MINOR_VERSION),
++ &error);
++ free(error);
++ if (reply == NULL)
++ return -1;
++
++ *major = reply->major_version;
++ *minor = reply->minor_version;
++ free(reply);
++
++ return 0;
++}
++
++static int has_dri3(Display *dpy)
++{
++ const xcb_query_extension_reply_t *ext;
++ int major, minor;
++
++ ext = xcb_get_extension_data(XGetXCBConnection(dpy), &xcb_dri3_id);
++ if (ext == NULL || !ext->present)
++ return 0;
++
++ if (dri3_query_version(dpy, &major, &minor) < 0)
++ return 0;
++
++ return major >= 0;
++}
++
++static int has_xfixes(Display *dpy)
++{
++ xcb_connection_t *c = XGetXCBConnection(dpy);
++ const xcb_query_extension_reply_t *ext;
++ void *reply;
++
++ ext = xcb_get_extension_data(c, &xcb_xfixes_id);
++ if (ext == NULL || !ext->present)
++ return 0;
++
++ reply = xcb_xfixes_query_version_reply(c,
++ xcb_xfixes_query_version(c,
++ XCB_XFIXES_MAJOR_VERSION,
++ XCB_XFIXES_MINOR_VERSION),
++ NULL);
++ free(reply);
++
++ return reply != NULL;
++}
++
++static inline XRRScreenResources *_XRRGetScreenResourcesCurrent(Display *dpy, Window window)
++{
++ XRRScreenResources *res;
++
++ res = XRRGetScreenResourcesCurrent(dpy, window);
++ if (res == NULL)
++ res = XRRGetScreenResources(dpy, window);
++
++ return res;
++}
++
++static XRRModeInfo *lookup_mode(XRRScreenResources *res, int id)
++{
++ int i;
++
++ for (i = 0; i < res->nmode; i++) {
++ if (res->modes[i].id == id)
++ return &res->modes[i];
++ }
++
++ return NULL;
++}
++
++static void fullscreen(Display *dpy, Window win)
++{
++ Atom atom = XInternAtom(dpy, "_NET_WM_STATE_FULLSCREEN", False);
++ XChangeProperty(dpy, win,
++ XInternAtom(dpy, "_NET_WM_STATE", False),
++ XA_ATOM, 32, PropModeReplace,
++ (unsigned char *)&atom, 1);
++}
++
++static void loop(Display *dpy, XRRScreenResources *res, unsigned options)
++{
++ Window root = DefaultRootWindow(dpy);
++ Window win;
++ XSetWindowAttributes attr;
++ int i, j;
++
++ attr.override_redirect = 1;
++
++ run(dpy, root, "off", options);
++ XSync(dpy, True);
++
++ for (i = 0; i < res->noutput; i++) {
++ XRROutputInfo *output;
++ XRRModeInfo *mode;
++
++ output = XRRGetOutputInfo(dpy, res, res->outputs[i]);
++ if (output == NULL)
++ continue;
++
++ mode = NULL;
++ if (res->nmode)
++ mode = lookup_mode(res, output->modes[0]);
++
++ for (j = 0; mode && j < 2*output->ncrtc; j++) {
++ int c = j;
++ if (c >= output->ncrtc)
++ c = 2*output->ncrtc - j - 1;
++
++ printf("[%d, %d] -- OUTPUT:%ld, CRTC:%ld: %dx%d\n",
++ i, c, (long)res->outputs[i], (long)output->crtcs[c],
++ mode->width, mode->height);
++ XRRSetCrtcConfig(dpy, res, output->crtcs[c], CurrentTime,
++ 0, 0, output->modes[0], RR_Rotate_0, &res->outputs[i], 1);
++
++ run(dpy, root, "root", options);
++ XSync(dpy, True);
++
++ win = XCreateWindow(dpy, root,
++ 0, 0, mode->width, mode->height, 0,
++ DefaultDepth(dpy, DefaultScreen(dpy)),
++ InputOutput,
++ DefaultVisual(dpy, DefaultScreen(dpy)),
++ CWOverrideRedirect, &attr);
++ fullscreen(dpy, win);
++ XMapWindow(dpy, win);
++ run(dpy, win, "fullscreen", options);
++ XDestroyWindow(dpy, win);
++ XSync(dpy, True);
++
++ win = XCreateWindow(dpy, root,
++ 0, 0, mode->width, mode->height, 0,
++ DefaultDepth(dpy, DefaultScreen(dpy)),
++ InputOutput,
++ DefaultVisual(dpy, DefaultScreen(dpy)),
++ CWOverrideRedirect, &attr);
++ XMapWindow(dpy, win);
++ run(dpy, win, "windowed", options);
++ XDestroyWindow(dpy, win);
++ XSync(dpy, True);
++
++ if (has_composite(dpy)) {
++ Damage damage;
++
++ _x_error_occurred = 0;
++ win = XCreateWindow(dpy, root,
++ 0, 0, mode->width, mode->height, 0,
++ DefaultDepth(dpy, DefaultScreen(dpy)),
++ InputOutput,
++ DefaultVisual(dpy, DefaultScreen(dpy)),
++ CWOverrideRedirect, &attr);
++ XCompositeRedirectWindow(dpy, win, CompositeRedirectManual);
++ damage = XDamageCreate(dpy, win, XDamageReportNonEmpty);
++ XMapWindow(dpy, win);
++ XSync(dpy, True);
++ if (!_x_error_occurred)
++ run(dpy, win, "composited", options);
++ XDamageDestroy(dpy, damage);
++ XDestroyWindow(dpy, win);
++ XSync(dpy, True);
++ }
++
++ win = XCreateWindow(dpy, root,
++ 0, 0, mode->width/2, mode->height/2, 0,
++ DefaultDepth(dpy, DefaultScreen(dpy)),
++ InputOutput,
++ DefaultVisual(dpy, DefaultScreen(dpy)),
++ CWOverrideRedirect, &attr);
++ XMapWindow(dpy, win);
++ run(dpy, win, "half", options);
++ XDestroyWindow(dpy, win);
++ XSync(dpy, True);
++
++ perpixel(dpy, mode->width, mode->height, options);
++
++ siblings(dpy, mode->width, mode->height,
++ sysconf(_SC_NPROCESSORS_ONLN),
++ options);
++
++ cousins(mode->width, mode->height,
++ sysconf(_SC_NPROCESSORS_ONLN),
++ options);
++
++ XRRSetCrtcConfig(dpy, res, output->crtcs[c], CurrentTime,
++ 0, 0, None, RR_Rotate_0, NULL, 0);
++ }
++
++ XRRFreeOutputInfo(output);
++ }
++
++}
++
++int main(void)
++{
++ Display *dpy;
++ XRRScreenResources *res;
++ XRRCrtcInfo **original_crtc;
++ int i;
++
++ XInitThreads();
++
++ dpy = XOpenDisplay(NULL);
++ if (dpy == NULL)
++ return 77;
++
++ if (!has_present(dpy))
++ return 77;
++
++ if (DPMSQueryExtension(dpy, &i, &i))
++ DPMSDisable(dpy);
++
++ signal(SIGALRM, SIG_IGN);
++ XSetErrorHandler(_check_error_handler);
++
++ res = NULL;
++ if (XRRQueryVersion(dpy, &i, &i))
++ res = _XRRGetScreenResourcesCurrent(dpy, DefaultRootWindow(dpy));
++ if (res == NULL)
++ return 77;
++
++ original_crtc = malloc(sizeof(XRRCrtcInfo *)*res->ncrtc);
++ for (i = 0; i < res->ncrtc; i++)
++ original_crtc[i] = XRRGetCrtcInfo(dpy, res, res->crtcs[i]);
++
++ printf("noutput=%d, ncrtc=%d\n", res->noutput, res->ncrtc);
++ for (i = 0; i < res->ncrtc; i++)
++ XRRSetCrtcConfig(dpy, res, res->crtcs[i], CurrentTime,
++ 0, 0, None, RR_Rotate_0, NULL, 0);
++
++ loop(dpy, res, 0);
++ loop(dpy, res, ASYNC);
++ if (has_xfixes(dpy))
++ loop(dpy, res, NOCOPY);
++ if (has_dri3(dpy)) {
++ loop(dpy, res, DRI3);
++ loop(dpy, res, DRI3 | ASYNC);
++ }
++
++ for (i = 0; i < res->ncrtc; i++)
++ XRRSetCrtcConfig(dpy, res, res->crtcs[i], CurrentTime,
++ original_crtc[i]->x,
++ original_crtc[i]->y,
++ original_crtc[i]->mode,
++ original_crtc[i]->rotation,
++ original_crtc[i]->outputs,
++ original_crtc[i]->noutput);
++
++ if (DPMSQueryExtension(dpy, &i, &i))
++ DPMSEnable(dpy);
++ return 0;
++}
+diff --git a/test/present-test.c b/test/present-test.c
+index 6b562eb0..5a12a24f 100644
+--- a/test/present-test.c
++++ b/test/present-test.c
+@@ -31,7 +31,9 @@
+ #include <X11/xshmfence.h>
+ #include <X11/Xutil.h>
+ #include <X11/Xlibint.h>
++#include <X11/extensions/dpms.h>
+ #include <X11/extensions/randr.h>
++#include <X11/extensions/Xcomposite.h>
+ #include <X11/extensions/Xrandr.h>
+ #include <X11/extensions/Xrender.h>
+ #include <X11/extensions/XShm.h>
+@@ -44,6 +46,8 @@
+ #endif
+ #include <xcb/xcb.h>
+ #include <xcb/present.h>
++#include <xcb/xfixes.h>
++#include <xcb/dri3.h>
+ #include <xf86drm.h>
+ #include <i915_drm.h>
+
+@@ -134,12 +138,14 @@ static void *setup_msc(Display *dpy, Window win)
+ return q;
+ }
+
+-static uint64_t check_msc(Display *dpy, Window win, void *q, uint64_t last_msc)
++static uint64_t check_msc(Display *dpy, Window win, void *q, uint64_t last_msc, uint64_t *ust)
+ {
+ xcb_connection_t *c = XGetXCBConnection(dpy);
++ static uint32_t serial = 1;
+ uint64_t msc = 0;
++ int complete = 0;
+
+- xcb_present_notify_msc(c, win, 0, 0, 0, 0);
++ xcb_present_notify_msc(c, win, serial ^ 0xcc00ffee, 0, 0, 0);
+ xcb_flush(c);
+
+ do {
+@@ -151,82 +157,1268 @@ static uint64_t check_msc(Display *dpy, Window win, void *q, uint64_t last_msc)
+ break;
+
+ ce = (xcb_present_complete_notify_event_t *)ev;
+- if (ce->kind != XCB_PRESENT_COMPLETE_KIND_PIXMAP)
++ if (ce->kind == XCB_PRESENT_COMPLETE_KIND_NOTIFY_MSC &&
++ ce->serial == (serial ^ 0xcc00ffee)) {
++ msc = ce->msc;
++ if (ust)
++ *ust = ce->ust;
++ complete = 1;
++ }
++ free(ev);
++ } while (!complete);
++
++ if ((int64_t)(msc - last_msc) < 0) {
++ printf("Invalid MSC: was %llu, now %llu\n",
++ (long long)last_msc, (long long)msc);
++ }
++
++ if (++serial == 0)
++ serial = 1;
++
++ return msc;
++}
++
++static uint64_t wait_vblank(Display *dpy, Window win, void *q)
++{
++ xcb_connection_t *c = XGetXCBConnection(dpy);
++ static uint32_t serial = 1;
++ uint64_t msc = 0;
++ int complete = 0;
++
++ xcb_present_notify_msc(c, win, serial ^ 0xdeadbeef, 0, 1, 0);
++ xcb_flush(c);
++
++ do {
++ xcb_present_complete_notify_event_t *ce;
++ xcb_generic_event_t *ev;
++
++ ev = xcb_wait_for_special_event(c, q);
++ if (ev == NULL)
++ break;
++
++ ce = (xcb_present_complete_notify_event_t *)ev;
++ if (ce->kind == XCB_PRESENT_COMPLETE_KIND_NOTIFY_MSC &&
++ ce->serial == (serial ^ 0xdeadbeef)) {
+ msc = ce->msc;
++ complete = 1;
++ }
++ free(ev);
++ } while (!complete);
++
++ if (++serial == 0)
++ serial = 1;
++
++ return msc;
++}
++
++static uint64_t msc_interval(Display *dpy, Window win, void *q)
++{
++ xcb_connection_t *c = XGetXCBConnection(dpy);
++ uint64_t msc, ust;
++ int complete = 0;
++
++ msc = check_msc(dpy, win, q, 0, NULL);
++
++ xcb_present_notify_msc(c, win, 0xc0ffee00, msc, 0, 0);
++ xcb_present_notify_msc(c, win, 0xc0ffee01, msc + 10, 0, 0);
++ xcb_flush(c);
++
++ ust = msc = 0;
++ do {
++ xcb_present_complete_notify_event_t *ce;
++ xcb_generic_event_t *ev;
++
++ ev = xcb_wait_for_special_event(c, q);
++ if (ev == NULL)
++ break;
++
++ ce = (xcb_present_complete_notify_event_t *)ev;
++ if (ce->kind == XCB_PRESENT_COMPLETE_KIND_NOTIFY_MSC &&
++ ce->serial == 0xc0ffee00) {
++ msc -= ce->msc;
++ ust -= ce->ust;
++ complete++;
++ }
++ if (ce->kind == XCB_PRESENT_COMPLETE_KIND_NOTIFY_MSC &&
++ ce->serial == 0xc0ffee01) {
++ msc += ce->msc;
++ ust += ce->ust;
++ complete++;
++ }
++ free(ev);
++ } while (complete != 2);
++
++ printf("10 frame interval: msc=%lld, ust=%lld\n",
++ (long long)msc, (long long)ust);
++ XSync(dpy, True);
++ if (msc == 0)
++ return 0;
++
++ return (ust + msc/2) / msc;
++}
++
++static void teardown_msc(Display *dpy, void *q)
++{
++ xcb_unregister_for_special_event(XGetXCBConnection(dpy), q);
++}
++
++static int test_whole(Display *dpy, Window win, const char *phase)
++{
++ xcb_connection_t *c = XGetXCBConnection(dpy);
++ Pixmap pixmap;
++ struct dri3_fence fence;
++ Window root;
++ unsigned int width, height;
++ unsigned border, depth;
++ int x, y, ret = 1;
++
++ XGetGeometry(dpy, win,
++ &root, &x, &y, &width, &height, &border, &depth);
++
++ if (dri3_create_fence(dpy, win, &fence))
++ return 0;
++
++ printf("%s: Testing simple flip: %dx%d\n", phase, width, height);
++ _x_error_occurred = 0;
++
++ xshmfence_reset(fence.addr);
++
++ pixmap = XCreatePixmap(dpy, win, width, height, depth);
++ xcb_present_pixmap(c, win, pixmap, 0,
++ 0, /* valid */
++ 0, /* update */
++ 0, /* x_off */
++ 0, /* y_off */
++ None,
++ None, /* wait fence */
++ fence.xid,
++ XCB_PRESENT_OPTION_NONE,
++ 0, /* target msc */
++ 0, /* divisor */
++ 0, /* remainder */
++ 0, NULL);
++ XFreePixmap(dpy, pixmap);
++
++ pixmap = XCreatePixmap(dpy, win, width, height, depth);
++ xcb_present_pixmap(c, win, pixmap, 0,
++ 0, /* valid */
++ 0, /* update */
++ 0, /* x_off */
++ 0, /* y_off */
++ None,
++ None, /* wait fence */
++ None, /* sync fence */
++ XCB_PRESENT_OPTION_NONE,
++ 0, /* target msc */
++ 0, /* divisor */
++ 0, /* remainder */
++ 0, NULL);
++ XFreePixmap(dpy, pixmap);
++ XFlush(dpy);
++
++ ret = !!xshmfence_await(fence.addr);
++ dri3_fence_free(dpy, &fence);
++
++ XSync(dpy, True);
++ ret += !!_x_error_occurred;
++
++ return ret;
++}
++
++static uint64_t flush_flips(Display *dpy, Window win, Pixmap pixmap, void *Q, uint64_t *ust)
++{
++ xcb_connection_t *c = XGetXCBConnection(dpy);
++ uint64_t msc;
++ int complete;
++
++ msc = check_msc(dpy, win, Q, 0, NULL);
++ xcb_present_pixmap(c, win, pixmap,
++ 0xdeadbeef, /* serial */
++ 0, /* valid */
++ 0, /* update */
++ 0, /* x_off */
++ 0, /* y_off */
++ None,
++ None, /* wait fence */
++ None,
++ XCB_PRESENT_OPTION_NONE,
++ msc + 60, /* target msc */
++ 0, /* divisor */
++ 0, /* remainder */
++ 0, NULL);
++ xcb_flush(c);
++ complete = 0;
++ do {
++ xcb_present_complete_notify_event_t *ce;
++ xcb_generic_event_t *ev;
++
++ ev = xcb_wait_for_special_event(c, Q);
++ if (ev == NULL)
++ break;
++
++ ce = (xcb_present_complete_notify_event_t *)ev;
++ complete = (ce->kind == XCB_PRESENT_COMPLETE_KIND_PIXMAP &&
++ ce->serial == 0xdeadbeef);
++ free(ev);
++ } while (!complete);
++ XSync(dpy, True);
++
++ return check_msc(dpy, win, Q, msc, ust);
++}
++
++static int test_double(Display *dpy, Window win, const char *phase, void *Q)
++{
++#define COUNT (15*60)
++ xcb_connection_t *c = XGetXCBConnection(dpy);
++ Pixmap pixmap;
++ Window root;
++ unsigned int width, height;
++ unsigned border, depth;
++ int x, y, n, ret;
++ struct {
++ uint64_t msc, ust;
++ } frame[COUNT+1];
++ int offset = 0;
++
++ XGetGeometry(dpy, win,
++ &root, &x, &y, &width, &height, &border, &depth);
++
++ printf("%s: Testing flip double buffering: %dx%d\n", phase, width, height);
++ _x_error_occurred = 0;
++
++ pixmap = XCreatePixmap(dpy, win, width, height, depth);
++ flush_flips(dpy, win, pixmap, Q, NULL);
++ for (n = 0; n <= COUNT; n++) {
++ int complete;
++
++ xcb_present_pixmap(c, win, pixmap, n,
++ 0, /* valid */
++ 0, /* update */
++ 0, /* x_off */
++ 0, /* y_off */
++ None,
++ None, /* wait fence */
++ None,
++ XCB_PRESENT_OPTION_NONE,
++ 0, /* target msc */
++ 0, /* divisor */
++ 0, /* remainder */
++ 0, NULL);
++ xcb_flush(c);
++
++ complete = 0;
++ do {
++ xcb_present_complete_notify_event_t *ce;
++ xcb_generic_event_t *ev;
++
++ ev = xcb_wait_for_special_event(c, Q);
++ if (ev == NULL)
++ break;
++
++ ce = (xcb_present_complete_notify_event_t *)ev;
++ if (ce->kind == XCB_PRESENT_COMPLETE_KIND_PIXMAP &&
++ ce->serial == n) {
++ frame[n].msc = ce->msc;
++ frame[n].ust = ce->ust;
++ complete = 1;
++ }
++ free(ev);
++ } while (!complete);
++ }
++ XFreePixmap(dpy, pixmap);
++
++ XSync(dpy, True);
++ ret = !!_x_error_occurred;
++
++ if (frame[COUNT].msc - frame[0].msc != COUNT) {
++ printf("Expected %d frames interval, %d elapsed instead\n",
++ COUNT, (int)(frame[COUNT].msc - frame[0].msc));
++ for (n = 0; n <= COUNT; n++) {
++ if (frame[n].msc - frame[0].msc != n + offset) {
++ printf("frame[%d]: msc=%03lld, ust=%lld\n", n,
++ (long long)(frame[n].msc - frame[0].msc),
++ (long long)(frame[n].ust - frame[0].ust));
++ offset = frame[n].msc - frame[0].msc - n;
++ ret++;
++ }
++ }
++ }
++
++ return ret;
++}
++
++static int test_future(Display *dpy, Window win, const char *phase, void *Q)
++{
++ xcb_connection_t *c = XGetXCBConnection(dpy);
++ Pixmap pixmap;
++ struct dri3_fence fence;
++ Window root;
++ unsigned int width, height;
++ unsigned border, depth;
++ int x, y, ret = 0, n;
++ uint64_t msc, ust;
++ int complete, count;
++ int early = 0, late = 0;
++ int earliest = 0, latest = 0;
++ uint64_t interval;
++
++ XGetGeometry(dpy, win,
++ &root, &x, &y, &width, &height, &border, &depth);
++
++ if (dri3_create_fence(dpy, win, &fence))
++ return 0;
++
++ printf("%s: Testing flips into the future: %dx%d\n", phase, width, height);
++ _x_error_occurred = 0;
++
++ interval = msc_interval(dpy, win, Q);
++ if (interval == 0) {
++ printf("Zero delay between frames\n");
++ return 1;
++ }
++
++ pixmap = XCreatePixmap(dpy, win, width, height, depth);
++ msc = flush_flips(dpy, win, pixmap, Q, &ust);
++ for (n = 1; n <= 10; n++)
++ xcb_present_pixmap(c, win, pixmap,
++ n, /* serial */
++ 0, /* valid */
++ 0, /* update */
++ 0, /* x_off */
++ 0, /* y_off */
++ None,
++ None, /* wait fence */
++ None,
++ XCB_PRESENT_OPTION_NONE,
++ msc + 60 + n*15*60, /* target msc */
++ 0, /* divisor */
++ 0, /* remainder */
++ 0, NULL);
++ xcb_present_pixmap(c, win, pixmap,
++ 0xdeadbeef, /* serial */
++ 0, /* valid */
++ 0, /* update */
++ 0, /* x_off */
++ 0, /* y_off */
++ None,
++ None, /* wait fence */
++ None,
++ XCB_PRESENT_OPTION_NONE,
++ msc + 60 + n*15*60, /* target msc */
++ 0, /* divisor */
++ 0, /* remainder */
++ 0, NULL);
++ xcb_flush(c);
++
++ complete = 0;
++ count = 0;
++ do {
++ xcb_present_complete_notify_event_t *ce;
++ xcb_generic_event_t *ev;
++
++ ev = xcb_wait_for_special_event(c, Q);
++ if (ev == NULL)
++ break;
++
++ ce = (xcb_present_complete_notify_event_t *)ev;
++ assert(ce->kind == XCB_PRESENT_COMPLETE_KIND_PIXMAP);
++
++ if (ce->serial == 0xdeadbeef) {
++ int64_t time;
++
++ time = ce->ust - (ust + (60 + 15*60*n) * interval);
++ if (time < -(int64_t)interval) {
++ fprintf(stderr,
++ "\tflips completed too early by %lldms\n",
++ (long long)(-time / 1000));
++ } else if (time > (int64_t)interval) {
++ fprintf(stderr,
++ "\tflips completed too late by %lldms\n",
++ (long long)(time / 1000));
++ }
++ complete = 1;
++ } else {
++ int diff = (int64_t)(ce->msc - (15*60*ce->serial + msc + 60));
++ if (diff < 0) {
++ if (-diff > earliest) {
++ fprintf(stderr, "\tframe %d displayed early by %d frames\n", ce->serial, -diff);
++ earliest = -diff;
++ }
++ early++;
++ ret++;
++ } else if (diff > 0) {
++ if (diff > latest) {
++ fprintf(stderr, "\tframe %d displayed late by %d frames\n", ce->serial, diff);
++ latest = diff;
++ }
++ late++;
++ ret++;
++ }
++ count++;
++ }
++ free(ev);
++ } while (!complete);
++
++ if (early)
++ printf("\t%d frames shown too early (worst %d)!\n", early, earliest);
++ if (late)
++ printf("\t%d frames shown too late (worst %d)!\n", late, latest);
++
++ if (count != 10) {
++ fprintf(stderr, "Sentinel frame received too early! %d frames outstanding\n", 10 - count);
++ ret++;
++
++ do {
++ xcb_present_complete_notify_event_t *ce;
++ xcb_generic_event_t *ev;
++
++ ev = xcb_wait_for_special_event(c, Q);
++ if (ev == NULL)
++ break;
++
++ ce = (xcb_present_complete_notify_event_t *)ev;
++ assert(ce->kind == XCB_PRESENT_COMPLETE_KIND_PIXMAP);
++ free(ev);
++ } while (++count != 10);
++ }
++
++ ret += !!_x_error_occurred;
++
++ return ret;
++}
++
++static int test_exhaustion(Display *dpy, Window win, const char *phase, void *Q)
++{
++#define N_VBLANKS 256 /* kernel event queue length: 128 vblanks */
++ xcb_connection_t *c = XGetXCBConnection(dpy);
++ Pixmap pixmap;
++ struct dri3_fence fence[2];
++ Window root;
++ xcb_xfixes_region_t region;
++ unsigned int width, height;
++ unsigned border, depth;
++ int x, y, ret = 0, n;
++ uint64_t target, final;
++
++ XGetGeometry(dpy, win,
++ &root, &x, &y, &width, &height, &border, &depth);
++
++ if (dri3_create_fence(dpy, win, &fence[0]) ||
++ dri3_create_fence(dpy, win, &fence[1]))
++ return 0;
++
++ printf("%s: Testing flips with long vblank queues: %dx%d\n", phase, width, height);
++ _x_error_occurred = 0;
++
++ region = xcb_generate_id(c);
++ xcb_xfixes_create_region(c, region, 0, NULL);
++
++ pixmap = XCreatePixmap(dpy, win, width, height, depth);
++ xshmfence_reset(fence[0].addr);
++ xshmfence_reset(fence[1].addr);
++ target = check_msc(dpy, win, Q, 0, NULL);
++ for (n = N_VBLANKS; n--; )
++ xcb_present_pixmap(c, win, pixmap, 0,
++ 0, /* valid */
++ region, /* update */
++ 0, /* x_off */
++ 0, /* y_off */
++ None,
++ None, /* wait fence */
++ None,
++ XCB_PRESENT_OPTION_NONE,
++ target + N_VBLANKS, /* target msc */
++ 1, /* divisor */
++ 0, /* remainder */
++ 0, NULL);
++ xcb_present_pixmap(c, win, pixmap, 0,
++ region, /* valid */
++ region, /* update */
++ 0, /* x_off */
++ 0, /* y_off */
++ None,
++ None, /* wait fence */
++ fence[0].xid,
++ XCB_PRESENT_OPTION_NONE,
++ target, /* target msc */
++ 0, /* divisor */
++ 0, /* remainder */
++ 0, NULL);
++ for (n = 1; n < N_VBLANKS; n++)
++ xcb_present_pixmap(c, win, pixmap, 0,
++ region, /* valid */
++ region, /* update */
++ 0, /* x_off */
++ 0, /* y_off */
++ None,
++ None, /* wait fence */
++ None,
++ XCB_PRESENT_OPTION_NONE,
++ target + n, /* target msc */
++ 0, /* divisor */
++ 0, /* remainder */
++ 0, NULL);
++ xcb_present_pixmap(c, win, pixmap, 0,
++ region, /* valid */
++ region, /* update */
++ 0, /* x_off */
++ 0, /* y_off */
++ None,
++ None, /* wait fence */
++ fence[1].xid,
++ XCB_PRESENT_OPTION_NONE,
++ target + N_VBLANKS, /* target msc */
++ 0, /* divisor */
++ 0, /* remainder */
++ 0, NULL);
++ xcb_flush(c);
++
++ ret += !!xshmfence_await(fence[0].addr);
++ final = check_msc(dpy, win, Q, 0, NULL);
++ if (final < target) {
++ printf("\tFirst flip too early, MSC was %llu, expected %llu\n",
++ (long long)final, (long long)target);
++ ret++;
++ } else if (final > target + 1) {
++ printf("\tFirst flip too late, MSC was %llu, expected %llu\n",
++ (long long)final, (long long)target);
++ ret++;
++ }
++
++ ret += !!xshmfence_await(fence[1].addr);
++ final = check_msc(dpy, win, Q, 0, NULL);
++ if (final < target + N_VBLANKS) {
++ printf("\tLast flip too early, MSC was %llu, expected %llu\n",
++ (long long)final, (long long)(target + N_VBLANKS));
++ ret++;
++ } else if (final > target + N_VBLANKS + 1) {
++ printf("\tLast flip too late, MSC was %llu, expected %llu\n",
++ (long long)final, (long long)(target + N_VBLANKS));
++ ret++;
++ }
++
++ flush_flips(dpy, win, pixmap, Q, NULL);
++
++ XFreePixmap(dpy, pixmap);
++ xcb_xfixes_destroy_region(c, region);
++ dri3_fence_free(dpy, &fence[1]);
++ dri3_fence_free(dpy, &fence[0]);
++
++ XSync(dpy, True);
++ ret += !!_x_error_occurred;
++
++ return ret;
++#undef N_VBLANKS
++}
++
++static int test_accuracy(Display *dpy, Window win, const char *phase, void *Q)
++{
++#define N_VBLANKS (60 * 120) /* ~2 minutes */
++ xcb_connection_t *c = XGetXCBConnection(dpy);
++ Pixmap pixmap;
++ Window root;
++ unsigned int width, height;
++ unsigned border, depth;
++ int x, y, ret = 0, n;
++ uint64_t target;
++ int early = 0, late = 0;
++ int earliest = 0, latest = 0;
++ int complete, count;
++
++ XGetGeometry(dpy, win,
++ &root, &x, &y, &width, &height, &border, &depth);
++
++ printf("%s: Testing flip accuracy: %dx%d\n", phase, width, height);
++ _x_error_occurred = 0;
++
++ pixmap = XCreatePixmap(dpy, win, width, height, depth);
++ target = flush_flips(dpy, win, pixmap, Q, NULL);
++ for (n = 0; n <= N_VBLANKS; n++)
++ xcb_present_pixmap(c, win, pixmap,
++ n, /* serial */
++ 0, /* valid */
++ 0, /* update */
++ 0, /* x_off */
++ 0, /* y_off */
++ None,
++ None, /* wait fence */
++ None,
++ XCB_PRESENT_OPTION_NONE,
++ target + 60 + n, /* target msc */
++ 0, /* divisor */
++ 0, /* remainder */
++ 0, NULL);
++ xcb_present_pixmap(c, win, pixmap,
++ 0xdeadbeef, /* serial */
++ 0, /* valid */
++ 0, /* update */
++ 0, /* x_off */
++ 0, /* y_off */
++ None,
++ None, /* wait fence */
++ None,
++ XCB_PRESENT_OPTION_NONE,
++ target + 60 + n, /* target msc */
++ 0, /* divisor */
++ 0, /* remainder */
++ 0, NULL);
++ xcb_flush(c);
++
++ complete = 0;
++ count = 0;
++ do {
++ xcb_present_complete_notify_event_t *ce;
++ xcb_generic_event_t *ev;
++
++ ev = xcb_wait_for_special_event(c, Q);
++ if (ev == NULL)
++ break;
++
++ ce = (xcb_present_complete_notify_event_t *)ev;
++ assert(ce->kind == XCB_PRESENT_COMPLETE_KIND_PIXMAP);
++
++ if (ce->serial != 0xdeadbeef) {
++ int diff = (int64_t)(ce->msc - (target + ce->serial + 60));
++ if (diff < 0) {
++ if (-diff > earliest) {
++ fprintf(stderr, "\tframe %d displayed early by %d frames\n", ce->serial, -diff);
++ earliest = -diff;
++ }
++ early++;
++ ret++;
++ } else if (diff > 0) {
++ if (diff > latest) {
++ fprintf(stderr, "\tframe %d displayed late by %d frames\n", ce->serial, diff);
++ latest = diff;
++ }
++ late++;
++ ret++;
++ }
++ count++;
++ } else
++ complete = 1;
+ free(ev);
+- } while (msc == 0);
++ } while (!complete);
++
++ if (early)
++ printf("\t%d frames shown too early (worst %d)!\n", early, earliest);
++ if (late)
++ printf("\t%d frames shown too late (worst %d)!\n", late, latest);
++
++ if (count != N_VBLANKS+1) {
++ fprintf(stderr, "Sentinel frame received too early! %d frames outstanding\n", N_VBLANKS+1 - count);
++ ret++;
++ do {
++ xcb_present_complete_notify_event_t *ce;
++ xcb_generic_event_t *ev;
++
++ ev = xcb_wait_for_special_event(c, Q);
++ if (ev == NULL)
++ break;
++
++ ce = (xcb_present_complete_notify_event_t *)ev;
++ assert(ce->kind == XCB_PRESENT_COMPLETE_KIND_PIXMAP);
++ free(ev);
++ } while (++count != N_VBLANKS+1);
++ }
++
++ XFreePixmap(dpy, pixmap);
++
++ XSync(dpy, True);
++ ret += !!_x_error_occurred;
++
++ return ret;
++#undef N_VBLANKS
++}
++
++static int test_modulus(Display *dpy, Window win, const char *phase, void *Q)
++{
++ xcb_connection_t *c = XGetXCBConnection(dpy);
++ Pixmap pixmap;
++ Window root;
++ unsigned int width, height;
++ unsigned border, depth;
++ xcb_xfixes_region_t region;
++ int x, y, ret = 0;
++ uint64_t target;
++ int early = 0, late = 0;
++ int earliest = 0, latest = 0;
++ int complete, expect, count;
++
++ XGetGeometry(dpy, win,
++ &root, &x, &y, &width, &height, &border, &depth);
++
++ printf("%s: Testing flip modulus: %dx%d\n", phase, width, height);
++ _x_error_occurred = 0;
++
++ region = xcb_generate_id(c);
++ xcb_xfixes_create_region(c, region, 0, NULL);
++
++ pixmap = XCreatePixmap(dpy, win, width, height, depth);
++ target = flush_flips(dpy, win, pixmap, Q, NULL);
++ expect = 0;
++ for (x = 1; x <= 7; x++) {
++ for (y = 0; y < x; y++) {
++ xcb_present_pixmap(c, win, pixmap,
++ y << 16 | x, /* serial */
++ region, /* valid */
++ region, /* update */
++ 0, /* x_off */
++ 0, /* y_off */
++ None,
++ None, /* wait fence */
++ None,
++ XCB_PRESENT_OPTION_NONE,
++ 0, /* target msc */
++ x, /* divisor */
++ y, /* remainder */
++ 0, NULL);
++ expect++;
++ }
++ }
++ xcb_present_pixmap(c, win, pixmap,
++ 0xdeadbeef, /* serial */
++ 0, /* valid */
++ 0, /* update */
++ 0, /* x_off */
++ 0, /* y_off */
++ None,
++ None, /* wait fence */
++ None,
++ XCB_PRESENT_OPTION_NONE,
++ target + 2*x, /* target msc */
++ 0, /* divisor */
++ 0, /* remainder */
++ 0, NULL);
++ xcb_flush(c);
++
++ complete = 0;
++ count = 0;
++ do {
++ xcb_present_complete_notify_event_t *ce;
++ xcb_generic_event_t *ev;
++
++ ev = xcb_wait_for_special_event(c, Q);
++ if (ev == NULL)
++ break;
++
++ ce = (xcb_present_complete_notify_event_t *)ev;
++ if (ce->kind != XCB_PRESENT_COMPLETE_KIND_PIXMAP)
++ break;
++
++ assert(ce->serial);
++ if (ce->serial != 0xdeadbeef) {
++ uint64_t msc;
++ int diff;
++
++ x = ce->serial & 0xffff;
++ y = ce->serial >> 16;
++
++ msc = target;
++ msc -= target % x;
++ msc += y;
++ if (msc <= target)
++ msc += x;
++
++ diff = (int64_t)(ce->msc - msc);
++ if (diff < 0) {
++ if (-diff > earliest) {
++ fprintf(stderr, "\tframe (%d, %d) displayed early by %d frames\n", y, x, -diff);
++ earliest = -diff;
++ }
++ early++;
++ ret++;
++ } else if (diff > 0) {
++ if (diff > latest) {
++ fprintf(stderr, "\tframe (%d, %d) displayed late by %d frames\n", y, x, diff);
++ latest = diff;
++ }
++ late++;
++ ret++;
++ }
++ count++;
++ } else
++ complete = 1;
++ free(ev);
++ } while (!complete);
++
++ if (early)
++ printf("\t%d frames shown too early (worst %d)!\n", early, earliest);
++ if (late)
++ printf("\t%d frames shown too late (worst %d)!\n", late, latest);
++
++ if (count != expect) {
++ fprintf(stderr, "Sentinel frame received too early! %d frames outstanding\n", expect - count);
++ ret++;
++ do {
++ xcb_present_complete_notify_event_t *ce;
++ xcb_generic_event_t *ev;
++
++ ev = xcb_wait_for_special_event(c, Q);
++ if (ev == NULL)
++ break;
++
++ ce = (xcb_present_complete_notify_event_t *)ev;
++ assert(ce->kind == XCB_PRESENT_COMPLETE_KIND_NOTIFY_MSC);
++ free(ev);
++ } while (++count != expect);
++ }
++
++ XFreePixmap(dpy, pixmap);
++ xcb_xfixes_destroy_region(c, region);
++
++ XSync(dpy, True);
++ ret += !!_x_error_occurred;
++
++ return ret;
++}
++
++static int test_future_msc(Display *dpy, void *Q)
++{
++ xcb_connection_t *c = XGetXCBConnection(dpy);
++ Window root = DefaultRootWindow(dpy);
++ int ret = 0, n;
++ uint64_t msc, ust;
++ int complete, count;
++ int early = 0, late = 0;
++ int earliest = 0, latest = 0;
++ uint64_t interval;
++
++ printf("Testing notifies into the future\n");
++ _x_error_occurred = 0;
++
++ interval = msc_interval(dpy, root, Q);
++ if (interval == 0) {
++ printf("Zero delay between frames\n");
++ return 1;
++ }
++ msc = check_msc(dpy, root, Q, 0, &ust);
++ printf("Initial msc=%llx, interval between frames %lldus\n",
++ (long long)msc, (long long)interval);
++
++ for (n = 1; n <= 10; n++)
++ xcb_present_notify_msc(c, root, n, msc + 60 + n*15*60, 0, 0);
++ xcb_present_notify_msc(c, root, 0xdeadbeef, msc + 60 + n*15*60, 0, 0);
++ xcb_flush(c);
++
++ complete = 0;
++ count = 0;
++ do {
++ xcb_present_complete_notify_event_t *ce;
++ xcb_generic_event_t *ev;
++
++ ev = xcb_wait_for_special_event(c, Q);
++ if (ev == NULL)
++ break;
++
++ ce = (xcb_present_complete_notify_event_t *)ev;
++ assert(ce->kind == XCB_PRESENT_COMPLETE_KIND_NOTIFY_MSC);
++
++ if (ce->serial == 0xdeadbeef) {
++ int64_t time, tolerance;
++
++ tolerance = 60 + 15*60*n/10;
++ if (tolerance < interval)
++ tolerance = interval;
++
++ time = ce->ust - (ust + (60 + 15*60*n) * interval);
++ if (time < -(int64_t)tolerance) {
++ fprintf(stderr,
++ "\tnotifies completed too early by %lldms, tolerance %lldus\n",
++ (long long)(-time / 1000), (long long)tolerance);
++ } else if (time > (int64_t)tolerance) {
++ fprintf(stderr,
++ "\tnotifies completed too late by %lldms, tolerance %lldus\n",
++ (long long)(time / 1000), (long long)tolerance);
++ }
++ complete = 1;
++ } else {
++ int diff = (int64_t)(ce->msc - (15*60*ce->serial + msc + 60));
++
++ if (ce->serial != count + 1) {
++ fprintf(stderr, "vblank received out of order! expected %d, received %d\n",
++ count + 1, (int)ce->serial);
++ ret++;
++ }
++ count++;
++
++ if (diff < 0) {
++ if (-diff > earliest) {
++ fprintf(stderr, "\tnotify %d early by %d msc\n", ce->serial, -diff);
++ earliest = -diff;
++ }
++ early++;
++ ret++;
++ } else if (diff > 0) {
++ if (diff > latest) {
++ fprintf(stderr, "\tnotify %d late by %d msc\n", ce->serial, diff);
++ latest = diff;
++ }
++ late++;
++ ret++;
++ }
++ }
++ free(ev);
++ } while (!complete);
++
++ if (early)
++ printf("\t%d notifies too early (worst %d)!\n", early, earliest);
++ if (late)
++ printf("\t%d notifies too late (worst %d)!\n", late, latest);
++
++ if (count != 10) {
++ fprintf(stderr, "Sentinel vblank received too early! %d waits outstanding\n", 10 - count);
++ ret++;
++ do {
++ xcb_present_complete_notify_event_t *ce;
++ xcb_generic_event_t *ev;
++
++ ev = xcb_wait_for_special_event(c, Q);
++ if (ev == NULL)
++ break;
++
++ ce = (xcb_present_complete_notify_event_t *)ev;
++ assert(ce->kind == XCB_PRESENT_COMPLETE_KIND_NOTIFY_MSC);
++ free(ev);
++ } while (++count != 10);
++ }
++
++ XSync(dpy, True);
++ ret += !!_x_error_occurred;
++
++ return ret;
++}
++
++static int test_wrap_msc(Display *dpy)
++{
++ xcb_connection_t *c = XGetXCBConnection(dpy);
++ Window root, win;
++ int x, y;
++ unsigned int width, height;
++ unsigned border, depth;
++ XSetWindowAttributes attr;
++ int ret = 0, n;
++ uint64_t msc, ust;
++ int complete;
++ uint64_t interval;
++ void *Q;
++
++ XGetGeometry(dpy, DefaultRootWindow(dpy),
++ &root, &x, &y, &width, &height, &border, &depth);
++
++ attr.override_redirect = 1;
++ win = XCreateWindow(dpy, root,
++ 0, 0, width, height, 0, depth,
++ InputOutput, DefaultVisual(dpy, DefaultScreen(dpy)),
++ CWOverrideRedirect, &attr);
++ XMapWindow(dpy, win);
++ XSync(dpy, True);
++ if (_x_error_occurred)
++ return 1;
+
+- if (msc < last_msc) {
+- printf("Invalid MSC: was %llu, now %llu\n",
+- (long long)last_msc, (long long)msc);
++ printf("Testing wraparound notifies\n");
++ _x_error_occurred = 0;
++
++ Q = setup_msc(dpy, win);
++ interval = msc_interval(dpy, win, Q);
++ if (interval == 0) {
++ printf("Zero delay between frames\n");
++ return 1;
+ }
++ msc = check_msc(dpy, win, Q, 0, &ust);
++ printf("Initial msc=%llx, interval between frames %lldus\n",
++ (long long)msc, (long long)interval);
++
++ for (n = 1; n <= 10; n++)
++ xcb_present_notify_msc(c, win, n,
++ msc + ((long long)n<<32) + n,
++ 0, 0);
++ for (n = 1; n <= 10; n++)
++ xcb_present_notify_msc(c, win, -n,
++ 0, (long long)n << 32, 0);
++ xcb_present_notify_msc(c, win, 0xdeadbeef, msc + 60*10, 0, 0);
++ xcb_flush(c);
+
+- return msc;
++ complete = 0;
++ do {
++ xcb_present_complete_notify_event_t *ce;
++ xcb_generic_event_t *ev;
++
++ ev = xcb_wait_for_special_event(c, Q);
++ if (ev == NULL)
++ break;
++
++ ce = (xcb_present_complete_notify_event_t *)ev;
++ assert(ce->kind == XCB_PRESENT_COMPLETE_KIND_NOTIFY_MSC);
++
++ if (ce->serial == 0xdeadbeef) {
++ complete = 1;
++ } else {
++ fprintf(stderr,
++ "\tnotify %d recieved at +%llu\n",
++ ce->serial, ce->msc - msc);
++ ret++;
++ }
++ free(ev);
++ } while (!complete);
++
++ teardown_msc(dpy, Q);
++ XDestroyWindow(dpy, win);
++ XSync(dpy, True);
++
++ return ret;
+ }
+
+-static void teardown_msc(Display *dpy, void *q)
++static int test_exhaustion_msc(Display *dpy, void *Q)
+ {
+- xcb_unregister_for_special_event(XGetXCBConnection(dpy), q);
++#define N_VBLANKS 256 /* kernel event queue length: 128 vblanks */
++ xcb_connection_t *c = XGetXCBConnection(dpy);
++ Window root = DefaultRootWindow(dpy);
++ int ret = 0, n, complete;
++ int earliest = 0, early = 0;
++ int latest = 0, late = 0;
++ uint64_t msc;
++
++ printf("Testing notifies with long queues\n");
++ _x_error_occurred = 0;
++
++ msc = check_msc(dpy, root, Q, 0, NULL);
++ for (n = N_VBLANKS; n--; )
++ xcb_present_notify_msc(c, root, N_VBLANKS, msc + N_VBLANKS, 0, 0);
++ for (n = 1; n <= N_VBLANKS ; n++)
++ xcb_present_notify_msc(c, root, n, msc + n, 0, 0);
++ xcb_flush(c);
++
++ complete = 2*N_VBLANKS;
++ do {
++ xcb_present_complete_notify_event_t *ce;
++ xcb_generic_event_t *ev;
++ int diff;
++
++ ev = xcb_wait_for_special_event(c, Q);
++ if (ev == NULL)
++ break;
++
++ ce = (xcb_present_complete_notify_event_t *)ev;
++ assert(ce->kind == XCB_PRESENT_COMPLETE_KIND_NOTIFY_MSC);
++
++ diff = (int64_t)(ce->msc - msc - ce->serial);
++ if (diff < 0) {
++ if (-diff > earliest) {
++ fprintf(stderr, "\tnotify %d early by %d msc\n",(int)ce->serial, -diff);
++ earliest = -diff;
++ }
++ early++;
++ ret++;
++ } else if (diff > 0) {
++ if (diff > latest) {
++ fprintf(stderr, "\tnotify %d late by %d msc\n", (int)ce->serial, diff);
++ latest = diff;
++ }
++ late++;
++ ret++;
++ }
++ free(ev);
++ } while (--complete);
++
++ if (early)
++ printf("\t%d notifies too early (worst %d)!\n", early, earliest);
++ if (late)
++ printf("\t%d notifies too late (worst %d)!\n", late, latest);
++
++ XSync(dpy, True);
++ ret += !!_x_error_occurred;
++
++ return ret;
++#undef N_VBLANKS
+ }
+-static int test_whole(Display *dpy)
++
++static int test_accuracy_msc(Display *dpy, void *Q)
+ {
+- Pixmap pixmap;
+- struct dri3_fence fence;
+- Window root;
+- unsigned int width, height;
+- unsigned border, depth;
+- int x, y, ret = 1;
++#define N_VBLANKS (60 * 120) /* ~2 minutes */
++ xcb_connection_t *c = XGetXCBConnection(dpy);
++ Window root = DefaultRootWindow(dpy);
++ int ret = 0, n;
++ uint64_t msc;
++ int early = 0, late = 0;
++ int earliest = 0, latest = 0;
++ int complete, count;
+
+- XGetGeometry(dpy, DefaultRootWindow(dpy),
+- &root, &x, &y, &width, &height, &border, &depth);
++ printf("Testing notify accuracy\n");
++ _x_error_occurred = 0;
+
+- if (dri3_create_fence(dpy, root, &fence))
+- return 0;
++ msc = check_msc(dpy, root, Q, 0, NULL);
++ for (n = 0; n <= N_VBLANKS; n++)
++ xcb_present_notify_msc(c, root, n, msc + 60 + n, 0, 0);
++ xcb_present_notify_msc(c, root, 0xdeadbeef, msc + 60 + n, 0, 0);
++ xcb_flush(c);
++
++ complete = 0;
++ count = 0;
++ do {
++ xcb_present_complete_notify_event_t *ce;
++ xcb_generic_event_t *ev;
++
++ ev = xcb_wait_for_special_event(c, Q);
++ if (ev == NULL)
++ break;
++
++ ce = (xcb_present_complete_notify_event_t *)ev;
++ assert(ce->kind == XCB_PRESENT_COMPLETE_KIND_NOTIFY_MSC);
++
++ if (ce->serial != 0xdeadbeef) {
++ int diff = (int64_t)(ce->msc - (msc + ce->serial + 60));
++ if (diff < 0) {
++ if (-diff > earliest) {
++ fprintf(stderr, "\tnotify %d early by %d msc\n", ce->serial, -diff);
++ earliest = -diff;
++ }
++ early++;
++ ret++;
++ } else if (diff > 0) {
++ if (diff > latest) {
++ fprintf(stderr, "\tnotify %d late by %d msc\n", ce->serial, diff);
++ latest = diff;
++ }
++ late++;
++ ret++;
++ }
++ count++;
++ } else
++ complete = 1;
++ free(ev);
++ } while (!complete);
++
++ if (early)
++ printf("\t%d notifies too early (worst %d)!\n", early, earliest);
++ if (late)
++ printf("\t%d notifies too late (worst %d)!\n", late, latest);
++
++ if (count != N_VBLANKS+1) {
++ fprintf(stderr, "Sentinel vblank received too early! %d waits outstanding\n", N_VBLANKS+1 - count);
++ ret++;
++ do {
++ xcb_present_complete_notify_event_t *ce;
++ xcb_generic_event_t *ev;
++
++ ev = xcb_wait_for_special_event(c, Q);
++ if (ev == NULL)
++ break;
++
++ ce = (xcb_present_complete_notify_event_t *)ev;
++ assert(ce->kind == XCB_PRESENT_COMPLETE_KIND_NOTIFY_MSC);
++ free(ev);
++ } while (++count != N_VBLANKS+1);
++ }
++
++ XSync(dpy, True);
++ ret += !!_x_error_occurred;
++
++ return ret;
++#undef N_VBLANKS
++}
+
+- printf("Testing whole screen flip: %dx%d\n", width, height);
++static int test_modulus_msc(Display *dpy, void *Q)
++{
++ xcb_connection_t *c = XGetXCBConnection(dpy);
++ Window root = DefaultRootWindow(dpy);
++ xcb_present_complete_notify_event_t *ce;
++ xcb_generic_event_t *ev;
++ int x, y, ret = 0;
++ uint64_t target;
++ int early = 0, late = 0;
++ int earliest = 0, latest = 0;
++ int complete, count, expect;
++
++ printf("Testing notify modulus\n");
+ _x_error_occurred = 0;
+
+- xshmfence_reset(fence.addr);
++ target = wait_vblank(dpy, root, Q);
+
+- pixmap = XCreatePixmap(dpy, root, width, height, depth);
+- xcb_present_pixmap(XGetXCBConnection(dpy),
+- root, pixmap,
+- 0, /* sbc */
+- 0, /* valid */
+- 0, /* update */
+- 0, /* x_off */
+- 0, /* y_off */
+- None,
+- None, /* wait fence */
+- fence.xid,
+- XCB_PRESENT_OPTION_NONE,
+- 0, /* target msc */
+- 0, /* divisor */
+- 0, /* remainder */
+- 0, NULL);
+- XFreePixmap(dpy, pixmap);
++ expect = 0;
++ xcb_present_notify_msc(c, root, 0, 0, 0, 0);
++ for (x = 1; x <= 19; x++) {
++ for (y = 0; y < x; y++) {
++ xcb_present_notify_msc(c, root, y << 16 | x, 0, x, y);
++ expect++;
++ }
++ }
++ xcb_present_notify_msc(c, root, 0xdeadbeef, target + 2*x, 0, 0);
++ xcb_flush(c);
+
+- pixmap = XCreatePixmap(dpy, root, width, height, depth);
+- xcb_present_pixmap(XGetXCBConnection(dpy),
+- root, pixmap,
+- 0, /* sbc */
+- 0, /* valid */
+- 0, /* update */
+- 0, /* x_off */
+- 0, /* y_off */
+- None,
+- None, /* wait fence */
+- None, /* sync fence */
+- XCB_PRESENT_OPTION_NONE,
+- 0, /* target msc */
+- 0, /* divisor */
+- 0, /* remainder */
+- 0, NULL);
+- XFreePixmap(dpy, pixmap);
+- XFlush(dpy);
++ ev = xcb_wait_for_special_event(c, Q);
++ if (ev) {
++ ce = (xcb_present_complete_notify_event_t *)ev;
++ assert(ce->kind == XCB_PRESENT_COMPLETE_KIND_NOTIFY_MSC);
++ assert(ce->serial == 0);
++ assert(target == ce->msc);
++ target = ce->msc;
++ }
+
+- ret = !!xshmfence_await(fence.addr);
+- dri3_fence_free(dpy, &fence);
++ complete = 0;
++ count = 0;
++ do {
++ ev = xcb_wait_for_special_event(c, Q);
++ if (ev == NULL)
++ break;
++
++ ce = (xcb_present_complete_notify_event_t *)ev;
++ assert(ce->kind == XCB_PRESENT_COMPLETE_KIND_NOTIFY_MSC);
++
++ assert(ce->serial);
++ if (ce->serial != 0xdeadbeef) {
++ uint64_t msc;
++ int diff;
++
++ x = ce->serial & 0xffff;
++ y = ce->serial >> 16;
++
++ msc = target;
++ msc -= target % x;
++ msc += y;
++ if (msc <= target)
++ msc += x;
++
++ diff = (int64_t)(ce->msc - msc);
++ if (diff < 0) {
++ if (-diff > earliest) {
++ fprintf(stderr, "\tnotify (%d, %d) early by %d msc (target %lld, reported %lld)\n", y, x, -diff, (long long)msc, (long long)ce->msc);
++ earliest = -diff;
++ }
++ early++;
++ ret++;
++ } else if (diff > 0) {
++ if (diff > latest) {
++ fprintf(stderr, "\tnotify (%d, %d) late by %d msc (target %lld, reported %lld)\n", y, x, diff, (long long)msc, (long long)ce->msc);
++ latest = diff;
++ }
++ late++;
++ ret++;
++ }
++ count++;
++ } else
++ complete = 1;
++ free(ev);
++ } while (!complete);
++
++ if (early)
++ printf("\t%d notifies too early (worst %d)!\n", early, earliest);
++ if (late)
++ printf("\t%d notifies too late (worst %d)!\n", late, latest);
++
++ if (count != expect) {
++ fprintf(stderr, "Sentinel vblank received too early! %d waits outstanding\n", expect - count);
++ ret++;
++ do {
++ ev = xcb_wait_for_special_event(c, Q);
++ if (ev == NULL)
++ break;
++
++ ce = (xcb_present_complete_notify_event_t *)ev;
++ assert(ce->kind == XCB_PRESENT_COMPLETE_KIND_NOTIFY_MSC);
++ free(ev);
++ } while (++count != expect);
++ }
+
+ XSync(dpy, True);
+ ret += !!_x_error_occurred;
+@@ -279,8 +1471,6 @@ static int for_each_crtc(Display *dpy,
+ for (i = 0; i < res->ncrtc; i++)
+ original_crtc[i] = XRRGetCrtcInfo(dpy, res, res->crtcs[i]);
+
+- printf("noutput=%d, ncrtc=%d\n", res->noutput, res->ncrtc);
+-
+ for (i = 0; i < res->noutput; i++) {
+ XRROutputInfo *output;
+ XRRModeInfo *mode;
+@@ -322,7 +1512,7 @@ static int for_each_crtc(Display *dpy,
+ free(original_crtc);
+ XRRFreeScreenResources(res);
+
+- return j;
++ return err;
+ }
+
+ struct test_crtc {
+@@ -335,6 +1525,7 @@ struct test_crtc {
+ uint64_t msc;
+ };
+ #define SYNC 0x1
++#define FUTURE 0x2
+
+ static int __test_crtc(Display *dpy, RRCrtc crtc,
+ int width, int height,
+@@ -344,7 +1535,7 @@ static int __test_crtc(Display *dpy, RRCrtc crtc,
+ Pixmap pixmap;
+ int err = 0;
+
+- test->msc = check_msc(dpy, test->win, test->queue, test->msc);
++ test->msc = check_msc(dpy, test->win, test->queue, test->msc, NULL);
+
+ if (test->flags & SYNC)
+ xshmfence_reset(test->fence.addr);
+@@ -361,16 +1552,14 @@ static int __test_crtc(Display *dpy, RRCrtc crtc,
+ None, /* wait fence */
+ test->flags & SYNC ? test->fence.xid : None,
+ XCB_PRESENT_OPTION_NONE,
+- 0, /* target msc */
++ test->msc, /* target msc */
+ 1, /* divisor */
+ 0, /* remainder */
+ 0, NULL);
+- XFreePixmap(dpy, pixmap);
+-
+ if (test->flags & SYNC) {
+- pixmap = XCreatePixmap(dpy, test->win, width, height, test->depth);
++ Pixmap tmp = XCreatePixmap(dpy, test->win, width, height, test->depth);
+ xcb_present_pixmap(XGetXCBConnection(dpy),
+- test->win, pixmap,
++ test->win, tmp,
+ 1, /* sbc */
+ 0, /* valid */
+ 0, /* update */
+@@ -380,16 +1569,17 @@ static int __test_crtc(Display *dpy, RRCrtc crtc,
+ None, /* wait fence */
+ None, /* sync fence */
+ XCB_PRESENT_OPTION_NONE,
+- 1, /* target msc */
++ test->msc + (test->flags & FUTURE ? 5 * 16 : 1), /* target msc */
+ 1, /* divisor */
+ 0, /* remainder */
+ 0, NULL);
+- XFreePixmap(dpy, pixmap);
++ XFreePixmap(dpy, tmp);
+ XFlush(dpy);
+ err += !!xshmfence_await(test->fence.addr);
+ }
++ XFreePixmap(dpy, pixmap);
+
+- test->msc = check_msc(dpy, test->win, test->queue, test->msc);
++ test->msc = check_msc(dpy, test->win, test->queue, test->msc, NULL);
+ return err;
+ }
+
+@@ -410,15 +1600,23 @@ static int test_crtc(Display *dpy, void *queue, uint64_t last_msc)
+
+ printf("Testing each crtc, without waiting for each flip\n");
+ test.flags = 0;
++ test.msc = check_msc(dpy, test.win, test.queue, test.msc, NULL);
+ err += for_each_crtc(dpy, __test_crtc, &test);
++ test.msc = check_msc(dpy, test.win, test.queue, test.msc, NULL);
+
+ printf("Testing each crtc, waiting for flips to complete\n");
+ test.flags = SYNC;
++ test.msc = check_msc(dpy, test.win, test.queue, test.msc, NULL);
+ err += for_each_crtc(dpy, __test_crtc, &test);
++ test.msc = check_msc(dpy, test.win, test.queue, test.msc, NULL);
+
+- test.msc = check_msc(dpy, test.win, test.queue, test.msc);
+- dri3_fence_free(dpy, &test.fence);
++ printf("Testing each crtc, with future flips\n");
++ test.flags = FUTURE | SYNC;
++ test.msc = check_msc(dpy, test.win, test.queue, test.msc, NULL);
++ err += for_each_crtc(dpy, __test_crtc, &test);
++ test.msc = check_msc(dpy, test.win, test.queue, test.msc, NULL);
+
++ dri3_fence_free(dpy, &test.fence);
+ XSync(dpy, True);
+ err += !!_x_error_occurred;
+
+@@ -536,6 +1734,31 @@ static int gem_set_caching(int fd, uint32_t handle, int caching)
+ return drmIoctl(fd, LOCAL_IOCTL_I915_GEM_SET_CACHING, &arg) == 0;
+ }
+
++static int gem_set_tiling(int fd, uint32_t handle, int tiling, int stride)
++{
++ struct drm_i915_gem_set_tiling set_tiling;
++ int err;
++
++restart:
++ set_tiling.handle = handle;
++ set_tiling.tiling_mode = tiling;
++ set_tiling.stride = stride;
++
++ if (drmIoctl(fd, DRM_IOCTL_I915_GEM_SET_TILING, &set_tiling) == 0)
++ return 1;
++
++ err = errno;
++ if (err == EINTR)
++ goto restart;
++
++ if (err == EAGAIN) {
++ sched_yield();
++ goto restart;
++ }
++
++ return 0;
++}
++
+ static int gem_export(int fd, uint32_t handle)
+ {
+ struct drm_prime_handle args;
+@@ -557,6 +1780,126 @@ static void gem_close(int fd, uint32_t handle)
+ (void)drmIoctl(fd, DRM_IOCTL_GEM_CLOSE, &close);
+ }
+
++static int test_dri3_tiling(Display *dpy)
++{
++ Window win = DefaultRootWindow(dpy);
++ const int tiling[] = { I915_TILING_NONE, I915_TILING_X, I915_TILING_Y };
++ Window root;
++ unsigned int width, height;
++ unsigned border, depth, bpp;
++ unsigned stride, size;
++ void *Q;
++ int x, y;
++ int device;
++ int line = -1;
++ int t;
++
++ device = dri3_open(dpy);
++ if (device < 0)
++ return 0;
++
++ if (!is_intel(device))
++ return 0;
++
++ printf("Opened Intel DRI3 device\n");
++
++ XGetGeometry(dpy, win, &root, &x, &y,
++ &width, &height, &border, &depth);
++
++ switch (depth) {
++ case 8: bpp = 8; break;
++ case 15: case 16: bpp = 16; break;
++ case 24: case 32: bpp = 32; break;
++ default: return 0;
++ }
++
++ stride = ALIGN(width * bpp/8, 512);
++ size = PAGE_ALIGN(stride * ALIGN(height, 32));
++ printf("Creating DRI3 %dx%d (source stride=%d, size=%d) for GTT\n",
++ width, height, stride, size);
++
++ _x_error_occurred = 0;
++ Q = setup_msc(dpy, root);
++
++ for (t = 0; t < sizeof(tiling)/sizeof(tiling[0]); t++) {
++ uint64_t msc;
++ uint32_t src;
++ int src_fd;
++ Pixmap src_pix;
++
++ src = gem_create(device, size);
++ if (!src) {
++ line = __LINE__;
++ goto fail;
++ }
++
++ gem_set_tiling(device, src, tiling[t], stride);
++
++ src_fd = gem_export(device, src);
++ if (src_fd < 0) {
++ line = __LINE__;
++ goto fail;
++ }
++
++ src_pix = dri3_create_pixmap(dpy, root,
++ width, height, depth,
++ src_fd, bpp, stride, size);
++
++ msc = wait_vblank(dpy, root, Q);
++
++ xcb_present_pixmap(XGetXCBConnection(dpy),
++ win, src_pix,
++ 0, /* sbc */
++ 0, /* valid */
++ 0, /* update */
++ 0, /* x_off */
++ 0, /* y_off */
++ None,
++ None, /* wait fence */
++ None,
++ XCB_PRESENT_OPTION_NONE,
++ msc + 2, /* target msc */
++ 1, /* divisor */
++ 0, /* remainder */
++ 0, NULL);
++
++ xcb_present_pixmap(XGetXCBConnection(dpy),
++ win, src_pix,
++ 0, /* sbc */
++ 0, /* valid */
++ 0, /* update */
++ 0, /* x_off */
++ 0, /* y_off */
++ None,
++ None, /* wait fence */
++ None,
++ XCB_PRESENT_OPTION_NONE,
++ msc + 3, /* target msc */
++ 1, /* divisor */
++ 0, /* remainder */
++ 0, NULL);
++
++ XSync(dpy, True);
++ if (_x_error_occurred) {
++ line = __LINE__;
++ goto fail;
++ }
++ XFreePixmap(dpy, src_pix);
++ _x_error_occurred = 0;
++
++ close(src_fd);
++ gem_close(device, src);
++ }
++
++ teardown_msc(dpy, Q);
++ return 0;
++
++fail:
++ printf("%s failed with tiling %d, line %d\n", __func__, tiling[t], line);
++ teardown_msc(dpy, Q);
++ return 1;
++}
++
+ static int test_dri3(Display *dpy)
+ {
+ Window win = DefaultRootWindow(dpy);
+@@ -670,8 +2013,32 @@ fail:
+ static int has_present(Display *dpy)
+ {
+ xcb_connection_t *c = XGetXCBConnection(dpy);
+- xcb_present_query_version_reply_t *reply;
+ xcb_generic_error_t *error = NULL;
++ void *reply;
++
++ reply = xcb_xfixes_query_version_reply(c,
++ xcb_xfixes_query_version(c,
++ XCB_XFIXES_MAJOR_VERSION,
++ XCB_XFIXES_MINOR_VERSION),
++ &error);
++ free(reply);
++ free(error);
++ if (reply == NULL) {
++ fprintf(stderr, "XFixes not supported on %s\n", DisplayString(dpy));
++ return 0;
++ }
++
++ reply = xcb_dri3_query_version_reply(c,
++ xcb_dri3_query_version(c,
++ XCB_DRI3_MAJOR_VERSION,
++ XCB_DRI3_MINOR_VERSION),
++ &error);
++ free(reply);
++ free(error);
++ if (reply == NULL) {
++ fprintf(stderr, "DRI3 not supported on %s\n", DisplayString(dpy));
++ return 0;
++ }
+
+ reply = xcb_present_query_version_reply(c,
+ xcb_present_query_version(c,
+@@ -681,14 +2048,32 @@ static int has_present(Display *dpy)
+
+ free(reply);
+ free(error);
++ if (reply == NULL) {
++ fprintf(stderr, "Present not supported on %s\n", DisplayString(dpy));
++ return 0;
++ }
++
++ return 1;
++}
++
++static int has_composite(Display *dpy)
++{
++ int event, error;
++ int major, minor;
++
++ if (!XCompositeQueryExtension(dpy, &event, &error))
++ return 0;
++
++ XCompositeQueryVersion(dpy, &major, &minor);
+
+- return reply != NULL;
++ return major > 0 || minor >= 4;
+ }
+
+ int main(void)
+ {
+ Display *dpy;
+ Window root;
++ int dummy;
+ int error = 0;
+ uint64_t last_msc;
+ void *queue;
+@@ -700,27 +2085,135 @@ int main(void)
+ if (!has_present(dpy))
+ return 77;
+
++ if (DPMSQueryExtension(dpy, &dummy, &dummy))
++ DPMSDisable(dpy);
++
+ root = DefaultRootWindow(dpy);
+
+ signal(SIGALRM, SIG_IGN);
+ XSetErrorHandler(_check_error_handler);
+
+ queue = setup_msc(dpy, root);
+- last_msc = check_msc(dpy, root, queue, 0);
++ last_msc = check_msc(dpy, root, queue, 0, NULL);
++
++ error += test_future_msc(dpy, queue);
++ last_msc = check_msc(dpy, root, queue, last_msc, NULL);
++
++ error += test_wrap_msc(dpy);
++ last_msc = check_msc(dpy, root, queue, last_msc, NULL);
++
++ error += test_accuracy_msc(dpy, queue);
++ last_msc = check_msc(dpy, root, queue, last_msc, NULL);
++
++ error += test_modulus_msc(dpy, queue);
++ last_msc = check_msc(dpy, root, queue, last_msc, NULL);
++
++ error += test_exhaustion_msc(dpy, queue);
++ last_msc = check_msc(dpy, root, queue, last_msc, NULL);
++
++ for (dummy = 0; dummy <= 3; dummy++) {
++ Window win;
++ uint64_t msc = 0;
++ XSetWindowAttributes attr;
++ Visual *visual = DefaultVisual(dpy, DefaultScreen(dpy));
++ unsigned int width, height;
++ unsigned border, depth;
++ const char *phase;
++ int x, y;
++ void *Q;
++
++ attr.override_redirect = 1;
++
++ XGetGeometry(dpy, root, &win, &x, &y,
++ &width, &height, &border, &depth);
++
++ _x_error_occurred = 0;
++ switch (dummy) {
++ case 0:
++ win = root;
++ phase = "root";
++ break;
++ case 1:
++ win = XCreateWindow(dpy, root,
++ 0, 0, width, height, 0, depth,
++ InputOutput, visual,
++ CWOverrideRedirect, &attr);
++ phase = "fullscreen";
++ break;
++ case 2:
++ win = XCreateWindow(dpy, root,
++ 0, 0, width/2, height/2, 0, depth,
++ InputOutput, visual,
++ CWOverrideRedirect, &attr);
++ phase = "window";
++ break;
++ case 3:
++ if (!has_composite(dpy))
++ continue;
++
++ win = XCreateWindow(dpy, root,
++ 0, 0, width, height, 0,
++ DefaultDepth(dpy, DefaultScreen(dpy)),
++ InputOutput,
++ DefaultVisual(dpy, DefaultScreen(dpy)),
++ CWOverrideRedirect, &attr);
++ XCompositeRedirectWindow(dpy, win, CompositeRedirectManual);
++ phase = "composite";
++ break;
++
++ default:
++ phase = "broken";
++ win = root;
++ abort();
++ break;
++ }
++
++ XMapWindow(dpy, win);
++ XSync(dpy, True);
++ if (_x_error_occurred)
++ continue;
++
++ Q = setup_msc(dpy, win);
++ msc = check_msc(dpy, win, Q, msc, NULL);
+
+- error += test_whole(dpy);
+- last_msc = check_msc(dpy, root, queue, last_msc);
++ error += test_whole(dpy, win, phase);
++ msc = check_msc(dpy, win, Q, msc, NULL);
++
++ error += test_double(dpy, win, phase, Q);
++ msc = check_msc(dpy, win, Q, msc, NULL);
++
++ error += test_future(dpy, win, phase, Q);
++ msc = check_msc(dpy, win, Q, msc, NULL);
++
++ error += test_accuracy(dpy, win, phase, Q);
++ msc = check_msc(dpy, win, Q, msc, NULL);
++
++ error += test_modulus(dpy, win, phase, Q);
++ msc = check_msc(dpy, win, Q, msc, NULL);
++
++ error += test_exhaustion(dpy, win, phase, Q);
++ msc = check_msc(dpy, win, Q, msc, NULL);
++
++ teardown_msc(dpy, Q);
++ if (win != root)
++ XDestroyWindow(dpy, win);
++ }
+
+ error += test_crtc(dpy, queue, last_msc);
+- last_msc = check_msc(dpy, root, queue, last_msc);
++ last_msc = check_msc(dpy, root, queue, last_msc, NULL);
+
+ error += test_shm(dpy);
+- last_msc = check_msc(dpy, root, queue, last_msc);
++ last_msc = check_msc(dpy, root, queue, last_msc, NULL);
+
+ error += test_dri3(dpy);
+- last_msc = check_msc(dpy, root, queue, last_msc);
++ last_msc = check_msc(dpy, root, queue, last_msc, NULL);
++
++ error += test_dri3_tiling(dpy);
++ last_msc = check_msc(dpy, root, queue, last_msc, NULL);
+
+ teardown_msc(dpy, queue);
+
++ if (DPMSQueryExtension(dpy, &dummy, &dummy))
++ DPMSEnable(dpy);
+ return !!error;
+ }
+diff --git a/test/render-glyphs.c b/test/render-glyphs.c
+new file mode 100644
+index 00000000..8822e36a
+--- /dev/null
++++ b/test/render-glyphs.c
+@@ -0,0 +1,441 @@
++#include <stdint.h>
++#include <stdio.h>
++#include <stdlib.h>
++#include <stdbool.h>
++#include <stdarg.h>
++#include <string.h>
++
++#include <X11/Xutil.h> /* for XDestroyImage */
++#include <pixman.h> /* for pixman blt functions */
++
++#include "test.h"
++
++static const XRenderColor colors[] = {
++ /* red, green, blue, alpha */
++ { 0 },
++ { 0, 0, 0, 0xffff },
++ { 0xffff, 0, 0, 0xffff },
++ { 0, 0xffff, 0, 0xffff },
++ { 0, 0, 0xffff, 0xffff },
++ { 0xffff, 0xffff, 0xffff, 0xffff },
++};
++
++static struct clip {
++ void *func;
++} clips[] = {
++ { NULL },
++};
++
++static int _x_error_occurred;
++
++static int
++_check_error_handler(Display *display,
++ XErrorEvent *event)
++{
++ _x_error_occurred = 1;
++ return False; /* ignored */
++}
++
++static void clear(struct test_display *dpy,
++ struct test_target *tt,
++ const XRenderColor *c)
++{
++ XRenderFillRectangle(dpy->dpy, PictOpClear, tt->picture, c,
++ 0, 0, tt->width, tt->height);
++}
++
++static bool check_op(struct test_display *dpy, int op, struct test_target *tt)
++{
++ XRenderColor render_color = {0};
++
++ XSync(dpy->dpy, True);
++ _x_error_occurred = 0;
++
++ XRenderFillRectangle(dpy->dpy, op,
++ tt->picture, &render_color,
++ 0, 0, 0, 0);
++
++ XSync(dpy->dpy, True);
++ return _x_error_occurred == 0;
++}
++
++struct glyph_iter {
++ enum {
++ GLYPHS, OP, DST, SRC, MASK, CLIP,
++ } stage;
++
++ int glyph_format;
++ int op;
++ int dst_color;
++ int src_color;
++ int mask_format;
++ int clip;
++
++ struct {
++ struct test_display *dpy;
++ struct test_target tt;
++ GlyphSet glyphset;
++ Picture src;
++ XRenderPictFormat *mask_format;
++ } ref, out;
++};
++
++static void glyph_iter_init(struct glyph_iter *gi,
++ struct test *t, enum target target)
++{
++ memset(gi, 0, sizeof(*gi));
++
++ gi->out.dpy = &t->out;
++ test_target_create_render(&t->out, target, &gi->out.tt);
++
++ gi->ref.dpy = &t->ref;
++ test_target_create_render(&t->ref, target, &gi->ref.tt);
++
++ gi->stage = GLYPHS;
++ gi->glyph_format = -1;
++ gi->op = -1;
++ gi->dst_color = -1;
++ gi->src_color = -1;
++ gi->mask_format = -1;
++ gi->clip = -1;
++}
++
++static void render_clear(char *image, int image_size, int bpp)
++{
++ memset(image, 0, image_size);
++}
++
++static void render_black(char *image, int image_size, int bpp)
++{
++ if (bpp == 4) {
++ uint32_t *p = (uint32_t *)image;
++ image_size /= 4;
++ while (image_size--)
++ *p++ = 0x000000ff;
++ } else
++ memset(image, 0x55, image_size);
++}
++
++static void render_green(char *image, int image_size, int bpp)
++{
++ if (bpp == 4) {
++ uint32_t *p = (uint32_t *)image;
++ image_size /= 4;
++ while (image_size--)
++ *p++ = 0xffff0000;
++ } else
++ memset(image, 0xaa, image_size);
++}
++
++static void render_white(char *image, int image_size, int bpp)
++{
++ memset(image, 0xff, image_size);
++}
++
++static GlyphSet create_glyphs(Display *dpy, int format_id)
++{
++#define N_GLYPHS 4
++ XRenderPictFormat *format;
++ XGlyphInfo glyph = { 8, 8, 0, 0, 8, 0 };
++ char image[4*8*8];
++ GlyphSet glyphset;
++ Glyph gid;
++ int image_size;
++ int bpp;
++ int n;
++
++ format = XRenderFindStandardFormat(dpy, format_id);
++ if (format == NULL)
++ return 0;
++
++ switch (format_id) {
++ case PictStandardARGB32:
++ case PictStandardRGB24:
++ image_size = 4 * 8 * 8;
++ bpp = 4;
++ break;
++ case PictStandardA8:
++ case PictStandardA4:
++ image_size = 8 * 8;
++ bpp = 1;
++ break;
++ case PictStandardA1:
++ image_size = 8;
++ bpp = 0;
++ break;
++ default:
++ return 0;
++ }
++
++ glyphset = XRenderCreateGlyphSet(dpy, format);
++ for (n = 0; n < N_GLYPHS; n++) {
++ gid = n;
++
++ switch (n) {
++ case 0: render_clear(image, image_size, bpp); break;
++ case 1: render_black(image, image_size, bpp); break;
++ case 2: render_green(image, image_size, bpp); break;
++ case 3: render_white(image, image_size, bpp); break;
++ }
++
++ XRenderAddGlyphs(dpy, glyphset,
++ &gid, &glyph, 1, image, image_size);
++ }
++
++ return glyphset;
++}
++
++static const char *glyph_name(int n)
++{
++ switch (n) {
++ case 0: return "clear";
++ case 1: return "black";
++ case 2: return "green";
++ case 3: return "white";
++ default: return "unknown";
++ }
++}
++
++static bool glyph_iter_next(struct glyph_iter *gi)
++{
++restart:
++ if (gi->stage == GLYPHS) {
++ if (++gi->glyph_format == PictStandardNUM)
++ return false;
++
++ if (gi->out.glyphset)
++ XRenderFreeGlyphSet(gi->out.dpy->dpy,
++ gi->out.glyphset);
++ gi->out.glyphset = create_glyphs(gi->out.dpy->dpy,
++ gi->glyph_format);
++
++ if (gi->ref.glyphset)
++ XRenderFreeGlyphSet(gi->ref.dpy->dpy,
++ gi->ref.glyphset);
++ gi->ref.glyphset = create_glyphs(gi->ref.dpy->dpy,
++ gi->glyph_format);
++
++ gi->stage++;
++ }
++
++ if (gi->stage == OP) {
++ do {
++ if (++gi->op == 255)
++ goto reset_op;
++ } while (!check_op(gi->out.dpy, gi->op, &gi->out.tt) ||
++ !check_op(gi->ref.dpy, gi->op, &gi->ref.tt));
++
++ gi->stage++;
++ }
++
++ if (gi->stage == DST) {
++ if (++gi->dst_color == ARRAY_SIZE(colors))
++ goto reset_dst;
++
++ gi->stage++;
++ }
++
++ if (gi->stage == SRC) {
++ if (++gi->src_color == ARRAY_SIZE(colors))
++ goto reset_src;
++
++ if (gi->ref.src)
++ XRenderFreePicture(gi->ref.dpy->dpy, gi->ref.src);
++ gi->ref.src = XRenderCreateSolidFill(gi->ref.dpy->dpy,
++ &colors[gi->src_color]);
++
++ if (gi->out.src)
++ XRenderFreePicture(gi->out.dpy->dpy, gi->out.src);
++ gi->out.src = XRenderCreateSolidFill(gi->out.dpy->dpy,
++ &colors[gi->src_color]);
++
++ gi->stage++;
++ }
++
++ if (gi->stage == MASK) {
++ if (++gi->mask_format > PictStandardNUM)
++ goto reset_mask;
++
++ if (gi->mask_format == PictStandardRGB24)
++ gi->mask_format++;
++
++ if (gi->mask_format < PictStandardNUM) {
++ gi->out.mask_format = XRenderFindStandardFormat(gi->out.dpy->dpy,
++ gi->mask_format);
++ gi->ref.mask_format = XRenderFindStandardFormat(gi->ref.dpy->dpy,
++ gi->mask_format);
++ } else {
++ gi->out.mask_format = NULL;
++ gi->ref.mask_format = NULL;
++ }
++
++ gi->stage++;
++ }
++
++ if (gi->stage == CLIP) {
++ if (++gi->clip == ARRAY_SIZE(clips))
++ goto reset_clip;
++
++ gi->stage++;
++ }
++
++ gi->stage--;
++ return true;
++
++reset_op:
++ gi->op = -1;
++reset_dst:
++ gi->dst_color = -1;
++reset_src:
++ gi->src_color = -1;
++reset_mask:
++ gi->mask_format = -1;
++reset_clip:
++ gi->clip = -1;
++ gi->stage--;
++ goto restart;
++}
++
++static void glyph_iter_fini(struct glyph_iter *gi)
++{
++ if (gi->out.glyphset)
++ XRenderFreeGlyphSet (gi->out.dpy->dpy, gi->out.glyphset);
++ if (gi->ref.glyphset)
++ XRenderFreeGlyphSet (gi->ref.dpy->dpy, gi->ref.glyphset);
++
++ test_target_destroy_render(gi->out.dpy, &gi->out.tt);
++ test_target_destroy_render(gi->ref.dpy, &gi->ref.tt);
++}
++
++static const char *stdformat_to_str(int id)
++{
++ switch (id) {
++ case PictStandardARGB32: return "ARGB32";
++ case PictStandardRGB24: return "RGB24";
++ case PictStandardA8: return "A8";
++ case PictStandardA4: return "A4";
++ case PictStandardA1: return "A1";
++ default: return "none";
++ }
++}
++
++static char *glyph_iter_to_string(struct glyph_iter *gi,
++ const char *format,
++ ...)
++{
++ static char buf[100];
++ va_list ap;
++ int len;
++
++ len = sprintf(buf, "glyphs=%s, op=%d, dst=%08x, src=%08x, mask=%s",
++ stdformat_to_str(gi->glyph_format), gi->op,
++ xrender_color(&colors[gi->dst_color]),
++ xrender_color(&colors[gi->src_color]),
++ stdformat_to_str(gi->mask_format));
++
++ if (format) {
++ buf[len++] = ' ';
++ va_start(ap, format);
++ vsprintf(buf+len, format, ap);
++ va_end(ap);
++ }
++
++ return buf;
++}
++
++static void single(struct test *t, enum target target)
++{
++ struct glyph_iter gi;
++ int n;
++
++ printf("Testing single glyph (%s): ", test_target_name(target));
++ fflush(stdout);
++
++ glyph_iter_init(&gi, t, target);
++ while (glyph_iter_next(&gi)) {
++ XGlyphElt8 elt;
++ char id[N_GLYPHS];
++
++ for (n = 0; n < N_GLYPHS; n++) {
++ id[n] = n;
++
++ elt.chars = &id[n];
++ elt.nchars = 1;
++ elt.xOff = 0;
++ elt.yOff = 0;
++
++ clear(gi.out.dpy, &gi.out.tt, &colors[gi.dst_color]);
++ elt.glyphset = gi.out.glyphset;
++ XRenderCompositeText8 (gi.out.dpy->dpy, gi.op,
++ gi.out.src,
++ gi.out.tt.picture,
++ gi.out.mask_format,
++ 0, 0,
++ 0, 8,
++ &elt, 1);
++
++ clear(gi.ref.dpy, &gi.ref.tt, &colors[gi.dst_color]);
++ elt.glyphset = gi.ref.glyphset;
++ XRenderCompositeText8 (gi.ref.dpy->dpy, gi.op,
++ gi.ref.src,
++ gi.ref.tt.picture,
++ gi.ref.mask_format,
++ 0, 0,
++ 0, 8,
++ &elt, 1);
++ test_compare(t,
++ gi.out.tt.draw, gi.out.tt.format,
++ gi.ref.tt.draw, gi.ref.tt.format,
++ 0, 0, gi.out.tt.width, gi.out.tt.height,
++ glyph_iter_to_string(&gi,
++ "glyph=%s",
++ glyph_name(n)));
++ }
++
++ elt.chars = &id[0];
++ elt.nchars = n;
++ clear(gi.out.dpy, &gi.out.tt, &colors[gi.dst_color]);
++ elt.glyphset = gi.out.glyphset;
++ XRenderCompositeText8 (gi.out.dpy->dpy, gi.op,
++ gi.out.src,
++ gi.out.tt.picture,
++ gi.out.mask_format,
++ 0, 0,
++ 0, 8,
++ &elt, 1);
++
++ clear(gi.ref.dpy, &gi.ref.tt, &colors[gi.dst_color]);
++ elt.glyphset = gi.ref.glyphset;
++ XRenderCompositeText8 (gi.ref.dpy->dpy, gi.op,
++ gi.ref.src,
++ gi.ref.tt.picture,
++ gi.ref.mask_format,
++ 0, 0,
++ 0, 8,
++ &elt, 1);
++ test_compare(t,
++ gi.out.tt.draw, gi.out.tt.format,
++ gi.ref.tt.draw, gi.ref.tt.format,
++ 0, 0, gi.out.tt.width, gi.out.tt.height,
++ glyph_iter_to_string(&gi, "all"));
++ }
++ glyph_iter_fini(&gi);
++}
++
++int main(int argc, char **argv)
++{
++ struct test test;
++ int t;
++
++ test_init(&test, argc, argv);
++ XSetErrorHandler(_check_error_handler);
++
++ for (t = TARGET_FIRST; t <= TARGET_LAST; t++) {
++ single(&test, t);
++ //overlapping(&test, t);
++ //gap(&test, t);
++ //mixed(&test, t);
++ }
++
++ return 0;
++}
+diff --git a/test/render-trapezoid.c b/test/render-trapezoid.c
+index cd990143..f15a78e3 100644
+--- a/test/render-trapezoid.c
++++ b/test/render-trapezoid.c
+@@ -403,16 +403,141 @@ static void trap_tests(struct test *t,
+ free(traps);
+ }
+
++enum edge {
++ EDGE_SHARP = PolyEdgeSharp,
++ EDGE_SMOOTH,
++};
++
++static const char *edge_name(enum edge edge)
++{
++ switch (edge) {
++ default:
++ case EDGE_SHARP: return "sharp";
++ case EDGE_SMOOTH: return "smooth";
++ }
++}
++
++static void set_edge(Display *dpy, Picture p, enum edge edge)
++{
++ XRenderPictureAttributes a;
++
++ a.poly_edge = edge;
++ XRenderChangePicture(dpy, p, CPPolyEdge, &a);
++}
++
++static void edge_test(struct test *t,
++ enum mask mask,
++ enum edge edge,
++ enum target target)
++{
++ struct test_target out, ref;
++ XRenderColor white = { 0xffff, 0xffff, 0xffff, 0xffff };
++ Picture src_ref, src_out;
++ XTrapezoid trap;
++ int left_or_right, p;
++
++ test_target_create_render(&t->out, target, &out);
++ set_edge(t->out.dpy, out.picture, edge);
++ src_out = XRenderCreateSolidFill(t->out.dpy, &white);
++
++ test_target_create_render(&t->ref, target, &ref);
++ set_edge(t->ref.dpy, ref.picture, edge);
++ src_ref = XRenderCreateSolidFill(t->ref.dpy, &white);
++
++ printf("Testing edges (with mask %s and %s edges) (%s): ",
++ mask_name(mask),
++ edge_name(edge),
++ test_target_name(target));
++ fflush(stdout);
++
++ for (left_or_right = 0; left_or_right <= 1; left_or_right++) {
++ for (p = -64; p <= out.width + 64; p++) {
++ char buf[80];
++
++ if (left_or_right) {
++ trap.left.p1.x = 0;
++ trap.left.p1.y = 0;
++ trap.left.p2.x = 0;
++ trap.left.p2.y = out.height << 16;
++
++ trap.right.p1.x = p << 16;
++ trap.right.p1.y = 0;
++ trap.right.p2.x = out.width << 16;
++ trap.right.p2.y = out.height << 16;
++ } else {
++ trap.right.p1.x = out.width << 16;
++ trap.right.p1.y = 0;
++ trap.right.p2.x = out.width << 16;
++ trap.right.p2.y = out.height << 16;
++
++ trap.left.p1.x = 0;
++ trap.left.p1.y = 0;
++ trap.left.p2.x = p << 16;
++ trap.left.p2.y = out.height << 16;
++ }
++
++ trap.top = 0;
++ trap.bottom = out.height << 16;
++
++ sprintf(buf,
++ "trap=((%d, %d), (%d, %d)), ((%d, %d), (%d, %d))\n",
++ trap.left.p1.x >> 16, trap.left.p1.y >> 16,
++ trap.left.p2.x >> 16, trap.left.p2.y >> 16,
++ trap.right.p1.x >> 16, trap.right.p1.y >> 16,
++ trap.right.p2.x >> 16, trap.right.p2.y >> 16);
++
++ clear(&t->out, &out);
++ XRenderCompositeTrapezoids(t->out.dpy,
++ PictOpSrc,
++ src_out,
++ out.picture,
++ mask_format(t->out.dpy, mask),
++ 0, 0,
++ &trap, 1);
++
++ clear(&t->ref, &ref);
++ XRenderCompositeTrapezoids(t->ref.dpy,
++ PictOpSrc,
++ src_ref,
++ ref.picture,
++ mask_format(t->ref.dpy, mask),
++ 0, 0,
++ &trap, 1);
++
++ test_compare(t,
++ out.draw, out.format,
++ ref.draw, ref.format,
++ 0, 0, out.width, out.height,
++ buf);
++ }
++ }
++
++ XRenderFreePicture(t->out.dpy, src_out);
++ test_target_destroy_render(&t->out, &out);
++
++ XRenderFreePicture(t->ref.dpy, src_ref);
++ test_target_destroy_render(&t->ref, &ref);
++
++ printf("pass\n");
++}
++
+ int main(int argc, char **argv)
+ {
+ struct test test;
+ int i, dx, dy;
+ enum target target;
+ enum mask mask;
++ enum edge edge;
+ enum trapezoid trapezoid;
+
+ test_init(&test, argc, argv);
+
++ for (target = TARGET_FIRST; target <= TARGET_LAST; target++) {
++ for (mask = MASK_NONE; mask <= MASK_A8; mask++)
++ for (edge = EDGE_SHARP; edge <= EDGE_SMOOTH; edge++)
++ edge_test(&test, mask, edge, target);
++ }
++
+ for (i = 0; i <= DEFAULT_ITERATIONS; i++) {
+ int reps = REPS(i), sets = SETS(i);
+
+diff --git a/test/render-triangle.c b/test/render-triangle.c
+new file mode 100644
+index 00000000..165834ce
+--- /dev/null
++++ b/test/render-triangle.c
+@@ -0,0 +1,180 @@
++#include <stdint.h>
++#include <stdio.h>
++#include <stdlib.h>
++
++#include "test.h"
++
++enum edge {
++ EDGE_SHARP = PolyEdgeSharp,
++ EDGE_SMOOTH,
++};
++
++static void set_edge(Display *dpy, Picture p, enum edge edge)
++{
++ XRenderPictureAttributes a;
++
++ a.poly_edge = edge;
++ XRenderChangePicture(dpy, p, CPPolyEdge, &a);
++}
++
++static XRenderPictFormat *mask_format(Display *dpy, enum mask mask)
++{
++ switch (mask) {
++ default:
++ case MASK_NONE: return NULL;
++ case MASK_A1: return XRenderFindStandardFormat(dpy, PictStandardA1);
++ case MASK_A8: return XRenderFindStandardFormat(dpy, PictStandardA8);
++ }
++}
++
++static const char *mask_name(enum mask mask)
++{
++ switch (mask) {
++ default:
++ case MASK_NONE: return "none";
++ case MASK_A1: return "a1";
++ case MASK_A8: return "a8";
++ }
++}
++
++static const char *edge_name(enum edge edge)
++{
++ switch (edge) {
++ default:
++ case EDGE_SHARP: return "sharp";
++ case EDGE_SMOOTH: return "smooth";
++ }
++}
++
++static void clear(struct test_display *dpy, struct test_target *tt)
++{
++ XRenderColor render_color = {0};
++ XRenderFillRectangle(dpy->dpy, PictOpClear, tt->picture, &render_color,
++ 0, 0, tt->width, tt->height);
++}
++
++static void step_to_point(int step, int width, int height, XPointFixed *p)
++{
++ do {
++ p->x = (step - 64) << 16;
++ p->y = -64 << 16;
++
++ step -= width - 128;
++ if (step <= 0)
++ return;
++
++ p->x = (width + 64) << 16;
++ p->y = (step - 64) << 16;
++ step -= height - 128;
++
++ if (step <= 0)
++ return;
++
++ p->x = (width + 64 - step) << 16;
++ p->y = (height + 64) << 16;
++ step -= width - 128;
++
++ if (step <= 0)
++ return;
++
++ p->x = -64 << 16;
++ p->y = (height + 64 - step) << 16;
++ step -= height - 128;
++ } while (step > 0);
++}
++
++static void edge_test(struct test *t,
++ enum mask mask,
++ enum edge edge,
++ enum target target)
++{
++ struct test_target out, ref;
++ XRenderColor white = { 0xffff, 0xffff, 0xffff, 0xffff };
++ Picture src_ref, src_out;
++ XTriangle tri;
++ unsigned step, max;
++
++ test_target_create_render(&t->out, target, &out);
++ set_edge(t->out.dpy, out.picture, edge);
++ src_out = XRenderCreateSolidFill(t->out.dpy, &white);
++
++ test_target_create_render(&t->ref, target, &ref);
++ set_edge(t->ref.dpy, ref.picture, edge);
++ src_ref = XRenderCreateSolidFill(t->ref.dpy, &white);
++
++ printf("Testing edges (with mask %s and %s edges) (%s): ",
++ mask_name(mask),
++ edge_name(edge),
++ test_target_name(target));
++ fflush(stdout);
++
++ max = 2*(out.width + 128 + out.height+128);
++ step = 0;
++ for (step = 0; step <= max; step++) {
++ char buf[80];
++
++ step_to_point(step, out.width, out.height, &tri.p1);
++ step_to_point(step + out.width + 128,
++ out.width, out.height,
++ &tri.p2);
++ step_to_point(step + out.height + 128 + 2*(out.width + 128),
++ out.width, out.height,
++ &tri.p3);
++
++ sprintf(buf,
++ "tri=((%d, %d), (%d, %d), (%d, %d))\n",
++ tri.p1.x >> 16, tri.p1.y >> 16,
++ tri.p2.x >> 16, tri.p2.y >> 16,
++ tri.p3.x >> 16, tri.p3.y >> 16);
++
++ clear(&t->out, &out);
++ XRenderCompositeTriangles(t->out.dpy,
++ PictOpSrc,
++ src_out,
++ out.picture,
++ mask_format(t->out.dpy, mask),
++ 0, 0,
++ &tri, 1);
++
++ clear(&t->ref, &ref);
++ XRenderCompositeTriangles(t->ref.dpy,
++ PictOpSrc,
++ src_ref,
++ ref.picture,
++ mask_format(t->ref.dpy, mask),
++ 0, 0,
++ &tri, 1);
++
++ test_compare(t,
++ out.draw, out.format,
++ ref.draw, ref.format,
++ 0, 0, out.width, out.height,
++ buf);
++ }
++
++ XRenderFreePicture(t->out.dpy, src_out);
++ test_target_destroy_render(&t->out, &out);
++
++ XRenderFreePicture(t->ref.dpy, src_ref);
++ test_target_destroy_render(&t->ref, &ref);
++
++ printf("pass\n");
++}
++
++int main(int argc, char **argv)
++{
++ struct test test;
++ enum target target;
++ enum mask mask;
++ enum edge edge;
++
++ test_init(&test, argc, argv);
++
++ for (target = TARGET_FIRST; target <= TARGET_LAST; target++) {
++ for (mask = MASK_NONE; mask <= MASK_A8; mask++)
++ for (edge = EDGE_SHARP; edge <= EDGE_SMOOTH; edge++)
++ edge_test(&test, mask, edge, target);
++ }
++
++ return 0;
++}
+diff --git a/test/test.h b/test/test.h
+index a3ef979d..9eec1cf9 100644
+--- a/test/test.h
++++ b/test/test.h
+@@ -107,6 +107,15 @@ static inline uint32_t color(uint8_t red, uint8_t green, uint8_t blue, uint8_t a
+ return alpha << 24 | ra >> 8 << 16 | ga >> 8 << 8 | ba >> 8;
+ }
+
++static inline uint32_t xrender_color(const XRenderColor *c)
++{
++ uint32_t ra = c->red * c->alpha;
++ uint32_t ga = c->green * c->alpha;
++ uint32_t ba = c->blue * c->alpha;
++
++ return c->alpha >> 8 << 24 | ra >> 24 << 16 | ga >> 24 << 8 | ba >> 24;
++}
++
+ void test_timer_start(struct test_display *t, struct timespec *tv);
+ double test_timer_stop(struct test_display *t, struct timespec *tv);
+
+diff --git a/test/test_image.c b/test/test_image.c
+index d15a8af8..1c076990 100644
+--- a/test/test_image.c
++++ b/test/test_image.c
+@@ -197,13 +197,10 @@ void test_compare(struct test *t,
+ const char *info)
+ {
+ XImage out_image, ref_image;
+- Pixmap tmp;
+- char *out, *ref;
++ uint32_t *out, *ref;
+ char buf[600];
+ uint32_t mask;
+ int i, j;
+- XGCValues gcv;
+- GC gc;
+
+ if (w * h * 4 > t->out.max_shm_size)
+ return test_compare_fallback(t,
+@@ -214,37 +211,24 @@ void test_compare(struct test *t,
+ test_init_image(&out_image, &t->out.shm, out_format, w, h);
+ test_init_image(&ref_image, &t->ref.shm, ref_format, w, h);
+
+- gcv.graphics_exposures = 0;
+-
+ die_unless(out_image.depth == ref_image.depth);
+ die_unless(out_image.bits_per_pixel == ref_image.bits_per_pixel);
+ die_unless(out_image.bits_per_pixel == 32);
+
+- mask = depth_mask(out_image.depth);
++ XShmGetImage(t->out.dpy, out_draw, &out_image, x, y, AllPlanes);
++ out = (uint32_t *)out_image.data;
+
+- tmp = XCreatePixmap(t->out.dpy, out_draw, w, h, out_image.depth);
+- gc = XCreateGC(t->out.dpy, tmp, GCGraphicsExposures, &gcv);
+- XCopyArea(t->out.dpy, out_draw, tmp, gc, x, y, w, h, 0, 0);
+- XShmGetImage(t->out.dpy, tmp, &out_image, 0, 0, AllPlanes);
+- XFreeGC(t->out.dpy, gc);
+- XFreePixmap(t->out.dpy, tmp);
+- out = out_image.data;
+-
+- tmp = XCreatePixmap(t->ref.dpy, ref_draw, w, h, ref_image.depth);
+- gc = XCreateGC(t->ref.dpy, tmp, GCGraphicsExposures, &gcv);
+- XCopyArea(t->ref.dpy, ref_draw, tmp, gc, x, y, w, h, 0, 0);
+- XShmGetImage(t->ref.dpy, tmp, &ref_image, 0, 0, AllPlanes);
+- XFreeGC(t->ref.dpy, gc);
+- XFreePixmap(t->ref.dpy, tmp);
+- ref = ref_image.data;
++ XShmGetImage(t->ref.dpy, ref_draw, &ref_image, x, y, AllPlanes);
++ ref = (uint32_t *)ref_image.data;
+
+ /* Start with an exact comparison. However, one quicky desires
+ * a fuzzy comparator to hide hardware inaccuracies...
+ */
++ mask = depth_mask(out_image.depth);
+ for (j = 0; j < h; j++) {
+ for (i = 0; i < w; i++) {
+- uint32_t a = ((uint32_t *)out)[i] & mask;
+- uint32_t b = ((uint32_t *)ref)[i] & mask;
++ uint32_t a = out[i] & mask;
++ uint32_t b = ref[i] & mask;
+ if (a != b && pixel_difference(a, b) > MAX_DELTA) {
+ show_pixels(buf,
+ &out_image, &ref_image,
+@@ -255,8 +239,8 @@ void test_compare(struct test *t,
+ x,i, y,j, a, b, pixel_difference(a, b), buf, info);
+ }
+ }
+- out += out_image.bytes_per_line;
+- ref += ref_image.bytes_per_line;
++ out = (uint32_t *)((char *)out + out_image.bytes_per_line);
++ ref = (uint32_t *)((char *)ref + ref_image.bytes_per_line);
+ }
+ }
+
+diff --git a/test/xvidmode.c b/test/xvidmode.c
+new file mode 100644
+index 00000000..5cde8286
+--- /dev/null
++++ b/test/xvidmode.c
+@@ -0,0 +1,54 @@
++#include <stdlib.h>
++#include <stdio.h>
++#include <string.h>
++#include <X11/Xlib.h>
++#include <X11/extensions/xf86vmode.h>
++
++int main(void)
++{
++ Display *dpy;
++ XF86VidModeModeLine current;
++ XF86VidModeModeInfo **modes;
++ int num_modes, i;
++ int saved_mode = -1;
++ int dotclock;
++
++ dpy = XOpenDisplay(NULL);
++ if (dpy == NULL)
++ dpy = XOpenDisplay(":0");
++
++ XF86VidModeGetModeLine(dpy, DefaultScreen(dpy), &dotclock, &current);
++ XF86VidModeGetAllModeLines(dpy, XDefaultScreen(dpy),
++ &num_modes, &modes);
++ for (i = 0; i < num_modes; i++) {
++ int this;
++
++ this = (current.hdisplay == modes[i]->hdisplay &&
++ current.vdisplay == modes[i]->vdisplay &&
++ dotclock == modes[i]->dotclock);
++ if (this && saved_mode == -1)
++ saved_mode = i;
++
++ printf("[%d] %dx%d%s\n",
++ i,
++ modes[i]->hdisplay,
++ modes[i]->vdisplay,
++ this ? "*" : "");
++ }
++
++ for (i = 0; i < num_modes; i++) {
++ printf("Switching to mode %dx%d\n",
++ modes[i]->hdisplay,
++ modes[i]->vdisplay);
++ XF86VidModeSwitchToMode(dpy, XDefaultScreen(dpy), modes[i]);
++ XSync(dpy, True);
++ }
++
++ if (saved_mode != -1) {
++ XF86VidModeSwitchToMode(dpy, XDefaultScreen(dpy),
++ modes[saved_mode]);
++ XFlush(dpy);
++ }
++
++ return 0;
++}
+diff --git a/tools/Makefile.am b/tools/Makefile.am
+index b5de2c96..92df266b 100644
+--- a/tools/Makefile.am
++++ b/tools/Makefile.am
+@@ -26,13 +26,30 @@ AM_CFLAGS = \
+ drivermandir = $(DRIVER_MAN_DIR)
+ policydir = $(datarootdir)/polkit-1/actions
+
++bin_PROGRAMS =
++noinst_PROGRAMS =
++libexec_PROGRAMS =
++
+ if BUILD_TOOLS
+-bin_PROGRAMS = intel-virtual-output
++bin_PROGRAMS += intel-virtual-output
+ driverman_DATA = intel-virtual-output.$(DRIVER_MAN_SUFFIX)
+ endif
+
++if BUILD_TOOL_CURSOR
++noinst_PROGRAMS += cursor
++cursor_CFLAGS = $(TOOL_CURSOR_CFLAGS)
++cursor_LDADD = $(TOOL_CURSOR_LIBS)
++endif
++
++if X11_DRI3
++noinst_PROGRAMS += dri3info
++dri3info_SOURCES = dri3info.c
++dri3info_CFLAGS = $(X11_DRI3_CFLAGS) $(DRI_CFLAGS)
++dri3info_LDADD = $(X11_DRI3_LIBS) $(DRI_LIBS)
++endif
++
+ if BUILD_BACKLIGHT_HELPER
+-libexec_PROGRAMS = xf86-video-intel-backlight-helper
++libexec_PROGRAMS += xf86-video-intel-backlight-helper
+ nodist_policy_DATA = org.x.xf86-video-intel.backlight-helper.policy
+
+ backlight_helper = $(libexecdir)/xf86-video-intel-backlight-helper
+diff --git a/tools/backlight_helper.c b/tools/backlight_helper.c
+index 8b2667dc..aadb8fac 100644
+--- a/tools/backlight_helper.c
++++ b/tools/backlight_helper.c
+@@ -1,3 +1,7 @@
++#ifdef HAVE_CONFIG_H
++#include "config.h"
++#endif
++
+ #include <stdio.h>
+ #include <string.h>
+ #include <stdarg.h>
+@@ -9,6 +13,12 @@
+ #include <sys/types.h>
+ #include <sys/stat.h>
+
++#if MAJOR_IN_MKDEV
++#include <sys/mkdev.h>
++#elif MAJOR_IN_SYSMACROS
++#include <sys/sysmacros.h>
++#endif
++
+ #define DBG 0
+
+ #if defined(__GNUC__) && (__GNUC__ > 3)
+diff --git a/tools/cursor.c b/tools/cursor.c
+new file mode 100644
+index 00000000..6a2438ad
+--- /dev/null
++++ b/tools/cursor.c
+@@ -0,0 +1,127 @@
++/*
++ * Copyright © 2015 Intel Corporation
++ *
++ * Permission is hereby granted, free of charge, to any person obtaining a
++ * copy of this software and associated documentation files (the "Software"),
++ * to deal in the Software without restriction, including without limitation
++ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
++ * and/or sell copies of the Software, and to permit persons to whom the
++ * Software is furnished to do so, subject to the following conditions:
++ *
++ * The above copyright notice and this permission notice (including the next
++ * paragraph) shall be included in all copies or substantial portions of the
++ * Software.
++ *
++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
++ * IN THE SOFTWARE.
++ *
++ */
++
++#ifdef HAVE_CONFIG_H
++#include "config.h"
++#endif
++
++#include <X11/Xlib.h>
++#include <X11/extensions/Xfixes.h>
++
++#include <stdint.h>
++#include <stdio.h>
++#include <stdlib.h>
++#include <png.h>
++
++int main(int argc, char **argv)
++{
++ Display *dpy;
++ XFixesCursorImage *cur;
++ unsigned long *src; /* XXX deep sigh */
++ unsigned x, y;
++ png_struct *png;
++ png_info *info;
++ png_byte **rows;
++ FILE *file;
++
++ dpy = XOpenDisplay(NULL);
++ if (dpy == NULL)
++ return 1;
++
++ if (!XFixesQueryExtension(dpy, (int *)&x, (int *)&y))
++ return 1;
++
++ cur = XFixesGetCursorImage(dpy);
++ if (cur == NULL)
++ return 1;
++
++ printf("Cursor on display '%s': %dx%d, (hotspot %dx%d)\n",
++ DisplayString(dpy),
++ cur->width, cur->height,
++ cur->xhot, cur->yhot);
++
++ if (1) {
++ int x, y;
++
++ src = cur->pixels;
++ for (y = 0; y < cur->height; y++) {
++ for (x = 0; x < cur->width; x++) {
++ if (x == cur->xhot && y == cur->yhot)
++ printf("+");
++ else
++ printf("%c", *src ? *src >> 24 >= 127 ? 'x' : '.' : ' ');
++ src++;
++ }
++ printf("\n");
++ }
++ }
++
++ file = fopen("cursor.png", "wb");
++ if (file == NULL)
++ return 2;
++
++ png = png_create_write_struct(PNG_LIBPNG_VER_STRING, NULL, NULL, NULL);
++ info = png_create_info_struct(png);
++ png_init_io(png, file);
++ png_set_IHDR(png, info,
++ cur->width, cur->height, 8,
++ PNG_COLOR_TYPE_RGB_ALPHA,
++ PNG_INTERLACE_NONE,
++ PNG_COMPRESSION_TYPE_DEFAULT,
++ PNG_FILTER_TYPE_DEFAULT);
++ png_write_info(png, info);
++
++ src = cur->pixels;
++ rows = malloc(cur->height*sizeof(png_byte*));
++ if (rows == NULL)
++ return 3;
++
++ for (y = 0; y < cur->height; y++) {
++ rows[y] = malloc(cur->width * 4);
++ for (x = 0; x < cur->width; x++) {
++ uint32_t p = *src++;
++ uint8_t r = p >> 0;
++ uint8_t g = p >> 8;
++ uint8_t b = p >> 16;
++ uint8_t a = p >> 24;
++
++ if (a > 0x00 && a < 0xff) {
++ r = (r * 0xff + a /2) / a;
++ g = (g * 0xff + a /2) / a;
++ b = (b * 0xff + a /2) / a;
++ }
++
++ rows[y][4*x + 0] = b;
++ rows[y][4*x + 1] = g;
++ rows[y][4*x + 2] = r;
++ rows[y][4*x + 3] = a;
++ }
++ }
++
++ png_write_image(png, rows);
++ png_write_end(png, NULL);
++ fclose(file);
++
++ return 0;
++}
+diff --git a/tools/dri3info.c b/tools/dri3info.c
+new file mode 100644
+index 00000000..0c33fc5a
+--- /dev/null
++++ b/tools/dri3info.c
+@@ -0,0 +1,329 @@
++/*
++ * Copyright (c) 2015 Intel Corporation
++ *
++ * Permission is hereby granted, free of charge, to any person obtaining a
++ * copy of this software and associated documentation files (the "Software"),
++ * to deal in the Software without restriction, including without limitation
++ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
++ * and/or sell copies of the Software, and to permit persons to whom the
++ * Software is furnished to do so, subject to the following conditions:
++ *
++ * The above copyright notice and this permission notice (including the next
++ * paragraph) shall be included in all copies or substantial portions of the
++ * Software.
++ *
++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
++ * SOFTWARE.
++ *
++ * To compile standalone: gcc -o dri3info dri3info.c `pkg-config --cflags --libs xcb-dri3 x11-xcb xrandr xxf86vm libdrm`
++ */
++
++#include <X11/Xlib.h>
++#include <X11/Xlib-xcb.h>
++#include <xcb/xcb.h>
++#include <xcb/dri3.h>
++#include <unistd.h>
++#include <stdio.h>
++#include <stdlib.h>
++#include <stdint.h>
++#include <string.h>
++#include <sys/stat.h>
++#include <drm.h>
++#include <xf86drm.h>
++
++#include <X11/extensions/Xrandr.h>
++#include <X11/extensions/xf86vmode.h>
++
++static int dri3_query_version(Display *dpy, int *major, int *minor)
++{
++ xcb_connection_t *c = XGetXCBConnection(dpy);
++ xcb_dri3_query_version_reply_t *reply;
++ xcb_generic_error_t *error;
++
++ *major = *minor = -1;
++
++ reply = xcb_dri3_query_version_reply(c,
++ xcb_dri3_query_version(c,
++ XCB_DRI3_MAJOR_VERSION,
++ XCB_DRI3_MINOR_VERSION),
++ &error);
++ free(error);
++ if (reply == NULL)
++ return -1;
++
++ *major = reply->major_version;
++ *minor = reply->minor_version;
++ free(reply);
++
++ return 0;
++}
++
++static int dri3_exists(Display *dpy)
++{
++ const xcb_query_extension_reply_t *ext;
++ int major, minor;
++
++ ext = xcb_get_extension_data(XGetXCBConnection(dpy), &xcb_dri3_id);
++ if (ext == NULL || !ext->present)
++ return 0;
++
++ if (dri3_query_version(dpy, &major, &minor) < 0)
++ return 0;
++
++ return major >= 0;
++}
++
++static int dri3_open(Display *dpy)
++{
++ xcb_connection_t *c = XGetXCBConnection(dpy);
++ xcb_dri3_open_cookie_t cookie;
++ xcb_dri3_open_reply_t *reply;
++
++ if (!dri3_exists(dpy))
++ return -1;
++
++ cookie = xcb_dri3_open(c, RootWindow(dpy, DefaultScreen(dpy)), None);
++ reply = xcb_dri3_open_reply(c, cookie, NULL);
++
++ if (!reply)
++ return -1;
++
++ if (reply->nfd != 1)
++ return -1;
++
++ return xcb_dri3_open_reply_fds(c, reply)[0];
++}
++
++static void get_device_path(int fd, char *buf, int len)
++{
++ struct stat remote, local;
++ int i;
++
++ if (fstat(fd, &remote))
++ goto out;
++
++ for (i = 0; i < 16; i++) {
++ snprintf(buf, len, "/dev/dri/card%d", i);
++ if (stat(buf, &local))
++ continue;
++
++ if (local.st_mode == remote.st_mode &&
++ local.st_rdev == remote.st_rdev)
++ return;
++
++ snprintf(buf, len, "/dev/dri/renderD%d", i + 128);
++ if (stat(buf, &local))
++ continue;
++
++ if (local.st_mode == remote.st_mode &&
++ local.st_rdev == remote.st_rdev)
++ return;
++ }
++
++out:
++ strncpy(buf, "unknown path", len);
++}
++
++static void get_driver_name(int fd, char *name, int len)
++{
++ drm_version_t version;
++
++ memset(name, 0, len);
++ memset(&version, 0, sizeof(version));
++ version.name_len = len;
++ version.name = name;
++
++ (void)drmIoctl(fd, DRM_IOCTL_VERSION, &version);
++}
++
++static int compute_refresh_rate_from_mode(long n, long d, unsigned flags,
++ int32_t *numerator,
++ int32_t *denominator)
++{
++ int i;
++
++ /* The mode flags are only defined privately to the Xserver (in xf86str.h)
++ * but they at least bit compatible between VidMode, RandR and DRM.
++ */
++# define V_INTERLACE 0x010
++# define V_DBLSCAN 0x020
++
++ if (flags & V_INTERLACE)
++ n *= 2;
++ else if (flags & V_DBLSCAN)
++ d *= 2;
++
++ /* The OML_sync_control spec requires that if the refresh rate is a
++ * whole number, that the returned numerator be equal to the refresh
++ * rate and the denominator be 1.
++ */
++
++ if (n % d == 0) {
++ n /= d;
++ d = 1;
++ }
++ else {
++ static const unsigned f[] = { 13, 11, 7, 5, 3, 2, 0 };
++
++ /* This is a poor man's way to reduce a fraction. It's far from
++ * perfect, but it will work well enough for this situation.
++ */
++
++ for (i = 0; f[i] != 0; i++) {
++ while (n % f[i] == 0 && d % f[i] == 0) {
++ d /= f[i];
++ n /= f[i];
++ }
++ }
++ }
++
++ *numerator = n;
++ *denominator = d;
++ return 1;
++}
++
++static int RRGetMscRate(Display *dpy, int32_t *numerator, int32_t *denominator)
++{
++ int ret = 0;
++ Window root = RootWindow(dpy, DefaultScreen(dpy));
++ XRRScreenResources *res;
++ int rr_event, rr_error;
++ RROutput primary;
++ RRMode mode = 0;
++ int n;
++
++ if (!XRRQueryExtension(dpy, &rr_event, &rr_error))
++ return ret;
++
++ res = XRRGetScreenResourcesCurrent(dpy, root);
++ if (res == NULL)
++ return ret;
++
++ /* Use the primary output if specified, otherwise
++ * use the mode on the first enabled crtc.
++ */
++ primary = XRRGetOutputPrimary(dpy, root);
++ if (primary) {
++ XRROutputInfo *output;
++
++ output = XRRGetOutputInfo(dpy, res, primary);
++ if (output != NULL) {
++ if (output->crtc) {
++ XRRCrtcInfo *crtc;
++
++ crtc = XRRGetCrtcInfo(dpy, res, output->crtc);
++ if (crtc) {
++ mode = crtc->mode;
++ XRRFreeCrtcInfo(crtc);
++ }
++ }
++ XRRFreeOutputInfo(output);
++ }
++ }
++
++ for (n = 0; mode == 0 && n < res->ncrtc; n++) {
++ XRRCrtcInfo *crtc;
++
++ crtc = XRRGetCrtcInfo(dpy, res, res->crtcs[n]);
++ if (crtc) {
++ mode = crtc->mode;
++ XRRFreeCrtcInfo(crtc);
++ }
++ }
++
++ for (n = 0; n < res->nmode; n++) {
++ if (res->modes[n].id == mode) {
++ ret = compute_refresh_rate_from_mode(res->modes[n].dotClock,
++ res->modes[n].hTotal*res->modes[n].vTotal,
++ res->modes[n].modeFlags,
++ numerator, denominator);
++ break;
++ }
++ }
++
++ XRRFreeScreenResources(res);
++ return ret;
++}
++
++static int VMGetMscRate(Display *dpy, int32_t *numerator, int32_t *denominator)
++{
++ XF86VidModeModeLine mode_line;
++ int dot_clock;
++ int i;
++
++ if (XF86VidModeQueryVersion(dpy, &i, &i) &&
++ XF86VidModeGetModeLine(dpy, DefaultScreen(dpy), &dot_clock, &mode_line))
++ return compute_refresh_rate_from_mode(dot_clock * 1000,
++ mode_line.vtotal * mode_line.htotal,
++ mode_line.flags,
++ numerator, denominator);
++
++ return 0;
++}
++
++static int get_refresh_rate(Display *dpy,
++ int32_t *numerator,
++ int32_t *denominator)
++{
++ if (RRGetMscRate(dpy, numerator, denominator))
++ return 1;
++
++ if (VMGetMscRate(dpy, numerator, denominator))
++ return 1;
++
++ return 0;
++}
++
++static void info(const char *dpyname)
++{
++ Display *dpy;
++ int device;
++ int32_t numerator, denominator;
++
++ dpy = XOpenDisplay(dpyname);
++ if (dpy == NULL) {
++ printf("Unable to connect to display '%s'\n",
++ dpyname ?: getenv("DISPLAY") ?: "unset");
++ return;
++ }
++
++ printf("Display '%s'\n", DisplayString(dpy));
++ device = dri3_open(dpy);
++ if (device < 0) {
++ printf("\tUnable to connect to DRI3\n");
++ } else {
++ char device_path[1024];
++ char driver_name[1024];
++
++ get_device_path(device, device_path, sizeof(device_path));
++ get_driver_name(device, driver_name, sizeof(driver_name));
++
++ printf("Connected to DRI3, using fd %d which matches %s, driver %s\n",
++ device, device_path, driver_name);
++ close(device);
++ }
++
++ if (get_refresh_rate(dpy, &numerator, &denominator))
++ printf("\tPrimary refresh rate: %d/%d (%.1fHz)\n",
++ numerator, denominator, numerator/(float)denominator);
++
++ XCloseDisplay(dpy);
++}
++
++int main(int argc, char **argv)
++{
++ int i;
++
++ if (argc > 1) {
++ for (i = 1; i < argc; i++)
++ info(argv[i]);
++ } else
++ info(NULL);
++
++ return 0;
++}
+diff --git a/tools/virtual.c b/tools/virtual.c
+index 8e2b4a22..fc8db2b9 100644
+--- a/tools/virtual.c
++++ b/tools/virtual.c
+@@ -31,6 +31,7 @@
+
+ #include <X11/Xlibint.h>
+ #include <X11/extensions/record.h>
++#include <X11/extensions/scrnsaver.h>
+ #include <X11/extensions/XShm.h>
+ #if HAVE_X11_EXTENSIONS_SHMPROTO_H
+ #include <X11/extensions/shmproto.h>
+@@ -79,13 +80,15 @@ static int verbose;
+ #define DRAW 0x8
+ #define DAMAGE 0x10
+ #define CURSOR 0x20
+-#define POLL 0x40
++#define SCREEN 0x40
++#define POLL 0x80
+
+ struct display {
+ Display *dpy;
+ struct clone *clone;
+ struct context *ctx;
+
++ int saver_event, saver_error, saver_active;
+ int damage_event, damage_error;
+ int xfixes_event, xfixes_error;
+ int rr_event, rr_error, rr_active;
+@@ -98,6 +101,7 @@ struct display {
+ int width;
+ int height;
+ int depth;
++ int active;
+
+ XRenderPictFormat *root_format;
+ XRenderPictFormat *rgb16_format;
+@@ -111,7 +115,7 @@ struct display {
+ Cursor invisible_cursor;
+ Cursor visible_cursor;
+
+- XcursorImage cursor_image;
++ XcursorImage cursor_image; /* first only */
+ int cursor_serial;
+ int cursor_x;
+ int cursor_y;
+@@ -123,6 +127,13 @@ struct display {
+ int send;
+ int skip_clone;
+ int skip_frame;
++
++ struct {
++ int timeout;
++ int interval;
++ int prefer_blank;
++ int allow_exp;
++ } saver;
+ };
+
+ struct output {
+@@ -145,6 +156,7 @@ struct output {
+ XRenderPictFormat *use_render;
+
+ int x, y;
++ int width, height;
+ XRRModeInfo mode;
+ Rotation rotation;
+ };
+@@ -218,6 +230,13 @@ static inline XRRScreenResources *_XRRGetScreenResourcesCurrent(Display *dpy, Wi
+ static int _x_error_occurred;
+
+ static int
++_io_error_handler(Display *display)
++{
++ fprintf(stderr, "XIO error on display %s\n", DisplayString(display));
++ abort();
++}
++
++static int
+ _check_error_handler(Display *display,
+ XErrorEvent *event)
+ {
+@@ -243,6 +262,10 @@ can_use_shm(Display *dpy,
+ XExtCodes *codes;
+ int major, minor, has_shm, has_pixmap;
+
++ *shm_event = 0;
++ *shm_opcode = 0;
++ *shm_pixmap = 0;
++
+ if (!XShmQueryExtension(dpy))
+ return 0;
+
+@@ -320,6 +343,7 @@ can_use_shm(Display *dpy,
+ #include <X11/Xlib-xcb.h>
+ #include <X11/xshmfence.h>
+ #include <xcb/xcb.h>
++#include <xcb/xcbext.h>
+ #include <xcb/dri3.h>
+ #include <xcb/sync.h>
+ static Pixmap dri3_create_pixmap(Display *dpy,
+@@ -357,6 +381,7 @@ static int dri3_query_version(Display *dpy, int *major, int *minor)
+ {
+ xcb_connection_t *c = XGetXCBConnection(dpy);
+ xcb_dri3_query_version_reply_t *reply;
++ xcb_generic_error_t *error;
+
+ *major = *minor = -1;
+
+@@ -364,7 +389,8 @@ static int dri3_query_version(Display *dpy, int *major, int *minor)
+ xcb_dri3_query_version(c,
+ XCB_DRI3_MAJOR_VERSION,
+ XCB_DRI3_MINOR_VERSION),
+- NULL);
++ &error);
++ free(error);
+ if (reply == NULL)
+ return -1;
+
+@@ -377,8 +403,13 @@ static int dri3_query_version(Display *dpy, int *major, int *minor)
+
+ static int dri3_exists(Display *dpy)
+ {
++ const xcb_query_extension_reply_t *ext;
+ int major, minor;
+
++ ext = xcb_get_extension_data(XGetXCBConnection(dpy), &xcb_dri3_id);
++ if (ext == NULL || !ext->present)
++ return 0;
++
+ if (dri3_query_version(dpy, &major, &minor) < 0)
+ return 0;
+
+@@ -809,6 +840,10 @@ static int clone_update_modes__fixed(struct clone *clone)
+ RRMode id;
+ int i, j, ret = ENOENT;
+
++ DBG(X11, ("%s-%s cloning modes fixed %dx%d\n",
++ DisplayString(clone->dst.dpy), clone->dst.name,
++ clone->dst.width, clone->dst.height));
++
+ assert(clone->src.rr_output);
+
+ res = _XRRGetScreenResourcesCurrent(clone->src.dpy, clone->src.window);
+@@ -837,8 +872,8 @@ static int clone_update_modes__fixed(struct clone *clone)
+
+ /* Create matching mode for the real output on the virtual */
+ memset(&mode, 0, sizeof(mode));
+- mode.width = clone->width;
+- mode.height = clone->height;
++ mode.width = clone->dst.width;
++ mode.height = clone->dst.height;
+ mode.nameLength = sprintf(mode_name, "FAKE-%dx%d", mode.width, mode.height);
+ mode.name = mode_name;
+
+@@ -942,6 +977,35 @@ out:
+ return rr_output;
+ }
+
++static int check_virtual(struct display *display)
++{
++ XRRScreenResources *res;
++ int found = -ENOENT;
++ int i;
++
++ res = _XRRGetScreenResourcesCurrent(display->dpy, display->root);
++ if (res == NULL)
++ return -ENOMEM;
++
++ for (i = 0; found == -ENOENT && i < res->noutput; i++) {
++ XRROutputInfo *output;
++
++ output = XRRGetOutputInfo(display->dpy, res, res->outputs[i]);
++ if (output == NULL)
++ continue;
++
++ if (strcmp(output->name, "VIRTUAL1") == 0)
++ found = 0;
++
++ XRRFreeOutputInfo(output);
++ }
++ XRRFreeScreenResources(res);
++
++ DBG(XRR, ("%s(%s): has VIRTUAL1? %d\n",
++ __func__, DisplayString(display->dpy), found));
++ return found;
++}
++
+ static int stride_for_depth(int width, int depth)
+ {
+ if (depth == 24)
+@@ -1082,20 +1146,20 @@ static int clone_init_xfer(struct clone *clone)
+ width = 0;
+ height = 0;
+ } else if (clone->dri3.xid) {
+- width = clone->dst.display->width;
+- height = clone->dst.display->height;
++ width = clone->dst.width;
++ height = clone->dst.height;
+ } else {
+ width = mode_width(&clone->src.mode, clone->src.rotation);
+ height = mode_height(&clone->src.mode, clone->src.rotation);
+ }
+
++ DBG(DRAW, ("%s-%s create xfer, %dx%d (currently %dx%d)\n",
++ DisplayString(clone->dst.dpy), clone->dst.name,
++ width, height, clone->width, clone->height));
++
+ if (width == clone->width && height == clone->height)
+ return 0;
+
+- DBG(DRAW, ("%s-%s create xfer, %dx%d\n",
+- DisplayString(clone->dst.dpy), clone->dst.name,
+- width, height));
+-
+ if (clone->shm.shmaddr) {
+ if (clone->src.use_shm)
+ XShmDetach(clone->src.dpy, &clone->src.shm);
+@@ -1225,6 +1289,56 @@ static void clone_update(struct clone *clone)
+ clone->rr_update = 0;
+ }
+
++static void screensaver_save(struct display *display)
++{
++ display->saver_active =
++ XScreenSaverQueryExtension(display->dpy,
++ &display->saver_event,
++ &display->saver_error);
++ DBG(SCREEN,
++ ("%s screen saver active? %d [event=%d, error=%d]\n",
++ DisplayString(display->dpy),
++ display->saver_active,
++ display->saver_event,
++ display->saver_error));
++
++ XGetScreenSaver(display->dpy,
++ &display->saver.timeout,
++ &display->saver.interval,
++ &display->saver.prefer_blank,
++ &display->saver.allow_exp);
++
++ DBG(SCREEN,
++ ("%s saving screen saver defaults: timeout=%d interval=%d prefer_blank=%d allow_exp=%d\n",
++ DisplayString(display->dpy),
++ display->saver.timeout,
++ display->saver.interval,
++ display->saver.prefer_blank,
++ display->saver.allow_exp));
++}
++
++static void screensaver_disable(struct display *display)
++{
++ DBG(SCREEN,
++ ("%s disabling screen saver\n", DisplayString(display->dpy)));
++
++ XSetScreenSaver(display->dpy, 0, 0, DefaultBlanking, DefaultExposures);
++ display_mark_flush(display);
++}
++
++static void screensaver_restore(struct display *display)
++{
++ DBG(SCREEN,
++ ("%s restoring screen saver\n", DisplayString(display->dpy)));
++
++ XSetScreenSaver(display->dpy,
++ display->saver.timeout,
++ display->saver.interval,
++ display->saver.prefer_blank,
++ display->saver.allow_exp);
++ display_mark_flush(display);
++}
++
+ static int context_update(struct context *ctx)
+ {
+ Display *dpy = ctx->display->dpy;
+@@ -1325,8 +1439,19 @@ static int context_update(struct context *ctx)
+ struct clone *clone;
+ int x1, x2, y1, y2;
+
+- if (display->rr_active == 0)
++ if (display->rr_active == 0) {
++ for (clone = display->clone; clone; clone = clone->next) {
++ struct output *output = &clone->src;
++ if (output->mode.id) {
++ clone->dst.mode.id = -1;
++ clone->dst.rr_crtc = -1;
++ } else {
++ clone->dst.mode.id = 0;
++ clone->dst.rr_crtc = 0;
++ }
++ }
+ continue;
++ }
+
+ x1 = y1 = INT_MAX;
+ x2 = y2 = INT_MIN;
+@@ -1570,6 +1695,13 @@ ungrab:
+ XUngrabServer(display->dpy);
+ }
+
++ for (n = 1; n < ctx->ndisplay; n++) {
++ struct display *display = &ctx->display[n];
++
++ display->active = 0;
++ screensaver_restore(display);
++ }
++
+ ctx->active = NULL;
+ for (n = 0; n < ctx->nclone; n++) {
+ struct clone *clone = &ctx->clones[n];
+@@ -1580,7 +1712,10 @@ ungrab:
+ continue;
+
+ DBG(XRR, ("%s-%s: added to active list\n",
+- DisplayString(clone->dst.display->dpy), clone->dst.name));
++ DisplayString(clone->dst.display->dpy), clone->dst.name));
++
++ if (clone->dst.display->active++ == 0)
++ screensaver_disable(clone->dst.display);
+
+ clone->active = ctx->active;
+ ctx->active = clone;
+@@ -1599,14 +1734,17 @@ static Cursor display_load_invisible_cursor(struct display *display)
+
+ static Cursor display_get_visible_cursor(struct display *display)
+ {
+- if (display->cursor_serial != display->cursor_image.size) {
+- DBG(CURSOR, ("%s updating cursor\n", DisplayString(display->dpy)));
++ struct display *first = display->ctx->display;
++
++ if (display->cursor_serial != first->cursor_serial) {
++ DBG(CURSOR, ("%s updating cursor %dx%d, serial %d\n",
++ DisplayString(display->dpy), first->cursor_image.width, first->cursor_image.height, first->cursor_serial));
+
+ if (display->visible_cursor)
+ XFreeCursor(display->dpy, display->visible_cursor);
+
+- display->visible_cursor = XcursorImageLoadCursor(display->dpy, &display->cursor_image);
+- display->cursor_serial = display->cursor_image.size;
++ display->visible_cursor = XcursorImageLoadCursor(display->dpy, &first->cursor_image);
++ display->cursor_serial = first->cursor_serial;
+ }
+
+ return display->visible_cursor;
+@@ -1629,7 +1767,7 @@ static void display_load_visible_cursor(struct display *display, XFixesCursorIma
+ display->cursor_image.height = cur->height;
+ display->cursor_image.xhot = cur->xhot;
+ display->cursor_image.yhot = cur->yhot;
+- display->cursor_image.size++;
++ display->cursor_serial++;
+
+ n = cur->width*cur->height;
+ src = cur->pixels;
+@@ -1637,11 +1775,24 @@ static void display_load_visible_cursor(struct display *display, XFixesCursorIma
+ while (n--)
+ *dst++ = *src++;
+
+- DBG(CURSOR, ("%s marking cursor changed\n", DisplayString(display->dpy)));
+- display->cursor_moved++;
+- if (display->cursor != display->invisible_cursor) {
+- display->cursor_visible++;
+- context_enable_timer(display->ctx);
++ if (verbose & CURSOR) {
++ int x, y;
++
++ printf("%s cursor image %dx%d, serial %d:\n",
++ DisplayString(display->dpy),
++ cur->width, cur->height,
++ display->cursor_serial);
++ dst = display->cursor_image.pixels;
++ for (y = 0; y < cur->height; y++) {
++ for (x = 0; x < cur->width; x++) {
++ if (x == cur->xhot && y == cur->yhot)
++ printf("+");
++ else
++ printf("%c", *dst ? *dst >> 24 >= 127 ? 'x' : '.' : ' ');
++ dst++;
++ }
++ printf("\n");
++ }
+ }
+ }
+
+@@ -1685,6 +1836,8 @@ static void display_flush_cursor(struct display *display)
+ if (cursor == None)
+ cursor = display->invisible_cursor;
+ if (cursor != display->cursor) {
++ DBG(CURSOR, ("%s setting cursor shape %lx\n",
++ DisplayString(display->dpy), (long)cursor));
+ XDefineCursor(display->dpy, display->root, cursor);
+ display->cursor = cursor;
+ }
+@@ -1762,6 +1915,8 @@ static void get_src(struct clone *c, const XRectangle *clip)
+ c->image.obdata = (char *)&c->src.shm;
+
+ if (c->src.use_render) {
++ DBG(DRAW, ("%s-%s get_src via XRender\n",
++ DisplayString(c->dst.dpy), c->dst.name));
+ XRenderComposite(c->src.dpy, PictOpSrc,
+ c->src.win_picture, 0, c->src.pix_picture,
+ clip->x, clip->y,
+@@ -1782,16 +1937,22 @@ static void get_src(struct clone *c, const XRectangle *clip)
+ &c->image, 0, 0);
+ }
+ } else if (c->src.pixmap) {
++ DBG(DRAW, ("%s-%s get_src XCopyArea (SHM/DRI3)\n",
++ DisplayString(c->dst.dpy), c->dst.name));
+ XCopyArea(c->src.dpy, c->src.window, c->src.pixmap, c->src.gc,
+ clip->x, clip->y,
+ clip->width, clip->height,
+ 0, 0);
+ XSync(c->src.dpy, False);
+ } else if (c->src.use_shm) {
++ DBG(DRAW, ("%s-%s get_src XShmGetImage\n",
++ DisplayString(c->dst.dpy), c->dst.name));
+ ximage_prepare(&c->image, clip->width, clip->height);
+ XShmGetImage(c->src.dpy, c->src.window, &c->image,
+ clip->x, clip->y, AllPlanes);
+ } else {
++ DBG(DRAW, ("%s-%s get_src XGetSubImage (slow)\n",
++ DisplayString(c->dst.dpy), c->dst.name));
+ ximage_prepare(&c->image, c->width, c->height);
+ XGetSubImage(c->src.dpy, c->src.window,
+ clip->x, clip->y, clip->width, clip->height,
+@@ -1838,7 +1999,7 @@ static void put_dst(struct clone *c, const XRectangle *clip)
+ clip->width, clip->height);
+ c->dst.display->send |= c->dst.use_shm;
+ } else if (c->dst.pixmap) {
+- DBG(DRAW, ("%s-%s using SHM pixmap\n",
++ DBG(DRAW, ("%s-%s using SHM or DRI3 pixmap\n",
+ DisplayString(c->dst.dpy), c->dst.name));
+ c->dst.serial = NextRequest(c->dst.dpy);
+ XCopyArea(c->dst.dpy, c->dst.pixmap, c->dst.window, c->dst.gc,
+@@ -1870,6 +2031,9 @@ static int clone_paint(struct clone *c)
+ {
+ XRectangle clip;
+
++ if (c->width == 0 || c->height == 0)
++ return 0;
++
+ DBG(DRAW, ("%s-%s paint clone, damaged (%d, %d), (%d, %d) [(%d, %d), (%d, %d)]\n",
+ DisplayString(c->dst.dpy), c->dst.name,
+ c->damaged.x1, c->damaged.y1,
+@@ -1944,6 +2108,10 @@ static int clone_paint(struct clone *c)
+ clip.height = c->damaged.y2 - c->damaged.y1;
+ get_src(c, &clip);
+
++ DBG(DRAW, ("%s-%s target offset %dx%d\n",
++ DisplayString(c->dst.dpy), c->dst.name,
++ c->dst.x - c->src.x, c->dst.y - c->src.y));
++
+ clip.x += c->dst.x - c->src.x;
+ clip.y += c->dst.y - c->src.y;
+ put_dst(c, &clip);
+@@ -1969,8 +2137,9 @@ static void clone_damage(struct clone *c, const XRectangle *rec)
+ if ((v = (int)rec->y + rec->height) > c->damaged.y2)
+ c->damaged.y2 = v;
+
+- DBG(DAMAGE, ("%s-%s damaged: (%d, %d), (%d, %d)\n",
++ DBG(DAMAGE, ("%s-%s damaged: +(%d,%d)x(%d, %d) -> (%d, %d), (%d, %d)\n",
+ DisplayString(c->dst.display->dpy), c->dst.name,
++ rec->x, rec->y, rec->width, rec->height,
+ c->damaged.x1, c->damaged.y1,
+ c->damaged.x2, c->damaged.y2));
+ }
+@@ -2252,6 +2421,8 @@ static int clone_init_depth(struct clone *clone)
+ if (ret)
+ return ret;
+
++ clone->depth = depth;
++
+ DBG(X11, ("%s-%s using depth %d, requires xrender for src? %d, for dst? %d\n",
+ DisplayString(clone->dst.dpy), clone->dst.name,
+ clone->depth,
+@@ -2312,6 +2483,8 @@ static int add_display(struct context *ctx, Display *dpy)
+ display->depth = DefaultDepth(dpy, DefaultScreen(dpy));
+ display->visual = DefaultVisual(dpy, DefaultScreen(dpy));
+
++ XSelectInput(dpy, display->root, ExposureMask);
++
+ display->has_shm = can_use_shm(dpy, display->root,
+ &display->shm_event,
+ &display->shm_opcode,
+@@ -2323,6 +2496,8 @@ static int add_display(struct context *ctx, Display *dpy)
+ display->shm_opcode,
+ display->has_shm_pixmap));
+
++ screensaver_save(display);
++
+ display->rr_active = XRRQueryExtension(dpy, &display->rr_event, &display->rr_error);
+ DBG(X11, ("%s: randr_active?=%d, event=%d, error=%d\n",
+ DisplayString(dpy),
+@@ -2592,6 +2767,11 @@ static int last_display_add_clones__randr(struct context *ctx)
+ return ret;
+ }
+
++ clone->dst.x = 0;
++ clone->dst.y = 0;
++ clone->dst.width = display->width;
++ clone->dst.height = display->height;
++
+ ret = clone_update_modes__randr(clone);
+ if (ret) {
+ fprintf(stderr, "Failed to clone output \"%s\" from display \"%s\"\n",
+@@ -2668,8 +2848,8 @@ static int last_display_add_clones__xinerama(struct context *ctx)
+ }
+
+ /* Replace the modes on the local VIRTUAL output with the remote Screen */
+- clone->width = xi[n].width;
+- clone->height = xi[n].height;
++ clone->dst.width = xi[n].width;
++ clone->dst.height = xi[n].height;
+ clone->dst.x = xi[n].x_org;
+ clone->dst.y = xi[n].y_org;
+ clone->dst.rr_crtc = -1;
+@@ -2698,64 +2878,67 @@ static int last_display_add_clones__display(struct context *ctx)
+ Display *dpy = display->dpy;
+ struct clone *clone;
+ Screen *scr;
++ int count, s;
+ char buf[80];
+ int ret;
+ RROutput id;
+
++ count = ScreenCount(dpy);
++ DBG(X11, ("%s(%s) - %d screens\n", __func__, DisplayString(dpy), count));
++ for (s = 0; s < count; s++) {
++ clone = add_clone(ctx);
++ if (clone == NULL)
++ return -ENOMEM;
+
+- DBG(X11, ("%s(%s)\n", __func__, DisplayString(dpy)));
+- clone = add_clone(ctx);
+- if (clone == NULL)
+- return -ENOMEM;
++ clone->depth = 24;
++ clone->next = display->clone;
++ display->clone = clone;
+
+- clone->depth = 24;
+- clone->next = display->clone;
+- display->clone = clone;
++ id = claim_virtual(ctx->display, buf, ctx->nclone);
++ if (id == 0) {
++ fprintf(stderr, "Failed to find available VirtualHead \"%s\" for on display \"%s\"\n",
++ buf, DisplayString(dpy));
++ }
++ ret = clone_output_init(clone, &clone->src, ctx->display, buf, id);
++ if (ret) {
++ fprintf(stderr, "Failed to add display \"%s\"\n",
++ DisplayString(ctx->display->dpy));
++ return ret;
++ }
+
+- id = claim_virtual(ctx->display, buf, ctx->nclone);
+- if (id == 0) {
+- fprintf(stderr, "Failed to find available VirtualHead \"%s\" for on display \"%s\"\n",
+- buf, DisplayString(dpy));
+- }
+- ret = clone_output_init(clone, &clone->src, ctx->display, buf, id);
+- if (ret) {
+- fprintf(stderr, "Failed to add display \"%s\"\n",
+- DisplayString(ctx->display->dpy));
+- return ret;
+- }
++ sprintf(buf, "SCREEN%d", s);
++ ret = clone_output_init(clone, &clone->dst, display, buf, 0);
++ if (ret) {
++ fprintf(stderr, "Failed to add display \"%s\"\n",
++ DisplayString(dpy));
++ return ret;
++ }
+
+- sprintf(buf, "WHOLE");
+- ret = clone_output_init(clone, &clone->dst, display, buf, 0);
+- if (ret) {
+- fprintf(stderr, "Failed to add display \"%s\"\n",
+- DisplayString(dpy));
+- return ret;
+- }
++ ret = clone_init_depth(clone);
++ if (ret) {
++ fprintf(stderr, "Failed to negotiate image format for display \"%s\"\n",
++ DisplayString(dpy));
++ return ret;
++ }
+
+- ret = clone_init_depth(clone);
+- if (ret) {
+- fprintf(stderr, "Failed to negotiate image format for display \"%s\"\n",
+- DisplayString(dpy));
+- return ret;
+- }
++ /* Replace the modes on the local VIRTUAL output with the remote Screen */
++ scr = ScreenOfDisplay(dpy, s);
++ clone->dst.width = scr->width;
++ clone->dst.height = scr->height;
++ clone->dst.x = 0;
++ clone->dst.y = 0;
++ clone->dst.rr_crtc = -1;
++ ret = clone_update_modes__fixed(clone);
++ if (ret) {
++ fprintf(stderr, "Failed to clone display \"%s\"\n",
++ DisplayString(dpy));
++ return ret;
++ }
+
+- /* Replace the modes on the local VIRTUAL output with the remote Screen */
+- scr = ScreenOfDisplay(dpy, DefaultScreen(dpy));
+- clone->width = scr->width;
+- clone->height = scr->height;
+- clone->dst.x = 0;
+- clone->dst.y = 0;
+- clone->dst.rr_crtc = -1;
+- ret = clone_update_modes__fixed(clone);
+- if (ret) {
+- fprintf(stderr, "Failed to clone display \"%s\"\n",
+- DisplayString(dpy));
+- return ret;
++ clone->active = ctx->active;
++ ctx->active = clone;
+ }
+
+- clone->active = ctx->active;
+- ctx->active = clone;
+-
+ return 0;
+ }
+
+@@ -3168,6 +3351,33 @@ static void context_cleanup(struct context *ctx)
+ XCloseDisplay(dpy);
+ }
+
++static void update_cursor_image(struct context *ctx)
++{
++ XFixesCursorImage *cur;
++ int i;
++
++ DBG(CURSOR, ("%s cursor changed\n",
++ DisplayString(ctx->display->dpy)));
++
++ cur = XFixesGetCursorImage(ctx->display->dpy);
++ if (cur == NULL)
++ return;
++
++ display_load_visible_cursor(&ctx->display[0], cur);
++ for (i = 1; i < ctx->ndisplay; i++) {
++ struct display *display = &ctx->display[i];
++
++ DBG(CURSOR, ("%s marking cursor changed\n", DisplayString(display->dpy)));
++ display->cursor_moved++;
++ if (display->cursor != display->invisible_cursor) {
++ display->cursor_visible++;
++ context_enable_timer(display->ctx);
++ }
++ }
++
++ XFree(cur);
++}
++
+ static int done;
+
+ static void signal_handler(int sig)
+@@ -3182,6 +3392,7 @@ int main(int argc, char **argv)
+ uint64_t count;
+ int daemonize = 1, bumblebee = 0, siblings = 0, singleton = 1;
+ int i, ret, open, fail;
++ int idle;
+
+ signal(SIGPIPE, SIG_IGN);
+
+@@ -3228,6 +3439,7 @@ int main(int argc, char **argv)
+ return -ret;
+
+ XSetErrorHandler(_check_error_handler);
++ XSetIOErrorHandler(_io_error_handler);
+
+ ret = add_fd(&ctx, display_open(&ctx, src_name));
+ if (ret) {
+@@ -3237,6 +3449,13 @@ int main(int argc, char **argv)
+ goto out;
+ }
+
++ ret = check_virtual(ctx.display);
++ if (ret) {
++ fprintf(stderr, "No VIRTUAL outputs on \"%s\".\n",
++ DisplayString(ctx.display->dpy));
++ goto out;
++ }
++
+ if (singleton) {
+ XSelectInput(ctx.display->dpy, ctx.display->root, PropertyChangeMask);
+ if (first_display_has_singleton(&ctx)) {
+@@ -3291,6 +3510,11 @@ int main(int argc, char **argv)
+ if (ret)
+ goto out;
+
++ if (ctx.display->saver_active)
++ XScreenSaverSelectInput(ctx.display->dpy,
++ ctx.display->root,
++ ScreenSaverNotifyMask);
++
+ if ((ctx.display->rr_event | ctx.display->rr_error) == 0) {
+ fprintf(stderr, "RandR extension not supported by %s\n", DisplayString(ctx.display->dpy));
+ ret = EINVAL;
+@@ -3348,25 +3572,60 @@ int main(int argc, char **argv)
+ signal(SIGTERM, signal_handler);
+
+ ctx.command_continuation = 0;
++ update_cursor_image(&ctx);
++
++ idle = 0;
+ while (!done) {
+ XEvent e;
+ int reconfigure = 0;
+ int rr_update = 0;
+
+- DBG(POLL, ("polling - enable timer? %d, nfd=%d, ndisplay=%d\n", ctx.timer_active, ctx.nfd, ctx.ndisplay));
+- ret = poll(ctx.pfd + !ctx.timer_active, ctx.nfd - !ctx.timer_active, -1);
+- if (ret <= 0)
+- break;
++ if (idle) {
++ DBG(POLL, ("polling - enable timer? %d, nfd=%d, ndisplay=%d\n", ctx.timer_active, ctx.nfd, ctx.ndisplay));
++ ret = poll(ctx.pfd + !ctx.timer_active, ctx.nfd - !ctx.timer_active, -1);
++ if (ret <= 0)
++ break;
++
++ DBG(POLL, ("poll reports %d fd awake\n", ret));
++ }
++ idle = 1;
+
+ /* pfd[0] is the timer, pfd[1] is the local display, pfd[2] is the mouse, pfd[3+] are the remotes */
+
+- DBG(POLL, ("poll reports %d fd awake\n", ret));
+ if (ctx.pfd[1].revents || XPending(ctx.display[0].dpy)) {
+ DBG(POLL,("%s woken up\n", DisplayString(ctx.display[0].dpy)));
++ ctx.pfd[1].revents = 0;
++ idle = 0;
++
+ do {
+ XNextEvent(ctx.display->dpy, &e);
+
+- if (e.type == ctx.display->damage_event + XDamageNotify ) {
++ DBG(POLL, ("%s received event %d\n", DisplayString(ctx.display[0].dpy), e.type));
++
++ if (e.type == ctx.display->saver_event + ScreenSaverNotify) {
++ const XScreenSaverNotifyEvent *se = (const XScreenSaverNotifyEvent *)&e;
++ DBG(SCREEN,
++ ("%s screen saver: state=%d, kind=%d, forced=%d\n",
++ DisplayString(ctx.display->dpy),
++ se->state, se->kind, se->forced));
++ for (i = 1; i < ctx.ndisplay; i++) {
++ struct display *display = &ctx.display[i];
++
++ if (!display->active)
++ continue;
++
++ DBG(SCREEN,
++ ("%s %s screen saver\n",
++ DisplayString(display->dpy),
++ se->state == ScreenSaverOn ? "activating" : "resetting\n"));
++
++ if (se->state == ScreenSaverOn)
++ XActivateScreenSaver(display->dpy);
++ else
++ XResetScreenSaver(display->dpy);
++ XFlush(display->dpy);
++ }
++ } else if (e.type == ctx.display->damage_event + XDamageNotify) {
+ const XDamageNotifyEvent *de = (const XDamageNotifyEvent *)&e;
+ struct clone *clone;
+
+@@ -3380,19 +3639,7 @@ int main(int argc, char **argv)
+ if (ctx.active)
+ context_enable_timer(&ctx);
+ } else if (e.type == ctx.display->xfixes_event + XFixesCursorNotify) {
+- XFixesCursorImage *cur;
+-
+- DBG(CURSOR, ("%s cursor changed\n",
+- DisplayString(ctx.display->dpy)));
+-
+- cur = XFixesGetCursorImage(ctx.display->dpy);
+- if (cur == NULL)
+- continue;
+-
+- for (i = 1; i < ctx.ndisplay; i++)
+- display_load_visible_cursor(&ctx.display[i], cur);
+-
+- XFree(cur);
++ update_cursor_image(&ctx);
+ } else if (e.type == ctx.display->rr_event + RRScreenChangeNotify) {
+ DBG(XRR, ("%s screen changed (reconfigure pending? %d)\n",
+ DisplayString(ctx.display->dpy), reconfigure));
+@@ -3426,13 +3673,41 @@ int main(int argc, char **argv)
+ if (ctx.pfd[i+2].revents == 0 && !XPending(ctx.display[i].dpy))
+ continue;
+
++ ctx.pfd[i+2].revents = 0;
++ idle = 0;
++
+ DBG(POLL, ("%s woken up\n", DisplayString(ctx.display[i].dpy)));
+ do {
+ XNextEvent(ctx.display[i].dpy, &e);
+
+ DBG(POLL, ("%s received event %d\n", DisplayString(ctx.display[i].dpy), e.type));
+- if (ctx.display[i].rr_active && e.type == ctx.display[i].rr_event + RRNotify) {
+- XRRNotifyEvent *re = (XRRNotifyEvent *)&e;
++ if (e.type == Expose) {
++ const XExposeEvent *xe = (XExposeEvent *)&e;
++ struct clone *clone;
++ int damaged = 0;
++
++ DBG(DAMAGE, ("%s exposed: (%d, %d)x(%d, %d)\n",
++ DisplayString(ctx.display[i].dpy),
++ xe->x, xe->y, xe->width, xe->height));
++
++ for (clone = ctx.active; clone; clone = clone->active) {
++ XRectangle r;
++
++ if (clone->dst.display != &ctx.display[i])
++ continue;
++
++ r.x = clone->src.x + xe->x;
++ r.y = clone->src.y + xe->y;
++ r.width = xe->width;
++ r.height = xe->height;
++ clone_damage(clone, &r);
++ damaged++;
++ }
++
++ if (damaged)
++ context_enable_timer(&ctx);
++ } else if (ctx.display[i].rr_active && e.type == ctx.display[i].rr_event + RRNotify) {
++ const XRRNotifyEvent *re = (XRRNotifyEvent *)&e;
+
+ DBG(XRR, ("%s received RRNotify, type %d\n", DisplayString(ctx.display[i].dpy), re->subtype));
+ if (re->subtype == RRNotify_OutputChange) {
+@@ -3480,6 +3755,7 @@ int main(int argc, char **argv)
+
+ DBG(TIMER, ("%s timer still active? %d\n", DisplayString(ctx.display->dpy), ret != 0));
+ ctx.timer_active = ret != 0;
++ idle = 0;
+ }
+ }
+
diff --git a/main/xf86-video-intel/xf86-video-intel-2.99.917-libdrm-kernel-4_0-crash.patch b/main/xf86-video-intel/xf86-video-intel-2.99.917-libdrm-kernel-4_0-crash.patch
deleted file mode 100644
index ea3aa30ed1..0000000000
--- a/main/xf86-video-intel/xf86-video-intel-2.99.917-libdrm-kernel-4_0-crash.patch
+++ /dev/null
@@ -1,65 +0,0 @@
-From 7fe2b2948652443ff43d907855bd7a051d54d309 Mon Sep 17 00:00:00 2001
-From: Chris Wilson <chris@chris-wilson.co.uk>
-Date: Thu, 19 Mar 2015 23:14:17 +0000
-Subject: sna: Protect against ABI breakage in recent versions of libdrm
-
-Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
-
-diff --git a/src/sna/kgem.c b/src/sna/kgem.c
-index 11f0828..6f16cba 100644
---- a/src/sna/kgem.c
-+++ b/src/sna/kgem.c
-@@ -182,6 +182,15 @@ struct local_i915_gem_caching {
- #define LOCAL_IOCTL_I915_GEM_SET_CACHING DRM_IOW(DRM_COMMAND_BASE + LOCAL_I915_GEM_SET_CACHING, struct local_i915_gem_caching)
- #define LOCAL_IOCTL_I915_GEM_GET_CACHING DRM_IOW(DRM_COMMAND_BASE + LOCAL_I915_GEM_GET_CACHING, struct local_i915_gem_caching)
-
-+struct local_i915_gem_mmap {
-+ uint32_t handle;
-+ uint32_t pad;
-+ uint64_t offset;
-+ uint64_t size;
-+ uint64_t addr_ptr;
-+};
-+#define LOCAL_IOCTL_I915_GEM_MMAP DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_GEM_MMAP, struct local_i915_gem_mmap)
-+
- struct local_i915_gem_mmap2 {
- uint32_t handle;
- uint32_t pad;
-@@ -514,15 +523,15 @@ retry_wc:
-
- static void *__kgem_bo_map__cpu(struct kgem *kgem, struct kgem_bo *bo)
- {
-- struct drm_i915_gem_mmap mmap_arg;
-+ struct local_i915_gem_mmap arg;
- int err;
-
- retry:
-- VG_CLEAR(mmap_arg);
-- mmap_arg.handle = bo->handle;
-- mmap_arg.offset = 0;
-- mmap_arg.size = bytes(bo);
-- if ((err = do_ioctl(kgem->fd, DRM_IOCTL_I915_GEM_MMAP, &mmap_arg))) {
-+ VG_CLEAR(arg);
-+ arg.handle = bo->handle;
-+ arg.offset = 0;
-+ arg.size = bytes(bo);
-+ if ((err = do_ioctl(kgem->fd, LOCAL_IOCTL_I915_GEM_MMAP, &arg))) {
- assert(err != EINVAL);
-
- if (__kgem_throttle_retire(kgem, 0))
-@@ -536,10 +545,10 @@ retry:
- return NULL;
- }
-
-- VG(VALGRIND_MAKE_MEM_DEFINED(mmap_arg.addr_ptr, bytes(bo)));
-+ VG(VALGRIND_MAKE_MEM_DEFINED(arg.addr_ptr, bytes(bo)));
-
- DBG(("%s: caching CPU vma for %d\n", __FUNCTION__, bo->handle));
-- return bo->map__cpu = (void *)(uintptr_t)mmap_arg.addr_ptr;
-+ return bo->map__cpu = (void *)(uintptr_t)arg.addr_ptr;
- }
-
- static int gem_write(int fd, uint32_t handle,
---
-cgit v0.10.2
-
diff --git a/main/xf86-video-intel/xorg-1.18.patch b/main/xf86-video-intel/xorg-1.18.patch
deleted file mode 100644
index ab3fb399a1..0000000000
--- a/main/xf86-video-intel/xorg-1.18.patch
+++ /dev/null
@@ -1,24 +0,0 @@
-From 2c5063938cc809f624e56efd4673041fa8141e81 Mon Sep 17 00:00:00 2001
-From: Martin Peres <martin.peres@linux.intel.com>
-Date: Thu, 9 Jul 2015 11:26:38 +0300
-Subject: uxa: fix the call to PixmapSyncDirtyHelper, broken by xserver's
- 90db5ed
-
-[ickle: switch to HAS_DIRTYTRACKING_ROTATION as suggested by Dave Airlie]
-Signed-off-by: Martin Peres <martin.peres@linux.intel.com>
-
-diff --git a/src/compat-api.h b/src/compat-api.h
-index aa93bee..293e9d7 100644
---- a/src/compat-api.h
-+++ b/src/compat-api.h
-@@ -247,3 +247,7 @@ static inline void FreePixmap(PixmapPtr pixmap)
- #endif
-
- #endif
-+
-+#if HAS_DIRTYTRACKING_ROTATION
-+#define PixmapSyncDirtyHelper(d, dd) PixmapSyncDirtyHelper(d)
-+#endif
---
-cgit v0.10.2
-
generated by cgit v1.2.3 (git 2.25.1) at 2024-07-02 13:56:37 +0000