nvidia-open-gpu-kernel-modules/kernel-open/nvidia-drm/nvidia-drm-modeset.c

/*
 * Copyright (c) 2015, NVIDIA CORPORATION. All rights reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 */

#include "nvidia-drm-conftest.h" /* NV_DRM_ATOMIC_MODESET_AVAILABLE */

#if defined(NV_DRM_ATOMIC_MODESET_AVAILABLE)

#include "nvidia-drm-priv.h"
#include "nvidia-drm-modeset.h"
#include "nvidia-drm-crtc.h"
#include "nvidia-drm-os-interface.h"
#include "nvidia-drm-helper.h"

#if defined(NV_DRM_DRMP_H_PRESENT)
#include <drm/drmP.h>
#endif

#if defined(NV_DRM_DRM_VBLANK_H_PRESENT)
#include <drm/drm_vblank.h>
#endif

#include <drm/drm_atomic.h>
#include <drm/drm_atomic_helper.h>
#include <drm/drm_crtc.h>

#if defined(NV_LINUX_NVHOST_H_PRESENT) && defined(CONFIG_TEGRA_GRHOST)
#include <linux/nvhost.h>
#elif defined(NV_LINUX_HOST1X_NEXT_H_PRESENT)            
#include <linux/host1x-next.h>
#endif

#if defined(NV_DRM_FENCE_AVAILABLE)
#include "nvidia-dma-fence-helper.h"
#endif

struct nv_drm_atomic_state {
    struct NvKmsKapiRequestedModeSetConfig config;
    struct drm_atomic_state base;
};

static inline struct nv_drm_atomic_state *to_nv_atomic_state(
    struct drm_atomic_state *state)
{
    return container_of(state, struct nv_drm_atomic_state, base);
}

struct drm_atomic_state *nv_drm_atomic_state_alloc(struct drm_device *dev)
{
    struct nv_drm_atomic_state *nv_state =
            nv_drm_calloc(1, sizeof(*nv_state));

    if (nv_state == NULL || drm_atomic_state_init(dev, &nv_state->base) < 0) {
        nv_drm_free(nv_state);
        return NULL;
    }

    return &nv_state->base;
}

void nv_drm_atomic_state_clear(struct drm_atomic_state *state)
{
    drm_atomic_state_default_clear(state);
}

void nv_drm_atomic_state_free(struct drm_atomic_state *state)
{
    struct nv_drm_atomic_state *nv_state =
                    to_nv_atomic_state(state);
    drm_atomic_state_default_release(state);
    nv_drm_free(nv_state);
}

/**
 * __will_generate_flip_event - Check whether event is going to be generated by
 * hardware when it flips from old crtc/plane state to current one. This
 * function is called after drm_atomic_helper_swap_state(), therefore new state
 * is swapped into current state.
 */
static bool __will_generate_flip_event(struct drm_crtc *crtc,
                                       struct drm_crtc_state *old_crtc_state)
{
    struct drm_crtc_state *new_crtc_state = crtc->state;
    struct nv_drm_crtc_state *nv_new_crtc_state =
        to_nv_crtc_state(new_crtc_state);
    struct drm_plane_state *old_plane_state = NULL;
    struct drm_plane *plane = NULL;
    int i;

    if (!old_crtc_state->active  && !new_crtc_state->active) {
        /*
         * crtc is not active in old and new states therefore all planes are
         * disabled, hardware can not generate flip events.
         */
        return false;
    }

    /* Find out whether primary & overlay flip done events will be generated. */
    nv_drm_for_each_plane_in_state(old_crtc_state->state,
        plane, old_plane_state, i) {
        if (old_plane_state->crtc != crtc) {
           continue;
        }

        if (plane->type == DRM_PLANE_TYPE_CURSOR) {
            continue;
        }

        /*
         * Hardware generates flip event for only those
         * planes which were active previously.
         */
        if (old_crtc_state->active && old_plane_state->fb != NULL) {
            nv_new_crtc_state->nv_flip->pending_events++;
        }
    }

    return nv_new_crtc_state->nv_flip->pending_events != 0;
}

static int __nv_drm_put_back_post_fence_fd(
    struct nv_drm_plane_state *plane_state,
    const struct NvKmsKapiLayerReplyConfig *layer_reply_config)
{
    int fd = layer_reply_config->postSyncptFd;
    int ret = 0;

    if ((fd >= 0) && (plane_state->fd_user_ptr != NULL)) {
        ret = copy_to_user(plane_state->fd_user_ptr, &fd, sizeof(fd));
        if (ret != 0) {
            return ret;
        }

        /*! set back to Null and let set_property specify it again */
        plane_state->fd_user_ptr = NULL;
    }

    return ret;
}

#if defined(NV_DRM_FENCE_AVAILABLE)
struct nv_drm_plane_fence_cb_data {
    nv_dma_fence_cb_t dma_fence_cb;
    struct nv_drm_device *nv_dev;
    NvU32 semaphore_index;
};

static void
__nv_drm_plane_fence_cb(
    nv_dma_fence_t *fence,
    nv_dma_fence_cb_t *cb_data
)
{
    struct nv_drm_plane_fence_cb_data *fence_data =
        container_of(cb_data, typeof(*fence_data), dma_fence_cb);
    struct nv_drm_device *nv_dev = fence_data->nv_dev;

    nv_dma_fence_put(fence);
    nvKms->signalDisplaySemaphore(nv_dev->pDevice, fence_data->semaphore_index);
    nv_drm_free(fence_data);
}

static int __nv_drm_convert_in_fences(
    struct nv_drm_device *nv_dev,
    struct drm_atomic_state *state,
    struct drm_crtc *crtc,
    struct drm_crtc_state *crtc_state)
{
    struct drm_plane *plane = NULL;
    struct drm_plane_state *plane_state = NULL;
    struct nv_drm_plane *nv_plane = NULL;
    struct NvKmsKapiLayerRequestedConfig *plane_req_config = NULL;
    struct NvKmsKapiHeadRequestedConfig *head_req_config =
        &to_nv_crtc_state(crtc_state)->req_config;
    struct nv_drm_plane_fence_cb_data *fence_data;
    uint32_t semaphore_index;
    int ret, i;

    if (!crtc_state->active) {
        return 0;
    }

    nv_drm_for_each_new_plane_in_state(state, plane, plane_state, i) {
        if ((plane->type == DRM_PLANE_TYPE_CURSOR) ||
            (plane_state->crtc != crtc) ||
            (plane_state->fence == NULL)) {
            continue;
        }

        nv_plane = to_nv_plane(plane);
        plane_req_config =
            &head_req_config->layerRequestedConfig[nv_plane->layer_idx];

        if (nv_dev->supportsSyncpts) {
#if defined(NV_LINUX_NVHOST_H_PRESENT) && defined(CONFIG_TEGRA_GRHOST)
#if defined(NV_NVHOST_DMA_FENCE_UNPACK_PRESENT)
            int ret =
                nvhost_dma_fence_unpack(
                    plane_state->fence,
                    &plane_req_config->config.syncParams.u.syncpt.preSyncptId,
                    &plane_req_config->config.syncParams.u.syncpt.preSyncptValue);
            if (ret == 0) {
                plane_req_config->config.syncParams.preSyncptSpecified = true;
                continue;
            }
#endif
#elif defined(NV_LINUX_HOST1X_NEXT_H_PRESENT)
            int ret =
                host1x_fence_extract(
                    plane_state->fence,
                    &plane_req_config->config.syncParams.u.syncpt.preSyncptId,
                    &plane_req_config->config.syncParams.u.syncpt.preSyncptValue);
            if (ret == 0) {
                plane_req_config->config.syncParams.preSyncptSpecified = true;
                continue;
            }
#endif
        }

        /*
         * Syncpt extraction failed, or syncpts are not supported.
         * Use general DRM fence support with semaphores instead.
         */
        if (plane_req_config->config.syncParams.postSyncptRequested) {
            // Can't mix Syncpts and semaphores in a given request.
            return -EINVAL;
        }

        semaphore_index = nv_drm_next_display_semaphore(nv_dev);

        if (!nvKms->resetDisplaySemaphore(nv_dev->pDevice, semaphore_index)) {
            NV_DRM_DEV_LOG_ERR(
                nv_dev,
                "Failed to initialize semaphore for plane fence");
            /*
             * This should only happen if the semaphore pool was somehow
             * exhausted. Waiting a bit and retrying may help in that case.
             */
            return -EAGAIN;
        }

        plane_req_config->config.syncParams.semaphoreSpecified = true;
        plane_req_config->config.syncParams.u.semaphore.index = semaphore_index;

        fence_data = nv_drm_calloc(1, sizeof(*fence_data));

        if (!fence_data) {
            NV_DRM_DEV_LOG_ERR(
                nv_dev,
                "Failed to allocate callback data for plane fence");
            nvKms->cancelDisplaySemaphore(nv_dev->pDevice, semaphore_index);
            return -ENOMEM;
        }

        fence_data->nv_dev = nv_dev;
        fence_data->semaphore_index = semaphore_index;

        ret = nv_dma_fence_add_callback(plane_state->fence,
                                        &fence_data->dma_fence_cb,
                                        __nv_drm_plane_fence_cb);

        switch (ret) {
        case -ENOENT:
            /* The fence is already signaled */
            __nv_drm_plane_fence_cb(plane_state->fence,
                                    &fence_data->dma_fence_cb);
#if defined(fallthrough)
            fallthrough;
#else
            /* Fallthrough */
#endif
        case 0:
            /*
             * The plane state's fence reference has either been consumed or
             * belongs to the outstanding callback now.
             */
            plane_state->fence = NULL;
            break;
        default:
            NV_DRM_DEV_LOG_ERR(
                nv_dev,
                "Failed plane fence callback registration");
            /* Fence callback registration failed */
            nvKms->cancelDisplaySemaphore(nv_dev->pDevice, semaphore_index);
            nv_drm_free(fence_data);
            return ret;
        }
    }

    return 0;
}
#endif /* defined(NV_DRM_FENCE_AVAILABLE) */

static int __nv_drm_get_syncpt_data(
    struct nv_drm_device *nv_dev,
    struct drm_crtc *crtc,
    struct drm_crtc_state *old_crtc_state,
    struct NvKmsKapiRequestedModeSetConfig *requested_config,
    struct NvKmsKapiModeSetReplyConfig *reply_config)
{
    struct nv_drm_crtc *nv_crtc = to_nv_crtc(crtc);
    struct NvKmsKapiHeadReplyConfig *head_reply_config;
    struct nv_drm_plane_state *plane_state;
    struct drm_crtc_state *new_crtc_state = crtc->state;
    struct drm_plane_state *old_plane_state = NULL;
    struct drm_plane_state *new_plane_state = NULL;
    struct drm_plane *plane = NULL;
    int i, ret;

    if (!old_crtc_state->active && !new_crtc_state->active) {
        /*
         * crtc is not active in old and new states therefore all planes are
         * disabled, exit early.
         */
        return 0;
    }

    head_reply_config = &reply_config->headReplyConfig[nv_crtc->head];

    nv_drm_for_each_plane_in_state(old_crtc_state->state, plane, old_plane_state, i) {
        struct nv_drm_plane *nv_plane = to_nv_plane(plane);

        if (plane->type == DRM_PLANE_TYPE_CURSOR || old_plane_state->crtc != crtc) {
            continue;
        }

        new_plane_state = plane->state;

        if (new_plane_state->crtc != crtc) {
            continue;
        }

        plane_state = to_nv_drm_plane_state(new_plane_state);

        ret = __nv_drm_put_back_post_fence_fd(
            plane_state,
            &head_reply_config->layerReplyConfig[nv_plane->layer_idx]);

        if (ret != 0) {
            return ret;
        }
    }

    return 0;
}

/**
 * nv_drm_atomic_commit - validate/commit modeset config
 * @dev: DRM device
 * @state: atomic state tracking atomic update
 * @commit: commit/check modeset config associated with atomic update
 *
 * @state tracks atomic update and modeset objects affected
 * by the atomic update, but the state of the modeset objects it contains
 * depends on the current stage of the update.
 * At the commit stage, the proposed state is already stored in the current
 * state, and @state contains old state for all affected modeset objects.
 * At the check/validation stage, @state contains the proposed state for
 * all affected objects.
 *
 * Sequence of atomic update -
 *   1. The check/validation of proposed atomic state,
 *   2. Do any other steps that might fail,
 *   3. Put the proposed state into the current state pointers,
 *   4. Actually commit the hardware state,
 *   5. Cleanup old state.
 *
 * The function nv_drm_atomic_apply_modeset_config() is getting called
 * at stages (1) and (4) after drm_atomic_helper_swap_state().
 */
static int
nv_drm_atomic_apply_modeset_config(struct drm_device *dev,
                                   struct drm_atomic_state *state,
                                   bool commit)
{
    struct nv_drm_device *nv_dev = to_nv_device(dev);
    struct NvKmsKapiRequestedModeSetConfig *requested_config =
        &(to_nv_atomic_state(state)->config);
    struct NvKmsKapiModeSetReplyConfig reply_config = { };
    struct drm_crtc *crtc;
    struct drm_crtc_state *crtc_state;
    int i;
    int ret;

    /*
     * If sub-owner permission was granted to another NVKMS client, disallow
     * modesets through the DRM interface.
     */
    if (nv_dev->subOwnershipGranted) {
        return -EINVAL;
    }

    memset(requested_config, 0, sizeof(*requested_config));

    /* Loop over affected crtcs and construct NvKmsKapiRequestedModeSetConfig */
    nv_drm_for_each_crtc_in_state(state, crtc, crtc_state, i) {
        /*
         * When committing a state, the new state is already stored in
         * crtc->state. When checking a proposed state, the proposed state is
         * stored in crtc_state.
         */
        struct drm_crtc_state *new_crtc_state =
                               commit ? crtc->state : crtc_state;
        struct nv_drm_crtc *nv_crtc = to_nv_crtc(crtc);

        if (commit) {
            struct drm_crtc_state *old_crtc_state = crtc_state;
            struct nv_drm_crtc_state *nv_new_crtc_state =
                to_nv_crtc_state(new_crtc_state);

            nv_new_crtc_state->nv_flip->event = new_crtc_state->event;
            nv_new_crtc_state->nv_flip->pending_events = 0;
            new_crtc_state->event = NULL;

            /*
             * If flip event will be generated by hardware
             * then defer flip object processing to flip event from hardware.
             */
            if (__will_generate_flip_event(crtc, old_crtc_state)) {
                nv_drm_crtc_enqueue_flip(nv_crtc,
                                         nv_new_crtc_state->nv_flip);

                nv_new_crtc_state->nv_flip = NULL;
            }

#if defined(NV_DRM_FENCE_AVAILABLE)
            ret = __nv_drm_convert_in_fences(nv_dev,
                                             state,
                                             crtc,
                                             new_crtc_state);

            if (ret != 0) {
                return ret;
            }
#endif /* defined(NV_DRM_FENCE_AVAILABLE) */
        }

        /*
         * Do this deep copy after calling __nv_drm_convert_in_fences,
         * which modifies the new CRTC state's req_config member
         */
        requested_config->headRequestedConfig[nv_crtc->head] =
            to_nv_crtc_state(new_crtc_state)->req_config;

        requested_config->headsMask |= 1 << nv_crtc->head;
    }

    if (commit && nvKms->systemInfo.bAllowWriteCombining) {
        /*
         * XXX This call is required only if dumb buffer is going
         * to be presented.
         */
         nv_drm_write_combine_flush();
    }

    if (!nvKms->applyModeSetConfig(nv_dev->pDevice,
                                   requested_config,
                                   &reply_config,
                                   commit)) {
        if (commit || reply_config.flipResult != NV_KMS_FLIP_RESULT_IN_PROGRESS) {
            return -EINVAL;
        }
    }

    if (commit && nv_dev->supportsSyncpts) {
        nv_drm_for_each_crtc_in_state(state, crtc, crtc_state, i) {
            /*! loop over affected crtcs and get NvKmsKapiModeSetReplyConfig */
            ret = __nv_drm_get_syncpt_data(
                      nv_dev, crtc, crtc_state, requested_config, &reply_config);
            if (ret != 0) {
                return ret;
            }
        }
    }

    if (commit && nv_dev->requiresVrrSemaphores && reply_config.vrrFlip) {
        nvKms->signalVrrSemaphore(nv_dev->pDevice, reply_config.vrrSemaphoreIndex);
    }

    return 0;
}

int nv_drm_atomic_check(struct drm_device *dev,
                        struct drm_atomic_state *state)
{
    int ret = 0;

#if defined(NV_DRM_COLOR_MGMT_AVAILABLE)
    struct drm_crtc *crtc;
    struct drm_crtc_state *crtc_state;
    int i;

    nv_drm_for_each_crtc_in_state(state, crtc, crtc_state, i) {
        /*
         * if the color management changed on the crtc, we need to update the
         * crtc's plane's CSC matrices, so add the crtc's planes to the commit
         */
        if (crtc_state->color_mgmt_changed) {
            if ((ret = drm_atomic_add_affected_planes(state, crtc)) != 0) {
                goto done;
            }
        }
    }
#endif /* NV_DRM_COLOR_MGMT_AVAILABLE */

    if ((ret = drm_atomic_helper_check(dev, state)) != 0) {
        goto done;
    }

    ret = nv_drm_atomic_apply_modeset_config(dev,
                                             state, false /* commit */);

done:
    return ret;
}

/**
 * __nv_drm_handle_flip_event - handle flip occurred event
 * @nv_crtc: crtc on which flip has been occurred
 *
 * This handler dequeues the first nv_drm_flip from the crtc's flip_list,
 * generates an event if requested at flip time, and frees the nv_drm_flip.
 */
static void __nv_drm_handle_flip_event(struct nv_drm_crtc *nv_crtc)
{
    struct drm_device *dev = nv_crtc->base.dev;
    struct nv_drm_device *nv_dev = to_nv_device(dev);
    struct nv_drm_flip *nv_flip;

    /*
     * Acquire event_lock before nv_flip object dequeue, otherwise immediate
     * flip event delivery from nv_drm_atomic_commit() races ahead and
     * messes up with event delivery order.
     */
    spin_lock(&dev->event_lock);
    nv_flip = nv_drm_crtc_dequeue_flip(nv_crtc);
    if (likely(nv_flip != NULL)) {
        struct nv_drm_flip *nv_deferred_flip, *nv_next_deferred_flip;

        if (nv_flip->event != NULL) {
            drm_crtc_send_vblank_event(&nv_crtc->base, nv_flip->event);
        }

        /*
         * Process flips that were deferred until processing of this nv_flip
         * object.
         */
        list_for_each_entry_safe(nv_deferred_flip,
                                 nv_next_deferred_flip,
                                 &nv_flip->deferred_flip_list, list_entry) {

            if (nv_deferred_flip->event != NULL) {
                drm_crtc_send_vblank_event(&nv_crtc->base,
                                           nv_deferred_flip->event);
            }
            list_del(&nv_deferred_flip->list_entry);

            nv_drm_free(nv_deferred_flip);
        }
    }
    spin_unlock(&dev->event_lock);

    wake_up_all(&nv_dev->flip_event_wq);

    nv_drm_free(nv_flip);
}

int nv_drm_atomic_commit(struct drm_device *dev,
                         struct drm_atomic_state *state,
                         bool nonblock)
{
    int ret = -EBUSY;

    int i;
    struct drm_crtc *crtc = NULL;
    struct drm_crtc_state *crtc_state = NULL;
    struct nv_drm_device *nv_dev = to_nv_device(dev);

    /*
     * XXX: drm_mode_config_funcs::atomic_commit() mandates to return -EBUSY
     * for nonblocking commit if the commit would need to wait for previous
     * updates (commit tasks/flip event) to complete. In case of blocking
     * commits it mandates to wait for previous updates to complete. However,
     * the kernel DRM-KMS documentation does explicitly allow maintaining a
     * queue of outstanding commits.
     *
     * Our system already implements such a queue, but due to
     * bug 4054608, it is currently not used.
     */
    nv_drm_for_each_crtc_in_state(state, crtc, crtc_state, i) {
        struct nv_drm_crtc *nv_crtc = to_nv_crtc(crtc);

        /*
         * Here you aren't required to hold nv_drm_crtc::flip_list_lock
         * because:
         *
         * The core DRM driver acquires lock for all affected crtcs before
         * calling into ->commit() hook, therefore it is not possible for
         * other threads to call into ->commit() hook affecting same crtcs
         * and enqueue flip objects into flip_list -
         *
         *   nv_drm_atomic_commit_internal()
         *     |-> nv_drm_atomic_apply_modeset_config(commit=true)
         *           |-> nv_drm_crtc_enqueue_flip()
         *
         * Only possibility is list_empty check races with code path
         * dequeuing flip object -
         *
         *   __nv_drm_handle_flip_event()
         *     |-> nv_drm_crtc_dequeue_flip()
         *
         * But this race condition can't lead list_empty() to return
         * incorrect result. nv_drm_crtc_dequeue_flip() in the middle of
         * updating the list could not trick us into thinking the list is
         * empty when it isn't.
         */
        if (nonblock) {
            if (!list_empty(&nv_crtc->flip_list)) {
                return -EBUSY;
            }
        } else {
            if (wait_event_timeout(
                    nv_dev->flip_event_wq,
                    list_empty(&nv_crtc->flip_list),
                    3 * HZ /* 3 second */) == 0) {
                NV_DRM_DEV_LOG_ERR(
                    nv_dev,
                    "Flip event timeout on head %u", nv_crtc->head);
            }
        }
    }

#if defined(NV_DRM_ATOMIC_HELPER_SWAP_STATE_HAS_STALL_ARG)

    /*
     * nv_drm_atomic_commit_internal()
     * implements blocking/non-blocking atomic commit using
     * nv_drm_crtc::flip_list, it does not require any help from core DRM
     * helper functions to stall commit processing.  Therefore passing false to
     * 'stall' parameter.
     * In this context, failure from drm_atomic_helper_swap_state() is not
     * expected.
     */

#if defined(NV_DRM_ATOMIC_HELPER_SWAP_STATE_RETURN_INT)
    ret = drm_atomic_helper_swap_state(state, false /* stall */);
    if (WARN_ON(ret != 0)) {
        return ret;
    }
#else
    drm_atomic_helper_swap_state(state, false /* stall */);
#endif

#else
    drm_atomic_helper_swap_state(dev, state);
#endif

    /*
     * nv_drm_atomic_commit_internal() must not return failure after
     * calling drm_atomic_helper_swap_state().
     */

    if ((ret = nv_drm_atomic_apply_modeset_config(
                    dev,
                    state, true /* commit */)) != 0) {
        NV_DRM_DEV_LOG_ERR(
            nv_dev,
            "Failed to apply atomic modeset.  Error code: %d",
            ret);

        goto done;
    }

    nv_drm_for_each_crtc_in_state(state, crtc, crtc_state, i) {
        struct nv_drm_crtc *nv_crtc = to_nv_crtc(crtc);
        struct nv_drm_crtc_state *nv_new_crtc_state =
            to_nv_crtc_state(crtc->state);

        /*
         * If nv_drm_atomic_apply_modeset_config() hasn't consumed the flip
         * object, no event will be generated for this flip, and we need process
         * it:
         */

        if (nv_new_crtc_state->nv_flip != NULL) {
            /*
             * First, defer processing of all pending flips for this crtc until
             * last flip in the queue has been processed. This is to ensure a
             * correct order in event delivery.
             */
            spin_lock(&nv_crtc->flip_list_lock);
            if (!list_empty(&nv_crtc->flip_list)) {
                struct nv_drm_flip *nv_last_flip =
                    list_last_entry(&nv_crtc->flip_list,
                                    struct nv_drm_flip, list_entry);

                list_add(&nv_new_crtc_state->nv_flip->list_entry,
                    &nv_last_flip->deferred_flip_list);

                nv_new_crtc_state->nv_flip = NULL;
            }
            spin_unlock(&nv_crtc->flip_list_lock);
        }

        if (nv_new_crtc_state->nv_flip != NULL) {
            /*
             * Then, if no more pending flips for this crtc, deliver event for the
             * current flip.
             */
            if (nv_new_crtc_state->nv_flip->event != NULL) {
                spin_lock(&dev->event_lock);
                drm_crtc_send_vblank_event(crtc,
                                           nv_new_crtc_state->nv_flip->event);
                spin_unlock(&dev->event_lock);
            }

            nv_drm_free(nv_new_crtc_state->nv_flip);
            nv_new_crtc_state->nv_flip = NULL;
        }

        if (!nonblock) {
            /*
             * Here you aren't required to hold nv_drm_crtc::flip_list_lock
             * because:
             *
             * The core DRM driver acquires lock for all affected crtcs before
             * calling into ->commit() hook, therefore it is not possible for
             * other threads to call into ->commit() hook affecting same crtcs
             * and enqueue flip objects into flip_list -
             *
             *   nv_drm_atomic_commit_internal()
             *     |-> nv_drm_atomic_apply_modeset_config(commit=true)
             *           |-> nv_drm_crtc_enqueue_flip()
             *
             * Only possibility is list_empty check races with code path
             * dequeuing flip object -
             *
             *   __nv_drm_handle_flip_event()
             *     |-> nv_drm_crtc_dequeue_flip()
             *
             * But this race condition can't lead list_empty() to return
             * incorrect result. nv_drm_crtc_dequeue_flip() in the middle of
             * updating the list could not trick us into thinking the list is
             * empty when it isn't.
             */
            if (wait_event_timeout(
                    nv_dev->flip_event_wq,
                    list_empty(&nv_crtc->flip_list),
                    3 * HZ /* 3 second */) == 0) {
                NV_DRM_DEV_LOG_ERR(
                    nv_dev,
                    "Flip event timeout on head %u", nv_crtc->head);
                while (!list_empty(&nv_crtc->flip_list)) {
                    __nv_drm_handle_flip_event(nv_crtc);
                }
            }
        }
    }

done:

#if defined(NV_DRM_ATOMIC_STATE_REF_COUNTING_PRESENT)
    /*
     * If ref counting is present, state will be freed when the caller
     * drops its reference after we return.
     */
#else
    drm_atomic_state_free(state);
#endif

    return 0;
}

void nv_drm_handle_flip_occurred(struct nv_drm_device *nv_dev,
                                 NvU32 head, NvU32 plane)
{
    struct nv_drm_crtc *nv_crtc = nv_drm_crtc_lookup(nv_dev, head);

    if (NV_DRM_WARN(nv_crtc == NULL)) {
        return;
    }

    __nv_drm_handle_flip_event(nv_crtc);
}

#endif