/*===================== begin_copyright_notice ==================================

Copyright (c) 2017 Intel Corporation

Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:

The above copyright notice and this permission notice shall be included
in all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.


======================= end_copyright_notice ==================================*/

#include "BuildIR.h"
#include "../Timer.h"

static const unsigned MESSAGE_PRECISION_SUBTYPE_OFFSET  = 30;
static const unsigned SIMD_MODE_2_OFFSET  = 29;


static uint32_t createSamplerMsgDesc(
    VISASampler3DSubOpCode samplerOp,
    bool isNativeSIMDSize,
    bool isFP16Return,
    bool isFP16Input)
{
    // Now create message descriptor
    // 7:0 - BTI
    // 11:8 - Sampler Index
    // 16:12 - Message Type
    // 18:17 - SIMD Mode[0:1]
    // 19 - Header Present
    // 24:20 - Response Length
    // 28:25 - Message Length
    // 29 - SIMD Mode[2]
    // 30 - Return Format
    // 31 - CPS Message LOD Compensation Enable
    // We only set message type, SIMD mode, and return format here.  The other fields
    // are set in Create_Send_Inst_For_CISA as they are common with other send messages
    uint32_t fc = 0;

    fc |= ((uint32_t)samplerOp & 0x1f) << 12;

    if (isNativeSIMDSize)
    {
        fc |= (1 << 17);
    }
    else
    {
        fc |= (2 << 17);
    }

    if (isFP16Return)
    {
        // 16-bit return type.  Note that this doesn't change the return length
        fc |= (1 << MESSAGE_PRECISION_SUBTYPE_OFFSET);
    }

    if (isFP16Input)
    {
        fc |= (1 << SIMD_MODE_2_OFFSET);
    }

    return fc;
}


int IR_Builder::translateVISASampleInfoInst(
    VISA_Exec_Size executionSize,
    VISA_EMask_Ctrl emask,
    ChannelMask chMask,
    G4_Operand* surface,
    G4_DstRegRegion* dst)
{
    TIME_SCOPE(VISA_BUILDER_IR_CONSTRUCTION);

    G4_ExecSize execSize {Get_VISA_Exec_Size(executionSize)};
    G4_InstOpts instOpt = Get_Gen4_Emask(emask, execSize);
    VISAChannelMask channels = chMask.getAPI();
    bool useFakeHeader = (getPlatform() < GENX_SKL) ? false :
        (channels == CHANNEL_MASK_R);
    bool preEmption = forceSamplerHeader();
    bool forceSplitSend = shouldForceSplitSend(surface);
    bool useHeader = true;
    // SAMPLEINFO has 0 parameters so its only header

    unsigned int numRows = 1;

    G4_Declare *msg = NULL;
    G4_SrcRegRegion *m0 = NULL;

    if (!useFakeHeader || forceSplitSend || preEmption)
    {
        msg = getSamplerHeader(false /*isBindlessSampler*/, false /*samperIndexGE16*/);

        unsigned int secondDword = chMask.getHWEncoding() << 12;

        G4_Imm* immOpndSecondDword = createImm(secondDword, Type_UD);

        // mov (1) msg(0,2) immOpndSecondDword
        auto payloadDstRgn = createDst(msg->getRegVar(), 0, 2, 1, Type_UD);

        G4_INST* movInst = createMov(g4::SIMD1, payloadDstRgn, immOpndSecondDword, InstOpt_NoOpt, true);
        movInst->setOptionOn(InstOpt_WriteEnable);

        m0 = Create_Src_Opnd_From_Dcl(msg, getRegionStride1());
    }
    else
    {
        useHeader = false;
        msg = createTempVar(8, Type_UD, Any);
        G4_DstRegRegion *dst = createDst(msg->getRegVar(), 0, 0, 1, Type_UD);
        G4_Imm* src0Imm = createImm(0, Type_UD);
        auto temp = createMov(g4::SIMD8, dst, src0Imm, InstOpt_NoOpt, true);
        temp->setOptionOn(InstOpt_WriteEnable);
        m0 = createSrcRegRegion(Mod_src_undef, Direct, msg->getRegVar(), 0, 0, getRegionStride1(), Type_UD);
    }
    // Now create message descriptor
    // 7:0 - BTI
    // 11:8 - Sampler Index
    // 16:12 - Message Type
    // 18:17 - SIMD Mode
    // 19 - Header Present
    // 24:20 - Response Length
    // 28:25 - Message Length
    // 29 - SIMD Mode
    // 30 - Return Format
    // 31 - CPS Message LOD Compensation Enable
    unsigned int fc = 0;

    fc |= ((unsigned int) VISA_3D_SAMPLEINFO & 0x1f) << 12;

    if (execSize == getNativeExecSize())
    {
        fc |= (1 << 17);
    }
    else
    {
        fc |= (2 << 17);
    }

    uint32_t retSize = (execSize == getNativeExecSize() ? chMask.getNumEnabledChannels() : chMask.getNumEnabledChannels() * 2);

    if (forceSplitSend)
    {
        Create_SplitSend_Inst_For_CISA(NULL, dst, m0, numRows,
            createNullSrc(Type_UD), 0, retSize,
            execSize, fc, SFID::SAMPLER, useHeader, SendAccess::READ_ONLY, surface, NULL, instOpt, false);
    }
    else
    {
        Create_Send_Inst_For_CISA(NULL, dst, m0, numRows, retSize,
            execSize, fc, SFID::SAMPLER, useHeader, SendAccess::READ_ONLY, surface, NULL, instOpt, false);
    }

    return VISA_SUCCESS;
}

int IR_Builder::translateVISAResInfoInst(
    VISA_Exec_Size executionSize,
    VISA_EMask_Ctrl emask,
    ChannelMask chMask,
    G4_Operand* surface,
    G4_SrcRegRegion* lod,
    G4_DstRegRegion* dst)
{
    TIME_SCOPE(VISA_BUILDER_IR_CONSTRUCTION);

    G4_ExecSize execSize {Get_VISA_Exec_Size(executionSize)};
    G4_InstOpts instOpt = Get_Gen4_Emask(emask, execSize);
    //For SKL if channels are continuous don't need header

    VISAChannelMask channels = chMask.getAPI();
    bool preEmption = forceSamplerHeader();
    bool useHeader = preEmption || (getPlatform() < GENX_SKL) ? channels != CHANNEL_MASK_RGBA :
        (channels != CHANNEL_MASK_R && channels != CHANNEL_MASK_RG && channels != CHANNEL_MASK_RGB && channels != CHANNEL_MASK_RGBA);

    // Setup number of rows = (header + lod) by default
    unsigned int numRows = (execSize == getNativeExecSize() ? 1 : 2);
    if (useHeader)
    {
        numRows++;
    }
    unsigned int regOff = 0;
    uint32_t returnLength = (execSize == getNativeExecSize() ? chMask.getNumEnabledChannels() : chMask.getNumEnabledChannels() * 2);

    bool useSplitSend = useSends();

    G4_Declare *msg = NULL;
    G4_Declare *payloadUD = NULL;
    if (useSplitSend)
    {
        if (useHeader)
        {
            --numRows;
        }
        unsigned int numElts = numRows * numEltPerGRF(Type_UB)/G4_Type_Table[Type_F].byteSize;
        msg = getSamplerHeader(false /*isBindlessSampler*/, false /*samperIndexGE16*/);
        payloadUD = createSendPayloadDcl(numElts, Type_UD);
    }
    else
    {
        unsigned int numElts = numRows * numEltPerGRF(Type_UB)/G4_Type_Table[Type_F].byteSize;
        msg = createSendPayloadDcl(numElts, Type_UD);
        payloadUD = createSendPayloadDcl(numElts - (useHeader ? GENX_SAMPLER_IO_SZ : 0), Type_UD);
        payloadUD->setAliasDeclare(msg, useHeader ? numEltPerGRF(Type_UB) : 0);

        if (useHeader)
        {
            // Both SAMPLEINFO and RESINFO use header
            Create_MOVR0_Inst(msg, 0, 0, true);
        }
    }

    if (useHeader)
    {
        unsigned int secondDword = 0;
        secondDword |= (chMask.getHWEncoding() << 12);

        G4_Imm* immOpndSecondDword = createImm(secondDword, Type_UD);

        // mov (1) msg(0,2) immOpndSecondDword
        auto payloadDstRgn = createDst(msg->getRegVar(), 0, 2, 1, Type_UD);

        G4_INST* movInst = createMov(g4::SIMD1, payloadDstRgn, immOpndSecondDword, InstOpt_NoOpt, true);
        movInst->setOptionOn(InstOpt_WriteEnable);
    }

    // Copy over lod vector operand to payload's 1st row
    Copy_SrcRegRegion_To_Payload(payloadUD, regOff, lod, execSize, instOpt);

    // Now create message descriptor
    // 7:0 - BTI
    // 11:8 - Sampler Index
    // 16:12 - Message Type
    // 18:17 - SIMD Mode
    // 19 - Header Present
    // 24:20 - Response Length
    // 28:25 - Message Length
    // 29 - SIMD Mode
    // 30 - Return Format
    // 31 - CPS Message LOD Compensation Enable
    unsigned int fc = 0;

    fc |= ((unsigned int) VISA_3D_RESINFO & 0x1f) << 12;

    if (execSize == getNativeExecSize())
    {
        fc |= (1 << 17);
    }
    else
    {
        fc |= (2 << 17);
    }

    if (useSplitSend)
    {
        G4_SrcRegRegion *m0 = nullptr;
        G4_SrcRegRegion *m1 = nullptr;
        unsigned int src0Size = 0;
        unsigned int src1Size = 0;

        if (useHeader)
        {
            m0 = Create_Src_Opnd_From_Dcl(msg, getRegionStride1());
            m1 = Create_Src_Opnd_From_Dcl(payloadUD, getRegionStride1());
            src0Size = 1;
            src1Size = numRows;
        }
        else
        {
            m0 = Create_Src_Opnd_From_Dcl(payloadUD, getRegionStride1());
            m1 = createNullSrc(Type_UD);
            src0Size = numRows;
            src1Size = 0;
        }
        Create_SplitSend_Inst_For_CISA(NULL, dst, m0, src0Size, m1, src1Size, returnLength,
            execSize, fc, SFID::SAMPLER, useHeader, SendAccess::READ_ONLY, surface, NULL, instOpt, false);
    }
    else
    {
        G4_SrcRegRegion *m = Create_Src_Opnd_From_Dcl(msg, getRegionStride1());
        Create_Send_Inst_For_CISA(NULL, dst, m, numRows, returnLength,
            execSize, fc, SFID::SAMPLER, useHeader, SendAccess::READ_ONLY, surface, NULL, instOpt, false);
    }

    return VISA_SUCCESS;
}



// generate a URB_SIMD8* message
// urbHandle -- 1 GRF holding 8 URB handles.  This is the header of the message
// perSlotOffset -- 1 GRF holding 8 DWord offsets.  If present, it must be immediately after the header
// channelMask -- 1 GRF holding 8 8-bit masks.  In vISA spec they have constant values and must be
//                identical.  If present,  occurs after the per slot message phase if the per slot
//                message phase exists else it occurs after the header.

int IR_Builder::translateVISAURBWrite3DInst(
    G4_Predicate* pred,
    VISA_Exec_Size executionSize,
    VISA_EMask_Ctrl emask,
    uint8_t numOut,
    uint16_t globalOffset,
    G4_SrcRegRegion* channelMask,
    G4_SrcRegRegion* urbHandle,
    G4_SrcRegRegion* perSlotOffset,
    G4_SrcRegRegion* vertexData)
{
    TIME_SCOPE(VISA_BUILDER_IR_CONSTRUCTION);

    G4_ExecSize execSize {Get_VISA_Exec_Size(executionSize)};
    G4_InstOpts instOpt = Get_Gen4_Emask(emask, execSize);

    if (numOut == 0)
    {
        MUST_BE_TRUE(vertexData->isNullReg(), "vertex payload must be null ARF when numOut is 0");
    }

    // header + channelMask + numOut
    unsigned int numRows = 2 + numOut;
    const bool useHeader = true;
    bool usePerSlotIndex = false;
    bool useChannelMask = true;

    if (!perSlotOffset->isNullReg())
    {
        usePerSlotIndex = true;
        numRows++;
    }

    if (channelMask->isNullReg())
    {
        useChannelMask = false;
        numRows--;
    }

    bool useSplitSend = useSends();
    // So far, we don't have a obvious cut except for header. As the result,
    // split-send is disabled once there's no header in the message.
    if (!useHeader)
        useSplitSend = false;

    if (numOut == 0)
    {
        // no split send if payload is null
        useSplitSend = false;
    }

    // msg is the header for split send, or the entire payload for regular send
    G4_Declare *msg = NULL;
    G4_Declare* payloadF = NULL;
    G4_Declare* payloadD = NULL;
    G4_Declare* payloadUD = NULL;
    if (useSplitSend)
    {
        ASSERT_USER(useHeader, "So far, split-send is only used when header is present!");
        --numRows;
        if (numRows > 0)
        {
            unsigned int numElts = numRows * numEltPerGRF(Type_UB)/G4_Type_Table[Type_F].byteSize;
            // we can use the urb handle directly since URB write will not modify its header
            //msg = createSendPayloadDcl(GENX_SAMPLER_IO_SZ, Type_UD);
            payloadUD = createSendPayloadDcl(numElts, Type_UD);
            payloadF = createSendPayloadDcl(numElts, Type_F);
            payloadD = createSendPayloadDcl(numElts, Type_D);
            payloadF->setAliasDeclare(payloadUD, 0);
            payloadD->setAliasDeclare(payloadUD, 0);
        }
    }
    else
    {
        unsigned int numElts = numRows * numEltPerGRF(Type_UB)/G4_Type_Table[Type_F].byteSize;
        msg = createSendPayloadDcl(numElts, Type_UD);
        if (numRows > 1)
        {
            payloadUD = createSendPayloadDcl(numElts - (useHeader ? GENX_SAMPLER_IO_SZ : 0), Type_UD);
            payloadF = createSendPayloadDcl(numElts - (useHeader ? GENX_SAMPLER_IO_SZ : 0), Type_F);
            payloadD = createSendPayloadDcl(numElts - (useHeader ? GENX_SAMPLER_IO_SZ : 0), Type_D);
            payloadUD->setAliasDeclare(msg, useHeader ? numEltPerGRF(Type_UB) : 0);
            payloadF->setAliasDeclare(msg, useHeader ? numEltPerGRF(Type_UB) : 0);
            payloadD->setAliasDeclare(msg, useHeader ? numEltPerGRF(Type_UB) : 0);
        }
    }

    unsigned int regOff = 0;
    // Setup header
    if (useHeader && msg != NULL)
    {
        unsigned ignoredOff = 0;
        Copy_SrcRegRegion_To_Payload(msg, ignoredOff, urbHandle, g4::SIMD8, instOpt);
    }

    if (usePerSlotIndex)
    {
        Copy_SrcRegRegion_To_Payload(payloadUD, regOff, perSlotOffset, g4::SIMD8, instOpt);
    }

    if (useChannelMask)
    {

        // shl (8) M2.0<1>:ud cmask<8;8,1>:ud 0x10:uw
        auto payloadUDRegRgnRow2 = createDst(payloadUD->getRegVar(), regOff++, 0, 1, Type_UD);

        createBinOp(G4_shl, g4::SIMD8, payloadUDRegRgnRow2, channelMask, createImm(16, Type_UW),
            instOpt, true);
    }

    G4_Declare* vertexDataDcl = numOut == 0 ? NULL : vertexData->getBase()->asRegVar()->getDeclare();

    bool needsDataMove = (!useSplitSend || usePerSlotIndex || useChannelMask);
    if (needsDataMove)
    {
        // we have to insert moves to make payload contiguous
        unsigned int startSrcRow = vertexData->getRegOff();

        for (int i = 0; i < numOut; i++)
        {
            G4_DstRegRegion payloadTypedRegRowi(Direct, payloadF->getRegVar(), regOff++, 0, 1, Type_F);
            G4_DstRegRegion* payloadTypedRegRowRgni = createDstRegRegion(payloadTypedRegRowi);

            G4_SrcRegRegion* vertexSrcRegRgnRowi = createSrcRegRegion(Mod_src_undef, Direct, vertexDataDcl->getRegVar(), startSrcRow++, 0, getRegionStride1(), Type_F);

            createMov(g4::SIMD8, payloadTypedRegRowRgni, vertexSrcRegRgnRowi, instOpt, true);
        }
    }
    else
    {
        payloadUD = vertexDataDcl;
    }

    // Msg descriptor
    unsigned int fc = 0;

    fc |= 0x7;

    fc |= (globalOffset << 4);

    if (useChannelMask)
    {
        fc |= (0x1 << 15);
    }

    if (usePerSlotIndex)
    {
        fc |= (0x1 << 17);
    }

    if (useSplitSend)
    {
        G4_SrcRegRegion *m0 = urbHandle;
        G4_SrcRegRegion *m1 = nullptr;

        if (needsDataMove)
        {
            m1 = Create_Src_Opnd_From_Dcl(payloadUD, getRegionStride1());
        }
        else
        {
            ASSERT_USER(payloadUD == vertexDataDcl,
                "If there is no need for data move then payloadUD == vertexDataDcl must hold!");

            m1 = createSrcRegRegion(
                Mod_src_undef,
                Direct,
                payloadUD->getRegVar(),
                vertexData->getRegOff(),
                vertexData->getSubRegOff(),
                getRegionStride1(),
                payloadUD->getElemType());
        }

        Create_SplitSend_Inst_For_CISA(pred, createNullDst(Type_UD), m0, 1, m1, numRows, 0,
            execSize, fc, SFID::URB, useHeader, SendAccess::WRITE_ONLY, NULL, NULL, instOpt, false);
    } else {
        G4_SrcRegRegion *m = Create_Src_Opnd_From_Dcl(msg, getRegionStride1());
        Create_Send_Inst_For_CISA(pred, createNullDst(Type_UD), m, numRows, 0,
            execSize, fc, SFID::URB, useHeader, SendAccess::WRITE_ONLY, nullptr, nullptr, instOpt, false);
    }
    return VISA_SUCCESS;
}

/*****************************************************************************\
ENUM: EU_GEN6_DATA_PORT_RENDER_TARGET_WRITE_CONTROL
\*****************************************************************************/
enum EU_GEN6_DATA_PORT_RENDER_TARGET_WRITE_CONTROL
{
    EU_GEN6_DATA_PORT_RENDER_TARGET_WRITE_CONTROL_SIMD16_SINGLE_SOURCE = 0,
    EU_GEN6_DATA_PORT_RENDER_TARGET_WRITE_CONTROL_SIMD16_SINGLE_SOURCE_REPLICATED = 1,
    EU_GEN6_DATA_PORT_RENDER_TARGET_WRITE_CONTROL_SIMD8_DUAL_SOURCE_LOW = 2,
    EU_GEN6_DATA_PORT_RENDER_TARGET_WRITE_CONTROL_SIMD8_DUAL_SOURCE_HIGH = 3,
    EU_GEN6_DATA_PORT_RENDER_TARGET_WRITE_CONTROL_SIMD8_SINGLE_SOURCE_LOW = 4,
    EU_GEN6_DATA_PORT_RENDER_TARGET_WRITE_CONTROL_SIMD8_IMAGE_WRITE = 5
};

int IR_Builder::translateVISARTWrite3DInst(
    G4_Predicate* pred,
    VISA_Exec_Size executionSize,
    VISA_EMask_Ctrl emask,
    G4_Operand *surface,
    G4_SrcRegRegion *r1HeaderOpnd,
    G4_Operand *rtIndex,
    vISA_RT_CONTROLS cntrls,
    G4_SrcRegRegion *sampleIndexOpnd,
    G4_Operand *cpsCounter,
    unsigned int numParms,
    G4_SrcRegRegion ** msgOpnds)
{
    TIME_SCOPE(VISA_BUILDER_IR_CONSTRUCTION);

    G4_ExecSize execSize = toExecSize(executionSize);
    G4_InstOpts instOpt = Get_Gen4_Emask(emask, execSize);
    bool useHeader = false;

    uint8_t varOffset = 0;
    G4_SrcRegRegion * s0a = NULL;
    //oMask
    G4_SrcRegRegion * oM  = NULL;
    if (cntrls.s0aPresent)
    {
        s0a = msgOpnds[varOffset];
        ++varOffset;
    }
    if (cntrls.oMPresent)
    {
        oM = msgOpnds[varOffset];
        ++varOffset;
    }

    G4_SrcRegRegion * R = msgOpnds[varOffset++];
    G4_SrcRegRegion * G = msgOpnds[varOffset++];
    G4_SrcRegRegion * B = msgOpnds[varOffset++];
    G4_SrcRegRegion * A = msgOpnds[varOffset++];
    //depth
    G4_SrcRegRegion * Z = NULL;

    if (cntrls.zPresent)
        Z = msgOpnds[varOffset++];

    //stencil
    G4_SrcRegRegion * S = NULL;
    if (cntrls.isStencil)
    {
        S = msgOpnds[varOffset++];
    }

    if (varOffset != numParms)
    {
        assert(0);
        return VISA_FAILURE;
    }

    bool FP16Data = R->getType() == Type_HF;
    if (FP16Data)
    {
        MUST_BE_TRUE((G->isNullReg() || G->getType() == Type_HF) &&
            (B->isNullReg() || B->getType() == Type_HF) &&
            (A->isNullReg() || A->getType() == Type_HF),
            "R,G,B,A for RT write must have the same type");
    }

    auto mult = (execSize == getNativeExecSize() ? 1 : 2);
    mult = (FP16Data)? 1 : mult;

    //RGBA sr0Alpha take up one GRF in SIMD8 and SIMD16 modes.
    //in SIMD8 upper DWORDs are reserved
    unsigned int numRows = numParms * mult;

    //Depth is always Float
    //For SIMD16 it is 2 grfs
    //For SIMD8  it is 1 grf
    if (FP16Data && cntrls.zPresent && executionSize == EXEC_SIZE_16)
    {
        ++numRows;
    }

    if (cntrls.oMPresent && mult == 2)
    {
        // oM is always 1 row irrespective of execSize
        numRows--;
    }

    //although for now HW only supports stencil in SIMD8 mode
    if (cntrls.isStencil && mult == 2)
    {
        // stencil is always 1 row irrespective of execSize
        numRows--;
    }

    // header is always 64 byte
    const int numDWInHeader = 16;
    const int headerBytes = numDWInHeader * sizeof(int);
    const int numHeaderGRF = numDWInHeader / getNativeExecSize();

    /*
    All other values should be set by default.
    Most of the time when renderTargetIndex != 0, src0Alpha is present also
    */
    bool isRTIdxNonzero = cntrls.RTIndexPresent &&
        (rtIndex->isSrcRegRegion() || (rtIndex->isImm() && rtIndex->asImm()->getImm() != 0));
    bool isRTIdxDynamic = cntrls.RTIndexPresent && rtIndex->isSrcRegRegion();
    bool needsHeaderForMRT = isRTIdxDynamic || cntrls.s0aPresent || (!hasHeaderlessMRTWrite() && isRTIdxNonzero);
    if (needsHeaderForMRT || cntrls.isSampleIndex)
    {
        useHeader = true;
        numRows += numHeaderGRF;
    }

    bool useSplitSend = useSends();
    // So far, we don't have a obvious cut except for header. As the result,
    // split-send is disabled once there's no header in the message.

    G4_SrcRegRegion* srcToUse   = NULL;
    G4_Declare *msg             = NULL;
    G4_Declare *msgF            = NULL;
    G4_Declare *payloadUD       = NULL;
    G4_Declare *payloadUW       = NULL;
    G4_Declare *payloadFOrHF    = NULL;
    G4_Declare *payloadF        = NULL;

    if (useSplitSend)
    {
        if (useHeader)
        {
            //subtracting Header
            numRows -= numHeaderGRF;
            //creating header
            msg = createSendPayloadDcl(numDWInHeader, Type_UD);
            msgF = createSendPayloadDcl(numDWInHeader, Type_F);
            msgF->setAliasDeclare(msg, 0);
        }
        //creating payload
        unsigned int numElts = numRows * numEltPerGRF(Type_UB) / G4_Type_Table[Type_F].byteSize;
        payloadUD = createSendPayloadDcl(numElts, Type_UD);
        payloadFOrHF = createSendPayloadDcl(numElts, FP16Data ? Type_HF : Type_F);
        payloadUW = createSendPayloadDcl(numElts, Type_UW);
        payloadF = createSendPayloadDcl(numElts, Type_F);

        payloadFOrHF->setAliasDeclare(payloadUD, 0);
        payloadUW->setAliasDeclare(payloadUD, 0);
        payloadF->setAliasDeclare(payloadUD, 0);
    }
    else
    {
        unsigned int numElts = numRows * numEltPerGRF(Type_UB)/G4_Type_Table[Type_F].byteSize;
        //creating enough space for header + payload
        msg = createSendPayloadDcl(numElts, Type_UD);
        msgF = createSendPayloadDcl(GENX_SAMPLER_IO_SZ * 2, Type_F);
        msgF->setAliasDeclare(msg, 0);

        //creating payload declarations.
        payloadUD = createSendPayloadDcl(numElts - (useHeader ? numDWInHeader : 0), Type_UD);
        payloadFOrHF = createSendPayloadDcl(numElts - (useHeader ? numDWInHeader : 0), FP16Data ? Type_HF : Type_F);
        payloadUW = createSendPayloadDcl(numElts - (useHeader ? numDWInHeader : 0), Type_UW);
        payloadF = createSendPayloadDcl(numElts, Type_F);

        //setting them to alias a top level decl with offset past the header
        payloadUD->setAliasDeclare(msg, useHeader ? headerBytes : 0);
        payloadFOrHF->setAliasDeclare(msg, useHeader ? headerBytes : 0);
        payloadUW->setAliasDeclare(msg, useHeader ? headerBytes : 0);
        payloadF->setAliasDeclare(payloadUD, 0);
    }

    if (useHeader)
    {
        ASSERT_USER(r1HeaderOpnd, "Second GRF for header that was passed in is NULL.");
        G4_DstRegRegion* payloadRegRgn = createDst(msg->getRegVar(), 0, 0, 1, Type_UD);

        G4_Declare* r0 = getBuiltinR0();
        G4_SrcRegRegion* r0RegRgn = createSrcRegRegion(Mod_src_undef, Direct, r0->getRegVar(), 0, 0, getRegionStride1(), Type_UD);

        //moves data from r0 to header portion of the message
        G4_INST* movInst = createMov(g4::SIMD8, payloadRegRgn, r0RegRgn, InstOpt_NoOpt, true);
        movInst->setOptionOn(InstOpt_WriteEnable);

        payloadRegRgn = createDst(msg->getRegVar(), 1, 0, 1, Type_UD);
        r1HeaderOpnd->setType(Type_UD);
        movInst = createMov(g4::SIMD8, payloadRegRgn, r1HeaderOpnd, InstOpt_NoOpt, true);
        movInst->setOptionOn(InstOpt_WriteEnable);

#define SAMPLE_INDEX_OFFSET 6
        if (cntrls.isSampleIndex)
        {
            G4_Declare* tmpDcl = createTempVar(2, Type_UD, Any);
            G4_DstRegRegion* tmpDst = createDst(tmpDcl->getRegVar(), 0, 0, 1, Type_UD);

            createBinOp(G4_shl, g4::SIMD1, tmpDst, sampleIndexOpnd, createImm(SAMPLE_INDEX_OFFSET, Type_UD), InstOpt_WriteEnable, true);

            G4_DstRegRegion* payloadUDRegRgn = createDst(msg->getRegVar(), 0, 0, 1, Type_UD);
            G4_SrcRegRegion* tmpSrc = createSrcRegRegion(Mod_src_undef, Direct, tmpDcl->getRegVar(), 0, 0, getRegionScalar(), Type_UD);
            G4_SrcRegRegion* payloadSrc = createSrcRegRegion(Mod_src_undef, Direct, msg->getRegVar(), 0, 0, getRegionScalar(), Type_UD);
            createBinOp(G4_or, g4::SIMD1, payloadUDRegRgn, payloadSrc, tmpSrc, InstOpt_WriteEnable, true);
        }

        if (isRTIdxNonzero)
        {
            G4_DstRegRegion* dstRTIRgn = createDst(msg->getRegVar(), 0, 2, 1, Type_UD);

            G4_INST* rtiMovInst = createMov(g4::SIMD1, dstRTIRgn, rtIndex, InstOpt_NoOpt, true);
            rtiMovInst->setOptionOn(InstOpt_WriteEnable);
        }

        //if header is used, then predication value will need to be stored
        //in the header
        if (useHeader && (pred || cntrls.isHeaderMaskfromCe0))
        {
            //moving pixelMask in to payload
            G4_DstRegRegion* dstPixelMaskRgn = createDst(
                msg->getRegVar(), 1, 14, 1, Type_UW);

            // setPixelMaskRgn when WA ce0 is needed
            auto setPixelMaskRgn = [=](G4_InstOption Option) -> void
            {
                G4_Declare* flagDecl = createTempFlag(2, "WAce0");
                G4_RegVar* flagVar = flagDecl->getRegVar();
                G4_DstRegRegion* flag = createDst(
                    flagVar, 0,
                    Option == InstOpt_M16 ? 1 : 0,
                    1, Type_UW);

                // (1) (W) mov (1|M0) WAce0.[0|1]:uw, 0
                //         M0 : WAce0.0; M16 : WAce0.1
                // (2)     cmp (16|[M0|M16]) (eq)WAce0.0 r0:uw r0:uw
                // (3) (W) mov(1|M0) dstPixelMaskRgn:uw  WAce0.[0|1]:uw
                //         M0 : WAce0.0; M16 : WAce0.1
                createMov(g4::SIMD1, flag, createImm(0, Type_UW), InstOpt_WriteEnable, true);

                G4_SrcRegRegion* r0_0 = createSrcRegRegion(
                    Mod_src_undef, Direct,
                    getRealR0()->getRegVar(), 0, 0,
                    getRegionStride1(), Type_UW);
                G4_SrcRegRegion* r0_1 = createSrcRegRegion(
                    Mod_src_undef, Direct,
                    getRealR0()->getRegVar(), 0, 0,
                    getRegionStride1(), Type_UW);
                G4_DstRegRegion* nullDst = createNullDst(Type_UW);
                G4_CondMod* flagCM = createCondMod(Mod_e, flagVar, 0);
                createInst(NULL, G4_cmp, flagCM, g4::NOSAT, g4::SIMD16, nullDst,
                    r0_0, r0_1, Option, true);

                G4_SrcRegRegion* flagSrc = createSrcRegRegion(
                    Mod_src_undef, Direct,
                    flagVar, 0,
                    Option == InstOpt_M16 ? 1 : 0,
                    getRegionScalar(), Type_UW);

                // move to dstPixelMaskRgn
                createMov(g4::SIMD1, dstPixelMaskRgn, flagSrc, InstOpt_WriteEnable, true);
            };

            G4_SrcRegRegion* pixelMask = NULL;
            if (emask == vISA_EMASK_M5_NM || emask == vISA_EMASK_M5)
            {
                if (pred)
                {
                    //this is a Second half of a SIMD32 RT write. We need to get second half of flag register.
                    //mov whole register in to GRF, move second word of it in to payload.

                    G4_SrcRegRegion* pixelMaskTmp = createSrcRegRegion(
                        Mod_src_undef, Direct,
                        pred->getBase()->asRegVar(), 0, 0,
                        getRegionScalar(), Type_UD);
                    G4_Declare* tmpDcl = createTempVar(1, Type_UD, Any);
                    G4_DstRegRegion* tmpDst = createDst(tmpDcl->getRegVar(), 0, 0, 1, Type_UD);
                    createMov(g4::SIMD1, tmpDst, pixelMaskTmp, InstOpt_WriteEnable, true);

                    pixelMask = createSrcRegRegion(Mod_src_undef, Direct,
                        tmpDcl->getRegVar(), 0, 1, getRegionScalar(), Type_UW);

                    // move from temp register to header
                    createMov(g4::SIMD1, dstPixelMaskRgn, pixelMask, InstOpt_WriteEnable, true);
                }
                else
                {
                    if (VISA_WA_CHECK(getPWaTable(), Wa_1406950495))
                    {
                        setPixelMaskRgn(InstOpt_M16);
                    }
                    else
                    {
                        G4_SrcRegRegion* ce0 = createSrcRegRegion(
                            Mod_src_undef, Direct,
                            phyregpool.getMask0Reg(), 0, 0,
                            getRegionScalar(), Type_UD);

                        // shr .14<1>:uw ce0:ud 16:uw
                        createBinOp(G4_shr, g4::SIMD1, dstPixelMaskRgn,
                            ce0, createImm(16, Type_UW), InstOpt_WriteEnable, true);
                    }
                }
            }
            else
            {
                if (pred)
                {
                    pixelMask = createSrcRegRegion(Mod_src_undef, Direct,
                        pred->getBase()->asRegVar(), 0, 0,
                        getRegionScalar(), Type_UW);

                    //clearing lower 15 bits
                    createMov(g4::SIMD1, dstPixelMaskRgn, pixelMask, InstOpt_WriteEnable, true);
                }
                else
                {
                    if (VISA_WA_CHECK(getPWaTable(), Wa_1406950495))
                    {
                        setPixelMaskRgn(InstOpt_M0);
                    }
                    else
                    {
                        G4_SrcRegRegion* ce0 = createSrcRegRegion(
                            Mod_src_undef, Direct,
                            phyregpool.getMask0Reg(), 0, 0,
                            getRegionScalar(), Type_UD);

                        // mov .14<1>:uw ce0:ud.  clearing lower 15 bits
                        createMov(g4::SIMD1, dstPixelMaskRgn, ce0, InstOpt_WriteEnable, true);
                    }
                }
            }

            pred = NULL;

        }
        unsigned int orImmVal = 0;

        //setting first DWORD of MHC_RT_C0 - Render Target Message Header Control

        if (cntrls.isStencil)
        {
            orImmVal = (0x1 << 14);
        }

        if (cntrls.zPresent)
        {
            orImmVal = (0x1 << 13);
        }

        if (cntrls.oMPresent)
        {
            orImmVal |= (0x1 << 12);
        }

        if (cntrls.s0aPresent)
        {
            orImmVal |= (0x1 << 11);
        }

        if (orImmVal != 0)
        {
            G4_SrcRegRegion* immSrcRegRgn = createSrcRegRegion(Mod_src_undef, Direct, msg->getRegVar(), 0, 0, getRegionScalar(), Type_UD);

            G4_DstRegRegion* immDstRegRgn = createDst(msg->getRegVar(), 0, 0, 1, Type_UD);

            G4_INST* immOrInst = createBinOp(G4_or, g4::SIMD1, immDstRegRgn, immSrcRegRgn, createImm(orImmVal, Type_UD), InstOpt_WriteEnable, true);
            immOrInst->setOptionOn(InstOpt_WriteEnable);
        }
    }

    // Check whether coalescing is possible
#define UNINITIALIZED_DWORD 0xffffffff
    unsigned int offset = UNINITIALIZED_DWORD;
    // If the header is not present or split-send is available, we will try to
    // coalesc payload by checking whether the source is already prepared in a
    // continuous region. If so, we could reuse the source region directly
    // instead of copying it again.
    bool canCoalesce = !useHeader || useSplitSend;
    G4_SrcRegRegion* prevRawOpnd = NULL;

    if (R->isNullReg()  ||
        G->isNullReg()  ||
        B->isNullReg()  ||
        A->isNullReg())
        canCoalesce = false;

    if (canCoalesce && cntrls.s0aPresent)
    {
        prevRawOpnd = s0a;
        offset = getByteOffsetSrcRegion(s0a);
    }

    if (canCoalesce && cntrls.oMPresent)
    {
        //by default it will check based on first opnd type, but that can be HF, F, we need second operand type
        //according to spec oM is UW
        canCoalesce = checkIfRegionsAreConsecutive(prevRawOpnd, oM, execSize, oM->getType());
        prevRawOpnd = oM;
        if (offset == UNINITIALIZED_DWORD)
        {
            offset = getByteOffsetSrcRegion(oM);
        }
    }

    if (canCoalesce)
    {
        if (execSize == 16 && cntrls.oMPresent)
        {
            // oM is 1 GRF for SIMD16 since it is UW type
            canCoalesce = checkIfRegionsAreConsecutive(oM, R, execSize, Type_UW);
            prevRawOpnd = R;
        }
        else
        {
            canCoalesce = checkIfRegionsAreConsecutive(prevRawOpnd, R, execSize);
            prevRawOpnd = R;
        }

        if (offset == UNINITIALIZED_DWORD)
        {
            offset = getByteOffsetSrcRegion(prevRawOpnd);
        }

        if (canCoalesce)
        {
            auto tempExecSize = execSize;
            if (FP16Data && execSize == 8)
                tempExecSize = g4::SIMD16;
            canCoalesce = checkIfRegionsAreConsecutive(prevRawOpnd, G, tempExecSize) &&
                checkIfRegionsAreConsecutive(G, B, tempExecSize) &&
                checkIfRegionsAreConsecutive(B, A, tempExecSize);
            prevRawOpnd = A;
            if (offset == UNINITIALIZED_DWORD)
            {
                offset = getByteOffsetSrcRegion(A);
                if (FP16Data && execSize == g4::SIMD8)
                    offset += 8;
            }
        }
    }

    if (canCoalesce && cntrls.zPresent)
    {
        canCoalesce = checkIfRegionsAreConsecutive(prevRawOpnd, Z, execSize);
        prevRawOpnd = Z;
    }

    if (canCoalesce && cntrls.isStencil)
    {
        canCoalesce = checkIfRegionsAreConsecutive(prevRawOpnd, S, execSize);
        prevRawOpnd = S;
    }

    if (canCoalesce == false)
    {
        // Copy parms to payload
        unsigned regOff = 0;

        if (cntrls.s0aPresent)
        {

            Copy_SrcRegRegion_To_Payload(payloadFOrHF, regOff, s0a, execSize, instOpt);
        }

        if (cntrls.oMPresent)
        {
            Copy_SrcRegRegion_To_Payload(payloadUW, regOff, oM, execSize, instOpt);
            //Copy_SrcRegRegion_To_Payload increments regOff by 1 if byteSize ==2
            //works for oM since in SIMD16 it occupies one GRF
        }


        //   When RT write is HF s0a,R, G, B, A are allowed to be HF.
        //   In SIMD8 upper DWORDS are reserved.
        //   In SIMD16 uppder DOWRDS contain second grf worth of values if type was F.
        //
        // Output can be only Depth, so V0 is passed in if RGBA don't need to be outputted
        auto offIncrement = 2;
        if (execSize == 8 || FP16Data)
            offIncrement = 1;

        if (!R->isNullReg())
            Copy_SrcRegRegion_To_Payload(payloadFOrHF, regOff, R, execSize, instOpt);
        else
            regOff+= offIncrement;

        if (!G->isNullReg())
            Copy_SrcRegRegion_To_Payload(payloadFOrHF, regOff, G, execSize, instOpt);
        else
            regOff+= offIncrement;

        if (!B->isNullReg())
            Copy_SrcRegRegion_To_Payload(payloadFOrHF, regOff, B, execSize, instOpt);
        else
            regOff+= offIncrement;

        if (!A->isNullReg())
            Copy_SrcRegRegion_To_Payload(payloadFOrHF, regOff, A, execSize, instOpt);
        else
            regOff += offIncrement;

        if (cntrls.zPresent)
        {
            Copy_SrcRegRegion_To_Payload(payloadF, regOff, Z, execSize, instOpt);
        }

        if (cntrls.isStencil)
        {
            Copy_SrcRegRegion_To_Payload(payloadFOrHF, regOff, S, execSize, InstOpt_WriteEnable);
        }

        srcToUse = Create_Src_Opnd_From_Dcl(payloadUD, getRegionStride1());
    }
    else
    {
        // Coalesce and directly use original raw operand
        G4_Declare *dcl = R->getBase()->asRegVar()->getDeclare();
        srcToUse = createSrcRegRegion(Mod_src_undef, Direct, dcl->getRegVar(), offset / 32, 0, getRegionStride1(), R->getType());
    }

    // Now create message message descriptor
    // 7:0 - BTI
    // 10:8 - Render Target Message Subtype
    // 11 - Slot Group Select
    // 12 - Last Render Target Select
    // 13 - Reserved (DevBDW)
    // 13 - Per-Sample PS Outputs Enable (DevSKL+)
    // 17:14 - Message Type
    // 18 - Reserved
    // 19 - Header Present
    // 24:20 - Response Length
    // 28:25 - Message Length
    // 29 - Reserved
    // 30 - Message Precision Subtype (DevBDW+)
    // 31 - Reserved (MBZ)
    unsigned int fc = 0;

    //making explicit
    EU_GEN6_DATA_PORT_RENDER_TARGET_WRITE_CONTROL messageType =
        (executionSize == EXEC_SIZE_8)
        ? EU_GEN6_DATA_PORT_RENDER_TARGET_WRITE_CONTROL_SIMD8_SINGLE_SOURCE_LOW
        : EU_GEN6_DATA_PORT_RENDER_TARGET_WRITE_CONTROL_SIMD16_SINGLE_SOURCE;

#define RENDER_TARGET_MESSAGE_SUBTYPE_OFFSET 8
    fc |= (messageType << RENDER_TARGET_MESSAGE_SUBTYPE_OFFSET);

#define SLOT_GROUP_SELECT_OFFSET 11
    //for SIMD32 for second RT Write setting this bit
    if (emask == vISA_EMASK_M5_NM || emask == vISA_EMASK_M5)
        fc |= (0x1 << SLOT_GROUP_SELECT_OFFSET);

    if (cntrls.isLastWrite)
    {
#define LAST_RENDER_TARGET_SELECT_OFFSET 12
        fc |= (0x1 << LAST_RENDER_TARGET_SELECT_OFFSET);
    }

    if (cntrls.isPerSample)
    {
#define PER_SAMPLE_PS_ENABLE_OFFSET 13
        fc += (0x1 << PER_SAMPLE_PS_ENABLE_OFFSET);
    }

    if (FP16Data)
    {
        fc |= 0x1 << MESSAGE_PRECISION_SUBTYPE_OFFSET;
    }

#define MESSAGE_TYPE 14
    fc |= (0xc << MESSAGE_TYPE);

#define COARSE_PIXEL_OUTPUT_ENABLE 18
    if (cntrls.isCoarseMode)
        fc |= 0x1 << COARSE_PIXEL_OUTPUT_ENABLE;
#define CPS_COUNTER_EXT_MSG_DESC_OFFSET 16

    uint16_t extFuncCtrl = 0;
    if (cntrls.isNullRT && getPlatform() >= GENX_TGLLP)
    {
        // extFuncCtrl is the 16:31 bits of extDesc. NullRT is the bit 20 of extDesc.
        // That says NullRT is the bit 4 of extFuncCtrl.
#define NULL_RENDER_TARGET 4
        extFuncCtrl |= 0x1 << NULL_RENDER_TARGET;
    }

    if (useSplitSend || cpsCounter)
    {
        G4_SendMsgDescriptor *msgDesc = NULL;
        G4_SrcRegRegion *m0 = NULL;
        bool indirectExDesc = false;
        if (useHeader)
        {
            m0 = Create_Src_Opnd_From_Dcl(msg, getRegionStride1());
            msgDesc = createSendMsgDesc(fc, 0, numHeaderGRF, SFID::DP_WRITE, numRows,
                extFuncCtrl, SendAccess::WRITE_ONLY, surface);
            msgDesc->setHeaderPresent(useHeader);
        }
        else
        {
            if (!isRTIdxNonzero && !cntrls.s0aPresent)
            {
                // direct imm is a-ok for ext desc
                msgDesc = createSendMsgDesc(fc, 0, numRows, SFID::DP_WRITE, 0,
                    extFuncCtrl, SendAccess::WRITE_ONLY, surface);
            }
            else
            {
                assert(rtIndex->isImm() && "RTIndex must be imm at this point");
                uint8_t RTIndex = (uint8_t)rtIndex->asImm()->getImm() & 0x7;
                uint32_t desc = G4_SendMsgDescriptor::createDesc(fc, false, numRows, 0);
                uint32_t extDesc = G4_SendMsgDescriptor::createMRTExtDesc(cntrls.s0aPresent, RTIndex,
                    false, 0, extFuncCtrl);
                msgDesc = createGeneralMsgDesc(desc, extDesc, SendAccess::WRITE_ONLY, surface);

                if (!canEncodeFullExtDesc())
                {
                    // we must use a0 for extended msg desc in this case as there aren't enough bits to encode
                    // the full ext desc
                    // mov (1) a0.2:ud extDesc
                    G4_DstRegRegion* dst = Create_Dst_Opnd_From_Dcl(getBuiltinA0Dot2(), 1);
                    createMov(g4::SIMD1, dst, createImm(extDesc, Type_UD), InstOpt_WriteEnable, true);
                    indirectExDesc = true;
                }
            }
        }

        /*
        If we need to set cps counter then ext_message descriptor
        needs to be a register.
        */
        if (cpsCounter)
        {
            ASSERT_USER(hasCPS(), "CPS counter is not supported");
            unsigned msgDescValue = msgDesc->getExtendedDesc();

            //shifting CPS counter by appropriate number of bits and storing in ext_descriptor operand
            G4_DstRegRegion *dstMove2 = Create_Dst_Opnd_From_Dcl(getBuiltinA0Dot2(), 1);
            G4_Imm *immedOpnd = createImm(msgDescValue, Type_UD);

            ///setting lower bits
            createBinOp(G4_or, g4::SIMD1, dstMove2, cpsCounter, immedOpnd, InstOpt_WriteEnable, true);
            indirectExDesc = true;
        }

        if (!useHeader)
        {
            m0 = srcToUse;
            srcToUse = createNullSrc(Type_UD);
        }

        Create_SplitSend_Inst_For_RT(
            pred,
            createNullDst(Type_UD),
            m0,
            srcToUse,
            indirectExDesc ? Create_Src_Opnd_From_Dcl(getBuiltinA0Dot2(), getRegionScalar()) : nullptr,
            execSize,
            msgDesc,
            instOpt);
    }
    else
    {
        G4_SrcRegRegion *m = srcToUse;
        if (useHeader)
            m = Create_Src_Opnd_From_Dcl(msg, getRegionStride1());
        Create_Send_Inst_For_CISA(pred, createNullDst(Type_UD), m, numRows, 0,
            execSize, fc, SFID::DP_WRITE, useHeader, SendAccess::WRITE_ONLY, surface, NULL, instOpt, true);
    }
    return VISA_SUCCESS;

}


// Bit 15 of aoffimmi is set in messages with sampler index >= 16.
static bool IsSamplerIndexGE16(G4_Operand* aoffimmi)
{
    bool ret = false;
    if (aoffimmi && aoffimmi->isImm())
    {
        const uint16_t aoffimmiVal = (uint16_t)aoffimmi->asImm()->getInt();
        ret = (aoffimmiVal & 0x8000) != 0;
    }
    return ret;
}


// return the contents of M0.2 for sampler messages.  It must be an immediate value
static uint32_t createSampleHeader0Dot2(VISASampler3DSubOpCode op,
    bool pixelNullMask,
    uint16_t aoffimmi,
    ChannelMask channels,
    IR_Builder* builder)
{
    uint32_t secondDword = aoffimmi & 0xfff;
    switch (op)
    {
    case VISA_3D_GATHER4:
    case VISA_3D_GATHER4_PO:
        //gather4 source channel select
        secondDword |= (channels.getSingleChannel() << 16);
        break;
    case VISA_3D_GATHER4_C:
    case VISA_3D_GATHER4_PO_C:
        // do nothing as channle must be Red (0)
        break;
    default:
        // RGBA write channel mask
        secondDword |= (channels.getHWEncoding() << 12);
        break;
    }

    // M0.2:23, Pixel Null Mask Enable.
    // Only valid for SKL+, and ignored otherwise.
    if (builder->hasPixelNullMask() && pixelNullMask)
    {
        secondDword |= 1 << 23;
    }

    return secondDword;
}

//
// Coarse Pixel Shading(CPS) LOD compensation enable.
//
// - must be disabled if the response length of the message is zero;
// - must be disabled if the messages is from a 32-pixel dispatch thread;
// - must be disabled unless SIMD Mode is SIMD8* or SIMD16*;
// - only available for sample, sample_b, sample_bc, sample_c, and LOD.
//
static void checkCPSEnable(VISASampler3DSubOpCode op,
    unsigned reponseLength,
    unsigned execSize)
{

    ASSERT_USER(reponseLength > 0,
        "CPS LOD Compensation Enable must be disabled if the "
        "response length is zero");

    ASSERT_USER(execSize == 8 || execSize == 16,
        "CPS LOD Compensation Enable only valid for SIMD8* or SIMD16*");

    ASSERT_USER(op == VISA_3D_SAMPLE ||
        op == VISA_3D_SAMPLE_B ||
        op == VISA_3D_SAMPLE_C ||
        op == VISA_3D_SAMPLE_B_C ||
        op == VISA_3D_LOD,
        "CPD LOD Compensation Enable only available for "
        "sample, sample_b, sample_bc, sample_c and LOD");
}

static G4_Operand* createSampleHeader(
    IR_Builder* builder, G4_Declare* header, VISASampler3DSubOpCode actualop,
    bool pixelNullMask, G4_Operand* aoffimmi, ChannelMask srcChannel, G4_Operand* sampler)
{
    G4_Operand* retSampler = sampler;
    uint16_t aoffimmiVal = aoffimmi->isImm() ? (uint16_t)aoffimmi->asImm()->getInt() : 0;

    unsigned int secondDword = createSampleHeader0Dot2(actualop, pixelNullMask, aoffimmiVal, srcChannel, builder);

    G4_Imm* immOpndSecondDword = builder->createImm(secondDword, Type_UD);
    G4_DstRegRegion* payloadDstRgn = builder->createDst(header->getRegVar(), 0, 2, 1, Type_UD);
    if (aoffimmi->isImm())
    {
        // mov (1) payload(0,2) immOpndSecondDword
        builder->createMov(g4::SIMD1, payloadDstRgn, immOpndSecondDword, InstOpt_WriteEnable, true);
    }
    else
    {
        // or (1) payload(0,2) aoffimmi<0;1,0>:uw immOpndSeconDword
        builder->createBinOp(G4_or, g4::SIMD1, payloadDstRgn,
            aoffimmi, immOpndSecondDword, InstOpt_WriteEnable, true);
    }

    if (sampler != nullptr)
    {
        builder->doSamplerHeaderMove(header, sampler);

        // Use bit 15 of aoffimmi to tell VISA the sample index could be greater
        // than 15.  In this case, we need to use msg header, and setup M0.3
        // to point to next 16 sampler state.
        if (IsSamplerIndexGE16(aoffimmi))
        {
            retSampler = builder->emitSampleIndexGE16(sampler, header);
        }
    }

    return retSampler;
}


static bool needsNoMaskCoordinates(VISASampler3DSubOpCode opcode)
{
    return opcode == VISA_3D_SAMPLE || opcode == VISA_3D_SAMPLE_B || opcode == VISA_3D_SAMPLE_C ||
        opcode == VISA_3D_SAMPLE_B_C || opcode == VISA_3D_LOD || opcode == VISA_3D_SAMPLE_KILLPIX;
}

static uint8_t getUPosition(VISASampler3DSubOpCode opcode)
{
    uint8_t position = 0;
    switch (opcode)
    {
    case VISA_3D_SAMPLE:
    case VISA_3D_LOD:
    case VISA_3D_SAMPLE_D:
    case VISA_3D_SAMPLE_LZ:
    case VISA_3D_SAMPLE_KILLPIX:
        position = 0;
        break;
    case VISA_3D_SAMPLE_B:
    case VISA_3D_SAMPLE_L:
    case VISA_3D_SAMPLE_C:
    case VISA_3D_SAMPLE_D_C:
    case VISA_3D_SAMPLE_C_LZ:
        position = 1;
        break;
    case VISA_3D_SAMPLE_B_C:
    case VISA_3D_SAMPLE_L_C:
        position = 2;
        break;
    default:
        MUST_BE_TRUE(false, "unexpected sampler operation");
        return 0;
    }
    return position;
}

static void setUniformSampler(G4_InstSend* sendInst, bool uniformSampler)
{
    if (!uniformSampler)
    {
        sendInst->setSerialize();
    }
}

/*
Need to split sample_d and sample_dc in to two simd8 sends since HW doesn't support it.
Also need to split any sample instruciton that has more then 5 parameters. Since there is a limit on msg length.
*/
static unsigned TmpSmplDstID = 0;

// TODO: use IR_Builder::getNameString....
const char* getNameString(
    Mem_Manager& mem, size_t size, const char* format, ...)
{
#ifdef _DEBUG
    char* name = (char*) mem.alloc(size);
    va_list args;
    va_start(args, format);
    std::vsnprintf(name, size, format, args);
    va_end(args);
    return name;
#else
    const char* name = "";
    return const_cast<char*>(name);
#endif
}

// split simd32/16 sampler messages into simd16/8 messages due to HW limitation.
int IR_Builder::splitSampleInst(
    VISASampler3DSubOpCode actualop,
    bool pixelNullMask,
    bool cpsEnable,
    G4_Predicate* pred,
    ChannelMask srcChannel,
    int numChannels,
    G4_Operand *aoffimmi,
    G4_Operand *sampler,
    G4_Operand *surface,
    G4_DstRegRegion* dst,
    VISA_EMask_Ctrl emask,
    bool useHeader,
    unsigned numRows, // msg length for each simd8
    unsigned int numParms,
    G4_SrcRegRegion ** params,
    bool uniformSampler)
{
    int status = VISA_SUCCESS;
    G4_SrcRegRegion *secondHalf[12];

    bool isHalfReturn = G4_Type_Table[dst->getType()].byteSize == 2;
    const bool halfInput = G4_Type_Table[params[0]->getType()].byteSize == 2;

    // Now, depending on message type emit out parms to payload
    unsigned regOff = (useHeader ? 1 : 0);
    G4_SrcRegRegion* temp = nullptr;
    G4_ExecSize execSize = getNativeExecSize();
    uint16_t numElts = numRows * numEltPerGRF(Type_F);
    G4_Declare* payloadF = createSendPayloadDcl(numElts, Type_F);
    G4_Declare* payloadUD = createTempVar(numElts, Type_UD, GRFALIGN);
    payloadUD->setAliasDeclare(payloadF, 0);
    G4_SrcRegRegion* srcToUse = createSrcRegRegion(Mod_src_undef, Direct, payloadUD->getRegVar(), 0, 0, getRegionStride1(), Type_UD);

    // even though we only use lower half of the GRF, we have to allocate full GRF
    G4_Declare* payloadHF = createTempVar(numElts * 2, Type_HF, Any);
    payloadHF->setAliasDeclare(payloadF, 0);

    /********* Creating temp destination, since results are interleaved **************/
    G4_DstRegRegion *dst1 = createNullDst(dst->getType());
    G4_Declare * originalDstDcl = nullptr;
    G4_Declare* tempDstDcl = nullptr;
    bool pixelNullMaskEnable = false;
    unsigned tmpDstRows = 0;
    if (!dst->isNullReg())
    {
        originalDstDcl = dst->getBase()->asRegVar()->getDeclare();
        tmpDstRows = numChannels;

        // If Pixel Null Mask is enabled, then one extra GRF is needed for the
        // write back message.
        pixelNullMaskEnable = hasPixelNullMask() && pixelNullMask;
        if (pixelNullMaskEnable) {
            ASSERT_USER(useHeader, "pixel null mask requires a header");
            ++tmpDstRows;
        }

        const char *name = getNameString(mem, 20, "%s%d", "TmpSmplDst_", TmpSmplDstID++);

        tempDstDcl = createDeclareNoLookup(name,
            originalDstDcl->getRegFile(),
            originalDstDcl->getNumElems(),
            (uint16_t)tmpDstRows,
            originalDstDcl->getElemType());

        dst1 = createDstRegRegion(dst->getRegAccess(),
            tempDstDcl->getRegVar(),
            0,
            0,
            1,
            dst->getType());
    }
    /********* End creating temp destination ***********************/

    G4_Declare* header = nullptr;

    if (useHeader)
    {
        const bool samplerIndexGE16 = IsSamplerIndexGE16(aoffimmi);
        bool bindlessSampler = sampler ? isBindlessSampler(sampler) : false;
        header = getSamplerHeader(bindlessSampler, samplerIndexGE16);
        sampler = createSampleHeader(this, header, actualop, pixelNullMask, aoffimmi, srcChannel, sampler);
        Create_MOV_Inst(payloadUD, 0, 0, g4::SIMD8, nullptr, nullptr,
            Create_Src_Opnd_From_Dcl(header, getRegionStride1()), true);
    }

    G4_InstOpts instOpt = Get_Gen4_Emask(emask, execSize);
    for (unsigned paramCounter = 0; paramCounter < numParms; ++paramCounter)
    {
        temp = params[paramCounter];
        uint32_t MovInstOpt = InstOpt_WriteEnable;
        if (G4_Type_Table[temp->getType()].byteSize == 2)
        {
            // we should generate
            // mov (8) dst<1>:hf src.0<8;8,1>:hf
            G4_DstRegRegion* dstHF = createDst(
                payloadHF->getRegVar(), regOff++, 0, 1, temp->getType());
            temp->setRegion(getRegionStride1());
            createMov(g4::SIMD8, dstHF, temp, MovInstOpt, true);
        }
        else
        {
            Copy_SrcRegRegion_To_Payload(payloadF, regOff, temp, execSize, MovInstOpt);
        }
    }

    uint32_t responseLength = getSamplerResponseLength(numChannels, isHalfReturn, execSize,
        pixelNullMaskEnable, dst->isNullReg());

    uint32_t fc = createSamplerMsgDesc(actualop, execSize == getNativeExecSize(), isHalfReturn, halfInput);
    uint32_t desc = G4_SendMsgDescriptor::createDesc(fc, useHeader, numRows, responseLength);

    if (cpsEnable)
    {
        checkCPSEnable(actualop, responseLength, 8);
    }
    G4_SendMsgDescriptor *msgDesc = createSampleMsgDesc(desc, cpsEnable, 0, surface, sampler);

    G4_InstSend* sendInst = nullptr;
    bool forceSplitSend = shouldForceSplitSend(surface);

    if (forceSplitSend)
    {
        sendInst = Create_SplitSend_Inst(
            pred, dst1, srcToUse, createNullSrc(Type_UD), execSize, msgDesc, instOpt, false);
    }
    else
    {
        sendInst = Create_Send_Inst_For_CISA(
            pred, dst1, srcToUse, execSize, msgDesc, instOpt, false);
    }
    setUniformSampler(sendInst, uniformSampler);

    // SKL+
    // For SIMD8
    //
    // W4.7:1 Reserved (not written): This W4 is only delivered when Pixel Null
    //        Mask Enable is enabled.
    //
    // W4.0  32:8 Reserved: always written as 0xffffff
    //        7:0 Pixel Null Mask: This field has the bit for all pixels set
    //            to 1 except those pixels in which a null page was source for
    //            at least one texel.
    //
    // Need to combine the results from the above two writewback messages.
    // Denote by U0[W4:0] the last row of the first writeback message, and
    // by U1[W4:0] the last row of the second writeback message. Then the last
    // row of the whole writeback message is to take the bitwise OR of
    // U0[W4:0] and U1[W4:0].
    G4_Declare *tempDstUD = 0;
    G4_Declare *tempDst2UD = 0;
    G4_Declare *origDstUD = 0;

    // temp dst for the second send
    G4_DstRegRegion *dst2 = createNullDst(dst->getType());
    G4_Declare* tempDstDcl2 = nullptr;
    if (!dst->isNullReg())
    {
        const char *name = getNameString(mem, 20, "%s%d", "TmpSmplDst2_", TmpSmplDstID++);

        tempDstDcl2 = createDeclareNoLookup(name,
            originalDstDcl->getRegFile(),
            originalDstDcl->getNumElems(),
            (uint16_t)tmpDstRows,
            originalDstDcl->getElemType());

        if (pixelNullMaskEnable)
        {
            unsigned int numElts = tempDstDcl->getNumElems() * tempDstDcl->getNumRows();
            tempDstUD = createTempVar(numElts, Type_UD, GRFALIGN);
            tempDstUD->setAliasDeclare(tempDstDcl, 0);

            numElts = tempDstDcl2->getNumElems() * tempDstDcl2->getNumRows();
            tempDst2UD = createTempVar(numElts, Type_UD, GRFALIGN);
            tempDst2UD->setAliasDeclare(tempDstDcl2, 0);

            numElts = originalDstDcl->getNumElems() * originalDstDcl->getNumRows();
            origDstUD = createTempVar(numElts, Type_UD, GRFALIGN);
            origDstUD->setAliasDeclare(originalDstDcl, 0);
        }

        dst2 = createDstRegRegion(dst->getRegAccess(),
            tempDstDcl2->getRegVar(),
            0,
            0,
            1,
            dst->getType());
    }
    // update emask
    emask = Get_Next_EMask(emask, execSize);
    G4_InstOpts instOpt2 = Get_Gen4_Emask(emask, execSize);

    auto dupPredicate = [this](G4_Predicate* pred)
    {
        G4_Predicate* pred2 = nullptr;
        if (pred)
        {
            pred2 = createPredicate(
                pred->getState(),
                pred->getBase(),
                0);
        }

        return pred2;
    };

    {
        /**************** SECOND HALF OF THE SEND *********************/
        // re-create payload declare so the two sends may be issued independently
        G4_Declare* payloadF = createSendPayloadDcl(numElts, Type_F);
        G4_Declare* payloadUD = createTempVar(numElts, Type_UD, GRFALIGN);
        payloadUD->setAliasDeclare(payloadF, 0);

        // even though we only use lower half of the GRF, we have to allocate full GRF
        G4_Declare* payloadHF = createTempVar(numElts * 2, Type_HF, Any);
        payloadHF->setAliasDeclare(payloadF, 0);

        G4_SrcRegRegion *srcToUse2 = createSrcRegRegion(Mod_src_undef, Direct, payloadUD->getRegVar(), 0, 0, getRegionStride1(), Type_UD);

        if (useHeader)
        {
            Create_MOV_Inst(payloadUD, 0, 0, g4::SIMD8, nullptr, nullptr,
                Create_Src_Opnd_From_Dcl(header, getRegionStride1()), true);
        }

        for (unsigned int i = 0; i < numParms; i++)
        {
            if (params[i]->isNullReg())
            {
                secondHalf[i] = params[i];
            }
            else if (getTypeSize(params[i]->getType()) == 2)
            {
                // V1(0,8)<8;8,1>
                secondHalf[i] = createSrcWithNewSubRegOff(params[i], execSize);
            }
            else
            {
                // V1(1,0)<8;8,1>
                secondHalf[i] = createSrcWithNewRegOff(params[i], params[i]->getRegOff() + 1);
            }
        }

        regOff = (useHeader ? 1 : 0);
        for (unsigned paramCounter = 0; paramCounter < numParms; ++paramCounter)
        {
            temp = secondHalf[paramCounter];
            uint32_t MovInstOpt = InstOpt_WriteEnable;

            if (G4_Type_Table[temp->getType()].byteSize == 2)
            {
                // we should generate
                // mov (8) dst<1>:hf src.8<8;8,1>:hf
                G4_DstRegRegion* dstHF = createDst(
                    payloadHF->getRegVar(), regOff++, 0, 1, temp->getType());
                createMov(execSize, dstHF, temp, MovInstOpt, true);
            }
            else
            {
                Copy_SrcRegRegion_To_Payload(payloadF, regOff, temp, execSize, MovInstOpt);
            }
        }

        G4_Operand *surface2 = duplicateOperand(surface);

        // sampler may be null for 3d load (specifically ld2dms_w)
        G4_Operand* sampler2 = sampler == nullptr ? nullptr : duplicateOperand(sampler);

        G4_Predicate* pred2 = dupPredicate(pred);

        G4_SendMsgDescriptor *msgDesc2 = createSampleMsgDesc(desc, cpsEnable, 0, surface2, sampler2);
        msgDesc2->setHeaderPresent(useHeader);

        if (forceSplitSend)
        {
            sendInst = Create_SplitSend_Inst(
                pred2, dst2, srcToUse2, createNullSrc(Type_UD), execSize, msgDesc2, instOpt2, false);
        }
        else
        {
            sendInst = Create_Send_Inst_For_CISA(
                pred2, dst2, srcToUse2, execSize, msgDesc2, instOpt2, false);
        }
        setUniformSampler(sendInst, uniformSampler);
    }

    {

        /**************** MOVING FROM TEMP TO DST, 1st half *********************/
        regOff = 0;
        for (unsigned i = 0; i < tmpDstRows; i++, regOff += 1)
        {
            // If Pixel Null Mask is enabled, then only copy the last double word.
            if (pixelNullMaskEnable && i == tmpDstRows - 1)
            {
                G4_DstRegRegion *origDstPtr = createDst(origDstUD->getRegVar(), short(regOff), 0, 1, Type_UD);
                G4_SrcRegRegion *src0Ptr = createSrcRegRegion(Mod_src_undef, Direct, tempDstUD->getRegVar(),
                    short(i), 0, getRegionScalar(), Type_UD);

                G4_Predicate* pred2 = dupPredicate(pred);

                // Copy the write mask message W4.0 into the dst. (No mask?)
                createInst(pred2, G4_mov, NULL, g4::NOSAT, g4::SIMD1,
                    origDstPtr, src0Ptr, NULL, NULL,
                    InstOpt_WriteEnable, true);
                // Skip the remaining part of the loop.
                break;
            }

            G4_SrcRegRegion *tmpSrcPnt = createSrcRegRegion(
                Mod_src_undef, Direct, tempDstDcl->getRegVar(), (short)i, 0, getRegionStride1(), tempDstDcl->getElemType());

            uint32_t MovInstOpt = instOpt;
            if (isHalfReturn)
            {
                // mov (8) dst(0,0)<1>:hf tmp(0,0)<8;8,1>:hf {Q1}
                G4_DstRegRegion* dst = createDst(
                    originalDstDcl->getRegVar(), (short)regOff, 0, 1, originalDstDcl->getElemType());
                createMov(execSize, dst, tmpSrcPnt, MovInstOpt, true);
            }
            else
            {
                Copy_SrcRegRegion_To_Payload(originalDstDcl, regOff, tmpSrcPnt, execSize, MovInstOpt);
            }
        }
    }

    {
        /**************** MOVING FROM TEMP TO DST, 2nd half *********************/
        regOff = isHalfReturn ? 0 : 1;
        for (unsigned i = 0; i < tmpDstRows; i++, regOff += 1)
        {
            // If Pixel Null Mask is enabled, copy the second half to the originai dst
            if (pixelNullMaskEnable && i == tmpDstRows - 1) {
                G4_Type secondHalfType = execSize == g4::SIMD8 ? Type_UB : Type_UW;
                G4_DstRegRegion* origDstPtr = createDst(origDstUD->getRegVar(), regOff - 1, 1, 1, secondHalfType);
                G4_SrcRegRegion* src0Ptr = createSrcRegRegion(Mod_src_undef, Direct, tempDst2UD->getRegVar(),
                    short(i), 0, getRegionScalar(), secondHalfType);

                G4_Predicate* pred2 = dupPredicate(pred);
                // write to dst.0[8:15]
                createInst(pred2, G4_mov, NULL, g4::NOSAT, g4::SIMD1,
                    origDstPtr, src0Ptr, NULL, InstOpt_WriteEnable, true);

                // Skip the remaining part of the loop.
                break;
            }

            G4_SrcRegRegion *tmpSrcPnt = createSrcRegRegion(
                Mod_src_undef, Direct, tempDstDcl2->getRegVar(), (short)i, 0, getRegionStride1(), tempDstDcl->getElemType());

            uint32_t MovInstOpt = instOpt2;
            if (isHalfReturn)
            {
                // mov (8) dst(0,8)<1>:hf tmp(0,0)<8;8,1>:hf {Q2}
                G4_DstRegRegion* dst = createDst(
                    originalDstDcl->getRegVar(), (short)regOff, execSize, 1, originalDstDcl->getElemType());
                createMov(execSize, dst, tmpSrcPnt, MovInstOpt, true);
            }
            else
            {
                Copy_SrcRegRegion_To_Payload(originalDstDcl, regOff, tmpSrcPnt, execSize, MovInstOpt);
            }
        }
    }
    return status;
}

void IR_Builder::doSamplerHeaderMove(G4_Declare* headerDcl, G4_Operand* sampler)
{
    if (isBindlessSampler(sampler))
    {
        // sampler index in msg desc will be 0, manipulate the sampler offset instead
        // mov (1) M0.3<1>:ud sampler<0;1,0>:ud the driver will send the handle with bit 0 already set
        G4_DstRegRegion* dst = createDst(headerDcl->getRegVar(), 0, 3, 1, Type_UD);
        createMov(g4::SIMD1, dst, sampler, InstOpt_WriteEnable, true);
    }
}

//
// generate the r0 move for the sampler message header, and return the dcl
// for CNL+, also set SSP to dynamic if message is not bindless
//
G4_Declare* IR_Builder::getSamplerHeader(bool isBindlessSampler, bool samplerIndexGE16)
{
    G4_Declare* dcl = nullptr;

    if (m_options->getOption(vISA_cacheSamplerHeader) && !isBindlessSampler)
    {
        dcl = builtinSamplerHeader;
        if (!builtinSamplerHeaderInitialized)
        {
            builtinSamplerHeaderInitialized = true;
            if (hasBindlessSampler())
            {
                // make sure we set bit 0 of M0.3:ud to be 0
                // and (1) M0.6<1>:uw M0.6<1>:uw 0xFFFE
                G4_DstRegRegion* dst = createDst(dcl->getRegVar(), 0, 6, 1, Type_UW);
                G4_SrcRegRegion* src0 = createSrcRegRegion(Mod_src_undef, Direct, dcl->getRegVar(), 0, 6, getRegionScalar(), Type_UW);
                G4_INST* SSPMove = createBinOp(G4_and, g4::SIMD1, dst, src0, createImm(0xFFFE, Type_UW), InstOpt_WriteEnable, false);
                instList.push_front(SSPMove);
            }
            G4_INST* r0Move = createMov(g4::SIMD8,
                Create_Dst_Opnd_From_Dcl(dcl, 1),
                Create_Src_Opnd_From_Dcl(builtinR0, getRegionStride1()),
                InstOpt_WriteEnable, false);
            instList.push_front(r0Move);
        }
        if (samplerIndexGE16)
        {
            // When sampler index is greater or equal 16 then the
            // createSamplerHeader() message overwrites the sampler states
            // pointer in the header -> cannot use the cached value in this
            // case.
            dcl = createSendPayloadDcl(GENX_DATAPORT_IO_SZ, Type_UD);
            dcl->setCapableOfReuse();
            G4_SrcRegRegion* src = createSrcRegRegion(Mod_src_undef, Direct, builtinSamplerHeader->getRegVar(), 0, 0, getRegionStride1(), Type_UD);
            Create_MOV_Inst(dcl, 0, 0, g4::SIMD8, NULL, NULL, src);
        }
    }
    else
    {
        dcl = createSendPayloadDcl(GENX_DATAPORT_IO_SZ, Type_UD);
        dcl->setCapableOfReuse();
        Create_MOVR0_Inst(dcl, 0, 0, true);
        if (hasBindlessSampler() && !isBindlessSampler)
        {
            // make sure we set bit 0 of M0.3:ud to be 0
            // and (1) M0.6<1>:uw M0.6<1>:uw 0xFFFE
            G4_DstRegRegion* dst = createDst(dcl->getRegVar(), 0, 6, 1, Type_UW);
            G4_SrcRegRegion* src0 = createSrcRegRegion(Mod_src_undef, Direct, dcl->getRegVar(), 0, 6, getRegionScalar(), Type_UW);
            createBinOp(G4_and, g4::SIMD1, dst, src0, createImm(0xFFFE, Type_UW), InstOpt_WriteEnable, true);
        }
    }

    return dcl;
}

// get the number of GRFs occupied by a sampler message's operand
static uint32_t getNumGRF(bool isFP16, int execSize)
{
    int numBytes = (isFP16 ? 2 : 4) * execSize;
    return (numBytes + getGRFSize() - 1) / getGRFSize();
}

uint32_t IR_Builder::getSamplerResponseLength(
    int numChannels, bool isFP16, int execSize, bool pixelNullMask, bool nullDst)
{
    if (nullDst)
    {
        hasNullReturnSampler = true;
        return 0;
    }
    uint32_t responseLength = numChannels * getNumGRF(isFP16, execSize);

    if (pixelNullMask)
    {
        ++responseLength;
    }
    return responseLength;
}

static bool needSamplerHeader(
    IR_Builder* builder, bool pixelNullMask, bool nonZeroAoffImmi,
    bool needHeaderForChannels, bool bindlessSampler, bool simd16HFReturn)
{
    return builder->forceSamplerHeader() ||
        (pixelNullMask && builder->hasPixelNullMask()) ||
        nonZeroAoffImmi || needHeaderForChannels || bindlessSampler ||
        (simd16HFReturn && VISA_WA_CHECK(builder->getPWaTable(), WaHeaderRequiredOnSimd16Sample16bit));
}

// This function assumes there are no gaps in parameter array. e.g. NULL pointers
// If there is a gap it must be RawOperand with value 0.
int IR_Builder::translateVISASampler3DInst(
    VISASampler3DSubOpCode actualop,
    bool pixelNullMask,
    bool cpsEnable,
    bool uniformSampler,
    G4_Predicate* pred,
    VISA_Exec_Size executionSize,
    VISA_EMask_Ctrl emask,
    ChannelMask chMask,
    G4_Operand *aoffimmi,
    G4_Operand *sampler,
    G4_Operand *surface,
    G4_DstRegRegion* dst,
    unsigned int numParms,
    G4_SrcRegRegion ** params)
{
    TIME_SCOPE(VISA_BUILDER_IR_CONSTRUCTION);

    G4_ExecSize execSize = toExecSize(executionSize);
    G4_InstOpts instOpt = Get_Gen4_Emask(emask, execSize);

    // First setup message header and message payload

    // Message header and payload size is numParms GRFs

    const bool FP16Return = G4_Type_Table[dst->getType()].byteSize == 2;
    const bool FP16Input = params[0]->getType() == Type_HF;

    bool useHeader = false;

    unsigned int numRows = numParms * getNumGRF(FP16Input, execSize);

    VISAChannelMask channels = chMask.getAPI();
    // For SKL+ channel mask R, RG, RGB, and RGBA may be derived from response length
    bool needHeaderForChannels = (getPlatform() < GENX_SKL) ? channels != CHANNEL_MASK_RGBA :
        (channels != CHANNEL_MASK_R && channels != CHANNEL_MASK_RG && channels != CHANNEL_MASK_RGB && channels != CHANNEL_MASK_RGBA);

    bool nonZeroAoffImmi = !(aoffimmi->isImm() && aoffimmi->asImm()->getInt() == 0);
    bool simd16HFReturn = FP16Return && execSize == 16;
    if (needSamplerHeader(this, pixelNullMask, nonZeroAoffImmi, needHeaderForChannels,
        isBindlessSampler(sampler), simd16HFReturn))
    {
        useHeader = true;
        ++numRows;
    }

    int numChannels = chMask.getNumEnabledChannels();

    if (execSize > getNativeExecSize() &&
        (numRows > 11 || actualop == VISA_3D_SAMPLE_D || actualop == VISA_3D_SAMPLE_D_C || actualop == VISA_3D_SAMPLE_KILLPIX))
    {
        // decrementing since we will produce SIMD8 code.
        // don't do this for SIMD16H since its message length is the same as SIMD8H
        if (!FP16Input)
        {
            numRows -= numParms;
        }

        return splitSampleInst(actualop, pixelNullMask, cpsEnable, pred, chMask,
            numChannels, aoffimmi, sampler, surface, dst,
            emask, useHeader, numRows, numParms, params, uniformSampler);
    }

    bool useSplitSend = useSends();

    G4_SrcRegRegion *header = 0;
    G4_Operand* samplerIdx = sampler;

        if (useHeader)
        {
            const bool samplerIndexGE16 = IsSamplerIndexGE16(aoffimmi);
            G4_Declare *dcl = getSamplerHeader(isBindlessSampler(sampler), samplerIndexGE16);
            samplerIdx = createSampleHeader(this, dcl, actualop, pixelNullMask, aoffimmi, chMask, sampler);
            header = Create_Src_Opnd_From_Dcl(dcl, getRegionStride1());
        }

    // Collect payload sources.
    unsigned len = numParms + (header ? 1 : 0);
    std::vector<PayloadSource> sources(len);
    unsigned i = 0;
    // Collect header if present.
    if (header) {
        sources[i].opnd = header;
        sources[i].execSize = g4::SIMD8;
        sources[i].instOpt = InstOpt_WriteEnable;
        ++i;
    }
    // Collect all parameters.
    bool needNoMask = needsNoMaskCoordinates(actualop);
    unsigned uPos = needNoMask ? getUPosition(actualop) : ~0u;
    for (unsigned j = 0; j != numParms; ++j) {
        sources[i].opnd = params[j];
        sources[i].execSize = execSize;
        sources[i].instOpt = (needNoMask && (uPos <= j && j < (uPos + 3))) ?
            InstOpt_WriteEnable : instOpt;
        ++i;
    }
    ASSERT_USER(i == len, "There's mismatching during payload source collecting!");

    G4_SrcRegRegion *msgs[2] = {0, 0};
    unsigned sizes[2] = {0, 0};
    preparePayload(msgs, sizes, execSize, useSplitSend, sources.data(), len);

    uint32_t responseLength = getSamplerResponseLength(numChannels, FP16Return, execSize,
        hasPixelNullMask() && pixelNullMask, dst->isNullReg());

    // Check if CPS LOD Compensation Enable is valid.
    if (cpsEnable)
    {
        checkCPSEnable(actualop, responseLength, execSize);
    }

    uint32_t fc = createSamplerMsgDesc(actualop, execSize == getNativeExecSize(), FP16Return, FP16Input);
    uint32_t desc = G4_SendMsgDescriptor::createDesc(fc, useHeader, sizes[0], responseLength);

    G4_InstSend* sendInst = nullptr;
    bool forceSplitSend = shouldForceSplitSend(surface);
    if (msgs[1] == 0 && !forceSplitSend)
    {
        ASSERT_USER(sizes[1] == 0, "Expect the 2nd part of the payload has zero size!");
        G4_SendMsgDescriptor *msgDesc = createSampleMsgDesc(desc, cpsEnable, 0, surface, samplerIdx);

        sendInst = Create_Send_Inst_For_CISA(pred, dst, msgs[0], execSize,
            msgDesc, instOpt, false);
    }
    else
    {
        G4_SendMsgDescriptor *msgDesc = createSampleMsgDesc(desc, cpsEnable, sizes[1], surface, samplerIdx);
        sendInst = Create_SplitSend_Inst(pred, dst, msgs[0], msgs[1],
            execSize, msgDesc, instOpt, false);
    }
    setUniformSampler(sendInst, uniformSampler);
    return VISA_SUCCESS;
}

int IR_Builder::translateVISALoad3DInst(
    VISASampler3DSubOpCode actualop,
    bool pixelNullMask,
    G4_Predicate *pred_opnd,
    VISA_Exec_Size executionSize,
    VISA_EMask_Ctrl em,
    ChannelMask channelMask,
    G4_Operand* aoffimmi,
    G4_Operand* surface,
    G4_DstRegRegion* dst,
    uint8_t numParms,
    G4_SrcRegRegion ** opndArray)
{
    TIME_SCOPE(VISA_BUILDER_IR_CONSTRUCTION);

    bool useHeader = false;

    G4_ExecSize execSize = toExecSize(executionSize);
    G4_InstOpts instOpt = Get_Gen4_Emask(em, execSize);

    const bool halfReturn = G4_Type_Table[dst->getType()].byteSize == 2;
    const bool halfInput = G4_Type_Table[opndArray[0]->getType()].byteSize == 2;

    unsigned int numRows = numParms * getNumGRF(halfInput, execSize);

    VISAChannelMask channels = channelMask.getAPI();
    // For SKL+ channel mask R, RG, RGB, and RGBA may be derived from response length
    bool needHeaderForChannels = (getPlatform() < GENX_SKL) ? channels != CHANNEL_MASK_RGBA :
        (channels != CHANNEL_MASK_R && channels != CHANNEL_MASK_RG && channels != CHANNEL_MASK_RGB && channels != CHANNEL_MASK_RGBA);

    bool nonZeroAoffImmi = !(aoffimmi->isImm() && aoffimmi->asImm()->getInt() == 0);
    bool simd16HFReturn = halfReturn && execSize == 16;
    if (needSamplerHeader(this, pixelNullMask, nonZeroAoffImmi, needHeaderForChannels, false,
        simd16HFReturn))
    {
        useHeader = true;
        ++numRows;
    }

    int numChannels = channelMask.getNumEnabledChannels();
    if (execSize > getNativeExecSize() && numRows > 11)
    {
        // decrementing since we will produce SIMD8 code.
        // don't do this for SIMD16H since its message length is the same as SIMD8H
        if (!halfInput)
        {
            numRows -= numParms;
        }
        return splitSampleInst(actualop, pixelNullMask, /*cpsEnable*/false,
            pred_opnd, channelMask, numChannels, aoffimmi,
            NULL, surface, dst, em, useHeader, numRows,
            numParms, opndArray);
    }

    bool useSplitSend = useSends();

    G4_SrcRegRegion *header = nullptr;
    if (useHeader)
    {
        G4_Declare *dcl = getSamplerHeader(false /*isBindlessSampler*/, false /*samperIndexGE16*/);
        (void) createSampleHeader(this, dcl, actualop, pixelNullMask, aoffimmi, channelMask, nullptr);
        header = Create_Src_Opnd_From_Dcl(dcl, getRegionStride1());
    }

    // Collect payload sources.
    unsigned len = numParms + (header ? 1 : 0);
    std::vector<PayloadSource> sources(len);
    unsigned i = 0;
    // Collect header if present.
    if (header) {
        sources[i].opnd = header;
        sources[i].execSize = g4::SIMD8;
        sources[i].instOpt = InstOpt_WriteEnable;
        ++i;
    }
    // Collect all parameters.
    bool needNoMask = needsNoMaskCoordinates(actualop);
    unsigned uPos = needNoMask ? getUPosition(actualop) : ~0u;
    for (unsigned j = 0; j != numParms; ++j) {
        sources[i].opnd = opndArray[j];
        sources[i].execSize = execSize;
        sources[i].instOpt = (needNoMask && (uPos <= j && j < (uPos + 3))) ?
            InstOpt_WriteEnable : instOpt;
        ++i;
    }
    ASSERT_USER(i == len, "There's mismatching during payload source collecting!");

    G4_SrcRegRegion *msgs[2] = {0, 0};
    unsigned sizes[2] = {0, 0};
    preparePayload(msgs, sizes, execSize, useSplitSend, sources.data(), len);

    uint32_t fc = createSamplerMsgDesc(actualop, execSize == getNativeExecSize(), halfReturn, halfInput);

    uint32_t responseLength = getSamplerResponseLength(numChannels, halfReturn, execSize,
        hasPixelNullMask() && pixelNullMask, dst->isNullReg());

    bool forceSplitSend = shouldForceSplitSend(surface);
    if (msgs[1] == 0 && !forceSplitSend)
    {
        Create_Send_Inst_For_CISA(pred_opnd, dst,
            msgs[0], sizes[0],
            responseLength,
            execSize, fc, SFID::SAMPLER,
            useHeader,
            SendAccess::READ_ONLY,
            surface, NULL,
            instOpt, false);
    }
    else
    {
        Create_SplitSend_Inst_For_CISA(pred_opnd, dst,
            msgs[0], sizes[0], msgs[1], sizes[1],
            responseLength,
            execSize, fc, SFID::SAMPLER,
            useHeader,
            SendAccess::READ_ONLY,
            surface, NULL,
            instOpt, false);
    }

    return VISA_SUCCESS;
}

int IR_Builder::translateVISAGather3dInst(
    VISASampler3DSubOpCode actualop,
    bool pixelNullMask,
    G4_Predicate* pred,
    VISA_Exec_Size executionSize,
    VISA_EMask_Ctrl em,
    ChannelMask channelMask,
    G4_Operand* aoffimmi,
    G4_Operand* sampler,
    G4_Operand* surface,
    G4_DstRegRegion* dst,
    unsigned int numOpnds,
    G4_SrcRegRegion ** opndArray)
{
    TIME_SCOPE(VISA_BUILDER_IR_CONSTRUCTION);

    bool useHeader = false;

    G4_ExecSize execSize = toExecSize(executionSize);
    G4_InstOpts instOpt = Get_Gen4_Emask(em, execSize);

    const bool FP16Return = G4_Type_Table[dst->getType()].byteSize == 2;
    const bool FP16Input = opndArray[0]->getType() == Type_HF;

    unsigned int numRows = numOpnds * getNumGRF(FP16Input, execSize);

    bool nonZeroAoffImmi = !(aoffimmi->isImm() && aoffimmi->asImm()->getInt() == 0);
    bool needHeaderForChannels = channelMask.getSingleChannel() != VISA_3D_GATHER4_CHANNEL_R;
    bool simd16HFReturn = FP16Return && execSize == 16;

    if (needSamplerHeader(this, pixelNullMask, nonZeroAoffImmi, needHeaderForChannels,
        isBindlessSampler(sampler), simd16HFReturn))
    {
        useHeader = true;
        ++numRows;
    }

    if (execSize > getNativeExecSize() && numRows > 11)
    {
        // decrementing since we will produce SIMD8 code.
        // don't do this for SIMD16H since its message length is the same as SIMD8H
        if (!FP16Input)
        {
            numRows -= numOpnds;
        }

        return splitSampleInst(actualop, pixelNullMask, /*cpsEnable*/false,
            pred, channelMask, 4, aoffimmi, sampler,
            surface, dst, em, useHeader, numRows,
            numOpnds, opndArray);
    }

    bool useSplitSend = useSends();

    G4_SrcRegRegion *header = nullptr;
    G4_Operand* samplerIdx = sampler;

    if (useHeader)
    {
        const bool samplerIndexGE16 = IsSamplerIndexGE16(aoffimmi);
        G4_Declare *dcl = getSamplerHeader(isBindlessSampler(sampler), samplerIndexGE16);
        samplerIdx = createSampleHeader(this, dcl, actualop, pixelNullMask, aoffimmi, channelMask, sampler);
        header = Create_Src_Opnd_From_Dcl(dcl, getRegionStride1());
    }

    // Collect payload sources.
    unsigned len = numOpnds + (header ? 1 : 0);
    std::vector<PayloadSource> sources(len);
    unsigned i = 0;
    // Collect header if present.
    if (header) {
        sources[i].opnd = header;
        sources[i].execSize = g4::SIMD8;
        sources[i].instOpt = InstOpt_WriteEnable;
        ++i;
    }
    // Collect all parameters.
    bool needNoMask = needsNoMaskCoordinates(actualop);
    unsigned uPos = needNoMask ? getUPosition(actualop) : ~0u;
    for (unsigned j = 0; j != numOpnds; ++j) {
        sources[i].opnd = opndArray[j];
        sources[i].execSize = execSize;
        sources[i].instOpt = (needNoMask && (uPos <= j && j < (uPos + 3))) ?
            InstOpt_WriteEnable : instOpt;
        ++i;
    }
    ASSERT_USER(i == len, "There's mismatching during payload source collecting!");

    G4_SrcRegRegion *msgs[2] = {0, 0};
    unsigned sizes[2] = {0, 0};
    preparePayload(msgs, sizes, execSize, useSplitSend, sources.data(), len);

    uint32_t fc = createSamplerMsgDesc(actualop, execSize == getNativeExecSize(), FP16Return, FP16Input);
    uint32_t responseLength = getSamplerResponseLength(4, FP16Return, execSize,
        hasPixelNullMask() && pixelNullMask, dst->isNullReg());

    bool forceSplitSend = shouldForceSplitSend(surface);
    if (msgs[1] == 0 && !forceSplitSend)
    {
        Create_Send_Inst_For_CISA(pred, dst, msgs[0], sizes[0],
            responseLength,
            execSize, fc, SFID::SAMPLER,
            useHeader,
            SendAccess::READ_ONLY,
            surface, samplerIdx,
            instOpt, false);
    }
    else
    {
        Create_SplitSend_Inst_For_CISA(pred, dst,
            msgs[0], sizes[0], msgs[1], sizes[1],
            responseLength,
            execSize, fc, SFID::SAMPLER,
            useHeader,
            SendAccess::READ_ONLY,
            surface, samplerIdx,
            instOpt, false);
    }

    return VISA_SUCCESS;
}


/*
* Translates Sampler Norm API intrinsic.
*
* Assuming: N = 4, channelMask=ABGR_ENABLE, surfIndex = 0x21, samplerIndex = 0x4,
*           then the generated code should look like the following for GT:
*
* .declare  VX Base=m ElementSize=4 Type=ud Total=16
* .declare  VY Base=r ElementSize=2 Type=uw Total=128
*
* mov  (8)     VX(0,0)<1>,  r0:ud
* mov  (1)     VX(0,2)<1>,  0
* mov  (1)     VX(1,1)<1>,  deltaU
* mov  (1)     VX(1,2)<1>,  u
* mov  (1)     VX(1,5)<1>,  deltaV
* mov  (1)     VX(1,6)<1>,  v
* send (16)    VY(0,0)<1>,  VX(0,0),    0x2,   0x048bc421
* mov  (128)   M(0,0)<1>,   VY(0,0)
*
* VX(0,0): message header
*
* VX(1,0): SIMD32 media payload
*
* ex_desc: 0x2 == 0010 (Target Function ID: Sampling Engine)
*
* desc: 0x048bc421 == Bit 31-29: 000 (Reserved)
*                     Bit 28-25: 0010 (Message Length =)
*                     Bit 24-20: 01000 (Response Message Length = 8)
*                     Bit 19:    1 (Header present)
*                     Bit 18:    0 (Reserved)
*                     Bit 17-16: 11 (SIMD Mode = SIMD32)
*                     Bit 15-12: 1100 (Message Type = sample_unorm media)
*                     Bit 11-8:  0000 + samplerIndex  (Sampler Index)
*                     Bit 7-0:   00000000 + surfIndex (Binding Table Index)
*
*/
int IR_Builder::translateVISASamplerNormInst(
    G4_Operand* surface,
    G4_Operand* sampler,
    ChannelMask channel,
    unsigned numEnabledChannels,
    G4_Operand* deltaUOpnd,
    G4_Operand* uOffOpnd,
    G4_Operand* deltaVOpnd,
    G4_Operand* vOffOpnd,
    G4_DstRegRegion* dst_opnd)
{
    TIME_SCOPE(VISA_BUILDER_IR_CONSTRUCTION);

    // mov (8)      VX(0,0)<1>,  r0:ud
    // add dcl for VX
    G4_Declare *dcl = createSendPayloadDcl(2 * GENX_SAMPLER_IO_SZ, Type_UD);

    // mov  VX(0,0)<1>, r0
    Create_MOVR0_Inst(dcl, 0, 0);
    /* mov (1)     VX(0,2)<1>,   0  */
    unsigned cmask = channel.getHWEncoding() << 12;
    Create_MOV_Inst(dcl, 0, 2, g4::SIMD1, NULL, NULL, createImm(cmask, Type_UD));

    G4_Declare *dcl1 = createSendPayloadDcl(GENX_DATAPORT_IO_SZ, Type_F);
    dcl1->setAliasDeclare(dcl, numEltPerGRF(Type_UB));

    // mov  (1)     VX(1,4)<1>,  deltaU
    Create_MOV_Inst(dcl1, 0, 4, g4::SIMD1, NULL, NULL, deltaUOpnd);
    // mov  (1)     VX(1,2)<1>,  u
    Create_MOV_Inst(dcl1, 0, 2, g4::SIMD1, NULL, NULL, uOffOpnd);
    // mov  (1)     VX(1,5)<1>,  deltaV
    Create_MOV_Inst(dcl1, 0, 5, g4::SIMD1, NULL, NULL, deltaVOpnd);
    // mov  (1)     VX(1,3)<1>,  v
    Create_MOV_Inst(dcl1, 0, 3, g4::SIMD1, NULL, NULL, vOffOpnd);

    // send's operands preparation
    // create a currDst for VX
    G4_SrcRegRegion* payload = Create_Src_Opnd_From_Dcl(dcl, getRegionStride1());

    G4_DstRegRegion* d = Check_Send_Dst(dst_opnd->asDstRegRegion());

    // Set bit 12-17 for the message descriptor
    unsigned temp = 0;
    temp += 0xc << 12;   // Bit 16-12 = 1100 for Sampler Message Type
    temp += 0x3 << 17;   // Bit 18-17 = 11 for SIMD32 mode

    Create_Send_Inst_For_CISA(
        NULL,
        d,
        payload,
        2,
        32*numEnabledChannels*G4_Type_Table[Type_UW].byteSize/numEltPerGRF(Type_UB),
        g4::SIMD32,
        temp,
        SFID::SAMPLER,
        1,
        SendAccess::READ_ONLY,
        surface,
        sampler,
        0,
        false);

    return VISA_SUCCESS;
}


/*
* Translates Sampler intrinsic.
*
* Assuming: N = 4, channelMask=ABGR_ENABLE, surfIndex = 0x21, samplerIndex = 0x4,
*           then the generated code should look like the following for GT:
*
* .declare  VX Base=m ElementSize=4 Type=f Total=72
* .declare  VY Base=r ElementSize=4 Type=f Total=64
* .declare  VZ Base=r ElementSize=2 Type=w Total=128 ALIAS(VY,0)
*
* mov  (8)     VX(0,0)<1>,  r0:ud
* mov  (1)     VX(0,2)<1>,  0
* mov  (16)    VX(1,0)<1>,  u
* mov  (16)    VX(3,0)<1>,  v
* mov  (16)    VX(5,0)<1>,  r
* mov  (16)    VX(7,0)<1>,  0
* send (16)    VY(0,0)<1>,  VX(0,0),    0x2,  0x128a0421
* mov  (64)    M(0,0)<1>,   VY(0,0)
*
* ex_desc: 0x2 == 0010 (Target Function ID: Sampling Engine)
*
* desc: 0x128a0421 == Bit 31-29: 000 (Reserved)
*                     Bit 28-25: 1001 (Message Length = 9 (1+2*4 for SIMD16))
*                     Bit 24-20: 01000 (Response Message Length = 8)
*                     Bit 19:    1 (Header present)
*                     Bit 18:    0 (Reserved)
*                     Bit 17-16: 10 (SIMD Mode = SIMD16)
*                     Bit 15-12: 0000 (Message Type = Sample)
*                     Bit 11-8:  0000 + samplerIndex  (Sampler Index)
*                     Bit 7-0:   00000000 + surfIndex (Binding Table Index)
*
*/
int IR_Builder::translateVISASamplerInst(
    unsigned simdMode,
    G4_Operand* surface,
    G4_Operand* sampler,
    ChannelMask channel,
    unsigned numEnabledChannels,
    G4_Operand* uOffOpnd,
    G4_Operand* vOffOpnd,
    G4_Operand* rOffOpnd,
    G4_DstRegRegion* dstOpnd)
{
    TIME_SCOPE(VISA_BUILDER_IR_CONSTRUCTION);

    // mov (8)      VX(0,0)<1>,  r0:ud
    // add dcl for VX
    unsigned num_payload_elt = simdMode/2 * numEltPerGRF(Type_UB)/G4_Type_Table[Type_UD].byteSize;
    G4_Declare *dcl = createSendPayloadDcl(num_payload_elt + GENX_SAMPLER_IO_SZ, Type_UD);

    // mov  VX(0,0)<1>, r0
    Create_MOVR0_Inst(dcl, 0, 0);
    unsigned cmask = channel.getHWEncoding() << 12;
    /* mov (1)     VX(0,2)<1>,   0  */
    Create_MOV_Inst(dcl, 0, 2, g4::SIMD1, NULL, NULL, createImm(cmask, Type_UD));

    // set up the message payload
    // lod is always uninitialized for us as we don't support it.
    G4_Declare *dcl1 = createSendPayloadDcl(num_payload_elt, Type_UD);
    dcl1->setAliasDeclare(dcl, numEltPerGRF(Type_UB));
    /* mov  (sample_mode)    VX(0,0)<1>,  u */
    Create_MOV_Send_Src_Inst(dcl1, 0, 0, simdMode, uOffOpnd, 0);
    if (sampler == NULL)
    {
        // ld
        if (getPlatform() < GENX_SKL)
        {
            // the order of paramters is
            // u    lod        v    r
            /* mov  (sample_mode)    VX(sample_mode/8, 0)<1>,  lod */
            Create_MOV_Send_Src_Inst(dcl1, simdMode/8, 0, simdMode, createImm(0, Type_UD), 0);
            /* mov  (sample_mode)    VX(2*sample_mode/8, 0)<1>,  v */
            Create_MOV_Send_Src_Inst(dcl1, 2*simdMode/8, 0, simdMode, vOffOpnd, 0);
            /* mov  (sample_mode)    VX(3*sampler_mode/8, 0)<1>,  r */
            Create_MOV_Send_Src_Inst(dcl1, 3*simdMode/8, 0, simdMode, rOffOpnd, 0);
        }
        else
        {
            // SKL+: the order of paramters is
            // u    v   lod r
            /* mov  (sample_mode)    VX(sample_mode/8, 0)<1>,  v */
            Create_MOV_Send_Src_Inst(dcl1, simdMode/8, 0, simdMode, vOffOpnd, 0);
            /* mov  (sample_mode)    VX(2*sample_mode/8, 0)<1>,  lod */
            Create_MOV_Send_Src_Inst(dcl1, 2*simdMode/8, 0, simdMode, createImm(0, Type_UD), 0);
            /* mov  (sample_mode)    VX(3*sampler_mode/8, 0)<1>,  r */
            Create_MOV_Send_Src_Inst(dcl1, 3*simdMode/8, 0, simdMode, rOffOpnd, 0);
        }
    }
    else
    {
        // sample
        /* mov  (sample_mode)    VX(1 + sample_mode/8, 0)<1>,  v */
        Create_MOV_Send_Src_Inst(dcl1, simdMode/8, 0, simdMode, vOffOpnd, 0);
        /* mov  (sample_mode)    VX(3,0)<1>,  r */
        Create_MOV_Send_Src_Inst(dcl1, 2*simdMode/8, 0, simdMode, rOffOpnd, 0);
        /* mov  (sample_mode)    VX(5,0)<1>,  0 */
        Create_MOV_Send_Src_Inst(dcl1, 3*simdMode/8, 0, simdMode, createImm(0, Type_UD), 0);
    }
    // send's operands preparation
    // create a currDst for VX
    G4_SrcRegRegion* payload = Create_Src_Opnd_From_Dcl(dcl, getRegionStride1());

    G4_DstRegRegion* d = Check_Send_Dst(dstOpnd->asDstRegRegion());

    // Set bit 9-8 for the message descriptor
    unsigned temp = 0;

    //Bit 17-18 = 10 for SIMD mode
    if (simdMode == 8)
    {
        temp += 0x1 << 17;
    }
    else
    {
        temp += 0x2 << 17;
    }

    if (sampler == NULL)
    {
#define SAMPLER_MESSAGE_TYPE_OFFSET    12
        //LD message
        temp += VISASampler3DSubOpCode::VISA_3D_LD << SAMPLER_MESSAGE_TYPE_OFFSET;
    }

    if (simdMode == 16) {
        // redefine the type and offset of post dst.
        if ((d->getType() != Type_W) &&
            (d->getType() != Type_UW)) {
            short new_SubRegOff = dstOpnd->asDstRegRegion()->getSubRegOff();
            if (dstOpnd->getRegAccess() == Direct) {
                new_SubRegOff = (dstOpnd->asDstRegRegion()->getSubRegOff() * G4_Type_Table[dstOpnd->getType()].byteSize) / G4_Type_Table[Type_W].byteSize;
            }
            G4_DstRegRegion new_dst(
                dstOpnd->getRegAccess(),
                dstOpnd->asDstRegRegion()->getBase(),
                dstOpnd->asDstRegRegion()->getRegOff(),
                new_SubRegOff,
                1,
                Type_W);
            d = createDstRegRegion(new_dst);
        }
    }

    Create_Send_Inst_For_CISA(
        NULL,
        d,
        payload,
        1 + simdMode/2,
        ((simdMode == 8) ? 32 : (numEnabledChannels*16))*G4_Type_Table[Type_F].byteSize/numEltPerGRF(Type_UB),
        G4_ExecSize(simdMode),
        temp,
        SFID::SAMPLER,
        1,
        SendAccess::READ_ONLY,
        surface,
        sampler,
        0,
        false);
    return VISA_SUCCESS;
}