/*===================== begin_copyright_notice ==================================

Copyright (c) 2017 Intel Corporation

Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:

The above copyright notice and this permission notice shall be included
in all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.


======================= end_copyright_notice ==================================*/

#include "BuildIR.h"
#include "../Timer.h"


#define FIX_OWORD_SEND_EXEC_SIZE(BLOCK_SIZE)(((BLOCK_SIZE) > 2)? 16: (BLOCK_SIZE*4))


static uint32_t buildDescForScatter(
    uint32_t msgType, VISA_SVM_Block_Num numBlocks, MDC_SM2 simdMode)
{
    uint32_t MD = (msgType & 0x1F) << 14;
    MD |= numBlocks << 10;
    MD |= 1 << 9;
    MD |= simdMode << 8;
    return MD;
}


bool IR_Builder::isMessageHeaderOptional(
    G4_Operand *surface, G4_Operand *Offset) const
{
    // Message header is require for T255 stateless surface on pre-SKL devices
    // as a workaround for HW issue.
    if (needsA32MsgHeader() && isStatelessSurface(surface))
    {
        return false;
    }

    // Message Header is optional when offset is 0.
    // When GlobalOffset is 0, message header is optional.
    // "If the header is not present, behavior is as if the message was sent
    // with all fields in the header set to zero."
    return Offset->isImm() && Offset->asImm()->isZero();
}




// if surface is PRED_SURF_255, lower it to PRED_SURF_253 so that it's non IA-coherent
// the surface is not changed otherwise
static G4_Operand* lowerSurface255To253(G4_Operand* surface, IR_Builder& builder)
{
    // disable due to OCL SVM atomics regression
#if 0
    if (surface && surface->isImm() && surface->asImm()->getImm() == PREDEF_SURF_255)
    {
        return builder.createImm(PREDEF_SURF_253, Type_UW);
    }
    else
#endif
    {
        return surface;
    }
}

static void BuildStatelessSurfaceMessageHeader(IR_Builder *IRB, G4_Declare *Header)
{
    // For A32, clearing off scratch space offset or Buffer Base Address is
    // always required once header is present.

    G4_Type ElemTy = Header->getElemType();

    // R0.5<31:10> is defined as Scratch Space Offset.
    // R0.5<8:0> is defined as FF Thread ID (FFTID) in SKL+ devices.
    // R0.5<7:0> is defined as FF Thread ID (FFTID) in pre-SKL devices.
    // We increase the bit range to <9:0> to copy reserved bits as well.
    const unsigned FFTID_Mask = 0x3ff;

    // Rx.5[31:0] = 0 | R0.5[9:0]
    G4_DstRegRegion *DstOpnd = IRB->createDst(Header->getRegVar(), 0, 5, 1, ElemTy);
    // R0.5
    G4_SrcRegRegion *SrcOpnd = IRB->createSrcRegRegion(Mod_src_undef, Direct,
        IRB->getBuiltinR0()->getRegVar(), 0, 5,
        IRB->getRegionScalar(), ElemTy);
    // Mask
    G4_Imm *Mask = IRB->createImm(FFTID_Mask, Type_UD);
    IRB->createBinOp(G4_and, g4::SIMD1, DstOpnd, SrcOpnd, Mask,
        InstOpt_WriteEnable, true);
}


// TODO: remove
#define SET_DATAPORT_MESSAGE_TYPE(dest, value)\
    dest |= value << 14;

uint32_t IR_Builder::setOwordForDesc(uint32_t desc, int numOword, bool isSLM) const
{
    static const uint32_t MESSAGE_SPECIFIC_CONTROL = 8;
    switch (numOword)
    {
    case 1:
        return desc;
    case 2:
        return desc | (0x2 << MESSAGE_SPECIFIC_CONTROL);
    case 4:
        return desc | (0x3 << MESSAGE_SPECIFIC_CONTROL);
    case 8:
        return desc | (0x4 << MESSAGE_SPECIFIC_CONTROL);
    case 16:
        assert(isSLM && has16OWordSLMBlockRW() && "16OWord block r/w not supported");
        return desc | (0x5 << MESSAGE_SPECIFIC_CONTROL);
    default:
        /// TODO(move to verifier): default: ASSERT_USER(false, "OWord block size must be 1/2/4/8.");
        return desc;
    }
}


/*
* Translates OWord Block read CISA inst.
*
* For GT, assume size is 8 then the code should look like
*
* .declare  VX Base=m ElementSize=4 Type=ud Total=8
* .declare  VY Base=r ElementSize=4 Type=ud Total=8
*
* mov  (8)     VX(0,0)<1>,  r0:ud
* mov  (1)     VX(0,2)<1>,  P
* send (8)     VY(0,0)<1>,  VX(0,0),    0x5,  0x02180200
* mov  (8)     v(0,0)<1>,   VY(0,0)
*
* P: M0.2 in the message header (Global offset)
*
* 0x5 == 0 (Not the EOT)
*
* 0x02180200 == Bit 31-29: 000 (Reserved)
*               Bit 28-25: 0001 (Msg. leng. = 1)
*               Bit 24-20: 00001 (Response msg. leng. = 1)
*               Bit 19:    1 (Header present)
*               Bit 18:    0 (Ignored)
*               Bit 17:    0 (Send write commit message; ignored for read message
*               Bit 16-13: 0000 (Msg. type = OWord block read - for Render Cache)
*               Bit 12-8:  00010 (Block size = 2 OWords) - can only be 1/2/4/8 for sampler/render cache
*               Bit 7-0:   00000000 + I (Binding table index)
*
*/
int IR_Builder::translateVISAOwordLoadInst(
    ISA_Opcode opcode,
    bool modified,
    G4_Operand* surface,
    VISA_Oword_Num size,
    G4_Operand* offOpnd,
    G4_DstRegRegion* dstOpnd)
{
    TIME_SCOPE(VISA_BUILDER_IR_CONSTRUCTION);

    surface = lowerSurface255To253(surface, *this);

    unsigned num_oword = Get_VISA_Oword_Num(size);
    bool unaligned = (opcode == ISA_OWORD_LD_UNALIGNED);

    // create dcl for VX
    G4_Declare *dcl = createSendPayloadDcl(GENX_DATAPORT_IO_SZ, Type_UD);

    if (isStatelessSurface(surface))
    {
        // Build stateless surface message header.
        BuildStatelessSurfaceMessageHeader(this, dcl);
    }

    /* mov (1)      VX(0,2)<1>,    P  */
    if (unaligned && (kernel.major_version == 3 && kernel.minor_version <= 1))
    {
        // for vISA3.1 and earlier
        // the offset for unaligned OW load is in unit of DW, tranlate it into BYTE.
        if (offOpnd->isImm())
        {
            // imm type must be UD as the result of shift could overflow word type
            G4_Imm *new_src_opnd1 = createImm(
                offOpnd->asImm()->getInt() << 2, Type_UD);
            Create_MOV_Inst(dcl, 0, 2, g4::SIMD1, NULL, NULL, new_src_opnd1, true);
        }
        else
        {
            G4_DstRegRegion* dstOpnd = createDst(dcl->getRegVar(), 0, 2, 1, dcl->getElemType());
            createBinOp(G4_shl, g4::SIMD1, dstOpnd, offOpnd,
                createImm(2, Type_UW), InstOpt_WriteEnable, true);
        }
    }
    else
    {
        dcl->setCapableOfReuse();
        Create_MOV_Inst(dcl, 0, 2, g4::SIMD1, NULL, NULL, offOpnd, true);
    }
    // send's operands preparation
    G4_SrcRegRegion* payload = Create_Src_Opnd_From_Dcl(dcl, getRegionStride1());
    G4_DstRegRegion* d = Check_Send_Dst(dstOpnd->asDstRegRegion());

    uint32_t temp = 0;

    if (unaligned)
    {
        SET_DATAPORT_MESSAGE_TYPE(temp, DC_ALIGNED_OWORD_BLOCK_READ)
    }

    // Set bit 12-8 for the message descriptor
    temp = setOwordForDesc(temp, num_oword, IsSLMSurface(surface));

    // !!!WHY???
    if (num_oword > 2)
    {
        // redefine the type and offset of post dst.
        if ((d->getType() != Type_W) &&
            (d->getType() != Type_UW)) {
            short new_SubRegOff = dstOpnd->asDstRegRegion()->getSubRegOff();
            if (dstOpnd->getRegAccess() == Direct) {
                new_SubRegOff = (dstOpnd->asDstRegRegion()->getSubRegOff() * G4_Type_Table[dstOpnd->getType()].byteSize) / G4_Type_Table[Type_W].byteSize;
            }
            G4_DstRegRegion new_dst(
                dstOpnd->getRegAccess(),
                dstOpnd->asDstRegRegion()->getBase(),
                dstOpnd->asDstRegRegion()->getRegOff(),
                new_SubRegOff,
                1,
                Type_W);
            d = createDstRegRegion(new_dst);
        }
    }

    SFID tf_id =  SFID::DP_DC;

    G4_ExecSize send_exec_size = G4_ExecSize(FIX_OWORD_SEND_EXEC_SIZE(num_oword));
    bool forceSplitSend = shouldForceSplitSend(surface);

    if (!forceSplitSend)
    {
        Create_Send_Inst_For_CISA(
            NULL, d,
            payload,
            1,
            (num_oword * 16 + getGRFSize() - 1) / getGRFSize(),
            send_exec_size,
            temp,
            tf_id,
            true,
            SendAccess::READ_ONLY,
            surface,
            NULL,
            InstOpt_WriteEnable,
            false);
    }
    else {
        G4_SrcRegRegion *m0 = Create_Src_Opnd_From_Dcl(dcl, getRegionStride1());
        Create_SplitSend_Inst_For_CISA(
            NULL, d, m0, 1,
            createNullSrc(Type_UD), 0,
            (num_oword * 16 + getGRFSize() - 1) / getGRFSize(),
            send_exec_size,
            temp,
            tf_id,
            true,
            SendAccess::READ_ONLY,
            surface,
            nullptr,
            InstOpt_WriteEnable,
            false);
    }

    return VISA_SUCCESS;
}

/*
* Translates OWord Block write intrinsic.
*
* write(I, P, vector<int, S> v)
*
* For GT, assume S = 8 then the code should look like
*
* .declare  VX Base=m ElementSize=4 Type=ud Total=16
* .declare  VY Base=m ElementSize=4 Type=ud Total=8  ALIAS(VX,8)
*
* mov  (8)     VX(0,0)<1>,  r0:ud
* mov  (8)     VY(0,0)<1>,  v       // mov  (8)     VX(1,0)<1>,  v
* mov  (1)     VX(0,2)<2>,  P
* send (8)     null<1>,  VX(0,0),  0x5,   0x04090200
*
* P: M0.2 in the message header (Global offset)
*
* 0x5 == 0 (Not the EOT)
*        0101 (Target Function ID: DP Render Cache)
*
* 0x04090200 == Bit 31-29: 000 (Reserved)
*               Bit 28-25: 0010 (Msg. leng. = 2)
*               Bit 24-20: 00000 (Response msg. leng. = 0)
*               Bit 19:    1 (Header present)
*               Bit 18:    0 (Ignored)
*               Bit 17:    0 (Send write commit message
*               Bit 16-13: 1000 (Msg. type = OWord block read - for Render Cache)
*               Bit 12-8:  00010 (Block size = 2 OWords) - can only be 1/2/4/8 for sampler/render cache
*               Bit 7-0:   00000000 + I (Binding table index)
*
*/
int IR_Builder::translateVISAOwordStoreInst(
    G4_Operand* surface,
    VISA_Oword_Num size,
    G4_Operand* offOpnd,
    G4_SrcRegRegion* srcOpnd)
{
    TIME_SCOPE(VISA_BUILDER_IR_CONSTRUCTION);

    surface = lowerSurface255To253(surface, *this);

    unsigned num_oword = Get_VISA_Oword_Num(size);
    unsigned obj_size = num_oword * 16; // size of obj in bytes

    unsigned funcCtrl = DC_OWORD_BLOCK_WRITE << 14;

    uint32_t payloadGRFSize = (num_oword * 16 + getGRFSize() - 1) / getGRFSize();

    // Set bit 12-8 for the message descriptor
    funcCtrl = setOwordForDesc(funcCtrl, num_oword, IsSLMSurface(surface));
    bool forceSplitSend = shouldForceSplitSend(surface);
    if (forceSplitSend || useSends())
    {
        G4_Declare *headerDcl = createSendPayloadDcl(GENX_DATAPORT_IO_SZ, Type_UD);

        if (isStatelessSurface(surface))
        {
            // Build stateless surface message header.
            BuildStatelessSurfaceMessageHeader(this, headerDcl);
        }

        /* mov (1)     VX(0,2)<1>,   P  */
        Create_MOV_Inst(headerDcl, 0, 2, g4::SIMD1, nullptr, nullptr, offOpnd, true);

        unsigned msgDesc = funcCtrl;
        unsigned extMsgLength = payloadGRFSize;
        uint16_t extFuncCtrl = 0;

        // message length = 1, response length = 0, header present = 1
        msgDesc += (1 << getSendMsgLengthBitOffset()) + (1 << getSendHeaderPresentBitOffset());

        G4_SendMsgDescriptor* desc = createSendMsgDesc(msgDesc, 0, 1, SFID::DP_DC,
            extMsgLength, extFuncCtrl, SendAccess::WRITE_ONLY, surface);

        G4_ExecSize sendSize = G4_ExecSize(FIX_OWORD_SEND_EXEC_SIZE(num_oword));

        G4_SrcRegRegion* src0 = Create_Src_Opnd_From_Dcl(headerDcl, getRegionStride1());
        G4_DstRegRegion* dst = createNullDst(sendSize > 8 ? Type_UW: Type_UD);

        Create_SplitSend_Inst(nullptr, dst, src0, srcOpnd, sendSize, desc, InstOpt_WriteEnable, false);
    }
    else
    {
        uint32_t temp =  obj_size/G4_Type_Table[Type_UD].byteSize + GENX_DATAPORT_IO_SZ;

        G4_Declare *dcl = createSendPayloadDcl(temp, Type_UD);

        /* mov  (c*r)    VX(1,0)<1>,  V */
        temp =  obj_size/G4_Type_Table[Type_UD].byteSize;

        Create_MOV_Send_Src_Inst(dcl, 1, 0, temp, srcOpnd, InstOpt_WriteEnable);

        if (isStatelessSurface(surface)) {
            // Build stateless surface message header.
            BuildStatelessSurfaceMessageHeader(this, dcl);
        } else {
            // Copy R0 header.
            Create_MOVR0_Inst(dcl, 0, 0, true);
        }

        /* mov (1)     VX(0,2)<1>,   P  */
        Create_MOV_Inst(dcl, 0, 2, g4::SIMD1, NULL, NULL, offOpnd, true);

        // send's operands preparation
        /* Size of whole operand in UINT elements */
        G4_SrcRegRegion* payload = Create_Src_Opnd_From_Dcl(dcl, getRegionStride1());

        unsigned send_size = FIX_OWORD_SEND_EXEC_SIZE(num_oword);
        G4_DstRegRegion *post_dst_opnd = createNullDst(send_size > 8 ? Type_UW: Type_UD);

        Create_Send_Inst_For_CISA(
            NULL,
            post_dst_opnd,
            payload,
            payloadGRFSize + 1,
            0,
            G4_ExecSize(send_size),
            funcCtrl,
            SFID::DP_DC,
            true,
            SendAccess::WRITE_ONLY,
            surface,
            NULL,
            InstOpt_WriteEnable,
            false);
    }

    return VISA_SUCCESS;
}

static const uint8_t mapExecSizeToNumElts[6] = {1, 2, 4, 8, 16, 32};

/*
* Translates scattered read intrinsic.
*
* For GT, assume N = 8 then the code should look like
*
* .declare  VX Base=m ElementSize=4 Type=ud Total=16
* .declare  VY Base=r ElementSize=4 Type=ud Total=8
*
* mov  (8)     VX(0,0)<1>,  r0:ud
* mov  (1)     VX(0,2)<1>,  P
* mov  (8)     VX(1,0)<1>,  E
* send (8)     VY(0,0)<1>,  VX(0,0),    0x5,  0x0418C200
*
* P: M0.2 in the message header (Global offset)
* E: M1 in the message payload (Element offsets)
* 0x5 == 0 (Not the EOT)
*        0101 (Target Function ID: DP Render Cache)
*
* 0x0418C200 == Bit 31-29: 000 (Reserved)
*               Bit 28-25: 0010 (Msg. leng. = 2)
*               Bit 24-20: 00001 (Response msg. leng. = 1)
*               Bit 19:    1 (Header present)
*               Bit 18:    0 (Ignored)
*               Bit 17:    0 (Send write commit message; ignored for read message
*               Bit 16-13: 0110 (Msg. type = DWord Scattered read - for Render Cache)
*               Bit 12-10: 010 Specifies the data size for each slot. 0: 1 byte; 1: 2 bytes; 2: 4 bytes; 3: Reserved
*               Bit 9-8:  00 (Block size = 8 DWords)
*               Bit 7-0:   00000000 + I (Binding table index)
*
*/
int IR_Builder::translateVISAGatherInst(
    VISA_EMask_Ctrl emask,
    bool modified,
    GATHER_SCATTER_ELEMENT_SIZE eltSize,
    VISA_Exec_Size executionSize,
    G4_Operand* surface,
    G4_Operand* gOffOpnd,
    G4_SrcRegRegion* eltOffOpnd,
    G4_DstRegRegion* dstOpnd)
{
    TIME_SCOPE(VISA_BUILDER_IR_CONSTRUCTION);

    surface = lowerSurface255To253(surface, *this);

    // Before GEN10, we translate DWORD GATHER on SLM to untyped GATHER4 on
    // SLM with only R channel enabled. The later is considered more
    // efficient without recalculating offsets in BYTE.
    if (eltSize == GATHER_SCATTER_DWORD && IsSLMSurface(surface)) {
        return translateVISAGather4Inst(emask, modified,
            ChannelMask::createFromAPI(CHANNEL_MASK_R),
            executionSize, surface, gOffOpnd,
            eltOffOpnd, dstOpnd);
    }

    G4_ExecSize exsize = G4_ExecSize(Get_VISA_Exec_Size(executionSize));
    unsigned int instOpt = Get_Gen4_Emask(emask, exsize);
    bool headerLess = isMessageHeaderOptional(surface, gOffOpnd);
    // Element size in gather/scatter message. Initially, we assume it's the
    // same as the request.
    GATHER_SCATTER_ELEMENT_SIZE msgEltSize = eltSize;

    // SLM access
    //              HEADLESS    BYTE    WORD    DWORD
    // BDW          Opt         YES     NO      NO
    // SKL          Req         YES     NO      NO
    // CNL          Req         YES     NO      YES

    G4_Predicate* pred = NULL; // for SIMD1 gather
    uint8_t numElt = mapExecSizeToNumElts[executionSize];
    // we need to treat simd1 as simd8 in several places during code gen
    uint8_t effectiveNumElt = (numElt == 1 ? 8 : numElt);

    if (!headerLess && noSLMMsgHeader() && IsSLMSurface(surface))
    {
        // From SKL, SLM messages forbid message header. Recalculate offset by
        // adding global offset and force headerLess.
        G4_Declare *dcl = createSendPayloadDcl(numElt, eltOffOpnd->getType());
        dcl->setSubRegAlign(GRFALIGN);
        G4_DstRegRegion *newEltOffOpnd = Create_Dst_Opnd_From_Dcl(dcl, 1);
        createBinOp(G4_add, G4_ExecSize(numElt), newEltOffOpnd, eltOffOpnd, gOffOpnd, instOpt, true);
        eltOffOpnd = Create_Src_Opnd_From_Dcl(dcl, numElt == 1 ? getRegionScalar() : getRegionStride1());
        headerLess = true;
    }

    bool useSplitSend = useSends();
    // When header is not required, split-send is not needed as there's only
    // one part in the message. When header is present, we will split the
    // message as (header, offset).
    if (headerLess)
        useSplitSend = false;

    G4_Declare *header = 0;
    G4_Declare *offset = createSendPayloadDcl(numElt, Type_UD);
    offset->setSubRegAlign(GRFALIGN);

    if (useSplitSend)
    {
        ASSERT_USER(!headerLess, "SplitSend should not be used when header is not required!");
        // Without header, it's unnecessary to split the message.
        header = createSendPayloadDcl(GENX_DATAPORT_IO_SZ, Type_UD);
    }
    else if (!headerLess)
    {
        header = createSendPayloadDcl(GENX_DATAPORT_IO_SZ + effectiveNumElt, Type_UD);
        offset->setAliasDeclare(header, numEltPerGRF(Type_UB));
    }

    G4_SrcRegRegion* msgSrcOpnd = NULL;

    if (headerLess)
    {
        ASSERT_USER(!header, "'header' should not be allocated when header is not required!");

        if (eltSize == GATHER_SCATTER_WORD ||
            (eltSize != GATHER_SCATTER_BYTE && IsSLMSurface(surface)))
        {
            // Use byte gather for WORD gather as well as SLM surfaces (only supports byte gather)
            // need a shift to make the offset to be byte offset
            // shl (8) tmp<1>:ud elt_off<8;8,1>:ud 0x2:uw
            // Don't do this for Dword because we use the dword scatter message instead
            G4_DstRegRegion* tmpDstOpnd = Create_Dst_Opnd_From_Dcl(offset, 1);
            createBinOp(G4_shl, G4_ExecSize(numElt), tmpDstOpnd, eltOffOpnd,
                createImm(unsigned(eltSize), Type_UD), instOpt, true);
            msgSrcOpnd = Create_Src_Opnd_From_Dcl(offset, getRegionStride1());
            msgEltSize = GATHER_SCATTER_BYTE;
        }
        else
        {
            msgSrcOpnd = eltOffOpnd;
        }
    }
    else
    {
        if (isStatelessSurface(surface)) {
            // Build stateless surface message header.
            BuildStatelessSurfaceMessageHeader(this, header);
        } else {
            // Copy R0 header.
            Create_MOVR0_Inst(header, 0, 0, true);
        }

        G4_DstRegRegion* dst1_opnd = createDst(offset->getRegVar(), 0, 0, 1, offset->getElemType());

        if (eltSize == GATHER_SCATTER_WORD || IsSLMSurface(surface))
        {
            // For non-SLM surface, WORD gather/scatter has no hardware
            // supportr and must be translated into BYTE gather/scatter.
            //
            // SLM surface supports only BYTE gather/scatter
            // support and also needs translating into BYTE gather/scatter.
            //
            /* mov (1)     VX(0,2)<1>,   P  */
            if (gOffOpnd->isImm())
            {
                G4_Imm *new_src_opnd1 = createImm(
                    gOffOpnd->asImm()->getInt() * (eltSize == GATHER_SCATTER_WORD ? 2 : 4),
                    gOffOpnd->getType());
                Create_MOV_Inst(header, 0, 2, g4::SIMD1, NULL, NULL, new_src_opnd1, true);
            }
            else
            {
                G4_DstRegRegion* dst2_opnd = createDst(header->getRegVar(), 0, 2, 1, header->getElemType());

                createBinOp(G4_shl, g4::SIMD1, dst2_opnd, gOffOpnd,
                    createImm((unsigned)eltSize, Type_UD), InstOpt_WriteEnable, true);
            }
            createBinOp(G4_shl, G4_ExecSize(numElt), dst1_opnd, eltOffOpnd,
                createImm((unsigned)eltSize, Type_UD), instOpt, true);
            msgEltSize = GATHER_SCATTER_BYTE;
        }
        else
        {
            /* mov (1)     VX(0,2)<1>,   P  */
            Create_MOV_Inst(header, 0, 2, g4::SIMD1, NULL, NULL, gOffOpnd, true);
            /* mov  (numElt)    VX(1,0)<1>,  E */
            createMov(G4_ExecSize(numElt), dst1_opnd,
                eltOffOpnd, instOpt, true);
        }

        // Create a <8;8,1> src region for the send payload
        msgSrcOpnd = Create_Src_Opnd_From_Dcl(header, getRegionStride1());
    }

    G4_DstRegRegion* d = dstOpnd->asDstRegRegion();

    SFID tf_id = SFID::DP_DC;
    unsigned temp = 0;
    // Set bit 9-8 for the message descriptor
    if (msgEltSize == GATHER_SCATTER_DWORD)
    {
        if (effectiveNumElt == 8)
        {
            temp += 2 << 8;
        }
        else {
            temp += 3 << 8;
        }
        temp += DC_DWORD_SCATTERED_READ << 14; // '0011' for DWORD scattered read
    }
    else
    {
        if (effectiveNumElt == 16)
        {
            temp += 1 << 8;
        }
        temp += (unsigned char)eltSize << 10;
        temp += DC_BYTE_SCATTERED_READ << 14;
    }

    if (useSplitSend)
    {
        ASSERT_USER(!headerLess, "SplitSend should only be used when header is required!");

        G4_SrcRegRegion *m0 = Create_Src_Opnd_From_Dcl(header, getRegionStride1());
        G4_SrcRegRegion *m1 = Create_Src_Opnd_From_Dcl(offset, getRegionStride1());
        Create_SplitSend_Inst_For_CISA(pred, d,
            m0, 1,
            m1, effectiveNumElt / GENX_DATAPORT_IO_SZ,
            effectiveNumElt / GENX_DATAPORT_IO_SZ,
            G4_ExecSize(numElt),
            temp,
            tf_id, true,
            SendAccess::READ_ONLY,
            surface, NULL, instOpt, false);
    }
    else
    {
        Create_Send_Inst_For_CISA(
            pred,
            d,
            msgSrcOpnd,
            headerLess ? effectiveNumElt/GENX_DATAPORT_IO_SZ : effectiveNumElt/GENX_DATAPORT_IO_SZ + 1,
            effectiveNumElt/GENX_DATAPORT_IO_SZ,
            G4_ExecSize(numElt),
            temp,
            tf_id,
            !headerLess,
            SendAccess::READ_ONLY,
            surface,
            nullptr,
            instOpt,
            false);
    }

    return VISA_SUCCESS;
}




/*
* Translates scattered write intrinsic.
*
* For GT, assume N = 8 then the code should look like
*
* .declare  VX Base=m ElementSize=4 Type=ud Total=24
*
* mov  (8)     VX(0,0)<1>,  r0:ud
* mov  (1)     VX(0,2)<1>,  P
* mov  (8)     VX(1,0)<1>,  E
* mov  (8)     VX(2,0)<1>,  V
* send (8)     null<1>,     VX(0,0),    0x5,  0x06096200
*
* P: M0.2 in the message header (Global offset)
* E: M1 in the message payload (Element offsets)
* v: M2 in the message payload (written data)
*
* 0x5 == 0 (Not the EOT)
*        0101 (Target Function ID: DP Render Cache)
*
* 0x06096200 == Bit 31-29: 000 (Reserved)
*               Bit 28-25: 0011 (Msg. leng. = 3)
*               Bit 24-20: 00000 (Response msg. leng. = 0)
*               Bit 19:    1 (Header present)
*               Bit 18:    0 (Ignored)
*               Bit 17:    0 (Send write commit message)
*               Bit 16-13: 1011 (Msg. type = DWord Scattered write - for Render Cache)
*               Bit 12-8:  00010 (Block size = 8 DWords)
*               Bit 7-0:   00000000 + I (Binding table index)
*
*/
int IR_Builder::translateVISAScatterInst(
    VISA_EMask_Ctrl emask,
    GATHER_SCATTER_ELEMENT_SIZE eltSize,
    VISA_Exec_Size executionSize,
    G4_Operand* surface,
    G4_Operand* gOffOpnd,
    G4_SrcRegRegion* eltOffOpnd,
    G4_SrcRegRegion* srcOpnd)
{
    // Before GEN10, we translate DWORD SCATTER on SLM to untyped GATHER4 on
    // SLM with only R channel enabled. The later is considered more
    // efficient without recalculating offsets in BYTE.
    if (eltSize == GATHER_SCATTER_DWORD && IsSLMSurface(surface)) {
        return translateVISAScatter4Inst(emask,
            ChannelMask::createFromAPI(CHANNEL_MASK_R),
            executionSize, surface, gOffOpnd,
            eltOffOpnd, srcOpnd);
    }

    TIME_SCOPE(VISA_BUILDER_IR_CONSTRUCTION);
    surface = lowerSurface255To253(surface, *this);

    G4_ExecSize exsize = G4_ExecSize(Get_VISA_Exec_Size(executionSize));
    G4_InstOpts instOpt = Get_Gen4_Emask(emask, exsize);
    G4_Predicate *pred = NULL;
    // Element size in gather/scatter message. Initially, we assume it's the same as the request.
    GATHER_SCATTER_ELEMENT_SIZE msgEltSize = eltSize;

    uint8_t numElt = mapExecSizeToNumElts[executionSize];
    // we need to treat simd1 as simd8 in several places during code gen
    uint8_t effectiveNumElt = (numElt == 1 ? 8 : numElt);

    bool headerLess = isMessageHeaderOptional(surface, gOffOpnd);
    G4_SrcRegRegion* msgSrcOpnd = NULL;

    // SLM access
    //              HEADLESS    BYTE    WORD    DWORD
    // BDW          Opt         YES     NO      NO
    // SKL          Req         YES     NO      NO
    // CNL          Req         YES     NO      YES

    if (!headerLess && noSLMMsgHeader() && IsSLMSurface(surface)) {
        // From SKL, SLM messages forbid message header. Recalculate offset by
        // adding global offset and force headerLess.
        G4_Declare *dcl = createSendPayloadDcl(numElt, eltOffOpnd->getType());
        G4_DstRegRegion *newEltOffOpnd = Create_Dst_Opnd_From_Dcl(dcl, 1);
        createBinOp(G4_add, G4_ExecSize(numElt), newEltOffOpnd, eltOffOpnd, gOffOpnd, instOpt, true);
        eltOffOpnd = Create_Src_Opnd_From_Dcl(dcl, numElt == 1 ? getRegionScalar() : getRegionStride1());
        headerLess = true;
    }

    if (headerLess)
    {
        // header size = 2 * #elt
        G4_Declare *dcl = createSendPayloadDcl(effectiveNumElt * 2, Type_UD);
        G4_DstRegRegion* tmpDstOpnd = Create_Dst_Opnd_From_Dcl(dcl, 1);
        if (eltSize == GATHER_SCATTER_WORD ||
            (eltSize != GATHER_SCATTER_BYTE && IsSLMSurface(surface)))
        {
            // For non-SLM surface,
            // need a shift to make the offset to be byte offset
            // shl (esize) tmp.0<1>:ud elt_off<8;8,1>:ud 0x2:uw
            // Don't do this for Dword because we use the dword scatter message instead
            //
            // SLM surface has only BYTE scattered
            // read/write support. Always use BYTE scater.
            createBinOp(G4_shl, G4_ExecSize(numElt), tmpDstOpnd, eltOffOpnd,
                createImm(unsigned(eltSize), Type_UD), instOpt, true);
            msgEltSize = GATHER_SCATTER_BYTE;
        }
        else
        {
            createMov(G4_ExecSize(numElt), tmpDstOpnd, eltOffOpnd, instOpt, true);
        }

        Create_MOV_Send_Src_Inst(dcl, effectiveNumElt/8, 0, numElt, srcOpnd, instOpt);
        msgSrcOpnd = Create_Src_Opnd_From_Dcl(dcl, getRegionStride1());
    }
    else
    {
        // mov (8)      VX(0,0)<1>,  r0:ud
        // add dcl for VX
        G4_Declare *dcl = createSendPayloadDcl(GENX_DATAPORT_IO_SZ + effectiveNumElt * 2, Type_UD);

        if (isStatelessSurface(surface)) {
            // Build stateless surface message header.
            BuildStatelessSurfaceMessageHeader(this, dcl);
        } else {
            // Copy R0 header.
            Create_MOVR0_Inst(dcl, 0, 0, true);
        }

        auto dst1_opnd = createDst(dcl->getRegVar(), 1, 0, 1, dcl->getElemType());

        if (eltSize == GATHER_SCATTER_WORD || IsSLMSurface(surface))
        {
            // For non-SLM surface, WORD gather/scatter has no hardware
            // supportr and must be translated into BYTE gather/scatter.
            //
            // For SLM surface, gen9 devices has only BYTE gather/scatter
            // support and also needs translating into BYTE gather/scatter.
            //
            /* mov (1)     VX(0,2)<1>,   P  */
            if (gOffOpnd->isImm())
            {
                G4_Imm *new_src_opnd1 = createImm(
                    gOffOpnd->asImm()->getInt() * (eltSize == GATHER_SCATTER_WORD ? 2 : 4),
                    gOffOpnd->getType());
                Create_MOV_Inst(dcl, 0, 2, g4::SIMD1, NULL, NULL, new_src_opnd1, true);
            }
            else
            {
                G4_DstRegRegion* dst2_opnd = createDst(dcl->getRegVar(), 0, 2, 1, dcl->getElemType());
                createBinOp(G4_shl, g4::SIMD1, dst2_opnd, gOffOpnd,
                    createImm((unsigned)eltSize, Type_UD), InstOpt_WriteEnable, true);
            }
            createBinOp(G4_shl, G4_ExecSize(numElt), dst1_opnd, eltOffOpnd,
                createImm((unsigned)eltSize, Type_UD), instOpt, true);
            msgEltSize = GATHER_SCATTER_BYTE;
        }
        else
        {
            /* mov (1)     VX(0,2)<1>,   P  */
            Create_MOV_Inst(dcl, 0, 2, g4::SIMD1, NULL, NULL, gOffOpnd, true);
            /* mov  (numElt)    VX(1,0)<1>,  E */
            createMov(G4_ExecSize(numElt), dst1_opnd,
                eltOffOpnd, instOpt, true);
        }

        /* mov  (numElt)    VX(numElt/8+1,0)<1>,  V */
        Create_MOV_Send_Src_Inst(dcl, (effectiveNumElt/8+1), 0, numElt, srcOpnd, instOpt);

        // send's operands preparation
        // create a currDst for VX
        msgSrcOpnd = Create_Src_Opnd_From_Dcl(dcl, getRegionStride1());
    }

    unsigned temp = 0;

    // Set bit 9-8 for the message descriptor
    if (msgEltSize == GATHER_SCATTER_DWORD)
    {
        if (effectiveNumElt == 8)
        {
            temp += 2 << 8;
        }
        else {
            temp += 3 << 8;
        }
        temp += DC_DWORD_SCATTERED_WRITE << 14;
    }
    else
    {
        if (effectiveNumElt == 16)
        {
            temp += 1 << 8;
        }
        temp += (unsigned char)eltSize << 10;
        temp += DC_BYTE_SCATTERED_WRITE << 14;
    }

    G4_DstRegRegion *post_dst_opnd = createNullDst(effectiveNumElt > 8 ? Type_UW : Type_UD);

    Create_Send_Inst_For_CISA(
        pred,
        post_dst_opnd,
        msgSrcOpnd,
        headerLess ? effectiveNumElt/GENX_DATAPORT_IO_SZ * 2 :
        effectiveNumElt/GENX_DATAPORT_IO_SZ * 2 + 1,
        0,
        G4_ExecSize(numElt),
        temp,
        SFID::DP_DC,
        !headerLess,
        SendAccess::WRITE_ONLY,
        surface,
        NULL,
        instOpt,
        false);

    return VISA_SUCCESS;
}


static void BuildUntypedStatelessSurfaceMessageHeader(IR_Builder *IRB, G4_Declare *Header)
{
    // Set PSM (Pixel Sample Mask) in MH1_A32_PSM
    G4_Type ElemTy = Header->getElemType();

    // R0.7<31:0> is defined as MHC_PSM where the lower 16 bits specify the
    // pixel sample mask.
    const unsigned PSM_Mask = 0xffff;

    // Rx.7[31:0] = 0xFFFF
    G4_DstRegRegion *DstOpnd = IRB->createDst(Header->getRegVar(), 0, 7, 1, ElemTy);
    // Mask
    G4_Imm *Mask = IRB->createImm(PSM_Mask, Type_UD);
    IRB->createMov(g4::SIMD1, DstOpnd, Mask, InstOpt_WriteEnable, true);

    BuildStatelessSurfaceMessageHeader(IRB, Header);
}


/*
* Translates untyped surface read.
*
* For GT, assume N = 8 then the code should look like
*
* .declare  VX Base=m ElementSize=4 Type=ud Total=16
* .declare  VY Base=r ElementSize=4 Type=ud Total=8
*
* mov  (8)     VX(0,0)<1>,  r0:ud
* mov  (8)     VX(1,0)<1>,  P+E
* send (8)     VY(0,0)<1>,  VX(0,0),    0x5,  0x0418C200
*
* E: M1 in the message payload (Element offsets in BYTEs)
* 1010 (Target Function ID: Data Cache)
*
* 0x0418C200 == Bit 31-29: 000 (Reserved)
*               Bit 28-25: 0010 (Msg. leng. = 2)
*               Bit 24-20: 00001 (Response msg. leng. = 1)
*               Bit 19:    1 (Header present)
*               Bit 18:    0 (Ignored)
*               Bit 17-14: 1101 (Msg. type = untyped write - for data Cache)
*               Bit 13-12:  0010 (SIMD mode = 8)
*               Bit 11-8:  0000 (masked channels)
*               Bit 7-0:   00000000 + I (Binding table index)
*
*/
int IR_Builder::translateVISAGather4Inst(
    VISA_EMask_Ctrl emask,
    bool modified,
    ChannelMask chMask,
    VISA_Exec_Size executionSize,
    G4_Operand* surface,
    G4_Operand* gOffOpnd,
    G4_SrcRegRegion* eltOffOpnd,
    G4_DstRegRegion* dstOpnd)
{
    TIME_SCOPE(VISA_BUILDER_IR_CONSTRUCTION);

    surface = lowerSurface255To253(surface, *this);

    G4_ExecSize exsize = G4_ExecSize(Get_VISA_Exec_Size(executionSize));
    G4_InstOpts instOpt = Get_Gen4_Emask(emask, exsize);
    unsigned int num_channel = chMask.getNumEnabledChannels();

    uint8_t numElt = mapExecSizeToNumElts[executionSize];
    uint8_t hdrSize = 0;

    bool useSplitSend = useSends();

    G4_Declare *header = 0;
    G4_Declare *offset = createSendPayloadDcl(numElt, Type_UD);

    if (surface && isStatelessSurface(surface) && needsA32MsgHeader())
    {
        // Header is required to work around a HW issue on pre-SKL devices.
        hdrSize = GENX_DATAPORT_IO_SZ;
        if (useSplitSend) {
            header = createSendPayloadDcl(GENX_DATAPORT_IO_SZ, Type_UD);
        } else {
            header = createSendPayloadDcl(GENX_DATAPORT_IO_SZ + numElt, Type_UD);
            offset->setAliasDeclare(header, numEltPerGRF(Type_UB));
        }
    } else {
        // When the surface is not stateless one, header is not used and therefore
        // split-send is not used.
        useSplitSend = false;
    }

    if (header) {
        // With 'header' allocated, we need prepare the header for the
        // (stateless) surface.
        ASSERT_USER(isStatelessSurface(surface), "With 'header' allocated, stateless surface is expected!");
        // Build stateless surface message header.
        BuildUntypedStatelessSurfaceMessageHeader(this, header);
    }

    // convert to byte address
    // shl (esize) offset<1>:ud elt_off<8;8,1>:ud 2:uw
    G4_DstRegRegion* dst1_opnd = createDst(offset->getRegVar(), 0, 0, 1, offset->getElemType());

    G4_Declare *tmp_dcl = createTempVar(numElt, Type_UD, GRFALIGN);
    G4_DstRegRegion* dst3_opnd = createDst(tmp_dcl->getRegVar(), 0, 0, 1, tmp_dcl->getElemType());

    createBinOp(G4_shl, G4_ExecSize(numElt), dst3_opnd, eltOffOpnd, createImm(2, Type_UW), instOpt, true);

    G4_SrcRegRegion* src2_opnd = createSrcRegRegion(Mod_src_undef, Direct, tmp_dcl->getRegVar(), 0, 0,
        getRegionStride1(), tmp_dcl->getElemType());

    // As untyped surface message use MH_IGNORE based header, if global offset
    // is non-zero, we need recalculate element offsets.
    if (gOffOpnd->isImm())
    {
        if (gOffOpnd->asImm()->getInt() != 0)
        {
            gOffOpnd = createImm(
                gOffOpnd->asImm()->getInt() * 4,
                gOffOpnd->getType());
            createBinOp(G4_add, G4_ExecSize(numElt), dst1_opnd, src2_opnd, gOffOpnd, instOpt, true);
        }
        else
        {
            createMov(G4_ExecSize(numElt), dst1_opnd, src2_opnd, instOpt, true);
        }
    }
    else
    {
        G4_Declare *tmp_dcl1 = createTempVar(1, gOffOpnd->getType(), Any);
        G4_DstRegRegion* dst2_opnd = createDst(tmp_dcl1->getRegVar(), 0, 0, 1, tmp_dcl1->getElemType());

        createBinOp(G4_shl, g4::SIMD1, dst2_opnd, gOffOpnd, createImm(2, Type_UW), InstOpt_WriteEnable, true);

        G4_SrcRegRegion* src1Opnd = createSrcRegRegion(Mod_src_undef, Direct, tmp_dcl1->getRegVar(), 0, 0,
            getRegionScalar(), tmp_dcl1->getElemType());

        createBinOp(G4_add, G4_ExecSize(numElt), dst1_opnd, src2_opnd, src1Opnd, instOpt, true);
    }

    // send's operands preparation

    G4_DstRegRegion* d = Check_Send_Dst(dstOpnd->asDstRegRegion());

    unsigned temp = 0;

    // Set bit 13-12 for the message descriptor
    if (numElt == 8)
    {
        temp += 2 << 12;
    }
    else
    {
        temp += 1 << 12;
    }

    SFID tf_id = SFID::DP_DC1;
    temp += DC1_UNTYPED_SURFACE_READ << 14;

    // bits 11-8: channel mask
    // HW defines 0 to mean the channel is on, so we have to flip it
    temp += chMask.getHWEncoding() << 8;

    if (surface == NULL)
    {
        temp |= 0xFE;
    }

    if (useSplitSend) {
        ASSERT_USER(header, "'header' should be allocated when split-send is to be used.");

        G4_SrcRegRegion *m0 = Create_Src_Opnd_From_Dcl(header, getRegionStride1());
        G4_SrcRegRegion *m1 = Create_Src_Opnd_From_Dcl(offset, getRegionStride1());
        Create_SplitSend_Inst_For_CISA(NULL, d,
            m0, 1, m1, numElt / GENX_DATAPORT_IO_SZ,
            (numElt / GENX_DATAPORT_IO_SZ)* num_channel,
            G4_ExecSize(numElt), temp, tf_id, hdrSize != 0,
            SendAccess::READ_ONLY,
            surface, NULL, instOpt, false);
    }
    else
    {
        G4_SrcRegRegion* payload = Create_Src_Opnd_From_Dcl(header ? header : offset, getRegionStride1());
        Create_Send_Inst_For_CISA(
            NULL,
            d,
            payload,
            (hdrSize + numElt)/GENX_DATAPORT_IO_SZ,
            (numElt/GENX_DATAPORT_IO_SZ) * num_channel,
            G4_ExecSize(numElt),
            temp,
            tf_id,
            hdrSize != 0,
            SendAccess::READ_ONLY,
            surface,
            NULL,
            instOpt,
            false);
    }

    return VISA_SUCCESS;
}


/*
* Translates untyped surface write intrinsic.
*
* For GT, assume N = 8 then the code should look like
*
* .declare  VX Base=m ElementSize=4 Type=ud Total=24
*
* mov  (8)     VX(0,0)<1>,  r0:ud
* mov  (8)     VX(1,0)<1>,  E + P
* mov  (8)     VX(2,0)<1>,  V
* send (8)     null<1>,     VX(0,0),    0x5,  0x06096200
*
* E: M1 in the message payload (Element offsets)
* v: M2 in the message payload (written data)
*
* 1010 (Target Function ID: DP Data Cache)
*
* 0x06096200 == Bit 31-29: 000 (Reserved)
*               Bit 28-25: 0011 (Msg. leng. = 3)
*               Bit 24-20: 00000 (Response msg. leng. = 0)
*               Bit 19:    1 (Header present)
*               Bit 18:    0 (Ignored)
*               Bit 17-14: 1101 (Msg. type = untyped write - for data Cache)
*               Bit 13-12:  0010 (SIMD mode = 8)
*                  Bit 11-8:  0000 (masked channels)
*               Bit 7-0:   00000000 + I (Binding table index)
*
*/
int IR_Builder::translateVISAScatter4Inst(
    VISA_EMask_Ctrl emask,
    ChannelMask chMask,
    VISA_Exec_Size executionSize,
    G4_Operand* surface,
    G4_Operand* gOffOpnd,
    G4_SrcRegRegion* eltOffOpnd,
    G4_SrcRegRegion* srcOpnd)
{
    TIME_SCOPE(VISA_BUILDER_IR_CONSTRUCTION);

    surface = lowerSurface255To253(surface, *this);

    G4_ExecSize exsize = G4_ExecSize(Get_VISA_Exec_Size(executionSize));
    G4_InstOpts instOpt = Get_Gen4_Emask(emask, exsize);

    unsigned int num_channel = chMask.getNumEnabledChannels();

    uint8_t numElt = mapExecSizeToNumElts[executionSize];
    uint8_t hdrSize = 0;

    unsigned int data_size = numElt * num_channel;
    G4_Declare *src_dcl = srcOpnd->asSrcRegRegion()->getBase()->asRegVar()->getDeclare();

    int payload_size = numElt + data_size;

    bool useSplitSend = useSends();

    G4_Declare *header = 0;
    G4_Declare *offset = 0;
    G4_Declare *data = createSendPayloadDcl(data_size, Type_UD);

    if (surface && isStatelessSurface(surface) && needsA32MsgHeader())
    {
        // Header is required to work around a HW issue on pre-SKL devices.
        hdrSize = GENX_DATAPORT_IO_SZ;
        offset = createSendPayloadDcl(numElt, Type_UD);
        if (useSplitSend) {
            // When header is required, we split the message as
            // (header, offset + data) if split-send is supported.
            header = createSendPayloadDcl(GENX_DATAPORT_IO_SZ, Type_UD);
            offset = createSendPayloadDcl(payload_size, Type_UD);
            data->setAliasDeclare(offset, (numElt/8) * numEltPerGRF(Type_UB));
        } else {
            header = createSendPayloadDcl(GENX_DATAPORT_IO_SZ + payload_size, Type_UD);
            offset->setAliasDeclare(header, numEltPerGRF(Type_UB));
            data->setAliasDeclare(header, numEltPerGRF(Type_UB) * ((numElt/8) + 1));
        }
    } else {
        if (useSplitSend) {
            // When header is not required, we split the message as (offset, data)
            // if split-send is supported.
            offset = createSendPayloadDcl(numElt, Type_UD);
        } else {
            offset = createSendPayloadDcl(payload_size, Type_UD);
            data->setAliasDeclare(offset, (numElt/8) * numEltPerGRF(Type_UB));
        }
    }

    if (header) {
        // With 'header' allocated, we need prepare the header for the
        // (stateless) surface.
        ASSERT_USER(isStatelessSurface(surface),
            "With 'header' allocated, stateless surface is expected!");
        // Build stateless surface message header.
        BuildUntypedStatelessSurfaceMessageHeader(this, header);
    }

    if (!header && useSplitSend)
    {
        data = src_dcl;
    } else
    {
        // Copy data from src operand.
        for (unsigned i = 0; i != num_channel; ++i)
        {
            G4_SrcRegRegion *s2_opnd =
                createSrcRegRegion(
                    Mod_src_undef, Direct, src_dcl->getRegVar(), (i * numElt) / 8, 0, getRegionStride1(), src_dcl->getElemType());
            Create_MOV_Send_Src_Inst(data, (i * numElt) / 8, 0, numElt, s2_opnd, instOpt);
        }
    }

    // mov  VX(0,0)<1>, r0
    // Create_MOVR0_Inst(header, 0, 0, true);

    G4_DstRegRegion* dst1_opnd = createDst(offset->getRegVar(), 0, 0, 1, offset->getElemType());

    G4_Declare *tmp_dcl = createTempVar(numElt, Type_UD, GRFALIGN);
    G4_DstRegRegion* dst3_opnd = createDst(tmp_dcl->getRegVar(), 0, 0, 1, tmp_dcl->getElemType());

    createBinOp(G4_shl, G4_ExecSize(numElt), dst3_opnd, eltOffOpnd, createImm(2, Type_UW), instOpt, true);

    G4_SrcRegRegion* src2_opnd =
        createSrcRegRegion(
            Mod_src_undef, Direct, tmp_dcl->getRegVar(), 0, 0, getRegionStride1(), tmp_dcl->getElemType());

    if (gOffOpnd->isImm())
    {
        if (gOffOpnd->asImm()->getInt() != 0)
        {
            gOffOpnd = createImm(
                gOffOpnd->asImm()->getInt() * 4,
                gOffOpnd->getType());
            createBinOp(G4_add, G4_ExecSize(numElt), dst1_opnd, src2_opnd, gOffOpnd, instOpt, true);
        }
        else
        {
            createMov(G4_ExecSize(numElt), dst1_opnd, src2_opnd, instOpt, true);
        }
    }
    else
    {
        G4_Declare *tmp_dcl1 = createTempVar(1, gOffOpnd->getType(), Any);
        G4_DstRegRegion* dst2_opnd = createDst(tmp_dcl1->getRegVar(), 0, 0, 1, tmp_dcl1->getElemType());

        createBinOp(G4_shl, g4::SIMD1, dst2_opnd, gOffOpnd, createImm(2, Type_UW), InstOpt_WriteEnable, true);

        G4_SrcRegRegion* src1Opnd = createSrcRegRegion(Mod_src_undef, Direct, tmp_dcl1->getRegVar(), 0, 0,
            getRegionScalar(), tmp_dcl1->getElemType());

        createBinOp(G4_add, G4_ExecSize(numElt), dst1_opnd, src2_opnd, src1Opnd, instOpt, true);
    }

    // send's operands preparation
    unsigned temp = 0;

    // Set bit 13-12 for the message descriptor
    if (numElt == 8) {
        temp += 2 << 12;
    } else {
        temp += 1 << 12;
    }

    SFID tf_id = SFID::DP_DC1;
    temp += DC1_UNTYPED_SURFACE_WRITE << 14;
    // bits 11-8: channel mask
    temp += chMask.getHWEncoding() << 8;

    // Set bit 9-8 for the message descriptor

    if (surface == NULL)
    {
        temp |= 0xFF - 1;
    }

    G4_DstRegRegion *post_dst_opnd = createNullDst(numElt > 8 ? Type_UW : Type_UD);

    if (useSplitSend) {
        G4_SrcRegRegion *m0 = 0; unsigned m0Len = 0;
        G4_SrcRegRegion *m1 = 0; unsigned m1Len = 0;
        if (header) {
            m0 = Create_Src_Opnd_From_Dcl(header, getRegionStride1());
            m0Len = 1;
            m1 = Create_Src_Opnd_From_Dcl(offset, getRegionStride1());
            m1Len = payload_size / GENX_DATAPORT_IO_SZ;
        } else {
            m0 = Create_Src_Opnd_From_Dcl(offset, getRegionStride1());
            m0Len = numElt / GENX_DATAPORT_IO_SZ;
            m1 = Create_Src_Opnd_From_Dcl(data, getRegionStride1());
            m1Len = data_size / GENX_DATAPORT_IO_SZ;
        }
        Create_SplitSend_Inst_For_CISA(NULL, post_dst_opnd,
            m0, m0Len, m1, m1Len, 0,
            G4_ExecSize(numElt),
            temp, tf_id, hdrSize != 0,
            SendAccess::WRITE_ONLY,
            surface, NULL,
            instOpt, false);
    }
    else
    {
        G4_SrcRegRegion* payload = Create_Src_Opnd_From_Dcl(header ? header : offset, getRegionStride1());
        Create_Send_Inst_For_CISA(
            NULL,
            post_dst_opnd,
            payload,
            (numElt * (num_channel + 1) + hdrSize)/GENX_DATAPORT_IO_SZ,
            0,
            G4_ExecSize(numElt),
            temp,
            tf_id,
            hdrSize != 0,
            SendAccess::WRITE_ONLY,
            surface,
            NULL,
            instOpt,
            false);
    }

    return VISA_SUCCESS;
}

static bool IsFloatAtomicOps(VISAAtomicOps op)
{
    return op == ATOMIC_FMAX || op == ATOMIC_FMIN || op == ATOMIC_FCMPWR;
}

static void BuildMH1_A32_PSM(IR_Builder *IRB, G4_Declare *header) {
    // Clear header. Ignore PSM so far.
    G4_DstRegRegion *h = IRB->createDst(header->getRegVar(),
        0, 0, 1, Type_UD);
    IRB->createMov(g4::SIMD8, h, IRB->createImm(0, Type_UD), InstOpt_WriteEnable, true);
    // Set PSM to all 1s.
    G4_DstRegRegion *h0_7 =
        IRB->createDst(header->getRegVar(), 0, 7, 1, Type_UD);
    G4_Imm *Mask = IRB->createImm(0xFFFF, Type_UD);
    IRB->createMov(g4::SIMD1, h0_7, Mask, InstOpt_WriteEnable, true);
}


static void BuildMH1_BTS_PSM(IR_Builder *IRB, G4_Declare *header) {
    // Clear header
    G4_DstRegRegion* h = IRB->createDst(header->getRegVar(),
        0, 0, 1, Type_UD);
    IRB->createMov(g4::SIMD8, h, IRB->createImm(0, Type_UD), InstOpt_WriteEnable, true);
    // Set PSM to 0xFFFF so far.
    G4_Operand* maskImm = IRB->createImm(0xFFFF, Type_UD);
    G4_DstRegRegion* pitchDst = IRB->createDst(
        header->getRegVar(),
        0, 7, 1, Type_UD);
    IRB->createMov(g4::SIMD1, pitchDst, maskImm, InstOpt_WriteEnable, true);
}


// This version takes byte offsets and predicates
int IR_Builder::translateVISADwordAtomicInst(
    VISAAtomicOps atomicOp,
    bool is16Bit,
    G4_Predicate *pred,
    VISA_Exec_Size execSize,
    VISA_EMask_Ctrl eMask,
    G4_Operand* surface,
    G4_SrcRegRegion* offsets,
    G4_SrcRegRegion* src0,
    G4_SrcRegRegion* src1,
    G4_DstRegRegion* dst)
{
    TIME_SCOPE(VISA_BUILDER_IR_CONSTRUCTION);

    ASSERT_USER(!IsFloatAtomicOps(atomicOp) || hasFloatAtomics(),
        "Float atomic operations are only supported on SKL+ devices");


    surface = lowerSurface255To253(surface, *this);

    VISA_Exec_Size instExecSize = execSize;
    execSize = roundUpExecSize(execSize);

    // always 8 or 16
    G4_ExecSize exSize = toExecSize(execSize);
    // can be 1 for scalar atomics
    G4_ExecSize instExSize = toExecSize(instExecSize);
    G4_InstOpts instOpt = Get_Gen4_Emask(eMask, instExSize);
    unsigned subOpc = Get_Atomic_Op(atomicOp);

    bool useSplitSend = useSends();
    bool hasRet = !dst->isNullReg();

    if (atomicOp == ATOMIC_CMPXCHG)
    {
        std::swap(src0, src1);
    }

    PayloadSource sources[4]; // optional header + offsets + [src0] + [src1]
    unsigned len = 0;

    bool useHeader = needsA32MsgHeader() && surface && isStatelessSurface(surface);
    if (useHeader) {
        G4_Declare *dcl = createSendPayloadDcl(GENX_DATAPORT_IO_SZ, Type_UD);

        BuildMH1_A32_PSM(this, dcl);

        G4_SrcRegRegion *header
            = Create_Src_Opnd_From_Dcl(dcl, getRegionStride1());
        sources[len].opnd = header;
        sources[len].execSize = g4::SIMD8;
        sources[len].instOpt = InstOpt_WriteEnable;
        ++len;
    }

    sources[len].opnd = offsets;
    sources[len].execSize = exSize;
    sources[len].instOpt = instOpt;
    ++len;

    if (src0 && !src0->isNullReg()) {
        sources[len].opnd = src0;
        sources[len].execSize = exSize;
        sources[len].instOpt = instOpt;
        ++len;
    }

    if (src1 && !src1->isNullReg()) {
        sources[len].opnd = src1;
        sources[len].execSize = exSize;
        sources[len].instOpt = instOpt;
        ++len;
    }

    G4_SrcRegRegion *msgs[2] = {0, 0};
    unsigned sizes[2] = {0, 0};
    preparePayload(msgs, sizes, exSize, useSplitSend, sources, len);

    SFID sfid = SFID::DP_DC1;
    unsigned MD = 0;
    bool IsFloatOp = IsFloatAtomicOps(atomicOp);

    // Bit 12 specifies the SIMD mode.
    MD |= (execSize == EXEC_SIZE_8 ? MDC_SM2R_SIMD8 : MDC_SM2R_SIMD16) << 12;
    if (is16Bit)
    {
        MD |= (IsFloatOp ? static_cast<unsigned>(DC1_UNTYPED_HALF_FLOAT_ATOMIC)
            : static_cast<unsigned>(DC1_UNTYPED_HALF_INTEGER_ATOMIC))
            << 14;
    }
    else
    {
        MD |= (IsFloatOp ? static_cast<unsigned>(DC1_UNTYPED_FLOAT_ATOMIC)
            : static_cast<unsigned>(DC1_UNTYPED_ATOMIC))
            << 14;
    }
    MD |= (hasRet ? 1 : 0) << 13;
    MD |= subOpc << 8;

    unsigned resLen = hasRet ? (exSize / GENX_DATAPORT_IO_SZ) : 0;
    bool forceSplitSend = shouldForceSplitSend(surface);
    if (msgs[1] == 0 && !forceSplitSend) {
        ASSERT_USER(sizes[1] == 0, "Expect the 2nd part of the payload has zero size!");
        Create_Send_Inst_For_CISA(pred, dst,
            msgs[0], sizes[0],
            resLen,
            instExSize,
            MD, sfid,
            useHeader,
            SendAccess::READ_WRITE,
            surface, NULL,
            instOpt, false);
    } else {
        Create_SplitSend_Inst_For_CISA(pred, dst,
            msgs[0], sizes[0], msgs[1], sizes[1],
            resLen,
            instExSize,
            MD, sfid,
            useHeader,
            SendAccess::READ_WRITE,
            surface, NULL,
            instOpt, false);
    }

    return VISA_SUCCESS;
}


// build the address payload for typed messages (read/write/atomic)
// sources stores the address payload, and its length len is also updated
void IR_Builder::buildTypedSurfaceAddressPayload(
    G4_SrcRegRegion* uOffsetOpnd,
    G4_SrcRegRegion* vOffsetOpnd,
    G4_SrcRegRegion* rOffsetOpnd,
    G4_SrcRegRegion* lodOpnd,
    G4_ExecSize exSize,
    G4_InstOpts instOpt,
    PayloadSource sources[],
    uint32_t& len)
{
    // Valid address payload pattern are listed below:
    // (* means the parameter is ignored by HW but must be included in payload)
    // U
    // U, V
    // U, V, R
    // U, *, *, LOD
    // U, V, *, LOD
    // U, V, R, LOD

    // Append U
    sources[len].opnd = uOffsetOpnd;
    sources[len].execSize = exSize;
    sources[len].instOpt = instOpt;
    ++len;

    // Append V if any.
    if (!vOffsetOpnd->isNullReg()) {
        sources[len].opnd = vOffsetOpnd;
        sources[len].execSize = exSize;
        sources[len].instOpt = instOpt;
        ++len;
    }
    else if (!lodOpnd->isNullReg()) {
        G4_SrcRegRegion *nullVOffset = createNullSrc(Type_UD);
        sources[len].opnd = nullVOffset;
        sources[len].execSize = exSize;
        sources[len].instOpt = instOpt;
        ++len;
    }

    // Append R if any.
    if (!rOffsetOpnd->isNullReg()) {
        ASSERT_USER(!vOffsetOpnd->isNullReg(),
            "r offset must be NULL if v offset is NULL");
        sources[len].opnd = rOffsetOpnd;
        sources[len].execSize = exSize;
        sources[len].instOpt = instOpt;
        ++len;
    }
    else if (!lodOpnd->isNullReg()) {
        G4_SrcRegRegion *nullROffset = createNullSrc(Type_UD);
        sources[len].opnd = nullROffset;
        sources[len].execSize = exSize;
        sources[len].instOpt = instOpt;
        ++len;
    }

    // Append LOD if any.
    if (!lodOpnd->isNullReg()) {
        sources[len].opnd = lodOpnd;
        sources[len].execSize = exSize;
        sources[len].instOpt = instOpt;
        ++len;
    }
}


// u must not be V0. v and r are allowed to be V0, in which case they will be
// skipped in payload.
int IR_Builder::translateVISAGather4TypedInst(
    G4_Predicate *pred,
    VISA_EMask_Ctrl emask,
    ChannelMask chMask,
    G4_Operand *surface,
    VISA_Exec_Size executionSize,
    G4_SrcRegRegion *uOffsetOpnd,
    G4_SrcRegRegion *vOffsetOpnd,
    G4_SrcRegRegion *rOffsetOpnd,
    G4_SrcRegRegion *lodOpnd,
    G4_DstRegRegion *dstOpnd)
{
    TIME_SCOPE(VISA_BUILDER_IR_CONSTRUCTION);

    G4_ExecSize exSize = executionSize == EXEC_SIZE_16 ? g4::SIMD16 : g4::SIMD8;
    assert((exSize == 8 || hasSIMD16TypedRW()) && "only simd8 is supported");
    G4_InstOpts instOpt = Get_Gen4_Emask(emask, exSize);
    int numEnabledChannels = chMask.getNumEnabledChannels();

    bool useSplitSend = useSends();

    bool hasHeader = getPlatform() == GENX_BDW;

    PayloadSource sources[5]; // (maybe header) + maximal 4 addresses
    unsigned len = 0;

    if (hasHeader)
    {
        // Build header
        G4_Declare *dcl = createSendPayloadDcl(GENX_DATAPORT_IO_SZ, Type_UD);
        BuildMH1_BTS_PSM(this, dcl);

        // Append header
        G4_SrcRegRegion *header
            = Create_Src_Opnd_From_Dcl(dcl, getRegionStride1());
        sources[len].opnd = header;
        sources[len].execSize = g4::SIMD8;
        sources[len].instOpt = InstOpt_WriteEnable;
        ++len;
    }

    buildTypedSurfaceAddressPayload(uOffsetOpnd, vOffsetOpnd, rOffsetOpnd, lodOpnd, exSize, instOpt, sources, len);
    G4_SrcRegRegion *msgs[2] = {0, 0};
    unsigned sizes[2] = {0, 0};
    preparePayload(msgs, sizes, exSize, useSplitSend, sources, len);

    //bit 8-11: RGBA channel enable
    unsigned msgDesc = chMask.getHWEncoding() << 8;
    SFID sfId;

    // DC1
    // bit14-17: 0101 (read), 1101 (write)
    msgDesc |= DC1_TYPED_SURFACE_READ << 14;
    // bit12-13: 01 (use low 8 slot)
    msgDesc |= MDC_SG3_SG8L << 12;
    sfId = SFID::DP_DC1;

    bool forceSplitSend = shouldForceSplitSend(surface);
    if (msgs[1] == 0 && !forceSplitSend) {
        ASSERT_USER(sizes[1] == 0, "Expect the 2nd part of the payload has zero size!");
        Create_Send_Inst_For_CISA(pred, dstOpnd,
            msgs[0], sizes[0],
            numEnabledChannels,
            exSize,
            msgDesc, sfId,
            hasHeader,
            SendAccess::READ_ONLY,
            surface, nullptr,
            instOpt, false);
    } else {
        Create_SplitSend_Inst_For_CISA(pred, dstOpnd,
            msgs[0], sizes[0], msgs[1], sizes[1],
            numEnabledChannels,
            exSize,
            msgDesc, sfId,
            hasHeader,
            SendAccess::READ_ONLY,
            surface, nullptr,
            instOpt, false);
    }

    return VISA_SUCCESS;
}

// u must not be V0. v and r are allowed to be V0, in which case they will be
// skipped in payload.
int IR_Builder::translateVISAScatter4TypedInst(
    G4_Predicate *pred,
    VISA_EMask_Ctrl emask,
    ChannelMask chMask,
    G4_Operand *surface,
    VISA_Exec_Size executionSize,
    G4_SrcRegRegion *uOffsetOpnd,
    G4_SrcRegRegion *vOffsetOpnd,
    G4_SrcRegRegion *rOffsetOpnd,
    G4_SrcRegRegion *lodOpnd,
    G4_SrcRegRegion *srcOpnd)
{
    TIME_SCOPE(VISA_BUILDER_IR_CONSTRUCTION);

    G4_ExecSize exSize = executionSize == EXEC_SIZE_16 ? g4::SIMD16 : g4::SIMD8;
    assert((exSize == g4::SIMD8 || hasSIMD16TypedRW()) && "only simd8 is supported");
    G4_InstOpts instOpt = Get_Gen4_Emask(emask, exSize);
    int numEnabledChannels = chMask.getNumEnabledChannels();

    bool useSplitSend = useSends();

    bool hasHeader = getPlatform() == GENX_BDW;

    PayloadSource sources[6]; // (maybe header) + maximal 4 addresses + source
    unsigned len = 0;

    if (hasHeader)
    {
        // Build header
        G4_Declare *dcl = createSendPayloadDcl(GENX_DATAPORT_IO_SZ, Type_UD);
        BuildMH1_BTS_PSM(this, dcl);

        // Append header
        G4_SrcRegRegion *header
            = Create_Src_Opnd_From_Dcl(dcl, getRegionStride1());
        sources[len].opnd = header;
        sources[len].execSize = g4::SIMD8;
        sources[len].instOpt = InstOpt_WriteEnable;
        ++len;
    }

    buildTypedSurfaceAddressPayload(uOffsetOpnd, vOffsetOpnd, rOffsetOpnd, lodOpnd, exSize, instOpt, sources, len);

    // Append source
    sources[len].opnd = srcOpnd;
    sources[len].execSize = G4_ExecSize(exSize * numEnabledChannels);
    sources[len].instOpt = instOpt;
    ++len;

    G4_SrcRegRegion *msgs[2] = {0, 0};
    unsigned sizes[2] = {0, 0};
    preparePayload(msgs, sizes, exSize, useSplitSend, sources, len);

    //bit 8-11: RGBA channel enable
    unsigned msgDesc = 0;
    SFID sfId;

    // DC1
    // bit14-17: 0101 (read), 1101 (write)
    msgDesc |= DC1_TYPED_SURFACE_WRITE << 14;
    // bit12-13: 01 (use low 8 slot)
    msgDesc |= MDC_SG3_SG8L << 12;
    sfId = SFID::DP_DC1;

    msgDesc |= chMask.getHWEncoding() << 8;

    G4_DstRegRegion* dstOpnd = createNullDst(Type_UD);

    bool forceSplitSend = shouldForceSplitSend(surface);
    if (msgs[1] == 0 && !forceSplitSend) {
        ASSERT_USER(sizes[1] == 0, "Expect the 2nd part of the payload has zero size!");
        Create_Send_Inst_For_CISA(
            pred, dstOpnd,
            msgs[0], sizes[0],
            0,
            exSize,
            msgDesc, sfId,
            hasHeader,
            SendAccess::WRITE_ONLY,
            surface, NULL,
            instOpt, false);
    } else
    {
        Create_SplitSend_Inst_For_CISA(
            pred, dstOpnd,
            msgs[0], sizes[0], msgs[1], sizes[1],
            0,
            exSize,
            msgDesc, sfId,
            hasHeader,
            SendAccess::WRITE_ONLY,
            surface, NULL,
            instOpt, false);
    }

    return VISA_SUCCESS;
}

int IR_Builder::translateVISATypedAtomicInst(
    VISAAtomicOps atomicOp,
    bool is16Bit,
    G4_Predicate *pred,
    VISA_EMask_Ctrl emask,
    VISA_Exec_Size execSize,
    G4_Operand *surface,
    G4_SrcRegRegion *uOffsetOpnd,
    G4_SrcRegRegion *vOffsetOpnd,
    G4_SrcRegRegion *rOffsetOpnd,
    G4_SrcRegRegion *lodOpnd,
    G4_SrcRegRegion *src0,
    G4_SrcRegRegion *src1,
    G4_DstRegRegion *dst)
{
    TIME_SCOPE(VISA_BUILDER_IR_CONSTRUCTION);

    VISA_Exec_Size instExecSize = execSize;
    assert(execSize <= (getNativeExecSize() == g4::SIMD8 ? EXEC_SIZE_8 : EXEC_SIZE_16) &&
        "send exec size must not exceed the platform's native execution size");

    unsigned op = Get_Atomic_Op(atomicOp);

    G4_ExecSize exSize {Get_VISA_Exec_Size(execSize)};
    G4_ExecSize instExSize {Get_VISA_Exec_Size(instExecSize)};
    G4_InstOpts instOpt = Get_Gen4_Emask(emask, instExSize);

    if (atomicOp == ATOMIC_CMPXCHG)
    {
        // we have to swap src0 and src1 since vISA has them in different order from HW
        G4_SrcRegRegion* tmp = src0;
        src0 = src1;
        src1 = tmp;
    }

    bool useSplitSend = useSends();

    PayloadSource sources[6]; // u, v, r, lod, src0, src1
    unsigned len = 0;

    buildTypedSurfaceAddressPayload(uOffsetOpnd, vOffsetOpnd, rOffsetOpnd, lodOpnd, exSize, instOpt, sources, len);

    if (src0 != nullptr && !src0->isNullReg())
    {
        sources[len].opnd = src0;
        sources[len].execSize = exSize;
        sources[len].instOpt = instOpt;
        ++len;
    }

    if (src1 != nullptr && !src1->isNullReg())
    {
        sources[len].opnd = src1;
        sources[len].execSize = exSize;
        sources[len].instOpt = instOpt;
        ++len;
    }

    G4_SrcRegRegion *msgs[2] = {0, 0};
    unsigned sizes[2] = {0, 0};
    preparePayload(msgs, sizes, exSize, useSplitSend, sources, len);

    unsigned dstLength = dst->isNullReg() ? 0 : 1;

    unsigned msgDesc = 0;
    // BTI is filled later
    msgDesc |= op << 8;
    msgDesc |= (dstLength != 0 ? 1 : 0) << 13;

    if (is16Bit)
    {
        msgDesc |= DC1_TYPED_HALF_INTEGER_ATOMIC << 14;
    }
    else
    {
        msgDesc |= DC1_TYPED_ATOMIC << 14;
    }

    bool forceSplitSend = shouldForceSplitSend(surface);
    if (msgs[1] == 0 && !forceSplitSend)
    {
        ASSERT_USER(sizes[1] == 0, "Expect the 2nd part of the payload has zero size!");
        Create_Send_Inst_For_CISA(pred, dst,
            msgs[0], sizes[0], dstLength, exSize,
            msgDesc, SFID::DP_DC1,
            false,
            SendAccess::READ_WRITE,
            surface, nullptr,
            instOpt, false);
    }
    else
    {
        Create_SplitSend_Inst_For_CISA(pred, dst,
            msgs[0], sizes[0], msgs[1], sizes[1],
            dstLength, exSize,
            msgDesc, SFID::DP_DC1,
            false,
            SendAccess::READ_WRITE,
            surface, nullptr,
            instOpt, false);
    }

    return VISA_SUCCESS;
}

static void BuildMH2_A32_PSM(
    IR_Builder *IRB, G4_Declare *header, uint16_t scale, G4_Operand *globalOffset)
{
    // Clear header
    G4_DstRegRegion *h = IRB->createDst(header->getRegVar(),
        0, 0, 1, Type_UD);
    IRB->createMov(g4::SIMD8, h, IRB->createImm(0, Type_UD), InstOpt_WriteEnable, true);
    // Copy global offset if necessary.
    if (!(globalOffset->isImm() && globalOffset->asImm()->isZero())) {
        G4_DstRegRegion* gOffDst = IRB->createDst(
            header->getRegVar(),
            0, 5, 1, Type_UD);
        IRB->createMov(g4::SIMD1, gOffDst, globalOffset, InstOpt_WriteEnable, true);
    }
    // Copy scale pitch if necessary.
    if (scale != 0) {
        G4_Operand* scaleImm = IRB->createImm(scale, Type_UD);
        G4_DstRegRegion* pitchDst = IRB->createDst(
            header->getRegVar(),
            0, 0, 1, Type_UD);
        IRB->createMov(g4::SIMD1, pitchDst, scaleImm, InstOpt_WriteEnable, true);
    }
    // Copy PSM which is set to 0xFFFF so far.
    G4_Operand* maskImm = IRB->createImm(0xFFFF, Type_UD);
    G4_DstRegRegion* pitchDst = IRB->createDst(
        header->getRegVar(),
        0, 7, 1, Type_UD);
    IRB->createMov(g4::SIMD1, pitchDst, maskImm, InstOpt_WriteEnable, true);
}

// apply the sideband offset (can be either imm or variable) to the message descriptor
void IR_Builder::applySideBandOffset(
    G4_Operand* sideBand, G4_SendMsgDescriptor* sendMsgDesc)
{
#define SIDEBAND_OFFSET_IN_EXDESC 12

    if (sideBand->isImm())
    {
        // mov (1) a0.0 sideband << 0xC
        uint32_t sidebandInDesc = (uint32_t)(sideBand->asImm()->getImm() << SIDEBAND_OFFSET_IN_EXDESC);
        G4_DstRegRegion* dst = Create_Dst_Opnd_From_Dcl(builtinA0, 1);
        createMov(g4::SIMD1, dst, createImm(sidebandInDesc, Type_UD), InstOpt_WriteEnable, true);
    }
    else
    {
        MUST_BE_TRUE(sideBand->isSrcRegRegion(), "sideband offset should be a srcRegRegion");
        // shl (1) a0.0 sideband 0xC
        G4_DstRegRegion* dst = Create_Dst_Opnd_From_Dcl(builtinA0, 1);
        createBinOp(G4_shl, g4::SIMD1, dst, sideBand,
            createImm(SIDEBAND_OFFSET_IN_EXDESC, Type_UW), InstOpt_WriteEnable, true);
    }

    // add (1) a0.0 a0.0 MD
    G4_DstRegRegion* a0Dst = Create_Dst_Opnd_From_Dcl(builtinA0, 1);
    G4_SrcRegRegion* a0Src = Create_Src_Opnd_From_Dcl(builtinA0, getRegionScalar());
    createBinOp(G4_add, g4::SIMD1, a0Dst, a0Src,
        createImm(sendMsgDesc->getExtendedDesc(), Type_UD), InstOpt_WriteEnable, true);
}

static void BuildMH2_A32(IR_Builder *IRB, G4_Declare *header,
    uint16_t scale, G4_Operand *globalOffset) {
    // Clear header
    G4_DstRegRegion *h = IRB->createDst(header->getRegVar(),
        0, 0, 1, Type_UD);
    IRB->createMov(g4::SIMD8, h, IRB->createImm(0, Type_UD), InstOpt_WriteEnable, true);
    // Copy global offset if necessary.
    if (!(globalOffset->isImm() && globalOffset->asImm()->isZero())) {
        G4_DstRegRegion* gOffDst = IRB->createDst(
            header->getRegVar(),
            0, 5, 1, Type_UD);
        IRB->createMov(g4::SIMD1, gOffDst, globalOffset, InstOpt_WriteEnable, true);
    }
    // Copy scale pitch if necessary.
    if (scale != 0) {
        G4_Operand* scaleImm = IRB->createImm(scale, Type_UD);
        G4_DstRegRegion* pitchDst = IRB->createDst(
            header->getRegVar(),
            0, 0, 1, Type_UD);
        IRB->createMov(g4::SIMD1, pitchDst, scaleImm, InstOpt_WriteEnable, true);
    }
}


int IR_Builder::translateVISASLMUntypedScaledInst(
    bool isRead,
    G4_Predicate *pred,
    VISA_Exec_Size execSize,
    VISA_EMask_Ctrl eMask,
    ChannelMask chMask,
    uint16_t scale,
    G4_Operand *sideBand,
    G4_SrcRegRegion *offsets,
    G4_Operand *srcOrDst)
{
    TIME_SCOPE(VISA_BUILDER_IR_CONSTRUCTION);

    G4_ExecSize exSize {Get_VISA_Exec_Size(execSize)};
    G4_InstOpts instOpt = Get_Gen4_Emask(eMask, exSize);

    PayloadSource sources[2]; // Maximal 2 sources, offset + source
    unsigned len = 0;

    sources[len].opnd = offsets;
    sources[len].execSize = exSize;
    sources[len].instOpt = instOpt;
    ++len;
    if (!isRead)
    {
        sources[len].opnd = srcOrDst->asSrcRegRegion();
        sources[len].execSize = G4_ExecSize(exSize * chMask.getNumEnabledChannels());
        sources[len].instOpt = instOpt;
        ++len;
    }

    G4_SrcRegRegion *msgs[2] = { 0, 0 };
    unsigned sizes[2] = { 0, 0 };
    preparePayload(msgs, sizes, exSize, true, sources, len);

    SFID sfid = SFID::DP_DC2;

    unsigned MD = 0;
    // Leave sidebind scale offset 0 as it is not used now.
    MD |= (isRead ? DC2_UNTYPED_SURFACE_READ : DC2_UNTYPED_SURFACE_WRITE) << 14;
    MD |= (execSize == EXEC_SIZE_8 ? MDC_SM3_SIMD8 : MDC_SM3_SIMD16) << 12;
    MD |= chMask.getHWEncoding() << 8;
    // SLM encodes scale pitch in MD.
    MD |= 1 << 7;
    MD |= scale & 0x7F;

    G4_DstRegRegion *dst = isRead ? srcOrDst->asDstRegRegion() : createNullDst(Type_UD);
    unsigned resLen = isRead ? (exSize / GENX_DATAPORT_IO_SZ) *
        chMask.getNumEnabledChannels() : 0;

    uint32_t exFuncCtrl = 0;
    G4_SendMsgDescriptor *sendMsgDesc = createSendMsgDesc(MD, resLen, sizes[0], sfid,
        sizes[1], (uint16_t)exFuncCtrl, isRead ? SendAccess::READ_ONLY : SendAccess::WRITE_ONLY);

    applySideBandOffset(sideBand, sendMsgDesc);

    createSplitSendInst(
        pred,
        G4_sends,
        exSize,
        dst,
        msgs[0],
        msgs[1],
        createImm(sendMsgDesc->getDesc(), Type_UD),
        instOpt,
        sendMsgDesc,
        Create_Src_Opnd_From_Dcl(builtinA0, getRegionScalar()),
        true);

    return VISA_SUCCESS;
}



int IR_Builder::translateVISAGather4ScaledInst(
    G4_Predicate     *pred,
    VISA_Exec_Size    execSize,
    VISA_EMask_Ctrl   eMask,
    ChannelMask       chMask,
    G4_Operand       *surface,
    G4_Operand       *globalOffset,
    G4_SrcRegRegion  *offsets,
    G4_DstRegRegion  *dst)
{
    surface = lowerSurface255To253(surface, *this);
    return translateGather4Inst(pred, execSize, eMask, chMask, surface,
        globalOffset, offsets, dst);
}

int IR_Builder::translateVISAScatter4ScaledInst(
    G4_Predicate     *pred,
    VISA_Exec_Size    execSize,
    VISA_EMask_Ctrl   eMask,
    ChannelMask       chMask,
    G4_Operand       *surface,
    G4_Operand       *globalOffset,
    G4_SrcRegRegion  *offsets,
    G4_SrcRegRegion  *src)
{
    surface = lowerSurface255To253(surface, *this);
    return translateScatter4Inst(pred, execSize, eMask, chMask, surface,
        globalOffset, offsets, src);
}

int IR_Builder::translateGather4Inst(
    G4_Predicate     *pred,
    VISA_Exec_Size    execSize,
    VISA_EMask_Ctrl   eMask,
    ChannelMask       chMask,
    G4_Operand       *surface,
    G4_Operand       *globalOffset,
    G4_SrcRegRegion  *offsets,
    G4_DstRegRegion  *dst)
{
    TIME_SCOPE(VISA_BUILDER_IR_CONSTRUCTION);

    ASSERT_USER(execSize == EXEC_SIZE_1 || execSize == EXEC_SIZE_2 ||
        execSize == EXEC_SIZE_4 || execSize == EXEC_SIZE_8 ||
        execSize == EXEC_SIZE_16,
        "Only support SIMD1, SIMD2, SIMD4, SIMD8 or SIMD16!");

    VISA_Exec_Size instExecSize = execSize;
    execSize = roundUpExecSize(execSize);

    G4_ExecSize exSize = toExecSize(execSize);
    G4_ExecSize instExSize = toExecSize(instExecSize);
    unsigned instOpt = Get_Gen4_Emask(eMask, exSize);

    bool useSplitSend = useSends();
    bool useHeader = needsA32MsgHeader() && surface && isStatelessSurface(surface);

    // In case non-zero global offset is specified, we need to recalculate
    // offsets.
    if (!globalOffset->isImm() || globalOffset->asImm()->getImm() != 0) {
        G4_Declare *dcl = createSendPayloadDcl(exSize, offsets->getType());
        G4_DstRegRegion *tmp = Create_Dst_Opnd_From_Dcl(dcl, 1);
        createInst(pred, G4_add, 0, g4::NOSAT, instExSize, tmp, offsets, globalOffset, instOpt, true);
        offsets = Create_Src_Opnd_From_Dcl(dcl, getRegionStride1());
    }

    PayloadSource sources[2]; // Maximal 2 sources, optional header + offsets
    unsigned len = 0;

    if (useHeader) {
        G4_Declare *dcl = createSendPayloadDcl(GENX_DATAPORT_IO_SZ, Type_UD);

        BuildMH1_A32_PSM(this, dcl);

        G4_SrcRegRegion *header
            = Create_Src_Opnd_From_Dcl(dcl, getRegionStride1());
        sources[len].opnd = header;
        sources[len].execSize = g4::SIMD8;
        sources[len].instOpt = InstOpt_WriteEnable;
        ++len;
    }

    sources[len].opnd = offsets;
    sources[len].execSize = exSize;
    sources[len].instOpt = instOpt;
    ++len;

    G4_SrcRegRegion *msgs[2] = {0, 0};
    unsigned sizes[2] = {0, 0};
    preparePayload(msgs, sizes, exSize, useSplitSend, sources, len);

    SFID sfid = SFID::DP_DC1;

    unsigned MD = 0;
    // Leave sidebind scale offset 0 as it is not used now.
    MD |= DC1_UNTYPED_SURFACE_READ << 14;
    MD |= (execSize == EXEC_SIZE_8 ? MDC_SM3_SIMD8 : MDC_SM3_SIMD16) << 12;
    MD |= chMask.getHWEncoding() << 8;

    unsigned resLen = (exSize / GENX_DATAPORT_IO_SZ) *
        chMask.getNumEnabledChannels();

    bool forceSplitSend = shouldForceSplitSend(surface);
    if (msgs[1] == 0 && !forceSplitSend) {
        ASSERT_USER(sizes[1] == 0, "Expect the 2nd part of the payload has zero size!");
        Create_Send_Inst_For_CISA(pred, dst,
            msgs[0], sizes[0],
            resLen,
            instExSize,
            MD, sfid,
            useHeader,
            SendAccess::READ_ONLY,
            surface, NULL,
            instOpt, false);
    } else {
        Create_SplitSend_Inst_For_CISA(pred, dst,
            msgs[0], sizes[0], msgs[1], sizes[1],
            resLen,
            instExSize,
            MD, sfid,
            useHeader,
            SendAccess::READ_ONLY,
            surface, NULL,
            instOpt, false);
    }

    return VISA_SUCCESS;
}


int IR_Builder::translateScatter4Inst(
    G4_Predicate          *pred,
    VISA_Exec_Size         execSize,
    VISA_EMask_Ctrl        eMask,
    ChannelMask            chMask,
    G4_Operand             *surface,
    G4_Operand             *globalOffset,
    G4_SrcRegRegion        *offsets,
    G4_SrcRegRegion        *src)
{
    TIME_SCOPE(VISA_BUILDER_IR_CONSTRUCTION);

    ASSERT_USER(
        execSize == EXEC_SIZE_1 || execSize == EXEC_SIZE_2 ||
        execSize == EXEC_SIZE_4 || execSize == EXEC_SIZE_8 ||
        execSize == EXEC_SIZE_16,
        "Only support SIMD1, SIMD2, SIMD4, SIMD8 or SIMD16!");

    VISA_Exec_Size instExecSize = execSize;
    execSize = roundUpExecSize(execSize);

    G4_ExecSize exSize = toExecSize(execSize);
    G4_ExecSize instExSize = toExecSize(instExecSize);
    unsigned instOpt = Get_Gen4_Emask(eMask, exSize);

    bool useSplitSend = useSends();
    bool useHeader = needsA32MsgHeader() && surface && isStatelessSurface(surface);

    // In case non-zero global offset is specified, we need to recalculate
    // offsets.
    if (!globalOffset->isImm() || globalOffset->asImm()->getImm() != 0) {
        G4_Declare *dcl = createSendPayloadDcl(exSize, offsets->getType());
        G4_DstRegRegion *tmp = Create_Dst_Opnd_From_Dcl(dcl, 1);
        createInst(pred, G4_add, 0, g4::NOSAT, instExSize, tmp, offsets, globalOffset, instOpt, true);
        offsets = Create_Src_Opnd_From_Dcl(dcl, getRegionStride1());
    }

    PayloadSource sources[3]; // Maximal 3 sources, optional header + offsets + src
    unsigned len = 0;

    if (useHeader) {
        G4_Declare *dcl = createSendPayloadDcl(GENX_DATAPORT_IO_SZ, Type_UD);

        // TODO: Get PSM supported on demand.
        BuildMH1_A32_PSM(this, dcl);

        G4_SrcRegRegion *header
            = Create_Src_Opnd_From_Dcl(dcl, getRegionStride1());
        sources[len].opnd = header;
        sources[len].execSize = g4::SIMD8;
        sources[len].instOpt = InstOpt_WriteEnable;
        ++len;
    }

    sources[len].opnd = offsets;
    sources[len].execSize = exSize;
    sources[len].instOpt = instOpt;
    ++len;
    sources[len].opnd = src;
    sources[len].execSize = G4_ExecSize(exSize * chMask.getNumEnabledChannels());
    sources[len].instOpt = instOpt;
    ++len;

    G4_SrcRegRegion *msgs[2] = {0, 0};
    unsigned sizes[2] = {0, 0};
    preparePayload(msgs, sizes, exSize, useSplitSend, sources, len);

    SFID sfid = SFID::DP_DC1;

    unsigned MD = 0;
    // Leave sidebind scale offset 0 as it is not used now.
    MD |= DC1_UNTYPED_SURFACE_WRITE << 14;
    MD |= (execSize == EXEC_SIZE_8 ? MDC_SM3_SIMD8 : MDC_SM3_SIMD16) << 12;
    MD |= chMask.getHWEncoding() << 8;

    G4_DstRegRegion *dst = createNullDst(Type_UD);
    bool forceSplitSend = shouldForceSplitSend(surface);
    if (msgs[1] == 0 && !forceSplitSend) {
        ASSERT_USER(sizes[1] == 0, "Expect the 2nd part of the payload has zero size!");
        Create_Send_Inst_For_CISA(pred, dst,
            msgs[0], sizes[0],
            0,
            instExSize,
            MD, sfid,
            useHeader,
            SendAccess::WRITE_ONLY,
            surface, NULL,
            instOpt, false);
    }
    else {
        Create_SplitSend_Inst_For_CISA(pred, dst,
            msgs[0], sizes[0], msgs[1], sizes[1],
            0,
            instExSize,
            MD, sfid,
            useHeader,
            SendAccess::WRITE_ONLY,
            surface, NULL,
            instOpt, false);
    }

    return VISA_SUCCESS;
}

/// GetNumBatch() - return the number of batches required to copy the raw
/// operand to message payload
static unsigned GetNumBatch(
    VISA_SVM_Block_Type blockSize, VISA_SVM_Block_Num  numBlocks)
{
    switch (blockSize) {
    case SVM_BLOCK_TYPE_BYTE:
        switch (numBlocks) {
        case SVM_BLOCK_NUM_1:
        case SVM_BLOCK_NUM_2:
        case SVM_BLOCK_NUM_4:
            return 1;
        case SVM_BLOCK_NUM_8:
            return 2;
        }
        break;
    case SVM_BLOCK_TYPE_DWORD:
        return Get_Common_ISA_SVM_Block_Num(numBlocks);
    case SVM_BLOCK_TYPE_QWORD:
        return Get_Common_ISA_SVM_Block_Num(numBlocks);
    }
    ASSERT_USER(false, "Unhandled sizes/numbers of block/element!");
    return 0;
}

int IR_Builder::translateVISAGatherScaledInst(
    G4_Predicate              *pred,
    VISA_Exec_Size             execSize,
    VISA_EMask_Ctrl            eMask,
    VISA_SVM_Block_Num         numBlocks,
    G4_Operand                *surface,
    G4_Operand                *globalOffset,
    G4_SrcRegRegion           *offsets,
    G4_DstRegRegion           *dst)
{
    surface = lowerSurface255To253(surface, *this);

    return translateByteGatherInst(pred, execSize, eMask, numBlocks,
        surface, globalOffset, offsets, dst);
}

int IR_Builder::translateVISAScatterScaledInst(
    G4_Predicate              *pred,
    VISA_Exec_Size             execSize,
    VISA_EMask_Ctrl            eMask,
    VISA_SVM_Block_Num         numBlocks,
    G4_Operand                *surface,
    G4_Operand                *globalOffset,
    G4_SrcRegRegion           *offsets,
    G4_SrcRegRegion           *src)
{

    surface = lowerSurface255To253(surface, *this);
    return translateByteScatterInst(pred, execSize, eMask, numBlocks,
        surface, globalOffset, offsets, src);
}

//
// For a SLM byte scaled inst with non-zero sideband, we must generate a split send
// and store the sideband into the extended message descriptor.  ExDesc must be indirect
// in this case
//
int IR_Builder::translateVISASLMByteScaledInst(
    bool isRead,
    G4_Predicate *pred,
    VISA_Exec_Size execSize,
    VISA_EMask_Ctrl eMask,
    VISA_SVM_Block_Type blockSize,
    VISA_SVM_Block_Num numBlocks,
    uint8_t scale,
    G4_Operand *sideBand,
    G4_SrcRegRegion *offsets,
    G4_Operand *srcOrDst)
{
    TIME_SCOPE(VISA_BUILDER_IR_CONSTRUCTION);

    G4_ExecSize exSize {Get_VISA_Exec_Size(execSize)};
    G4_InstOpts instOpt = Get_Gen4_Emask(eMask, exSize);
    unsigned numBatch = GetNumBatch(blockSize, numBlocks);

    uint16_t exFuncCtrl = 0;
    PayloadSource sources[3];
    unsigned len = 0;

    sources[len].opnd = offsets;
    sources[len].execSize = exSize;
    sources[len].instOpt = instOpt;
    ++len;
    if (!isRead)
    {
        sources[len].opnd = srcOrDst->asSrcRegRegion();
        sources[len].execSize = G4_ExecSize(exSize * numBatch);
        sources[len].instOpt = instOpt;
        ++len;
    }

    G4_SrcRegRegion *msgs[2] = { 0, 0 };
    unsigned sizes[2] = { 0, 0 };
    preparePayload(msgs, sizes, exSize, true, sources, len);

    SFID sfid = SFID::DP_DC2;

    unsigned MD = 0;
    // Leave sidebind scale offset 0 as it is not used now.
    MD |= (isRead ? DC2_BYTE_SCATTERED_READ : DC2_BYTE_SCATTERED_WRITE) << 14;
    MD |= numBlocks << 10;
    MD |= (execSize == EXEC_SIZE_8 ? MDC_SM2_SIMD8 : MDC_SM2_SIMD16) << 8;
    MD |= 1 << 7;
    MD |= scale & 0x7F;

    G4_DstRegRegion *dst = isRead ? srcOrDst->asDstRegRegion() : createNullDst(Type_UD);
    unsigned resLen = isRead ? (exSize / GENX_DATAPORT_IO_SZ) * numBatch : 0;

    G4_SendMsgDescriptor *sendMsgDesc = createSendMsgDesc(MD, resLen, sizes[0], sfid,
        sizes[1], exFuncCtrl, isRead ? SendAccess::READ_ONLY : SendAccess::WRITE_ONLY);

    applySideBandOffset(sideBand, sendMsgDesc);

    createSplitSendInst(
        pred,
        G4_sends,
        exSize,
        dst,
        msgs[0],
        msgs[1],
        createImm(sendMsgDesc->getDesc(), Type_UD),
        instOpt,
        sendMsgDesc,
        Create_Src_Opnd_From_Dcl(builtinA0, getRegionScalar()),
        true);

    return VISA_SUCCESS;
}

static void BuildMH_A32_GO(
    IR_Builder *IRB, G4_Declare *header, G4_Operand *globalOffset = 0)
{
    // Clear header
    G4_DstRegRegion *h = IRB->createDst(header->getRegVar(),
        0, 0, 1, Type_UD);
    IRB->createMov(g4::SIMD8, h, IRB->createImm(0, Type_UD), InstOpt_WriteEnable, true);
    // Copy global offset if necessary.
    if (globalOffset &&
        !(globalOffset->isImm() &&
            globalOffset->asImm()->isZero())) {
        G4_DstRegRegion* gOffDst = IRB->createDst(
            header->getRegVar(),
            0, 2, 1, Type_UD);
        IRB->createMov(g4::SIMD1, gOffDst, globalOffset, InstOpt_WriteEnable, true);
    }
}

int IR_Builder::translateByteGatherInst(
    G4_Predicate *pred,
    VISA_Exec_Size execSize,
    VISA_EMask_Ctrl eMask,
    VISA_SVM_Block_Num numBlocks,
    G4_Operand *surface,
    G4_Operand *globalOffset,
    G4_SrcRegRegion *offsets,
    G4_DstRegRegion *dst)
{
    TIME_SCOPE(VISA_BUILDER_IR_CONSTRUCTION);

    ASSERT_USER(execSize == EXEC_SIZE_1 || execSize == EXEC_SIZE_2 ||
        execSize == EXEC_SIZE_4 || execSize == EXEC_SIZE_8 ||
        execSize == EXEC_SIZE_16,
        "Only support SIMD1, SIMD2, SIMD4, SIMD8 or SIMD16!");
    ASSERT_USER(numBlocks == SVM_BLOCK_NUM_1 ||
        numBlocks == SVM_BLOCK_NUM_2 ||
        numBlocks == SVM_BLOCK_NUM_4,
        "Byte gather ONLY supports 1, 2, and 4 elements per slot!");

    VISA_Exec_Size instExecSize = execSize;
    execSize = roundUpExecSize(execSize);

    G4_ExecSize exSize {Get_VISA_Exec_Size(execSize)};
    G4_ExecSize instExSize {Get_VISA_Exec_Size(instExecSize)};
    G4_InstOpts instOpt = Get_Gen4_Emask(eMask, instExSize);
    unsigned numBatch = GetNumBatch(SVM_BLOCK_TYPE_BYTE, numBlocks);

    bool isSLM = IsSLMSurface(surface);
    // SLM forbids header. Header is optional in A32 when both scale and global
    // offset are 0s.
    bool useHeader = !isSLM && needsA32MsgHeader();
    bool useSplitSend = useSends();

    // In case non-zero global offset is specified, we need to recalculate
    // offsets.
    //
    // NOTE: Even though pre-SKL devices require header, eliminating global
    //       offset by adjusting offsets will simplify the header generation.
    if (!globalOffset->isImm() || globalOffset->asImm()->getImm() != 0)
    {
        G4_Declare *dcl = createSendPayloadDcl(exSize, offsets->getType());
        G4_DstRegRegion *tmp = Create_Dst_Opnd_From_Dcl(dcl, 1);
        createBinOp(G4_add, instExSize, tmp, offsets, globalOffset, instOpt, true);
        offsets = Create_Src_Opnd_From_Dcl(dcl, getRegionStride1());
    }

    PayloadSource sources[2]; // Maximal 2 sources, optional header + offsets
    unsigned len = 0;

    if (useHeader) {
        G4_Declare *dcl = createSendPayloadDcl(GENX_DATAPORT_IO_SZ, Type_UD);

        // TODO: Get BTS supported on demand.
        BuildMH_A32_GO(this, dcl);

        G4_SrcRegRegion *header
            = Create_Src_Opnd_From_Dcl(dcl, getRegionStride1());
        sources[len].opnd = header;
        sources[len].execSize = g4::SIMD8;
        sources[len].instOpt = InstOpt_WriteEnable;
        ++len;
    }

    sources[len].opnd = offsets;
    sources[len].execSize = exSize;
    sources[len].instOpt = instOpt;
    ++len;

    G4_SrcRegRegion *msgs[2] = {0, 0};
    unsigned sizes[2] = {0, 0};
    preparePayload(msgs, sizes, exSize, useSplitSend, sources, len);

    SFID sfid = SFID::DP_DC;

    unsigned MD = 0;
    MD |= DC_BYTE_SCATTERED_READ << 14;
    MD |= numBlocks << 10;
    MD |= (execSize == EXEC_SIZE_8 ? MDC_SM2_SIMD8 : MDC_SM2_SIMD16) << 8;

    unsigned resLen = (exSize / GENX_DATAPORT_IO_SZ) * numBatch;
    bool forceSplitSend = shouldForceSplitSend(surface);
    if (msgs[1] == 0 && !forceSplitSend) {
        ASSERT_USER(sizes[1] == 0, "Expect the 2nd part of the payload has zero size!");
        Create_Send_Inst_For_CISA(pred, dst,
            msgs[0], sizes[0],
            resLen,
            instExSize,
            MD, sfid,
            useHeader,
            SendAccess::READ_ONLY,
            surface, NULL,
            instOpt, false);
    }
    else {
        Create_SplitSend_Inst_For_CISA(pred, dst,
            msgs[0], sizes[0], msgs[1], sizes[1],
            resLen,
            instExSize,
            MD, sfid,
            useHeader,
            SendAccess::READ_ONLY,
            surface, NULL,
            instOpt, false);
    }

    return VISA_SUCCESS;
}

int IR_Builder::translateByteScatterInst(
    G4_Predicate *pred,
    VISA_Exec_Size execSize,
    VISA_EMask_Ctrl eMask,
    VISA_SVM_Block_Num numBlocks,
    G4_Operand *surface,
    G4_Operand *globalOffset,
    G4_SrcRegRegion *offsets,
    G4_SrcRegRegion *src)
{
    TIME_SCOPE(VISA_BUILDER_IR_CONSTRUCTION);

    ASSERT_USER(execSize == EXEC_SIZE_1 || execSize == EXEC_SIZE_2 ||
        execSize == EXEC_SIZE_4 || execSize == EXEC_SIZE_8 ||
        execSize == EXEC_SIZE_16,
        "Only support SIMD1, SIMD2, SIMD4, SIMD8 or SIMD16!");
    ASSERT_USER(numBlocks == SVM_BLOCK_NUM_1 ||
        numBlocks == SVM_BLOCK_NUM_2 ||
        numBlocks == SVM_BLOCK_NUM_4,
        "Byte scatter ONLY supports 1, 2, and 4 elements per slot!");

    VISA_Exec_Size instExecSize = execSize;
    execSize = roundUpExecSize(execSize);

    G4_ExecSize exSize {Get_VISA_Exec_Size(execSize)};
    G4_ExecSize instExSize {Get_VISA_Exec_Size(instExecSize)};
    G4_InstOpts instOpt = Get_Gen4_Emask(eMask, exSize);
    unsigned numBatch = GetNumBatch(SVM_BLOCK_TYPE_BYTE, numBlocks);

    bool isSLM = IsSLMSurface(surface);
    // SLM forbids header. Header is optional in A32 when both scale and global
    // offset are 0s.
    bool useHeader = !isSLM && needsA32MsgHeader();
    bool useSplitSend = useSends();

    // In case non-zero global offset is specified, we need to recalculate
    // offsets.
    //
    // NOTE: Even though pre-SKL devices require header, eliminating global
    //       offset by adjusting offsets will simplify the header generation.
    if (!globalOffset->isImm() || globalOffset->asImm()->getImm() != 0)
    {
        G4_Declare *dcl = createSendPayloadDcl(exSize, offsets->getType());
        G4_DstRegRegion *tmp = Create_Dst_Opnd_From_Dcl(dcl, 1);
        createBinOp(G4_add, instExSize, tmp, offsets, globalOffset, instOpt, true);
        offsets = Create_Src_Opnd_From_Dcl(dcl, getRegionStride1());
    }

    PayloadSource sources[3]; // Maximal 2 sources, optional header + offsets + src
    unsigned len = 0;

    if (useHeader) {
        G4_Declare *dcl = createSendPayloadDcl(GENX_DATAPORT_IO_SZ, Type_UD);

        // TODO: Get BTS supported on demand.
        BuildMH_A32_GO(this, dcl);

        G4_SrcRegRegion *header
            = Create_Src_Opnd_From_Dcl(dcl, getRegionStride1());
        sources[len].opnd = header;
        sources[len].execSize = g4::SIMD8;
        sources[len].instOpt = InstOpt_WriteEnable;
        ++len;
    }

    sources[len].opnd = offsets;
    sources[len].execSize = exSize;
    sources[len].instOpt = instOpt;
    ++len;
    sources[len].opnd = src;
    sources[len].execSize = G4_ExecSize(exSize * numBatch);
    sources[len].instOpt = instOpt;
    ++len;

    G4_SrcRegRegion *msgs[2] = {0, 0};
    unsigned sizes[2] = {0, 0};
    preparePayload(msgs, sizes, exSize, useSplitSend, sources, len);

    SFID sfid = SFID::DP_DC;

    unsigned MD = 0;
    // Leave sidebind scale offset 0 as it is not used now.
    MD |= DC_BYTE_SCATTERED_WRITE << 14;
    MD |= numBlocks << 10;
    MD |= (execSize == EXEC_SIZE_8 ? MDC_SM2_SIMD8 : MDC_SM2_SIMD16) << 8;

    G4_DstRegRegion *dst = createNullDst(Type_UD);
    bool forceSplitSend = shouldForceSplitSend(surface);
    if (msgs[1] == 0 && !forceSplitSend) {
        ASSERT_USER(sizes[1] == 0, "Expect the 2nd part of the payload has zero size!");
        Create_Send_Inst_For_CISA(pred, dst,
            msgs[0], sizes[0],
            0,
            instExSize,
            MD, sfid,
            useHeader,
            SendAccess::WRITE_ONLY,
            surface, NULL,
            instOpt, false);
    } else {
        Create_SplitSend_Inst_For_CISA(pred, dst,
            msgs[0], sizes[0], msgs[1], sizes[1],
            0,
            instExSize,
            MD, sfid,
            useHeader,
            SendAccess::WRITE_ONLY,
            surface, NULL,
            instOpt, false);
    }

    return VISA_SUCCESS;
}


///
/// Bits 31-29: Reserved
/// Bits 28-25: Message Length: Total 256bit registers expected to be sent.
/// Bits 24-20: Response Length: Total 256bit registers expected in response.
/// Bit  19:    Does this Message Descriptor have a header? 1 Yes, 0 No.
/// Bits 18-14: Message Type: 10100: A64 Block Read, 10101: A64 Block Write
/// Bit  13:    Ignore
/// Bits 12-11: Message sub-type (00 for OWord Block Read/Write, 01 for Unaligned OWord Block Read/Write)
/// Bits 10-8:  Block Size, 000 for 1 OWord, 001 for 2 OWords, 010 for 4 OWords, 100 for 8 OWords.
/// Bits 7-0:   Binding Table Index: Set to 0xFF for stateless memory space used bu A64 SVM Data Port.
int IR_Builder::translateVISASVMBlockReadInst(
    VISA_Oword_Num size,
    bool unaligned,
    G4_Operand* address,
    G4_DstRegRegion* dst)
{
    TIME_SCOPE(VISA_BUILDER_IR_CONSTRUCTION);

    unsigned numOword = Get_VISA_Oword_Num(size);
    G4_Declare* dcl = createSendPayloadDcl(GENX_DATAPORT_IO_SZ, Type_UD);
    if (noInt64())
    {
        G4_SrcRegRegion *region = address->asSrcRegRegion();
        G4_SrcRegRegion *tmp;
        tmp = createSrcRegRegion(Mod_src_undef,
            region->getRegAccess(),
            region->getBase(),
            region->getRegOff(),
            region->getSubRegOff() * 2,
            region->getRegion(), Type_UD);
        Create_MOV_Inst(dcl, 0, 0, g4::SIMD1, NULL, NULL, tmp, true);
        tmp = createSrcRegRegion(Mod_src_undef,
            region->getRegAccess(),
            region->getBase(),
            region->getRegOff(),
            region->getSubRegOff() * 2 + 1,
            region->getRegion(), Type_UD);
        Create_MOV_Inst(dcl, 0, 1, g4::SIMD1, NULL, NULL, tmp, true);
    }
    else
    {
        G4_Declare* dclAsUQ = createSendPayloadDcl(GENX_DATAPORT_IO_SZ / 2, Type_UQ);
        dclAsUQ->setAliasDeclare(dcl, 0);
        Create_MOV_Inst(dclAsUQ, 0, 0, g4::SIMD1, NULL, NULL, address, true);
    }

    G4_SrcRegRegion* src = Create_Src_Opnd_From_Dcl(dcl, getRegionStride1());

    DATA_CACHE1_MESSAGES msgSubOpcode = DC1_A64_BLOCK_READ;
    unsigned rspLength = ((numOword * 16 - 1) / getGRFSize() + 1);

    unsigned desc = getA64BTI() |
        (unaligned ? A64_BLOCK_MSG_OWORD_UNALIGNED_READ : A64_BLOCK_MSG_OWORD_RW) << A64_BLOCK_MSG_SUBTYPE_OFFSET |
        msgSubOpcode << SEND_GT_MSG_TYPE_BIT;

    desc = setOwordForDesc(desc, numOword);

    G4_ExecSize sendExecSize {FIX_OWORD_SEND_EXEC_SIZE(numOword)};
    dst->setType(Type_UD);

    Create_Send_Inst_For_CISA(
        NULL, dst, src, 1, rspLength, sendExecSize, desc,
        SFID::DP_DC1, true, SendAccess::READ_ONLY, NULL, NULL, InstOpt_WriteEnable, false);

    return VISA_SUCCESS;
}

int IR_Builder::translateVISASVMBlockWriteInst(
    VISA_Oword_Num size,
    G4_Operand* address,
    G4_SrcRegRegion* src)
{
    TIME_SCOPE(VISA_BUILDER_IR_CONSTRUCTION);

    unsigned numOword = Get_VISA_Oword_Num(size);
    unsigned srcNumGRF = (numOword * 16 + getGRFSize() - 1) / getGRFSize();
    G4_ExecSize sendExecSize {FIX_OWORD_SEND_EXEC_SIZE(numOword)};

    // FIXME: may want to apply this to FIX_OWORD_SEND_EXEC_SIZE instead
    if (sendExecSize < g4::SIMD8)
    {
        sendExecSize = g4::SIMD8;
    }

    G4_Declare* dcl = createSendPayloadDcl(GENX_DATAPORT_IO_SZ, Type_UD);
    if (noInt64())
    {
        G4_SrcRegRegion *region = address->asSrcRegRegion();
        G4_SrcRegRegion *tmp;
        tmp = createSrcRegRegion(Mod_src_undef,
            region->getRegAccess(),
            region->getBase(),
            region->getRegOff(),
            region->getSubRegOff() * 2,
            region->getRegion(), Type_UD);
        Create_MOV_Inst(dcl, 0, 0, g4::SIMD1, NULL, NULL, tmp, true);
        tmp = createSrcRegRegion(Mod_src_undef,
            region->getRegAccess(),
            region->getBase(),
            region->getRegOff(),
            region->getSubRegOff() * 2 + 1,
            region->getRegion(), Type_UD);
        Create_MOV_Inst(dcl, 0, 1, g4::SIMD1, NULL, NULL, tmp, true);
    } else {
        G4_Declare* dclAsUQ = createSendPayloadDcl(GENX_DATAPORT_IO_SZ / 2, Type_UQ);
        dclAsUQ->setAliasDeclare(dcl, 0);
        Create_MOV_Inst(dclAsUQ, 0, 0, g4::SIMD1, NULL, NULL, address, true);
    }

    bool useSplitSend = useSends();
    PayloadSource sources[2];
    unsigned len = 0;

    sources[len].opnd = Create_Src_Opnd_From_Dcl(dcl, getRegionStride1());
    sources[len].execSize = g4::SIMD8;
    sources[len].instOpt = InstOpt_WriteEnable;
    ++len;

    if (src->getElemSize() < getTypeSize(Type_UD))
    {
        // use D for size computation. Src is guaranteed to be GRF-aligend per vISA spec
        src->setType(Type_UD);
    }
    sources[len].opnd = src;

    G4_ExecSize movExecSize {0};

    auto scale = getGRFSize() / src->getElemSize();
    switch (src->getElemSize())
    {
    case 4:
        sources[len].execSize = G4_ExecSize(scale * srcNumGRF);
        movExecSize = G4_ExecSize(scale);
        break;
    case 8:
        sources[len].execSize = G4_ExecSize(scale * srcNumGRF);
        movExecSize = G4_ExecSize(scale);
        break;
    }

    sources[len].instOpt = InstOpt_WriteEnable;
    ++len;

    G4_SrcRegRegion *msgs[2] = {0, 0};
    unsigned sizes[2] = {0, 0};
    preparePayload(msgs, sizes, movExecSize, useSplitSend, sources, len);

    DATA_CACHE1_MESSAGES msgSubOpcode = DC1_A64_BLOCK_WRITE;

    unsigned desc = getA64BTI() |
        A64_BLOCK_MSG_OWORD_RW << A64_BLOCK_MSG_SUBTYPE_OFFSET |
        msgSubOpcode << SEND_GT_MSG_TYPE_BIT;

    desc = setOwordForDesc(desc, numOword);

    G4_DstRegRegion* sendDst = createNullDst(Type_UD);

    if (msgs[1] == 0)
    {
        Create_Send_Inst_For_CISA(NULL, sendDst,
            msgs[0], sizes[0],
            0, sendExecSize,
            desc, SFID::DP_DC1,
            true,
            SendAccess::WRITE_ONLY,
            NULL, NULL,
            InstOpt_WriteEnable, false);
    }
    else
    {
        Create_SplitSend_Inst_For_CISA(NULL, sendDst,
            msgs[0], sizes[0],
            msgs[1], sizes[1],
            0, sendExecSize,
            desc, SFID::DP_DC1,
            true,
            SendAccess::WRITE_ONLY,
            NULL, NULL,
            InstOpt_WriteEnable, false);
    }

    return VISA_SUCCESS;
}

int IR_Builder::translateVISASVMScatterReadInst(
    VISA_Exec_Size execSize,
    VISA_EMask_Ctrl eMask,
    G4_Predicate* pred,
    VISA_SVM_Block_Type blockSize,
    VISA_SVM_Block_Num numBlocks,
    G4_SrcRegRegion* addresses,
    G4_DstRegRegion* dst)
{
    TIME_SCOPE(VISA_BUILDER_IR_CONSTRUCTION);

    ASSERT_USER(execSize == EXEC_SIZE_1 || execSize == EXEC_SIZE_2 ||
        execSize == EXEC_SIZE_4 || execSize == EXEC_SIZE_8 ||
        execSize == EXEC_SIZE_16,
        "Only support SIMD1, SIMD2, SIMD4, SIMD8 or SIMD16!");

    VISA_Exec_Size instExecSize = execSize;
    execSize = roundUpExecSize(execSize);

    bool is8ByteMsg = blockSize == SVM_BLOCK_TYPE_BYTE && numBlocks == SVM_BLOCK_NUM_8;
    assert((!is8ByteMsg || has8ByteA64Gather()) && "A64 8-byte scatter not supported on this platform");

    G4_ExecSize exSize {Get_VISA_Exec_Size(execSize)};
    G4_ExecSize instExSize {Get_VISA_Exec_Size(instExecSize)};
    G4_InstOpts instOpt = Get_Gen4_Emask(eMask, instExSize);

    uint32_t messageLength = (8 * exSize) / getGRFSize();
    uint32_t numDWperLane = 0;

    // ToDo: remove this as it should be done in HWConformity
    if (instExSize < 8 && WaDisableSendSrcDstOverlap())
    {
        // as message length is set to 2 (HW requirements),
        // we have to even align both src/dst to satisfy the WA
        G4_Declare* srcDcl = addresses->getTopDcl()->getRootDeclare();
        if (srcDcl->getByteSize() <= numEltPerGRF(Type_UB))
        {
            srcDcl->setEvenAlign();
        }
        G4_Declare* dstDcl = dst->getTopDcl()->getRootDeclare();
        if (dstDcl->getByteSize() <= numEltPerGRF(Type_UB))
        {
            dstDcl->setEvenAlign();
        }
    }

    switch (blockSize)
    {
    case SVM_BLOCK_TYPE_BYTE:
        numDWperLane = (numBlocks == SVM_BLOCK_NUM_8) ? 2 : 1;
        break;
    case SVM_BLOCK_TYPE_DWORD:
        numDWperLane = Get_Common_ISA_SVM_Block_Num(numBlocks);
        break;
    case SVM_BLOCK_TYPE_QWORD:
        numDWperLane = Get_Common_ISA_SVM_Block_Num(numBlocks) * 2;
        break;
    default:
        MUST_BE_TRUE(false, "Illegal SVM block type");
    }
    uint32_t responseLength = (numDWperLane * 4 * exSize) / getGRFSize();

    unsigned desc = 0;
    desc |= getA64BTI();
    desc |= blockSize << 8;
    desc |= numBlocks << 10;
    desc |= (exSize == 8 ? 0 : 1) << 12;
    desc |= DC1_A64_SCATTERED_READ << 14;

    Create_Send_Inst_For_CISA(pred, dst, addresses, messageLength, responseLength, instExSize, desc,
        SFID::DP_DC1, false, SendAccess::READ_ONLY, NULL, NULL, instOpt, false);

    return VISA_SUCCESS;
}

int IR_Builder::translateVISASVMScatterWriteInst(
    VISA_Exec_Size execSize,
    VISA_EMask_Ctrl eMask,
    G4_Predicate* pred,
    VISA_SVM_Block_Type blockSize,
    VISA_SVM_Block_Num numBlocks,
    G4_SrcRegRegion* addresses,
    G4_SrcRegRegion* src)
{
    TIME_SCOPE(VISA_BUILDER_IR_CONSTRUCTION);

    ASSERT_USER(execSize == EXEC_SIZE_1 || execSize == EXEC_SIZE_2 ||
        execSize == EXEC_SIZE_4 || execSize == EXEC_SIZE_8 ||
        execSize == EXEC_SIZE_16,
        "Only support SIMD1, SIMD2, SIMD4, SIMD8 or SIMD16!");

    bool is8ByteMsg = blockSize == SVM_BLOCK_TYPE_BYTE && numBlocks == SVM_BLOCK_NUM_8;
    assert((!is8ByteMsg || has8ByteA64Gather()) && "A64 8-byte scatter not supported on this platform");
    VISA_Exec_Size instExecSize = execSize;
    execSize = roundUpExecSize(execSize);

    G4_ExecSize exSize {Get_VISA_Exec_Size(execSize)};
    G4_ExecSize instExSize {Get_VISA_Exec_Size(instExecSize)};
    G4_InstOpts instOpt = Get_Gen4_Emask(eMask, instExSize);

    bool useSplitSend = useSends();

    PayloadSource sources[2]; // Maximal 2 sources, optional header + offsets
    unsigned len = 0;

    sources[len].opnd = addresses;
    sources[len].execSize = exSize;
    sources[len].instOpt = instOpt;
    ++len;

    unsigned numElems = 1;
    // NOTE that BYTE scatter always has numElems set to 1 as
    // - when the number of data elements is 1, 2, or 4, the writeback payload
    //   is always 1 MDP_DW_SIMD8/_SIMD16.
    // - when the number of data elements is 8, the write payload is always 1
    //   MDP_QW_SIMD8/_SIMD16.
    // This ALSO implies the RAW operand should be in type of UQ when the
    // number of data elements is 8.
    if (blockSize != SVM_BLOCK_TYPE_BYTE)
        numElems = Get_Common_ISA_SVM_Block_Num(numBlocks);

    sources[len].opnd = src;
    sources[len].execSize = G4_ExecSize(exSize * numElems);
    sources[len].instOpt = instOpt;
    ++len;

    G4_SrcRegRegion *msgs[2] = {0, 0};
    unsigned sizes[2] = {0, 0};

    // adjust src type
    // PreparePayload takes src type to calculate src1 size. The src type have to be DW
    // for byte scatter read
    G4_Type srcType = src->getType();
    if ((blockSize == SVM_BLOCK_TYPE_BYTE) &&
        (numBlocks == SVM_BLOCK_NUM_1 || numBlocks == SVM_BLOCK_NUM_2) &&
        (G4_Type_Table[srcType].byteSize != 4))
        src->setType(Type_UD);

    preparePayload(msgs, sizes, exSize, useSplitSend, sources, len);

    // set the type back in case we changed it for preparePayload
    src->setType(srcType);

    unsigned desc = 0;
    desc |= getA64BTI();
    desc |= blockSize << 8;
    desc |= numBlocks << 10;
    desc |= (exSize == 8 ? 0 : 1) << 12;
    desc |= DC1_A64_SCATTERED_WRITE << 14;

    G4_DstRegRegion* dst = createNullDst(Type_UD);
    if (msgs[1] == 0) {
        ASSERT_USER(sizes[1] == 0, "Expect the 2nd part of the payload has zero size!");
        Create_Send_Inst_For_CISA(pred, dst,
            msgs[0], sizes[0],
            0, instExSize,
            desc, SFID::DP_DC1,
            false,
            SendAccess::WRITE_ONLY,
            NULL, NULL,
            instOpt, false);
    }
    else {
        Create_SplitSend_Inst_For_CISA(pred, dst,
            msgs[0], sizes[0],
            msgs[1], sizes[1],
            0, instExSize,
            desc, SFID::DP_DC1,
            false,
            SendAccess::WRITE_ONLY,
            NULL, NULL,
            instOpt, false);
    }

    return VISA_SUCCESS;
}




// is16Bit indicates if this is a 16bit atomic op. The input source (if
// any) and the writeback (if any) have the same datalayout as dword messages.
// Only the lower 16 bits of each dword is used.
//
static void FillSVMAtomicMsgDesc(bool is16Bit, bool isFloatOp, uint32_t &msgDesc)
{
    if (is16Bit)
    {
        if (isFloatOp)
        {
            msgDesc |= DC1_A64_UNTYPED_HALF_FLOAT_ATOMIC << 14;
        }
        else
        {
            msgDesc |= DC1_A64_UNTYPED_HALF_INTEGER_ATOMIC << 14;
        }
    }
    else
    {
        if (isFloatOp)
        {
            msgDesc |= DC1_A64_UNTYPED_FLOAT_ATOMIC << 14;
        }
        else
        {
            msgDesc |= DC1_A64_ATOMIC << 14;
        }
    }
}


int IR_Builder::translateVISASVMAtomicInst(
    VISAAtomicOps atomicOp,
    unsigned short bitwidth,
    VISA_Exec_Size execSize,
    VISA_EMask_Ctrl emask,
    G4_Predicate* pred,
    G4_SrcRegRegion* addresses,
    G4_SrcRegRegion* src0,
    G4_SrcRegRegion* src1,
    G4_DstRegRegion* dst)
{
    TIME_SCOPE(VISA_BUILDER_IR_CONSTRUCTION);

    MUST_BE_TRUE(bitwidth == 16 || bitwidth == 32 || bitwidth == 64,
        "bitwidth must be 16/32/64");


    VISA_Exec_Size instExecSize = execSize;
    execSize = roundUpExecSize(execSize);

    unsigned op = Get_Atomic_Op(atomicOp);

    G4_ExecSize exSize {Get_VISA_Exec_Size(execSize)};
    G4_ExecSize instExSize {Get_VISA_Exec_Size(instExecSize)};
    G4_InstOpts instOpt = Get_Gen4_Emask(emask, instExSize);

    if (atomicOp == ATOMIC_CMPXCHG)
    {
        // we have to swap src0 and src1 since vISA has them in different order from HW
        G4_SrcRegRegion* tmp = src0;
        src0 = src1;
        src1 = tmp;
    }

    bool useSplitSend = useSends();

    PayloadSource sources[3]; // addresses, src0, and src1
    unsigned len = 0;

    sources[len].opnd = addresses;
    sources[len].execSize = exSize;
    sources[len].instOpt = instOpt;
    ++len;

    if (src0 != NULL && !src0->isNullReg())
    {
        sources[len].opnd = src0;
        sources[len].execSize = exSize;
        sources[len].instOpt = instOpt;
        ++len;
    }

    if (src1 != NULL && !src1->isNullReg())
    {
        sources[len].opnd = src1;
        sources[len].execSize = exSize;
        sources[len].instOpt = instOpt;
        ++len;
    }

    G4_SrcRegRegion *msgs[2] = {0, 0};
    unsigned sizes[2] = {0, 0};
    preparePayload(msgs, sizes, exSize, useSplitSend, sources, len);
    unsigned dstLength = dst->isNullReg() ? 0 : ((bitwidth == 16 || bitwidth == 32) ? 1 : 2);
    unsigned msgDesc = 0;
    msgDesc |= getA64BTI();
    msgDesc |= op << 8;
#define A64_ATOMIC_RETURN_DATA_CONTROL_BIT 13
    msgDesc |= (dstLength ? 1 : 0) << A64_ATOMIC_RETURN_DATA_CONTROL_BIT;
    msgDesc |= ((bitwidth == 16 || bitwidth == 32) ? 0 : 1) << 12;

    // Fill remaining bits.
    FillSVMAtomicMsgDesc(bitwidth == 16, IsFloatAtomicOps(atomicOp), msgDesc);

    if (msgs[1] == 0) {
        Create_Send_Inst_For_CISA(pred, dst,
            msgs[0], sizes[0], dstLength,
            instExSize,
            msgDesc, SFID::DP_DC1,
            false,
            SendAccess::READ_WRITE,
            NULL, NULL,
            instOpt, false);
    }
    else {
        Create_SplitSend_Inst_For_CISA(pred, dst,
            msgs[0], sizes[0],
            msgs[1], sizes[1],
            dstLength,
            instExSize,
            msgDesc, SFID::DP_DC1,
            false,
            SendAccess::READ_WRITE,
            NULL, NULL,
            instOpt, false);
    }

    return VISA_SUCCESS;
}

G4_SrcRegRegion* IR_Builder::getSVMOffset(
    G4_Operand* globalOffset, G4_SrcRegRegion* offsets, uint16_t exSize,
    G4_Predicate* pred, uint32_t mask)
{
    G4_Declare* dcl = createSendPayloadDcl(exSize, offsets->getType());
    G4_DstRegRegion* tmp = Create_Dst_Opnd_From_Dcl(dcl, 1);
    createInst(pred, G4_add, 0, g4::NOSAT, g4::SIMD8, tmp, offsets, globalOffset, mask, true);
    if (exSize == 16)
    {
        // do second half of the 64-bit add
        int offset = (8 * sizeof(uint64_t)) / getGRFSize();
        auto dst = createDst(dcl->getRegVar(), offset, 0, 1, offsets->getType());
        auto src = createSrcRegRegion(Mod_src_undef, Direct, offsets->getBase(),
            offsets->getRegOff() + offset, offsets->getSubRegOff(), getRegionStride1(), offsets->getType());
        createInst(duplicateOperand(pred), G4_add, 0, g4::NOSAT, g4::SIMD8, dst, src,
            duplicateOperand(globalOffset), getSplitHiEMask(16, mask), true);
    }
    return Create_Src_Opnd_From_Dcl(dcl, getRegionStride1());
}

int IR_Builder::translateSVMGather4Inst(
    VISA_Exec_Size          execSize,
    VISA_EMask_Ctrl         eMask,
    ChannelMask             chMask,
    G4_Predicate            *pred,
    G4_Operand              *globalOffset,
    G4_SrcRegRegion         *offsets,
    G4_DstRegRegion         *dst)
{
    TIME_SCOPE(VISA_BUILDER_IR_CONSTRUCTION);

    ASSERT_USER(execSize == EXEC_SIZE_8 || execSize == EXEC_SIZE_16,
        "Only support SIMD8 or SIMD16!");

    G4_ExecSize exSize {Get_VISA_Exec_Size(execSize)};
    G4_InstOpts instOpt = Get_Gen4_Emask(eMask, exSize);

    bool useSplitSend = useSends();

    // In case non-zero global offset is specified, we need to recalculate
    // offsets.
    if (!globalOffset->isImm() || globalOffset->asImm()->getImm() != 0)
    {
        offsets = getSVMOffset(globalOffset, offsets, exSize, pred, instOpt);
    }

    PayloadSource sources[1]; // Maximal 1 sources, offsets
    unsigned len = 0;

    sources[len].opnd = offsets;
    sources[len].execSize = exSize;
    sources[len].instOpt = instOpt;
    ++len;

    G4_SrcRegRegion *msgs[2] = {0, 0};
    unsigned sizes[2] = {0, 0};
    preparePayload(msgs, sizes, exSize, useSplitSend, sources, len);

    SFID sfid = SFID::DP_DC1;

    unsigned FC = 0;
    // Leave sidebind scaled offset 0 as it is not used now.
    FC |= DC1_A64_UNTYPED_SURFACE_READ << 14;
    FC |= (execSize == EXEC_SIZE_8 ? MDC_SM3_SIMD8 : MDC_SM3_SIMD16) << 12;
    FC |= chMask.getHWEncoding() << 8;
    FC |= getA64BTI();

    unsigned resLen = (exSize / GENX_DATAPORT_IO_SZ) *
        chMask.getNumEnabledChannels();
    if (msgs[1] == 0) {
        ASSERT_USER(sizes[1] == 0, "Expect the 2nd part of the payload has zero size!");
        Create_Send_Inst_For_CISA(pred, dst,
            msgs[0], sizes[0],
            resLen,
            exSize,
            FC, sfid,
            false,
            SendAccess::READ_ONLY,
            NULL, NULL,
            instOpt, false);
    }
    else {
        Create_SplitSend_Inst_For_CISA(pred, dst,
            msgs[0], sizes[0], msgs[1], sizes[1],
            resLen,
            exSize,
            FC, sfid,
            false,
            SendAccess::READ_ONLY,
            NULL, NULL,
            instOpt, false);
    }

    return VISA_SUCCESS;
}

int IR_Builder::translateSVMScatter4Inst(
    VISA_Exec_Size         execSize,
    VISA_EMask_Ctrl        eMask,
    ChannelMask            chMask,
    G4_Predicate           *pred,
    G4_Operand             *globalOffset,
    G4_SrcRegRegion        *offsets,
    G4_SrcRegRegion        *src)
{
    TIME_SCOPE(VISA_BUILDER_IR_CONSTRUCTION);

    ASSERT_USER(execSize == EXEC_SIZE_8 || execSize == EXEC_SIZE_16,
        "Only support SIMD8 or SIMD16!");

    G4_ExecSize exSize {Get_VISA_Exec_Size(execSize)};
    G4_InstOpts instOpt = Get_Gen4_Emask(eMask, exSize);
    bool useSplitSend = useSends();

    // In case non-zero global offset is specified, we need to recalculate
    // offsets.
    if (!globalOffset->isImm() || globalOffset->asImm()->getImm() != 0)
    {
        offsets = getSVMOffset(globalOffset, offsets, exSize, pred, instOpt);
    }

    PayloadSource sources[2]; // Maximal 2 sources, offsets + src
    unsigned len = 0;

    sources[len].opnd = offsets;
    sources[len].execSize = exSize;
    sources[len].instOpt = instOpt;
    ++len;
    sources[len].opnd = src;
    sources[len].execSize = G4_ExecSize(exSize * chMask.getNumEnabledChannels());
    sources[len].instOpt = instOpt;
    ++len;

    G4_SrcRegRegion *msgs[2] = {0, 0};
    unsigned sizes[2] = {0, 0};
    preparePayload(msgs, sizes, exSize, useSplitSend, sources, len);

    SFID sfid = SFID::DP_DC1;

    unsigned FC = 0;
    // Leave sidebind scaled offset 0 as it is not used now.
    FC |= DC1_A64_UNTYPED_SURFACE_WRITE << 14;
    FC |= (execSize == EXEC_SIZE_8 ? MDC_SM3_SIMD8 : MDC_SM3_SIMD16) << 12;
    FC |= chMask.getHWEncoding() << 8;
    FC |= getA64BTI();

    G4_DstRegRegion *dst = createNullDst(Type_UD);
    if (msgs[1] == 0) {
        ASSERT_USER(sizes[1] == 0, "Expect the 2nd part of the payload has zero size!");
        Create_Send_Inst_For_CISA(pred, dst,
            msgs[0], sizes[0],
            0,
            exSize,
            FC, sfid,
            false,
            SendAccess::WRITE_ONLY,
            NULL, NULL,
            instOpt, false);
    }
    else {
        Create_SplitSend_Inst_For_CISA(pred, dst,
            msgs[0], sizes[0], msgs[1], sizes[1],
            0,
            exSize,
            FC, sfid,
            false,
            SendAccess::WRITE_ONLY,
            NULL, NULL,
            instOpt, false);
    }

    return VISA_SUCCESS;
}

int IR_Builder::translateVISASVMGather4ScaledInst(
    VISA_Exec_Size            execSize,
    VISA_EMask_Ctrl           eMask,
    ChannelMask               chMask,
    G4_Predicate              *pred,
    G4_Operand                *globalOffset,
    G4_SrcRegRegion           *offsets,
    G4_DstRegRegion           *dst)
{
    return translateSVMGather4Inst(execSize, eMask, chMask, pred,
        globalOffset, offsets, dst);
}

int IR_Builder::translateVISASVMScatter4ScaledInst(
    VISA_Exec_Size           execSize,
    VISA_EMask_Ctrl          eMask,
    ChannelMask              chMask,
    G4_Predicate             *pred,
    G4_Operand               *globalOffset,
    G4_SrcRegRegion          *offsets,
    G4_SrcRegRegion          *src)
{
    return translateSVMScatter4Inst(execSize, eMask, chMask, pred,
        globalOffset, offsets, src);
}

