/*===================== begin_copyright_notice ==================================

Copyright (c) 2017 Intel Corporation

Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:

The above copyright notice and this permission notice shall be included
in all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.


======================= end_copyright_notice ==================================*/
//
/// GenXLowering
/// ------------
///
/// GenXLowering is a function pass that lowers certain LLVM IR instructions
/// that the rest of the GenX backend cannot deal with, or to implement peephole
/// optimizations.
///
/// It also performs a few other tasks:
///
/// 1. It implements add sinking for a variable index in a region/element
///    access. This ensures that, in a sequence of operations to calculate a
///    variable index for a region/element access, any add constant is sunk to
///    the end, such that it can become a constant offset in an indirect
///    operand, and give GenXAddressCommoning more chance to common up address
///    calculations.
///
/// 2. It splits struct values where possible, by splitting all struct phi nodes
///    before running the main pass, then removing an extractvalue by using the
///    corresponding insertvalue's input instead. Any struct value used as an
///    arg or return value still remains, and needs to be dealt with by register
///    allocation.
///
/// 3. It widens some byte vector operations to short vector.
///
///    Gen has restrictions on byte operands. The jitter copes with that, but
///    sometimes it needs to do even-odd splitting, which can lead to suboptimal
///    code if cmps and predicates are involved.
///    Here we attempt to pick up the common cases by converting a byte
///    operation to short.
///
///    Note that we might end up with the extends being baled into the
///    instruction anyway, resulting in a byte operation in vISA.
///
/// 4. Certain uses of shufflevector are lowered:
///
///    a. a splat (copy of one element across a vector);
///    b. a boolean slice (extract of a subvector) becomes rdpredregion;
///    c. a boolean unslice (insert subvector) becomes wrpredregion.
///    d. non-boolean shufflevector is lowered to sequence of rd/wrregions
///
///    The only one case of shufflevector allowed is shufflevector of
///    predicate and undef with replicated mask.
///
/// 5. A Trunc is lowered to a bitcast then a region/element read with a stride.
///    GenXCoalescing will coalesce the bitcast, and possibly bale in the region
///    read, so this will hopefully save an instruction or two.
///
/// 6. Certain floating point comparison instructions are lowered.
///
/// **IR restriction**: LLVM IR instructions not supported after this pass:
///
/// * insertelement
/// * extractelement
/// * trunc
/// * zext/sext/uitofp from (vector of) i1
/// * select on vector of i1
/// * ``llvm.uadd.with.overflow`` (the other
///   overflowing arithmetic intrinsics are not allowed by the GenX backend
///   anyway.)
///
///
/// **IR restriction**: all gather/scatter/atomic must have the width supported
/// by the hardware target.
///
/// **IR restriction**: rdpredregion intrinsic (which is generated by this pass
/// from certain cases of shufflevector, and represents a use of part of a
/// predicate) can only be used in select, wrregion, wrpredpredregion.
///
/// **IR restriction**: wrpredregion intrinsic (which is generated by this pass
/// from certain cases of shufflevector, and represents the write of part of a
/// predicate) must have a compare as its "new value" input.
///
/// **IR restriction**: No phi node of struct type after this pass. This is only
/// a general rule; subsequent passes have been known to reintroduce them so
/// GenXLiveness has another go at splitting them up.
///
//===----------------------------------------------------------------------===//

#include "IGC/common/debug/DebugMacros.hpp"
#include "GenX.h"
#include "GenXGotoJoin.h"
#include "GenXIntrinsics.h"
#include "GenXModule.h"
#include "GenXRegion.h"
#include "GenXSubtarget.h"
#include "GenXTargetMachine.h"
#include "GenXUtil.h"
#include "GenXVisa.h"
#include "visa_igc_common_header.h"
#include "llvm/ADT/PostOrderIterator.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/Analysis/CFG.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/CodeGen/TargetPassConfig.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/Module.h"
#include "llvm/Pass.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"

#include "llvmWrapper/IR/DerivedTypes.h"
#include "llvmWrapper/IR/Instructions.h"
#include "llvmWrapper/Support/TypeSize.h"

#include <algorithm>
#include <iterator>
#include <numeric>
#include "Probe/Assertion.h"

using namespace llvm;
using namespace genx;

static cl::opt<bool>
    EnableGenXByteWidening("enable-genx-byte-widening", cl::init(true),
                           cl::Hidden, cl::desc("Enable GenX byte widening."));
namespace {

// GenXLowering : legalize execution widths and GRF crossing
class GenXLowering : public FunctionPass {
  DominatorTree *DT = nullptr;
  const GenXSubtarget *ST = nullptr;
  SmallVector<Instruction *, 8> ToErase;
  const DataLayout *DL = nullptr;

public:
  static char ID;
  explicit GenXLowering() : FunctionPass(ID), DT(nullptr) {}
  virtual StringRef getPassName() const { return "GenX lowering"; }
  void getAnalysisUsage(AnalysisUsage &AU) const;
  bool runOnFunction(Function &F);
  static bool splitStructPhi(PHINode *Phi);

private:
  bool splitGatherScatter(CallInst *CI, unsigned IID);
  bool processTwoAddressOpnd(CallInst *CI);
  bool processInst(Instruction *Inst);
  bool lowerRdRegion(Instruction *Inst);
  bool lowerWrRegion(Instruction *Inst);
  bool lowerRdPredRegion(Instruction *Inst);
  bool lowerWrPredRegion(Instruction *Inst);
  bool lowerInsertElement(Instruction *Inst);
  bool lowerExtractElement(Instruction *Inst);
  Value *scaleInsertExtractElementIndex(Value *IdxVal, Type *ElTy,
                                        Instruction *InsertBefore);
  bool lowerTrunc(Instruction *Inst);
  bool lowerCast(Instruction *Inst);
  bool lowerBoolScalarSelect(SelectInst *SI);
  bool lowerBoolVectorSelect(SelectInst *SI);
  bool lowerBoolShuffle(ShuffleVectorInst *Inst);
  bool lowerBoolSplat(ShuffleVectorInst *SI, Value *In, unsigned Idx);
  bool lowerSelect(SelectInst* SI);
  bool lowerShuffle(ShuffleVectorInst *Inst);
  void lowerShuffleSplat(ShuffleVectorInst *SI,
                         ShuffleVectorAnalyzer::SplatInfo Splat);
  bool lowerShuffleToSelect(ShuffleVectorInst *Inst);
  void lowerShuffleToMove(ShuffleVectorInst *SI);
  bool lowerShr(Instruction *Inst);
  bool lowerExtractValue(ExtractValueInst *Inst);
  bool lowerInsertValue(InsertValueInst *Inst);
  bool lowerUAddWithOverflow(CallInst *CI);
  bool lowerCtpop(CallInst *CI);
  bool lowerFCmpInst(FCmpInst *Inst);
  bool lowerUnorderedFCmpInst(FCmpInst *Inst);
  bool widenByteOp(Instruction *Inst);
  bool lowerLoadStore(Instruction *Inst);
  bool lowerMul64(Instruction *Inst);
  bool lowerTrap(CallInst *CI);
  bool generatePredicatedWrrForNewLoad(CallInst *CI);
};

} // end namespace

char GenXLowering::ID = 0;
namespace llvm {
void initializeGenXLoweringPass(PassRegistry &);
}
INITIALIZE_PASS_BEGIN(GenXLowering, "GenXLowering", "GenXLowering", false,
                      false)
INITIALIZE_PASS_END(GenXLowering, "GenXLowering", "GenXLowering", false, false)

FunctionPass *llvm::createGenXLoweringPass() {
  initializeGenXLoweringPass(*PassRegistry::getPassRegistry());
  return new GenXLowering;
}

void GenXLowering::getAnalysisUsage(AnalysisUsage &AU) const {
  AU.addRequired<TargetPassConfig>();
  AU.addPreserved<DominatorTreeWrapperPass>();
  AU.addPreserved<LoopInfoWrapperPass>();
  AU.addPreserved<GenXModule>();
}

/***********************************************************************
 * GenXLowering::runOnFunction : process one function to
 *    lower instructions as required for GenX backend.
 *
 * This does a postordered depth first traversal of the CFG,
 * processing instructions within a basic block in reverse, to
 * ensure that we see a def after its uses (ignoring phi node uses).
 * This helps peephole optimizations which generally want to be
 * approached from the top down. For example, add sinking in the index
 * of an indirect region/element wants to see the trunc before the trunc
 * is lowered to a bitcast and an element access.
 */
bool GenXLowering::runOnFunction(Function &F) {
  auto *DTWP = getAnalysisIfAvailable<DominatorTreeWrapperPass>();
  DT = DTWP ? &DTWP->getDomTree() : nullptr;
  ST = &getAnalysis<TargetPassConfig>()
            .getTM<GenXTargetMachine>()
            .getGenXSubtarget();
  DL = &F.getParent()->getDataLayout();
  // First split any phi nodes with struct type.
  splitStructPhis(&F);
  // Create a list of basic blocks in the order we want to process them, before
  // we start the lowering. This is because lowering can split a basic block.
  SmallVector<BasicBlock *, 8> BBs;
  for (auto i = po_begin(&F.getEntryBlock()), e = po_end(&F.getEntryBlock());
       i != e; ++i)
    BBs.push_back(*i);
  // Process each basic block.
  for (auto i = BBs.begin(), e = BBs.end(); i != e; ++i) {
    BasicBlock *BB = *i;
    // The effect of this loop is that we process the instructions in reverse
    // order, and we re-process anything inserted before the instruction
    // being processed.
    for (Instruction *Inst = BB->getTerminator();;) {
      processInst(Inst);
      BasicBlock *Parent = Inst->getParent();
      if (Inst != &Parent->front())
        Inst = Inst->getPrevNode();
      else {
        if (Parent == BB)
          break;
        // We have reached the start of the basic block, but it is a different
        // basic block to BB, so lowering must have split a BB. Just go back to
        // the end of the previous one.
        Inst = Parent->getPrevNode()->getTerminator();
      }
    }
  }
  // Erase the instructions that we saved in ToErase.
  for (SmallVectorImpl<Instruction *>::iterator i = ToErase.begin(),
                                                e = ToErase.end();
       i != e; ++i)
    (*i)->eraseFromParent();
  ToErase.clear();
  return true;
}

// Optimize two address operands if any.
//
// An instruction with a two address opernd should be predicated. If predicate
// is a constant splat, then the old value will be over-written. In this case,
// replace the old value with undef which allows more optimizations to kick in.
//
bool GenXLowering::processTwoAddressOpnd(CallInst *CI) {
  int OpNum = getTwoAddressOperandNum(CI);
  // Skip write regions whose OpNum is 0.
  if (OpNum > 0) {
    Type *Ty = CI->getArgOperand(OpNum)->getType();
    IGC_ASSERT_MESSAGE(Ty == CI->getType(), "two address op type out of sync");

    for (unsigned i = 0; i < CI->getNumArgOperands(); ++i) {
      auto Op = dyn_cast<Constant>(CI->getArgOperand(i));
      // Check if the predicate operand is all true.
      if (Op && Op->getType()->getScalarSizeInBits() == 1) {
        if (Op->getType()->isVectorTy())
          Op = Op->getSplatValue();
        if (Op && Op->isOneValue()) {
          CI->setOperand(OpNum, UndefValue::get(Ty));
          return true;
        }
        return false;
      }
    }
  }

  return false;
}

// Check whether given intrinsic is new load
// without predicate and old value arguments.
static bool isNewLoadInst(CallInst *Inst) {
  unsigned IID = GenXIntrinsic::getGenXIntrinsicID(Inst);
  switch (IID) {
  case GenXIntrinsic::genx_gather4_scaled2:
  case GenXIntrinsic::genx_gather_scaled2:
    return true;
  default:
    return false;
  }
}

// Find single wrregion user of load instruction.
// Returns nullptr on failure.
static CallInst *getLoadWrregion(CallInst *Inst) {
  IGC_ASSERT_MESSAGE(isNewLoadInst(Inst), "Expected new load intrinsics");
  if (Inst->getNumUses() != 1)
    return nullptr;

  auto *WrR = dyn_cast<CallInst>(Inst->user_back());
  if (!WrR)
    return nullptr;
  return GenXIntrinsic::isWrRegion(WrR) ? WrR : nullptr;
}

// Find single select user of load instruction.
// Returns nullptr on failure.
// TODO: maybe just lower every select to wrregion in lowerSelect?
static SelectInst *getLoadSelect(CallInst *Inst) {
  IGC_ASSERT_MESSAGE(isNewLoadInst(Inst), "Expected new load intrinsics");
  if (Inst->getNumUses() != 1)
    return nullptr;

  auto *SI = dyn_cast<SelectInst>(Inst->user_back());
  if (!SI)
    return nullptr;
  // TODO: handle inverted selects.
  // Need to regenerate mask in this case.
  if (SI->getTrueValue() != Inst)
    return nullptr;
  return SI;
}

// Generate predicate for wrregion of splitted load.
// Returns new predicate.
static Value *generatePredicateForLoadWrregion(
    Value *OldPred, unsigned Offset, unsigned Width, unsigned NumChannels,
    Instruction *InsertBefore, const DebugLoc &DL, const Twine &Name) {
  if (isa<ConstantInt>(OldPred))
    return OldPred;

  Value *Pred = OldPred;
  // If old predicate is result of rdpredregion or shufflevector then
  // we can reuse their predicate and offset to avoid double read of predicate.
  if (GenXIntrinsic::getGenXIntrinsicID(OldPred) == GenXIntrinsic::genx_rdpredregion) {
    auto *OldPredInst = cast<CallInst>(OldPred);
    Offset += cast<ConstantInt>(OldPredInst->getArgOperand(1))->getZExtValue();
    Pred = OldPredInst->getArgOperand(0);
  } else if (auto *SVI = dyn_cast<ShuffleVectorInst>(OldPred)) {
    Offset +=
        ShuffleVectorAnalyzer::getReplicatedSliceDescriptor(SVI).InitialOffset;
    Pred = SVI->getOperand(0);
  }

  // Replicate mask across channels.
  SmallVector<Constant *, 16> NewMaskVals(Width);
  IGC_ASSERT(NumChannels);
  unsigned ChannelWidth = Width / NumChannels;
  Type *Int32Ty = IntegerType::getInt32Ty(Pred->getContext());
  for (unsigned i = 0; i < NumChannels; ++i)
    std::generate_n(NewMaskVals.begin() + ChannelWidth * i, ChannelWidth,
                    [Int32Ty, Offset]() mutable {
                      return ConstantInt::get(Int32Ty, Offset++);
                    });
  Constant *NewMask = ConstantVector::get(NewMaskVals);

  Value *Undef = UndefValue::get(Pred->getType());
  auto *Res = new ShuffleVectorInst(Pred, Undef, NewMask, Name, InsertBefore);
  Res->setDebugLoc(DL);
  return Res;
}

static Value *generatePredicatedWrregion(Value *OldVal, Value *NewVal,
                                         Value *Pred, unsigned Offset,
                                         Instruction *InsertBefore,
                                         const llvm::Twine &Name = "") {
  Type *NewValType = NewVal->getType();

  Region WrR(NewValType);
  WrR.Mask = Pred;
  WrR.Offset = Offset;
  return WrR.createWrRegion(OldVal, NewVal, Name, InsertBefore,
                            InsertBefore->getDebugLoc());
}


// Generate partial write for result of splitted 1-channel load instruction.
// Initially we could have following sequence for illegal load (on gather_scaled example):
//   res = gather_scaled <32>
//   mask = rdpredregion <32> pred, offset
//   newV = wrregion <32> oldV, res, wroffset, mask
// After splitting we want to get as less extra code as possible.
// To achieve this we generate following pattern:
// bale {
//   res1 = gather_scaled <16>
//   mask1 = rdpredregion <16> pred, offset
//   partialV = wrregion <16> oldV, res1, mask1
// }
// bale {
//   res2 = gather_scaled <16>
//   mask2 = rdpredregion <16> pred, offset + 16
//   newV = wrregion <16> partialV, res2, wroffset + 16 * elemsize, mask2
// }
// Bale markers show how this will be baled later.
static Value *generate1ChannelWrrregion(Value *Target, unsigned InitialOffset,
                                        CallInst *Load, Value *OldPred,
                                        unsigned SplitNum,
                                        Instruction *InsertBefore) {
  const DebugLoc &DL = Load->getDebugLoc();
  Type *LoadType = Load->getType();
  unsigned LoadWidth = cast<VectorType>(LoadType)->getNumElements();

  Value *Pred =
      generatePredicateForLoadWrregion(OldPred, LoadWidth * SplitNum, LoadWidth,
                                       1, InsertBefore, DL, "load1.pred.split");
  Region WrR(LoadType);
  WrR.Mask = Pred;
  WrR.Offset = InitialOffset +
               LoadWidth * SplitNum * (LoadType->getScalarSizeInBits() / 8);
  return WrR.createWrRegion(Target, Load, "load1.join", InsertBefore, DL);
}

// Generate partial write for result of splitted N-channel load.
// For channelled loads we need to also shuffle result of splitted
// instructions to write back them to destination in expected order.
// Temporary splits should always be predicated in case of atomics
// because latter load and store at the same time.
// Example for gather4_scaled (with two channels enabled). Before:
//   res = gather4_scaled <32> RG
//   mask = rdpredregion <64> pred, offset ; mask is replicated across channels
//   newV = wrregion <64> oldV, res, wroffset, mask
// After:
// bale {
//   res1temp = gather4_scaled <16> RG ; create temporary (unnecessary in case of non-atomics)
//   splitmask1 = rdpredregion <32> pred, offset ; replicated
//   res1 = wrregion <32> undef, res1temp, 0, splitmask1
// }
// bale {
//   res1R = rdregion <16> res1, 0
//   mask1R = rdpredregion <16> pred, offset ; same for all channels
//   partialVR = wrregion <16> oldV, res1R, wroffset, mask1R
// }
// bale {
//   res1G = rdregion <16> res1, 16 * elemsize
//   mask1G = rdpredregion <16> pred, offset
//   partialV = wrregion <16> partialVR, res1G, wroffset + 32 * elemsize, mask1G
// }
// bale {
//   res2temp = gather4_scaled <16> RG ; second temporary
//   splitmask2 = rdpredregion <32> pred, offset + 16
//   res2 = wrregion <32> undef, res2temp, 0, splitmask2
// }
// bale {
//   res2R = rdregion <16> res2, 0
//   mask2R = rdpredregion <16> pred, offset + 16
//   newVR = wrregion <16> partialV, res2R, wroffset + 16 * elemsize, mask2R
// }
// bale {
//   res2G = rdregion <16> res2, 16 * elemsize
//   mask2G = rdpredregion <16> pred, offset + 16
//   newV = wrregion <16> newVR, res2G, wroffset + 48 * elemsize, mask2G
// }
// As it can be noticed, splitting of channeled loads is quite expensive.
// We should hope that later passes (like region collapsing) can optimize it
// by analyzing how resulting value was assembled.
static Value *generateNChannelWrregion(Value *Target, unsigned InitialOffset,
                                       CallInst *Load, Value *OldPred,
                                       unsigned SplitNum, unsigned NumSplits,
                                       unsigned NumChannels,
                                       Instruction *InsertBefore) {
  const DebugLoc &DL = Load->getDebugLoc();
  Type *LoadType = Load->getType();
  unsigned LoadWidth = cast<VectorType>(LoadType)->getNumElements();
  IGC_ASSERT(NumChannels);
  unsigned ChannelWidth = LoadWidth / NumChannels;
  unsigned MaskOffset = ChannelWidth * SplitNum;

  // Generate temporary for load.
  Value *Pred = generatePredicateForLoadWrregion(
      OldPred, MaskOffset, LoadWidth, NumChannels, InsertBefore, DL, "loadN.pred.split");
  Region WrR(LoadType);
  WrR.Mask = Pred;
  Value *SplitRes = WrR.createWrRegion(UndefValue::get(LoadType), Load,
                                       "loadN.split", InsertBefore, DL);

  // Generate shuffle writes to the target.
  unsigned ElemByteSize = LoadType->getScalarSizeInBits() / 8;
  Type *ShuffleType =
      IGCLLVM::FixedVectorType::get(LoadType->getScalarType(), ChannelWidth);
  Region ChannelRdR(ShuffleType);
  Region ChannelWrR(ShuffleType);
  Value *ResChannel = nullptr;
  for (unsigned i = 0; i < NumChannels; ++i) {
    ChannelRdR.Offset = ChannelWidth * i * ElemByteSize;
    ResChannel = ChannelRdR.createRdRegion(SplitRes, "loadN.channel.read.join",
                                           InsertBefore, DL);
    Pred = generatePredicateForLoadWrregion(OldPred, MaskOffset, ChannelWidth,
                                            1, InsertBefore, DL,
                                            "loadN.channel.pred.join");
    ChannelWrR.Offset =
        InitialOffset +
        (ChannelWidth * SplitNum + ChannelWidth * NumSplits * i) * ElemByteSize;
    ChannelWrR.Mask = Pred;
    Target = ChannelWrR.createWrRegion(Target, ResChannel, "loadN.channel.join",
                                       InsertBefore, DL);
  }
  return Target;
}

// Get target for wrregions of splitted load.
// Returns tuple consisted of:
//  1. Target for wrregions
//  2. Predicate
//  3. Initial offset of target
//  4. Instruction to replace later
static std::tuple<Value *, Value *, unsigned, Instruction *>
getLoadTarget(CallInst *Load, const GenXSubtarget *ST) {
  Value *LoadPred;
  if (CallInst *LoadWrr = getLoadWrregion(Load)) {
    // If we found wrregion user, then use its predicate for splitted instructions.
    LoadPred =
        LoadWrr->getArgOperand(GenXIntrinsic::GenXRegion::PredicateOperandNum);

    // If wrregion can be represented as raw operand, we can reuse its target and offset.
    if (genx::isValueRegionOKForRaw(LoadWrr, true /* IsWrite */, ST)) {
      // TODO: mark wrregion to be erased once issue with ToErase and
      // iteration order will be resolved.
      Value *Target =
          LoadWrr->getArgOperand(GenXIntrinsic::GenXRegion::OldValueOperandNum);
      Value *Offset =
          LoadWrr->getArgOperand(GenXIntrinsic::GenXRegion::WrIndexOperandNum);
      unsigned InitialOffset = cast<ConstantInt>(Offset)->getZExtValue();
      return {Target, LoadPred, InitialOffset, LoadWrr};
    }
  } else if (SelectInst *SI = getLoadSelect(Load)) {
    LoadPred = SI->getCondition();
    Value *Target = SI->getFalseValue();
    return {Target, LoadPred, 0, SI};
  } else {
    // No wrregion user, load is not predicated.
    LoadPred = ConstantInt::get(IntegerType::getInt1Ty(Load->getContext()), 1);
  }

  // Create new target for load.
  Value *Target = UndefValue::get(Load->getType());
  return {Target, LoadPred, 0, Load};
}

/***********************************************************************
 * splitGatherScatter : lower gather/scatter/atomic to the width support
 * by the hardware platform.
 *
 * This performs two functions:
 *
 * 1. If the operation is wider than what hardware can support, splits it
 *    into the legal width.
 *
 * 2. For typed gather4/scatter4, when r or both v and r are zero, replace
 *    with undef so that they are not encoded in the vISA instruction and the
 *    message skips them.
 */
bool GenXLowering::splitGatherScatter(CallInst *CI, unsigned IID) {
  enum {
    MASK_IDX = 0,
    PRED_IDX = 1,
    SURF_IDX = 2,
    U_IDX = 3,
    DATA_IDX = 6,
    NONEED = 11
  };

  unsigned MaskIdx = NONEED;
  unsigned PredIdx = NONEED;
  unsigned AddrIdx = NONEED;
  unsigned DataIdx = NONEED;
  unsigned AtomicSrcIdx = NONEED;
  bool IsTyped = false;
  int AtomicNumSrc = (-1); // -1 means not-an-atomic
  unsigned NumVectorElements = 1;

  switch (IID) {
  case GenXIntrinsic::genx_typed_atomic_add:
  case GenXIntrinsic::genx_typed_atomic_and:
  case GenXIntrinsic::genx_typed_atomic_fmax:
  case GenXIntrinsic::genx_typed_atomic_fmin:
  case GenXIntrinsic::genx_typed_atomic_imax:
  case GenXIntrinsic::genx_typed_atomic_imin:
  case GenXIntrinsic::genx_typed_atomic_max:
  case GenXIntrinsic::genx_typed_atomic_min:
  case GenXIntrinsic::genx_typed_atomic_or:
  case GenXIntrinsic::genx_typed_atomic_sub:
  case GenXIntrinsic::genx_typed_atomic_xchg:
  case GenXIntrinsic::genx_typed_atomic_xor:
    AtomicSrcIdx = 2;
    PredIdx = 0;
    AddrIdx = 3;
    IsTyped = true;
    AtomicNumSrc = 1;
    break;
  case GenXIntrinsic::genx_typed_atomic_dec:
  case GenXIntrinsic::genx_typed_atomic_inc:
    PredIdx = 0;
    AddrIdx = 2;
    IsTyped = true;
    AtomicNumSrc = 0;
    break;
  case GenXIntrinsic::genx_typed_atomic_cmpxchg:
  case GenXIntrinsic::genx_typed_atomic_fcmpwr:
    AtomicSrcIdx = 2;
    PredIdx = 0;
    AddrIdx = 4;
    IsTyped = true;
    AtomicNumSrc = 2;
    break;
  case GenXIntrinsic::genx_scatter4_typed:
  case GenXIntrinsic::genx_gather4_typed:
    DataIdx = DATA_IDX;
    MaskIdx = MASK_IDX;
    PredIdx = PRED_IDX;
    AddrIdx = U_IDX;
    IsTyped = true;
    break;
  case GenXIntrinsic::genx_scatter4_scaled:
  case GenXIntrinsic::genx_gather4_scaled:
    DataIdx = 6;
    PredIdx = 0;
    MaskIdx = 1;
    AddrIdx = 5;
    break;
  case GenXIntrinsic::genx_gather4_scaled2:
    MaskIdx = 0;
    AddrIdx = 4;
    break;
  case GenXIntrinsic::genx_svm_scatter4_scaled:
  case GenXIntrinsic::genx_svm_gather4_scaled:
    DataIdx = 5;
    PredIdx = 0;
    MaskIdx = 1;
    AddrIdx = 4;
    break;
  case GenXIntrinsic::genx_scatter_scaled:
  case GenXIntrinsic::genx_gather_scaled:
    DataIdx = 6;
    PredIdx = 0;
    AddrIdx = 5;
    break;
  case GenXIntrinsic::genx_gather_scaled2:
    AddrIdx = 4;
    break;
  case GenXIntrinsic::genx_svm_scatter:
  case GenXIntrinsic::genx_svm_gather:
    DataIdx = 3;
    PredIdx = 0;
    AddrIdx = 2;
    break;
  case GenXIntrinsic::genx_svm_atomic_dec:
  case GenXIntrinsic::genx_svm_atomic_inc:
    DataIdx = 2;
    PredIdx = 0;
    AddrIdx = 1;
    AtomicNumSrc = 0;
  case GenXIntrinsic::genx_svm_atomic_add:
  case GenXIntrinsic::genx_svm_atomic_and:
  case GenXIntrinsic::genx_svm_atomic_fmax:
  case GenXIntrinsic::genx_svm_atomic_fmin:
  case GenXIntrinsic::genx_svm_atomic_imax:
  case GenXIntrinsic::genx_svm_atomic_imin:
  case GenXIntrinsic::genx_svm_atomic_max:
  case GenXIntrinsic::genx_svm_atomic_min:
  case GenXIntrinsic::genx_svm_atomic_or:
  case GenXIntrinsic::genx_svm_atomic_sub:
  case GenXIntrinsic::genx_svm_atomic_xchg:
  case GenXIntrinsic::genx_svm_atomic_xor:
    DataIdx = 3;
    PredIdx = 0;
    AddrIdx = 1;
    AtomicSrcIdx = 2;
    AtomicNumSrc = 1;
    break;
  case GenXIntrinsic::genx_svm_atomic_cmpxchg:
  case GenXIntrinsic::genx_svm_atomic_fcmpwr:
    DataIdx = 4;
    PredIdx = 0;
    AddrIdx = 1;
    AtomicSrcIdx = 2;
    AtomicNumSrc = 2;
    break;
  case GenXIntrinsic::genx_dword_atomic_add:
  case GenXIntrinsic::genx_dword_atomic_and:
  case GenXIntrinsic::genx_dword_atomic_fmax:
  case GenXIntrinsic::genx_dword_atomic_fmin:
  case GenXIntrinsic::genx_dword_atomic_imax:
  case GenXIntrinsic::genx_dword_atomic_imin:
  case GenXIntrinsic::genx_dword_atomic_max:
  case GenXIntrinsic::genx_dword_atomic_min:
  case GenXIntrinsic::genx_dword_atomic_or:
  case GenXIntrinsic::genx_dword_atomic_sub:
  case GenXIntrinsic::genx_dword_atomic_xchg:
  case GenXIntrinsic::genx_dword_atomic_xor:
    DataIdx = 4;
    PredIdx = 0;
    AddrIdx = 2;
    AtomicSrcIdx = 3;
    AtomicNumSrc = 1;
    break;
  case GenXIntrinsic::genx_dword_atomic_cmpxchg:
  case GenXIntrinsic::genx_dword_atomic_fcmpwr:
    DataIdx = 5;
    PredIdx = 0;
    AddrIdx = 2;
    AtomicSrcIdx = 3;
    AtomicNumSrc = 2;
    break;
  case GenXIntrinsic::genx_dword_atomic_dec:
  case GenXIntrinsic::genx_dword_atomic_inc:
    DataIdx = 3;
    PredIdx = 0;
    AddrIdx = 2;
    AtomicNumSrc = 0;
    break;

  default:
    return false;
  }

  // nulling unused inputs for typed gather/scatter/atomic
  if (IsTyped) {
    Constant *V = dyn_cast<Constant>(CI->getArgOperand(AddrIdx + 1));
    Constant *R = dyn_cast<Constant>(CI->getArgOperand(AddrIdx + 2));
    // Only continue when R is known to be zero.
    if (R && R->isNullValue()) {
      CI->setOperand(AddrIdx + 2, UndefValue::get(R->getType()));
      if (V && V->isNullValue())
        CI->setOperand(AddrIdx + 1, UndefValue::get(V->getType()));
    }
    // check if LOD is zero for atomic
    if (AtomicNumSrc >= 0) {
      Constant *LOD = dyn_cast<Constant>(CI->getArgOperand(AddrIdx + 3));
      if (LOD && LOD->isNullValue())
        CI->setOperand(AddrIdx + 3, UndefValue::get(LOD->getType()));
    }
  }
  // Deduce intrinsic width: check predicate if exists, then check address vector.
  unsigned WidthOperand;
  if (PredIdx != NONEED)
    WidthOperand = PredIdx;
  else if (AddrIdx != NONEED)
    WidthOperand = AddrIdx;
  else
    IGC_ASSERT_EXIT_MESSAGE(0, "Cannot infer execution width of intrinsic (checked pred and addr operands)");
  auto Width = cast<VectorType>(CI->getArgOperand(WidthOperand)->getType())
                   ->getNumElements();
  unsigned TargetWidth = IsTyped ? 8 : 16;
  if (Width <= TargetWidth)
    return false;
  IGC_ASSERT(TargetWidth);
  IGC_ASSERT((Width % TargetWidth) == 0);
  auto NumSplits = Width / TargetWidth;
  IGC_ASSERT(NumSplits == 2 || NumSplits == 4);
  unsigned NumChannels = NumVectorElements;
  if (MaskIdx != NONEED) {
    NumChannels = (unsigned)cast<ConstantInt>(CI->getArgOperand(MaskIdx))
                      ->getZExtValue();
    NumChannels = (NumChannels & 1) + ((NumChannels & 2) >> 1) +
                  ((NumChannels & 4) >> 2) + ((NumChannels & 8) >> 3);
  }
  
  unsigned NumBlks = 1;
  if (IID == GenXIntrinsic::genx_svm_scatter ||
      IID == GenXIntrinsic::genx_svm_gather) {
    NumBlks = (unsigned)cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
    NumBlks = (1 << NumBlks);
    auto ElmSz = CI->getArgOperand(DataIdx)->getType()->getScalarSizeInBits() / 8;
    if (ElmSz == 1 && NumBlks < 4)
      NumBlks = 4;
    else if (ElmSz == 2 && NumBlks < 2)
      NumBlks = 2;
  }
  const DebugLoc &DL = CI->getDebugLoc();
  Value *NewResult = nullptr;
  if (CI->getType() && CI->getType()->isVectorTy() &&
      cast<VectorType>(CI->getType())->getNumElements() >=
          Width * NumChannels * NumBlks) {
    if (DataIdx != NONEED)
      NewResult = CI->getArgOperand(DataIdx);
    else
      NewResult = UndefValue::get(CI->getType());
  }

  bool IsNewLoad = isNewLoadInst(CI);
  Value *LoadPred = nullptr;
  unsigned InitialOffset = 0;
  Instruction *InstToReplace = CI;
  if (IsNewLoad)
    std::tie(NewResult, LoadPred, InitialOffset, InstToReplace) =
        getLoadTarget(CI, ST);

  for (unsigned i = 0; i < NumSplits; ++i) {
    SmallVector<Value *, 8> Args;
    // initialize the args with the old values
    for (unsigned ArgI = 0; ArgI < CI->getNumArgOperands(); ++ArgI)
      Args.push_back(CI->getArgOperand(ArgI));
    // Predicate
    if (PredIdx != NONEED) {
      Value *V = CI->getArgOperand(PredIdx);
      if (auto C = dyn_cast<Constant>(V))
        Args[PredIdx] = getConstantSubvector(C, i * TargetWidth, TargetWidth);
      else
        Args[PredIdx] = Region::createRdPredRegion(
          V, i * TargetWidth, TargetWidth, "predsplit", CI, DL);
    }
    // address source
    unsigned NumAddrs = 1;
    if (IsTyped)
      NumAddrs = (AtomicNumSrc >= 0) ? 4 : 3;
    for (unsigned AddrI = 0; AddrI < NumAddrs; ++AddrI) {
      Value *V = CI->getArgOperand(AddrIdx + AddrI);
      Region R(V);
      R.Width = R.NumElements = TargetWidth;
      R.Offset = i * TargetWidth * V->getType()->getScalarSizeInBits()/8; // in bytes
      Args[AddrIdx + AddrI] = R.createRdRegion(V, "addrsplit", CI, DL);
    }
    // data source
    // We need to construct a new vector with 8 elements per enabled
    // color.
    if (DataIdx != NONEED) {
      Value *V = CI->getArgOperand(DataIdx);
      auto DataTy = IGCLLVM::FixedVectorType::get(
          V->getType()->getScalarType(), TargetWidth * NumChannels * NumBlks);
      auto ElmSz = V->getType()->getScalarSizeInBits() / 8;
      Value *NewVec = UndefValue::get(DataTy);
      if (!isa<UndefValue>(V)) {
        for (unsigned Channel = 0; Channel < NumChannels; ++Channel) {
          Region RdR(V);
          RdR.Width = RdR.NumElements = TargetWidth * NumBlks;
          RdR.Offset = 4 * (Width * NumBlks * Channel + TargetWidth * NumBlks * i);
          auto Rd = RdR.createRdRegion(V, "datasplit", CI, DL);
          if (NumChannels > 1) {
            Region WrR(DataTy);
            WrR.Width = WrR.NumElements = TargetWidth * NumBlks;
            WrR.Offset = ElmSz * TargetWidth * NumBlks * Channel;
            NewVec = WrR.createWrRegion(NewVec, Rd, "datasplit", CI, DL);
          } else
            NewVec = Rd;
        }
      }
      Args[DataIdx] = NewVec;
    }
    // atomic source operands
    if (AtomicSrcIdx != NONEED) {
      for (int SrcI = 0; SrcI < AtomicNumSrc; ++SrcI) {
        Value *V = CI->getArgOperand(AtomicSrcIdx + SrcI);
        Region R(V);
        R.Width = R.NumElements = TargetWidth;
        R.Offset = i * TargetWidth * V->getType()->getScalarSizeInBits()/8; // in bytes
        Args[AtomicSrcIdx + SrcI] = R.createRdRegion(V, "addrsplit", CI, DL);
        }
    }
    // now create the new narrower instruction
    if (NewResult) {
      Type *DstTy = nullptr;
      if (DataIdx != NONEED)
        DstTy = Args[DataIdx]->getType();
      else {
        DstTy =
            IGCLLVM::FixedVectorType::get(CI->getType()->getScalarType(),
                                          TargetWidth * NumBlks * NumChannels);
      }
      SmallVector<Type *, 4> Tys = {DstTy};
      if (PredIdx != NONEED)
        Tys.push_back(Args[PredIdx]->getType());
      if (AddrIdx != NONEED)
        Tys.push_back(Args[AddrIdx]->getType());
      auto Decl = GenXIntrinsic::getAnyDeclaration(
          CI->getParent()->getParent()->getParent(), IID, Tys);
      auto *Gather = CallInst::Create(Decl, Args, CI->getName() + ".split", CI);
      Gather->setDebugLoc(DL);
      if (IsNewLoad) {
        if (NumChannels == 1)
          NewResult = generate1ChannelWrrregion(NewResult, InitialOffset,
                                                Gather, LoadPred, i, CI);
        else
          NewResult =
              generateNChannelWrregion(NewResult, InitialOffset, Gather,
                                       LoadPred, i, NumSplits, NumChannels, CI);
        continue;
      }
      // Join the results together, starting with the old value.
      auto ElmSz = DstTy->getScalarSizeInBits() / 8;
      if (NumChannels > 1) {
        Region RdR(Gather);
        RdR.Width = RdR.NumElements = TargetWidth * NumBlks;
        Region WrR(NewResult);
        WrR.Width = WrR.NumElements = TargetWidth * NumBlks;
        WrR.Mask = Args[PredIdx];
        for (unsigned Channel = 0; Channel != NumChannels; ++Channel) {
          RdR.Offset = ElmSz * TargetWidth * NumBlks * Channel;
          auto Rd = RdR.createRdRegion(Gather, "joint", CI, DL);
          WrR.Offset = 4 * (Width * NumBlks * Channel + TargetWidth * NumBlks * i);
          NewResult = WrR.createWrRegion(NewResult, Rd, "join", CI, DL);
        }
      } else {
        Region WrR(NewResult);
        WrR.Width = WrR.NumElements = TargetWidth * NumBlks;
        WrR.Offset = ElmSz * TargetWidth * NumBlks * i;
        WrR.Mask = Args[PredIdx];
        NewResult = WrR.createWrRegion(NewResult, Gather, "join", CI, DL);
      }
    } else {
      IGC_ASSERT(CI->use_empty());
      IGC_ASSERT(DataIdx != NONEED);
      // Create the target-wide scatter instructions.
      Type *Tys[] = {Args[PredIdx]->getType(), Args[AddrIdx]->getType(),
                     Args[DataIdx]->getType()};
      auto Decl = GenXIntrinsic::getAnyDeclaration(
          CI->getParent()->getParent()->getParent(), IID, Tys);
      auto NewInst = CallInst::Create(Decl, Args, "", CI);
      NewInst->setDebugLoc(DL);
    }
  }

  if (NewResult)
    InstToReplace->replaceAllUsesWith(NewResult);

  if (InstToReplace != CI)
    ToErase.push_back(InstToReplace);
  ToErase.push_back(CI);
  return true;
}


/***********************************************************************
 * generatePrecicatedWrrForNewLoad : Generate predicated wrr if result
 *                                   of a load that needs no splits
 * Return: true if predicated wrr was generated
 */

bool GenXLowering::generatePredicatedWrrForNewLoad(CallInst *CI) {
  IGC_ASSERT(isNewLoadInst(CI) && "New load expected");
  // Generate predicated wrr if result of a load is predicated with a select
  if (auto *SI = getLoadSelect(CI)) {
    Value *NewResult = UndefValue::get(CI->getType());
    Value *LoadPred = SI->getCondition();
    NewResult =
        generatePredicatedWrregion(NewResult, CI, LoadPred, 0 /* Offset */,
                                   SI->getNextNode(), "lowerpred");
    SI->replaceAllUsesWith(NewResult);
    ToErase.push_back(SI);
    return true;
  }
  return false;
}

/***********************************************************************
 * processInst : process one instruction in GenXLowering
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 */
bool GenXLowering::processInst(Instruction *Inst) {
  if (isa<InsertElementInst>(Inst))
    return lowerInsertElement(Inst);
  if (isa<ExtractElementInst>(Inst))
    return lowerExtractElement(Inst);
  if (isa<TruncInst>(Inst))
    return lowerTrunc(Inst);
  if (isa<CastInst>(Inst))
    return lowerCast(Inst);
  if (auto SI = dyn_cast<SelectInst>(Inst)) {
    if (SI->getType()->getScalarType()->isIntegerTy(1)) {
      if (SI->getType() == SI->getCondition()->getType())
        return lowerBoolVectorSelect(SI);
      return lowerBoolScalarSelect(SI);
    }
    // Try lowering a non-bool select to wrregion. If lowerSelect decides
    // not to, and it is a byte operation, widen it if necessary.
    return lowerSelect(SI) || widenByteOp(SI);
  }
  if (auto SI = dyn_cast<ShuffleVectorInst>(Inst)) {
    if (SI->getType()->getScalarType()->isIntegerTy(1))
      return lowerBoolShuffle(SI);
    return lowerShuffle(SI);
  }
  if (isa<BinaryOperator>(Inst)) {
    if (widenByteOp(Inst))
      return true;
    if (Inst->getOpcode() == Instruction::AShr ||
        Inst->getOpcode() == Instruction::LShr)
      return lowerShr(Inst);
    if (Inst->getOpcode() == Instruction::Mul)
      return lowerMul64(Inst);
    return false;
  }
  if (Inst->getOpcode() == Instruction::ICmp)
    return widenByteOp(Inst);
  else if (auto CI = dyn_cast<FCmpInst>(Inst))
    return lowerFCmpInst(CI);
  if (CallInst *CI = dyn_cast<CallInst>(Inst)) {
    if (CI->isInlineAsm())
      return false;
    processTwoAddressOpnd(CI);
    unsigned IntrinsicID = GenXIntrinsic::not_any_intrinsic;
    if (Function *Callee = CI->getCalledFunction()) {
      IntrinsicID = GenXIntrinsic::getAnyIntrinsicID(Callee);
      IGC_ASSERT(CI->getNumArgOperands() < GenXIntrinsicInfo::OPNDMASK);
    }
       // split gather/scatter/atomic into the width legal to the target
    if (splitGatherScatter(CI, IntrinsicID))
      return true;
    else if (isNewLoadInst(CI))
      return generatePredicatedWrrForNewLoad(CI);

    switch (IntrinsicID) {
    case GenXIntrinsic::genx_rdregioni:
    case GenXIntrinsic::genx_rdregionf:
      return lowerRdRegion(Inst);
    case GenXIntrinsic::genx_wrregioni:
    case GenXIntrinsic::genx_wrregionf:
      return lowerWrRegion(Inst);
    case GenXIntrinsic::genx_rdpredregion:
      return lowerRdPredRegion(Inst);
    case GenXIntrinsic::genx_wrpredregion:
      return lowerWrPredRegion(Inst);
    case GenXIntrinsic::not_any_intrinsic:
      break;
    case Intrinsic::dbg_value:
    case GenXIntrinsic::genx_absf:
    case GenXIntrinsic::genx_absi:
      break;
    default:
    case GenXIntrinsic::genx_constantpred:
    case GenXIntrinsic::genx_constanti:
    case GenXIntrinsic::genx_constantf:
      break; // ignore
    case GenXIntrinsic::genx_vload: {
      if (!Inst->use_empty()) {
        Value *Ptr = Inst->getOperand(0);
        LoadInst *LI = new LoadInst(Ptr->getType()->getPointerElementType(),
                                    Ptr, "", /*volatile*/ true, Inst);
        LI->takeName(Inst);
        LI->setDebugLoc(Inst->getDebugLoc());
        Inst->replaceAllUsesWith(LI);
      }
      ToErase.push_back(Inst);
      return true;
    }
    case GenXIntrinsic::genx_vstore: {
      Value *Val = Inst->getOperand(0);
      Value *Ptr = Inst->getOperand(1);
      auto ST = new StoreInst(Val, Ptr, /*volatile*/ true, Inst);
      ST->setDebugLoc(Inst->getDebugLoc());
      ToErase.push_back(Inst);
      return true;
    }
    case Intrinsic::trap:
      return lowerTrap(CI);
    case Intrinsic::ctpop:
      return lowerCtpop(CI);
    case Intrinsic::uadd_with_overflow:
      return lowerUAddWithOverflow(CI);
    case Intrinsic::sadd_with_overflow:
    case Intrinsic::ssub_with_overflow:
    case Intrinsic::usub_with_overflow:
    case Intrinsic::smul_with_overflow:
    case Intrinsic::umul_with_overflow:
      Inst->getContext().emitError(
          Inst, "GenX backend cannot handle overflowing intrinsics yet");
      break;
    }
    return false;
  }
  if (ExtractValueInst *EV = dyn_cast<ExtractValueInst>(Inst))
    return lowerExtractValue(EV);
  if (InsertValueInst *IV = dyn_cast<InsertValueInst>(Inst))
    return lowerInsertValue(IV);
  if (isa<LoadInst>(Inst) || isa<StoreInst>(Inst))
    return lowerLoadStore(Inst);
  if (isa<AllocaInst>(Inst))
    Inst->getContext().emitError(Inst,
                                 "GenX backend cannot handle allocas yet");
  return false;
}

/***********************************************************************
 * lowerRdRegion : handle read region instruction
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 *
 * 1. If index is variable do add sinking on it. (This in itself does not
 *    cause this function to return true, because it does not cause the
 *    original instruction to be replaced.)
 */
bool GenXLowering::lowerRdRegion(Instruction *Inst) {
  // Sink add in address calculation.
  Use *U = &Inst->getOperandUse(GenXIntrinsic::GenXRegion::RdIndexOperandNum);
  *U = sinkAdd(*U);
  return false;
}

/***********************************************************************
 * lowerWrRegion : handle write region instruction
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 *
 * 1. If index is variable do add sinking on it. (This in itself does not
 *    cause this function to return true, because it does not cause the
 *    original instruction to be replaced.)
 *
 * 2. If it is a predicated byte wrregion, see if it can be widened.
 */
bool GenXLowering::lowerWrRegion(Instruction *Inst) {
  // Sink add in address calculation.
  Use *U = &Inst->getOperandUse(GenXIntrinsic::GenXRegion::WrIndexOperandNum);
  *U = sinkAdd(*U);
  // See if a predicated byte wrregion can be widened.
  return widenByteOp(Inst);
}

/***********************************************************************
 * lowerRdPredRegion : handle read predicate region instruction
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 *
 * rdpredregion is a GenX backend internal intrinsic, and was thus created
 * within this GenXLowering pass. However it is considered legal only if its
 * uses are all in select or wrregion or wrpredpredregion; if not we lower
 * it further here. If a use is in rdpredregion, we need to combine the two
 * rdpredregions into one.
 */
bool GenXLowering::lowerRdPredRegion(Instruction *Inst) {
  SmallVector<CallInst *, 4> RdPredRegionUsers;
  bool Ok = true;
  for (auto ui = Inst->use_begin(), ue = Inst->use_end(); ui != ue; ++ui) {
    auto User = cast<Instruction>(ui->getUser());
    if (isa<SelectInst>(User))
      continue;
    unsigned IID = GenXIntrinsic::getAnyIntrinsicID(User);
    if (GenXIntrinsic::isWrRegion(IID))
      continue;
    if (IID == GenXIntrinsic::genx_wrpredpredregion)
      continue;
    if (IID == GenXIntrinsic::genx_rdpredregion) {
      RdPredRegionUsers.push_back(cast<CallInst>(User));
      continue;
    }
    if (IID == GenXIntrinsic::not_any_intrinsic) {
      Ok = false;
      break;
    }
    if (cast<CallInst>(User)->doesNotAccessMemory()) {
      Ok = false;
      break;
    }
  }
  unsigned Start = cast<ConstantInt>(Inst->getOperand(1))->getZExtValue();
  unsigned Size = cast<VectorType>(Inst->getType())->getNumElements();
  if (Ok) {
    // All uses in select/wrregion/rdpredregion/non-ALU intrinsic, so we can
    // keep the rdpredregion.  Check for uses in another rdpredregion; we need
    // to combine those.
    for (auto ui = RdPredRegionUsers.begin(), ue = RdPredRegionUsers.end();
         ui != ue; ++ui) {
      auto User = *ui;
      unsigned UserStart =
          cast<ConstantInt>(User->getOperand(1))->getZExtValue();
      unsigned UserSize = cast<VectorType>(User->getType())->getNumElements();
      auto Combined =
          Region::createRdPredRegion(Inst->getOperand(0), Start + UserStart,
                                     UserSize, "", User, User->getDebugLoc());
      Combined->takeName(User);
      User->replaceAllUsesWith(Combined);
      ToErase.push_back(User);
    }
    return false;
  }
  // Need to lower it further.
  const DebugLoc &DL = Inst->getDebugLoc();
  // Convert input to vector of short.
  auto In = Inst->getOperand(0);
  Type *I16Ty = Type::getInt16Ty(Inst->getContext());
  Type *InI16Ty = IGCLLVM::FixedVectorType::get(
      I16Ty, cast<VectorType>(In->getType())->getNumElements());
  auto InI16 = CastInst::Create(Instruction::ZExt, In, InI16Ty,
                                Inst->getName() + ".lower1", Inst);
  InI16->setDebugLoc(DL);
  // Use rdregion to extract the region.
  Region R(InI16);
  R.getSubregion(Start, Size);
  auto Rd = R.createRdRegion(InI16, Inst->getName() + ".lower3", Inst, DL);
  // Convert back to predicate.
  auto Res = CmpInst::Create(Instruction::ICmp, CmpInst::ICMP_NE, Rd,
                             Constant::getNullValue(Rd->getType()),
                             Inst->getName() + ".lower4", Inst);
  Res->setDebugLoc(DL);
  // Replace uses and erase.
  Inst->replaceAllUsesWith(Res);
  ToErase.push_back(Inst);
  return true;
}

/***********************************************************************
 * lowerWrPredRegion : handle write predicate region instruction
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 *
 * wrpredregion is a GenX backend internal intrinsic, and was thus created
 * within this GenXLowering pass. However it is considered legal only if its
 * "new value" input is a compare; if not we lower it further here.
 */
bool GenXLowering::lowerWrPredRegion(Instruction *Inst) {
  auto NewVal = Inst->getOperand(1);
  if (isa<CmpInst>(NewVal))
    return false;
  // Need to lower it further.
  const DebugLoc &DL = Inst->getDebugLoc();
  // Convert "old value" input to vector of short.
  auto OldVal = Inst->getOperand(0);
  Type *I16Ty = Type::getInt16Ty(Inst->getContext());
  Type *OldValI16Ty = IGCLLVM::FixedVectorType::get(
      I16Ty, cast<VectorType>(OldVal->getType())->getNumElements());
  auto OldValI16 = CastInst::Create(Instruction::ZExt, OldVal, OldValI16Ty,
                                    Inst->getName() + ".lower1", Inst);
  OldValI16->setDebugLoc(DL);
  // Convert "new value" input to vector of short.
  Type *NewValI16Ty = IGCLLVM::FixedVectorType::get(
      I16Ty, cast<VectorType>(NewVal->getType())->getNumElements());
  auto NewValI16 = CastInst::Create(Instruction::ZExt, NewVal, NewValI16Ty,
                                    Inst->getName() + ".lower2", Inst);
  NewValI16->setDebugLoc(DL);
  // Use wrregion to write the new value into the old value.
  Region R(OldValI16);
  R.getSubregion(cast<ConstantInt>(Inst->getOperand(2))->getZExtValue(),
                 cast<VectorType>(NewValI16Ty)->getNumElements());
  auto Wr = R.createWrRegion(OldValI16, NewValI16, Inst->getName() + ".lower3",
                             Inst, DL);
  // Convert back to predicate.
  auto Res = CmpInst::Create(Instruction::ICmp, CmpInst::ICMP_NE, Wr,
                             Constant::getNullValue(Wr->getType()),
                             Inst->getName() + ".lower4", Inst);
  Res->setDebugLoc(DL);
  // Replace uses and erase.
  Inst->replaceAllUsesWith(Res);
  ToErase.push_back(Inst);
  return true;
}

/***********************************************************************
 * lowerInsertElement : lower InsertElement to wrregion, multiplying the
 *      index by the element size
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 */
bool GenXLowering::lowerInsertElement(Instruction *Inst) {
  Instruction *NewInst = NULL;
  // Special case - if the result has 1 element (usually turning scalar into 1
  // element vector) then simply transform the insert element into a bitcast We
  // don't need to worry about the index since if it is not zero the result is
  // undef anyway (and can be set to anything we like) We also don't need to
  // worry about what the original vector is (usually undef) since it will be
  // overwritten or undef
  VectorType *VT = dyn_cast<VectorType>(Inst->getType());
  IGC_ASSERT(VT);
  unsigned NumElements = VT->getNumElements();
  const DebugLoc &DL = Inst->getDebugLoc();
  if (NumElements == 1) {
    Value *ToInsert = Inst->getOperand(1);
    NewInst = CastInst::Create(Instruction::BitCast, ToInsert, VT,
                               Inst->getName(), Inst);
    NewInst->setDebugLoc(DL);
  } else if (!Inst->getType()->getScalarType()->isIntegerTy(1)) {
    // Cast and scale the index.
    Value *IdxVal = scaleInsertExtractElementIndex(
        Inst->getOperand(2), Inst->getOperand(1)->getType(), Inst);
    // Sink adds in the address calculation.
    IdxVal = sinkAdd(IdxVal);
    // Create the new wrregion
    Value *Src = Inst->getOperand(1);
    Region R(Src);
    R.Indirect = IdxVal;
    NewInst = cast<Instruction>(R.createWrRegion(
        Inst->getOperand(0), Src, Inst->getName(), Inst /*InsertBefore*/, DL));
  } else {
    // Boolean insertelement. We have to cast everything to i16, do the
    // insertelement, and cast it back again. All this gets further lowered
    // subsequently.
    auto I16Ty = Type::getIntNTy(Inst->getContext(), 16);
    auto VecTy = IGCLLVM::FixedVectorType::get(
        I16Ty, cast<VectorType>(Inst->getType())->getNumElements());
    auto CastVec =
        CastInst::Create(Instruction::ZExt, Inst->getOperand(0), VecTy,
                         Inst->getOperand(0)->getName() + ".casti16", Inst);
    CastVec->setDebugLoc(DL);
    auto CastEl =
        CastInst::Create(Instruction::ZExt, Inst->getOperand(1), I16Ty,
                         Inst->getOperand(1)->getName() + ".casti16", Inst);
    CastEl->setDebugLoc(DL);
    auto NewInsert = InsertElementInst::Create(CastVec, CastEl,
                                               Inst->getOperand(2), "", Inst);
    NewInsert->takeName(Inst);
    NewInsert->setDebugLoc(DL);
    NewInst = CmpInst::Create(Instruction::ICmp, CmpInst::ICMP_NE, NewInsert,
                              Constant::getNullValue(VecTy),
                              NewInsert->getName() + ".casti1", Inst);
    NewInst->setDebugLoc(DL);
  }
  // Change uses and mark the old inst for erasing.
  Inst->replaceAllUsesWith(NewInst);
  ToErase.push_back(Inst);
  return true;
}

/***********************************************************************
 * lowerExtractElement : lower ExtractElement to rdregion, multiplying the
 *      index by the element size
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 */
bool GenXLowering::lowerExtractElement(Instruction *Inst) {
  Instruction *NewInst = nullptr;
  if (!Inst->getType()->isIntegerTy(1)) {
    // Cast and scale the index.
    Type *ElTy = Inst->getType();
    Value *IdxVal =
        scaleInsertExtractElementIndex(Inst->getOperand(1), ElTy, Inst);
    // Sink adds in the address calculation.
    IdxVal = sinkAdd(IdxVal);
    // Create the new rdregion.
    Region R(Inst);
    R.Indirect = IdxVal;
    NewInst = R.createRdRegion(Inst->getOperand(0), Inst->getName(),
                               Inst /*InsertBefore*/, Inst->getDebugLoc(),
                               true /*AllowScalar*/);
  } else {
    // Boolean extractelement. We have to cast everything to i16, do the
    // extractelement, and cast it back again. All this gets further lowered
    // subsequently.
    auto I16Ty = Type::getIntNTy(Inst->getContext(), 16);
    auto VecTy = IGCLLVM::FixedVectorType::get(
        I16Ty,
        cast<VectorType>(Inst->getOperand(0)->getType())->getNumElements());
    auto CastVec =
        CastInst::Create(Instruction::ZExt, Inst->getOperand(0), VecTy,
                         Inst->getOperand(0)->getName() + ".casti16", Inst);
    const DebugLoc &DL = Inst->getDebugLoc();
    CastVec->setDebugLoc(DL);
    auto NewExtract =
        ExtractElementInst::Create(CastVec, Inst->getOperand(1), "", Inst);
    NewExtract->takeName(Inst);
    NewExtract->setDebugLoc(DL);
    NewInst = CmpInst::Create(Instruction::ICmp, CmpInst::ICMP_NE, NewExtract,
                              Constant::getNullValue(I16Ty),
                              NewExtract->getName() + ".casti1", Inst);
    NewInst->setDebugLoc(DL);
  }
  // Change uses and mark the old inst for erasing.
  Inst->replaceAllUsesWith(NewInst);
  ToErase.push_back(Inst);
  return true;
}

/***********************************************************************
 * scaleInsertExtractElementIndex : scale index by element byte size,
 *      and ensure it is an i16
 */
Value *GenXLowering::scaleInsertExtractElementIndex(Value *IdxVal, Type *ElTy,
                                                    Instruction *InsertBefore) {
  // Do the cast and multiply.
  static_assert(genx::ByteBits);
  unsigned ElementBytes = DL->getTypeSizeInBits(ElTy) / genx::ByteBits;
  IntegerType *I16Ty = Type::getInt16Ty(IdxVal->getContext());
  if (ConstantInt *CI = dyn_cast<ConstantInt>(IdxVal))
    return ConstantInt::get(I16Ty, CI->getSExtValue() * ElementBytes);
  // Ensure the variable offset is i16.
  Instruction *IdxInst = CastInst::CreateIntegerCast(
      IdxVal, I16Ty, false /*isSigned*/, "cast", InsertBefore);
  IdxInst->setDebugLoc(InsertBefore->getDebugLoc());
  // Multiply it by the element size in bytes.
  if (ElementBytes != 1) {
    IdxInst = BinaryOperator::Create(
        Instruction::Shl, IdxInst,
        ConstantInt::get(I16Ty, genx::log2(ElementBytes)), "scale",
        InsertBefore);
    IdxInst->setDebugLoc(InsertBefore->getDebugLoc());
  }
  return IdxInst;
}

/***********************************************************************
 * lowerTrunc : lower a TruncInst
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 *
 * A Trunc is lowered to a bitcast then a region/element read with a stride.
 * GenXCoalescing will coalesce the bitcast, so this will hopefully save
 * an instruction.
 */
bool GenXLowering::lowerTrunc(Instruction *Inst) {
  Value *InValue = Inst->getOperand(0);
  // Check for the trunc's input being a sext/zext where the original element
  // size is the same as the result of the trunc. We can just remove the
  // whole thing then. (This can arise from GenXReduceIntSize.)
  if (auto CI = dyn_cast<CastInst>(InValue)) {
    if ((isa<SExtInst>(CI) || isa<ZExtInst>(CI)) &&
        CI->getOperand(0)->getType() == Inst->getType()) {
      // Just replace uses with the original unextended value.
      Inst->replaceAllUsesWith(CI->getOperand(0));
      ToErase.push_back(Inst);
      return true;
    }
  }

  // Lower "trunc i8 %v to i1" into "cmp.ne (%v & 1), 0"
  if (Inst->getType()->isIntOrIntVectorTy(1)) {
    IRBuilder<> Builder(Inst);
    auto V =
        Builder.CreateAnd(InValue, ConstantInt::get(InValue->getType(), 1));
    V = Builder.CreateICmpNE(V, ConstantInt::get(V->getType(), 0));
    if (auto I = dyn_cast<Instruction>(V))
      I->setDebugLoc(Inst->getDebugLoc());
    Inst->replaceAllUsesWith(V);
    ToErase.push_back(Inst);
    return true;
  }

  Type *InElementTy = InValue->getType();
  Type *OutElementTy = Inst->getType();
  unsigned NumElements = 1;
  if (VectorType *VT = dyn_cast<VectorType>(InElementTy)) {
    InElementTy = VT->getElementType();
    OutElementTy = cast<VectorType>(OutElementTy)->getElementType();
    NumElements = VT->getNumElements();
  }

  // Lower "trunc <32 x i16> %v to <32 x i1>" into "cmp.ne (%v & 1), 0"
  if (NumElements > 1 && OutElementTy->isIntegerTy(1)) {
    IRBuilder<> Builder(Inst);
    unsigned N = NumElements;
    Value *Os = ConstantVector::getSplat(IGCLLVM::getElementCount(N),
                                         ConstantInt::get(InElementTy, 1));
    Value *Zs = ConstantVector::getSplat(IGCLLVM::getElementCount(N),
                                         ConstantInt::get(InElementTy, 0));
    auto V = Builder.CreateAnd(InValue, Os);
    if (auto I = dyn_cast<Instruction>(V))
      I->setDebugLoc(Inst->getDebugLoc());
    V = Builder.CreateICmpNE(V, Zs);
    if (auto I = dyn_cast<Instruction>(V))
      I->setDebugLoc(Inst->getDebugLoc());
    Inst->replaceAllUsesWith(V);
    ToErase.push_back(Inst);
    return true;
  }

  IGC_ASSERT(OutElementTy->getPrimitiveSizeInBits());
  unsigned Stride = InElementTy->getPrimitiveSizeInBits() /
                    OutElementTy->getPrimitiveSizeInBits();
  // Create the new bitcast.
  Instruction *BC = CastInst::Create(
      Instruction::BitCast, InValue,
      IGCLLVM::FixedVectorType::get(OutElementTy, Stride * NumElements),
      Inst->getName(), Inst /*InsertBefore*/);
  BC->setDebugLoc(Inst->getDebugLoc());
  // Create the new rdregion.
  Region R(BC);
  R.NumElements = NumElements;
  R.Stride = Stride;
  R.Width = NumElements;
  R.VStride = R.Stride * R.Width;
  Instruction *NewInst = R.createRdRegion(
      BC, Inst->getName(), Inst /*InsertBefore*/, Inst->getDebugLoc(),
      !isa<VectorType>(Inst->getType()) /*AllowScalar*/);
  // Change uses and mark the old inst for erasing.
  Inst->replaceAllUsesWith(NewInst);
  ToErase.push_back(Inst);
  return true;
}

/***********************************************************************
 * lowerCast : lower a CastInst
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 */
bool GenXLowering::lowerCast(Instruction *Inst) {
  // If it is zext/sext/UIToFP from (vector of) i1, turn into a select.
  if (Inst->getOperand(0)->getType()->getScalarType()->isIntegerTy(1) &&
      Inst->getOpcode() != Instruction::BitCast) {
    int OneVal = 0;
    switch (Inst->getOpcode()) {
    case Instruction::ZExt:
      OneVal = 1;
      break;
    case Instruction::SExt:
      OneVal = -1;
      break;
    case Instruction::UIToFP:
      OneVal = 1;
      break;
    default:
      IGC_ASSERT_MESSAGE(0, "unknown opcode in lowerCast");
    }

    Instruction *NewInst;
    if (Inst->getType()->isFPOrFPVectorTy())
      NewInst = SelectInst::Create(
          Inst->getOperand(0), ConstantFP::get(Inst->getType(), OneVal),
          ConstantFP::get(Inst->getType(), 0), Inst->getName(), Inst);
    else
      NewInst = SelectInst::Create(
          Inst->getOperand(0), ConstantInt::get(Inst->getType(), OneVal),
          ConstantInt::get(Inst->getType(), 0), Inst->getName(), Inst);
    NewInst->setDebugLoc(Inst->getDebugLoc());
    Inst->replaceAllUsesWith(NewInst);
    ToErase.push_back(Inst);
    return true;
  }
  return false;
}

/***********************************************************************
 * lowerSelect : lower a non-i1 select
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 *
 *          Lower select into predicated  wrr. This transform is profitable
 *          if we can bale into resulting wrr later
 */
bool GenXLowering::lowerSelect(SelectInst *SI) {
  IGC_ASSERT(SI);

  if (!isa<VectorType>(SI->getOperand(0)->getType()))
    return false; // scalar selector

  // Do not lower byte select, because byte wrr then will be widened
  if (SI->getTrueValue()->getType()->getScalarType()->isIntegerTy(8))
    return false;

  Value *Cond = SI->getCondition();
  Value *TrueVal = SI->getTrueValue();
  Value *FalseVal = SI->getFalseValue();

  // Do not transform if one of the sources is constant.
  // Now post-legalization generarates redundant moves for constants.
  // It's also required for correct baling of function pointers' PtrToInts
  // into select.
  // This check can be relaxed.
  if (isa<Constant>(TrueVal) || isa<Constant>(FalseVal))
    return false;

  // If select is used by unmasked wrr than we do not apply transformation too
  // because wrr+wrr is not optimal. In this case select itself will bale into
  // wrr. There might be some cases where wrr user of
  // select can be eliminated too.
  if (SI->hasOneUse() && GenXIntrinsic::isWrRegion(SI->user_back())) {
    auto *I = cast<Instruction>(SI->user_back());
    if ((I->getOperand(GenXIntrinsic::GenXRegion::NewValueOperandNum) == SI) &&
        !Region(I, BaleInfo()).Mask)
      return false;
  }

  // GenXPatternMatch tries to convert cmp + select
  // into min/max instructions. So do not transform in this case
  // This check can be relaxed too.
  if (isa<CmpInst>(Cond))
    return false;

  bool TrueValUsedOnce = TrueVal->hasOneUse();
  bool FalseValUsedOnce = FalseVal->hasOneUse();

  // Baling produces better code if balable
  // value has single use
  if (!FalseValUsedOnce && !TrueValUsedOnce)
    return false;

  // So select this value
  bool InvertPred = false;
  Value *OldWrrVal = FalseVal;
  Value *NewWrrVal = TrueVal;
  if (GotoJoin::isEMValue(Cond) && !TrueValUsedOnce) {
    // Conversion only for true val if EM since
    // EM is implicit, inverting it will require extra instructions
    return false;
  }

  if (FalseValUsedOnce && !TrueValUsedOnce) {
    std::swap(OldWrrVal, NewWrrVal);
    InvertPred = true;
  }

  // Main check: profitable only if we can bale later
  Region R(SI);
  R.Mask = Cond;
  if (!GenXBaling::isBalableNewValueIntoWrr(NewWrrVal, R, ST))
    return false;

  // Inverting predicate if false value of select was choosen
  // as new value for wrr
  if (InvertPred) {
    R.Mask = BinaryOperator::Create(
        Instruction::Xor, R.Mask, Constant::getAllOnesValue(R.Mask->getType()),
        SI->getName() + ".invertpred", SI);
    cast<Instruction>(R.Mask)->setDebugLoc(SI->getDebugLoc());
  }

  auto NewWrRegion = cast<Instruction>(R.createWrRegion(
      OldWrrVal, NewWrrVal, SI->getName() + ".lower", SI, SI->getDebugLoc()));
  SI->replaceAllUsesWith(NewWrRegion);
  ToErase.push_back(SI);
  return true;
}

/***********************************************************************
 * lowerBoolScalarSelect : lower a SelectInst on vector of i1
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 *
 * This is a select on vector of i1 where the condition is scalar. This only
 * happens in simd control flow where an LLVM pass has optimized away the
 * conditional branch. We restore the conditional branch and create an
 * if..else..endif.
 */
bool GenXLowering::lowerBoolScalarSelect(SelectInst *SI) {
  //         BB1
  //        /  |
  // false /   | true
  //      /    |
  //   BB2     |
  //      \    |
  //       \   |
  //        \  |
  //         BB4
  //
  auto BB1 = SI->getParent();
  auto BB2 = SplitBlock(BB1, SI, DT);
  auto BB4 = SplitEdge(BB1, BB2, DT);
  BB2->setName("select.false");
  BB4->setName("select.true");

  auto OldTerm = BB1->getTerminator();
  BranchInst::Create(BB4, BB2, SI->getCondition(), OldTerm);
  OldTerm->eraseFromParent();
  // Since additional edge is added between BB1 and BB4 instead of through BB2
  // only. BB4 is not immediately dominated by BB2 anymore. Instead, BB4 is
  // dominated by BB1 immediately.
  if (DT)
    DT->changeImmediateDominator(BB4, BB1);
  // Replace 'select' with 'phi'
  auto Phi = PHINode::Create(SI->getType(), /*NumReservedValues=*/2, "",
                             &BB4->front());
  Phi->takeName(SI);
  Phi->addIncoming(SI->getTrueValue(), BB1);
  Phi->addIncoming(SI->getFalseValue(), BB2);
  SI->replaceAllUsesWith(Phi);
  ToErase.push_back(SI);
  // Split the (critical) edge from BB1 to BB4 to avoid having critical edge.
  auto BB3 = SplitEdge(BB1, BB4, DT);
  BB3->setName("select.crit");
  return true;
}

/***********************************************************************
 * lowerBoolVectorSelect : lower a SelectInst on (vector of) i1
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 *
 * A select on (vector of) i1 is lowered to the equivalent and/or/xor
 * instructions. No simplification is done even if an input is a constant.
 *
 * However, if the selector looks like an EM value, and the "true" operand is
 * a cmp, it is instead lowered to an llvm.genx.wrpredpredregion. Baling will
 * bale the cmp into it, resulting in a masked cmp instruction that sets bits
 * of the flag only if the corresponding EM bit is set.
 *
 * FIXME: I have seen a case where the two inputs are all false and all true.
 * Rather than try and simplify that here in the GenX backend, we should
 * try and work out how to stop LLVM generating it in the first place.
 */
bool GenXLowering::lowerBoolVectorSelect(SelectInst *Inst) {
  if (isa<CmpInst>(Inst->getTrueValue())) {
    // Check for the condition being an EM value. It might be a shufflevector
    // that slices the EM value at index 0.
    bool IsEM = GotoJoin::isEMValue(Inst->getCondition());
    if (!IsEM) {
      if (auto SV = dyn_cast<ShuffleVectorInst>(Inst->getCondition())) {
        ShuffleVectorAnalyzer SVA(SV);
        if (!SVA.getAsSlice()) {
          // Slice at index 0.
          IsEM = GotoJoin::isEMValue(SV->getOperand(0));
        }
      }
    }
    if (IsEM) {
      // Can be lowered to llvm.genx.wrpredpredregion. It always has an index of
      // 0 and the "new value" operand the same vector width as the whole vector
      // here. That might get changed if it is split up in legalization.
      auto NewInst = Region::createWrPredPredRegion(
          Inst->getFalseValue(), Inst->getTrueValue(), 0, Inst->getCondition(),
          "", Inst, Inst->getDebugLoc());
      NewInst->takeName(Inst);
      Inst->replaceAllUsesWith(NewInst);
      ToErase.push_back(Inst);
      return true;
    }
  }
  // Normal lowering to some bit twiddling.
  Instruction *NewInst1 =
      BinaryOperator::Create(BinaryOperator::And, Inst->getOperand(0),
                             Inst->getOperand(1), Inst->getName(), Inst);
  NewInst1->setDebugLoc(Inst->getDebugLoc());
  Instruction *NewInst2 = BinaryOperator::Create(
      BinaryOperator::Xor, Inst->getOperand(0),
      Constant::getAllOnesValue(Inst->getType()), Inst->getName(), Inst);
  NewInst2->setDebugLoc(Inst->getDebugLoc());
  Instruction *NewInst3 =
      BinaryOperator::Create(BinaryOperator::And, Inst->getOperand(2), NewInst2,
                             Inst->getName(), Inst);
  NewInst3->setDebugLoc(Inst->getDebugLoc());
  Instruction *NewInst4 = BinaryOperator::Create(
      BinaryOperator::Or, NewInst1, NewInst3, Inst->getName(), Inst);
  NewInst4->setDebugLoc(Inst->getDebugLoc());
  Inst->replaceAllUsesWith(NewInst4);
  ToErase.push_back(Inst);
  return true;
}

/***********************************************************************
 * lowerBoolShuffle : lower a shufflevector (element type i1)
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 *
 * We handle four cases:
 *
 * 1. A slice of the vector, which can be turned into rdpredregion.
 *
 * 2. A splat. By default we need to lower that to a select to
 *    0 or -1 then a bitcast to the vector of i1. But if the input is the
 *    result of a cmp then we can splat the cmp as an optimization.
 *
 * 3. An unslice of the vector, which can be turned into wrpredregion.
 *
 * 4. General case. Like in the splat case we convert input via select and
 *    result is then bitcasted back to vector of i1. Converted vectors are
 *    then handled by lowerShuffleToMove
 */
bool GenXLowering::lowerBoolShuffle(ShuffleVectorInst *SI) {
  ShuffleVectorAnalyzer SVA(SI);
  // 1. Check for a slice.
  int SliceStart = SVA.getAsSlice();
  if (SliceStart >= 0) {
    unsigned Width = cast<VectorType>(SI->getType())->getNumElements();
    auto RPR = Region::createRdPredRegion(SI->getOperand(0), SliceStart, Width,
                                          "", SI, SI->getDebugLoc());
    RPR->takeName(SI);
    SI->replaceAllUsesWith(RPR);
    ToErase.push_back(SI);
    return true;
  }
  // 2. Check for a splat.
  auto Splat = SVA.getAsSplat();
  if (Splat.Input)
    return lowerBoolSplat(SI, Splat.Input, Splat.Index);
  // 3. Check for an unslice. The "old value" input is operand 0 of the
  // shufflevector; the "new value" input is operand 0 of the shufflevector
  // that is operand 1 of SI. We create a wrpredregion, but GenXLowering might
  // subsequently decide that it is illegal because its "new value" input is not
  // a compare, in which case it is further lowered.
  int UnsliceStart = SVA.getAsUnslice();
  if (UnsliceStart >= 0) {
    auto InnerSI = cast<ShuffleVectorInst>(SI->getOperand(1));
    auto WPR =
        Region::createWrPredRegion(SI->getOperand(0), InnerSI->getOperand(0),
                                   UnsliceStart, "", SI, SI->getDebugLoc());
    WPR->takeName(SI);
    SI->replaceAllUsesWith(WPR);
    // Undef out the operand for InnerSI in SI, so we can directly erase InnerSI
    // if SI was its only use.
    SI->setOperand(1, UndefValue::get(InnerSI->getType()));
    ToErase.push_back(SI);
    if (InnerSI->use_empty())
      InnerSI->eraseFromParent();
    return true;
  }

  // Do not lower replicated slices.
  if (SVA.isReplicatedSlice())
    return false;

  // 4. General case.

  // The idea is to convert input i1 vector to i16 vector via select,
  // then do a shufflevector lowering for non-bool case
  // and convert back to i1 vector via icmp instruction.

  IRBuilder<> B(SI);
  unsigned WidthInput =
      cast<VectorType>(SI->getOperand(0)->getType())->getNumElements();
  unsigned WidthResult = cast<VectorType>(SI->getType())->getNumElements();
  Constant *C1 = ConstantVector::getSplat(IGCLLVM::getElementCount(WidthInput),
                                          B.getInt16(1));
  Constant *C0 = ConstantVector::getSplat(IGCLLVM::getElementCount(WidthInput),
                                          B.getInt16(0));
  Value *V1 = B.CreateSelect(SI->getOperand(0), C1, C0);
  Value *V2 = B.CreateSelect(SI->getOperand(1), C1, C0);
  Value *SI1 = B.CreateShuffleVector(V1, V2, IGCLLVM::getShuffleMaskForBitcode(SI), SI->getName());
  Constant *C2 = ConstantVector::getSplat(IGCLLVM::getElementCount(WidthResult),
                                          B.getInt16(0));
  Value *Result = B.CreateICmpNE(SI1, C2);
  SI->replaceAllUsesWith(Result);
  ToErase.push_back(SI);

  return true;
}

/***********************************************************************
 * lowerBoolSplat : lower a shufflevector (element type i1) that is a splat
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 */
bool GenXLowering::lowerBoolSplat(ShuffleVectorInst *SI, Value *In,
                                  unsigned Idx) {
  unsigned Width = cast<VectorType>(SI->getType())->getNumElements();
  if (isa<VectorType>(In->getType())) {
    IRBuilder<> B(SI);
    Constant *C1 = ConstantVector::getSplat(IGCLLVM::getElementCount(Width),
                                            B.getInt16(1));
    Constant *C0 = ConstantVector::getSplat(IGCLLVM::getElementCount(Width),
                                            B.getInt16(0));
    Value *V = B.CreateSelect(In, C1, C0);
    Region R(V);
    R.NumElements = Width;
    R.Stride = 0;
    R.VStride = 0;
    R.Offset = (int)Idx;
    V = R.createRdRegion(V, "splat", SI, SI->getDebugLoc());
    V = B.CreateICmpNE(V, C0);
    SI->replaceAllUsesWith(V);
    ToErase.push_back(SI);
    return true;
  }
  // This is a splat. See if the input is a cmp, possibly via a bitcast.
  if (auto BC = dyn_cast<BitCastInst>(In))
    In = BC->getOperand(0);
  if (auto Cmp = dyn_cast<CmpInst>(In)) {
    // Create a splatted version of the cmp.
    Value *CmpOpnds[2];
    Region R(Cmp->getOperand(0));
    R.NumElements = Width;
    R.Width = R.NumElements;
    R.Stride = 0;
    R.VStride = 0;
    for (unsigned i = 0; i != 2; ++i) {
      auto Opnd = Cmp->getOperand(i);
      if (auto C = dyn_cast<Constant>(Opnd)) {
        CmpOpnds[i] = ConstantVector::getSplat(
            IGCLLVM::getElementCount(R.NumElements), C);
        continue;
      }
      if (!isa<VectorType>(Opnd->getType())) {
        auto NewBC =
            CastInst::Create(Instruction::BitCast, Opnd,
                             IGCLLVM::FixedVectorType::get(Opnd->getType(), 1),
                             Opnd->getName() + ".bc", Cmp);
        NewBC->setDebugLoc(Cmp->getDebugLoc());
        Opnd = NewBC;
      }
      CmpOpnds[i] =
          R.createRdRegion(Opnd, Cmp->getOperand(i)->getName() + ".splat",
                           Cmp /*InsertBefore*/, Cmp->getDebugLoc());
    }
    auto NewCmp = CmpInst::Create(
        Cmp->getOpcode(), Cmp->getPredicate(), CmpOpnds[0], CmpOpnds[1],
        Cmp->getName() + ".splat", Cmp /*InsertBefore*/);
    NewCmp->setDebugLoc(Cmp->getDebugLoc());
    SI->replaceAllUsesWith(NewCmp);
    ToErase.push_back(SI);
    return true;
  }
  // Default code. Select int and bitcast to vector of i1.
  if (isa<VectorType>(In->getType())) {
    // First convert v1i1 to i1.
    auto NewBC = CastInst::Create(Instruction::BitCast, In,
                                  In->getType()->getScalarType(),
                                  In->getName() + ".scalar", SI);
    NewBC->setDebugLoc(SI->getDebugLoc());
    In = NewBC;
  }
  if (Width == 8 || Width == 16 || Width == 32) {
    auto IntTy = Type::getIntNTy(SI->getContext(), Width);
    auto Sel = SelectInst::Create(In, Constant::getAllOnesValue(IntTy),
                                  Constant::getNullValue(IntTy),
                                  SI->getName() + ".sel", SI);
    Sel->setDebugLoc(SI->getDebugLoc());
    auto NewBC =
        CastInst::Create(Instruction::BitCast, Sel, SI->getType(), "", SI);
    NewBC->takeName(SI);
    NewBC->setDebugLoc(SI->getDebugLoc());
    SI->replaceAllUsesWith(NewBC);
    ToErase.push_back(SI);
    return true;
  }

  IRBuilder<> Builder(SI);
  auto Val = Builder.CreateSelect(In, Builder.getInt16(1), Builder.getInt16(0),
                                  SI->getName() + ".sel");
  if (auto Inst = dyn_cast<Instruction>(Val))
    Inst->setDebugLoc(SI->getDebugLoc());
  Val = Builder.CreateBitCast(
      Val, IGCLLVM::FixedVectorType::get(Builder.getInt16Ty(), 1));
  if (auto Inst = dyn_cast<Instruction>(Val))
    Inst->setDebugLoc(SI->getDebugLoc());

  Region R(Val);
  R.Offset = 0;
  R.Width = 1;
  R.Stride = R.VStride = 0;
  R.NumElements = Width;
  Val = R.createRdRegion(Val, "", SI, SI->getDebugLoc());
  Val = Builder.CreateICmpNE(Val, ConstantVector::getNullValue(Val->getType()));
  Val->takeName(SI);
  if (auto Inst = dyn_cast<Instruction>(Val))
    Inst->setDebugLoc(SI->getDebugLoc());
  SI->replaceAllUsesWith(Val);
  ToErase.push_back(SI);
  return true;
}

/***********************************************************************
 * lowerShuffleSplat : lower a ShuffleInst (element type not i1) when it is
 *                     a splat (repetition of the same element)
 */
void GenXLowering::lowerShuffleSplat(ShuffleVectorInst *SI,
                                     ShuffleVectorAnalyzer::SplatInfo Splat) {
  // This is a splat. Turn it into a splatting rdregion.
  if (!isa<VectorType>(Splat.Input->getType())) {
    // The input is a scalar rather than a 1-vector. Bitcast it to a 1-vector.
    auto *BC = CastInst::Create(
        Instruction::BitCast, Splat.Input,
        IGCLLVM::FixedVectorType::get(Splat.Input->getType(), 1), SI->getName(),
        SI);
    BC->setDebugLoc(SI->getDebugLoc());
    Splat.Input = BC;
  }
  // Create a rdregion with a stride of 0 to represent this splat
  Region R(Splat.Input);
  R.NumElements = cast<VectorType>(SI->getType())->getNumElements();
  R.Width = R.NumElements;
  R.Stride = 0;
  R.VStride = 0;
  R.Offset = Splat.Index * R.ElementBytes;
  Instruction *NewInst =
      R.createRdRegion(Splat.Input, "", SI /*InsertBefore*/, SI->getDebugLoc());
  NewInst->takeName(SI);
  NewInst->setDebugLoc(SI->getDebugLoc());
  SI->replaceAllUsesWith(NewInst);
  ToErase.push_back(SI);
}

/***********************************************************************
 * lowerShuffle : lower a ShuffleInst (element type not i1)
 *
 * Mostly these are splats. These are lowered to a rdregion
 * Any other shuffle is currently unsupported
 */
bool GenXLowering::lowerShuffle(ShuffleVectorInst *SI) {
  auto Splat = ShuffleVectorAnalyzer(SI).getAsSplat();
  if (Splat.Input) {
    lowerShuffleSplat(SI, Splat);
    return true;
  }
  if (lowerShuffleToSelect(SI))
    return true;
  lowerShuffleToMove(SI);
  return true;
}

// Lower those shufflevector that can be implemented efficiently as select.
bool GenXLowering::lowerShuffleToSelect(ShuffleVectorInst *SI) {
  int NumElements = cast<VectorType>(SI->getType())->getNumElements();
  int NumOpnd = SI->getNumOperands();
  for (int i = 0; i < NumOpnd; ++i) {
    if (cast<VectorType>(SI->getOperand(i)->getType())->getNumElements() !=
        NumElements)
      return false;
  }
  for (int i = 0; i < NumElements; ++i) {
    int idx = SI->getMaskValue(i);
    // undef index returns -1.
    if (idx < 0)
      continue;
    if (idx != i && idx != i + NumElements)
      return false;
  }
  IRBuilder<> Builder(SI);
  Type *Int1Ty = Builder.getInt1Ty();
  SmallVector<Constant *, 16> MaskVec;
  MaskVec.reserve(NumElements);
  for (int i = 0; i < NumElements; ++i) {
    int idx = SI->getMaskValue(i);
    // undef index returns -1.
    if (idx == i || idx < 0)
      MaskVec.push_back(ConstantInt::get(Int1Ty, 1));
    else
      MaskVec.push_back(ConstantInt::get(Int1Ty, 0));
  }
  Value *Mask = ConstantVector::get(MaskVec);
  auto NewSel =
      SelectInst::Create(Mask, SI->getOperand(0), SI->getOperand(1), "", SI);
  NewSel->takeName(SI);
  NewSel->setDebugLoc(SI->getDebugLoc());
  SI->replaceAllUsesWith(NewSel);
  ToErase.push_back(SI);
  return true;
}

template <typename Iter> Iter skipUndefs(Iter First, Iter Last) {
  return std::find_if(First, Last, [](int MaskVal) { return MaskVal != -1; });
}

/***********************************************************************
 * lowerShuffleToMove : lower a ShuffleInst (element type is not i1) to a
 *                      sequence of rd/wrregion intrinsics
 */
void GenXLowering::lowerShuffleToMove(ShuffleVectorInst *SI) {
  ShuffleVectorAnalyzer Analyzer(SI);
  std::vector<ShuffleVectorAnalyzer::OperandRegionInfo> RdRegions;
  std::vector<Region> WrRegions;
  auto MaskVals = SI->getShuffleMask();

  // Filling read and write regions based on shuffle mask.
  for (auto It = skipUndefs(MaskVals.begin(), MaskVals.end());
       It != MaskVals.end();
       It = skipUndefs(std::next(It, RdRegions.back().R.NumElements),
                       MaskVals.end())) {
    int Idx = It - MaskVals.begin();
    auto OpRegion = Analyzer.getMaskRegionPrefix(Idx);
    IGC_ASSERT_MESSAGE(OpRegion.R.NumElements > 0,
      "should've match at least 1 element region");
    Region WrRegion(SI);
    WrRegion.Offset = Idx * WrRegion.ElementBytes;
    WrRegion.NumElements = WrRegion.Width = OpRegion.R.NumElements;
    RdRegions.push_back(std::move(OpRegion));
    WrRegions.push_back(std::move(WrRegion));
  }

  // Building rdregion intrinsics or promoting the operand if possible.
  std::vector<Value *> RdRegionInsts;
  std::transform(
      RdRegions.begin(), RdRegions.end(), std::back_inserter(RdRegionInsts),
      [SI](ShuffleVectorAnalyzer::OperandRegionInfo &OpRegion) -> Value * {
        if (OpRegion.R.isWhole(OpRegion.Op->getType()))
          return OpRegion.Op;
        return OpRegion.R.createRdRegion(
            OpRegion.Op, SI->getName() + ".shuffle.rd", SI, SI->getDebugLoc());
      });

  // Obtaining SI replacement (sequence of wrregion intrinsics in the
  // most common case).
  Value *Result;
  if (WrRegions.size() == 0)
    Result = UndefValue::get(SI->getType());
  else if (WrRegions.size() == 1 &&
           WrRegions.front().NumElements ==
               cast<VectorType>(SI->getType())->getNumElements())
    Result = RdRegionInsts.back();
  else {
    auto WrRegionArgs = zip(WrRegions, RdRegionInsts);
    Result = std::accumulate(
        WrRegionArgs.begin(), WrRegionArgs.end(),
        static_cast<Value *>(UndefValue::get(SI->getType())),
        [SI](Value *PrevWrRegionInst,
             const std::tuple<Region &, Value *> &Args) {
          return std::get<0>(Args).createWrRegion(
              PrevWrRegionInst, std::get<1>(Args),
              SI->getName() + ".shuffle.wr", SI, SI->getDebugLoc());
        });
  }

  SI->replaceAllUsesWith(Result);
  ToErase.push_back(SI);
}

/***********************************************************************
 * lowerShr : lower Shl followed by AShr/LShr by the same amount
 *    into trunc+sext/zext
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 *
 * See convertShlShr below.
 */
bool GenXLowering::lowerShr(Instruction *Inst) {
  Instruction *NewInst = convertShlShr(Inst);
  if (!NewInst)
    return false; // no conversion done
  ToErase.push_back(Inst);
  auto Shl = cast<Instruction>(Inst->getOperand(0));
  if (Shl->hasOneUse())
    ToErase.push_back(Shl);
  return true;
}

/***********************************************************************
 * convertShlShr : convert Shl followed by AShr/LShr by the same amount
 *    into trunc+sext/zext
 *
 * Enter:   Inst = the AShr or LShr instruction
 *
 * Return:  0 if no conversion done, else the new SExt/ZExt instruction.
 *          The original AShr/LShr is now unused, but neither original
 *          instruction is erased.
 *
 * This is the opposite to what instruction combining does! We want to change
 * it back to trunc then extend because the trunc can then be lowered into
 * a region, and the extend can sometimes be baled into whatever uses it.
 *
 * This is a separately callable global function so it can also be used
 * from GenXReduceIntSize, which for other reasons of convenience runs before
 * GenXLowering.
 */
Instruction *llvm::genx::convertShlShr(Instruction *Inst) {
  unsigned NumBits = Inst->getType()->getScalarType()->getPrimitiveSizeInBits();
  auto C = dyn_cast<Constant>(Inst->getOperand(1));
  if (!C)
    return nullptr;
  auto Shl = dyn_cast<Instruction>(Inst->getOperand(0));
  if (!Shl)
    return nullptr;
  if (Shl->getOpcode() != Instruction::Shl)
    return nullptr;
  if (Shl->getOperand(1) != C)
    return nullptr;
  if (isa<VectorType>(C->getType())) {
    C = C->getSplatValue();
    if (!C)
      return nullptr;
  }
  unsigned ShiftBits = cast<ConstantInt>(C)->getSExtValue();
  unsigned RemainingBits = NumBits - ShiftBits;
  if (RemainingBits != 8 && RemainingBits != 16)
    return nullptr;
  // We have Shl+AShr or Shl+LShr that can be turned into trunc+sext/zext.
  Type *ConvTy = Type::getIntNTy(Inst->getContext(), RemainingBits);
  if (auto VT = dyn_cast<VectorType>(Inst->getType()))
    ConvTy = IGCLLVM::FixedVectorType::get(ConvTy, VT->getNumElements());
  auto Trunc = CastInst::Create(Instruction::Trunc, Shl->getOperand(0), ConvTy,
                                "", Inst);
  Trunc->takeName(Shl);
  Trunc->setDebugLoc(Shl->getDebugLoc());
  auto Ext = CastInst::Create(Inst->getOpcode() == Instruction::AShr
                                  ? Instruction::SExt
                                  : Instruction::ZExt,
                              Trunc, Inst->getType(), "", Inst);
  Ext->takeName(Inst);
  Ext->setDebugLoc(Inst->getDebugLoc());
  Inst->replaceAllUsesWith(Ext);
  return Ext;
}

/***********************************************************************
 * splitStructPhis : find struct phi nodes and split them
 *
 * Return:  whether code modified
 *
 * Each struct phi node is split into a separate phi node for each struct
 * element. This is needed because the GenX backend's liveness and coalescing
 * code cannot cope with a struct phi.
 *
 * This is run in two places: firstly in GenXLowering, so that pass can then
 * simplify any InsertElement and ExtractElement instructions added by the
 * struct phi splitting. But then it needs to be run again in GenXLiveness,
 * because other passes can re-insert a struct phi. The case I saw in
 * hevc_speed was something commoning up the struct return from two calls in an
 * if..else..endif.
 */
bool genx::splitStructPhis(Function *F) {
  bool Modified = false;
  for (Function::iterator fi = F->begin(), fe = F->end(); fi != fe; ++fi) {
    BasicBlock *BB = &*fi;
    for (BasicBlock::iterator bi = BB->begin();;) {
      PHINode *Phi = dyn_cast<PHINode>(&*bi);
      if (!Phi)
        break;
      ++bi; // increment here as splitStructPhi removes old phi node
      if (isa<StructType>(Phi->getType()))
        Modified |= GenXLowering::splitStructPhi(Phi);
    }
  }
  return Modified;
}

/***********************************************************************
 * splitStructPhi : split a phi node with struct type by splitting into
 *                  struct elements
 */
bool GenXLowering::splitStructPhi(PHINode *Phi) {
  StructType *Ty = cast<StructType>(Phi->getType());
  // Find where we need to insert the combine instructions.
  Instruction *CombineInsertBefore = Phi->getParent()->getFirstNonPHI();
  // Now split the phi.
  Value *Combined = UndefValue::get(Ty);
  // For each struct element...
  for (unsigned Idx = 0, e = Ty->getNumElements(); Idx != e; ++Idx) {
    Type *ElTy = Ty->getTypeAtIndex(Idx);
    // Create the new phi node.
    PHINode *NewPhi =
        PHINode::Create(ElTy, Phi->getNumIncomingValues(),
                        Phi->getName() + ".element" + Twine(Idx), Phi);
    NewPhi->setDebugLoc(Phi->getDebugLoc());
    // Combine the new phi.
    Instruction *Combine = InsertValueInst::Create(
        Combined, NewPhi, Idx, NewPhi->getName(), CombineInsertBefore);
    Combine->setDebugLoc(Phi->getDebugLoc());
    Combined = Combine;
    // For each incoming...
    for (unsigned In = 0, InEnd = Phi->getNumIncomingValues(); In != InEnd;
         ++In) {
      // Create an extractelement to get the individual element value.
      // This needs to go before the terminator of the incoming block.
      BasicBlock *IncomingBB = Phi->getIncomingBlock(In);
      Value *Incoming = Phi->getIncomingValue(In);
      Instruction *Extract = ExtractValueInst::Create(
          Incoming, Idx, Phi->getName() + ".element" + Twine(Idx),
          IncomingBB->getTerminator());
      Extract->setDebugLoc(Phi->getDebugLoc());
      // Add as an incoming of the new phi node.
      NewPhi->addIncoming(Extract, IncomingBB);
    }
  }
  Phi->replaceAllUsesWith(Combined);
  Phi->eraseFromParent();
  return true;
}

/***********************************************************************
 * lowerExtractValue : remove extractvalue if possible
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 *
 * If we can trace the input of the extractvalue to the point where the
 * value was inserted, use that value instead.
 *
 * Because we have already split struct phi nodes, we should just be left
 * with insertvalue/extractvalue pairs that we can remove here. The
 * exception is when a struct is passed in to or returned from a call.
 * Then we leave the extractvalue for later handling in the register
 * allocator.
 */
bool GenXLowering::lowerExtractValue(ExtractValueInst *Inst) {
  ArrayRef<unsigned> EVIndices = Inst->getIndices();
  ArrayRef<unsigned> Indices = EVIndices;
  Value *V = Inst->getAggregateOperand();
  for (;;) {
    InsertValueInst *IV = dyn_cast<InsertValueInst>(V);
    if (!IV) {
      // If we used up any indices, create a new extractvalue for the
      // remaining ones.
      if (Indices.size() != EVIndices.size()) {
        Instruction *NewIV = ExtractValueInst::Create(
            Inst->getAggregateOperand(), Indices, Inst->getName(), Inst);
        NewIV->setDebugLoc(Inst->getDebugLoc());
        Inst->replaceAllUsesWith(NewIV);
        ToErase.push_back(Inst);
        return true;
      }
      return false;
    }
    // We have an insertvalue. See how many of the indices agree.
    ArrayRef<unsigned> IVIndices = IV->getIndices();
    unsigned Match = 0;
    while (Match < Indices.size() && Match < IVIndices.size() &&
           Indices[Match] == IVIndices[Match])
      ++Match;
    if (!Match) {
      // No match at all. Go back to the previous insertvalue.
      V = IV->getAggregateOperand();
      continue;
    }
    // Use the inserted value here.
    V = IV->getInsertedValueOperand();
    // Chop off the indices we have used up. If none left, we have finished.
    Indices = Indices.slice(Match);
    if (!Indices.size())
      break;
  }
  // We have found the struct element value V.
  Inst->replaceAllUsesWith(V);
  ToErase.push_back(Inst);
  return true;
}

/***********************************************************************
 * lowerInsertValue : remove insertvalue if possible
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 *
 * In most cases, by the time we get to an insertvalue, it will be unused
 * because of extractvalue removal.
 *
 * In a case where it is still used (probably because this function has an
 * arg or return value that is a struct, or we call a function like that),
 * the struct value is dealt with in register allocation.
 */
bool GenXLowering::lowerInsertValue(InsertValueInst *Inst) {
  if (Inst->use_empty()) {
    ToErase.push_back(Inst);
    return true;
  }
  return false;
}

/***********************************************************************
 * lowerUAddWithOverflow : lower llvm.uadd.with.overflow
 *
 * This could potentially be implemented with the vISA addc instruction.
 * However an intrinsic for that would need extra GenX backend support for
 * returning a struct containing two vectors, and that support does not exist
 * now.
 *
 * So for now we use the old DEC Alpha trick of comparing the result with
 * one of the operands.
 */
bool GenXLowering::lowerUAddWithOverflow(CallInst *CI) {
  const DebugLoc &DL = CI->getDebugLoc();
  // Do the add.
  auto Add =
      BinaryOperator::Create(Instruction::Add, CI->getArgOperand(0),
                             CI->getArgOperand(1), CI->getName() + ".add", CI);
  Add->setDebugLoc(DL);
  // Do the comparison. (An unsigned add has overflowed if the result is
  // smaller than one of the operands, and, if it has overflowed, the result
  // is smaller than both of the operands. So it doesn't matter which operand
  // we use for the comparison.)
  auto Cmp = CmpInst::Create(Instruction::ICmp, CmpInst::ICMP_ULT, Add,
                             CI->getArgOperand(1), CI->getName() + ".cmp", CI);
  Cmp->setDebugLoc(DL);
  // For any extractvalue use of the result of the original add with overflow,
  // replace it directly.
  SmallVector<ExtractValueInst *, 4> Extracts;
  for (auto ui = CI->use_begin(), ue = CI->use_end(); ui != ue; ++ui)
    if (auto EVI = dyn_cast<ExtractValueInst>(ui->getUser()))
      Extracts.push_back(EVI);
  for (auto ei = Extracts.begin(), ee = Extracts.end(); ei != ee; ++ei) {
    auto EVI = *ei;
    EVI->replaceAllUsesWith(EVI->getIndices()[0] ? (Value *)Cmp : (Value *)Add);
    EVI->setOperand(0, UndefValue::get(CI->getType()));
    ToErase.push_back(EVI);
  }
  // If any uses of the original intrinsic remain, recreate the struct value.
  if (!CI->use_empty()) {
    auto Insert = InsertValueInst::Create(UndefValue::get(CI->getType()), Add,
                                          0, CI->getName() + ".insertadd", CI);
    Insert->setDebugLoc(DL);
    Insert = InsertValueInst::Create(Insert, Cmp, 1,
                                     CI->getName() + ".insertcmp", CI);
    Insert->setDebugLoc(DL);
    // ... and use it to replace the original intrinsic.
    CI->replaceAllUsesWith(Insert);
  }
  ToErase.push_back(CI);
  return true;
}

bool GenXLowering::lowerTrap(CallInst *CI) {
  Module *M = CI->getModule();
  IRBuilder<> Builder(CI);
  auto &Ctx = CI->getContext();
  unsigned EMWidth = 32;
  Type *ArgTypes[] = {
      IGCLLVM::FixedVectorType::get(Type::getInt1Ty(Ctx), EMWidth),
      IGCLLVM::FixedVectorType::get(Type::getInt16Ty(Ctx), EMWidth)};
  auto Fn = GenXIntrinsic::getGenXDeclaration(M,
    GenXIntrinsic::genx_raw_send_noresult, ArgTypes);
  SmallVector<Value *, 8> Args;
  // send
  Args.push_back(ConstantInt::get(Type::getInt32Ty(Ctx), 0));
  // predicate all lanes
  Args.push_back(ConstantVector::getSplat(IGCLLVM::getElementCount(EMWidth),
                                          ConstantInt::getTrue(Ctx)));
  // EOT
  Args.push_back(ConstantInt::get(Type::getInt32Ty(Ctx), 0x27));
  Args.push_back(ConstantInt::get(Type::getInt32Ty(Ctx), 0x02000010));
  Args.push_back(
      ConstantVector::getSplat(IGCLLVM::getElementCount(EMWidth),
                               Constant::getNullValue(Type::getInt16Ty(Ctx))));
  Builder.CreateCall(Fn, Args);
  ToErase.push_back(CI);

  return true;
}

bool GenXLowering::lowerCtpop(CallInst *CI) {
  Module *M = CI->getModule();
  IRBuilder<> Builder(CI);
  Builder.SetCurrentDebugLocation(CI->getDebugLoc());

  Type *Int32Ty = IntegerType::getInt32Ty(CI->getContext());
  Type *RetTy = nullptr;
  if (auto *VT = dyn_cast<VectorType>(CI->getType()))
    RetTy = IGCLLVM::FixedVectorType::get(Int32Ty, VT->getNumElements());
  else
    RetTy = Int32Ty;

  auto *CBitDecl = GenXIntrinsic::getGenXDeclaration(
      M, GenXIntrinsic::genx_cbit, {RetTy, CI->getType()});
  Value *CBitInst =
      Builder.CreateCall(CBitDecl, CI->getOperand(0), CI->getName());

  CBitInst = Builder.CreateZExtOrTrunc(CBitInst, CI->getType());
  CI->replaceAllUsesWith(CBitInst);
  ToErase.push_back(CI);

  return true;
}

// Lower cmp instructions that GenX cannot deal with.
bool GenXLowering::lowerFCmpInst(FCmpInst *Inst) {
  IRBuilder<> Builder(Inst);
  Builder.SetCurrentDebugLocation(Inst->getDebugLoc());
  Value *Ops[] = {Inst->getOperand(0), Inst->getOperand(1)};

  switch (Inst->getPredicate()) {
  default:
    return lowerUnorderedFCmpInst(Inst);
  case CmpInst::FCMP_ORD: // True if ordered (no nans)
  {
    // %c = fcmp ord %a %b
    // =>
    // %1 = fcmp oeq %a %a
    // %2 = fcmp oeq %b %b
    // %c = and %1 %2
    Value *LHS = Builder.CreateFCmpOEQ(Ops[0], Ops[0]);
    Value *RHS = Builder.CreateFCmpOEQ(Ops[1], Ops[1]);
    Value *New = Builder.CreateAnd(LHS, RHS);
    Inst->replaceAllUsesWith(New);
    ToErase.push_back(Inst);
    return true;
  }
  case CmpInst::FCMP_UNO: // True if unordered: isnan(X) | isnan(Y)
  {
    // %c = fcmp uno %a %b
    // =>
    // %1 = fcmp une %a %a
    // %2 = fcmp une %b %b
    // %c = or %1 %2
    Value *LHS = Builder.CreateFCmpUNE(Ops[0], Ops[0]);
    Value *RHS = Builder.CreateFCmpUNE(Ops[1], Ops[1]);
    Value *New = Builder.CreateOr(LHS, RHS);
    Inst->replaceAllUsesWith(New);
    ToErase.push_back(Inst);
    return true;
  }
  case CmpInst::FCMP_UEQ: // UEQ cannot be replaced with NOT ONE because we do
                          // not have ONE
  {
    // %c = fcmp ueq %a %b
    // =>
    // %1 = fcmp olt %a %b
    // %2 = fcmp ogt %a %b
    // %3 = or %1 %2
    // %c = not %3
    Value *LHS = Builder.CreateFCmpOLT(Ops[0], Ops[1]);
    Value *RHS = Builder.CreateFCmpOGT(Ops[0], Ops[1]);
    Value *Or = Builder.CreateOr(LHS, RHS);
    Value *New = Builder.CreateNot(Or);
    Inst->replaceAllUsesWith(New);
    ToErase.push_back(Inst);
    return true;
  }
  case CmpInst::FCMP_ONE: // NE is unordered
  {
    // %c = fcmp one %a %b
    // =>
    // %1 = fcmp olt %a %b
    // %2 = fcmp ogt %a %b
    // %c = or %1 %2
    Value *LHS = Builder.CreateFCmpOLT(Ops[0], Ops[1]);
    Value *RHS = Builder.CreateFCmpOGT(Ops[0], Ops[1]);
    Value *New = Builder.CreateOr(LHS, RHS);
    Inst->replaceAllUsesWith(New);
    ToErase.push_back(Inst);
    return true;
  }
  }

  return false;
}

// FCmp with NE is the only one supported unordered cmp inst. All the rest must
// be lowered.
bool GenXLowering::lowerUnorderedFCmpInst(FCmpInst *Inst) {
  CmpInst::Predicate Pred = Inst->getPredicate();
  if (CmpInst::isOrdered(Pred))
    return false;

  // We support UNE.
  if (Pred == CmpInst::FCMP_UNE)
    return false;

  // For UNO and UEQ we have replacement in lowerFCmpInst.
  IGC_ASSERT(Pred != CmpInst::FCMP_UNO);
  IGC_ASSERT(Pred != CmpInst::FCMP_UEQ);

  CmpInst *InverseFCmp = CmpInst::Create(
      Inst->getOpcode(), CmpInst::getInversePredicate(Pred),
      Inst->getOperand(0), Inst->getOperand(1),
      Inst->getName() + ".ordered.inversed", Inst->getNextNode());
  Instruction *Result = BinaryOperator::CreateNot(
      InverseFCmp, InverseFCmp->getName() + ".not", InverseFCmp->getNextNode());
  Inst->replaceAllUsesWith(Result);
  ToErase.push_back(Inst);

  return true;
}

// Lower cmp instructions that GenX cannot deal with.
bool GenXLowering::lowerMul64(Instruction *Inst) {

  IVSplitter SplitBuilder(*Inst);
  if (!SplitBuilder.IsI64Operation())
    return false;

  IRBuilder<> Builder(Inst);
  Builder.SetCurrentDebugLocation(Inst->getDebugLoc());

  auto Src0 = SplitBuilder.splitOperandLoHi(0);
  auto Src1 = SplitBuilder.splitOperandLoHi(1);

  // create muls and adds
  auto *ResL = Builder.CreateMul(Src0.Lo, Src1.Lo);
  // create the mulh intrinsic to the get the carry-part
  Type *tys[2] = {ResL->getType(), Src0.Lo->getType()};
  // build argument list
  SmallVector<llvm::Value *, 2> args{Src0.Lo, Src1.Lo};
  auto *M = Inst->getModule();
  Function *IntrinFunc =
      GenXIntrinsic::getGenXDeclaration(M, GenXIntrinsic::genx_umulh, tys);

  auto *Cari = Builder.CreateCall(IntrinFunc, args, ".cari");
  auto *Temp0 = Builder.CreateMul(Src0.Lo, Src1.Hi);
  auto *Temp1 = Builder.CreateAdd(Cari, Temp0);
  auto *Temp2 = Builder.CreateMul(Src0.Hi, Src1.Lo);
  auto *ResH = Builder.CreateAdd(Temp2, Temp1);

  // create the bitcast to the destination-type
  auto *Replace = SplitBuilder.combineLoHiSplit({ ResL, ResH }, "mul64",
                                                Inst->getType()->isIntegerTy());
  Inst->replaceAllUsesWith(Replace);
  ToErase.push_back(Inst);
  return true;
}
/***********************************************************************
 * widenByteOp : widen a vector byte operation to short if that might
 *               improve code
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 *
 * Gen has restrictions on byte operands. The jitter copes with that, but
 * sometimes it needs to do even-odd splitting, which can lead to suboptimal
 * code if cmps and predicates are involved.
 * Here we attempt to pick up the common cases by converting a byte operation
 * to short.
 *
 * Note that we might end up with the extends being baled into the instruction
 * anyway, resulting in a byte operation in vISA.
 */
bool GenXLowering::widenByteOp(Instruction *Inst) {
  if (!EnableGenXByteWidening)
    return false;
  Type *Ty = Inst->getType();
  if (isa<CmpInst>(Inst))
    Ty = Inst->getOperand(0)->getType();
  if (!isa<VectorType>(Ty) || !Ty->getScalarType()->isIntegerTy(8))
    return false; // not byte operation
  if (Inst->use_empty())
    return false; // result unused
  // check use, if use is a phi, stop widenning
  if (!isa<CmpInst>(Inst)) {
    for (auto ui = Inst->use_begin(), ue = Inst->use_end(); ui != ue; ++ui) {
      auto User = cast<Instruction>(ui->getUser());
      if (isa<PHINode>(User))
        return false;
    }
  }
  // For a predicated wrregion, widen by separating the predication into a
  // rdregion and select, which can then be widened.
  if (GenXIntrinsic::isWrRegion(Inst)) {
    Region R(Inst, BaleInfo());
    if (R.NumElements == 1 || !R.Mask)
      return false;
    // Can only do this if the predicate is the right size. (We could handle
    // the wrong size case by adding an rdpredregion, but then we would need
    // to ensure that GenXLegalization can cope with an arbitrary size
    // rdpredregion.)
    if (cast<VectorType>(R.Mask->getType())->getNumElements() != R.NumElements)
      return false;
    // Create the rdregion and select.
    auto NewRd =
        R.createRdRegion(Inst->getOperand(0), Inst->getName() + ".byteselrdr",
                         Inst, Inst->getDebugLoc());
    auto NewSel =
        SelectInst::Create(R.Mask, Inst->getOperand(1), NewRd, "", Inst);
    NewSel->takeName(Inst);
    NewSel->setDebugLoc(Inst->getDebugLoc());
    // Modify the existing wrregion.
    Inst->setName(NewSel->getName() + ".byteselwrr");
    Inst->setOperand(1, NewSel);
    Inst->setOperand(GenXIntrinsic::GenXRegion::PredicateOperandNum,
                     Constant::getAllOnesValue(R.Mask->getType()));
    // Fall through for the select to get widened.
    Inst = NewSel;
  }
  // Do the widening for:
  // 1. a compare or select
  // 2. used in a zext that indicates that the user has probably already been
  //    widened by this code.
  bool Widen = false;
  if (isa<CmpInst>(Inst) || isa<SelectInst>(Inst))
    Widen = true;
  else {
    auto user = cast<Instruction>(Inst->use_begin()->getUser());
    if (isa<ZExtInst>(user))
      Widen = true;
  }
  if (!Widen)
    return false;
  // Widen to short.
  // Decide whether to zero or sign extend. Also decide whether the result is
  // guaranteed to have all 0 bits in the extended part.
  Instruction::CastOps ExtOpcode = Instruction::ZExt;
  bool ExtendedIsZero = false;
  switch (Inst->getOpcode()) {
  case Instruction::SDiv:
  case Instruction::AShr:
    ExtOpcode = Instruction::SExt;
    break;
  case Instruction::And:
  case Instruction::Or:
  case Instruction::Xor:
  case Instruction::LShr:
    ExtendedIsZero = true;
    break;
  case Instruction::ICmp:
    if (cast<CmpInst>(Inst)->isSigned())
      ExtOpcode = Instruction::SExt;
    break;
  default:
    break;
  }
  // Get the range of operands to process.
  unsigned StartIdx = 0, EndIdx = Inst->getNumOperands();
  if (auto CI = dyn_cast<CallInst>(Inst))
    EndIdx = CI->getNumArgOperands();
  else if (isa<SelectInst>(Inst))
    StartIdx = 1;
  // Extend the operands.
  auto ExtTy = IGCLLVM::FixedVectorType::get(
      Type::getInt16Ty(Inst->getContext()),
      cast<VectorType>(Inst->getOperand(StartIdx)->getType())
          ->getNumElements());
  SmallVector<Value *, 4> Opnds;
  for (unsigned Idx = 0; Idx != EndIdx; ++Idx) {
    Value *Opnd = Inst->getOperand(Idx);
    if (Idx >= StartIdx) {
      if (auto C = dyn_cast<Constant>(Opnd))
        Opnd = ConstantExpr::getCast(ExtOpcode, C, ExtTy);
      else {
        auto NewExt = CastInst::Create(ExtOpcode, Opnd, ExtTy,
                                       Inst->getName() + ".byteext", Inst);
        NewExt->setDebugLoc(Inst->getDebugLoc());
        Opnd = NewExt;
      }
    }
    Opnds.push_back(Opnd);
  }
  // Create the replacement instruction.
  Instruction *NewInst = nullptr;
  if (isa<BinaryOperator>(Inst))
    NewInst = BinaryOperator::Create((Instruction::BinaryOps)Inst->getOpcode(),
                                     Opnds[0], Opnds[1], "", Inst);
  else if (auto CI = dyn_cast<CmpInst>(Inst))
    NewInst = CmpInst::Create(CI->getOpcode(), CI->getPredicate(), Opnds[0],
                              Opnds[1], "", CI);
  else if (isa<SelectInst>(Inst))
    NewInst = SelectInst::Create(Opnds[0], Opnds[1], Opnds[2], "", Inst);
  else
    IGC_ASSERT_EXIT_MESSAGE(0, "unhandled instruction in widenByteOp");
  NewInst->takeName(Inst);
  NewInst->setDebugLoc(Inst->getDebugLoc());
  if (ExtendedIsZero) {
    // We know that the extended part of the result contains 0 bits. If we
    // find that any use is a zext (probably from also being byte widened
    // in this code), we can replace the use directly and save the
    // trunc/zext pair. First put the uses in a vector as the use list will
    // change under our feet.
    SmallVector<Use *, 4> Uses;
    for (auto ui = Inst->use_begin(), ue = Inst->use_end(); ui != ue; ++ui)
      Uses.push_back(&*ui);
    for (auto ui = Uses.begin(), ue = Uses.end(); ui != ue; ++ui) {
      if (auto user = dyn_cast<ZExtInst>((*ui)->getUser())) {
        if (user->getType() == NewInst->getType()) {
          user->replaceAllUsesWith(NewInst);
          ToErase.push_back(user);
          // Remove the use of Inst from the trunc so we can tell whether there
          // are any uses left below.
          *(*ui) = UndefValue::get(Inst->getType());
        }
      }
    }
  }
  if (!Inst->use_empty()) {
    // Truncate the result.
    if (!isa<CmpInst>(Inst)) {
      NewInst = CastInst::Create(Instruction::Trunc, NewInst, Inst->getType(),
                                 Inst->getName() + ".bytetrunc", Inst);
      NewInst->setDebugLoc(Inst->getDebugLoc());
    }
    // Replace uses.
    Inst->replaceAllUsesWith(NewInst);
  }
  ToErase.push_back(Inst);
  return true;
}

namespace {

// Helper class to translate load/store into proper GenX intrinsic calls.
class LoadStoreResolver {
  Instruction *Inst;
  const GenXSubtarget *ST;
  IRBuilder<> Builder;

public:
  LoadStoreResolver(Instruction *Inst, const GenXSubtarget *ST)
      : Inst(Inst), ST(ST), Builder(Inst) {}

  // Resolve this instruction and return true on success.
  bool resolve();

private:
  bool isLoad() const { return isa<LoadInst>(Inst); }
  bool isStore() const { return isa<StoreInst>(Inst); }

  const DataLayout &getDL() const {
    Function *F = Inst->getParent()->getParent();
    return F->getParent()->getDataLayout();
  }

  // Find a proper GenX intrinsic ID for this load/store instruction.
  GenXIntrinsic::ID getGenXIntrinsicID() const;

  unsigned getPointerSizeInBits() const {
    unsigned AddrSp = 0;
    if (auto LI = dyn_cast<LoadInst>(Inst))
      AddrSp = LI->getPointerAddressSpace();
    else if (auto SI = dyn_cast<StoreInst>(Inst))
      AddrSp = SI->getPointerAddressSpace();
    return getDL().getPointerSizeInBits(AddrSp);
  }

  unsigned getValueSizeInBits(Type *T) const {
    if (auto PT = dyn_cast<PointerType>(T)) {
      unsigned AddrSp = PT->getAddressSpace();
      return getDL().getPointerSizeInBits(AddrSp);
    }
    return T->getPrimitiveSizeInBits();
  }

  // Return true if this load/store can be translated.
  bool isSupported() const;

  // Emit actual intrinsic calls.
  bool emitGather();
  bool emitScatter();
  bool emitSVMGather();
  bool emitSVMScatter();
};

} // namespace

// Translate store instructions into genx builtins.
bool GenXLowering::lowerLoadStore(Instruction *Inst) {
  LoadStoreResolver Resolver(Inst, ST);
  if (Resolver.resolve()) {
    ToErase.push_back(Inst);
    return true;
  }
  return false;
}

bool LoadStoreResolver::resolve() {
  if (!isSupported())
    return false;

  GenXIntrinsic::ID ID = getGenXIntrinsicID();
  switch (ID) {
  case GenXIntrinsic::genx_gather_scaled:
    return emitGather();
  case GenXIntrinsic::genx_scatter_scaled:
    return emitScatter();
  case GenXIntrinsic::genx_svm_gather:
    return emitSVMGather();
  case GenXIntrinsic::genx_svm_scatter:
    return emitSVMScatter();
  default:
    break;
  }

  return false;
}

// Return true if this load/store can be translated.
bool LoadStoreResolver::isSupported() const {
  auto IsGlobalLoadStore = [=]() {
    Value *Ptr = nullptr;
    if (auto LI = dyn_cast<LoadInst>(Inst))
      Ptr = LI->getPointerOperand();
    if (auto SI = dyn_cast<StoreInst>(Inst))
      Ptr = SI->getPointerOperand();
    return getUnderlyingGlobalVariable(Ptr) != nullptr;
  };

  if (IsGlobalLoadStore())
    return false;

  Type *ValTy = Inst->getType();
  if (auto SI = dyn_cast<StoreInst>(Inst))
    ValTy = SI->getValueOperand()->getType();

  // Only scalar data types.
  if (!ValTy->isFloatingPointTy() && !ValTy->isIntegerTy() &&
      !ValTy->isPointerTy()) {
    Inst->getContext().emitError(Inst, "unsupported type for load/store");
    return false;
  }

  // Only legal types: float, double, half, i8, i16, 132, i64, pointer types.
  unsigned NumBits = getValueSizeInBits(ValTy);
  if (NumBits < 8 || NumBits > 64 || !isPowerOf2_32(NumBits)) {
    Inst->getContext().emitError("unsupported integer type for load/store");
    return false;
  }

  // Translate this instruction.
  return true;
}

// Find a proper GenX intrinsic ID for this load/store instruction.
GenXIntrinsic::ID LoadStoreResolver::getGenXIntrinsicID() const {
  // A32 byte scattered stateless messages only work on CNL+.
  unsigned NBits = getPointerSizeInBits();
  if (NBits == 32 && ST && !ST->WaNoA32ByteScatteredStatelessMessages())
    return isLoad() ? GenXIntrinsic::genx_gather_scaled
                    : GenXIntrinsic::genx_scatter_scaled;
  return isLoad() ? GenXIntrinsic::genx_svm_gather
                  : GenXIntrinsic::genx_svm_scatter;
}

bool LoadStoreResolver::emitGather() {
  unsigned NBits = getPointerSizeInBits();
  Type *IntTy = IntegerType::get(Inst->getContext(), NBits);
  auto LI = cast<LoadInst>(Inst);

  // Global offset.
  Value *Addr = LI->getPointerOperand();
  Addr = Builder.CreatePtrToInt(Addr, IntTy);

  unsigned NBlocks = getValueSizeInBits(LI->getType()) / 8;
  unsigned NBlocksLog2 = llvm::Log2_32(NBlocks);

  // If this is more than 4 bytes, use a larger SIMD size.
  unsigned SIMD = 1;
  if (NBlocks > 4) {
    SIMD = NBlocks / 4;
    NBlocksLog2 = 2;
  }

  // The old value is undef.
  Type *ValTy = LI->getType();
  if (ValTy->isPointerTy())
    ValTy = Builder.getIntNTy(getValueSizeInBits(ValTy));
  Type *DataTy = IGCLLVM::FixedVectorType::get(ValTy, 1);
  if (SIMD > 1)
    DataTy = IGCLLVM::FixedVectorType::get(Builder.getInt32Ty(), SIMD);
  Value *OldVal = UndefValue::get(DataTy);

  // Offset.
  Type *EltOffsetTy = IGCLLVM::FixedVectorType::get(Builder.getInt32Ty(), SIMD);
  Value *EltOffset = Constant::getNullValue(EltOffsetTy);
  if (SIMD > 1) {
    SmallVector<uint32_t, 2> Offsets(SIMD);
    for (unsigned i = 0; i < SIMD; ++i)
      // Increase offset by 4 bytes for each lane.
      Offsets[i] = i * 4;
    EltOffset = ConstantDataVector::get(Inst->getContext(), Offsets);
  }

  // Arguments.
  Value *Args[] = {
      Constant::getAllOnesValue(
          IGCLLVM::FixedVectorType::get(Builder.getInt1Ty(), SIMD)),
      Builder.getInt32(NBlocksLog2), // log[2](NBlocks)
      Builder.getInt16(0),           // scale
      Builder.getInt32(visa::getReservedSurfaceIndex(
          PreDefined_Surface::PREDEFINED_SURFACE_T255)), // surface
      Addr,                                              // global offset
      EltOffset,                                         // element offset
      OldVal                                             // old value
  };

  // Overload with return type, predicate type and element offset type
  Type *Tys[] = {OldVal->getType(), Args[0]->getType(), EltOffsetTy};
  Module *M = Inst->getParent()->getParent()->getParent();
  auto Fn = GenXIntrinsic::getGenXDeclaration(M, GenXIntrinsic::genx_gather_scaled, Tys);

  Value *NewVal = Builder.CreateCall(Fn, Args);
  NewVal = Builder.CreateBitCast(NewVal, ValTy);
  LI->replaceAllUsesWith(NewVal);
  return true;
}

bool LoadStoreResolver::emitScatter() {
  unsigned NBits = getPointerSizeInBits();
  Type *IntTy = IntegerType::get(Inst->getContext(), NBits);
  auto SI = cast<StoreInst>(Inst);

  // Global offset.
  Value *Addr = SI->getPointerOperand();
  Addr = Builder.CreatePtrToInt(Addr, IntTy);

  Value *Val = SI->getValueOperand();
  unsigned NBlocks = getValueSizeInBits(Val->getType()) / 8;
  unsigned NBlocksLog2 = llvm::Log2_32(NBlocks);

  // If this is more than 4 bytes, use a larger SIMD size.
  unsigned SIMD = 1;
  if (NBlocks > 4) {
    SIMD = NBlocks / 4;
    NBlocksLog2 = 2;
  }

  // Value to write.
  Type *ValTy = (SIMD > 1) ? Builder.getInt32Ty() : Val->getType();
  if (ValTy->isPointerTy())
    ValTy = Builder.getIntNTy(getValueSizeInBits(ValTy));
  Val = Builder.CreateBitCast(Val, IGCLLVM::FixedVectorType::get(ValTy, SIMD));

  // Offset.
  Type *EltOffsetTy = IGCLLVM::FixedVectorType::get(Builder.getInt32Ty(), SIMD);
  Value *EltOffset = Constant::getNullValue(EltOffsetTy);
  if (SIMD > 1) {
    SmallVector<uint32_t, 2> Offsets(SIMD);
    // Increase offset by 4 bytes for each lane.
    for (unsigned i = 0; i < SIMD; ++i)
      Offsets[i] = i * 4;
    EltOffset = ConstantDataVector::get(Inst->getContext(), Offsets);
  }

  // Arguments.
  Value *Args[] = {
      Constant::getAllOnesValue(
          IGCLLVM::FixedVectorType::get(Builder.getInt1Ty(), SIMD)),
      Builder.getInt32(NBlocksLog2), // log[2](NBlocks)
      Builder.getInt16(0),           // scale
      Builder.getInt32(visa::getReservedSurfaceIndex(
          PreDefined_Surface::PREDEFINED_SURFACE_T255)), // surface
      Addr,                                              // global offset
      EltOffset,                                         // element offset
      Val                                                // value to write
  };

  // Overload with predicate type, element offset type, value to write type.
  Type *Tys[] = {Args[0]->getType(), EltOffsetTy, Val->getType()};
  Module *M = Inst->getParent()->getParent()->getParent();
  auto Fn = GenXIntrinsic::getGenXDeclaration(M, GenXIntrinsic::genx_scatter_scaled, Tys);
  Builder.CreateCall(Fn, Args);
  return true;
}

// Compute the block size and the number of blocks for svm gather/scatter.
//
// Block_Size, 1, 4, 8
// Num_Blocks, 1, 2, 4,
//             8 only valid for 4 byte blocks and execution size 8.
//
static unsigned getBlockCount(Type *Ty) {
  unsigned NumBytes = Ty->getPrimitiveSizeInBits() / 8;
  IGC_ASSERT_MESSAGE(NumBytes <= 8, "out of sync");

  // If this is N = 2 byte data, use 2 blocks;
  // otherwise, use 1 block of N bytes.
  return (NumBytes == 2) ? NumBytes : 1U;
}

// Translate store to svm scatter.
bool LoadStoreResolver::emitSVMGather() {
  unsigned NBits = getPointerSizeInBits();
  Type *IntTy = IntegerType::get(Inst->getContext(), NBits);
  auto LI = cast<LoadInst>(Inst);

  // Address.
  Value *Addr = LI->getPointerOperand();
  Addr = Builder.CreatePtrToInt(Addr, IntTy);
  if (NBits == 32)
    Addr = Builder.CreateZExt(Addr, Builder.getInt64Ty());
  Addr = Builder.CreateBitCast(
      Addr, IGCLLVM::FixedVectorType::get(Addr->getType(), 1));

  // The old value is undef.
  Type *ValTy = LI->getType();
  if (ValTy->isPointerTy())
    ValTy = Builder.getIntNTy(getValueSizeInBits(ValTy));
  Type *DataTy = IGCLLVM::FixedVectorType::get(ValTy, 1);
  Value *OldVal = UndefValue::get(DataTy);

  // Num of blocks.
  unsigned NBlocks = getBlockCount(OldVal->getType());
  unsigned NBlocksLog2 = llvm::Log2_32(NBlocks);

  Value *Args[] = {
      Constant::getAllOnesValue(
          IGCLLVM::FixedVectorType::get(Builder.getInt1Ty(), 1)),
      Builder.getInt32(NBlocksLog2), // log2(num_of_blocks)
      Addr,                          // addresses
      OldVal                         // old value
  };

  // Overload with return type, predicate type and address vector type
  Type *Tys[] = {OldVal->getType(), Args[0]->getType(), Addr->getType()};
  Module *M = Inst->getParent()->getParent()->getParent();
  auto Fn = GenXIntrinsic::getGenXDeclaration(M, GenXIntrinsic::genx_svm_gather, Tys);

  Value *NewVal = Builder.CreateCall(Fn, Args);
  NewVal = Builder.CreateBitCast(NewVal, ValTy);
  if (LI->getType()->isPointerTy())
    NewVal = Builder.CreateIntToPtr(NewVal, LI->getType());
  LI->replaceAllUsesWith(NewVal);
  return true;
}

bool LoadStoreResolver::emitSVMScatter() {
  unsigned NBits = getPointerSizeInBits();
  Type *IntTy = IntegerType::get(Inst->getContext(), NBits);
  auto SI = cast<StoreInst>(Inst);

  // Address
  Value *Addr = SI->getPointerOperand();
  Addr = Builder.CreatePtrToInt(Addr, IntTy);
  if (NBits == 32)
    Addr = Builder.CreateZExt(Addr, Builder.getInt64Ty());
  Addr = Builder.CreateBitCast(
      Addr, IGCLLVM::FixedVectorType::get(Addr->getType(), 1));

  // data to write.
  Value *Val = SI->getValueOperand();
  Type *ValTy = Val->getType();
  if (ValTy->isPointerTy()) {
    ValTy = Builder.getIntNTy(getValueSizeInBits(ValTy));
    Val = Builder.CreatePtrToInt(Val, ValTy);
  }
  Val = Builder.CreateBitCast(Val, IGCLLVM::FixedVectorType::get(ValTy, 1));

  // Num of blocks.
  unsigned NBlocks = getBlockCount(Val->getType());
  unsigned NBlocksLog2 = llvm::Log2_32(NBlocks);

  Value *Args[] = {
      Constant::getAllOnesValue(
          IGCLLVM::FixedVectorType::get(Builder.getInt1Ty(), 1)),
      Builder.getInt32(NBlocksLog2), // log2(num_of_blocks)
      Addr,                          // addresses
      Val                            // value to write
  };

  // Overload with predicate type, address vector type, and data type
  Type *Tys[] = {Args[0]->getType(), Addr->getType(), Val->getType()};
  Module *M = Inst->getParent()->getParent()->getParent();
  auto Fn = GenXIntrinsic::getGenXDeclaration(M, GenXIntrinsic::genx_svm_scatter, Tys);

  Builder.CreateCall(Fn, Args);
  return true;
}
