compute_op.cc 21.7 KB
Newer Older
1 2 3 4 5 6 7 8
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
9
 *
10
 *   http://www.apache.org/licenses/LICENSE-2.0
11
 *
12 13 14 15 16 17 18 19
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

20 21 22 23
/*!
 * \brief Compute Op.
 * \file compute_op.cc
 */
24
#include <tvm/runtime/registry.h>
25
#include <tvm/te/operation.h>
26
#include <tvm/arith/analyzer.h>
27 28 29
#include <tvm/tir/expr.h>
#include <tvm/tir/ir_pass.h>
#include <tvm/tir/stmt_functor.h>
30
#include <unordered_set>
31
#include <string>
32
#include <utility>
33 34
#include "compute_op.h"
#include "op_util.h"
35
#include "../schedule/message_passing.h"
36 37
#include "../../arith/compute_expr.h"
#include "../../arith/interval_set.h"
38 39

namespace tvm {
40
namespace te {
41
using namespace tir;
42

43 44
TVM_STATIC_IR_FUNCTOR(ReprPrinter, vtable)
.set_dispatch<ComputeOpNode>([](const ObjectRef& node, ReprPrinter* p) {
45
    auto* op = static_cast<const ComputeOpNode*>(node.get());
46 47 48 49 50
    p->stream << "compute(" << op->name << ", " << op << ")";
});

TVM_REGISTER_NODE_TYPE(ComputeOpNode);

51 52 53
/// Verify if ComputeOp is valid with respect to Reduce operations.
static void VerifyComputeOp(const ComputeOpNode *op);

54
inline bool ReduceEqual(const tir::ReduceNode* a, const tir::ReduceNode* b) {
55 56 57 58 59 60
  return (a->combiner.same_as(b->combiner)) &&
         (a->source.same_as(b->source)) &&
         (a->axis.same_as(b->axis)) &&
         (a->condition.same_as(b->condition));
}

61
int ComputeOpNode::num_outputs() const {
62
  return body.size();
63 64
}

65
Array<IterVar> BaseComputeOpNode::root_iter_vars() const {
66 67 68 69 70 71 72 73
  if (reduce_axis.size() == 0) return axis;
  Array<IterVar> ret = axis;
  for (IterVar iv : reduce_axis) {
    ret.push_back(iv);
  }
  return ret;
}

74
DataType ComputeOpNode::output_dtype(size_t idx) const {
75
  CHECK_LT(idx, num_outputs());
76
  return body[idx].dtype();
77 78
}

79
Array<PrimExpr> BaseComputeOpNode::output_shape(size_t idx) const {
80
  CHECK_LT(idx, num_outputs());
81
  // for now, all outputs of a BaseComputeOp have the same shape
82
  Array<PrimExpr> shape;
83 84
  for (const auto& ivar : this->axis) {
    const Range& r = ivar->dom;
85 86
    shape.push_back(r->extent);
  }
87
  return shape;
88 89
}

90
Tensor compute(Array<PrimExpr> shape,
91 92
               FCompute fcompute,
               std::string name,
93
               std::string tag,
94 95
               Map<std::string, ObjectRef> attrs) {
  auto op_node = make_object<ComputeOpNode>();
96 97 98 99 100 101 102 103
  // compute dimension.
  size_t ndim = shape.size();
  std::vector<IterVar> axis;
  std::vector<Var> args;
  for (size_t i = 0; i < ndim; ++i) {
    std::ostringstream os;
    os << "ax" << i;
    axis.emplace_back(IterVarNode::make(
104
        Range(0, shape[i]), Var(os.str(), shape[i].dtype()), kDataPar));
105 106 107
    args.push_back(axis.back()->var);
  }

108 109
  return ComputeOpNode::make(
      name, tag, attrs, axis, {fcompute(args)}).output(0);
110 111
}

112
Array<Tensor> compute(Array<PrimExpr> shape,
113 114
                      FBatchCompute fcompute,
                      std::string name,
115
                      std::string tag,
116 117
                      Map<std::string, ObjectRef> attrs) {
  auto op_node = make_object<ComputeOpNode>();
118 119 120 121 122 123 124 125
  // compute dimension.
  size_t ndim = shape.size();
  std::vector<IterVar> axis;
  std::vector<Var> args;
  for (size_t i = 0; i < ndim; ++i) {
    std::ostringstream os;
    os << "ax" << i;
    axis.emplace_back(IterVarNode::make(
126
        Range(0, shape[i]), Var(os.str(), shape[i].dtype()), kDataPar));
127 128 129
    args.push_back(axis.back()->var);
  }

130
  Operation op = ComputeOpNode::make(name, tag, attrs, axis, fcompute(args));
131 132 133 134 135 136 137
  Array<Tensor> outputs;
  for (int idx = 0; idx < op->num_outputs(); ++idx) {
    outputs.push_back(op.output(idx));
  }
  return outputs;
}

138
Operation ComputeOpNode::make(std::string name,
139
                              std::string tag,
140
                              Map<std::string, ObjectRef> attrs,
141
                              Array<IterVar> axis,
142
                              Array<PrimExpr> body) {
143
  if (!attrs.defined()) {
144
    attrs = Map<std::string, ObjectRef>();
145
  }
146
  auto n = make_object<ComputeOpNode>();
147 148 149 150 151
  n->name = std::move(name);
  n->tag = std::move(tag);
  n->attrs = std::move(attrs);
  n->axis = std::move(axis);
  n->body = std::move(body);
152 153
  if (n->body[0]->IsInstance<tir::ReduceNode>()) {
    const tir::ReduceNode* reduce = n->body[0].as<tir::ReduceNode>();
154
    n->reduce_axis = reduce->axis;
155
  }
156
  VerifyComputeOp(n.get());
157 158 159
  return Operation(n);
}

160 161 162 163
TVM_REGISTER_GLOBAL("te.ComputeOp")
.set_body_typed(ComputeOpNode::make);


164 165 166 167
// The schedule related logics
Array<Tensor> ComputeOpNode::InputTensors() const {
  Array<Tensor> ret;
  std::unordered_set<Tensor> visited;
168
  for (auto& e : body) {
169 170
    tir::PostOrderVisit(e, [&ret, &visited](const ObjectRef& n) {
        const tir::CallNode *call = n.as<tir::CallNode>();
171
        if (call != nullptr && call->func.defined()) {
172
          Tensor t = Downcast<Operation>(call->func).output(call->value_index);
173 174 175 176
          if (!visited.count(t)) {
            ret.push_back(t);
            visited.insert(t);
          }
177
        }
178 179
      });
  }
180 181 182 183 184 185 186
  return ret;
}

Operation ComputeOpNode::ReplaceInputs(
    const Operation& self,
    const std::unordered_map<Tensor, Tensor>& rmap) const {
  CHECK_EQ(self.operator->(), this);
187
  VerifyComputeOp(this);
188
  Array<PrimExpr> arr;
189
  if (this->body[0]->IsInstance<tir::ReduceNode>()) {
190 191
    // Specially handle reduce so the replaced op
    // still share all the components
192
    PrimExpr new_reduce = te::ReplaceTensor(this->body[0], rmap);
193
    if (!new_reduce.same_as(this->body[0])) {
194
      const tir::ReduceNode* r = new_reduce.as<tir::ReduceNode>();
195
      for (size_t k = 0; k < this->body.size(); ++k) {
196
        auto n = make_object<tir::ReduceNode>(*r);
197
        n->value_index = static_cast<int>(k);
198
        n->dtype = r->source[k].dtype();
199
        arr.push_back(PrimExpr(n));
200 201 202 203 204
      }
    } else {
      arr = this->body;
    }
  } else {
205
    arr = UpdateArray(this->body, [&rmap] (const PrimExpr& e) {
206
        return te::ReplaceTensor(e, rmap);
207 208
      });
  }
209
  if (!arr.same_as(this->body)) {
210 211
    return ComputeOpNode::make(
        this->name, this->tag, this->attrs, this->axis, arr);
212 213 214 215 216 217 218
  } else {
    return self;
  }
}

void ComputeOpNode::PropBoundToInputs(
    const Operation& self,
219
    arith::Analyzer* analyzer,
220
    const std::unordered_map<const VarNode*, IntSet>& dom_map,
221 222
    std::unordered_map<Tensor, TensorDom>* out_dom_map) const {
  CHECK_EQ(self.operator->(), this);
223
  auto fvisit = [&dom_map, out_dom_map, analyzer](const ObjectRef& n) {
224
    auto *call = n.as<tir::CallNode>();
225
    if (call != nullptr && call->func.defined()) {
226
      Tensor t = Downcast<Operation>(call->func).output(call->value_index);
227 228 229
      if (t->op.defined() && out_dom_map->count(t)) {
        TensorDom& dom = out_dom_map->at(t);
        for (size_t i = 0; i < t.ndim(); ++i) {
230 231 232 233 234 235 236
          // We assume that the value of the argument cannot be out of bounds (otherwise it is
          // undefined behaviour), so we can intersect the estimated set of the argument with the
          // range expected by the tensor. However, intersection may result in overly complex
          // expressions, so we perform a more relaxed form of intersection.
          IntSet arg_intset = EvalSet(call->args[i], dom_map);
          const arith::IntervalSetNode* arg_interval = arg_intset.as<arith::IntervalSetNode>();
          if (arg_interval) {
237 238 239 240
            PrimExpr shape_i_min_value = make_zero(t->shape[i].dtype());
            PrimExpr shape_i_max_value = t->shape[i] - 1;
            PrimExpr min_value = arg_interval->min_value;
            PrimExpr max_value = arg_interval->max_value;
241 242 243 244 245 246 247 248 249 250 251 252 253
            // Prefer the shape bounds only when we can prove they are tighter.
            if (arith::is_neg_inf(min_value) ||
                analyzer->CanProve(shape_i_min_value >= min_value)) {
              min_value = shape_i_min_value;
            }
            if (arith::is_pos_inf(max_value) ||
                analyzer->CanProve(shape_i_max_value <= max_value)) {
              max_value = shape_i_max_value;
            }
            dom.data[i].push_back(IntSet::interval(min_value, max_value));
          } else {
            dom.data[i].push_back(arg_intset);
          }
254 255 256 257
        }
      }
    }
  };
258
  for (auto& e : body) tir::PostOrderVisit(e, fvisit);
259 260
}

261
void BaseComputeOpNode::GatherBound(
262 263 264
    const Operation& self,
    const std::unordered_map<Tensor, TensorDom>& tensor_dom,
    std::unordered_map<IterVar, Range>* out_dom_map) const {
265
  CHECK_EQ(self.operator->(), this);
266 267 268 269 270 271 272 273 274 275 276 277
  const TensorDom& tdom = tensor_dom.at(self.output(0));
  for (size_t i = 0; i < this->axis.size(); ++i) {
    Range r = arith::Union(tdom.data.at(i)).cover_range(this->axis[i]->dom);
    CHECK(!out_dom_map->count(this->axis[i]));
    (*out_dom_map)[this->axis[i]] = r;
  }
  for (size_t i = 0; i < this->reduce_axis.size(); ++i) {
    CHECK(!out_dom_map->count(this->reduce_axis[i]));
    (*out_dom_map)[this->reduce_axis[i]] = this->reduce_axis[i]->dom;
  }
}

278
Stmt BaseComputeOpNode::BuildRealize(
279
    const Stage& stage,
280
    const std::unordered_map<IterVar, Range>& realize_map,
281
    const Stmt& body) const {
282
  CHECK_EQ(stage->op.get(), this);
283
  Region bounds;
284 285 286
  for (IterVar iv : this->axis) {
    bounds.push_back(realize_map.at(iv));
  }
287
  Stmt realize = body;
288 289
  for (int i = this->num_outputs(); i > 0; --i) {
    Tensor t = stage->op.output(i-1);
290
    realize = tir::RealizeNode::make(t->op, t->value_index,
291
      t->dtype, bounds, const_true(), realize);
292
    // alignment requirement, only useful for compute
293
    for (size_t i = 0; i < num_schedulable_dims(); ++i) {
294 295 296 297
      auto it = stage->iter_var_attrs.find(this->axis[i]);
      if (it != stage->iter_var_attrs.end()) {
        IterVarAttr attr = (*it).second;
        if (attr->dim_align_factor != 0) {
298
          Array<PrimExpr> tuple = {static_cast<int>(i),
299 300
                               attr->dim_align_factor,
                               attr->dim_align_offset};
301 302
          realize = tir::AttrStmtNode::make(
              t, tir::attr::buffer_dim_align,
303
              CallNode::make(DataType::Handle(),
304
                             tir::intrinsic::tvm_tuple,
305
                             tuple, CallNode::Intrinsic),
306 307 308 309
              realize);
        }
      }
    }
310 311
  }
  return realize;
312 313
}

314 315 316 317
size_t ComputeOpNode::num_schedulable_dims() const {
  return axis.size();
}

318 319
// Build a reduction body.
void MakeReduction(const ComputeOpNode* op,
320
                   const Array<Tensor>& tensors,
321 322
                   Stmt* init,
                   Stmt* provide) {
323
  Array<PrimExpr>  args;
324 325 326
  for (IterVar iv : op->axis) {
    args.push_back(iv->var);
  }
327 328 329
  std::vector<Stmt> inits, provides;

  size_t size = op->body.size();
330
  const ReduceNode* reduce = op->body[0].as<ReduceNode>();
331
  CHECK(reduce);
ziheng committed
332 333
  const CommReducerNode* combiner = reduce->combiner.as<CommReducerNode>();
  CHECK(combiner);
334
  Array<PrimExpr> lhs;
335 336 337
  for (size_t i = 0; i < size; ++i) {
    lhs.push_back(tensors[i](args));
  }
338 339
  Array<PrimExpr> init_value = combiner->identity_element;
  Array<PrimExpr> update_value = (*combiner)(lhs, reduce->source);
340 341
  for (size_t i = 0; i < size; ++i) {
    Tensor t = tensors[i];
342
    inits.emplace_back(ProvideNode::make(
343
          t->op, t->value_index, init_value[i], args));
344
    provides.emplace_back(ProvideNode::make(
345 346
          t->op, t->value_index, update_value[i], args));
  }
347 348
  *init = SeqStmt::Flatten(inits);
  *provide = SeqStmt::Flatten(provides);
349
  if (!is_one(reduce->condition)) {
350
    *provide = IfThenElseNode::make(reduce->condition, *provide);
351
  }
352 353
}

354
// Normal computation.
355 356
Stmt MakeProvide(const ComputeOpNode* op,
                 const Tensor& t) {
357
  Array<PrimExpr> args;
358 359 360
  for (IterVar iv : op->axis) {
    args.push_back(iv->var);
  }
361
  return ProvideNode::make(t->op, t->value_index, op->body[t->value_index], args);
362 363
}

364 365
Stmt MakeComputeStmt(const ComputeOpNode* self,
                     const Stage& stage,
366
                     const std::unordered_map<IterVar, Range>& dom_map,
367
                     bool debug_keep_trivial_loop) {
368
  // grab the nest structure
369
  ComputeLoopNest n = ComputeLoopNest::make(self, stage, dom_map, debug_keep_trivial_loop);
370
  // Normal loop structure
371 372
  n.init_nest.emplace_back(MakeIfNest(n.init_predicates));
  n.main_nest.emplace_back(MakeIfNest(n.main_predicates));
373 374 375 376 377 378 379 380 381
  if (self->reduce_axis.size() != 0) {
    // make reduction.
    Stmt init, provide;
    Array<Tensor> source;
    for (size_t i = 0; i < self->body.size(); ++i) {
      source.push_back(stage->op.output(i));
    }
    MakeReduction(self, source, &init, &provide);
    init = MergeNest(n.init_nest, init);
382
    init = Substitute(init, n.init_vmap);
383 384 385 386 387 388
    // common nest
    std::vector<std::vector<Stmt> > common(
        n.main_nest.begin(), n.main_nest.begin() + n.num_common_loop + 1);
    std::vector<std::vector<Stmt> > reduce(
        n.main_nest.begin() + n.num_common_loop + 1, n.main_nest.end());
    provide = MergeNest(reduce, provide);
389
    if (debug_keep_trivial_loop) {
390
      provide = MergeNest(common, provide);
391
    } else {
392
      provide = MergeNest(common, SeqStmt::Flatten(init, provide));
393
    }
394 395
    // run substitution in the on the full nest, because  loop condition
    // could depend on outer loops.
396
    return Substitute(provide, n.main_vmap);
397 398 399 400 401
  } else {
    std::vector<Stmt> provides;
    for (size_t i = 0; i < self->body.size(); ++i) {
      provides.emplace_back(MakeProvide(self, stage->op.output(i)));
    }
402
    Stmt provide = SeqStmt::Flatten(provides);
403 404 405
    provide = MergeNest(n.main_nest, provide);
    // run substitution in the on the full nest, because  loop condition
    // could depend on outer loops.
406
    return Substitute(provide, n.main_vmap);
407 408
  }
}
409

410 411 412 413 414 415 416
enum class ComputeType {
  kNormal,
  kCrossThreadReduction,
  kTensorize
};

ComputeType DetectComputeType(const ComputeOpNode* self,
417
                              const Stage& stage) {
418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454
  // Verify correctness of leaf nest.
  int normal_red = 0, thread_red = 0, tensorize = 0;

  for (IterVar iv : stage->leaf_iter_vars) {
    IterVarAttr attr;
    auto it = stage->iter_var_attrs.find(iv);
    if (it != stage->iter_var_attrs.end()) {
      attr = (*it).second;
    }
    if (attr.defined() && attr->iter_type == kTensorized) {
      ++tensorize;
    }
    if (iv->iter_type == kCommReduce) {
      if (attr.defined() && attr->bind_thread.defined()) {
        ++thread_red;
      } else {
        ++normal_red;
      }
    } else {
      CHECK_EQ(thread_red, 0)
          << "Cross thread reduce cannot swap with normal data axis";
    }
  }
  if (tensorize != 0) {
    CHECK(thread_red == 0)
        << "Cannot mix cross thread reduction with Tensorize";
    return ComputeType::kTensorize;
  }
  CHECK(normal_red == 0 || thread_red == 0)
      << "Cannot mix normal reduction with thread reduce";
  if (thread_red != 0) {
    return ComputeType::kCrossThreadReduction;
  } else {
    return ComputeType::kNormal;
  }
}

455 456 457
// implement the provide utility.
Stmt ComputeOpNode::BuildProvide(
    const Stage& stage,
458
    const std::unordered_map<IterVar, Range>& dom_map,
459
    bool debug_keep_trivial_loop) const {
460
  CHECK_EQ(stage->op.operator->(), this);
461 462
  ComputeType ctype = DetectComputeType(this, stage);
  if (ctype == ComputeType::kCrossThreadReduction) {
463
    // specially handle cross thread reduction.
464
    return MakeCrossThreadReduction(this, stage, dom_map, debug_keep_trivial_loop);
465
  } else if (ctype == ComputeType::kTensorize) {
466
    return MakeTensorize(this, stage, dom_map, debug_keep_trivial_loop);
467
  } else {
468
    return MakeComputeStmt(this, stage, dom_map, debug_keep_trivial_loop);
469 470 471 472
  }
}

ComputeLoopNest ComputeLoopNest::make(
473
    const BaseComputeOpNode* self,
474
    const Stage& stage,
475
    const std::unordered_map<IterVar, Range>& dom_map,
476
    bool debug_keep_trivial_loop) {
477 478 479
  CHECK_EQ(stage->op.operator->(), self);
  ComputeLoopNest ret;
  // make main loop nest
480
  ret.main_nest = MakeLoopNest(
481 482
      stage, dom_map, 0, false, std::unordered_set<IterVar>(), &ret.main_vmap,
      debug_keep_trivial_loop);
483
  ret.main_predicates = MakeBoundCheck(
484 485
      stage, dom_map, ret.main_vmap, false,
      std::unordered_set<IterVar>());
486 487
  for (auto& e : ret.main_predicates) {
    e = likely(e);
488
  }
489
  if (stage->store_predicate.defined()) {
490
    ret.main_predicates.push_back(stage->store_predicate);
491
  }
492
  if (self->reduce_axis.size() != 0) {
493 494 495
    // try to find the location to insert the initialization.
    // Fuse the initialization and provide loop when possible.
    std::unordered_map<IterVar, int> update_state;
496
    for (IterVar iv : self->reduce_axis) {
497 498
      update_state[iv] = 2;
    }
499 500
    for (size_t i = 0; i < self->num_schedulable_dims(); ++i) {
      update_state[self->axis[i]] = 1;
501 502
    }
    // find which iter var is related to reduction and which is related to axis.
503
    te::PassDownBitMaskOr(stage, &update_state);
504 505 506 507 508 509 510 511 512
    auto leaf_iter_vars = stage->leaf_iter_vars;
    // first first loop that is related to reduction.
    size_t begin_loop = leaf_iter_vars.size();
    for (size_t i = 0; i < leaf_iter_vars.size(); ++i) {
      auto iv = leaf_iter_vars[i];
      int flag = update_state.at(iv);
      if ((flag & 2) != 0) {
        begin_loop = i; break;
      }
513
      ret.init_vmap[iv] = ret.main_vmap.at(iv);
514
    }
515
    ret.num_common_loop = begin_loop;
516
    // skip loops that are related to reduction and are unrelated to axis.
517 518 519
    std::unordered_set<IterVar> skip_iter;
    for (auto kv : update_state) {
      int flag = kv.second;
520
      if (flag == 2) skip_iter.insert(kv.first);
521
    }
522
    ret.init_nest = MakeLoopNest(
523
        stage, dom_map, begin_loop, true,
524
        skip_iter, &(ret.init_vmap), debug_keep_trivial_loop);
525
    ret.init_predicates = MakeBoundCheck(
526
        stage, dom_map, ret.init_vmap, true, skip_iter);
527 528 529 530
    for (auto& e : ret.init_predicates) {
      e = likely(e);
    }
  } else {
531 532
    CHECK_EQ(ret.main_nest.size(), stage->leaf_iter_vars.size() + 1);
    ret.num_common_loop = stage->leaf_iter_vars.size();
533 534 535 536
  }
  // copy elison here.
  return ret;
}
537 538 539 540 541 542 543 544 545 546 547

namespace {
/*!
 * \brief Verify if ComputeOp is valid with respect to Reduce operations.
 *
 *  The following two properties are verified:
 *  (1) All Reduce operations must exist at top level.
 *  (2) For a list of operations, if one is Reduce, then the others
 *      must be Reduce as well; and their inputs should have the
 *      same attribute except value_index.
 */
548
class ComputeVerifier final : protected tir::ExprVisitor {
549 550 551 552
 public:
  /// Special member functions
  //@{
  explicit ComputeVerifier(const ComputeOpNode* compute)
553
      : compute_(compute), reduce_(compute->body[0].as<tir::ReduceNode>()) {}
554 555 556 557 558 559 560 561 562
  virtual ~ComputeVerifier() = default;
  ComputeVerifier(const ComputeVerifier&) = delete;
  ComputeVerifier(ComputeVerifier&&) = delete;
  ComputeVerifier& operator=(const ComputeVerifier&) = delete;
  ComputeVerifier& operator=(ComputeVerifier&&) = delete;
  //@}

  /// Interface to perform compute verification
  void Run() {
563
    for (const PrimExpr e : compute_->body) {
564
      // Check for consistency of top level reductions
565
      const tir::ReduceNode* reduce = e.as<tir::ReduceNode>();
566 567 568 569 570 571 572 573 574 575 576
      CHECK((reduce && reduce_) || (!reduce && !reduce_))
          << "All ComputeOp should be consistent "
          << "with being Reduce operation or not.";

      if (reduce && reduce_) {
        CHECK(ReduceEqual(reduce, reduce_))
            << "The Reduce inputs of ComputeOp should "
            << "have the same attribute except value_index";
      }

      level_ = 0;
577
      ExprVisitor::VisitExpr(e);
578 579 580 581 582 583
    }
  }

 protected:
  /// Visitor implementation
  //@{
584
  void VisitExpr(const PrimExpr& n) final {
585
    ++level_;
586
    ExprVisitor::VisitExpr(n);
587 588 589
    --level_;
  }

590
  void VisitExpr_(const tir::ReduceNode* op) final {
591 592 593 594 595 596 597 598 599
    // Check for non top level reductions
    CHECK(0 == level_)
        << "Reductions are only allowed at the top level of compute. "
        << "Please create another tensor for further composition.";
  }
  //@}

 private:
  const ComputeOpNode* compute_{nullptr};  ///< ComputeOpNode to verify
600
  const tir::ReduceNode* reduce_{nullptr};      ///< Top level Reduce operation
601 602 603 604 605 606 607 608 609 610
  int level_{0};                           ///< Level of op being processed
};
}  // namespace

/// Verify if ComputeOp is valid with respect to Reduce operations.
static void VerifyComputeOp(const ComputeOpNode* op) {
  ComputeVerifier v(op);
  v.Run();
}

611 612 613 614 615
Stmt TransformUpdate(const Stage& stage,
                     const std::unordered_map<IterVar, Range>& dom_map,
                     const ComputeLoopNest& n,
                     Stmt body,
                     Stmt update) {
616
  Array<PrimExpr> conds;
617
  std::unordered_set<const VarNode*> banned;
618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634
  for (size_t i = 0; i < stage->leaf_iter_vars.size(); ++i) {
    IterVar iv = stage->leaf_iter_vars[i];
    auto iit = stage->iter_var_attrs.find(iv);
    if (iit != stage->iter_var_attrs.end()) {
      const IterVarAttr& attr = (*iit).second;
      if (attr->iter_type == kTensorized) {
        break;
      }
    }
    if (iv->iter_type == kCommReduce) {
      auto vit = dom_map.find(iv);
      CHECK(vit != dom_map.end());
      const Range& vrange = vit->second;
      conds.push_back(likely(iv->var > vrange->min));
      banned.insert(iv->var.get());
    }
  }
635
  for (const PrimExpr& pred : n.main_predicates) {
636
    if (tir::ExprUseVar(pred, banned)) {
637 638 639 640 641
      LOG(FATAL) << "Tensorize update transform failed, the condition "
                 << pred << " has a conflict with the reset condition";
    }
  }

642
  return IfThenElseNode::make(arith::ComputeReduce<tir::OrNode>(conds, const_true(1)),
643 644
                          update, body);
}
645

646
}  // namespace te
647
}  // namespace tvm