From 3f5726598792060071ff8221337120553e38b0df Mon Sep 17 00:00:00 2001 From: chenhu-wang Date: Sat, 8 Feb 2025 15:44:51 +0800 Subject: [PATCH 1/4] fix_list_iterators_incompatible --- src/common/snippets/src/lowered/loop_manager.cpp | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/common/snippets/src/lowered/loop_manager.cpp b/src/common/snippets/src/lowered/loop_manager.cpp index e7e83361ee0a39..ee151fc2c78760 100644 --- a/src/common/snippets/src/lowered/loop_manager.cpp +++ b/src/common/snippets/src/lowered/loop_manager.cpp @@ -358,15 +358,6 @@ size_t LinearIR::LoopManager::replace_with_new_loop(const LinearIR& linear_ir, for (auto expr_it = loop_begin_pos; expr_it != loop_end_pos; ++expr_it) { replace_loop_id(*expr_it, old_id, loop_id); } - - const auto old_loop_info = this->get_loop_info(old_id); - const auto old_loop_begin_pos = linear_ir.find(old_loop_info->get_entry_points().front().expr_port->get_expr()); - const auto old_loop_end_pos = linear_ir.find(old_loop_info->get_exit_points().back().expr_port->get_expr()); - // If new bounds are equal to old loop bounds, this means that old Loop is removed totally from LIR - // In this case old loop info must be completely removed from loop manager - if (loop_begin_pos == old_loop_begin_pos && loop_end_pos == old_loop_end_pos) { - this->remove_loop_info(old_id); - } return loop_id; } From 22568aa2e24f986d63e10f8465175bda0d3a5c10 Mon Sep 17 00:00:00 2001 From: chenhu-wang Date: Mon, 17 Feb 2025 17:01:43 +0800 Subject: [PATCH 2/4] back port fix --- .../lowered/pass/insert_tail_loop.hpp | 4 ++- .../snippets/src/lowered/loop_manager.cpp | 9 +++++++ .../src/lowered/pass/insert_tail_loop.cpp | 25 +++++++++---------- .../x64/pass/lowered/brgemm_blocking.cpp | 12 +++++---- 4 files changed, 31 insertions(+), 19 deletions(-) diff --git a/src/common/snippets/include/snippets/lowered/pass/insert_tail_loop.hpp b/src/common/snippets/include/snippets/lowered/pass/insert_tail_loop.hpp index faafd8186b8448..bd255339543e01 100644 --- a/src/common/snippets/include/snippets/lowered/pass/insert_tail_loop.hpp +++ b/src/common/snippets/include/snippets/lowered/pass/insert_tail_loop.hpp @@ -24,7 +24,9 @@ class InsertTailLoop : public Pass { public: OPENVINO_RTTI("InsertTailLoop", "Pass") bool run(LinearIR& linear_ir) override; - static LinearIR::container copy_loop(const LinearIR& linear_ir, const size_t loop_id); + static LinearIR::constExprIt insert_copy_loop(LinearIR& linear_ir, + const size_t loop_id, + const LinearIR::constExprIt& insert_pos); static constexpr size_t existing_subtensor_value = SIZE_MAX; static void propagate_updated_subtensor_through_loop(const LinearIR& linear_ir, diff --git a/src/common/snippets/src/lowered/loop_manager.cpp b/src/common/snippets/src/lowered/loop_manager.cpp index ee151fc2c78760..e7e83361ee0a39 100644 --- a/src/common/snippets/src/lowered/loop_manager.cpp +++ b/src/common/snippets/src/lowered/loop_manager.cpp @@ -358,6 +358,15 @@ size_t LinearIR::LoopManager::replace_with_new_loop(const LinearIR& linear_ir, for (auto expr_it = loop_begin_pos; expr_it != loop_end_pos; ++expr_it) { replace_loop_id(*expr_it, old_id, loop_id); } + + const auto old_loop_info = this->get_loop_info(old_id); + const auto old_loop_begin_pos = linear_ir.find(old_loop_info->get_entry_points().front().expr_port->get_expr()); + const auto old_loop_end_pos = linear_ir.find(old_loop_info->get_exit_points().back().expr_port->get_expr()); + // If new bounds are equal to old loop bounds, this means that old Loop is removed totally from LIR + // In this case old loop info must be completely removed from loop manager + if (loop_begin_pos == old_loop_begin_pos && loop_end_pos == old_loop_end_pos) { + this->remove_loop_info(old_id); + } return loop_id; } diff --git a/src/common/snippets/src/lowered/pass/insert_tail_loop.cpp b/src/common/snippets/src/lowered/pass/insert_tail_loop.cpp index cc685c1851157a..6c23d77536a0de 100644 --- a/src/common/snippets/src/lowered/pass/insert_tail_loop.cpp +++ b/src/common/snippets/src/lowered/pass/insert_tail_loop.cpp @@ -127,12 +127,14 @@ void InsertTailLoop::propagate_updated_subtensor_through_loop(const LinearIR& li (*expr_it)->updateShapes(); } -LinearIR::container InsertTailLoop::copy_loop(const LinearIR& linear_ir, const size_t loop_id) { +LinearIR::constExprIt InsertTailLoop::insert_copy_loop(LinearIR& linear_ir, const size_t loop_id, const LinearIR::constExprIt& insert_pos) { const auto& loop_manager = linear_ir.get_loop_manager(); LinearIR::constExprIt loop_begin_pos, loop_end_pos; loop_manager->get_loop_bounds(linear_ir, loop_id, loop_begin_pos, loop_end_pos, true); ExressionMap expression_map; const auto& loop_copy_range = LinearIR::deep_copy_range(loop_begin_pos, std::next(loop_end_pos), expression_map); + const auto new_loop_begin_pos = linear_ir.insert(insert_pos, loop_copy_range.begin(), loop_copy_range.end()); + const auto new_loop_end_pos = insert_pos; const auto original_loop_info = loop_manager->get_loop_info(loop_id); std::vector new_entry_points, new_exit_points; @@ -156,11 +158,9 @@ LinearIR::container InsertTailLoop::copy_loop(const LinearIR& linear_ir, const s loop_manager->update_loops_port(outer_loop_ids, expr->get_output_port(i), {expr->get_output_port(i), new_expr->get_output_port(i)}, false); } - const auto new_loop_begin_pos = loop_copy_range.begin(); - const auto new_loop_end_pos = loop_copy_range.end(); const auto new_id = loop_manager->replace_with_new_loop(linear_ir, - std::next(new_loop_begin_pos), - std::prev(new_loop_end_pos), + new_loop_begin_pos, + new_loop_end_pos, original_loop_info->get_work_amount(), original_loop_info->get_increment(), new_entry_points, @@ -169,7 +169,7 @@ LinearIR::container InsertTailLoop::copy_loop(const LinearIR& linear_ir, const s const auto loop_end = ov::as_type_ptr(std::prev(new_loop_end_pos)->get()->get_node()); OPENVINO_ASSERT(loop_end, "Cloned Loop does not contain LoopEnd op at the expected place."); loop_end->set_id(new_id); - return loop_copy_range; + return new_loop_begin_pos; } void InsertTailLoop::create_tail_loop(LinearIR& linear_ir, @@ -186,17 +186,16 @@ void InsertTailLoop::create_tail_loop(LinearIR& linear_ir, auto original_loop_info = loop_manager->get_loop_info(original_loop_id); auto tail_loop_info = original_loop_info; if (need_vector_loop) { - const auto new_loop_range = copy_loop(linear_ir, original_loop_id); - const auto new_loop_end = ov::as_type_ptr(std::prev(new_loop_range.end())->get()->get_node()); - OPENVINO_ASSERT(new_loop_end, "Cloned Loop does not contain LoopEnd op at the expected place."); - tail_loop_info = original_loop_info; - original_loop_info = loop_manager->get_loop_info(new_loop_end->get_id()); - // Note: new loop body is inserted before the original loop // So new loop becomes a main vector loop, the original loop becomes tail loop // This is done in such way to have original ops from the main body at the end: // this allows us to conveniently interact with outer loops in further passes - linear_ir.insert(begin, new_loop_range.begin(), new_loop_range.end()); + const auto new_loop_begin_pos = insert_copy_loop(linear_ir, original_loop_id, begin); + const auto new_loop_begin = ov::as_type_ptr(new_loop_begin_pos->get()->get_node()); + OPENVINO_ASSERT(new_loop_begin, "Cloned Loop does not contain LoopBegin op at the expected place."); + const auto new_loop_end = new_loop_begin->get_loop_end(); + tail_loop_info = original_loop_info; + original_loop_info = loop_manager->get_loop_info(new_loop_end->get_id()); const auto new_vector_loop_wa = original_loop_info->get_work_amount() - tail_size; original_loop_info->set_work_amount(new_vector_loop_wa); diff --git a/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/lowered/brgemm_blocking.cpp b/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/lowered/brgemm_blocking.cpp index fc9aeeac10ee92..cf6752c8fb67e3 100644 --- a/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/lowered/brgemm_blocking.cpp +++ b/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/lowered/brgemm_blocking.cpp @@ -160,16 +160,18 @@ bool BrgemmBlocking::run(LinearIR& linear_ir) { if (work_amount <= increment) return false; - auto new_loop_range = snippets::lowered::pass::InsertTailLoop::copy_loop(linear_ir, loop_id); - const auto firt_iter_loop_end = ov::as_type_ptr(std::prev(new_loop_range.end())->get()->get_node()); + const auto loop_begin_it = linear_ir.find(linear_ir.get_expr_by_node(loop_end->get_loop_begin())); + const auto new_loop_begin_pos = + snippets::lowered::pass::InsertTailLoop::insert_copy_loop(linear_ir, loop_id, loop_begin_it); + const auto new_loop_begin = + ov::as_type_ptr(new_loop_begin_pos->get()->get_node()); + OPENVINO_ASSERT(new_loop_begin, "Cloned Loop does not contain LoopBegin op at the expected place."); + const auto firt_iter_loop_end = new_loop_begin->get_loop_end(); auto first_iter_loop_info = loop_manager->get_loop_info(firt_iter_loop_end->get_id()); firt_iter_loop_end->set_work_amount(increment); first_iter_loop_info->set_work_amount(increment); firt_iter_loop_end->set_finalization_offsets(std::vector(loop_end->get_finalization_offsets().size(), 0)); - const auto loop_begin_it = linear_ir.find(linear_ir.get_expr_by_node(loop_end->get_loop_begin())); - linear_ir.insert(loop_begin_it, new_loop_range.begin(), new_loop_range.end()); - const auto new_work_amount = work_amount - increment; loop_info->set_work_amount(new_work_amount); loop_end->set_work_amount(new_work_amount); From cb45c6de98c0ba9fb53c48dc2208d4f169b58469 Mon Sep 17 00:00:00 2001 From: chenhu-wang Date: Wed, 19 Feb 2025 15:15:29 +0800 Subject: [PATCH 3/4] back port full fix --- .../include/snippets/lowered/loop_manager.hpp | 65 ++++++++++--- .../snippets/src/lowered/loop_manager.cpp | 96 ++++++++++++------- .../snippets/src/lowered/pass/fuse_loops.cpp | 11 +-- .../snippets/src/lowered/pass/init_loops.cpp | 49 ++++------ .../src/lowered/pass/insert_buffers.cpp | 10 +- .../src/lowered/pass/insert_loops.cpp | 7 +- .../src/lowered/pass/insert_tail_loop.cpp | 17 ++-- .../lowered/pass/move_result_out_of_loop.cpp | 4 +- .../snippets/src/lowered/pass/split_loops.cpp | 15 ++- 9 files changed, 155 insertions(+), 119 deletions(-) diff --git a/src/common/snippets/include/snippets/lowered/loop_manager.hpp b/src/common/snippets/include/snippets/lowered/loop_manager.hpp index 93d1620f5fdbe7..602f5fcd49641b 100644 --- a/src/common/snippets/include/snippets/lowered/loop_manager.hpp +++ b/src/common/snippets/include/snippets/lowered/loop_manager.hpp @@ -85,6 +85,10 @@ class LinearIR::LoopManager { void set_outer_splited_loop(bool outer_splited_loop); void set_first_iter_handler(FirstIterHandler handler); + // Update the parameters of existing LoopPorts + void update_entry_points(const std::function& updater); + void update_exit_points(const std::function& updater); + private: size_t m_work_amount = 0; size_t m_increment = 0; @@ -101,8 +105,6 @@ class LinearIR::LoopManager { using LoopInfoPtr = std::shared_ptr; std::shared_ptr clone_with_new_expr(const ExressionMap& expr_map) const; - size_t add_loop_info(const LoopInfoPtr& loop); - void remove_loop_info(size_t index); LoopInfoPtr get_loop_info(size_t index) const; size_t get_loop_count() const { return m_map.size(); } const std::map& get_map() const; @@ -182,23 +184,55 @@ class LinearIR::LoopManager { void expression_replacement(constExprIt new_expr_begin, constExprIt new_expr_end, const ExpressionPtr& decomposed_expr, size_t loop_id, const std::vector& new_entries, const std::vector& exits); - // Note: these methods find iterators of first entry loop point and last exit point (bounds of Loop) - // If there are already inserted LoopBegin and LoopEnd in Linear IR, the methods can find them as well if `loop_ops_inserted` = true - void get_loop_bounds(const LinearIR& linear_ir, - size_t loop_id, - LinearIR::constExprIt& loop_begin_pos, - LinearIR::constExprIt& loop_end_pos, - bool loop_ops_inserted = false) const; - static void get_loop_bounds(const LinearIR& linear_ir, - const std::vector& entries, - const std::vector& exits, - LinearIR::constExprIt& loop_begin_pos, - LinearIR::constExprIt& loop_end_pos, - size_t loop_id, bool loop_ops_inserted = false); + /** + * @brief Find bounds of Loop: + * - If the explicit Loop exprs with the target `loop_id` have been inserted, + * Loop bounds are these iterators of the corresponding LoopBegin and LoopEnd. + * - Otherwise Loop bounds are iterators of the first entry loop port (or Scalar, VectorBuffer and another LoopBegin that + * are in this Loop but have another `loop_id`) and the next iterator of the last exit loop port (or another LoopEnd that + * are in this Loop but have another `loop_id`). + * @param linear_ir linear IR + * @param loop_id target Loop ID + * @return the pair of loop_begin_pos and loop_end_pos iterators + */ + std::pair get_loop_bounds(const LinearIR& linear_ir, size_t loop_id) const; + /** + * @brief Find bounds of Loop: + * - If the explicit Loop exprs with the target `loop_id` have been inserted, + * Loop bounds are these iterators of the corresponding LoopBegin and LoopEnd. + * - Otherwise Loop bounds are iterators of the first entry loop port (or Scalar, VectorBuffer and another LoopBegin that + * are in this Loop but have another `loop_id`) and the next iterator of the last exit loop port (or another LoopEnd that + * are in this Loop but have another `loop_id`). + * @param linear_ir linear IR + * @param loop_id target Loop ID + * @param entries input loop ports + * @param exits output loop ports + * @return the pair of loop_begin_pos and loop_end_pos iterators + */ + static std::pair get_loop_bounds(const LinearIR& linear_ir, size_t loop_id, + const std::vector& entries, const std::vector& exits); LoopPort get_loop_port_by_expr_port(const ExpressionPort& expr_port, const size_t loop_id); private: + /** + * @brief Add new Loop Info to the map + * @param loop target loop info + * @return the loop ID + */ + size_t add_loop_info(const LoopInfoPtr& loop); + /** + * @brief Remove LoopInfo from the map + * @param index the target index of Loop + */ + void remove_loop_info(size_t index); + /** + * @brief Find expression ports in bounds that are connected to consumers or parent that aren't in these bounds + * @param loop_begin_pos the first expression iterator of the Loop + * @param loop_end_pos the next iterator after the last expression + * @param entries found input expression ports + * @param exits found output expression ports + */ static void get_io_loop_ports(LinearIR::constExprIt loop_begin_pos, LinearIR::constExprIt loop_end_pos, std::vector& entries, @@ -220,6 +254,7 @@ class LinearIR::LoopManager { // for `before` the new Loop is the most outer Loop void insert_loop_id(const ExpressionPtr& expr, size_t new_id, bool before = true, size_t target_id = SIZE_MAX); void insert_loop_ids(const ExpressionPtr& expr, const std::vector& new_ids, bool before = true, size_t target_id = SIZE_MAX); + static bool is_loop_id_found(const ExpressionPtr& expr, size_t id); std::map m_map = {}; size_t next_id = 0; diff --git a/src/common/snippets/src/lowered/loop_manager.cpp b/src/common/snippets/src/lowered/loop_manager.cpp index e7e83361ee0a39..5920f75197ff0f 100644 --- a/src/common/snippets/src/lowered/loop_manager.cpp +++ b/src/common/snippets/src/lowered/loop_manager.cpp @@ -141,6 +141,14 @@ void LoopInfo::set_first_iter_handler(LoopInfo::FirstIterHandler first_iter_hand m_first_iter_handler = std::move(first_iter_handler); } +void LoopInfo::update_entry_points(const std::function& updater) { + std::for_each(m_entry_points.begin(), m_entry_points.end(), updater); +} + +void LoopInfo::update_exit_points(const std::function& updater) { + std::for_each(m_exit_points.begin(), m_exit_points.end(), updater); +} + bool operator==(const LinearIR::LoopManager::LoopPort& lhs, const LinearIR::LoopManager::LoopPort& rhs) { if (&lhs == &rhs) return true; @@ -194,26 +202,19 @@ std::vector LinearIR::LoopManager::get_outer_expr_loops(const Expression return std::vector(loop_ids.cbegin(), it); } -void LinearIR::LoopManager::get_loop_bounds(const LinearIR &linear_ir, - size_t loop_id, - LinearIR::constExprIt &loop_begin_pos, - LinearIR::constExprIt &loop_end_pos, - bool loop_ops_inserted) const { +std::pair LinearIR::LoopManager::get_loop_bounds(const LinearIR &linear_ir, size_t loop_id) const { const auto loop_info = get_loop_info(loop_id); - get_loop_bounds(linear_ir, loop_info->get_entry_points(), loop_info->get_exit_points(), loop_begin_pos, loop_end_pos, loop_id, loop_ops_inserted); + return get_loop_bounds(linear_ir, loop_id, loop_info->get_entry_points(), loop_info->get_exit_points()); } -void LinearIR::LoopManager::get_loop_bounds(const LinearIR &linear_ir, - const std::vector& entries, - const std::vector& exits, - LinearIR::constExprIt &loop_begin_pos, - LinearIR::constExprIt &loop_end_pos, - size_t loop_id, bool loop_ops_inserted) { +std::pair LinearIR::LoopManager::get_loop_bounds(const LinearIR &linear_ir, size_t loop_id, + const std::vector& entries, + const std::vector& exits) { OPENVINO_ASSERT(!entries.empty(), "Loop must have entry points"); OPENVINO_ASSERT(!exits.empty(), "Loop must have entry points"); - const auto& entry_expr = entries.front().expr_port->get_expr(); - loop_begin_pos = linear_ir.find(entry_expr); + const auto& entry_expr = entries.front().expr_port->get_expr(); + auto loop_begin_pos = linear_ir.find(entry_expr); // Some operations in Loop can be before first entry points: Scalars, VectorBuffer. // We should iterate by them till the expr is in the corresponding Loop auto prev_loop_ids = (*std::prev(loop_begin_pos))->get_loop_ids(); @@ -222,18 +223,28 @@ void LinearIR::LoopManager::get_loop_bounds(const LinearIR &linear_ir, prev_loop_ids = (*std::prev(loop_begin_pos))->get_loop_ids(); } - if (loop_ops_inserted) { - const auto loop_begin = ov::as_type_ptr((*std::prev(loop_begin_pos))->get_node()); - OPENVINO_ASSERT(loop_begin, "Failed explicit loop bounds getting: LoopBegin has not been found"); - const auto loop_end = loop_begin->get_loop_end(); - OPENVINO_ASSERT(loop_end->get_id() == loop_id, "Failed explicit loop bounds getting: Loop bounds with correct ID have not been found"); - loop_begin_pos = std::prev(loop_begin_pos); - loop_end_pos = linear_ir.find_after(loop_begin_pos, linear_ir.get_expr_by_node(loop_end)); - } else { - // At the moment all Loops must have exit points - const auto& exit_expr = exits.back().expr_port->get_expr(); - loop_end_pos = std::next(linear_ir.find_after(loop_begin_pos, exit_expr)); + const auto& exit_expr = exits.back().expr_port->get_expr(); + auto loop_end_pos = std::next(linear_ir.find_after(loop_begin_pos, exit_expr)); + // There might be LoopEnd with another `loop_id` but in the target Loop as well. + auto current_loop_ids = (*loop_end_pos)->get_loop_ids(); + while (std::find(current_loop_ids.begin(), current_loop_ids.end(), loop_id) != current_loop_ids.end()) { + loop_end_pos = std::next(loop_end_pos); + current_loop_ids = (*loop_end_pos)->get_loop_ids(); } + + // Check for the existing LoopBegin/LoopEnd + if (const auto loop_end = ov::as_type_ptr((*loop_end_pos)->get_node())) { + if (loop_end->get_id() == loop_id) { + // loop_begin_pos is iterator of LoopBegin now + // loop_end_pos is iterator of LoopEnd now + loop_begin_pos = std::prev(loop_begin_pos); + const auto loop_begin = loop_end->get_loop_begin(); + OPENVINO_ASSERT((*loop_begin_pos)->get_node() == loop_begin, "LoopBegin has not been found!"); + } + } + + OPENVINO_ASSERT(loop_begin_pos != linear_ir.cend() && loop_end_pos != linear_ir.cend(), "Loop bounds haven't been found!"); + return std::make_pair(loop_begin_pos, loop_end_pos); } LinearIR::LoopManager::LoopPort LinearIR::LoopManager::get_loop_port_by_expr_port(const ExpressionPort& expr_port, const size_t loop_id) { @@ -353,27 +364,35 @@ size_t LinearIR::LoopManager::replace_with_new_loop(const LinearIR& linear_ir, const std::vector& entries, const std::vector& exits, const size_t old_id) { - const auto loop_info = std::make_shared(work_amount, increment, entries, exits); - const auto loop_id = this->add_loop_info(loop_info); - for (auto expr_it = loop_begin_pos; expr_it != loop_end_pos; ++expr_it) { + const auto is_bound_explicit_loop_begin = ov::is_type(loop_begin_pos->get()->get_node()); + const auto is_bound_explicit_loop_end = ov::is_type(std::prev(loop_end_pos)->get()->get_node()); + OPENVINO_ASSERT((is_bound_explicit_loop_begin && is_bound_explicit_loop_end) || + (!is_bound_explicit_loop_begin && !is_bound_explicit_loop_end), + "Incorrect LoopBounds!"); + const auto explicit_loop_bounds = is_bound_explicit_loop_begin && is_bound_explicit_loop_end; + + const auto loop_id = + this->add_loop_info(std::make_shared(work_amount, increment, entries, exits)); + const auto begin = explicit_loop_bounds ? std::next(loop_begin_pos) : loop_begin_pos; + const auto end = explicit_loop_bounds ? std::prev(loop_end_pos) : loop_end_pos; + for (auto expr_it = begin; expr_it != end; ++expr_it) { replace_loop_id(*expr_it, old_id, loop_id); } - const auto old_loop_info = this->get_loop_info(old_id); - const auto old_loop_begin_pos = linear_ir.find(old_loop_info->get_entry_points().front().expr_port->get_expr()); - const auto old_loop_end_pos = linear_ir.find(old_loop_info->get_exit_points().back().expr_port->get_expr()); + // Check that other expression in LinearIR doesn't have the old loop ID - otherwise completely removed from loop + // manager + const auto old_loop_bounds = get_loop_bounds(linear_ir, old_id); // If new bounds are equal to old loop bounds, this means that old Loop is removed totally from LIR // In this case old loop info must be completely removed from loop manager - if (loop_begin_pos == old_loop_begin_pos && loop_end_pos == old_loop_end_pos) { + if (loop_begin_pos == old_loop_bounds.first && loop_end_pos == old_loop_bounds.second) { this->remove_loop_info(old_id); } return loop_id; } void LinearIR::LoopManager::fuse_loops(const LinearIR& linear_ir, size_t loop_id_upper, size_t loop_id_lower, bool fuse_into_upper) { - LinearIR::constExprIt loop_begin_target, loop_end_target; - get_loop_bounds(linear_ir, fuse_into_upper ? loop_id_lower : loop_id_upper, loop_begin_target, loop_end_target); - fuse_loops(loop_begin_target, loop_end_target, loop_id_upper, loop_id_lower, fuse_into_upper); + const auto loop_bounds = get_loop_bounds(linear_ir, fuse_into_upper ? loop_id_lower : loop_id_upper); + fuse_loops(loop_bounds.first, loop_bounds.second, loop_id_upper, loop_id_lower, fuse_into_upper); } void LinearIR::LoopManager::fuse_loops(LinearIR::constExprIt loop_begin_target, LinearIR::constExprIt loop_end_target, @@ -541,6 +560,7 @@ void LinearIR::LoopManager::sort_loop_ports(LinearIR::constExprIt& loop_begin_po void LinearIR::LoopManager::insert_loop_id(const ExpressionPtr& expr, size_t new_id, bool before, size_t target_id) { OPENVINO_ASSERT(m_map.count(new_id) == 1, "Failed marking expression by Loop ID: the Loop with this ID hasn't registered"); + OPENVINO_ASSERT(!is_loop_id_found(expr, new_id), "Expression cannot have several the same Loop IDs"); auto& loop_ids = expr->m_loop_ids; OPENVINO_ASSERT(std::find(loop_ids.cbegin(), loop_ids.cend(), new_id) == loop_ids.cend(), "Expression cannot have several the same Loop IDs"); @@ -568,6 +588,7 @@ void LinearIR::LoopManager::insert_loop_ids(const ExpressionPtr& expr, const std void LinearIR::LoopManager::replace_loop_id(const ExpressionPtr& expr, size_t prev_id, size_t new_id) { OPENVINO_ASSERT(m_map.count(new_id), "Failed marking expression by Loop ID: the Loop with this ID hasn't registered"); + OPENVINO_ASSERT(!is_loop_id_found(expr, new_id), "Expression cannot have several the same Loop IDs"); auto& loop_ids = expr->m_loop_ids; OPENVINO_ASSERT(std::find(loop_ids.cbegin(), loop_ids.cend(), new_id) == loop_ids.cend(), "Expression already has the Loop with ID " + std::to_string(new_id)); @@ -584,6 +605,11 @@ void LinearIR::LoopManager::remove_loop_id(const ExpressionPtr& expr, size_t id) loop_ids.erase(it); } +bool LinearIR::LoopManager::is_loop_id_found(const ExpressionPtr& expr, size_t id) { + const auto loop_ids = expr->get_loop_ids(); + return std::find(loop_ids.cbegin(), loop_ids.cend(), id) != loop_ids.cend(); +} + }// namespace lowered }// namespace snippets }// namespace ov diff --git a/src/common/snippets/src/lowered/pass/fuse_loops.cpp b/src/common/snippets/src/lowered/pass/fuse_loops.cpp index 1738d6d8fe9574..154d74fa89ee16 100644 --- a/src/common/snippets/src/lowered/pass/fuse_loops.cpp +++ b/src/common/snippets/src/lowered/pass/fuse_loops.cpp @@ -68,10 +68,9 @@ void FuseLoops::move(LinearIR& linear_ir, const LinearIR::LoopManagerPtr& loop_m std::map> outer_loops; // The map: LoopID -> [ LoopBegin, LoopEnd ] const auto outer_loop_ids = LinearIR::LoopManager::get_outer_expr_loops(*loop_begin_pos, loop_id); for (const auto& loop_id : outer_loop_ids) { - LinearIR::constExprIt begin, end; - loop_manager->get_loop_bounds(linear_ir, loop_id, begin, end); + const auto loop_bounds = loop_manager->get_loop_bounds(linear_ir, loop_id); // save previos iterator since the current iterator can be moved - outer_loops[loop_id] = {std::prev(begin), end}; + outer_loops[loop_id] = {std::prev(loop_bounds.first), loop_bounds.second}; } // Secondly, move expressions for (auto it = loop_begin_pos; it != loop_end_pos;) { @@ -122,7 +121,7 @@ bool FuseLoops::fuse_upper_into_current(LinearIR& linear_ir, const LinearIR::Loo return false; LinearIR::constExprIt target_loop_begin_pos, target_loop_end_pos; - loop_manager->get_loop_bounds(linear_ir, target_loop_id, target_loop_begin_pos, target_loop_end_pos); + std::tie(target_loop_begin_pos, target_loop_end_pos) = loop_manager->get_loop_bounds(linear_ir, target_loop_id); loop_manager->fuse_loops(target_loop_begin_pos, target_loop_end_pos, target_loop_id, current_loop_id, false); // Update work_amount for Loop (increment is constant because increments must be the identical for fusion): loop_current->set_work_amount(std::max(loop_current->get_work_amount(), loop_target->get_work_amount())); @@ -167,7 +166,7 @@ bool FuseLoops::fuse_lower_into_current(LinearIR& linear_ir, const LinearIR::Loo return false; LinearIR::constExprIt target_loop_begin_pos, target_loop_end_pos; - loop_manager->get_loop_bounds(linear_ir, target_loop_id, target_loop_begin_pos, target_loop_end_pos); + std::tie(target_loop_begin_pos, target_loop_end_pos) = loop_manager->get_loop_bounds(linear_ir, target_loop_id); loop_manager->fuse_loops(target_loop_begin_pos, target_loop_end_pos, current_loop_id, target_loop_id); // Update work_amount for Loop (increment is constant because increments must be the identical for fusion): loop_current->set_work_amount(std::max(loop_current->get_work_amount(), loop_target->get_work_amount())); @@ -212,7 +211,7 @@ bool FuseLoops::run(LinearIR& linear_ir) { const auto current_loop_info = loop_manager->get_loop_info(current_loop_id); LinearIR::constExprIt current_loop_begin_pos, current_loop_end_pos; - loop_manager->get_loop_bounds(linear_ir, current_loop_id, current_loop_begin_pos, current_loop_end_pos); + std::tie(current_loop_begin_pos, current_loop_end_pos) = loop_manager->get_loop_bounds(linear_ir, current_loop_id); // We fuse upper Loops into the current till we can do it. // After that we fuse lower Loops into the current till we can do it. diff --git a/src/common/snippets/src/lowered/pass/init_loops.cpp b/src/common/snippets/src/lowered/pass/init_loops.cpp index 68e8cc7757e13f..6d014142de71b5 100644 --- a/src/common/snippets/src/lowered/pass/init_loops.cpp +++ b/src/common/snippets/src/lowered/pass/init_loops.cpp @@ -39,15 +39,12 @@ InitLoops::InitLoops() : Pass() {} void InitLoops::init_ptr_increments(const LinearIR::LoopManager::LoopInfoPtr& loop_info) { const auto work_amount = loop_info->get_work_amount(); - auto loop_entries = loop_info->get_entry_points(); - auto loop_exits = loop_info->get_exit_points(); - for (auto& loop_entry : loop_entries) { + auto init_entry_port_increment = [&work_amount](LoopPort& loop_entry) { loop_entry.ptr_increment = 0; if (loop_entry.is_incremented) { const auto& port = loop_entry.expr_port; const auto source = *port->get_connected_ports().begin(); - const auto loop_ids = port->get_expr()->get_loop_ids(); const auto& layout = port->get_descriptor_ptr()->get_layout(); const auto& shape = port->get_descriptor_ptr()->get_shape(); const auto& dim = *(layout.rbegin() + loop_entry.dim_idx); @@ -57,13 +54,11 @@ void InitLoops::init_ptr_increments(const LinearIR::LoopManager::LoopInfoPtr& lo loop_entry.ptr_increment = get_input_stride(dim, source.get_descriptor_ptr()->get_layout(), shape); } } - } - - for (auto& loop_exit : loop_exits) { + }; + auto init_exit_port_increment = [&work_amount](LoopPort& loop_exit) { loop_exit.ptr_increment = 0; if (loop_exit.is_incremented) { const auto& port = loop_exit.expr_port; - const auto loop_ids = port->get_expr()->get_loop_ids(); const auto& layout = port->get_descriptor_ptr()->get_layout(); const auto& shape = port->get_descriptor_ptr()->get_shape(); const auto original_dim = layout.size() - 1 - loop_exit.dim_idx; @@ -74,38 +69,34 @@ void InitLoops::init_ptr_increments(const LinearIR::LoopManager::LoopInfoPtr& lo loop_exit.ptr_increment = get_output_stride(dim, shape); } } - } - loop_info->set_entry_points(loop_entries); - loop_info->set_exit_points(loop_exits); + }; + + loop_info->update_entry_points(init_entry_port_increment); + loop_info->update_exit_points(init_exit_port_increment); } void InitLoops::init_finalization_offsets(const LinearIR::LoopManager::LoopInfoPtr& loop_info) { const auto work_amount = loop_info->get_work_amount(); - auto loop_entries = loop_info->get_entry_points(); - auto loop_exits = loop_info->get_exit_points(); - for (auto& loop_entry : loop_entries) { - loop_entry.finalization_offset = -1 * loop_entry.ptr_increment * work_amount; - } - for (auto& loop_exit : loop_exits) { - loop_exit.finalization_offset = -1 * loop_exit.ptr_increment * work_amount; - } - loop_info->set_entry_points(loop_entries); - loop_info->set_exit_points(loop_exits); + auto init_port_finalization_offset = [&work_amount](LoopPort& loop_port) { + loop_port.finalization_offset = -1 * loop_port.ptr_increment * work_amount; + }; + + loop_info->update_entry_points(init_port_finalization_offset); + loop_info->update_exit_points(init_port_finalization_offset); } void InitLoops::init_element_type_sizes(const LinearIR::LoopManager::LoopInfoPtr& loop_info) { - auto loop_entries = loop_info->get_entry_points(); - auto loop_exits = loop_info->get_exit_points(); - for (auto& loop_entry : loop_entries) { + auto init_entry_port_data_size = [](LoopPort& loop_entry) { const auto& port = loop_entry.expr_port; loop_entry.data_size = static_cast(port->get_expr()->get_node()->get_input_element_type(port->get_index()).size()); - } - for (auto& loop_exit : loop_exits) { + }; + auto init_exit_port_data_size = [](LoopPort& loop_exit) { const auto& port = loop_exit.expr_port; loop_exit.data_size = static_cast(port->get_expr()->get_node()->get_output_element_type(port->get_index()).size()); - } - loop_info->set_entry_points(loop_entries); - loop_info->set_exit_points(loop_exits); + }; + + loop_info->update_entry_points(init_entry_port_data_size); + loop_info->update_exit_points(init_exit_port_data_size); } bool InitLoops::run(LinearIR& linear_ir) { diff --git a/src/common/snippets/src/lowered/pass/insert_buffers.cpp b/src/common/snippets/src/lowered/pass/insert_buffers.cpp index 81835a4ca390ae..a5b0df38cfdfa7 100644 --- a/src/common/snippets/src/lowered/pass/insert_buffers.cpp +++ b/src/common/snippets/src/lowered/pass/insert_buffers.cpp @@ -126,18 +126,12 @@ LinearIR::constExprIt InsertBuffers::insertion_position(const LinearIR& linear_i // If upper expression is inside Loop, we should insert Buffer after this Loop if (loop_idx < up_loop_count) { const auto up_loop_id = up_loops[loop_idx]; - const auto loop_info = loop_manager->get_loop_info(up_loop_id); - LinearIR::constExprIt loop_begin_pos, loop_end_pos; - loop_manager->get_loop_bounds(linear_ir, up_loop_id, loop_begin_pos, loop_end_pos); - return loop_end_pos; + return loop_manager->get_loop_bounds(linear_ir, up_loop_id).second; } // If lower expression is inside Loop, we should insert Buffer before this Loop if (loop_idx < down_loop_count) { const auto down_loop_id = down_loops[loop_idx]; - const auto loop_info = loop_manager->get_loop_info(down_loop_id); - LinearIR::constExprIt loop_begin_pos, loop_end_pos; - loop_manager->get_loop_bounds(linear_ir, down_loop_id, loop_begin_pos, loop_end_pos); - return loop_begin_pos; + return loop_manager->get_loop_bounds(linear_ir, down_loop_id).first; } OPENVINO_THROW("Incorrect configuration for Buffer insertion!"); } diff --git a/src/common/snippets/src/lowered/pass/insert_loops.cpp b/src/common/snippets/src/lowered/pass/insert_loops.cpp index 3eab6e97df33fb..8e594f8e64682b 100644 --- a/src/common/snippets/src/lowered/pass/insert_loops.cpp +++ b/src/common/snippets/src/lowered/pass/insert_loops.cpp @@ -60,8 +60,7 @@ void InsertLoops::insertion(LinearIR& linear_ir, const LinearIR::LoopManagerPtr& const auto work_amount = loop_info->get_work_amount(); const auto work_amount_increment = loop_info->get_increment(); - LinearIR::constExprIt loop_begin_pos, loop_end_pos; - loop_manager->get_loop_bounds(linear_ir, loop_id, loop_begin_pos, loop_end_pos); + const auto loop_bounds = loop_manager->get_loop_bounds(linear_ir, loop_id); // Remove non MemoryAccess ports since Loop can have only GPR inputs filter_ports(loop_entries, loop_exits); @@ -90,7 +89,7 @@ void InsertLoops::insertion(LinearIR& linear_ir, const LinearIR::LoopManagerPtr& const auto& loop_begin = std::make_shared(); const auto& loop_begin_expr = linear_ir.create_expression(loop_begin, std::vector{}); - linear_ir.insert(loop_begin_pos, loop_begin_expr); + linear_ir.insert(loop_bounds.first, loop_begin_expr); const auto& loop_end = std::make_shared( loop_begin->output(0), work_amount, work_amount_increment, is_incremented, ptr_increments, @@ -101,7 +100,7 @@ void InsertLoops::insertion(LinearIR& linear_ir, const LinearIR::LoopManagerPtr& loop_end_inputs.push_back(loop_begin_expr->get_output_port_connector(0)); const auto& loop_end_expr = linear_ir.create_expression(loop_end, loop_end_inputs); - const auto& it = linear_ir.insert(loop_end_pos, loop_end_expr); + const auto& it = linear_ir.insert(loop_bounds.second, loop_end_expr); const auto outer_loop_ids = get_outer_loop_ids(*std::prev(it), loop_id); loop_begin_expr->set_loop_ids(outer_loop_ids); diff --git a/src/common/snippets/src/lowered/pass/insert_tail_loop.cpp b/src/common/snippets/src/lowered/pass/insert_tail_loop.cpp index 6c23d77536a0de..3823a4417f87f6 100644 --- a/src/common/snippets/src/lowered/pass/insert_tail_loop.cpp +++ b/src/common/snippets/src/lowered/pass/insert_tail_loop.cpp @@ -129,10 +129,10 @@ void InsertTailLoop::propagate_updated_subtensor_through_loop(const LinearIR& li LinearIR::constExprIt InsertTailLoop::insert_copy_loop(LinearIR& linear_ir, const size_t loop_id, const LinearIR::constExprIt& insert_pos) { const auto& loop_manager = linear_ir.get_loop_manager(); - LinearIR::constExprIt loop_begin_pos, loop_end_pos; - loop_manager->get_loop_bounds(linear_ir, loop_id, loop_begin_pos, loop_end_pos, true); + const auto loop_bounds = loop_manager->get_loop_bounds(linear_ir, loop_id); + ExressionMap expression_map; - const auto& loop_copy_range = LinearIR::deep_copy_range(loop_begin_pos, std::next(loop_end_pos), expression_map); + const auto& loop_copy_range = LinearIR::deep_copy_range(loop_bounds.first, std::next(loop_bounds.second), expression_map); const auto new_loop_begin_pos = linear_ir.insert(insert_pos, loop_copy_range.begin(), loop_copy_range.end()); const auto new_loop_end_pos = insert_pos; @@ -158,14 +158,9 @@ LinearIR::constExprIt InsertTailLoop::insert_copy_loop(LinearIR& linear_ir, cons loop_manager->update_loops_port(outer_loop_ids, expr->get_output_port(i), {expr->get_output_port(i), new_expr->get_output_port(i)}, false); } - const auto new_id = loop_manager->replace_with_new_loop(linear_ir, - new_loop_begin_pos, - new_loop_end_pos, - original_loop_info->get_work_amount(), - original_loop_info->get_increment(), - new_entry_points, - new_exit_points, - loop_id); + const auto new_id = loop_manager->replace_with_new_loop(linear_ir, new_loop_begin_pos, new_loop_end_pos, + original_loop_info->get_work_amount(), original_loop_info->get_increment(), + new_entry_points, new_exit_points, loop_id); const auto loop_end = ov::as_type_ptr(std::prev(new_loop_end_pos)->get()->get_node()); OPENVINO_ASSERT(loop_end, "Cloned Loop does not contain LoopEnd op at the expected place."); loop_end->set_id(new_id); diff --git a/src/common/snippets/src/lowered/pass/move_result_out_of_loop.cpp b/src/common/snippets/src/lowered/pass/move_result_out_of_loop.cpp index b423eeda46a5cb..d92cfaaa6e0386 100644 --- a/src/common/snippets/src/lowered/pass/move_result_out_of_loop.cpp +++ b/src/common/snippets/src/lowered/pass/move_result_out_of_loop.cpp @@ -51,8 +51,8 @@ bool MoveResultOutOfLoop::run(LinearIR& linear_ir) { continue; } - LinearIR::constExprIt loop_begin_pos, loop_end_pos; - loop_manager->get_loop_bounds(linear_ir, *(parent_loop_ids.cbegin()), loop_begin_pos, loop_end_pos); + const auto loop_bounds = loop_manager->get_loop_bounds(linear_ir, *(parent_loop_ids.cbegin())); + const auto loop_end_pos = loop_bounds.second; // If the Result isn't found after Outer LoopEnd, need to move it to there if (std::find(loop_end_pos, linear_ir.cend(), expr) == linear_ir.cend()) { expr_it = std::prev(expr_it); // save iterator before moving diff --git a/src/common/snippets/src/lowered/pass/split_loops.cpp b/src/common/snippets/src/lowered/pass/split_loops.cpp index ba036eca8011f9..d17b1cef91a35b 100644 --- a/src/common/snippets/src/lowered/pass/split_loops.cpp +++ b/src/common/snippets/src/lowered/pass/split_loops.cpp @@ -67,15 +67,12 @@ bool SplitLoops::run(LinearIR& linear_ir) { const auto& loop_to_fuse = !split_parent ? parent_loop : loop; loop_to_split->set_work_amount(loop_to_fuse->get_increment()); - LinearIR::constExprIt loop_begin_pos, loop_end_pos; - LoopManager::get_loop_bounds(linear_ir, - loop_to_split->get_entry_points(), - loop_to_split->get_exit_points(), - loop_begin_pos, - loop_end_pos, - loop_to_split_id); - const auto split_loop_id = loop_manager->mark_loop(loop_begin_pos, - loop_end_pos, + const auto loop_bounds = LoopManager::get_loop_bounds(linear_ir, + loop_to_split_id, + loop_to_split->get_entry_points(), + loop_to_split->get_exit_points()); + const auto split_loop_id = loop_manager->mark_loop(loop_bounds.first, + loop_bounds.second, loop_to_fuse->get_work_amount(), loop_to_fuse->get_increment(), loop_to_split->get_dim_idx(), From 492f9da58a61b84eec2cdc3e1ae6dc451b744b9e Mon Sep 17 00:00:00 2001 From: chenhu-wang Date: Tue, 25 Feb 2025 14:39:53 +0800 Subject: [PATCH 4/4] exclude changes that is not relavent --- .../include/snippets/lowered/loop_manager.hpp | 5 -- .../lowered/pass/insert_tail_loop.hpp | 4 +- .../snippets/src/lowered/loop_manager.cpp | 21 +------- .../snippets/src/lowered/pass/init_loops.cpp | 49 +++++++++++-------- .../x64/pass/lowered/brgemm_blocking.cpp | 6 +-- 5 files changed, 34 insertions(+), 51 deletions(-) diff --git a/src/common/snippets/include/snippets/lowered/loop_manager.hpp b/src/common/snippets/include/snippets/lowered/loop_manager.hpp index 602f5fcd49641b..f38fab34730e0c 100644 --- a/src/common/snippets/include/snippets/lowered/loop_manager.hpp +++ b/src/common/snippets/include/snippets/lowered/loop_manager.hpp @@ -85,10 +85,6 @@ class LinearIR::LoopManager { void set_outer_splited_loop(bool outer_splited_loop); void set_first_iter_handler(FirstIterHandler handler); - // Update the parameters of existing LoopPorts - void update_entry_points(const std::function& updater); - void update_exit_points(const std::function& updater); - private: size_t m_work_amount = 0; size_t m_increment = 0; @@ -254,7 +250,6 @@ class LinearIR::LoopManager { // for `before` the new Loop is the most outer Loop void insert_loop_id(const ExpressionPtr& expr, size_t new_id, bool before = true, size_t target_id = SIZE_MAX); void insert_loop_ids(const ExpressionPtr& expr, const std::vector& new_ids, bool before = true, size_t target_id = SIZE_MAX); - static bool is_loop_id_found(const ExpressionPtr& expr, size_t id); std::map m_map = {}; size_t next_id = 0; diff --git a/src/common/snippets/include/snippets/lowered/pass/insert_tail_loop.hpp b/src/common/snippets/include/snippets/lowered/pass/insert_tail_loop.hpp index bd255339543e01..5fe8634959fb51 100644 --- a/src/common/snippets/include/snippets/lowered/pass/insert_tail_loop.hpp +++ b/src/common/snippets/include/snippets/lowered/pass/insert_tail_loop.hpp @@ -24,9 +24,7 @@ class InsertTailLoop : public Pass { public: OPENVINO_RTTI("InsertTailLoop", "Pass") bool run(LinearIR& linear_ir) override; - static LinearIR::constExprIt insert_copy_loop(LinearIR& linear_ir, - const size_t loop_id, - const LinearIR::constExprIt& insert_pos); + static LinearIR::constExprIt insert_copy_loop(LinearIR& linear_ir, const size_t loop_id, const LinearIR::constExprIt& insert_pos); static constexpr size_t existing_subtensor_value = SIZE_MAX; static void propagate_updated_subtensor_through_loop(const LinearIR& linear_ir, diff --git a/src/common/snippets/src/lowered/loop_manager.cpp b/src/common/snippets/src/lowered/loop_manager.cpp index 5920f75197ff0f..6723b115d983fc 100644 --- a/src/common/snippets/src/lowered/loop_manager.cpp +++ b/src/common/snippets/src/lowered/loop_manager.cpp @@ -141,14 +141,6 @@ void LoopInfo::set_first_iter_handler(LoopInfo::FirstIterHandler first_iter_hand m_first_iter_handler = std::move(first_iter_handler); } -void LoopInfo::update_entry_points(const std::function& updater) { - std::for_each(m_entry_points.begin(), m_entry_points.end(), updater); -} - -void LoopInfo::update_exit_points(const std::function& updater) { - std::for_each(m_exit_points.begin(), m_exit_points.end(), updater); -} - bool operator==(const LinearIR::LoopManager::LoopPort& lhs, const LinearIR::LoopManager::LoopPort& rhs) { if (&lhs == &rhs) return true; @@ -366,8 +358,7 @@ size_t LinearIR::LoopManager::replace_with_new_loop(const LinearIR& linear_ir, const size_t old_id) { const auto is_bound_explicit_loop_begin = ov::is_type(loop_begin_pos->get()->get_node()); const auto is_bound_explicit_loop_end = ov::is_type(std::prev(loop_end_pos)->get()->get_node()); - OPENVINO_ASSERT((is_bound_explicit_loop_begin && is_bound_explicit_loop_end) || - (!is_bound_explicit_loop_begin && !is_bound_explicit_loop_end), + OPENVINO_ASSERT((is_bound_explicit_loop_begin && is_bound_explicit_loop_end) || (!is_bound_explicit_loop_begin && !is_bound_explicit_loop_end), "Incorrect LoopBounds!"); const auto explicit_loop_bounds = is_bound_explicit_loop_begin && is_bound_explicit_loop_end; @@ -379,8 +370,7 @@ size_t LinearIR::LoopManager::replace_with_new_loop(const LinearIR& linear_ir, replace_loop_id(*expr_it, old_id, loop_id); } - // Check that other expression in LinearIR doesn't have the old loop ID - otherwise completely removed from loop - // manager + // Check that other expression in LinearIR doesn't have the old loop ID - otherwise completely removed from loop manager const auto old_loop_bounds = get_loop_bounds(linear_ir, old_id); // If new bounds are equal to old loop bounds, this means that old Loop is removed totally from LIR // In this case old loop info must be completely removed from loop manager @@ -560,7 +550,6 @@ void LinearIR::LoopManager::sort_loop_ports(LinearIR::constExprIt& loop_begin_po void LinearIR::LoopManager::insert_loop_id(const ExpressionPtr& expr, size_t new_id, bool before, size_t target_id) { OPENVINO_ASSERT(m_map.count(new_id) == 1, "Failed marking expression by Loop ID: the Loop with this ID hasn't registered"); - OPENVINO_ASSERT(!is_loop_id_found(expr, new_id), "Expression cannot have several the same Loop IDs"); auto& loop_ids = expr->m_loop_ids; OPENVINO_ASSERT(std::find(loop_ids.cbegin(), loop_ids.cend(), new_id) == loop_ids.cend(), "Expression cannot have several the same Loop IDs"); @@ -588,7 +577,6 @@ void LinearIR::LoopManager::insert_loop_ids(const ExpressionPtr& expr, const std void LinearIR::LoopManager::replace_loop_id(const ExpressionPtr& expr, size_t prev_id, size_t new_id) { OPENVINO_ASSERT(m_map.count(new_id), "Failed marking expression by Loop ID: the Loop with this ID hasn't registered"); - OPENVINO_ASSERT(!is_loop_id_found(expr, new_id), "Expression cannot have several the same Loop IDs"); auto& loop_ids = expr->m_loop_ids; OPENVINO_ASSERT(std::find(loop_ids.cbegin(), loop_ids.cend(), new_id) == loop_ids.cend(), "Expression already has the Loop with ID " + std::to_string(new_id)); @@ -605,11 +593,6 @@ void LinearIR::LoopManager::remove_loop_id(const ExpressionPtr& expr, size_t id) loop_ids.erase(it); } -bool LinearIR::LoopManager::is_loop_id_found(const ExpressionPtr& expr, size_t id) { - const auto loop_ids = expr->get_loop_ids(); - return std::find(loop_ids.cbegin(), loop_ids.cend(), id) != loop_ids.cend(); -} - }// namespace lowered }// namespace snippets }// namespace ov diff --git a/src/common/snippets/src/lowered/pass/init_loops.cpp b/src/common/snippets/src/lowered/pass/init_loops.cpp index 6d014142de71b5..68e8cc7757e13f 100644 --- a/src/common/snippets/src/lowered/pass/init_loops.cpp +++ b/src/common/snippets/src/lowered/pass/init_loops.cpp @@ -39,12 +39,15 @@ InitLoops::InitLoops() : Pass() {} void InitLoops::init_ptr_increments(const LinearIR::LoopManager::LoopInfoPtr& loop_info) { const auto work_amount = loop_info->get_work_amount(); + auto loop_entries = loop_info->get_entry_points(); + auto loop_exits = loop_info->get_exit_points(); - auto init_entry_port_increment = [&work_amount](LoopPort& loop_entry) { + for (auto& loop_entry : loop_entries) { loop_entry.ptr_increment = 0; if (loop_entry.is_incremented) { const auto& port = loop_entry.expr_port; const auto source = *port->get_connected_ports().begin(); + const auto loop_ids = port->get_expr()->get_loop_ids(); const auto& layout = port->get_descriptor_ptr()->get_layout(); const auto& shape = port->get_descriptor_ptr()->get_shape(); const auto& dim = *(layout.rbegin() + loop_entry.dim_idx); @@ -54,11 +57,13 @@ void InitLoops::init_ptr_increments(const LinearIR::LoopManager::LoopInfoPtr& lo loop_entry.ptr_increment = get_input_stride(dim, source.get_descriptor_ptr()->get_layout(), shape); } } - }; - auto init_exit_port_increment = [&work_amount](LoopPort& loop_exit) { + } + + for (auto& loop_exit : loop_exits) { loop_exit.ptr_increment = 0; if (loop_exit.is_incremented) { const auto& port = loop_exit.expr_port; + const auto loop_ids = port->get_expr()->get_loop_ids(); const auto& layout = port->get_descriptor_ptr()->get_layout(); const auto& shape = port->get_descriptor_ptr()->get_shape(); const auto original_dim = layout.size() - 1 - loop_exit.dim_idx; @@ -69,34 +74,38 @@ void InitLoops::init_ptr_increments(const LinearIR::LoopManager::LoopInfoPtr& lo loop_exit.ptr_increment = get_output_stride(dim, shape); } } - }; - - loop_info->update_entry_points(init_entry_port_increment); - loop_info->update_exit_points(init_exit_port_increment); + } + loop_info->set_entry_points(loop_entries); + loop_info->set_exit_points(loop_exits); } void InitLoops::init_finalization_offsets(const LinearIR::LoopManager::LoopInfoPtr& loop_info) { const auto work_amount = loop_info->get_work_amount(); - auto init_port_finalization_offset = [&work_amount](LoopPort& loop_port) { - loop_port.finalization_offset = -1 * loop_port.ptr_increment * work_amount; - }; - - loop_info->update_entry_points(init_port_finalization_offset); - loop_info->update_exit_points(init_port_finalization_offset); + auto loop_entries = loop_info->get_entry_points(); + auto loop_exits = loop_info->get_exit_points(); + for (auto& loop_entry : loop_entries) { + loop_entry.finalization_offset = -1 * loop_entry.ptr_increment * work_amount; + } + for (auto& loop_exit : loop_exits) { + loop_exit.finalization_offset = -1 * loop_exit.ptr_increment * work_amount; + } + loop_info->set_entry_points(loop_entries); + loop_info->set_exit_points(loop_exits); } void InitLoops::init_element_type_sizes(const LinearIR::LoopManager::LoopInfoPtr& loop_info) { - auto init_entry_port_data_size = [](LoopPort& loop_entry) { + auto loop_entries = loop_info->get_entry_points(); + auto loop_exits = loop_info->get_exit_points(); + for (auto& loop_entry : loop_entries) { const auto& port = loop_entry.expr_port; loop_entry.data_size = static_cast(port->get_expr()->get_node()->get_input_element_type(port->get_index()).size()); - }; - auto init_exit_port_data_size = [](LoopPort& loop_exit) { + } + for (auto& loop_exit : loop_exits) { const auto& port = loop_exit.expr_port; loop_exit.data_size = static_cast(port->get_expr()->get_node()->get_output_element_type(port->get_index()).size()); - }; - - loop_info->update_entry_points(init_entry_port_data_size); - loop_info->update_exit_points(init_exit_port_data_size); + } + loop_info->set_entry_points(loop_entries); + loop_info->set_exit_points(loop_exits); } bool InitLoops::run(LinearIR& linear_ir) { diff --git a/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/lowered/brgemm_blocking.cpp b/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/lowered/brgemm_blocking.cpp index cf6752c8fb67e3..5a6236d1916b13 100644 --- a/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/lowered/brgemm_blocking.cpp +++ b/src/plugins/intel_cpu/src/transformations/snippets/x64/pass/lowered/brgemm_blocking.cpp @@ -161,10 +161,8 @@ bool BrgemmBlocking::run(LinearIR& linear_ir) { return false; const auto loop_begin_it = linear_ir.find(linear_ir.get_expr_by_node(loop_end->get_loop_begin())); - const auto new_loop_begin_pos = - snippets::lowered::pass::InsertTailLoop::insert_copy_loop(linear_ir, loop_id, loop_begin_it); - const auto new_loop_begin = - ov::as_type_ptr(new_loop_begin_pos->get()->get_node()); + const auto new_loop_begin_pos = snippets::lowered::pass::InsertTailLoop::insert_copy_loop(linear_ir, loop_id, loop_begin_it); + const auto new_loop_begin = ov::as_type_ptr(new_loop_begin_pos->get()->get_node()); OPENVINO_ASSERT(new_loop_begin, "Cloned Loop does not contain LoopBegin op at the expected place."); const auto firt_iter_loop_end = new_loop_begin->get_loop_end(); auto first_iter_loop_info = loop_manager->get_loop_info(firt_iter_loop_end->get_id());