Skip to content

R
riscv-gcc-1
Commit d54a098e by Richard Sandiford Committed by Richard Sandiford
Options

[1/n] PR85694: Allow pattern definition statements to be reused 

This patch is the first part of a series to fix to PR85694.
Later patches can make the pattern for a statement S2 reuse the
results of a PATTERN_DEF_SEQ statement attached to an earlier
statement S1.  Although vect_mark_stmts_to_be_vectorized handled
this fine, vect_analyze_stmt and vect_transform_loop both skipped the
PATTERN_DEF_SEQ for S1 if S1's main pattern wasn't live or relevant.

I couldn't wrap my head around the flow in vect_transform_loop,
so ended up moving the per-statement handling into a subroutine.
That makes the patch look bigger than it actually is.

2018-06-20  Richard Sandiford  <richard.sandiford@arm.com>

gcc/
	* tree-vect-stmts.c (vect_analyze_stmt): Move the handling of pattern
	definition statements before the early exit for statements that aren't
	live or relevant.
	* tree-vect-loop.c (vect_transform_loop_stmt): New function,
	split out from...
	(vect_transform_loop): ...here.  Process pattern definition
	statements without first checking whether the main pattern
	statement is live or relevant.

From-SVN: r261784
parent 036a9082
Showing with 152 additions and 204 deletions
gcc/ChangeLog
2018-06-20 Richard Sandiford <richard.sandiford@arm.com>
* tree-vect-stmts.c (vect_analyze_stmt): Move the handling of pattern
definition statements before the early exit for statements that aren't
live or relevant.
* tree-vect-loop.c (vect_transform_loop_stmt): New function,
split out from...
(vect_transform_loop): ...here. Process pattern definition
statements without first checking whether the main pattern
statement is live or relevant.
2018-06-19 Eric Botcazou <ebotcazou@adacore.com> 2018-06-19 Eric Botcazou <ebotcazou@adacore.com>
* tree-cfgcleanup.c (tree_forwarder_block_p): Do not return false at * tree-cfgcleanup.c (tree_forwarder_block_p): Do not return false at
......
gcc/tree-vect-loop.c
...@@ -8290,6 +8290,74 @@ scale_profile_for_vect_loop (struct loop *loop, unsigned vf) ...@@ -8290,6 +8290,74 @@ scale_profile_for_vect_loop (struct loop *loop, unsigned vf)
scale_bbs_frequencies (&loop->latch, 1, exit_l->probability / prob); scale_bbs_frequencies (&loop->latch, 1, exit_l->probability / prob);
} }
/* Vectorize STMT if relevant, inserting any new instructions before GSI.
When vectorizing STMT as a store, set *SEEN_STORE to its stmt_vec_info.
*SLP_SCHEDULE is a running record of whether we have called
vect_schedule_slp. */
static void
vect_transform_loop_stmt (loop_vec_info loop_vinfo, gimple *stmt,
gimple_stmt_iterator *gsi,
stmt_vec_info *seen_store, bool *slp_scheduled)
{
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
if (!stmt_info)
return;
if (dump_enabled_p ())
{
dump_printf_loc (MSG_NOTE, vect_location,
"------>vectorizing statement: ");
dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt, 0);
}
if (MAY_HAVE_DEBUG_BIND_STMTS && !STMT_VINFO_LIVE_P (stmt_info))
vect_loop_kill_debug_uses (loop, stmt);
if (!STMT_VINFO_RELEVANT_P (stmt_info)
&& !STMT_VINFO_LIVE_P (stmt_info))
return;
if (STMT_VINFO_VECTYPE (stmt_info))
{
poly_uint64 nunits
= TYPE_VECTOR_SUBPARTS (STMT_VINFO_VECTYPE (stmt_info));
if (!STMT_SLP_TYPE (stmt_info)
&& maybe_ne (nunits, vf)
&& dump_enabled_p ())
/* For SLP VF is set according to unrolling factor, and not
to vector size, hence for SLP this print is not valid. */
dump_printf_loc (MSG_NOTE, vect_location, "multiple-types.\n");
}
/* SLP. Schedule all the SLP instances when the first SLP stmt is
reached. */
if (STMT_SLP_TYPE (stmt_info))
{
if (!*slp_scheduled)
{
*slp_scheduled = true;
DUMP_VECT_SCOPE ("scheduling SLP instances");
vect_schedule_slp (loop_vinfo);
}
/* Hybrid SLP stmts must be vectorized in addition to SLP. */
if (PURE_SLP_STMT (stmt_info))
return;
}
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location, "transform statement.\n");
bool grouped_store = false;
if (vect_transform_stmt (stmt, gsi, &grouped_store, NULL, NULL))
*seen_store = stmt_info;
}
/* Function vect_transform_loop. /* Function vect_transform_loop.
The analysis phase has determined that the loop is vectorizable. The analysis phase has determined that the loop is vectorizable.
...@@ -8310,12 +8378,8 @@ vect_transform_loop (loop_vec_info loop_vinfo) ...@@ -8310,12 +8378,8 @@ vect_transform_loop (loop_vec_info loop_vinfo)
tree niters_vector_mult_vf = NULL_TREE; tree niters_vector_mult_vf = NULL_TREE;
poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo); poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
unsigned int lowest_vf = constant_lower_bound (vf); unsigned int lowest_vf = constant_lower_bound (vf);
bool grouped_store;
bool slp_scheduled = false; bool slp_scheduled = false;
gimple *stmt, *pattern_stmt; gimple *stmt;
gimple_seq pattern_def_seq = NULL;
gimple_stmt_iterator pattern_def_si = gsi_none ();
bool transform_pattern_stmt = false;
bool check_profitability = false; bool check_profitability = false;
unsigned int th; unsigned int th;
...@@ -8466,194 +8530,67 @@ vect_transform_loop (loop_vec_info loop_vinfo) ...@@ -8466,194 +8530,67 @@ vect_transform_loop (loop_vec_info loop_vinfo)
} }
} }
pattern_stmt = NULL;
for (gimple_stmt_iterator si = gsi_start_bb (bb); for (gimple_stmt_iterator si = gsi_start_bb (bb);
!gsi_end_p (si) || transform_pattern_stmt;) !gsi_end_p (si);)
{ {
bool is_store; stmt = gsi_stmt (si);
/* During vectorization remove existing clobber stmts. */
if (transform_pattern_stmt) if (gimple_clobber_p (stmt))
stmt = pattern_stmt;
else
{ {
stmt = gsi_stmt (si); unlink_stmt_vdef (stmt);
/* During vectorization remove existing clobber stmts. */ gsi_remove (&si, true);
if (gimple_clobber_p (stmt)) release_defs (stmt);
{
unlink_stmt_vdef (stmt);
gsi_remove (&si, true);
release_defs (stmt);
continue;
}
} }
else
if (dump_enabled_p ())
{
dump_printf_loc (MSG_NOTE, vect_location,
"------>vectorizing statement: ");
dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt, 0);
}
stmt_info = vinfo_for_stmt (stmt);
/* vector stmts created in the outer-loop during vectorization of
stmts in an inner-loop may not have a stmt_info, and do not
need to be vectorized. */
if (!stmt_info)
{ {
gsi_next (&si); stmt_info = vinfo_for_stmt (stmt);
continue;
}
if (MAY_HAVE_DEBUG_BIND_STMTS && !STMT_VINFO_LIVE_P (stmt_info))
vect_loop_kill_debug_uses (loop, stmt);
if (!STMT_VINFO_RELEVANT_P (stmt_info) /* vector stmts created in the outer-loop during vectorization of
&& !STMT_VINFO_LIVE_P (stmt_info)) stmts in an inner-loop may not have a stmt_info, and do not
{ need to be vectorized. */
if (STMT_VINFO_IN_PATTERN_P (stmt_info) stmt_vec_info seen_store = NULL;
&& (pattern_stmt = STMT_VINFO_RELATED_STMT (stmt_info)) if (stmt_info)
&& (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_stmt))
|| STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_stmt))))
{
stmt = pattern_stmt;
stmt_info = vinfo_for_stmt (stmt);
}
else
{
gsi_next (&si);
continue;
}
}
else if (STMT_VINFO_IN_PATTERN_P (stmt_info)
&& (pattern_stmt = STMT_VINFO_RELATED_STMT (stmt_info))
&& (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_stmt))
|| STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_stmt))))
transform_pattern_stmt = true;
/* If pattern statement has def stmts, vectorize them too. */
if (is_pattern_stmt_p (stmt_info))
{
if (pattern_def_seq == NULL)
{ {
pattern_def_seq = STMT_VINFO_PATTERN_DEF_SEQ (stmt_info); if (STMT_VINFO_IN_PATTERN_P (stmt_info))
pattern_def_si = gsi_start (pattern_def_seq);
}
else if (!gsi_end_p (pattern_def_si))
gsi_next (&pattern_def_si);
if (pattern_def_seq != NULL)
{
gimple *pattern_def_stmt = NULL;
stmt_vec_info pattern_def_stmt_info = NULL;
while (!gsi_end_p (pattern_def_si))
{ {
pattern_def_stmt = gsi_stmt (pattern_def_si); gimple *def_seq = STMT_VINFO_PATTERN_DEF_SEQ (stmt_info);
pattern_def_stmt_info for (gimple_stmt_iterator subsi = gsi_start (def_seq);
= vinfo_for_stmt (pattern_def_stmt); !gsi_end_p (subsi); gsi_next (&subsi))
if (STMT_VINFO_RELEVANT_P (pattern_def_stmt_info) vect_transform_loop_stmt (loop_vinfo,
|| STMT_VINFO_LIVE_P (pattern_def_stmt_info)) gsi_stmt (subsi), &si,
break; &seen_store,
gsi_next (&pattern_def_si); &slp_scheduled);
} gimple *pat_stmt = STMT_VINFO_RELATED_STMT (stmt_info);
vect_transform_loop_stmt (loop_vinfo, pat_stmt, &si,
if (!gsi_end_p (pattern_def_si)) &seen_store, &slp_scheduled);
{
if (dump_enabled_p ())
{
dump_printf_loc (MSG_NOTE, vect_location,
"==> vectorizing pattern def "
"stmt: ");
dump_gimple_stmt (MSG_NOTE, TDF_SLIM,
pattern_def_stmt, 0);
}
stmt = pattern_def_stmt;
stmt_info = pattern_def_stmt_info;
}
else
{
pattern_def_si = gsi_none ();
transform_pattern_stmt = false;
} }
vect_transform_loop_stmt (loop_vinfo, stmt, &si,
&seen_store, &slp_scheduled);
} }
else if (seen_store)
transform_pattern_stmt = false;
}
if (STMT_VINFO_VECTYPE (stmt_info))
{
poly_uint64 nunits
= TYPE_VECTOR_SUBPARTS (STMT_VINFO_VECTYPE (stmt_info));
if (!STMT_SLP_TYPE (stmt_info)
&& maybe_ne (nunits, vf)
&& dump_enabled_p ())
/* For SLP VF is set according to unrolling factor, and not
to vector size, hence for SLP this print is not valid. */
dump_printf_loc (MSG_NOTE, vect_location, "multiple-types.\n");
}
/* SLP. Schedule all the SLP instances when the first SLP stmt is
reached. */
if (STMT_SLP_TYPE (stmt_info))
{
if (!slp_scheduled)
{ {
slp_scheduled = true; if (STMT_VINFO_GROUPED_ACCESS (seen_store))
DUMP_VECT_SCOPE ("scheduling SLP instances");
vect_schedule_slp (loop_vinfo);
}
/* Hybrid SLP stmts must be vectorized in addition to SLP. */
if (!vinfo_for_stmt (stmt) || PURE_SLP_STMT (stmt_info))
{
if (!transform_pattern_stmt && gsi_end_p (pattern_def_si))
{ {
pattern_def_seq = NULL; /* Interleaving. If IS_STORE is TRUE, the
vectorization of the interleaving chain was
completed - free all the stores in the chain. */
gsi_next (&si); gsi_next (&si);
vect_remove_stores (DR_GROUP_FIRST_ELEMENT (seen_store));
}
else
{
/* Free the attached stmt_vec_info and remove the
stmt. */
free_stmt_vec_info (stmt);
unlink_stmt_vdef (stmt);
gsi_remove (&si, true);
release_defs (stmt);
} }
continue;
}
}
/* -------- vectorize statement ------------ */
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location, "transform statement.\n");
grouped_store = false;
is_store = vect_transform_stmt (stmt, &si, &grouped_store, NULL, NULL);
if (is_store)
{
if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
{
/* Interleaving. If IS_STORE is TRUE, the vectorization of the
interleaving chain was completed - free all the stores in
the chain. */
gsi_next (&si);
vect_remove_stores (DR_GROUP_FIRST_ELEMENT (stmt_info));
} }
else else
{ gsi_next (&si);
/* Free the attached stmt_vec_info and remove the stmt. */
gimple *store = gsi_stmt (si);
free_stmt_vec_info (store);
unlink_stmt_vdef (store);
gsi_remove (&si, true);
release_defs (store);
}
/* Stores can only appear at the end of pattern statements. */
gcc_assert (!transform_pattern_stmt);
pattern_def_seq = NULL;
} }
else if (!transform_pattern_stmt && gsi_end_p (pattern_def_si)) }
{
pattern_def_seq = NULL;
gsi_next (&si);
}
} /* stmts in BB */
/* Stub out scalar statements that must not survive vectorization. /* Stub out scalar statements that must not survive vectorization.
Doing this here helps with grouped statements, or statements that Doing this here helps with grouped statements, or statements that
......
gcc/tree-vect-stmts.c
...@@ -9393,6 +9393,34 @@ vect_analyze_stmt (gimple *stmt, bool *need_to_vectorize, slp_tree node, ...@@ -9393,6 +9393,34 @@ vect_analyze_stmt (gimple *stmt, bool *need_to_vectorize, slp_tree node,
return false; return false;
} }
if (STMT_VINFO_IN_PATTERN_P (stmt_info)
&& node == NULL
&& (pattern_def_seq = STMT_VINFO_PATTERN_DEF_SEQ (stmt_info)))
{
gimple_stmt_iterator si;
for (si = gsi_start (pattern_def_seq); !gsi_end_p (si); gsi_next (&si))
{
gimple *pattern_def_stmt = gsi_stmt (si);
if (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_def_stmt))
|| STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_def_stmt)))
{
/* Analyze def stmt of STMT if it's a pattern stmt. */
if (dump_enabled_p ())
{
dump_printf_loc (MSG_NOTE, vect_location,
"==> examining pattern def statement: ");
dump_gimple_stmt (MSG_NOTE, TDF_SLIM, pattern_def_stmt, 0);
}
if (!vect_analyze_stmt (pattern_def_stmt,
need_to_vectorize, node, node_instance,
cost_vec))
return false;
}
}
}
/* Skip stmts that do not need to be vectorized. In loops this is expected /* Skip stmts that do not need to be vectorized. In loops this is expected
to include: to include:
- the COND_EXPR which is the loop exit condition - the COND_EXPR which is the loop exit condition
...@@ -9453,34 +9481,6 @@ vect_analyze_stmt (gimple *stmt, bool *need_to_vectorize, slp_tree node, ...@@ -9453,34 +9481,6 @@ vect_analyze_stmt (gimple *stmt, bool *need_to_vectorize, slp_tree node,
return false; return false;
} }
if (is_pattern_stmt_p (stmt_info)
&& node == NULL
&& (pattern_def_seq = STMT_VINFO_PATTERN_DEF_SEQ (stmt_info)))
{
gimple_stmt_iterator si;
for (si = gsi_start (pattern_def_seq); !gsi_end_p (si); gsi_next (&si))
{
gimple *pattern_def_stmt = gsi_stmt (si);
if (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_def_stmt))
|| STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_def_stmt)))
{
/* Analyze def stmt of STMT if it's a pattern stmt. */
if (dump_enabled_p ())
{
dump_printf_loc (MSG_NOTE, vect_location,
"==> examining pattern def statement: ");
dump_gimple_stmt (MSG_NOTE, TDF_SLIM, pattern_def_stmt, 0);
}
if (!vect_analyze_stmt (pattern_def_stmt,
need_to_vectorize, node, node_instance,
cost_vec))
return false;
}
}
}
switch (STMT_VINFO_DEF_TYPE (stmt_info)) switch (STMT_VINFO_DEF_TYPE (stmt_info))
{ {
case vect_internal_def: case vect_internal_def:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment