Skip to content

R
riscv-gcc-1
Commit 32e8e429 by Richard Sandiford Committed by Richard Sandiford
Options

[32/46] Use stmt_vec_info in function interfaces (part 2) 

This second part handles the mechanical change from a gimple stmt
argument to a stmt_vec_info argument.  It updates the function
comments if they referred to the argument by name, but it doesn't
try to retrofit mentions to other functions.

2018-07-31  Richard Sandiford  <richard.sandiford@arm.com>

gcc/
	* tree-vectorizer.h (nested_in_vect_loop_p): Move further down
	file and take a stmt_vec_info instead of a gimple stmt.
	(supportable_widening_operation, vect_finish_replace_stmt)
	(vect_finish_stmt_generation, vect_get_store_rhs)
	(vect_get_vec_def_for_operand_1, vect_get_vec_def_for_operand)
	(vect_get_vec_defs, vect_init_vector, vect_transform_stmt)
	(vect_remove_stores, vect_analyze_stmt, vectorizable_condition)
	(vect_get_smallest_scalar_type, vect_check_gather_scatter)
	(vect_create_data_ref_ptr, bump_vector_ptr)
	(vect_permute_store_chain, vect_setup_realignment)
	(vect_transform_grouped_load, vect_record_grouped_load_vectors)
	(vect_create_addr_base_for_vector_ref, vectorizable_live_operation)
	(vectorizable_reduction, vectorizable_induction)
	(get_initial_def_for_reduction, is_simple_and_all_uses_invariant)
	(vect_get_place_in_interleaving_chain): Take stmt_vec_infos rather
	than gimple stmts as arguments.
	* tree-vect-data-refs.c (vect_get_smallest_scalar_type)
	(vect_preserves_scalar_order_p, vect_slp_analyze_node_dependences)
	(can_group_stmts_p, vect_check_gather_scatter)
	(vect_create_addr_base_for_vector_ref, vect_create_data_ref_ptr)
	(bump_vector_ptr, vect_permute_store_chain, vect_setup_realignment)
	(vect_permute_load_chain, vect_shift_permute_load_chain)
	(vect_transform_grouped_load)
	(vect_record_grouped_load_vectors): Likewise.
	* tree-vect-loop.c (vect_fixup_reduc_chain)
	(get_initial_def_for_reduction, vect_create_epilog_for_reduction)
	(vectorize_fold_left_reduction, is_nonwrapping_integer_induction)
	(vectorizable_reduction, vectorizable_induction)
	(vectorizable_live_operation, vect_loop_kill_debug_uses): Likewise.
	* tree-vect-patterns.c (type_conversion_p, adjust_bool_stmts)
	(vect_get_load_store_mask): Likewise.
	* tree-vect-slp.c (vect_get_place_in_interleaving_chain)
	(vect_analyze_slp_instance, vect_mask_constant_operand_p): Likewise.
	* tree-vect-stmts.c (vect_mark_relevant)
	(is_simple_and_all_uses_invariant)
	(exist_non_indexing_operands_for_use_p, process_use)
	(vect_init_vector_1, vect_init_vector, vect_get_vec_def_for_operand_1)
	(vect_get_vec_def_for_operand, vect_get_vec_defs)
	(vect_finish_stmt_generation_1, vect_finish_replace_stmt)
	(vect_finish_stmt_generation, vect_truncate_gather_scatter_offset)
	(compare_step_with_zero, vect_get_store_rhs, get_group_load_store_type)
	(get_negative_load_store_type, get_load_store_type)
	(vect_check_load_store_mask, vect_check_store_rhs)
	(vect_build_gather_load_calls, vect_get_strided_load_store_ops)
	(vectorizable_bswap, vectorizable_call, vectorizable_simd_clone_call)
	(vect_create_vectorized_demotion_stmts, vectorizable_conversion)
	(vectorizable_assignment, vectorizable_shift, vectorizable_operation)
	(get_group_alias_ptr_type, vectorizable_store, hoist_defs_of_uses)
	(vectorizable_load, vectorizable_condition, vectorizable_comparison)
	(vect_analyze_stmt, vect_transform_stmt, vect_remove_stores)
	(supportable_widening_operation): Likewise.

From-SVN: r263147
parent 82570274
Showing with 382 additions and 359 deletions
gcc/ChangeLog
2018-07-31 Richard Sandiford <richard.sandiford@arm.com> 2018-07-31 Richard Sandiford <richard.sandiford@arm.com>
* tree-vectorizer.h (nested_in_vect_loop_p): Move further down
file and take a stmt_vec_info instead of a gimple stmt.
(supportable_widening_operation, vect_finish_replace_stmt)
(vect_finish_stmt_generation, vect_get_store_rhs)
(vect_get_vec_def_for_operand_1, vect_get_vec_def_for_operand)
(vect_get_vec_defs, vect_init_vector, vect_transform_stmt)
(vect_remove_stores, vect_analyze_stmt, vectorizable_condition)
(vect_get_smallest_scalar_type, vect_check_gather_scatter)
(vect_create_data_ref_ptr, bump_vector_ptr)
(vect_permute_store_chain, vect_setup_realignment)
(vect_transform_grouped_load, vect_record_grouped_load_vectors)
(vect_create_addr_base_for_vector_ref, vectorizable_live_operation)
(vectorizable_reduction, vectorizable_induction)
(get_initial_def_for_reduction, is_simple_and_all_uses_invariant)
(vect_get_place_in_interleaving_chain): Take stmt_vec_infos rather
than gimple stmts as arguments.
* tree-vect-data-refs.c (vect_get_smallest_scalar_type)
(vect_preserves_scalar_order_p, vect_slp_analyze_node_dependences)
(can_group_stmts_p, vect_check_gather_scatter)
(vect_create_addr_base_for_vector_ref, vect_create_data_ref_ptr)
(bump_vector_ptr, vect_permute_store_chain, vect_setup_realignment)
(vect_permute_load_chain, vect_shift_permute_load_chain)
(vect_transform_grouped_load)
(vect_record_grouped_load_vectors): Likewise.
* tree-vect-loop.c (vect_fixup_reduc_chain)
(get_initial_def_for_reduction, vect_create_epilog_for_reduction)
(vectorize_fold_left_reduction, is_nonwrapping_integer_induction)
(vectorizable_reduction, vectorizable_induction)
(vectorizable_live_operation, vect_loop_kill_debug_uses): Likewise.
* tree-vect-patterns.c (type_conversion_p, adjust_bool_stmts)
(vect_get_load_store_mask): Likewise.
* tree-vect-slp.c (vect_get_place_in_interleaving_chain)
(vect_analyze_slp_instance, vect_mask_constant_operand_p): Likewise.
* tree-vect-stmts.c (vect_mark_relevant)
(is_simple_and_all_uses_invariant)
(exist_non_indexing_operands_for_use_p, process_use)
(vect_init_vector_1, vect_init_vector, vect_get_vec_def_for_operand_1)
(vect_get_vec_def_for_operand, vect_get_vec_defs)
(vect_finish_stmt_generation_1, vect_finish_replace_stmt)
(vect_finish_stmt_generation, vect_truncate_gather_scatter_offset)
(compare_step_with_zero, vect_get_store_rhs, get_group_load_store_type)
(get_negative_load_store_type, get_load_store_type)
(vect_check_load_store_mask, vect_check_store_rhs)
(vect_build_gather_load_calls, vect_get_strided_load_store_ops)
(vectorizable_bswap, vectorizable_call, vectorizable_simd_clone_call)
(vect_create_vectorized_demotion_stmts, vectorizable_conversion)
(vectorizable_assignment, vectorizable_shift, vectorizable_operation)
(get_group_alias_ptr_type, vectorizable_store, hoist_defs_of_uses)
(vectorizable_load, vectorizable_condition, vectorizable_comparison)
(vect_analyze_stmt, vect_transform_stmt, vect_remove_stores)
(supportable_widening_operation): Likewise.
2018-07-31 Richard Sandiford <richard.sandiford@arm.com>
* tree-vect-data-refs.c (vect_describe_gather_scatter_call): Take * tree-vect-data-refs.c (vect_describe_gather_scatter_call): Take
a stmt_vec_info instead of a gcall. a stmt_vec_info instead of a gcall.
(vect_check_gather_scatter): Update call accordingly. (vect_check_gather_scatter): Update call accordingly.
......
gcc/tree-vect-data-refs.c
...@@ -99,7 +99,7 @@ vect_lanes_optab_supported_p (const char *name, convert_optab optab, ...@@ -99,7 +99,7 @@ vect_lanes_optab_supported_p (const char *name, convert_optab optab,
} }
/* Return the smallest scalar part of STMT. /* Return the smallest scalar part of STMT_INFO.
This is used to determine the vectype of the stmt. We generally set the This is used to determine the vectype of the stmt. We generally set the
vectype according to the type of the result (lhs). For stmts whose vectype according to the type of the result (lhs). For stmts whose
result-type is different than the type of the arguments (e.g., demotion, result-type is different than the type of the arguments (e.g., demotion,
...@@ -117,10 +117,11 @@ vect_lanes_optab_supported_p (const char *name, convert_optab optab, ...@@ -117,10 +117,11 @@ vect_lanes_optab_supported_p (const char *name, convert_optab optab,
types. */ types. */
tree tree
vect_get_smallest_scalar_type (gimple *stmt, HOST_WIDE_INT *lhs_size_unit, vect_get_smallest_scalar_type (stmt_vec_info stmt_info,
HOST_WIDE_INT *rhs_size_unit) HOST_WIDE_INT *lhs_size_unit,
HOST_WIDE_INT *rhs_size_unit)
{ {
tree scalar_type = gimple_expr_type (stmt); tree scalar_type = gimple_expr_type (stmt_info->stmt);
HOST_WIDE_INT lhs, rhs; HOST_WIDE_INT lhs, rhs;
/* During the analysis phase, this function is called on arbitrary /* During the analysis phase, this function is called on arbitrary
...@@ -130,7 +131,7 @@ vect_get_smallest_scalar_type (gimple *stmt, HOST_WIDE_INT *lhs_size_unit, ...@@ -130,7 +131,7 @@ vect_get_smallest_scalar_type (gimple *stmt, HOST_WIDE_INT *lhs_size_unit,
lhs = rhs = TREE_INT_CST_LOW (TYPE_SIZE_UNIT (scalar_type)); lhs = rhs = TREE_INT_CST_LOW (TYPE_SIZE_UNIT (scalar_type));
gassign *assign = dyn_cast <gassign *> (stmt); gassign *assign = dyn_cast <gassign *> (stmt_info->stmt);
if (assign if (assign
&& (gimple_assign_cast_p (assign) && (gimple_assign_cast_p (assign)
|| gimple_assign_rhs_code (assign) == DOT_PROD_EXPR || gimple_assign_rhs_code (assign) == DOT_PROD_EXPR
...@@ -191,16 +192,14 @@ vect_check_nonzero_value (loop_vec_info loop_vinfo, tree value) ...@@ -191,16 +192,14 @@ vect_check_nonzero_value (loop_vec_info loop_vinfo, tree value)
LOOP_VINFO_CHECK_NONZERO (loop_vinfo).safe_push (value); LOOP_VINFO_CHECK_NONZERO (loop_vinfo).safe_push (value);
} }
/* Return true if we know that the order of vectorized STMT_A and /* Return true if we know that the order of vectorized STMTINFO_A and
vectorized STMT_B will be the same as the order of STMT_A and STMT_B. vectorized STMTINFO_B will be the same as the order of STMTINFO_A and
At least one of the statements is a write. */ STMTINFO_B. At least one of the statements is a write. */
static bool static bool
vect_preserves_scalar_order_p (gimple *stmt_a, gimple *stmt_b) vect_preserves_scalar_order_p (stmt_vec_info stmtinfo_a,
stmt_vec_info stmtinfo_b)
{ {
stmt_vec_info stmtinfo_a = vinfo_for_stmt (stmt_a);
stmt_vec_info stmtinfo_b = vinfo_for_stmt (stmt_b);
/* Single statements are always kept in their original order. */ /* Single statements are always kept in their original order. */
if (!STMT_VINFO_GROUPED_ACCESS (stmtinfo_a) if (!STMT_VINFO_GROUPED_ACCESS (stmtinfo_a)
&& !STMT_VINFO_GROUPED_ACCESS (stmtinfo_b)) && !STMT_VINFO_GROUPED_ACCESS (stmtinfo_b))
...@@ -666,7 +665,7 @@ vect_slp_analyze_data_ref_dependence (struct data_dependence_relation *ddr) ...@@ -666,7 +665,7 @@ vect_slp_analyze_data_ref_dependence (struct data_dependence_relation *ddr)
static bool static bool
vect_slp_analyze_node_dependences (slp_instance instance, slp_tree node, vect_slp_analyze_node_dependences (slp_instance instance, slp_tree node,
vec<stmt_vec_info> stores, vec<stmt_vec_info> stores,
gimple *last_store) stmt_vec_info last_store_info)
{ {
/* This walks over all stmts involved in the SLP load/store done /* This walks over all stmts involved in the SLP load/store done
in NODE verifying we can sink them up to the last stmt in the in NODE verifying we can sink them up to the last stmt in the
...@@ -712,7 +711,7 @@ vect_slp_analyze_node_dependences (slp_instance instance, slp_tree node, ...@@ -712,7 +711,7 @@ vect_slp_analyze_node_dependences (slp_instance instance, slp_tree node,
been sunk to (and we verify if we can do that as well). */ been sunk to (and we verify if we can do that as well). */
if (gimple_visited_p (stmt)) if (gimple_visited_p (stmt))
{ {
if (stmt_info != last_store) if (stmt_info != last_store_info)
continue; continue;
unsigned i; unsigned i;
stmt_vec_info store_info; stmt_vec_info store_info;
...@@ -2843,20 +2842,20 @@ strip_conversion (tree op) ...@@ -2843,20 +2842,20 @@ strip_conversion (tree op)
return gimple_assign_rhs1 (stmt); return gimple_assign_rhs1 (stmt);
} }
/* Return true if vectorizable_* routines can handle statements STMT1 /* Return true if vectorizable_* routines can handle statements STMT1_INFO
and STMT2 being in a single group. */ and STMT2_INFO being in a single group. */
static bool static bool
can_group_stmts_p (gimple *stmt1, gimple *stmt2) can_group_stmts_p (stmt_vec_info stmt1_info, stmt_vec_info stmt2_info)
{ {
if (gimple_assign_single_p (stmt1)) if (gimple_assign_single_p (stmt1_info->stmt))
return gimple_assign_single_p (stmt2); return gimple_assign_single_p (stmt2_info->stmt);
gcall *call1 = dyn_cast <gcall *> (stmt1); gcall *call1 = dyn_cast <gcall *> (stmt1_info->stmt);
if (call1 && gimple_call_internal_p (call1)) if (call1 && gimple_call_internal_p (call1))
{ {
/* Check for two masked loads or two masked stores. */ /* Check for two masked loads or two masked stores. */
gcall *call2 = dyn_cast <gcall *> (stmt2); gcall *call2 = dyn_cast <gcall *> (stmt2_info->stmt);
if (!call2 || !gimple_call_internal_p (call2)) if (!call2 || !gimple_call_internal_p (call2))
return false; return false;
internal_fn ifn = gimple_call_internal_fn (call1); internal_fn ifn = gimple_call_internal_fn (call1);
...@@ -3643,17 +3642,16 @@ vect_describe_gather_scatter_call (stmt_vec_info stmt_info, ...@@ -3643,17 +3642,16 @@ vect_describe_gather_scatter_call (stmt_vec_info stmt_info,
info->memory_type = TREE_TYPE (DR_REF (dr)); info->memory_type = TREE_TYPE (DR_REF (dr));
} }
/* Return true if a non-affine read or write in STMT is suitable for a /* Return true if a non-affine read or write in STMT_INFO is suitable for a
gather load or scatter store. Describe the operation in *INFO if so. */ gather load or scatter store. Describe the operation in *INFO if so. */
bool bool
vect_check_gather_scatter (gimple *stmt, loop_vec_info loop_vinfo, vect_check_gather_scatter (stmt_vec_info stmt_info, loop_vec_info loop_vinfo,
gather_scatter_info *info) gather_scatter_info *info)
{ {
HOST_WIDE_INT scale = 1; HOST_WIDE_INT scale = 1;
poly_int64 pbitpos, pbitsize; poly_int64 pbitpos, pbitsize;
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo); struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info); struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
tree offtype = NULL_TREE; tree offtype = NULL_TREE;
tree decl = NULL_TREE, base, off; tree decl = NULL_TREE, base, off;
...@@ -4473,7 +4471,7 @@ vect_duplicate_ssa_name_ptr_info (tree name, data_reference *dr) ...@@ -4473,7 +4471,7 @@ vect_duplicate_ssa_name_ptr_info (tree name, data_reference *dr)
that will be accessed for a data reference. that will be accessed for a data reference.
Input: Input:
STMT: The statement containing the data reference. STMT_INFO: The statement containing the data reference.
NEW_STMT_LIST: Must be initialized to NULL_TREE or a statement list. NEW_STMT_LIST: Must be initialized to NULL_TREE or a statement list.
OFFSET: Optional. If supplied, it is be added to the initial address. OFFSET: Optional. If supplied, it is be added to the initial address.
LOOP: Specify relative to which loop-nest should the address be computed. LOOP: Specify relative to which loop-nest should the address be computed.
...@@ -4502,12 +4500,11 @@ vect_duplicate_ssa_name_ptr_info (tree name, data_reference *dr) ...@@ -4502,12 +4500,11 @@ vect_duplicate_ssa_name_ptr_info (tree name, data_reference *dr)
FORNOW: We are only handling array accesses with step 1. */ FORNOW: We are only handling array accesses with step 1. */
tree tree
vect_create_addr_base_for_vector_ref (gimple *stmt, vect_create_addr_base_for_vector_ref (stmt_vec_info stmt_info,
gimple_seq *new_stmt_list, gimple_seq *new_stmt_list,
tree offset, tree offset,
tree byte_offset) tree byte_offset)
{ {
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info); struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
const char *base_name; const char *base_name;
tree addr_base; tree addr_base;
...@@ -4588,26 +4585,26 @@ vect_create_addr_base_for_vector_ref (gimple *stmt, ...@@ -4588,26 +4585,26 @@ vect_create_addr_base_for_vector_ref (gimple *stmt,
/* Function vect_create_data_ref_ptr. /* Function vect_create_data_ref_ptr.
Create a new pointer-to-AGGR_TYPE variable (ap), that points to the first Create a new pointer-to-AGGR_TYPE variable (ap), that points to the first
location accessed in the loop by STMT, along with the def-use update location accessed in the loop by STMT_INFO, along with the def-use update
chain to appropriately advance the pointer through the loop iterations. chain to appropriately advance the pointer through the loop iterations.
Also set aliasing information for the pointer. This pointer is used by Also set aliasing information for the pointer. This pointer is used by
the callers to this function to create a memory reference expression for the callers to this function to create a memory reference expression for
vector load/store access. vector load/store access.
Input: Input:
1. STMT: a stmt that references memory. Expected to be of the form 1. STMT_INFO: a stmt that references memory. Expected to be of the form
GIMPLE_ASSIGN <name, data-ref> or GIMPLE_ASSIGN <name, data-ref> or
GIMPLE_ASSIGN <data-ref, name>. GIMPLE_ASSIGN <data-ref, name>.
2. AGGR_TYPE: the type of the reference, which should be either a vector 2. AGGR_TYPE: the type of the reference, which should be either a vector
or an array. or an array.
3. AT_LOOP: the loop where the vector memref is to be created. 3. AT_LOOP: the loop where the vector memref is to be created.
4. OFFSET (optional): an offset to be added to the initial address accessed 4. OFFSET (optional): an offset to be added to the initial address accessed
by the data-ref in STMT. by the data-ref in STMT_INFO.
5. BSI: location where the new stmts are to be placed if there is no loop 5. BSI: location where the new stmts are to be placed if there is no loop
6. ONLY_INIT: indicate if ap is to be updated in the loop, or remain 6. ONLY_INIT: indicate if ap is to be updated in the loop, or remain
pointing to the initial address. pointing to the initial address.
7. BYTE_OFFSET (optional, defaults to NULL): a byte offset to be added 7. BYTE_OFFSET (optional, defaults to NULL): a byte offset to be added
to the initial address accessed by the data-ref in STMT. This is to the initial address accessed by the data-ref in STMT_INFO. This is
similar to OFFSET, but OFFSET is counted in elements, while BYTE_OFFSET similar to OFFSET, but OFFSET is counted in elements, while BYTE_OFFSET
in bytes. in bytes.
8. IV_STEP (optional, defaults to NULL): the amount that should be added 8. IV_STEP (optional, defaults to NULL): the amount that should be added
...@@ -4643,14 +4640,13 @@ vect_create_addr_base_for_vector_ref (gimple *stmt, ...@@ -4643,14 +4640,13 @@ vect_create_addr_base_for_vector_ref (gimple *stmt,
4. Return the pointer. */ 4. Return the pointer. */
tree tree
vect_create_data_ref_ptr (gimple *stmt, tree aggr_type, struct loop *at_loop, vect_create_data_ref_ptr (stmt_vec_info stmt_info, tree aggr_type,
tree offset, tree *initial_address, struct loop *at_loop, tree offset,
gimple_stmt_iterator *gsi, gimple **ptr_incr, tree *initial_address, gimple_stmt_iterator *gsi,
bool only_init, bool *inv_p, tree byte_offset, gimple **ptr_incr, bool only_init, bool *inv_p,
tree iv_step) tree byte_offset, tree iv_step)
{ {
const char *base_name; const char *base_name;
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
struct loop *loop = NULL; struct loop *loop = NULL;
bool nested_in_vect_loop = false; bool nested_in_vect_loop = false;
...@@ -4905,7 +4901,7 @@ vect_create_data_ref_ptr (gimple *stmt, tree aggr_type, struct loop *at_loop, ...@@ -4905,7 +4901,7 @@ vect_create_data_ref_ptr (gimple *stmt, tree aggr_type, struct loop *at_loop,
the loop. The increment amount across iterations is expected the loop. The increment amount across iterations is expected
to be vector_size. to be vector_size.
BSI - location where the new update stmt is to be placed. BSI - location where the new update stmt is to be placed.
STMT - the original scalar memory-access stmt that is being vectorized. STMT_INFO - the original scalar memory-access stmt that is being vectorized.
BUMP - optional. The offset by which to bump the pointer. If not given, BUMP - optional. The offset by which to bump the pointer. If not given,
the offset is assumed to be vector_size. the offset is assumed to be vector_size.
...@@ -4915,9 +4911,8 @@ vect_create_data_ref_ptr (gimple *stmt, tree aggr_type, struct loop *at_loop, ...@@ -4915,9 +4911,8 @@ vect_create_data_ref_ptr (gimple *stmt, tree aggr_type, struct loop *at_loop,
tree tree
bump_vector_ptr (tree dataref_ptr, gimple *ptr_incr, gimple_stmt_iterator *gsi, bump_vector_ptr (tree dataref_ptr, gimple *ptr_incr, gimple_stmt_iterator *gsi,
gimple *stmt, tree bump) stmt_vec_info stmt_info, tree bump)
{ {
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info); struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
tree vectype = STMT_VINFO_VECTYPE (stmt_info); tree vectype = STMT_VINFO_VECTYPE (stmt_info);
tree update = TYPE_SIZE_UNIT (vectype); tree update = TYPE_SIZE_UNIT (vectype);
...@@ -5217,11 +5212,10 @@ vect_store_lanes_supported (tree vectype, unsigned HOST_WIDE_INT count, ...@@ -5217,11 +5212,10 @@ vect_store_lanes_supported (tree vectype, unsigned HOST_WIDE_INT count,
void void
vect_permute_store_chain (vec<tree> dr_chain, vect_permute_store_chain (vec<tree> dr_chain,
unsigned int length, unsigned int length,
gimple *stmt, stmt_vec_info stmt_info,
gimple_stmt_iterator *gsi, gimple_stmt_iterator *gsi,
vec<tree> *result_chain) vec<tree> *result_chain)
{ {
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
tree vect1, vect2, high, low; tree vect1, vect2, high, low;
gimple *perm_stmt; gimple *perm_stmt;
tree vectype = STMT_VINFO_VECTYPE (stmt_info); tree vectype = STMT_VINFO_VECTYPE (stmt_info);
...@@ -5368,12 +5362,12 @@ vect_permute_store_chain (vec<tree> dr_chain, ...@@ -5368,12 +5362,12 @@ vect_permute_store_chain (vec<tree> dr_chain,
dr_explicit_realign_optimized. dr_explicit_realign_optimized.
The code above sets up a new (vector) pointer, pointing to the first The code above sets up a new (vector) pointer, pointing to the first
location accessed by STMT, and a "floor-aligned" load using that pointer. location accessed by STMT_INFO, and a "floor-aligned" load using that
It also generates code to compute the "realignment-token" (if the relevant pointer. It also generates code to compute the "realignment-token"
target hook was defined), and creates a phi-node at the loop-header bb (if the relevant target hook was defined), and creates a phi-node at the
whose arguments are the result of the prolog-load (created by this loop-header bb whose arguments are the result of the prolog-load (created
function) and the result of a load that takes place in the loop (to be by this function) and the result of a load that takes place in the loop
created by the caller to this function). (to be created by the caller to this function).
For the case of dr_explicit_realign_optimized: For the case of dr_explicit_realign_optimized:
The caller to this function uses the phi-result (msq) to create the The caller to this function uses the phi-result (msq) to create the
...@@ -5392,8 +5386,8 @@ vect_permute_store_chain (vec<tree> dr_chain, ...@@ -5392,8 +5386,8 @@ vect_permute_store_chain (vec<tree> dr_chain,
result = realign_load (msq, lsq, realignment_token); result = realign_load (msq, lsq, realignment_token);
Input: Input:
STMT - (scalar) load stmt to be vectorized. This load accesses STMT_INFO - (scalar) load stmt to be vectorized. This load accesses
a memory location that may be unaligned. a memory location that may be unaligned.
BSI - place where new code is to be inserted. BSI - place where new code is to be inserted.
ALIGNMENT_SUPPORT_SCHEME - which of the two misalignment handling schemes ALIGNMENT_SUPPORT_SCHEME - which of the two misalignment handling schemes
is used. is used.
...@@ -5404,13 +5398,12 @@ vect_permute_store_chain (vec<tree> dr_chain, ...@@ -5404,13 +5398,12 @@ vect_permute_store_chain (vec<tree> dr_chain,
Return value - the result of the loop-header phi node. */ Return value - the result of the loop-header phi node. */
tree tree
vect_setup_realignment (gimple *stmt, gimple_stmt_iterator *gsi, vect_setup_realignment (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
tree *realignment_token, tree *realignment_token,
enum dr_alignment_support alignment_support_scheme, enum dr_alignment_support alignment_support_scheme,
tree init_addr, tree init_addr,
struct loop **at_loop) struct loop **at_loop)
{ {
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
tree vectype = STMT_VINFO_VECTYPE (stmt_info); tree vectype = STMT_VINFO_VECTYPE (stmt_info);
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info); struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
...@@ -5839,11 +5832,10 @@ vect_load_lanes_supported (tree vectype, unsigned HOST_WIDE_INT count, ...@@ -5839,11 +5832,10 @@ vect_load_lanes_supported (tree vectype, unsigned HOST_WIDE_INT count,
static void static void
vect_permute_load_chain (vec<tree> dr_chain, vect_permute_load_chain (vec<tree> dr_chain,
unsigned int length, unsigned int length,
gimple *stmt, stmt_vec_info stmt_info,
gimple_stmt_iterator *gsi, gimple_stmt_iterator *gsi,
vec<tree> *result_chain) vec<tree> *result_chain)
{ {
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
tree data_ref, first_vect, second_vect; tree data_ref, first_vect, second_vect;
tree perm_mask_even, perm_mask_odd; tree perm_mask_even, perm_mask_odd;
tree perm3_mask_low, perm3_mask_high; tree perm3_mask_low, perm3_mask_high;
...@@ -6043,11 +6035,10 @@ vect_permute_load_chain (vec<tree> dr_chain, ...@@ -6043,11 +6035,10 @@ vect_permute_load_chain (vec<tree> dr_chain,
static bool static bool
vect_shift_permute_load_chain (vec<tree> dr_chain, vect_shift_permute_load_chain (vec<tree> dr_chain,
unsigned int length, unsigned int length,
gimple *stmt, stmt_vec_info stmt_info,
gimple_stmt_iterator *gsi, gimple_stmt_iterator *gsi,
vec<tree> *result_chain) vec<tree> *result_chain)
{ {
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
tree vect[3], vect_shift[3], data_ref, first_vect, second_vect; tree vect[3], vect_shift[3], data_ref, first_vect, second_vect;
tree perm2_mask1, perm2_mask2, perm3_mask; tree perm2_mask1, perm2_mask2, perm3_mask;
tree select_mask, shift1_mask, shift2_mask, shift3_mask, shift4_mask; tree select_mask, shift1_mask, shift2_mask, shift3_mask, shift4_mask;
...@@ -6311,10 +6302,9 @@ vect_shift_permute_load_chain (vec<tree> dr_chain, ...@@ -6311,10 +6302,9 @@ vect_shift_permute_load_chain (vec<tree> dr_chain,
*/ */
void void
vect_transform_grouped_load (gimple *stmt, vec<tree> dr_chain, int size, vect_transform_grouped_load (stmt_vec_info stmt_info, vec<tree> dr_chain,
gimple_stmt_iterator *gsi) int size, gimple_stmt_iterator *gsi)
{ {
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
machine_mode mode; machine_mode mode;
vec<tree> result_chain = vNULL; vec<tree> result_chain = vNULL;
...@@ -6337,13 +6327,13 @@ vect_transform_grouped_load (gimple *stmt, vec<tree> dr_chain, int size, ...@@ -6337,13 +6327,13 @@ vect_transform_grouped_load (gimple *stmt, vec<tree> dr_chain, int size,
} }
/* RESULT_CHAIN contains the output of a group of grouped loads that were /* RESULT_CHAIN contains the output of a group of grouped loads that were
generated as part of the vectorization of STMT. Assign the statement generated as part of the vectorization of STMT_INFO. Assign the statement
for each vector to the associated scalar statement. */ for each vector to the associated scalar statement. */
void void
vect_record_grouped_load_vectors (gimple *stmt, vec<tree> result_chain) vect_record_grouped_load_vectors (stmt_vec_info stmt_info,
vec<tree> result_chain)
{ {
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
vec_info *vinfo = stmt_info->vinfo; vec_info *vinfo = stmt_info->vinfo;
stmt_vec_info first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info); stmt_vec_info first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
unsigned int i, gap_count; unsigned int i, gap_count;
......
gcc/tree-vect-loop.c
...@@ -648,12 +648,12 @@ vect_analyze_scalar_cycles (loop_vec_info loop_vinfo) ...@@ -648,12 +648,12 @@ vect_analyze_scalar_cycles (loop_vec_info loop_vinfo)
vect_analyze_scalar_cycles_1 (loop_vinfo, loop->inner); vect_analyze_scalar_cycles_1 (loop_vinfo, loop->inner);
} }
/* Transfer group and reduction information from STMT to its pattern stmt. */ /* Transfer group and reduction information from STMT_INFO to its
pattern stmt. */
static void static void
vect_fixup_reduc_chain (gimple *stmt) vect_fixup_reduc_chain (stmt_vec_info stmt_info)
{ {
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
stmt_vec_info firstp = STMT_VINFO_RELATED_STMT (stmt_info); stmt_vec_info firstp = STMT_VINFO_RELATED_STMT (stmt_info);
stmt_vec_info stmtp; stmt_vec_info stmtp;
gcc_assert (!REDUC_GROUP_FIRST_ELEMENT (firstp) gcc_assert (!REDUC_GROUP_FIRST_ELEMENT (firstp)
...@@ -3998,15 +3998,15 @@ vect_model_induction_cost (stmt_vec_info stmt_info, int ncopies, ...@@ -3998,15 +3998,15 @@ vect_model_induction_cost (stmt_vec_info stmt_info, int ncopies,
/* Function get_initial_def_for_reduction /* Function get_initial_def_for_reduction
Input: Input:
STMT - a stmt that performs a reduction operation in the loop. STMT_VINFO - a stmt that performs a reduction operation in the loop.
INIT_VAL - the initial value of the reduction variable INIT_VAL - the initial value of the reduction variable
Output: Output:
ADJUSTMENT_DEF - a tree that holds a value to be added to the final result ADJUSTMENT_DEF - a tree that holds a value to be added to the final result
of the reduction (used for adjusting the epilog - see below). of the reduction (used for adjusting the epilog - see below).
Return a vector variable, initialized according to the operation that STMT Return a vector variable, initialized according to the operation that
performs. This vector will be used as the initial value of the STMT_VINFO performs. This vector will be used as the initial value
vector of partial results. of the vector of partial results.
Option1 (adjust in epilog): Initialize the vector as follows: Option1 (adjust in epilog): Initialize the vector as follows:
add/bit or/xor: [0,0,...,0,0] add/bit or/xor: [0,0,...,0,0]
...@@ -4027,7 +4027,7 @@ vect_model_induction_cost (stmt_vec_info stmt_info, int ncopies, ...@@ -4027,7 +4027,7 @@ vect_model_induction_cost (stmt_vec_info stmt_info, int ncopies,
for (i=0;i<n;i++) for (i=0;i<n;i++)
s = s + a[i]; s = s + a[i];
STMT is 's = s + a[i]', and the reduction variable is 's'. STMT_VINFO is 's = s + a[i]', and the reduction variable is 's'.
For a vector of 4 units, we want to return either [0,0,0,init_val], For a vector of 4 units, we want to return either [0,0,0,init_val],
or [0,0,0,0] and let the caller know that it needs to adjust or [0,0,0,0] and let the caller know that it needs to adjust
the result at the end by 'init_val'. the result at the end by 'init_val'.
...@@ -4039,10 +4039,9 @@ vect_model_induction_cost (stmt_vec_info stmt_info, int ncopies, ...@@ -4039,10 +4039,9 @@ vect_model_induction_cost (stmt_vec_info stmt_info, int ncopies,
A cost model should help decide between these two schemes. */ A cost model should help decide between these two schemes. */
tree tree
get_initial_def_for_reduction (gimple *stmt, tree init_val, get_initial_def_for_reduction (stmt_vec_info stmt_vinfo, tree init_val,
tree *adjustment_def) tree *adjustment_def)
{ {
stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt);
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo); loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo); struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
tree scalar_type = TREE_TYPE (init_val); tree scalar_type = TREE_TYPE (init_val);
...@@ -4321,7 +4320,7 @@ get_initial_defs_for_reduction (slp_tree slp_node, ...@@ -4321,7 +4320,7 @@ get_initial_defs_for_reduction (slp_tree slp_node,
VECT_DEFS is list of vector of partial results, i.e., the lhs's of vector VECT_DEFS is list of vector of partial results, i.e., the lhs's of vector
reduction statements. reduction statements.
STMT is the scalar reduction stmt that is being vectorized. STMT_INFO is the scalar reduction stmt that is being vectorized.
NCOPIES is > 1 in case the vectorization factor (VF) is bigger than the NCOPIES is > 1 in case the vectorization factor (VF) is bigger than the
number of elements that we can fit in a vectype (nunits). In this case number of elements that we can fit in a vectype (nunits). In this case
we have to generate more than one vector stmt - i.e - we need to "unroll" we have to generate more than one vector stmt - i.e - we need to "unroll"
...@@ -4334,7 +4333,7 @@ get_initial_defs_for_reduction (slp_tree slp_node, ...@@ -4334,7 +4333,7 @@ get_initial_defs_for_reduction (slp_tree slp_node,
statement that is defined by REDUCTION_PHI. statement that is defined by REDUCTION_PHI.
DOUBLE_REDUC is TRUE if double reduction phi nodes should be handled. DOUBLE_REDUC is TRUE if double reduction phi nodes should be handled.
SLP_NODE is an SLP node containing a group of reduction statements. The SLP_NODE is an SLP node containing a group of reduction statements. The
first one in this group is STMT. first one in this group is STMT_INFO.
INDUC_VAL is for INTEGER_INDUC_COND_REDUCTION the value to use for the case INDUC_VAL is for INTEGER_INDUC_COND_REDUCTION the value to use for the case
when the COND_EXPR is never true in the loop. For MAX_EXPR, it needs to when the COND_EXPR is never true in the loop. For MAX_EXPR, it needs to
be smaller than any value of the IV in the loop, for MIN_EXPR larger than be smaller than any value of the IV in the loop, for MIN_EXPR larger than
...@@ -4359,8 +4358,8 @@ get_initial_defs_for_reduction (slp_tree slp_node, ...@@ -4359,8 +4358,8 @@ get_initial_defs_for_reduction (slp_tree slp_node,
loop: loop:
vec_def = phi <null, null> # REDUCTION_PHI vec_def = phi <null, null> # REDUCTION_PHI
VECT_DEF = vector_stmt # vectorized form of STMT VECT_DEF = vector_stmt # vectorized form of STMT_INFO
s_loop = scalar_stmt # (scalar) STMT s_loop = scalar_stmt # (scalar) STMT_INFO
loop_exit: loop_exit:
s_out0 = phi <s_loop> # (scalar) EXIT_PHI s_out0 = phi <s_loop> # (scalar) EXIT_PHI
use <s_out0> use <s_out0>
...@@ -4370,8 +4369,8 @@ get_initial_defs_for_reduction (slp_tree slp_node, ...@@ -4370,8 +4369,8 @@ get_initial_defs_for_reduction (slp_tree slp_node,
loop: loop:
vec_def = phi <vec_init, VECT_DEF> # REDUCTION_PHI vec_def = phi <vec_init, VECT_DEF> # REDUCTION_PHI
VECT_DEF = vector_stmt # vectorized form of STMT VECT_DEF = vector_stmt # vectorized form of STMT_INFO
s_loop = scalar_stmt # (scalar) STMT s_loop = scalar_stmt # (scalar) STMT_INFO
loop_exit: loop_exit:
s_out0 = phi <s_loop> # (scalar) EXIT_PHI s_out0 = phi <s_loop> # (scalar) EXIT_PHI
v_out1 = phi <VECT_DEF> # NEW_EXIT_PHI v_out1 = phi <VECT_DEF> # NEW_EXIT_PHI
...@@ -4383,7 +4382,8 @@ get_initial_defs_for_reduction (slp_tree slp_node, ...@@ -4383,7 +4382,8 @@ get_initial_defs_for_reduction (slp_tree slp_node,
*/ */
static void static void
vect_create_epilog_for_reduction (vec<tree> vect_defs, gimple *stmt, vect_create_epilog_for_reduction (vec<tree> vect_defs,
stmt_vec_info stmt_info,
gimple *reduc_def_stmt, gimple *reduc_def_stmt,
int ncopies, internal_fn reduc_fn, int ncopies, internal_fn reduc_fn,
vec<stmt_vec_info> reduction_phis, vec<stmt_vec_info> reduction_phis,
...@@ -4393,7 +4393,6 @@ vect_create_epilog_for_reduction (vec<tree> vect_defs, gimple *stmt, ...@@ -4393,7 +4393,6 @@ vect_create_epilog_for_reduction (vec<tree> vect_defs, gimple *stmt,
tree induc_val, enum tree_code induc_code, tree induc_val, enum tree_code induc_code,
tree neutral_op) tree neutral_op)
{ {
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
stmt_vec_info prev_phi_info; stmt_vec_info prev_phi_info;
tree vectype; tree vectype;
machine_mode mode; machine_mode mode;
...@@ -5816,9 +5815,9 @@ vect_expand_fold_left (gimple_stmt_iterator *gsi, tree scalar_dest, ...@@ -5816,9 +5815,9 @@ vect_expand_fold_left (gimple_stmt_iterator *gsi, tree scalar_dest,
return lhs; return lhs;
} }
/* Perform an in-order reduction (FOLD_LEFT_REDUCTION). STMT is the /* Perform an in-order reduction (FOLD_LEFT_REDUCTION). STMT_INFO is the
statement that sets the live-out value. REDUC_DEF_STMT is the phi statement that sets the live-out value. REDUC_DEF_STMT is the phi
statement. CODE is the operation performed by STMT and OPS are statement. CODE is the operation performed by STMT_INFO and OPS are
its scalar operands. REDUC_INDEX is the index of the operand in its scalar operands. REDUC_INDEX is the index of the operand in
OPS that is set by REDUC_DEF_STMT. REDUC_FN is the function that OPS that is set by REDUC_DEF_STMT. REDUC_FN is the function that
implements in-order reduction, or IFN_LAST if we should open-code it. implements in-order reduction, or IFN_LAST if we should open-code it.
...@@ -5826,14 +5825,14 @@ vect_expand_fold_left (gimple_stmt_iterator *gsi, tree scalar_dest, ...@@ -5826,14 +5825,14 @@ vect_expand_fold_left (gimple_stmt_iterator *gsi, tree scalar_dest,
that should be used to control the operation in a fully-masked loop. */ that should be used to control the operation in a fully-masked loop. */
static bool static bool
vectorize_fold_left_reduction (gimple *stmt, gimple_stmt_iterator *gsi, vectorize_fold_left_reduction (stmt_vec_info stmt_info,
gimple_stmt_iterator *gsi,
stmt_vec_info *vec_stmt, slp_tree slp_node, stmt_vec_info *vec_stmt, slp_tree slp_node,
gimple *reduc_def_stmt, gimple *reduc_def_stmt,
tree_code code, internal_fn reduc_fn, tree_code code, internal_fn reduc_fn,
tree ops[3], tree vectype_in, tree ops[3], tree vectype_in,
int reduc_index, vec_loop_masks *masks) int reduc_index, vec_loop_masks *masks)
{ {
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo); struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
tree vectype_out = STMT_VINFO_VECTYPE (stmt_info); tree vectype_out = STMT_VINFO_VECTYPE (stmt_info);
...@@ -5962,16 +5961,16 @@ vectorize_fold_left_reduction (gimple *stmt, gimple_stmt_iterator *gsi, ...@@ -5962,16 +5961,16 @@ vectorize_fold_left_reduction (gimple *stmt, gimple_stmt_iterator *gsi,
/* Function is_nonwrapping_integer_induction. /* Function is_nonwrapping_integer_induction.
Check if STMT (which is part of loop LOOP) both increments and Check if STMT_VINO (which is part of loop LOOP) both increments and
does not cause overflow. */ does not cause overflow. */
static bool static bool
is_nonwrapping_integer_induction (gimple *stmt, struct loop *loop) is_nonwrapping_integer_induction (stmt_vec_info stmt_vinfo, struct loop *loop)
{ {
stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt); gphi *phi = as_a <gphi *> (stmt_vinfo->stmt);
tree base = STMT_VINFO_LOOP_PHI_EVOLUTION_BASE_UNCHANGED (stmt_vinfo); tree base = STMT_VINFO_LOOP_PHI_EVOLUTION_BASE_UNCHANGED (stmt_vinfo);
tree step = STMT_VINFO_LOOP_PHI_EVOLUTION_PART (stmt_vinfo); tree step = STMT_VINFO_LOOP_PHI_EVOLUTION_PART (stmt_vinfo);
tree lhs_type = TREE_TYPE (gimple_phi_result (stmt)); tree lhs_type = TREE_TYPE (gimple_phi_result (phi));
widest_int ni, max_loop_value, lhs_max; widest_int ni, max_loop_value, lhs_max;
wi::overflow_type overflow = wi::OVF_NONE; wi::overflow_type overflow = wi::OVF_NONE;
...@@ -6004,17 +6003,18 @@ is_nonwrapping_integer_induction (gimple *stmt, struct loop *loop) ...@@ -6004,17 +6003,18 @@ is_nonwrapping_integer_induction (gimple *stmt, struct loop *loop)
/* Function vectorizable_reduction. /* Function vectorizable_reduction.
Check if STMT performs a reduction operation that can be vectorized. Check if STMT_INFO performs a reduction operation that can be vectorized.
If VEC_STMT is also passed, vectorize the STMT: create a vectorized If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
stmt to replace it, put it in VEC_STMT, and insert it at GSI. stmt to replace it, put it in VEC_STMT, and insert it at GSI.
Return FALSE if not a vectorizable STMT, TRUE otherwise. Return true if STMT_INFO is vectorizable in this way.
This function also handles reduction idioms (patterns) that have been This function also handles reduction idioms (patterns) that have been
recognized in advance during vect_pattern_recog. In this case, STMT may be recognized in advance during vect_pattern_recog. In this case, STMT_INFO
of this form: may be of this form:
X = pattern_expr (arg0, arg1, ..., X) X = pattern_expr (arg0, arg1, ..., X)
and it's STMT_VINFO_RELATED_STMT points to the last stmt in the original and its STMT_VINFO_RELATED_STMT points to the last stmt in the original
sequence that had been detected and replaced by the pattern-stmt (STMT). sequence that had been detected and replaced by the pattern-stmt
(STMT_INFO).
This function also handles reduction of condition expressions, for example: This function also handles reduction of condition expressions, for example:
for (int i = 0; i < N; i++) for (int i = 0; i < N; i++)
...@@ -6026,9 +6026,9 @@ is_nonwrapping_integer_induction (gimple *stmt, struct loop *loop) ...@@ -6026,9 +6026,9 @@ is_nonwrapping_integer_induction (gimple *stmt, struct loop *loop)
index into the vector of results. index into the vector of results.
In some cases of reduction patterns, the type of the reduction variable X is In some cases of reduction patterns, the type of the reduction variable X is
different than the type of the other arguments of STMT. different than the type of the other arguments of STMT_INFO.
In such cases, the vectype that is used when transforming STMT into a vector In such cases, the vectype that is used when transforming STMT_INFO into
stmt is different than the vectype that is used to determine the a vector stmt is different than the vectype that is used to determine the
vectorization factor, because it consists of a different number of elements vectorization factor, because it consists of a different number of elements
than the actual number of elements that are being operated upon in parallel. than the actual number of elements that are being operated upon in parallel.
...@@ -6052,14 +6052,13 @@ is_nonwrapping_integer_induction (gimple *stmt, struct loop *loop) ...@@ -6052,14 +6052,13 @@ is_nonwrapping_integer_induction (gimple *stmt, struct loop *loop)
does *NOT* necessarily hold for reduction patterns. */ does *NOT* necessarily hold for reduction patterns. */
bool bool
vectorizable_reduction (gimple *stmt, gimple_stmt_iterator *gsi, vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
stmt_vec_info *vec_stmt, slp_tree slp_node, stmt_vec_info *vec_stmt, slp_tree slp_node,
slp_instance slp_node_instance, slp_instance slp_node_instance,
stmt_vector_for_cost *cost_vec) stmt_vector_for_cost *cost_vec)
{ {
tree vec_dest; tree vec_dest;
tree scalar_dest; tree scalar_dest;
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
tree vectype_out = STMT_VINFO_VECTYPE (stmt_info); tree vectype_out = STMT_VINFO_VECTYPE (stmt_info);
tree vectype_in = NULL_TREE; tree vectype_in = NULL_TREE;
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
...@@ -6247,7 +6246,7 @@ vectorizable_reduction (gimple *stmt, gimple_stmt_iterator *gsi, ...@@ -6247,7 +6246,7 @@ vectorizable_reduction (gimple *stmt, gimple_stmt_iterator *gsi,
inside the loop body. The last operand is the reduction variable, inside the loop body. The last operand is the reduction variable,
which is defined by the loop-header-phi. */ which is defined by the loop-header-phi. */
gcc_assert (is_gimple_assign (stmt)); gassign *stmt = as_a <gassign *> (stmt_info->stmt);
/* Flatten RHS. */ /* Flatten RHS. */
switch (get_gimple_rhs_class (gimple_assign_rhs_code (stmt))) switch (get_gimple_rhs_class (gimple_assign_rhs_code (stmt)))
...@@ -7240,18 +7239,17 @@ vect_worthwhile_without_simd_p (vec_info *vinfo, tree_code code) ...@@ -7240,18 +7239,17 @@ vect_worthwhile_without_simd_p (vec_info *vinfo, tree_code code)
/* Function vectorizable_induction /* Function vectorizable_induction
Check if PHI performs an induction computation that can be vectorized. Check if STMT_INFO performs an induction computation that can be vectorized.
If VEC_STMT is also passed, vectorize the induction PHI: create a vectorized If VEC_STMT is also passed, vectorize the induction PHI: create a vectorized
phi to replace it, put it in VEC_STMT, and add it to the same basic block. phi to replace it, put it in VEC_STMT, and add it to the same basic block.
Return FALSE if not a vectorizable STMT, TRUE otherwise. */ Return true if STMT_INFO is vectorizable in this way. */
bool bool
vectorizable_induction (gimple *phi, vectorizable_induction (stmt_vec_info stmt_info,
gimple_stmt_iterator *gsi ATTRIBUTE_UNUSED, gimple_stmt_iterator *gsi ATTRIBUTE_UNUSED,
stmt_vec_info *vec_stmt, slp_tree slp_node, stmt_vec_info *vec_stmt, slp_tree slp_node,
stmt_vector_for_cost *cost_vec) stmt_vector_for_cost *cost_vec)
{ {
stmt_vec_info stmt_info = vinfo_for_stmt (phi);
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo); struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
unsigned ncopies; unsigned ncopies;
...@@ -7276,9 +7274,9 @@ vectorizable_induction (gimple *phi, ...@@ -7276,9 +7274,9 @@ vectorizable_induction (gimple *phi,
edge latch_e; edge latch_e;
tree loop_arg; tree loop_arg;
gimple_stmt_iterator si; gimple_stmt_iterator si;
basic_block bb = gimple_bb (phi);
if (gimple_code (phi) != GIMPLE_PHI) gphi *phi = dyn_cast <gphi *> (stmt_info->stmt);
if (!phi)
return false; return false;
if (!STMT_VINFO_RELEVANT_P (stmt_info)) if (!STMT_VINFO_RELEVANT_P (stmt_info))
...@@ -7426,6 +7424,7 @@ vectorizable_induction (gimple *phi, ...@@ -7426,6 +7424,7 @@ vectorizable_induction (gimple *phi,
} }
/* Find the first insertion point in the BB. */ /* Find the first insertion point in the BB. */
basic_block bb = gimple_bb (phi);
si = gsi_after_labels (bb); si = gsi_after_labels (bb);
/* For SLP induction we have to generate several IVs as for example /* For SLP induction we have to generate several IVs as for example
...@@ -7791,17 +7790,16 @@ vectorizable_induction (gimple *phi, ...@@ -7791,17 +7790,16 @@ vectorizable_induction (gimple *phi,
/* Function vectorizable_live_operation. /* Function vectorizable_live_operation.
STMT computes a value that is used outside the loop. Check if STMT_INFO computes a value that is used outside the loop. Check if
it can be supported. */ it can be supported. */
bool bool
vectorizable_live_operation (gimple *stmt, vectorizable_live_operation (stmt_vec_info stmt_info,
gimple_stmt_iterator *gsi ATTRIBUTE_UNUSED, gimple_stmt_iterator *gsi ATTRIBUTE_UNUSED,
slp_tree slp_node, int slp_index, slp_tree slp_node, int slp_index,
stmt_vec_info *vec_stmt, stmt_vec_info *vec_stmt,
stmt_vector_for_cost *) stmt_vector_for_cost *)
{ {
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo); struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
imm_use_iterator imm_iter; imm_use_iterator imm_iter;
...@@ -7908,8 +7906,9 @@ vectorizable_live_operation (gimple *stmt, ...@@ -7908,8 +7906,9 @@ vectorizable_live_operation (gimple *stmt,
} }
/* If stmt has a related stmt, then use that for getting the lhs. */ /* If stmt has a related stmt, then use that for getting the lhs. */
if (is_pattern_stmt_p (stmt_info)) gimple *stmt = (is_pattern_stmt_p (stmt_info)
stmt = STMT_VINFO_RELATED_STMT (stmt_info); ? STMT_VINFO_RELATED_STMT (stmt_info)->stmt
: stmt_info->stmt);
lhs = (is_a <gphi *> (stmt)) ? gimple_phi_result (stmt) lhs = (is_a <gphi *> (stmt)) ? gimple_phi_result (stmt)
: gimple_get_lhs (stmt); : gimple_get_lhs (stmt);
...@@ -8010,17 +8009,17 @@ vectorizable_live_operation (gimple *stmt, ...@@ -8010,17 +8009,17 @@ vectorizable_live_operation (gimple *stmt,
return true; return true;
} }
/* Kill any debug uses outside LOOP of SSA names defined in STMT. */ /* Kill any debug uses outside LOOP of SSA names defined in STMT_INFO. */
static void static void
vect_loop_kill_debug_uses (struct loop *loop, gimple *stmt) vect_loop_kill_debug_uses (struct loop *loop, stmt_vec_info stmt_info)
{ {
ssa_op_iter op_iter; ssa_op_iter op_iter;
imm_use_iterator imm_iter; imm_use_iterator imm_iter;
def_operand_p def_p; def_operand_p def_p;
gimple *ustmt; gimple *ustmt;
FOR_EACH_PHI_OR_STMT_DEF (def_p, stmt, op_iter, SSA_OP_DEF) FOR_EACH_PHI_OR_STMT_DEF (def_p, stmt_info->stmt, op_iter, SSA_OP_DEF)
{ {
FOR_EACH_IMM_USE_STMT (ustmt, imm_iter, DEF_FROM_PTR (def_p)) FOR_EACH_IMM_USE_STMT (ustmt, imm_iter, DEF_FROM_PTR (def_p))
{ {
......
gcc/tree-vect-patterns.c
...@@ -236,22 +236,20 @@ vect_get_internal_def (vec_info *vinfo, tree op) ...@@ -236,22 +236,20 @@ vect_get_internal_def (vec_info *vinfo, tree op)
return NULL; return NULL;
} }
/* Check whether NAME, an ssa-name used in USE_STMT, /* Check whether NAME, an ssa-name used in STMT_VINFO,
is a result of a type promotion, such that: is a result of a type promotion, such that:
DEF_STMT: NAME = NOP (name0) DEF_STMT: NAME = NOP (name0)
If CHECK_SIGN is TRUE, check that either both types are signed or both are If CHECK_SIGN is TRUE, check that either both types are signed or both are
unsigned. */ unsigned. */
static bool static bool
type_conversion_p (tree name, gimple *use_stmt, bool check_sign, type_conversion_p (tree name, stmt_vec_info stmt_vinfo, bool check_sign,
tree *orig_type, gimple **def_stmt, bool *promotion) tree *orig_type, gimple **def_stmt, bool *promotion)
{ {
stmt_vec_info stmt_vinfo;
tree type = TREE_TYPE (name); tree type = TREE_TYPE (name);
tree oprnd0; tree oprnd0;
enum vect_def_type dt; enum vect_def_type dt;
stmt_vinfo = vinfo_for_stmt (use_stmt);
stmt_vec_info def_stmt_info; stmt_vec_info def_stmt_info;
if (!vect_is_simple_use (name, stmt_vinfo->vinfo, &dt, &def_stmt_info, if (!vect_is_simple_use (name, stmt_vinfo->vinfo, &dt, &def_stmt_info,
def_stmt)) def_stmt))
...@@ -3498,15 +3496,13 @@ sort_after_uid (const void *p1, const void *p2) ...@@ -3498,15 +3496,13 @@ sort_after_uid (const void *p1, const void *p2)
} }
/* Create pattern stmts for all stmts participating in the bool pattern /* Create pattern stmts for all stmts participating in the bool pattern
specified by BOOL_STMT_SET and its root STMT with the desired type specified by BOOL_STMT_SET and its root STMT_INFO with the desired type
OUT_TYPE. Return the def of the pattern root. */ OUT_TYPE. Return the def of the pattern root. */
static tree static tree
adjust_bool_stmts (hash_set <gimple *> &bool_stmt_set, adjust_bool_stmts (hash_set <gimple *> &bool_stmt_set,
tree out_type, gimple *stmt) tree out_type, stmt_vec_info stmt_info)
{ {
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
/* Gather original stmts in the bool pattern in their order of appearance /* Gather original stmts in the bool pattern in their order of appearance
in the IL. */ in the IL. */
auto_vec<gimple *> bool_stmts (bool_stmt_set.elements ()); auto_vec<gimple *> bool_stmts (bool_stmt_set.elements ());
...@@ -4126,19 +4122,19 @@ vect_recog_mask_conversion_pattern (stmt_vec_info stmt_vinfo, tree *type_out) ...@@ -4126,19 +4122,19 @@ vect_recog_mask_conversion_pattern (stmt_vec_info stmt_vinfo, tree *type_out)
return pattern_stmt; return pattern_stmt;
} }
/* STMT is a load or store. If the load or store is conditional, return /* STMT_INFO is a load or store. If the load or store is conditional, return
the boolean condition under which it occurs, otherwise return null. */ the boolean condition under which it occurs, otherwise return null. */
static tree static tree
vect_get_load_store_mask (gimple *stmt) vect_get_load_store_mask (stmt_vec_info stmt_info)
{ {
if (gassign *def_assign = dyn_cast <gassign *> (stmt)) if (gassign *def_assign = dyn_cast <gassign *> (stmt_info->stmt))
{ {
gcc_assert (gimple_assign_single_p (def_assign)); gcc_assert (gimple_assign_single_p (def_assign));
return NULL_TREE; return NULL_TREE;
} }
if (gcall *def_call = dyn_cast <gcall *> (stmt)) if (gcall *def_call = dyn_cast <gcall *> (stmt_info->stmt))
{ {
internal_fn ifn = gimple_call_internal_fn (def_call); internal_fn ifn = gimple_call_internal_fn (def_call);
int mask_index = internal_fn_mask_index (ifn); int mask_index = internal_fn_mask_index (ifn);
......
gcc/tree-vect-slp.c
...@@ -195,14 +195,14 @@ vect_free_oprnd_info (vec<slp_oprnd_info> &oprnds_info) ...@@ -195,14 +195,14 @@ vect_free_oprnd_info (vec<slp_oprnd_info> &oprnds_info)
} }
/* Find the place of the data-ref in STMT in the interleaving chain that starts /* Find the place of the data-ref in STMT_INFO in the interleaving chain
from FIRST_STMT. Return -1 if the data-ref is not a part of the chain. */ that starts from FIRST_STMT_INFO. Return -1 if the data-ref is not a part
of the chain. */
int int
vect_get_place_in_interleaving_chain (gimple *stmt, gimple *first_stmt) vect_get_place_in_interleaving_chain (stmt_vec_info stmt_info,
stmt_vec_info first_stmt_info)
{ {
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
stmt_vec_info first_stmt_info = vinfo_for_stmt (first_stmt);
stmt_vec_info next_stmt_info = first_stmt_info; stmt_vec_info next_stmt_info = first_stmt_info;
int result = 0; int result = 0;
...@@ -1918,9 +1918,8 @@ calculate_unrolling_factor (poly_uint64 nunits, unsigned int group_size) ...@@ -1918,9 +1918,8 @@ calculate_unrolling_factor (poly_uint64 nunits, unsigned int group_size)
static bool static bool
vect_analyze_slp_instance (vec_info *vinfo, vect_analyze_slp_instance (vec_info *vinfo,
gimple *stmt, unsigned max_tree_size) stmt_vec_info stmt_info, unsigned max_tree_size)
{ {
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
slp_instance new_instance; slp_instance new_instance;
slp_tree node; slp_tree node;
unsigned int group_size; unsigned int group_size;
...@@ -3118,13 +3117,12 @@ vect_slp_bb (basic_block bb) ...@@ -3118,13 +3117,12 @@ vect_slp_bb (basic_block bb)
/* Return 1 if vector type of boolean constant which is OPNUM /* Return 1 if vector type of boolean constant which is OPNUM
operand in statement STMT is a boolean vector. */ operand in statement STMT_VINFO is a boolean vector. */
static bool static bool
vect_mask_constant_operand_p (gimple *stmt, int opnum) vect_mask_constant_operand_p (stmt_vec_info stmt_vinfo, int opnum)
{ {
stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt); enum tree_code code = gimple_expr_code (stmt_vinfo->stmt);
enum tree_code code = gimple_expr_code (stmt);
tree op, vectype; tree op, vectype;
enum vect_def_type dt; enum vect_def_type dt;
...@@ -3132,6 +3130,7 @@ vect_mask_constant_operand_p (gimple *stmt, int opnum) ...@@ -3132,6 +3130,7 @@ vect_mask_constant_operand_p (gimple *stmt, int opnum)
on the other comparison operand. */ on the other comparison operand. */
if (TREE_CODE_CLASS (code) == tcc_comparison) if (TREE_CODE_CLASS (code) == tcc_comparison)
{ {
gassign *stmt = as_a <gassign *> (stmt_vinfo->stmt);
if (opnum) if (opnum)
op = gimple_assign_rhs1 (stmt); op = gimple_assign_rhs1 (stmt);
else else
...@@ -3145,6 +3144,7 @@ vect_mask_constant_operand_p (gimple *stmt, int opnum) ...@@ -3145,6 +3144,7 @@ vect_mask_constant_operand_p (gimple *stmt, int opnum)
if (code == COND_EXPR) if (code == COND_EXPR)
{ {
gassign *stmt = as_a <gassign *> (stmt_vinfo->stmt);
tree cond = gimple_assign_rhs1 (stmt); tree cond = gimple_assign_rhs1 (stmt);
if (TREE_CODE (cond) == SSA_NAME) if (TREE_CODE (cond) == SSA_NAME)
......
gcc/tree-vect-stmts.c
...@@ -192,13 +192,12 @@ vect_clobber_variable (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, ...@@ -192,13 +192,12 @@ vect_clobber_variable (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
/* Function vect_mark_relevant. /* Function vect_mark_relevant.
Mark STMT as "relevant for vectorization" and add it to WORKLIST. */ Mark STMT_INFO as "relevant for vectorization" and add it to WORKLIST. */
static void static void
vect_mark_relevant (vec<stmt_vec_info> *worklist, gimple *stmt, vect_mark_relevant (vec<stmt_vec_info> *worklist, stmt_vec_info stmt_info,
enum vect_relevant relevant, bool live_p) enum vect_relevant relevant, bool live_p)
{ {
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
enum vect_relevant save_relevant = STMT_VINFO_RELEVANT (stmt_info); enum vect_relevant save_relevant = STMT_VINFO_RELEVANT (stmt_info);
bool save_live_p = STMT_VINFO_LIVE_P (stmt_info); bool save_live_p = STMT_VINFO_LIVE_P (stmt_info);
...@@ -229,7 +228,6 @@ vect_mark_relevant (vec<stmt_vec_info> *worklist, gimple *stmt, ...@@ -229,7 +228,6 @@ vect_mark_relevant (vec<stmt_vec_info> *worklist, gimple *stmt,
gcc_assert (STMT_VINFO_RELATED_STMT (stmt_info) == old_stmt_info); gcc_assert (STMT_VINFO_RELATED_STMT (stmt_info) == old_stmt_info);
save_relevant = STMT_VINFO_RELEVANT (stmt_info); save_relevant = STMT_VINFO_RELEVANT (stmt_info);
save_live_p = STMT_VINFO_LIVE_P (stmt_info); save_live_p = STMT_VINFO_LIVE_P (stmt_info);
stmt = stmt_info->stmt;
} }
STMT_VINFO_LIVE_P (stmt_info) |= live_p; STMT_VINFO_LIVE_P (stmt_info) |= live_p;
...@@ -251,15 +249,17 @@ vect_mark_relevant (vec<stmt_vec_info> *worklist, gimple *stmt, ...@@ -251,15 +249,17 @@ vect_mark_relevant (vec<stmt_vec_info> *worklist, gimple *stmt,
/* Function is_simple_and_all_uses_invariant /* Function is_simple_and_all_uses_invariant
Return true if STMT is simple and all uses of it are invariant. */ Return true if STMT_INFO is simple and all uses of it are invariant. */
bool bool
is_simple_and_all_uses_invariant (gimple *stmt, loop_vec_info loop_vinfo) is_simple_and_all_uses_invariant (stmt_vec_info stmt_info,
loop_vec_info loop_vinfo)
{ {
tree op; tree op;
ssa_op_iter iter; ssa_op_iter iter;
if (!is_gimple_assign (stmt)) gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
if (!stmt)
return false; return false;
FOR_EACH_SSA_TREE_OPERAND (op, stmt, iter, SSA_OP_USE) FOR_EACH_SSA_TREE_OPERAND (op, stmt, iter, SSA_OP_USE)
...@@ -361,14 +361,13 @@ vect_stmt_relevant_p (stmt_vec_info stmt_info, loop_vec_info loop_vinfo, ...@@ -361,14 +361,13 @@ vect_stmt_relevant_p (stmt_vec_info stmt_info, loop_vec_info loop_vinfo,
/* Function exist_non_indexing_operands_for_use_p /* Function exist_non_indexing_operands_for_use_p
USE is one of the uses attached to STMT. Check if USE is USE is one of the uses attached to STMT_INFO. Check if USE is
used in STMT for anything other than indexing an array. */ used in STMT_INFO for anything other than indexing an array. */
static bool static bool
exist_non_indexing_operands_for_use_p (tree use, gimple *stmt) exist_non_indexing_operands_for_use_p (tree use, stmt_vec_info stmt_info)
{ {
tree operand; tree operand;
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
/* USE corresponds to some operand in STMT. If there is no data /* USE corresponds to some operand in STMT. If there is no data
reference in STMT, then any operand that corresponds to USE reference in STMT, then any operand that corresponds to USE
...@@ -428,7 +427,7 @@ exist_non_indexing_operands_for_use_p (tree use, gimple *stmt) ...@@ -428,7 +427,7 @@ exist_non_indexing_operands_for_use_p (tree use, gimple *stmt)
Function process_use. Function process_use.
Inputs: Inputs:
- a USE in STMT in a loop represented by LOOP_VINFO - a USE in STMT_VINFO in a loop represented by LOOP_VINFO
- RELEVANT - enum value to be set in the STMT_VINFO of the stmt - RELEVANT - enum value to be set in the STMT_VINFO of the stmt
that defined USE. This is done by calling mark_relevant and passing it that defined USE. This is done by calling mark_relevant and passing it
the WORKLIST (to add DEF_STMT to the WORKLIST in case it is relevant). the WORKLIST (to add DEF_STMT to the WORKLIST in case it is relevant).
...@@ -438,25 +437,24 @@ exist_non_indexing_operands_for_use_p (tree use, gimple *stmt) ...@@ -438,25 +437,24 @@ exist_non_indexing_operands_for_use_p (tree use, gimple *stmt)
Outputs: Outputs:
Generally, LIVE_P and RELEVANT are used to define the liveness and Generally, LIVE_P and RELEVANT are used to define the liveness and
relevance info of the DEF_STMT of this USE: relevance info of the DEF_STMT of this USE:
STMT_VINFO_LIVE_P (DEF_STMT_info) <-- live_p STMT_VINFO_LIVE_P (DEF_stmt_vinfo) <-- live_p
STMT_VINFO_RELEVANT (DEF_STMT_info) <-- relevant STMT_VINFO_RELEVANT (DEF_stmt_vinfo) <-- relevant
Exceptions: Exceptions:
- case 1: If USE is used only for address computations (e.g. array indexing), - case 1: If USE is used only for address computations (e.g. array indexing),
which does not need to be directly vectorized, then the liveness/relevance which does not need to be directly vectorized, then the liveness/relevance
of the respective DEF_STMT is left unchanged. of the respective DEF_STMT is left unchanged.
- case 2: If STMT is a reduction phi and DEF_STMT is a reduction stmt, we - case 2: If STMT_VINFO is a reduction phi and DEF_STMT is a reduction stmt,
skip DEF_STMT cause it had already been processed. we skip DEF_STMT cause it had already been processed.
- case 3: If DEF_STMT and STMT are in different nests, then "relevant" will - case 3: If DEF_STMT and STMT_VINFO are in different nests, then
be modified accordingly. "relevant" will be modified accordingly.
Return true if everything is as expected. Return false otherwise. */ Return true if everything is as expected. Return false otherwise. */
static bool static bool
process_use (gimple *stmt, tree use, loop_vec_info loop_vinfo, process_use (stmt_vec_info stmt_vinfo, tree use, loop_vec_info loop_vinfo,
enum vect_relevant relevant, vec<stmt_vec_info> *worklist, enum vect_relevant relevant, vec<stmt_vec_info> *worklist,
bool force) bool force)
{ {
stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt);
stmt_vec_info dstmt_vinfo; stmt_vec_info dstmt_vinfo;
basic_block bb, def_bb; basic_block bb, def_bb;
enum vect_def_type dt; enum vect_def_type dt;
...@@ -1342,12 +1340,12 @@ vect_get_load_cost (stmt_vec_info stmt_info, int ncopies, ...@@ -1342,12 +1340,12 @@ vect_get_load_cost (stmt_vec_info stmt_info, int ncopies,
} }
/* Insert the new stmt NEW_STMT at *GSI or at the appropriate place in /* Insert the new stmt NEW_STMT at *GSI or at the appropriate place in
the loop preheader for the vectorized stmt STMT. */ the loop preheader for the vectorized stmt STMT_VINFO. */
static void static void
vect_init_vector_1 (gimple *stmt, gimple *new_stmt, gimple_stmt_iterator *gsi) vect_init_vector_1 (stmt_vec_info stmt_vinfo, gimple *new_stmt,
gimple_stmt_iterator *gsi)
{ {
stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt);
if (gsi) if (gsi)
vect_finish_stmt_generation (stmt_vinfo, new_stmt, gsi); vect_finish_stmt_generation (stmt_vinfo, new_stmt, gsi);
else else
...@@ -1396,12 +1394,12 @@ vect_init_vector_1 (gimple *stmt, gimple *new_stmt, gimple_stmt_iterator *gsi) ...@@ -1396,12 +1394,12 @@ vect_init_vector_1 (gimple *stmt, gimple *new_stmt, gimple_stmt_iterator *gsi)
Place the initialization at BSI if it is not NULL. Otherwise, place the Place the initialization at BSI if it is not NULL. Otherwise, place the
initialization at the loop preheader. initialization at the loop preheader.
Return the DEF of INIT_STMT. Return the DEF of INIT_STMT.
It will be used in the vectorization of STMT. */ It will be used in the vectorization of STMT_INFO. */
tree tree
vect_init_vector (gimple *stmt, tree val, tree type, gimple_stmt_iterator *gsi) vect_init_vector (stmt_vec_info stmt_info, tree val, tree type,
gimple_stmt_iterator *gsi)
{ {
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
gimple *init_stmt; gimple *init_stmt;
tree new_temp; tree new_temp;
...@@ -1456,15 +1454,15 @@ vect_init_vector (gimple *stmt, tree val, tree type, gimple_stmt_iterator *gsi) ...@@ -1456,15 +1454,15 @@ vect_init_vector (gimple *stmt, tree val, tree type, gimple_stmt_iterator *gsi)
/* Function vect_get_vec_def_for_operand_1. /* Function vect_get_vec_def_for_operand_1.
For a defining stmt DEF_STMT of a scalar stmt, return a vector def with type For a defining stmt DEF_STMT_INFO of a scalar stmt, return a vector def
DT that will be used in the vectorized stmt. */ with type DT that will be used in the vectorized stmt. */
tree tree
vect_get_vec_def_for_operand_1 (gimple *def_stmt, enum vect_def_type dt) vect_get_vec_def_for_operand_1 (stmt_vec_info def_stmt_info,
enum vect_def_type dt)
{ {
tree vec_oprnd; tree vec_oprnd;
stmt_vec_info vec_stmt_info; stmt_vec_info vec_stmt_info;
stmt_vec_info def_stmt_info = NULL;
switch (dt) switch (dt)
{ {
...@@ -1478,8 +1476,6 @@ vect_get_vec_def_for_operand_1 (gimple *def_stmt, enum vect_def_type dt) ...@@ -1478,8 +1476,6 @@ vect_get_vec_def_for_operand_1 (gimple *def_stmt, enum vect_def_type dt)
case vect_internal_def: case vect_internal_def:
{ {
/* Get the def from the vectorized stmt. */ /* Get the def from the vectorized stmt. */
def_stmt_info = vinfo_for_stmt (def_stmt);
vec_stmt_info = STMT_VINFO_VEC_STMT (def_stmt_info); vec_stmt_info = STMT_VINFO_VEC_STMT (def_stmt_info);
/* Get vectorized pattern statement. */ /* Get vectorized pattern statement. */
if (!vec_stmt_info if (!vec_stmt_info
...@@ -1501,10 +1497,9 @@ vect_get_vec_def_for_operand_1 (gimple *def_stmt, enum vect_def_type dt) ...@@ -1501,10 +1497,9 @@ vect_get_vec_def_for_operand_1 (gimple *def_stmt, enum vect_def_type dt)
case vect_nested_cycle: case vect_nested_cycle:
case vect_induction_def: case vect_induction_def:
{ {
gcc_assert (gimple_code (def_stmt) == GIMPLE_PHI); gcc_assert (gimple_code (def_stmt_info->stmt) == GIMPLE_PHI);
/* Get the def from the vectorized stmt. */ /* Get the def from the vectorized stmt. */
def_stmt_info = vinfo_for_stmt (def_stmt);
vec_stmt_info = STMT_VINFO_VEC_STMT (def_stmt_info); vec_stmt_info = STMT_VINFO_VEC_STMT (def_stmt_info);
if (gphi *phi = dyn_cast <gphi *> (vec_stmt_info->stmt)) if (gphi *phi = dyn_cast <gphi *> (vec_stmt_info->stmt))
vec_oprnd = PHI_RESULT (phi); vec_oprnd = PHI_RESULT (phi);
...@@ -1521,8 +1516,8 @@ vect_get_vec_def_for_operand_1 (gimple *def_stmt, enum vect_def_type dt) ...@@ -1521,8 +1516,8 @@ vect_get_vec_def_for_operand_1 (gimple *def_stmt, enum vect_def_type dt)
/* Function vect_get_vec_def_for_operand. /* Function vect_get_vec_def_for_operand.
OP is an operand in STMT. This function returns a (vector) def that will be OP is an operand in STMT_VINFO. This function returns a (vector) def
used in the vectorized stmt for STMT. that will be used in the vectorized stmt for STMT_VINFO.
In the case that OP is an SSA_NAME which is defined in the loop, then In the case that OP is an SSA_NAME which is defined in the loop, then
STMT_VINFO_VEC_STMT of the defining stmt holds the relevant def. STMT_VINFO_VEC_STMT of the defining stmt holds the relevant def.
...@@ -1532,12 +1527,11 @@ vect_get_vec_def_for_operand_1 (gimple *def_stmt, enum vect_def_type dt) ...@@ -1532,12 +1527,11 @@ vect_get_vec_def_for_operand_1 (gimple *def_stmt, enum vect_def_type dt)
vector invariant. */ vector invariant. */
tree tree
vect_get_vec_def_for_operand (tree op, gimple *stmt, tree vectype) vect_get_vec_def_for_operand (tree op, stmt_vec_info stmt_vinfo, tree vectype)
{ {
gimple *def_stmt; gimple *def_stmt;
enum vect_def_type dt; enum vect_def_type dt;
bool is_simple_use; bool is_simple_use;
stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt);
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo); loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
if (dump_enabled_p ()) if (dump_enabled_p ())
...@@ -1683,12 +1677,11 @@ vect_get_vec_defs_for_stmt_copy (enum vect_def_type *dt, ...@@ -1683,12 +1677,11 @@ vect_get_vec_defs_for_stmt_copy (enum vect_def_type *dt,
/* Get vectorized definitions for OP0 and OP1. */ /* Get vectorized definitions for OP0 and OP1. */
void void
vect_get_vec_defs (tree op0, tree op1, gimple *stmt, vect_get_vec_defs (tree op0, tree op1, stmt_vec_info stmt_info,
vec<tree> *vec_oprnds0, vec<tree> *vec_oprnds0,
vec<tree> *vec_oprnds1, vec<tree> *vec_oprnds1,
slp_tree slp_node) slp_tree slp_node)
{ {
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
if (slp_node) if (slp_node)
{ {
int nops = (op1 == NULL_TREE) ? 1 : 2; int nops = (op1 == NULL_TREE) ? 1 : 2;
...@@ -1727,9 +1720,8 @@ vect_get_vec_defs (tree op0, tree op1, gimple *stmt, ...@@ -1727,9 +1720,8 @@ vect_get_vec_defs (tree op0, tree op1, gimple *stmt,
statement and create and return a stmt_vec_info for it. */ statement and create and return a stmt_vec_info for it. */
static stmt_vec_info static stmt_vec_info
vect_finish_stmt_generation_1 (gimple *stmt, gimple *vec_stmt) vect_finish_stmt_generation_1 (stmt_vec_info stmt_info, gimple *vec_stmt)
{ {
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
vec_info *vinfo = stmt_info->vinfo; vec_info *vinfo = stmt_info->vinfo;
stmt_vec_info vec_stmt_info = vinfo->add_stmt (vec_stmt); stmt_vec_info vec_stmt_info = vinfo->add_stmt (vec_stmt);
...@@ -1752,14 +1744,13 @@ vect_finish_stmt_generation_1 (gimple *stmt, gimple *vec_stmt) ...@@ -1752,14 +1744,13 @@ vect_finish_stmt_generation_1 (gimple *stmt, gimple *vec_stmt)
return vec_stmt_info; return vec_stmt_info;
} }
/* Replace the scalar statement STMT with a new vector statement VEC_STMT, /* Replace the scalar statement STMT_INFO with a new vector statement VEC_STMT,
which sets the same scalar result as STMT did. Create and return a which sets the same scalar result as STMT_INFO did. Create and return a
stmt_vec_info for VEC_STMT. */ stmt_vec_info for VEC_STMT. */
stmt_vec_info stmt_vec_info
vect_finish_replace_stmt (gimple *stmt, gimple *vec_stmt) vect_finish_replace_stmt (stmt_vec_info stmt_info, gimple *vec_stmt)
{ {
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
gcc_assert (gimple_get_lhs (stmt_info->stmt) == gimple_get_lhs (vec_stmt)); gcc_assert (gimple_get_lhs (stmt_info->stmt) == gimple_get_lhs (vec_stmt));
gimple_stmt_iterator gsi = gsi_for_stmt (stmt_info->stmt); gimple_stmt_iterator gsi = gsi_for_stmt (stmt_info->stmt);
...@@ -1768,14 +1759,13 @@ vect_finish_replace_stmt (gimple *stmt, gimple *vec_stmt) ...@@ -1768,14 +1759,13 @@ vect_finish_replace_stmt (gimple *stmt, gimple *vec_stmt)
return vect_finish_stmt_generation_1 (stmt_info, vec_stmt); return vect_finish_stmt_generation_1 (stmt_info, vec_stmt);
} }
/* Add VEC_STMT to the vectorized implementation of STMT and insert it /* Add VEC_STMT to the vectorized implementation of STMT_INFO and insert it
before *GSI. Create and return a stmt_vec_info for VEC_STMT. */ before *GSI. Create and return a stmt_vec_info for VEC_STMT. */
stmt_vec_info stmt_vec_info
vect_finish_stmt_generation (gimple *stmt, gimple *vec_stmt, vect_finish_stmt_generation (stmt_vec_info stmt_info, gimple *vec_stmt,
gimple_stmt_iterator *gsi) gimple_stmt_iterator *gsi)
{ {
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
gcc_assert (gimple_code (stmt_info->stmt) != GIMPLE_LABEL); gcc_assert (gimple_code (stmt_info->stmt) != GIMPLE_LABEL);
if (!gsi_end_p (*gsi) if (!gsi_end_p (*gsi)
...@@ -1976,22 +1966,21 @@ prepare_load_store_mask (tree mask_type, tree loop_mask, tree vec_mask, ...@@ -1976,22 +1966,21 @@ prepare_load_store_mask (tree mask_type, tree loop_mask, tree vec_mask,
} }
/* Determine whether we can use a gather load or scatter store to vectorize /* Determine whether we can use a gather load or scatter store to vectorize
strided load or store STMT by truncating the current offset to a smaller strided load or store STMT_INFO by truncating the current offset to a
width. We need to be able to construct an offset vector: smaller width. We need to be able to construct an offset vector:
{ 0, X, X*2, X*3, ... } { 0, X, X*2, X*3, ... }
without loss of precision, where X is STMT's DR_STEP. without loss of precision, where X is STMT_INFO's DR_STEP.
Return true if this is possible, describing the gather load or scatter Return true if this is possible, describing the gather load or scatter
store in GS_INFO. MASKED_P is true if the load or store is conditional. */ store in GS_INFO. MASKED_P is true if the load or store is conditional. */
static bool static bool
vect_truncate_gather_scatter_offset (gimple *stmt, loop_vec_info loop_vinfo, vect_truncate_gather_scatter_offset (stmt_vec_info stmt_info,
bool masked_p, loop_vec_info loop_vinfo, bool masked_p,
gather_scatter_info *gs_info) gather_scatter_info *gs_info)
{ {
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
data_reference *dr = STMT_VINFO_DATA_REF (stmt_info); data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
tree step = DR_STEP (dr); tree step = DR_STEP (dr);
if (TREE_CODE (step) != INTEGER_CST) if (TREE_CODE (step) != INTEGER_CST)
...@@ -2112,14 +2101,13 @@ vect_use_strided_gather_scatters_p (stmt_vec_info stmt_info, ...@@ -2112,14 +2101,13 @@ vect_use_strided_gather_scatters_p (stmt_vec_info stmt_info,
return true; return true;
} }
/* STMT is a non-strided load or store, meaning that it accesses /* STMT_INFO is a non-strided load or store, meaning that it accesses
elements with a known constant step. Return -1 if that step elements with a known constant step. Return -1 if that step
is negative, 0 if it is zero, and 1 if it is greater than zero. */ is negative, 0 if it is zero, and 1 if it is greater than zero. */
static int static int
compare_step_with_zero (gimple *stmt) compare_step_with_zero (stmt_vec_info stmt_info)
{ {
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
data_reference *dr = STMT_VINFO_DATA_REF (stmt_info); data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
return tree_int_cst_compare (vect_dr_behavior (dr)->step, return tree_int_cst_compare (vect_dr_behavior (dr)->step,
size_zero_node); size_zero_node);
...@@ -2144,29 +2132,29 @@ perm_mask_for_reverse (tree vectype) ...@@ -2144,29 +2132,29 @@ perm_mask_for_reverse (tree vectype)
return vect_gen_perm_mask_checked (vectype, indices); return vect_gen_perm_mask_checked (vectype, indices);
} }
/* STMT is either a masked or unconditional store. Return the value /* STMT_INFO is either a masked or unconditional store. Return the value
being stored. */ being stored. */
tree tree
vect_get_store_rhs (gimple *stmt) vect_get_store_rhs (stmt_vec_info stmt_info)
{ {
if (gassign *assign = dyn_cast <gassign *> (stmt)) if (gassign *assign = dyn_cast <gassign *> (stmt_info->stmt))
{ {
gcc_assert (gimple_assign_single_p (assign)); gcc_assert (gimple_assign_single_p (assign));
return gimple_assign_rhs1 (assign); return gimple_assign_rhs1 (assign);
} }
if (gcall *call = dyn_cast <gcall *> (stmt)) if (gcall *call = dyn_cast <gcall *> (stmt_info->stmt))
{ {
internal_fn ifn = gimple_call_internal_fn (call); internal_fn ifn = gimple_call_internal_fn (call);
int index = internal_fn_stored_value_index (ifn); int index = internal_fn_stored_value_index (ifn);
gcc_assert (index >= 0); gcc_assert (index >= 0);
return gimple_call_arg (stmt, index); return gimple_call_arg (call, index);
} }
gcc_unreachable (); gcc_unreachable ();
} }
/* A subroutine of get_load_store_type, with a subset of the same /* A subroutine of get_load_store_type, with a subset of the same
arguments. Handle the case where STMT is part of a grouped load arguments. Handle the case where STMT_INFO is part of a grouped load
or store. or store.
For stores, the statements in the group are all consecutive For stores, the statements in the group are all consecutive
...@@ -2175,12 +2163,11 @@ vect_get_store_rhs (gimple *stmt) ...@@ -2175,12 +2163,11 @@ vect_get_store_rhs (gimple *stmt)
as well as at the end. */ as well as at the end. */
static bool static bool
get_group_load_store_type (gimple *stmt, tree vectype, bool slp, get_group_load_store_type (stmt_vec_info stmt_info, tree vectype, bool slp,
bool masked_p, vec_load_store_type vls_type, bool masked_p, vec_load_store_type vls_type,
vect_memory_access_type *memory_access_type, vect_memory_access_type *memory_access_type,
gather_scatter_info *gs_info) gather_scatter_info *gs_info)
{ {
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
vec_info *vinfo = stmt_info->vinfo; vec_info *vinfo = stmt_info->vinfo;
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
struct loop *loop = loop_vinfo ? LOOP_VINFO_LOOP (loop_vinfo) : NULL; struct loop *loop = loop_vinfo ? LOOP_VINFO_LOOP (loop_vinfo) : NULL;
...@@ -2350,15 +2337,14 @@ get_group_load_store_type (gimple *stmt, tree vectype, bool slp, ...@@ -2350,15 +2337,14 @@ get_group_load_store_type (gimple *stmt, tree vectype, bool slp,
} }
/* A subroutine of get_load_store_type, with a subset of the same /* A subroutine of get_load_store_type, with a subset of the same
arguments. Handle the case where STMT is a load or store that arguments. Handle the case where STMT_INFO is a load or store that
accesses consecutive elements with a negative step. */ accesses consecutive elements with a negative step. */
static vect_memory_access_type static vect_memory_access_type
get_negative_load_store_type (gimple *stmt, tree vectype, get_negative_load_store_type (stmt_vec_info stmt_info, tree vectype,
vec_load_store_type vls_type, vec_load_store_type vls_type,
unsigned int ncopies) unsigned int ncopies)
{ {
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info); struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
dr_alignment_support alignment_support_scheme; dr_alignment_support alignment_support_scheme;
...@@ -2400,7 +2386,7 @@ get_negative_load_store_type (gimple *stmt, tree vectype, ...@@ -2400,7 +2386,7 @@ get_negative_load_store_type (gimple *stmt, tree vectype,
return VMAT_CONTIGUOUS_REVERSE; return VMAT_CONTIGUOUS_REVERSE;
} }
/* Analyze load or store statement STMT of type VLS_TYPE. Return true /* Analyze load or store statement STMT_INFO of type VLS_TYPE. Return true
if there is a memory access type that the vectorized form can use, if there is a memory access type that the vectorized form can use,
storing it in *MEMORY_ACCESS_TYPE if so. If we decide to use gathers storing it in *MEMORY_ACCESS_TYPE if so. If we decide to use gathers
or scatters, fill in GS_INFO accordingly. or scatters, fill in GS_INFO accordingly.
...@@ -2411,12 +2397,12 @@ get_negative_load_store_type (gimple *stmt, tree vectype, ...@@ -2411,12 +2397,12 @@ get_negative_load_store_type (gimple *stmt, tree vectype,
NCOPIES is the number of vector statements that will be needed. */ NCOPIES is the number of vector statements that will be needed. */
static bool static bool
get_load_store_type (gimple *stmt, tree vectype, bool slp, bool masked_p, get_load_store_type (stmt_vec_info stmt_info, tree vectype, bool slp,
vec_load_store_type vls_type, unsigned int ncopies, bool masked_p, vec_load_store_type vls_type,
unsigned int ncopies,
vect_memory_access_type *memory_access_type, vect_memory_access_type *memory_access_type,
gather_scatter_info *gs_info) gather_scatter_info *gs_info)
{ {
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
vec_info *vinfo = stmt_info->vinfo; vec_info *vinfo = stmt_info->vinfo;
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype); poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
...@@ -2496,12 +2482,12 @@ get_load_store_type (gimple *stmt, tree vectype, bool slp, bool masked_p, ...@@ -2496,12 +2482,12 @@ get_load_store_type (gimple *stmt, tree vectype, bool slp, bool masked_p,
} }
/* Return true if boolean argument MASK is suitable for vectorizing /* Return true if boolean argument MASK is suitable for vectorizing
conditional load or store STMT. When returning true, store the type conditional load or store STMT_INFO. When returning true, store the type
of the definition in *MASK_DT_OUT and the type of the vectorized mask of the definition in *MASK_DT_OUT and the type of the vectorized mask
in *MASK_VECTYPE_OUT. */ in *MASK_VECTYPE_OUT. */
static bool static bool
vect_check_load_store_mask (gimple *stmt, tree mask, vect_check_load_store_mask (stmt_vec_info stmt_info, tree mask,
vect_def_type *mask_dt_out, vect_def_type *mask_dt_out,
tree *mask_vectype_out) tree *mask_vectype_out)
{ {
...@@ -2521,7 +2507,6 @@ vect_check_load_store_mask (gimple *stmt, tree mask, ...@@ -2521,7 +2507,6 @@ vect_check_load_store_mask (gimple *stmt, tree mask,
return false; return false;
} }
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
enum vect_def_type mask_dt; enum vect_def_type mask_dt;
tree mask_vectype; tree mask_vectype;
if (!vect_is_simple_use (mask, stmt_info->vinfo, &mask_dt, &mask_vectype)) if (!vect_is_simple_use (mask, stmt_info->vinfo, &mask_dt, &mask_vectype))
...@@ -2566,13 +2551,14 @@ vect_check_load_store_mask (gimple *stmt, tree mask, ...@@ -2566,13 +2551,14 @@ vect_check_load_store_mask (gimple *stmt, tree mask,
} }
/* Return true if stored value RHS is suitable for vectorizing store /* Return true if stored value RHS is suitable for vectorizing store
statement STMT. When returning true, store the type of the statement STMT_INFO. When returning true, store the type of the
definition in *RHS_DT_OUT, the type of the vectorized store value in definition in *RHS_DT_OUT, the type of the vectorized store value in
*RHS_VECTYPE_OUT and the type of the store in *VLS_TYPE_OUT. */ *RHS_VECTYPE_OUT and the type of the store in *VLS_TYPE_OUT. */
static bool static bool
vect_check_store_rhs (gimple *stmt, tree rhs, vect_def_type *rhs_dt_out, vect_check_store_rhs (stmt_vec_info stmt_info, tree rhs,
tree *rhs_vectype_out, vec_load_store_type *vls_type_out) vect_def_type *rhs_dt_out, tree *rhs_vectype_out,
vec_load_store_type *vls_type_out)
{ {
/* In the case this is a store from a constant make sure /* In the case this is a store from a constant make sure
native_encode_expr can handle it. */ native_encode_expr can handle it. */
...@@ -2584,7 +2570,6 @@ vect_check_store_rhs (gimple *stmt, tree rhs, vect_def_type *rhs_dt_out, ...@@ -2584,7 +2570,6 @@ vect_check_store_rhs (gimple *stmt, tree rhs, vect_def_type *rhs_dt_out,
return false; return false;
} }
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
enum vect_def_type rhs_dt; enum vect_def_type rhs_dt;
tree rhs_vectype; tree rhs_vectype;
if (!vect_is_simple_use (rhs, stmt_info->vinfo, &rhs_dt, &rhs_vectype)) if (!vect_is_simple_use (rhs, stmt_info->vinfo, &rhs_dt, &rhs_vectype))
...@@ -2666,18 +2651,19 @@ vect_build_zero_merge_argument (stmt_vec_info stmt_info, tree vectype) ...@@ -2666,18 +2651,19 @@ vect_build_zero_merge_argument (stmt_vec_info stmt_info, tree vectype)
return vect_init_vector (stmt_info, merge, vectype, NULL); return vect_init_vector (stmt_info, merge, vectype, NULL);
} }
/* Build a gather load call while vectorizing STMT. Insert new instructions /* Build a gather load call while vectorizing STMT_INFO. Insert new
before GSI and add them to VEC_STMT. GS_INFO describes the gather load instructions before GSI and add them to VEC_STMT. GS_INFO describes
operation. If the load is conditional, MASK is the unvectorized the gather load operation. If the load is conditional, MASK is the
condition and MASK_DT is its definition type, otherwise MASK is null. */ unvectorized condition and MASK_DT is its definition type, otherwise
MASK is null. */
static void static void
vect_build_gather_load_calls (gimple *stmt, gimple_stmt_iterator *gsi, vect_build_gather_load_calls (stmt_vec_info stmt_info,
gimple_stmt_iterator *gsi,
stmt_vec_info *vec_stmt, stmt_vec_info *vec_stmt,
gather_scatter_info *gs_info, tree mask, gather_scatter_info *gs_info,
vect_def_type mask_dt) tree mask, vect_def_type mask_dt)
{ {
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo); struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
tree vectype = STMT_VINFO_VECTYPE (stmt_info); tree vectype = STMT_VINFO_VECTYPE (stmt_info);
...@@ -2897,7 +2883,7 @@ vect_get_gather_scatter_ops (struct loop *loop, stmt_vec_info stmt_info, ...@@ -2897,7 +2883,7 @@ vect_get_gather_scatter_ops (struct loop *loop, stmt_vec_info stmt_info,
/* Prepare to implement a grouped or strided load or store using /* Prepare to implement a grouped or strided load or store using
the gather load or scatter store operation described by GS_INFO. the gather load or scatter store operation described by GS_INFO.
STMT is the load or store statement. STMT_INFO is the load or store statement.
Set *DATAREF_BUMP to the amount that should be added to the base Set *DATAREF_BUMP to the amount that should be added to the base
address after each copy of the vectorized statement. Set *VEC_OFFSET address after each copy of the vectorized statement. Set *VEC_OFFSET
...@@ -2905,11 +2891,11 @@ vect_get_gather_scatter_ops (struct loop *loop, stmt_vec_info stmt_info, ...@@ -2905,11 +2891,11 @@ vect_get_gather_scatter_ops (struct loop *loop, stmt_vec_info stmt_info,
I * DR_STEP / SCALE. */ I * DR_STEP / SCALE. */
static void static void
vect_get_strided_load_store_ops (gimple *stmt, loop_vec_info loop_vinfo, vect_get_strided_load_store_ops (stmt_vec_info stmt_info,
loop_vec_info loop_vinfo,
gather_scatter_info *gs_info, gather_scatter_info *gs_info,
tree *dataref_bump, tree *vec_offset) tree *dataref_bump, tree *vec_offset)
{ {
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info); struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo); struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
tree vectype = STMT_VINFO_VECTYPE (stmt_info); tree vectype = STMT_VINFO_VECTYPE (stmt_info);
...@@ -2963,13 +2949,13 @@ vect_get_data_ptr_increment (data_reference *dr, tree aggr_type, ...@@ -2963,13 +2949,13 @@ vect_get_data_ptr_increment (data_reference *dr, tree aggr_type,
/* Check and perform vectorization of BUILT_IN_BSWAP{16,32,64}. */ /* Check and perform vectorization of BUILT_IN_BSWAP{16,32,64}. */
static bool static bool
vectorizable_bswap (gimple *stmt, gimple_stmt_iterator *gsi, vectorizable_bswap (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
stmt_vec_info *vec_stmt, slp_tree slp_node, stmt_vec_info *vec_stmt, slp_tree slp_node,
tree vectype_in, enum vect_def_type *dt, tree vectype_in, enum vect_def_type *dt,
stmt_vector_for_cost *cost_vec) stmt_vector_for_cost *cost_vec)
{ {
tree op, vectype; tree op, vectype;
stmt_vec_info stmt_info = vinfo_for_stmt (stmt); gcall *stmt = as_a <gcall *> (stmt_info->stmt);
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
unsigned ncopies; unsigned ncopies;
unsigned HOST_WIDE_INT nunits, num_bytes; unsigned HOST_WIDE_INT nunits, num_bytes;
...@@ -3103,13 +3089,13 @@ simple_integer_narrowing (tree vectype_out, tree vectype_in, ...@@ -3103,13 +3089,13 @@ simple_integer_narrowing (tree vectype_out, tree vectype_in,
/* Function vectorizable_call. /* Function vectorizable_call.
Check if GS performs a function call that can be vectorized. Check if STMT_INFO performs a function call that can be vectorized.
If VEC_STMT is also passed, vectorize the STMT: create a vectorized If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
stmt to replace it, put it in VEC_STMT, and insert it at BSI. stmt to replace it, put it in VEC_STMT, and insert it at GSI.
Return FALSE if not a vectorizable STMT, TRUE otherwise. */ Return true if STMT_INFO is vectorizable in this way. */
static bool static bool
vectorizable_call (gimple *gs, gimple_stmt_iterator *gsi, vectorizable_call (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
stmt_vec_info *vec_stmt, slp_tree slp_node, stmt_vec_info *vec_stmt, slp_tree slp_node,
stmt_vector_for_cost *cost_vec) stmt_vector_for_cost *cost_vec)
{ {
...@@ -3118,7 +3104,7 @@ vectorizable_call (gimple *gs, gimple_stmt_iterator *gsi, ...@@ -3118,7 +3104,7 @@ vectorizable_call (gimple *gs, gimple_stmt_iterator *gsi,
tree scalar_dest; tree scalar_dest;
tree op; tree op;
tree vec_oprnd0 = NULL_TREE, vec_oprnd1 = NULL_TREE; tree vec_oprnd0 = NULL_TREE, vec_oprnd1 = NULL_TREE;
stmt_vec_info stmt_info = vinfo_for_stmt (gs), prev_stmt_info; stmt_vec_info prev_stmt_info;
tree vectype_out, vectype_in; tree vectype_out, vectype_in;
poly_uint64 nunits_in; poly_uint64 nunits_in;
poly_uint64 nunits_out; poly_uint64 nunits_out;
...@@ -3747,14 +3733,15 @@ simd_clone_subparts (tree vectype) ...@@ -3747,14 +3733,15 @@ simd_clone_subparts (tree vectype)
/* Function vectorizable_simd_clone_call. /* Function vectorizable_simd_clone_call.
Check if STMT performs a function call that can be vectorized Check if STMT_INFO performs a function call that can be vectorized
by calling a simd clone of the function. by calling a simd clone of the function.
If VEC_STMT is also passed, vectorize the STMT: create a vectorized If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
stmt to replace it, put it in VEC_STMT, and insert it at BSI. stmt to replace it, put it in VEC_STMT, and insert it at GSI.
Return FALSE if not a vectorizable STMT, TRUE otherwise. */ Return true if STMT_INFO is vectorizable in this way. */
static bool static bool
vectorizable_simd_clone_call (gimple *stmt, gimple_stmt_iterator *gsi, vectorizable_simd_clone_call (stmt_vec_info stmt_info,
gimple_stmt_iterator *gsi,
stmt_vec_info *vec_stmt, slp_tree slp_node, stmt_vec_info *vec_stmt, slp_tree slp_node,
stmt_vector_for_cost *) stmt_vector_for_cost *)
{ {
...@@ -3762,7 +3749,7 @@ vectorizable_simd_clone_call (gimple *stmt, gimple_stmt_iterator *gsi, ...@@ -3762,7 +3749,7 @@ vectorizable_simd_clone_call (gimple *stmt, gimple_stmt_iterator *gsi,
tree scalar_dest; tree scalar_dest;
tree op, type; tree op, type;
tree vec_oprnd0 = NULL_TREE; tree vec_oprnd0 = NULL_TREE;
stmt_vec_info stmt_info = vinfo_for_stmt (stmt), prev_stmt_info; stmt_vec_info prev_stmt_info;
tree vectype; tree vectype;
unsigned int nunits; unsigned int nunits;
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
...@@ -3778,7 +3765,8 @@ vectorizable_simd_clone_call (gimple *stmt, gimple_stmt_iterator *gsi, ...@@ -3778,7 +3765,8 @@ vectorizable_simd_clone_call (gimple *stmt, gimple_stmt_iterator *gsi,
vec<constructor_elt, va_gc> *ret_ctor_elts = NULL; vec<constructor_elt, va_gc> *ret_ctor_elts = NULL;
/* Is STMT a vectorizable call? */ /* Is STMT a vectorizable call? */
if (!is_gimple_call (stmt)) gcall *stmt = dyn_cast <gcall *> (stmt_info->stmt);
if (!stmt)
return false; return false;
fndecl = gimple_call_fndecl (stmt); fndecl = gimple_call_fndecl (stmt);
...@@ -4487,7 +4475,8 @@ vect_get_loop_based_defs (tree *oprnd, stmt_vec_info stmt_info, ...@@ -4487,7 +4475,8 @@ vect_get_loop_based_defs (tree *oprnd, stmt_vec_info stmt_info,
static void static void
vect_create_vectorized_demotion_stmts (vec<tree> *vec_oprnds, vect_create_vectorized_demotion_stmts (vec<tree> *vec_oprnds,
int multi_step_cvt, gimple *stmt, int multi_step_cvt,
stmt_vec_info stmt_info,
vec<tree> vec_dsts, vec<tree> vec_dsts,
gimple_stmt_iterator *gsi, gimple_stmt_iterator *gsi,
slp_tree slp_node, enum tree_code code, slp_tree slp_node, enum tree_code code,
...@@ -4495,7 +4484,6 @@ vect_create_vectorized_demotion_stmts (vec<tree> *vec_oprnds, ...@@ -4495,7 +4484,6 @@ vect_create_vectorized_demotion_stmts (vec<tree> *vec_oprnds,
{ {
unsigned int i; unsigned int i;
tree vop0, vop1, new_tmp, vec_dest; tree vop0, vop1, new_tmp, vec_dest;
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
vec_dest = vec_dsts.pop (); vec_dest = vec_dsts.pop ();
...@@ -4606,13 +4594,13 @@ vect_create_vectorized_promotion_stmts (vec<tree> *vec_oprnds0, ...@@ -4606,13 +4594,13 @@ vect_create_vectorized_promotion_stmts (vec<tree> *vec_oprnds0,
} }
/* Check if STMT performs a conversion operation, that can be vectorized. /* Check if STMT_INFO performs a conversion operation that can be vectorized.
If VEC_STMT is also passed, vectorize the STMT: create a vectorized If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
stmt to replace it, put it in VEC_STMT, and insert it at GSI. stmt to replace it, put it in VEC_STMT, and insert it at GSI.
Return FALSE if not a vectorizable STMT, TRUE otherwise. */ Return true if STMT_INFO is vectorizable in this way. */
static bool static bool
vectorizable_conversion (gimple *stmt, gimple_stmt_iterator *gsi, vectorizable_conversion (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
stmt_vec_info *vec_stmt, slp_tree slp_node, stmt_vec_info *vec_stmt, slp_tree slp_node,
stmt_vector_for_cost *cost_vec) stmt_vector_for_cost *cost_vec)
{ {
...@@ -4620,7 +4608,6 @@ vectorizable_conversion (gimple *stmt, gimple_stmt_iterator *gsi, ...@@ -4620,7 +4608,6 @@ vectorizable_conversion (gimple *stmt, gimple_stmt_iterator *gsi,
tree scalar_dest; tree scalar_dest;
tree op0, op1 = NULL_TREE; tree op0, op1 = NULL_TREE;
tree vec_oprnd0 = NULL_TREE, vec_oprnd1 = NULL_TREE; tree vec_oprnd0 = NULL_TREE, vec_oprnd1 = NULL_TREE;
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
enum tree_code code, code1 = ERROR_MARK, code2 = ERROR_MARK; enum tree_code code, code1 = ERROR_MARK, code2 = ERROR_MARK;
enum tree_code codecvt1 = ERROR_MARK, codecvt2 = ERROR_MARK; enum tree_code codecvt1 = ERROR_MARK, codecvt2 = ERROR_MARK;
...@@ -4655,7 +4642,8 @@ vectorizable_conversion (gimple *stmt, gimple_stmt_iterator *gsi, ...@@ -4655,7 +4642,8 @@ vectorizable_conversion (gimple *stmt, gimple_stmt_iterator *gsi,
&& ! vec_stmt) && ! vec_stmt)
return false; return false;
if (!is_gimple_assign (stmt)) gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
if (!stmt)
return false; return false;
if (TREE_CODE (gimple_assign_lhs (stmt)) != SSA_NAME) if (TREE_CODE (gimple_assign_lhs (stmt)) != SSA_NAME)
...@@ -5220,20 +5208,19 @@ vectorizable_conversion (gimple *stmt, gimple_stmt_iterator *gsi, ...@@ -5220,20 +5208,19 @@ vectorizable_conversion (gimple *stmt, gimple_stmt_iterator *gsi,
/* Function vectorizable_assignment. /* Function vectorizable_assignment.
Check if STMT performs an assignment (copy) that can be vectorized. Check if STMT_INFO performs an assignment (copy) that can be vectorized.
If VEC_STMT is also passed, vectorize the STMT: create a vectorized If VEC_STMT is also passed, vectorize the STMT_INFO: create a vectorized
stmt to replace it, put it in VEC_STMT, and insert it at BSI. stmt to replace it, put it in VEC_STMT, and insert it at GSI.
Return FALSE if not a vectorizable STMT, TRUE otherwise. */ Return true if STMT_INFO is vectorizable in this way. */
static bool static bool
vectorizable_assignment (gimple *stmt, gimple_stmt_iterator *gsi, vectorizable_assignment (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
stmt_vec_info *vec_stmt, slp_tree slp_node, stmt_vec_info *vec_stmt, slp_tree slp_node,
stmt_vector_for_cost *cost_vec) stmt_vector_for_cost *cost_vec)
{ {
tree vec_dest; tree vec_dest;
tree scalar_dest; tree scalar_dest;
tree op; tree op;
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
tree new_temp; tree new_temp;
enum vect_def_type dt[1] = {vect_unknown_def_type}; enum vect_def_type dt[1] = {vect_unknown_def_type};
...@@ -5256,7 +5243,8 @@ vectorizable_assignment (gimple *stmt, gimple_stmt_iterator *gsi, ...@@ -5256,7 +5243,8 @@ vectorizable_assignment (gimple *stmt, gimple_stmt_iterator *gsi,
return false; return false;
/* Is vectorizable assignment? */ /* Is vectorizable assignment? */
if (!is_gimple_assign (stmt)) gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
if (!stmt)
return false; return false;
scalar_dest = gimple_assign_lhs (stmt); scalar_dest = gimple_assign_lhs (stmt);
...@@ -5422,13 +5410,13 @@ vect_supportable_shift (enum tree_code code, tree scalar_type) ...@@ -5422,13 +5410,13 @@ vect_supportable_shift (enum tree_code code, tree scalar_type)
/* Function vectorizable_shift. /* Function vectorizable_shift.
Check if STMT performs a shift operation that can be vectorized. Check if STMT_INFO performs a shift operation that can be vectorized.
If VEC_STMT is also passed, vectorize the STMT: create a vectorized If VEC_STMT is also passed, vectorize the STMT_INFO: create a vectorized
stmt to replace it, put it in VEC_STMT, and insert it at BSI. stmt to replace it, put it in VEC_STMT, and insert it at GSI.
Return FALSE if not a vectorizable STMT, TRUE otherwise. */ Return true if STMT_INFO is vectorizable in this way. */
static bool static bool
vectorizable_shift (gimple *stmt, gimple_stmt_iterator *gsi, vectorizable_shift (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
stmt_vec_info *vec_stmt, slp_tree slp_node, stmt_vec_info *vec_stmt, slp_tree slp_node,
stmt_vector_for_cost *cost_vec) stmt_vector_for_cost *cost_vec)
{ {
...@@ -5436,7 +5424,6 @@ vectorizable_shift (gimple *stmt, gimple_stmt_iterator *gsi, ...@@ -5436,7 +5424,6 @@ vectorizable_shift (gimple *stmt, gimple_stmt_iterator *gsi,
tree scalar_dest; tree scalar_dest;
tree op0, op1 = NULL; tree op0, op1 = NULL;
tree vec_oprnd1 = NULL_TREE; tree vec_oprnd1 = NULL_TREE;
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
tree vectype; tree vectype;
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
enum tree_code code; enum tree_code code;
...@@ -5470,7 +5457,8 @@ vectorizable_shift (gimple *stmt, gimple_stmt_iterator *gsi, ...@@ -5470,7 +5457,8 @@ vectorizable_shift (gimple *stmt, gimple_stmt_iterator *gsi,
return false; return false;
/* Is STMT a vectorizable binary/unary operation? */ /* Is STMT a vectorizable binary/unary operation? */
if (!is_gimple_assign (stmt)) gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
if (!stmt)
return false; return false;
if (TREE_CODE (gimple_assign_lhs (stmt)) != SSA_NAME) if (TREE_CODE (gimple_assign_lhs (stmt)) != SSA_NAME)
...@@ -5789,21 +5777,20 @@ vectorizable_shift (gimple *stmt, gimple_stmt_iterator *gsi, ...@@ -5789,21 +5777,20 @@ vectorizable_shift (gimple *stmt, gimple_stmt_iterator *gsi,
/* Function vectorizable_operation. /* Function vectorizable_operation.
Check if STMT performs a binary, unary or ternary operation that can Check if STMT_INFO performs a binary, unary or ternary operation that can
be vectorized. be vectorized.
If VEC_STMT is also passed, vectorize the STMT: create a vectorized If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
stmt to replace it, put it in VEC_STMT, and insert it at BSI. stmt to replace it, put it in VEC_STMT, and insert it at GSI.
Return FALSE if not a vectorizable STMT, TRUE otherwise. */ Return true if STMT_INFO is vectorizable in this way. */
static bool static bool
vectorizable_operation (gimple *stmt, gimple_stmt_iterator *gsi, vectorizable_operation (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
stmt_vec_info *vec_stmt, slp_tree slp_node, stmt_vec_info *vec_stmt, slp_tree slp_node,
stmt_vector_for_cost *cost_vec) stmt_vector_for_cost *cost_vec)
{ {
tree vec_dest; tree vec_dest;
tree scalar_dest; tree scalar_dest;
tree op0, op1 = NULL_TREE, op2 = NULL_TREE; tree op0, op1 = NULL_TREE, op2 = NULL_TREE;
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
tree vectype; tree vectype;
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
enum tree_code code, orig_code; enum tree_code code, orig_code;
...@@ -5836,7 +5823,8 @@ vectorizable_operation (gimple *stmt, gimple_stmt_iterator *gsi, ...@@ -5836,7 +5823,8 @@ vectorizable_operation (gimple *stmt, gimple_stmt_iterator *gsi,
return false; return false;
/* Is STMT a vectorizable binary/unary operation? */ /* Is STMT a vectorizable binary/unary operation? */
if (!is_gimple_assign (stmt)) gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
if (!stmt)
return false; return false;
if (TREE_CODE (gimple_assign_lhs (stmt)) != SSA_NAME) if (TREE_CODE (gimple_assign_lhs (stmt)) != SSA_NAME)
...@@ -6215,12 +6203,11 @@ ensure_base_align (struct data_reference *dr) ...@@ -6215,12 +6203,11 @@ ensure_base_align (struct data_reference *dr)
/* Function get_group_alias_ptr_type. /* Function get_group_alias_ptr_type.
Return the alias type for the group starting at FIRST_STMT. */ Return the alias type for the group starting at FIRST_STMT_INFO. */
static tree static tree
get_group_alias_ptr_type (gimple *first_stmt) get_group_alias_ptr_type (stmt_vec_info first_stmt_info)
{ {
stmt_vec_info first_stmt_info = vinfo_for_stmt (first_stmt);
struct data_reference *first_dr, *next_dr; struct data_reference *first_dr, *next_dr;
first_dr = STMT_VINFO_DATA_REF (first_stmt_info); first_dr = STMT_VINFO_DATA_REF (first_stmt_info);
...@@ -6244,21 +6231,20 @@ get_group_alias_ptr_type (gimple *first_stmt) ...@@ -6244,21 +6231,20 @@ get_group_alias_ptr_type (gimple *first_stmt)
/* Function vectorizable_store. /* Function vectorizable_store.
Check if STMT defines a non scalar data-ref (array/pointer/structure) that Check if STMT_INFO defines a non scalar data-ref (array/pointer/structure)
can be vectorized. that can be vectorized.
If VEC_STMT is also passed, vectorize the STMT: create a vectorized If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
stmt to replace it, put it in VEC_STMT, and insert it at BSI. stmt to replace it, put it in VEC_STMT, and insert it at GSI.
Return FALSE if not a vectorizable STMT, TRUE otherwise. */ Return true if STMT_INFO is vectorizable in this way. */
static bool static bool
vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, vectorizable_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
stmt_vec_info *vec_stmt, slp_tree slp_node, stmt_vec_info *vec_stmt, slp_tree slp_node,
stmt_vector_for_cost *cost_vec) stmt_vector_for_cost *cost_vec)
{ {
tree data_ref; tree data_ref;
tree op; tree op;
tree vec_oprnd = NULL_TREE; tree vec_oprnd = NULL_TREE;
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info), *first_dr = NULL; struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info), *first_dr = NULL;
tree elem_type; tree elem_type;
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
...@@ -7350,19 +7336,19 @@ permute_vec_elements (tree x, tree y, tree mask_vec, stmt_vec_info stmt_info, ...@@ -7350,19 +7336,19 @@ permute_vec_elements (tree x, tree y, tree mask_vec, stmt_vec_info stmt_info,
return data_ref; return data_ref;
} }
/* Hoist the definitions of all SSA uses on STMT out of the loop LOOP, /* Hoist the definitions of all SSA uses on STMT_INFO out of the loop LOOP,
inserting them on the loops preheader edge. Returns true if we inserting them on the loops preheader edge. Returns true if we
were successful in doing so (and thus STMT can be moved then), were successful in doing so (and thus STMT_INFO can be moved then),
otherwise returns false. */ otherwise returns false. */
static bool static bool
hoist_defs_of_uses (gimple *stmt, struct loop *loop) hoist_defs_of_uses (stmt_vec_info stmt_info, struct loop *loop)
{ {
ssa_op_iter i; ssa_op_iter i;
tree op; tree op;
bool any = false; bool any = false;
FOR_EACH_SSA_TREE_OPERAND (op, stmt, i, SSA_OP_USE) FOR_EACH_SSA_TREE_OPERAND (op, stmt_info->stmt, i, SSA_OP_USE)
{ {
gimple *def_stmt = SSA_NAME_DEF_STMT (op); gimple *def_stmt = SSA_NAME_DEF_STMT (op);
if (!gimple_nop_p (def_stmt) if (!gimple_nop_p (def_stmt)
...@@ -7390,7 +7376,7 @@ hoist_defs_of_uses (gimple *stmt, struct loop *loop) ...@@ -7390,7 +7376,7 @@ hoist_defs_of_uses (gimple *stmt, struct loop *loop)
if (!any) if (!any)
return true; return true;
FOR_EACH_SSA_TREE_OPERAND (op, stmt, i, SSA_OP_USE) FOR_EACH_SSA_TREE_OPERAND (op, stmt_info->stmt, i, SSA_OP_USE)
{ {
gimple *def_stmt = SSA_NAME_DEF_STMT (op); gimple *def_stmt = SSA_NAME_DEF_STMT (op);
if (!gimple_nop_p (def_stmt) if (!gimple_nop_p (def_stmt)
...@@ -7407,14 +7393,14 @@ hoist_defs_of_uses (gimple *stmt, struct loop *loop) ...@@ -7407,14 +7393,14 @@ hoist_defs_of_uses (gimple *stmt, struct loop *loop)
/* vectorizable_load. /* vectorizable_load.
Check if STMT reads a non scalar data-ref (array/pointer/structure) that Check if STMT_INFO reads a non scalar data-ref (array/pointer/structure)
can be vectorized. that can be vectorized.
If VEC_STMT is also passed, vectorize the STMT: create a vectorized If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
stmt to replace it, put it in VEC_STMT, and insert it at BSI. stmt to replace it, put it in VEC_STMT, and insert it at GSI.
Return FALSE if not a vectorizable STMT, TRUE otherwise. */ Return true if STMT_INFO is vectorizable in this way. */
static bool static bool
vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, vectorizable_load (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
stmt_vec_info *vec_stmt, slp_tree slp_node, stmt_vec_info *vec_stmt, slp_tree slp_node,
slp_instance slp_node_instance, slp_instance slp_node_instance,
stmt_vector_for_cost *cost_vec) stmt_vector_for_cost *cost_vec)
...@@ -7422,11 +7408,10 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, ...@@ -7422,11 +7408,10 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi,
tree scalar_dest; tree scalar_dest;
tree vec_dest = NULL; tree vec_dest = NULL;
tree data_ref = NULL; tree data_ref = NULL;
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
stmt_vec_info prev_stmt_info; stmt_vec_info prev_stmt_info;
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
struct loop *loop = NULL; struct loop *loop = NULL;
struct loop *containing_loop = (gimple_bb (stmt))->loop_father; struct loop *containing_loop = gimple_bb (stmt_info->stmt)->loop_father;
bool nested_in_vect_loop = false; bool nested_in_vect_loop = false;
struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info), *first_dr = NULL; struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info), *first_dr = NULL;
tree elem_type; tree elem_type;
...@@ -8532,6 +8517,7 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, ...@@ -8532,6 +8517,7 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi,
&& !nested_in_vect_loop && !nested_in_vect_loop
&& hoist_defs_of_uses (stmt_info, loop)) && hoist_defs_of_uses (stmt_info, loop))
{ {
gassign *stmt = as_a <gassign *> (stmt_info->stmt);
if (dump_enabled_p ()) if (dump_enabled_p ())
{ {
dump_printf_loc (MSG_NOTE, vect_location, dump_printf_loc (MSG_NOTE, vect_location,
...@@ -8730,19 +8716,19 @@ vect_is_simple_cond (tree cond, vec_info *vinfo, ...@@ -8730,19 +8716,19 @@ vect_is_simple_cond (tree cond, vec_info *vinfo,
/* vectorizable_condition. /* vectorizable_condition.
Check if STMT is conditional modify expression that can be vectorized. Check if STMT_INFO is conditional modify expression that can be vectorized.
If VEC_STMT is also passed, vectorize the STMT: create a vectorized If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
stmt using VEC_COND_EXPR to replace it, put it in VEC_STMT, and insert it stmt using VEC_COND_EXPR to replace it, put it in VEC_STMT, and insert it
at GSI. at GSI.
When STMT is vectorized as nested cycle, REDUC_DEF is the vector variable When STMT_INFO is vectorized as a nested cycle, REDUC_DEF is the vector
to be used at REDUC_INDEX (in then clause if REDUC_INDEX is 1, and in variable to be used at REDUC_INDEX (in then clause if REDUC_INDEX is 1,
else clause if it is 2). and in else clause if it is 2).
Return FALSE if not a vectorizable STMT, TRUE otherwise. */ Return true if STMT_INFO is vectorizable in this way. */
bool bool
vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi, vectorizable_condition (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
stmt_vec_info *vec_stmt, tree reduc_def, stmt_vec_info *vec_stmt, tree reduc_def,
int reduc_index, slp_tree slp_node, int reduc_index, slp_tree slp_node,
stmt_vector_for_cost *cost_vec) stmt_vector_for_cost *cost_vec)
...@@ -8751,7 +8737,6 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi, ...@@ -8751,7 +8737,6 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi,
tree vec_dest = NULL_TREE; tree vec_dest = NULL_TREE;
tree cond_expr, cond_expr0 = NULL_TREE, cond_expr1 = NULL_TREE; tree cond_expr, cond_expr0 = NULL_TREE, cond_expr1 = NULL_TREE;
tree then_clause, else_clause; tree then_clause, else_clause;
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
tree comp_vectype = NULL_TREE; tree comp_vectype = NULL_TREE;
tree vec_cond_lhs = NULL_TREE, vec_cond_rhs = NULL_TREE; tree vec_cond_lhs = NULL_TREE, vec_cond_rhs = NULL_TREE;
tree vec_then_clause = NULL_TREE, vec_else_clause = NULL_TREE; tree vec_then_clause = NULL_TREE, vec_else_clause = NULL_TREE;
...@@ -8800,7 +8785,8 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi, ...@@ -8800,7 +8785,8 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi,
} }
/* Is vectorizable conditional operation? */ /* Is vectorizable conditional operation? */
if (!is_gimple_assign (stmt)) gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
if (!stmt)
return false; return false;
code = gimple_assign_rhs_code (stmt); code = gimple_assign_rhs_code (stmt);
...@@ -9138,19 +9124,18 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi, ...@@ -9138,19 +9124,18 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi,
/* vectorizable_comparison. /* vectorizable_comparison.
Check if STMT is comparison expression that can be vectorized. Check if STMT_INFO is comparison expression that can be vectorized.
If VEC_STMT is also passed, vectorize the STMT: create a vectorized If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
comparison, put it in VEC_STMT, and insert it at GSI. comparison, put it in VEC_STMT, and insert it at GSI.
Return FALSE if not a vectorizable STMT, TRUE otherwise. */ Return true if STMT_INFO is vectorizable in this way. */
static bool static bool
vectorizable_comparison (gimple *stmt, gimple_stmt_iterator *gsi, vectorizable_comparison (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
stmt_vec_info *vec_stmt, tree reduc_def, stmt_vec_info *vec_stmt, tree reduc_def,
slp_tree slp_node, stmt_vector_for_cost *cost_vec) slp_tree slp_node, stmt_vector_for_cost *cost_vec)
{ {
tree lhs, rhs1, rhs2; tree lhs, rhs1, rhs2;
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
tree vectype1 = NULL_TREE, vectype2 = NULL_TREE; tree vectype1 = NULL_TREE, vectype2 = NULL_TREE;
tree vectype = STMT_VINFO_VECTYPE (stmt_info); tree vectype = STMT_VINFO_VECTYPE (stmt_info);
tree vec_rhs1 = NULL_TREE, vec_rhs2 = NULL_TREE; tree vec_rhs1 = NULL_TREE, vec_rhs2 = NULL_TREE;
...@@ -9197,7 +9182,8 @@ vectorizable_comparison (gimple *stmt, gimple_stmt_iterator *gsi, ...@@ -9197,7 +9182,8 @@ vectorizable_comparison (gimple *stmt, gimple_stmt_iterator *gsi,
return false; return false;
} }
if (!is_gimple_assign (stmt)) gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
if (!stmt)
return false; return false;
code = gimple_assign_rhs_code (stmt); code = gimple_assign_rhs_code (stmt);
...@@ -9446,10 +9432,10 @@ can_vectorize_live_stmts (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, ...@@ -9446,10 +9432,10 @@ can_vectorize_live_stmts (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
/* Make sure the statement is vectorizable. */ /* Make sure the statement is vectorizable. */
bool bool
vect_analyze_stmt (gimple *stmt, bool *need_to_vectorize, slp_tree node, vect_analyze_stmt (stmt_vec_info stmt_info, bool *need_to_vectorize,
slp_instance node_instance, stmt_vector_for_cost *cost_vec) slp_tree node, slp_instance node_instance,
stmt_vector_for_cost *cost_vec)
{ {
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
vec_info *vinfo = stmt_info->vinfo; vec_info *vinfo = stmt_info->vinfo;
bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info); bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
enum vect_relevant relevance = STMT_VINFO_RELEVANT (stmt_info); enum vect_relevant relevance = STMT_VINFO_RELEVANT (stmt_info);
...@@ -9525,7 +9511,6 @@ vect_analyze_stmt (gimple *stmt, bool *need_to_vectorize, slp_tree node, ...@@ -9525,7 +9511,6 @@ vect_analyze_stmt (gimple *stmt, bool *need_to_vectorize, slp_tree node,
|| STMT_VINFO_LIVE_P (pattern_stmt_info))) || STMT_VINFO_LIVE_P (pattern_stmt_info)))
{ {
/* Analyze PATTERN_STMT instead of the original stmt. */ /* Analyze PATTERN_STMT instead of the original stmt. */
stmt = pattern_stmt_info->stmt;
stmt_info = pattern_stmt_info; stmt_info = pattern_stmt_info;
if (dump_enabled_p ()) if (dump_enabled_p ())
{ {
...@@ -9682,14 +9667,13 @@ vect_analyze_stmt (gimple *stmt, bool *need_to_vectorize, slp_tree node, ...@@ -9682,14 +9667,13 @@ vect_analyze_stmt (gimple *stmt, bool *need_to_vectorize, slp_tree node,
/* Function vect_transform_stmt. /* Function vect_transform_stmt.
Create a vectorized stmt to replace STMT, and insert it at BSI. */ Create a vectorized stmt to replace STMT_INFO, and insert it at BSI. */
bool bool
vect_transform_stmt (gimple *stmt, gimple_stmt_iterator *gsi, vect_transform_stmt (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
bool *grouped_store, slp_tree slp_node, bool *grouped_store, slp_tree slp_node,
slp_instance slp_node_instance) slp_instance slp_node_instance)
{ {
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
vec_info *vinfo = stmt_info->vinfo; vec_info *vinfo = stmt_info->vinfo;
bool is_store = false; bool is_store = false;
stmt_vec_info vec_stmt = NULL; stmt_vec_info vec_stmt = NULL;
...@@ -9703,6 +9687,7 @@ vect_transform_stmt (gimple *stmt, gimple_stmt_iterator *gsi, ...@@ -9703,6 +9687,7 @@ vect_transform_stmt (gimple *stmt, gimple_stmt_iterator *gsi,
(LOOP_VINFO_LOOP (STMT_VINFO_LOOP_VINFO (stmt_info)), (LOOP_VINFO_LOOP (STMT_VINFO_LOOP_VINFO (stmt_info)),
stmt_info)); stmt_info));
gimple *stmt = stmt_info->stmt;
switch (STMT_VINFO_TYPE (stmt_info)) switch (STMT_VINFO_TYPE (stmt_info))
{ {
case type_demotion_vec_info_type: case type_demotion_vec_info_type:
...@@ -9861,9 +9846,9 @@ vect_transform_stmt (gimple *stmt, gimple_stmt_iterator *gsi, ...@@ -9861,9 +9846,9 @@ vect_transform_stmt (gimple *stmt, gimple_stmt_iterator *gsi,
stmt_vec_info. */ stmt_vec_info. */
void void
vect_remove_stores (gimple *first_stmt) vect_remove_stores (stmt_vec_info first_stmt_info)
{ {
stmt_vec_info next_stmt_info = vinfo_for_stmt (first_stmt); stmt_vec_info next_stmt_info = first_stmt_info;
gimple_stmt_iterator next_si; gimple_stmt_iterator next_si;
while (next_stmt_info) while (next_stmt_info)
...@@ -10329,13 +10314,12 @@ vect_is_simple_use (tree operand, vec_info *vinfo, enum vect_def_type *dt, ...@@ -10329,13 +10314,12 @@ vect_is_simple_use (tree operand, vec_info *vinfo, enum vect_def_type *dt,
widening operation (short in the above example). */ widening operation (short in the above example). */
bool bool
supportable_widening_operation (enum tree_code code, gimple *stmt, supportable_widening_operation (enum tree_code code, stmt_vec_info stmt_info,
tree vectype_out, tree vectype_in, tree vectype_out, tree vectype_in,
enum tree_code *code1, enum tree_code *code2, enum tree_code *code1, enum tree_code *code2,
int *multi_step_cvt, int *multi_step_cvt,
vec<tree> *interm_types) vec<tree> *interm_types)
{ {
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
loop_vec_info loop_info = STMT_VINFO_LOOP_VINFO (stmt_info); loop_vec_info loop_info = STMT_VINFO_LOOP_VINFO (stmt_info);
struct loop *vect_loop = NULL; struct loop *vect_loop = NULL;
machine_mode vec_mode; machine_mode vec_mode;
......
gcc/tree-vectorizer.h
...@@ -627,13 +627,6 @@ loop_vec_info_for_loop (struct loop *loop) ...@@ -627,13 +627,6 @@ loop_vec_info_for_loop (struct loop *loop)
return (loop_vec_info) loop->aux; return (loop_vec_info) loop->aux;
} }
static inline bool
nested_in_vect_loop_p (struct loop *loop, gimple *stmt)
{
return (loop->inner
&& (loop->inner == (gimple_bb (stmt))->loop_father));
}
typedef struct _bb_vec_info : public vec_info typedef struct _bb_vec_info : public vec_info
{ {
_bb_vec_info (gimple_stmt_iterator, gimple_stmt_iterator, vec_info_shared *); _bb_vec_info (gimple_stmt_iterator, gimple_stmt_iterator, vec_info_shared *);
...@@ -1119,6 +1112,13 @@ set_vinfo_for_stmt (gimple *stmt, stmt_vec_info info) ...@@ -1119,6 +1112,13 @@ set_vinfo_for_stmt (gimple *stmt, stmt_vec_info info)
} }
} }
static inline bool
nested_in_vect_loop_p (struct loop *loop, stmt_vec_info stmt_info)
{
return (loop->inner
&& (loop->inner == (gimple_bb (stmt_info->stmt))->loop_father));
}
/* Return the earlier statement between STMT1_INFO and STMT2_INFO. */ /* Return the earlier statement between STMT1_INFO and STMT2_INFO. */
static inline stmt_vec_info static inline stmt_vec_info
...@@ -1493,8 +1493,8 @@ extern bool vect_is_simple_use (tree, vec_info *, enum vect_def_type *, ...@@ -1493,8 +1493,8 @@ extern bool vect_is_simple_use (tree, vec_info *, enum vect_def_type *,
extern bool vect_is_simple_use (tree, vec_info *, enum vect_def_type *, extern bool vect_is_simple_use (tree, vec_info *, enum vect_def_type *,
tree *, stmt_vec_info * = NULL, tree *, stmt_vec_info * = NULL,
gimple ** = NULL); gimple ** = NULL);
extern bool supportable_widening_operation (enum tree_code, gimple *, tree, extern bool supportable_widening_operation (enum tree_code, stmt_vec_info,
tree, enum tree_code *, tree, tree, enum tree_code *,
enum tree_code *, int *, enum tree_code *, int *,
vec<tree> *); vec<tree> *);
extern bool supportable_narrowing_operation (enum tree_code, tree, tree, extern bool supportable_narrowing_operation (enum tree_code, tree, tree,
...@@ -1505,26 +1505,26 @@ extern void free_stmt_vec_info (gimple *stmt); ...@@ -1505,26 +1505,26 @@ extern void free_stmt_vec_info (gimple *stmt);
extern unsigned record_stmt_cost (stmt_vector_for_cost *, int, extern unsigned record_stmt_cost (stmt_vector_for_cost *, int,
enum vect_cost_for_stmt, stmt_vec_info, enum vect_cost_for_stmt, stmt_vec_info,
int, enum vect_cost_model_location); int, enum vect_cost_model_location);
extern stmt_vec_info vect_finish_replace_stmt (gimple *, gimple *); extern stmt_vec_info vect_finish_replace_stmt (stmt_vec_info, gimple *);
extern stmt_vec_info vect_finish_stmt_generation (gimple *, gimple *, extern stmt_vec_info vect_finish_stmt_generation (stmt_vec_info, gimple *,
gimple_stmt_iterator *); gimple_stmt_iterator *);
extern bool vect_mark_stmts_to_be_vectorized (loop_vec_info); extern bool vect_mark_stmts_to_be_vectorized (loop_vec_info);
extern tree vect_get_store_rhs (gimple *); extern tree vect_get_store_rhs (stmt_vec_info);
extern tree vect_get_vec_def_for_operand_1 (gimple *, enum vect_def_type); extern tree vect_get_vec_def_for_operand_1 (stmt_vec_info, enum vect_def_type);
extern tree vect_get_vec_def_for_operand (tree, gimple *, tree = NULL); extern tree vect_get_vec_def_for_operand (tree, stmt_vec_info, tree = NULL);
extern void vect_get_vec_defs (tree, tree, gimple *, vec<tree> *, extern void vect_get_vec_defs (tree, tree, stmt_vec_info, vec<tree> *,
vec<tree> *, slp_tree); vec<tree> *, slp_tree);
extern void vect_get_vec_defs_for_stmt_copy (enum vect_def_type *, extern void vect_get_vec_defs_for_stmt_copy (enum vect_def_type *,
vec<tree> *, vec<tree> *); vec<tree> *, vec<tree> *);
extern tree vect_init_vector (gimple *, tree, tree, extern tree vect_init_vector (stmt_vec_info, tree, tree,
gimple_stmt_iterator *); gimple_stmt_iterator *);
extern tree vect_get_vec_def_for_stmt_copy (enum vect_def_type, tree); extern tree vect_get_vec_def_for_stmt_copy (enum vect_def_type, tree);
extern bool vect_transform_stmt (gimple *, gimple_stmt_iterator *, extern bool vect_transform_stmt (stmt_vec_info, gimple_stmt_iterator *,
bool *, slp_tree, slp_instance); bool *, slp_tree, slp_instance);
extern void vect_remove_stores (gimple *); extern void vect_remove_stores (stmt_vec_info);
extern bool vect_analyze_stmt (gimple *, bool *, slp_tree, slp_instance, extern bool vect_analyze_stmt (stmt_vec_info, bool *, slp_tree, slp_instance,
stmt_vector_for_cost *); stmt_vector_for_cost *);
extern bool vectorizable_condition (gimple *, gimple_stmt_iterator *, extern bool vectorizable_condition (stmt_vec_info, gimple_stmt_iterator *,
stmt_vec_info *, tree, int, slp_tree, stmt_vec_info *, tree, int, slp_tree,
stmt_vector_for_cost *); stmt_vector_for_cost *);
extern void vect_get_load_cost (stmt_vec_info, int, bool, extern void vect_get_load_cost (stmt_vec_info, int, bool,
...@@ -1546,7 +1546,7 @@ extern tree vect_get_mask_type_for_stmt (stmt_vec_info); ...@@ -1546,7 +1546,7 @@ extern tree vect_get_mask_type_for_stmt (stmt_vec_info);
extern bool vect_can_force_dr_alignment_p (const_tree, unsigned int); extern bool vect_can_force_dr_alignment_p (const_tree, unsigned int);
extern enum dr_alignment_support vect_supportable_dr_alignment extern enum dr_alignment_support vect_supportable_dr_alignment
(struct data_reference *, bool); (struct data_reference *, bool);
extern tree vect_get_smallest_scalar_type (gimple *, HOST_WIDE_INT *, extern tree vect_get_smallest_scalar_type (stmt_vec_info, HOST_WIDE_INT *,
HOST_WIDE_INT *); HOST_WIDE_INT *);
extern bool vect_analyze_data_ref_dependences (loop_vec_info, unsigned int *); extern bool vect_analyze_data_ref_dependences (loop_vec_info, unsigned int *);
extern bool vect_slp_analyze_instance_dependence (slp_instance); extern bool vect_slp_analyze_instance_dependence (slp_instance);
...@@ -1558,36 +1558,36 @@ extern bool vect_analyze_data_ref_accesses (vec_info *); ...@@ -1558,36 +1558,36 @@ extern bool vect_analyze_data_ref_accesses (vec_info *);
extern bool vect_prune_runtime_alias_test_list (loop_vec_info); extern bool vect_prune_runtime_alias_test_list (loop_vec_info);
extern bool vect_gather_scatter_fn_p (bool, bool, tree, tree, unsigned int, extern bool vect_gather_scatter_fn_p (bool, bool, tree, tree, unsigned int,
signop, int, internal_fn *, tree *); signop, int, internal_fn *, tree *);
extern bool vect_check_gather_scatter (gimple *, loop_vec_info, extern bool vect_check_gather_scatter (stmt_vec_info, loop_vec_info,
gather_scatter_info *); gather_scatter_info *);
extern bool vect_find_stmt_data_reference (loop_p, gimple *, extern bool vect_find_stmt_data_reference (loop_p, gimple *,
vec<data_reference_p> *); vec<data_reference_p> *);
extern bool vect_analyze_data_refs (vec_info *, poly_uint64 *); extern bool vect_analyze_data_refs (vec_info *, poly_uint64 *);
extern void vect_record_base_alignments (vec_info *); extern void vect_record_base_alignments (vec_info *);
extern tree vect_create_data_ref_ptr (gimple *, tree, struct loop *, tree, extern tree vect_create_data_ref_ptr (stmt_vec_info, tree, struct loop *, tree,
tree *, gimple_stmt_iterator *, tree *, gimple_stmt_iterator *,
gimple **, bool, bool *, gimple **, bool, bool *,
tree = NULL_TREE, tree = NULL_TREE); tree = NULL_TREE, tree = NULL_TREE);
extern tree bump_vector_ptr (tree, gimple *, gimple_stmt_iterator *, gimple *, extern tree bump_vector_ptr (tree, gimple *, gimple_stmt_iterator *,
tree); stmt_vec_info, tree);
extern void vect_copy_ref_info (tree, tree); extern void vect_copy_ref_info (tree, tree);
extern tree vect_create_destination_var (tree, tree); extern tree vect_create_destination_var (tree, tree);
extern bool vect_grouped_store_supported (tree, unsigned HOST_WIDE_INT); extern bool vect_grouped_store_supported (tree, unsigned HOST_WIDE_INT);
extern bool vect_store_lanes_supported (tree, unsigned HOST_WIDE_INT, bool); extern bool vect_store_lanes_supported (tree, unsigned HOST_WIDE_INT, bool);
extern bool vect_grouped_load_supported (tree, bool, unsigned HOST_WIDE_INT); extern bool vect_grouped_load_supported (tree, bool, unsigned HOST_WIDE_INT);
extern bool vect_load_lanes_supported (tree, unsigned HOST_WIDE_INT, bool); extern bool vect_load_lanes_supported (tree, unsigned HOST_WIDE_INT, bool);
extern void vect_permute_store_chain (vec<tree> ,unsigned int, gimple *, extern void vect_permute_store_chain (vec<tree> ,unsigned int, stmt_vec_info,
gimple_stmt_iterator *, vec<tree> *); gimple_stmt_iterator *, vec<tree> *);
extern tree vect_setup_realignment (gimple *, gimple_stmt_iterator *, tree *, extern tree vect_setup_realignment (stmt_vec_info, gimple_stmt_iterator *,
enum dr_alignment_support, tree, tree *, enum dr_alignment_support, tree,
struct loop **); struct loop **);
extern void vect_transform_grouped_load (gimple *, vec<tree> , int, extern void vect_transform_grouped_load (stmt_vec_info, vec<tree> , int,
gimple_stmt_iterator *); gimple_stmt_iterator *);
extern void vect_record_grouped_load_vectors (gimple *, vec<tree> ); extern void vect_record_grouped_load_vectors (stmt_vec_info, vec<tree>);
extern tree vect_get_new_vect_var (tree, enum vect_var_kind, const char *); extern tree vect_get_new_vect_var (tree, enum vect_var_kind, const char *);
extern tree vect_get_new_ssa_name (tree, enum vect_var_kind, extern tree vect_get_new_ssa_name (tree, enum vect_var_kind,
const char * = NULL); const char * = NULL);
extern tree vect_create_addr_base_for_vector_ref (gimple *, gimple_seq *, extern tree vect_create_addr_base_for_vector_ref (stmt_vec_info, gimple_seq *,
tree, tree = NULL_TREE); tree, tree = NULL_TREE);
/* In tree-vect-loop.c. */ /* In tree-vect-loop.c. */
...@@ -1613,16 +1613,16 @@ extern tree vect_get_loop_mask (gimple_stmt_iterator *, vec_loop_masks *, ...@@ -1613,16 +1613,16 @@ extern tree vect_get_loop_mask (gimple_stmt_iterator *, vec_loop_masks *,
/* Drive for loop transformation stage. */ /* Drive for loop transformation stage. */
extern struct loop *vect_transform_loop (loop_vec_info); extern struct loop *vect_transform_loop (loop_vec_info);
extern loop_vec_info vect_analyze_loop_form (struct loop *, vec_info_shared *); extern loop_vec_info vect_analyze_loop_form (struct loop *, vec_info_shared *);
extern bool vectorizable_live_operation (gimple *, gimple_stmt_iterator *, extern bool vectorizable_live_operation (stmt_vec_info, gimple_stmt_iterator *,
slp_tree, int, stmt_vec_info *, slp_tree, int, stmt_vec_info *,
stmt_vector_for_cost *); stmt_vector_for_cost *);
extern bool vectorizable_reduction (gimple *, gimple_stmt_iterator *, extern bool vectorizable_reduction (stmt_vec_info, gimple_stmt_iterator *,
stmt_vec_info *, slp_tree, slp_instance, stmt_vec_info *, slp_tree, slp_instance,
stmt_vector_for_cost *); stmt_vector_for_cost *);
extern bool vectorizable_induction (gimple *, gimple_stmt_iterator *, extern bool vectorizable_induction (stmt_vec_info, gimple_stmt_iterator *,
stmt_vec_info *, slp_tree, stmt_vec_info *, slp_tree,
stmt_vector_for_cost *); stmt_vector_for_cost *);
extern tree get_initial_def_for_reduction (gimple *, tree, tree *); extern tree get_initial_def_for_reduction (stmt_vec_info, tree, tree *);
extern bool vect_worthwhile_without_simd_p (vec_info *, tree_code); extern bool vect_worthwhile_without_simd_p (vec_info *, tree_code);
extern int vect_get_known_peeling_cost (loop_vec_info, int, int *, extern int vect_get_known_peeling_cost (loop_vec_info, int, int *,
stmt_vector_for_cost *, stmt_vector_for_cost *,
...@@ -1643,13 +1643,13 @@ extern void vect_detect_hybrid_slp (loop_vec_info); ...@@ -1643,13 +1643,13 @@ extern void vect_detect_hybrid_slp (loop_vec_info);
extern void vect_get_slp_defs (vec<tree> , slp_tree, vec<vec<tree> > *); extern void vect_get_slp_defs (vec<tree> , slp_tree, vec<vec<tree> > *);
extern bool vect_slp_bb (basic_block); extern bool vect_slp_bb (basic_block);
extern stmt_vec_info vect_find_last_scalar_stmt_in_slp (slp_tree); extern stmt_vec_info vect_find_last_scalar_stmt_in_slp (slp_tree);
extern bool is_simple_and_all_uses_invariant (gimple *, loop_vec_info); extern bool is_simple_and_all_uses_invariant (stmt_vec_info, loop_vec_info);
extern bool can_duplicate_and_interleave_p (unsigned int, machine_mode, extern bool can_duplicate_and_interleave_p (unsigned int, machine_mode,
unsigned int * = NULL, unsigned int * = NULL,
tree * = NULL, tree * = NULL); tree * = NULL, tree * = NULL);
extern void duplicate_and_interleave (gimple_seq *, tree, vec<tree>, extern void duplicate_and_interleave (gimple_seq *, tree, vec<tree>,
unsigned int, vec<tree> &); unsigned int, vec<tree> &);
extern int vect_get_place_in_interleaving_chain (gimple *, gimple *); extern int vect_get_place_in_interleaving_chain (stmt_vec_info, stmt_vec_info);
/* In tree-vect-patterns.c. */ /* In tree-vect-patterns.c. */
/* Pattern recognition functions. /* Pattern recognition functions.
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment