Skip to content

R
riscv-gcc-1
Commit 134c85ca by Richard Sandiford Committed by Richard Sandiford
Options

[4/7] Add a gather_scatter_info structure 

This patch just refactors the gather/scatter support so that all
information is in a single structure, rather than separate variables.
This reduces the number of arguments to a function added in patch 6.

Tested on aarch64-linux-gnu and x86_64-linux-gnu.

gcc/
	* tree-vectorizer.h (gather_scatter_info): New structure.
	(vect_check_gather_scatter): Return a bool rather than a decl.
	Replace return-by-pointer arguments with a single
	gather_scatter_info *.
	* tree-vect-data-refs.c (vect_check_gather_scatter): Likewise.
	(vect_analyze_data_refs): Update call accordingly.
	* tree-vect-stmts.c (vect_mark_stmts_to_be_vectorized): Likewise.
	(vectorizable_mask_load_store): Likewise.  Also record the
	offset dt and vectype in the gather_scatter_info.
	(vectorizable_store): Likewise.
	(vectorizable_load): Likewise.

From-SVN: r238036
parent 071e8018
Showing with 110 additions and 83 deletions
gcc/ChangeLog
2016-07-06 Richard Sandiford <richard.sandiford@arm.com> 2016-07-06 Richard Sandiford <richard.sandiford@arm.com>
* tree-vectorizer.h (gather_scatter_info): New structure.
(vect_check_gather_scatter): Return a bool rather than a decl.
Replace return-by-pointer arguments with a single
gather_scatter_info *.
* tree-vect-data-refs.c (vect_check_gather_scatter): Likewise.
(vect_analyze_data_refs): Update call accordingly.
* tree-vect-stmts.c (vect_mark_stmts_to_be_vectorized): Likewise.
(vectorizable_mask_load_store): Likewise. Also record the
offset dt and vectype in the gather_scatter_info.
(vectorizable_store): Likewise.
(vectorizable_load): Likewise.
2016-07-06 Richard Sandiford <richard.sandiford@arm.com>
* tree-vect-stmts.c (vect_model_store_cost): For non-SLP * tree-vect-stmts.c (vect_model_store_cost): For non-SLP
strided groups, use the cost of N scalar accesses instead strided groups, use the cost of N scalar accesses instead
of ncopies vector accesses. of ncopies vector accesses.
......
gcc/tree-vect-data-refs.c
...@@ -3187,12 +3187,12 @@ vect_prune_runtime_alias_test_list (loop_vec_info loop_vinfo) ...@@ -3187,12 +3187,12 @@ vect_prune_runtime_alias_test_list (loop_vec_info loop_vinfo)
return true; return true;
} }
/* Check whether a non-affine read or write in stmt is suitable for gather load /* Return true if a non-affine read or write in STMT is suitable for a
or scatter store and if so, return a builtin decl for that operation. */ gather load or scatter store. Describe the operation in *INFO if so. */
tree bool
vect_check_gather_scatter (gimple *stmt, loop_vec_info loop_vinfo, tree *basep, vect_check_gather_scatter (gimple *stmt, loop_vec_info loop_vinfo,
tree *offp, int *scalep) gather_scatter_info *info)
{ {
HOST_WIDE_INT scale = 1, pbitpos, pbitsize; HOST_WIDE_INT scale = 1, pbitpos, pbitsize;
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo); struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
...@@ -3266,7 +3266,7 @@ vect_check_gather_scatter (gimple *stmt, loop_vec_info loop_vinfo, tree *basep, ...@@ -3266,7 +3266,7 @@ vect_check_gather_scatter (gimple *stmt, loop_vec_info loop_vinfo, tree *basep,
if (!expr_invariant_in_loop_p (loop, base)) if (!expr_invariant_in_loop_p (loop, base))
{ {
if (!integer_zerop (off)) if (!integer_zerop (off))
return NULL_TREE; return false;
off = base; off = base;
base = size_int (pbitpos / BITS_PER_UNIT); base = size_int (pbitpos / BITS_PER_UNIT);
} }
...@@ -3292,7 +3292,7 @@ vect_check_gather_scatter (gimple *stmt, loop_vec_info loop_vinfo, tree *basep, ...@@ -3292,7 +3292,7 @@ vect_check_gather_scatter (gimple *stmt, loop_vec_info loop_vinfo, tree *basep,
gimple *def_stmt = SSA_NAME_DEF_STMT (off); gimple *def_stmt = SSA_NAME_DEF_STMT (off);
if (expr_invariant_in_loop_p (loop, off)) if (expr_invariant_in_loop_p (loop, off))
return NULL_TREE; return false;
if (gimple_code (def_stmt) != GIMPLE_ASSIGN) if (gimple_code (def_stmt) != GIMPLE_ASSIGN)
break; break;
...@@ -3304,7 +3304,7 @@ vect_check_gather_scatter (gimple *stmt, loop_vec_info loop_vinfo, tree *basep, ...@@ -3304,7 +3304,7 @@ vect_check_gather_scatter (gimple *stmt, loop_vec_info loop_vinfo, tree *basep,
else else
{ {
if (get_gimple_rhs_class (TREE_CODE (off)) == GIMPLE_TERNARY_RHS) if (get_gimple_rhs_class (TREE_CODE (off)) == GIMPLE_TERNARY_RHS)
return NULL_TREE; return false;
code = TREE_CODE (off); code = TREE_CODE (off);
extract_ops_from_tree (off, &code, &op0, &op1); extract_ops_from_tree (off, &code, &op0, &op1);
} }
...@@ -3379,7 +3379,7 @@ vect_check_gather_scatter (gimple *stmt, loop_vec_info loop_vinfo, tree *basep, ...@@ -3379,7 +3379,7 @@ vect_check_gather_scatter (gimple *stmt, loop_vec_info loop_vinfo, tree *basep,
defined in the loop, punt. */ defined in the loop, punt. */
if (TREE_CODE (off) != SSA_NAME if (TREE_CODE (off) != SSA_NAME
|| expr_invariant_in_loop_p (loop, off)) || expr_invariant_in_loop_p (loop, off))
return NULL_TREE; return false;
if (offtype == NULL_TREE) if (offtype == NULL_TREE)
offtype = TREE_TYPE (off); offtype = TREE_TYPE (off);
...@@ -3392,15 +3392,15 @@ vect_check_gather_scatter (gimple *stmt, loop_vec_info loop_vinfo, tree *basep, ...@@ -3392,15 +3392,15 @@ vect_check_gather_scatter (gimple *stmt, loop_vec_info loop_vinfo, tree *basep,
offtype, scale); offtype, scale);
if (decl == NULL_TREE) if (decl == NULL_TREE)
return NULL_TREE; return false;
if (basep) info->decl = decl;
*basep = base; info->base = base;
if (offp) info->offset = off;
*offp = off; info->offset_dt = vect_unknown_def_type;
if (scalep) info->offset_vectype = NULL_TREE;
*scalep = scale; info->scale = scale;
return decl; return true;
} }
/* Function vect_analyze_data_refs. /* Function vect_analyze_data_refs.
...@@ -3878,10 +3878,10 @@ again: ...@@ -3878,10 +3878,10 @@ again:
if (gatherscatter != SG_NONE) if (gatherscatter != SG_NONE)
{ {
tree off; gather_scatter_info gs_info;
if (!vect_check_gather_scatter (stmt, as_a <loop_vec_info> (vinfo), if (!vect_check_gather_scatter (stmt, as_a <loop_vec_info> (vinfo),
NULL, &off, NULL) &gs_info)
|| get_vectype_for_scalar_type (TREE_TYPE (off)) == NULL_TREE) || !get_vectype_for_scalar_type (TREE_TYPE (gs_info.offset)))
{ {
STMT_VINFO_DATA_REF (stmt_info) = NULL; STMT_VINFO_DATA_REF (stmt_info) = NULL;
free_data_ref (dr); free_data_ref (dr);
......
gcc/tree-vect-stmts.c
...@@ -772,10 +772,11 @@ vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo) ...@@ -772,10 +772,11 @@ vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo)
if (STMT_VINFO_GATHER_SCATTER_P (stmt_vinfo)) if (STMT_VINFO_GATHER_SCATTER_P (stmt_vinfo))
{ {
tree off; gather_scatter_info gs_info;
tree decl = vect_check_gather_scatter (stmt, loop_vinfo, NULL, &off, NULL); if (!vect_check_gather_scatter (stmt, loop_vinfo, &gs_info))
gcc_assert (decl); gcc_unreachable ();
if (!process_use (stmt, off, loop_vinfo, relevant, &worklist, true)) if (!process_use (stmt, gs_info.offset, loop_vinfo, relevant,
&worklist, true))
return false; return false;
} }
} /* while worklist */ } /* while worklist */
...@@ -1703,10 +1704,7 @@ vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi, ...@@ -1703,10 +1704,7 @@ vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi,
int ncopies; int ncopies;
int i, j; int i, j;
bool inv_p; bool inv_p;
tree gather_base = NULL_TREE, gather_off = NULL_TREE; gather_scatter_info gs_info;
tree gather_off_vectype = NULL_TREE, gather_decl = NULL_TREE;
int gather_scale = 1;
enum vect_def_type gather_dt = vect_unknown_def_type;
bool is_store; bool is_store;
tree mask; tree mask;
gimple *def_stmt; gimple *def_stmt;
...@@ -1774,11 +1772,10 @@ vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi, ...@@ -1774,11 +1772,10 @@ vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi,
if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)) if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
{ {
gimple *def_stmt; gimple *def_stmt;
gather_decl = vect_check_gather_scatter (stmt, loop_vinfo, &gather_base, if (!vect_check_gather_scatter (stmt, loop_vinfo, &gs_info))
&gather_off, &gather_scale); gcc_unreachable ();
gcc_assert (gather_decl); if (!vect_is_simple_use (gs_info.offset, loop_vinfo, &def_stmt,
if (!vect_is_simple_use (gather_off, loop_vinfo, &def_stmt, &gather_dt, &gs_info.offset_dt, &gs_info.offset_vectype))
&gather_off_vectype))
{ {
if (dump_enabled_p ()) if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
...@@ -1786,7 +1783,7 @@ vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi, ...@@ -1786,7 +1783,7 @@ vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi,
return false; return false;
} }
tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gather_decl)); tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gs_info.decl));
tree masktype tree masktype
= TREE_VALUE (TREE_CHAIN (TREE_CHAIN (TREE_CHAIN (arglist)))); = TREE_VALUE (TREE_CHAIN (TREE_CHAIN (TREE_CHAIN (arglist))));
if (TREE_CODE (masktype) == INTEGER_TYPE) if (TREE_CODE (masktype) == INTEGER_TYPE)
...@@ -1825,7 +1822,7 @@ vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi, ...@@ -1825,7 +1822,7 @@ vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi,
if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)) if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
{ {
tree vec_oprnd0 = NULL_TREE, op; tree vec_oprnd0 = NULL_TREE, op;
tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gather_decl)); tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gs_info.decl));
tree rettype, srctype, ptrtype, idxtype, masktype, scaletype; tree rettype, srctype, ptrtype, idxtype, masktype, scaletype;
tree ptr, vec_mask = NULL_TREE, mask_op = NULL_TREE, var, scale; tree ptr, vec_mask = NULL_TREE, mask_op = NULL_TREE, var, scale;
tree perm_mask = NULL_TREE, prev_res = NULL_TREE; tree perm_mask = NULL_TREE, prev_res = NULL_TREE;
...@@ -1834,9 +1831,9 @@ vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi, ...@@ -1834,9 +1831,9 @@ vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi,
gimple_seq seq; gimple_seq seq;
basic_block new_bb; basic_block new_bb;
enum { NARROW, NONE, WIDEN } modifier; enum { NARROW, NONE, WIDEN } modifier;
int gather_off_nunits = TYPE_VECTOR_SUBPARTS (gather_off_vectype); int gather_off_nunits = TYPE_VECTOR_SUBPARTS (gs_info.offset_vectype);
rettype = TREE_TYPE (TREE_TYPE (gather_decl)); rettype = TREE_TYPE (TREE_TYPE (gs_info.decl));
srctype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist); srctype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
ptrtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist); ptrtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
idxtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist); idxtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
...@@ -1855,7 +1852,7 @@ vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi, ...@@ -1855,7 +1852,7 @@ vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi,
for (i = 0; i < gather_off_nunits; ++i) for (i = 0; i < gather_off_nunits; ++i)
sel[i] = i | nunits; sel[i] = i | nunits;
perm_mask = vect_gen_perm_mask_checked (gather_off_vectype, sel); perm_mask = vect_gen_perm_mask_checked (gs_info.offset_vectype, sel);
} }
else if (nunits == gather_off_nunits * 2) else if (nunits == gather_off_nunits * 2)
{ {
...@@ -1877,7 +1874,7 @@ vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi, ...@@ -1877,7 +1874,7 @@ vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi,
vec_dest = vect_create_destination_var (gimple_call_lhs (stmt), vectype); vec_dest = vect_create_destination_var (gimple_call_lhs (stmt), vectype);
ptr = fold_convert (ptrtype, gather_base); ptr = fold_convert (ptrtype, gs_info.base);
if (!is_gimple_min_invariant (ptr)) if (!is_gimple_min_invariant (ptr))
{ {
ptr = force_gimple_operand (ptr, &seq, true, NULL_TREE); ptr = force_gimple_operand (ptr, &seq, true, NULL_TREE);
...@@ -1885,7 +1882,7 @@ vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi, ...@@ -1885,7 +1882,7 @@ vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi,
gcc_assert (!new_bb); gcc_assert (!new_bb);
} }
scale = build_int_cst (scaletype, gather_scale); scale = build_int_cst (scaletype, gs_info.scale);
prev_stmt_info = NULL; prev_stmt_info = NULL;
for (j = 0; j < ncopies; ++j) for (j = 0; j < ncopies; ++j)
...@@ -1895,10 +1892,10 @@ vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi, ...@@ -1895,10 +1892,10 @@ vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi,
perm_mask, stmt, gsi); perm_mask, stmt, gsi);
else if (j == 0) else if (j == 0)
op = vec_oprnd0 op = vec_oprnd0
= vect_get_vec_def_for_operand (gather_off, stmt); = vect_get_vec_def_for_operand (gs_info.offset, stmt);
else else
op = vec_oprnd0 op = vec_oprnd0
= vect_get_vec_def_for_stmt_copy (gather_dt, vec_oprnd0); = vect_get_vec_def_for_stmt_copy (gs_info.offset_dt, vec_oprnd0);
if (!useless_type_conversion_p (idxtype, TREE_TYPE (op))) if (!useless_type_conversion_p (idxtype, TREE_TYPE (op)))
{ {
...@@ -1940,7 +1937,7 @@ vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi, ...@@ -1940,7 +1937,7 @@ vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi,
} }
new_stmt new_stmt
= gimple_build_call (gather_decl, 5, mask_op, ptr, op, mask_op, = gimple_build_call (gs_info.decl, 5, mask_op, ptr, op, mask_op,
scale); scale);
if (!useless_type_conversion_p (vectype, rettype)) if (!useless_type_conversion_p (vectype, rettype))
...@@ -5207,10 +5204,7 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, ...@@ -5207,10 +5204,7 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt,
bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info); bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
vec_info *vinfo = stmt_info->vinfo; vec_info *vinfo = stmt_info->vinfo;
tree aggr_type; tree aggr_type;
tree scatter_base = NULL_TREE, scatter_off = NULL_TREE; gather_scatter_info gs_info;
tree scatter_off_vectype = NULL_TREE, scatter_decl = NULL_TREE;
int scatter_scale = 1;
enum vect_def_type scatter_idx_dt = vect_unknown_def_type;
enum vect_def_type scatter_src_dt = vect_unknown_def_type; enum vect_def_type scatter_src_dt = vect_unknown_def_type;
gimple *new_stmt; gimple *new_stmt;
int vf; int vf;
...@@ -5374,11 +5368,10 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, ...@@ -5374,11 +5368,10 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt,
if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)) if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
{ {
gimple *def_stmt; gimple *def_stmt;
scatter_decl = vect_check_gather_scatter (stmt, loop_vinfo, &scatter_base, if (!vect_check_gather_scatter (stmt, loop_vinfo, &gs_info))
&scatter_off, &scatter_scale); gcc_unreachable ();
gcc_assert (scatter_decl); if (!vect_is_simple_use (gs_info.offset, vinfo, &def_stmt,
if (!vect_is_simple_use (scatter_off, vinfo, &def_stmt, &scatter_idx_dt, &gs_info.offset_dt, &gs_info.offset_vectype))
&scatter_off_vectype))
{ {
if (dump_enabled_p ()) if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
...@@ -5404,14 +5397,14 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, ...@@ -5404,14 +5397,14 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt,
if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)) if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
{ {
tree vec_oprnd0 = NULL_TREE, vec_oprnd1 = NULL_TREE, op, src; tree vec_oprnd0 = NULL_TREE, vec_oprnd1 = NULL_TREE, op, src;
tree arglist = TYPE_ARG_TYPES (TREE_TYPE (scatter_decl)); tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gs_info.decl));
tree rettype, srctype, ptrtype, idxtype, masktype, scaletype; tree rettype, srctype, ptrtype, idxtype, masktype, scaletype;
tree ptr, mask, var, scale, perm_mask = NULL_TREE; tree ptr, mask, var, scale, perm_mask = NULL_TREE;
edge pe = loop_preheader_edge (loop); edge pe = loop_preheader_edge (loop);
gimple_seq seq; gimple_seq seq;
basic_block new_bb; basic_block new_bb;
enum { NARROW, NONE, WIDEN } modifier; enum { NARROW, NONE, WIDEN } modifier;
int scatter_off_nunits = TYPE_VECTOR_SUBPARTS (scatter_off_vectype); int scatter_off_nunits = TYPE_VECTOR_SUBPARTS (gs_info.offset_vectype);
if (nunits == (unsigned int) scatter_off_nunits) if (nunits == (unsigned int) scatter_off_nunits)
modifier = NONE; modifier = NONE;
...@@ -5423,7 +5416,7 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, ...@@ -5423,7 +5416,7 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt,
for (i = 0; i < (unsigned int) scatter_off_nunits; ++i) for (i = 0; i < (unsigned int) scatter_off_nunits; ++i)
sel[i] = i | nunits; sel[i] = i | nunits;
perm_mask = vect_gen_perm_mask_checked (scatter_off_vectype, sel); perm_mask = vect_gen_perm_mask_checked (gs_info.offset_vectype, sel);
gcc_assert (perm_mask != NULL_TREE); gcc_assert (perm_mask != NULL_TREE);
} }
else if (nunits == (unsigned int) scatter_off_nunits * 2) else if (nunits == (unsigned int) scatter_off_nunits * 2)
...@@ -5441,7 +5434,7 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, ...@@ -5441,7 +5434,7 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt,
else else
gcc_unreachable (); gcc_unreachable ();
rettype = TREE_TYPE (TREE_TYPE (scatter_decl)); rettype = TREE_TYPE (TREE_TYPE (gs_info.decl));
ptrtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist); ptrtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
masktype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist); masktype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
idxtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist); idxtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
...@@ -5451,7 +5444,7 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, ...@@ -5451,7 +5444,7 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt,
gcc_checking_assert (TREE_CODE (masktype) == INTEGER_TYPE gcc_checking_assert (TREE_CODE (masktype) == INTEGER_TYPE
&& TREE_CODE (rettype) == VOID_TYPE); && TREE_CODE (rettype) == VOID_TYPE);
ptr = fold_convert (ptrtype, scatter_base); ptr = fold_convert (ptrtype, gs_info.base);
if (!is_gimple_min_invariant (ptr)) if (!is_gimple_min_invariant (ptr))
{ {
ptr = force_gimple_operand (ptr, &seq, true, NULL_TREE); ptr = force_gimple_operand (ptr, &seq, true, NULL_TREE);
...@@ -5464,7 +5457,7 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, ...@@ -5464,7 +5457,7 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt,
mask = build_int_cst (masktype, -1); mask = build_int_cst (masktype, -1);
mask = vect_init_vector (stmt, mask, masktype, NULL); mask = vect_init_vector (stmt, mask, masktype, NULL);
scale = build_int_cst (scaletype, scatter_scale); scale = build_int_cst (scaletype, gs_info.scale);
prev_stmt_info = NULL; prev_stmt_info = NULL;
for (j = 0; j < ncopies; ++j) for (j = 0; j < ncopies; ++j)
...@@ -5474,7 +5467,7 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, ...@@ -5474,7 +5467,7 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt,
src = vec_oprnd1 src = vec_oprnd1
= vect_get_vec_def_for_operand (gimple_assign_rhs1 (stmt), stmt); = vect_get_vec_def_for_operand (gimple_assign_rhs1 (stmt), stmt);
op = vec_oprnd0 op = vec_oprnd0
= vect_get_vec_def_for_operand (scatter_off, stmt); = vect_get_vec_def_for_operand (gs_info.offset, stmt);
} }
else if (modifier != NONE && (j & 1)) else if (modifier != NONE && (j & 1))
{ {
...@@ -5490,7 +5483,8 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, ...@@ -5490,7 +5483,8 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt,
src = permute_vec_elements (vec_oprnd1, vec_oprnd1, perm_mask, src = permute_vec_elements (vec_oprnd1, vec_oprnd1, perm_mask,
stmt, gsi); stmt, gsi);
op = vec_oprnd0 op = vec_oprnd0
= vect_get_vec_def_for_stmt_copy (scatter_idx_dt, vec_oprnd0); = vect_get_vec_def_for_stmt_copy (gs_info.offset_dt,
vec_oprnd0);
} }
else else
gcc_unreachable (); gcc_unreachable ();
...@@ -5500,7 +5494,8 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, ...@@ -5500,7 +5494,8 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt,
src = vec_oprnd1 src = vec_oprnd1
= vect_get_vec_def_for_stmt_copy (scatter_src_dt, vec_oprnd1); = vect_get_vec_def_for_stmt_copy (scatter_src_dt, vec_oprnd1);
op = vec_oprnd0 op = vec_oprnd0
= vect_get_vec_def_for_stmt_copy (scatter_idx_dt, vec_oprnd0); = vect_get_vec_def_for_stmt_copy (gs_info.offset_dt,
vec_oprnd0);
} }
if (!useless_type_conversion_p (srctype, TREE_TYPE (src))) if (!useless_type_conversion_p (srctype, TREE_TYPE (src)))
...@@ -5526,7 +5521,7 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, ...@@ -5526,7 +5521,7 @@ vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt,
} }
new_stmt new_stmt
= gimple_build_call (scatter_decl, 5, ptr, mask, op, src, scale); = gimple_build_call (gs_info.decl, 5, ptr, mask, op, src, scale);
vect_finish_stmt_generation (stmt, new_stmt, gsi); vect_finish_stmt_generation (stmt, new_stmt, gsi);
...@@ -6221,10 +6216,7 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, ...@@ -6221,10 +6216,7 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt,
bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info); bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
int vf; int vf;
tree aggr_type; tree aggr_type;
tree gather_base = NULL_TREE, gather_off = NULL_TREE; gather_scatter_info gs_info;
tree gather_off_vectype = NULL_TREE, gather_decl = NULL_TREE;
int gather_scale = 1;
enum vect_def_type gather_dt = vect_unknown_def_type;
vec_info *vinfo = stmt_info->vinfo; vec_info *vinfo = stmt_info->vinfo;
if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo) if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
...@@ -6421,11 +6413,10 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, ...@@ -6421,11 +6413,10 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt,
if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)) if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
{ {
gimple *def_stmt; gimple *def_stmt;
gather_decl = vect_check_gather_scatter (stmt, loop_vinfo, &gather_base, if (!vect_check_gather_scatter (stmt, loop_vinfo, &gs_info))
&gather_off, &gather_scale); gcc_unreachable ();
gcc_assert (gather_decl); if (!vect_is_simple_use (gs_info.offset, vinfo, &def_stmt,
if (!vect_is_simple_use (gather_off, vinfo, &def_stmt, &gather_dt, &gs_info.offset_dt, &gs_info.offset_vectype))
&gather_off_vectype))
{ {
if (dump_enabled_p ()) if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
...@@ -6500,14 +6491,14 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, ...@@ -6500,14 +6491,14 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt,
if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)) if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
{ {
tree vec_oprnd0 = NULL_TREE, op; tree vec_oprnd0 = NULL_TREE, op;
tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gather_decl)); tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gs_info.decl));
tree rettype, srctype, ptrtype, idxtype, masktype, scaletype; tree rettype, srctype, ptrtype, idxtype, masktype, scaletype;
tree ptr, mask, var, scale, merge, perm_mask = NULL_TREE, prev_res = NULL_TREE; tree ptr, mask, var, scale, merge, perm_mask = NULL_TREE, prev_res = NULL_TREE;
edge pe = loop_preheader_edge (loop); edge pe = loop_preheader_edge (loop);
gimple_seq seq; gimple_seq seq;
basic_block new_bb; basic_block new_bb;
enum { NARROW, NONE, WIDEN } modifier; enum { NARROW, NONE, WIDEN } modifier;
int gather_off_nunits = TYPE_VECTOR_SUBPARTS (gather_off_vectype); int gather_off_nunits = TYPE_VECTOR_SUBPARTS (gs_info.offset_vectype);
if (nunits == gather_off_nunits) if (nunits == gather_off_nunits)
modifier = NONE; modifier = NONE;
...@@ -6519,7 +6510,7 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, ...@@ -6519,7 +6510,7 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt,
for (i = 0; i < gather_off_nunits; ++i) for (i = 0; i < gather_off_nunits; ++i)
sel[i] = i | nunits; sel[i] = i | nunits;
perm_mask = vect_gen_perm_mask_checked (gather_off_vectype, sel); perm_mask = vect_gen_perm_mask_checked (gs_info.offset_vectype, sel);
} }
else if (nunits == gather_off_nunits * 2) else if (nunits == gather_off_nunits * 2)
{ {
...@@ -6536,7 +6527,7 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, ...@@ -6536,7 +6527,7 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt,
else else
gcc_unreachable (); gcc_unreachable ();
rettype = TREE_TYPE (TREE_TYPE (gather_decl)); rettype = TREE_TYPE (TREE_TYPE (gs_info.decl));
srctype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist); srctype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
ptrtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist); ptrtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
idxtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist); idxtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
...@@ -6546,7 +6537,7 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, ...@@ -6546,7 +6537,7 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt,
vec_dest = vect_create_destination_var (scalar_dest, vectype); vec_dest = vect_create_destination_var (scalar_dest, vectype);
ptr = fold_convert (ptrtype, gather_base); ptr = fold_convert (ptrtype, gs_info.base);
if (!is_gimple_min_invariant (ptr)) if (!is_gimple_min_invariant (ptr))
{ {
ptr = force_gimple_operand (ptr, &seq, true, NULL_TREE); ptr = force_gimple_operand (ptr, &seq, true, NULL_TREE);
...@@ -6578,7 +6569,7 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, ...@@ -6578,7 +6569,7 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt,
else else
gcc_unreachable (); gcc_unreachable ();
scale = build_int_cst (scaletype, gather_scale); scale = build_int_cst (scaletype, gs_info.scale);
if (TREE_CODE (TREE_TYPE (rettype)) == INTEGER_TYPE) if (TREE_CODE (TREE_TYPE (rettype)) == INTEGER_TYPE)
merge = build_int_cst (TREE_TYPE (rettype), 0); merge = build_int_cst (TREE_TYPE (rettype), 0);
...@@ -6604,10 +6595,10 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, ...@@ -6604,10 +6595,10 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt,
perm_mask, stmt, gsi); perm_mask, stmt, gsi);
else if (j == 0) else if (j == 0)
op = vec_oprnd0 op = vec_oprnd0
= vect_get_vec_def_for_operand (gather_off, stmt); = vect_get_vec_def_for_operand (gs_info.offset, stmt);
else else
op = vec_oprnd0 op = vec_oprnd0
= vect_get_vec_def_for_stmt_copy (gather_dt, vec_oprnd0); = vect_get_vec_def_for_stmt_copy (gs_info.offset_dt, vec_oprnd0);
if (!useless_type_conversion_p (idxtype, TREE_TYPE (op))) if (!useless_type_conversion_p (idxtype, TREE_TYPE (op)))
{ {
...@@ -6622,7 +6613,7 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt, ...@@ -6622,7 +6613,7 @@ vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt,
} }
new_stmt new_stmt
= gimple_build_call (gather_decl, 5, merge, ptr, op, mask, scale); = gimple_build_call (gs_info.decl, 5, merge, ptr, op, mask, scale);
if (!useless_type_conversion_p (vectype, rettype)) if (!useless_type_conversion_p (vectype, rettype))
{ {
......
gcc/tree-vectorizer.h
...@@ -608,6 +608,28 @@ typedef struct _stmt_vec_info { ...@@ -608,6 +608,28 @@ typedef struct _stmt_vec_info {
unsigned int num_slp_uses; unsigned int num_slp_uses;
} *stmt_vec_info; } *stmt_vec_info;
/* Information about a gather/scatter call. */
struct gather_scatter_info {
/* The FUNCTION_DECL for the built-in gather/scatter function. */
tree decl;
/* The loop-invariant base value. */
tree base;
/* The original scalar offset, which is a non-loop-invariant SSA_NAME. */
tree offset;
/* Each offset element should be multiplied by this amount before
being added to the base. */
int scale;
/* The definition type for the vectorized offset. */
enum vect_def_type offset_dt;
/* The type of the vectorized offset. */
tree offset_vectype;
};
/* Access Functions. */ /* Access Functions. */
#define STMT_VINFO_TYPE(S) (S)->type #define STMT_VINFO_TYPE(S) (S)->type
#define STMT_VINFO_STMT(S) (S)->stmt #define STMT_VINFO_STMT(S) (S)->stmt
...@@ -1031,8 +1053,8 @@ extern bool vect_verify_datarefs_alignment (loop_vec_info); ...@@ -1031,8 +1053,8 @@ extern bool vect_verify_datarefs_alignment (loop_vec_info);
extern bool vect_slp_analyze_and_verify_instance_alignment (slp_instance); extern bool vect_slp_analyze_and_verify_instance_alignment (slp_instance);
extern bool vect_analyze_data_ref_accesses (vec_info *); extern bool vect_analyze_data_ref_accesses (vec_info *);
extern bool vect_prune_runtime_alias_test_list (loop_vec_info); extern bool vect_prune_runtime_alias_test_list (loop_vec_info);
extern tree vect_check_gather_scatter (gimple *, loop_vec_info, tree *, tree *, extern bool vect_check_gather_scatter (gimple *, loop_vec_info,
int *); gather_scatter_info *);
extern bool vect_analyze_data_refs (vec_info *, int *); extern bool vect_analyze_data_refs (vec_info *, int *);
extern tree vect_create_data_ref_ptr (gimple *, tree, struct loop *, tree, extern tree vect_create_data_ref_ptr (gimple *, tree, struct loop *, tree,
tree *, gimple_stmt_iterator *, tree *, gimple_stmt_iterator *,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment