[8/n] PR85694: Make patterns check for target support

This patch makes pattern recognisers do their own checking for vector
types and target support.  Previously some recognisers did this
themselves and some left it to vect_pattern_recog_1.

Doing this means we can get rid of the type_in argument, which was
ignored if the recogniser did its own checking.  It also means
we create fewer junk statements.

2018-06-20  Richard Sandiford  <richard.sandiford@arm.com>

gcc/
	* tree-vectorizer.h (NUM_PATTERNS, vect_recog_func_ptr): Move to
	tree-vect-patterns.c.
	* tree-vect-patterns.c (vect_supportable_direct_optab_p): New function.
	(vect_recog_dot_prod_pattern): Use it.  Remove the type_in argument.
	(vect_recog_sad_pattern): Likewise.
	(vect_recog_widen_sum_pattern): Likewise.
	(vect_recog_pow_pattern): Likewise.  Check for a null vectype.
	(vect_recog_widen_shift_pattern): Remove the type_in argument.
	(vect_recog_rotate_pattern): Likewise.
	(vect_recog_mult_pattern): Likewise.
	(vect_recog_vector_vector_shift_pattern): Likewise.
	(vect_recog_divmod_pattern): Likewise.
	(vect_recog_mixed_size_cond_pattern): Likewise.
	(vect_recog_bool_pattern): Likewise.
	(vect_recog_mask_conversion_pattern): Likewise.
	(vect_try_gather_scatter_pattern): Likewise.
	(vect_recog_widen_mult_pattern): Likewise.  Check for a null vectype.
	(vect_recog_over_widening_pattern): Likewise.
	(vect_recog_gather_scatter_pattern): Likewise.
	(vect_recog_func_ptr): Move from tree-vectorizer.h
	(vect_vect_recog_func_ptrs): Move further down the file.
	(vect_recog_func): Likewise.  Remove the third argument.
	(NUM_PATTERNS): Define based on vect_vect_recog_func_ptrs.
	(vect_pattern_recog_1): Expect the pattern function to do any
	necessary target tests.  Also expect it to provide a vector type.
	Remove the type_in handling.

From-SVN: r261791

gcc/ChangeLog

 2018-06-20  Richard Sandiford  <richard.sandiford@arm.com>
 
+	* tree-vectorizer.h (NUM_PATTERNS, vect_recog_func_ptr): Move to
+	tree-vect-patterns.c.
+	* tree-vect-patterns.c (vect_supportable_direct_optab_p): New function.
+	(vect_recog_dot_prod_pattern): Use it.  Remove the type_in argument.
+	(vect_recog_sad_pattern): Likewise.
+	(vect_recog_widen_sum_pattern): Likewise.
+	(vect_recog_pow_pattern): Likewise.  Check for a null vectype.
+	(vect_recog_widen_shift_pattern): Remove the type_in argument.
+	(vect_recog_rotate_pattern): Likewise.
+	(vect_recog_mult_pattern): Likewise.
+	(vect_recog_vector_vector_shift_pattern): Likewise.
+	(vect_recog_divmod_pattern): Likewise.
+	(vect_recog_mixed_size_cond_pattern): Likewise.
+	(vect_recog_bool_pattern): Likewise.
+	(vect_recog_mask_conversion_pattern): Likewise.
+	(vect_try_gather_scatter_pattern): Likewise.
+	(vect_recog_widen_mult_pattern): Likewise.  Check for a null vectype.
+	(vect_recog_over_widening_pattern): Likewise.
+	(vect_recog_gather_scatter_pattern): Likewise.
+	(vect_recog_func_ptr): Move from tree-vectorizer.h
+	(vect_vect_recog_func_ptrs): Move further down the file.
+	(vect_recog_func): Likewise.  Remove the third argument.
+	(NUM_PATTERNS): Define based on vect_vect_recog_func_ptrs.
+	(vect_pattern_recog_1): Expect the pattern function to do any
+	necessary target tests.  Also expect it to provide a vector type.
+	Remove the type_in handling.
+
+2018-06-20  Richard Sandiford  <richard.sandiford@arm.com>
+
 	* tree-vect-patterns.c (vect_pattern_detected): New function.
 	(vect_recog_dot_prod_patternm, vect_recog_sad_pattern)
 	(vect_recog_widen_mult_pattern, vect_recog_widen_sum_pattern)

gcc/tree-vect-patterns.c

@@ -47,66 +47,6 @@ along with GCC; see the file COPYING3.  If not see
 #include "omp-simd-clone.h"
 #include "predict.h"
 
-/* Pattern recognition functions  */
-static gimple *vect_recog_widen_sum_pattern (vec<gimple *> *, tree *,
-					    tree *);
-static gimple *vect_recog_widen_mult_pattern (vec<gimple *> *, tree *,
-					     tree *);
-static gimple *vect_recog_dot_prod_pattern (vec<gimple *> *, tree *,
-					   tree *);
-static gimple *vect_recog_sad_pattern (vec<gimple *> *, tree *,
-				      tree *);
-static gimple *vect_recog_pow_pattern (vec<gimple *> *, tree *, tree *);
-static gimple *vect_recog_over_widening_pattern (vec<gimple *> *, tree *,
-                                                 tree *);
-static gimple *vect_recog_widen_shift_pattern (vec<gimple *> *,
-	                                tree *, tree *);
-static gimple *vect_recog_rotate_pattern (vec<gimple *> *, tree *, tree *);
-static gimple *vect_recog_vector_vector_shift_pattern (vec<gimple *> *,
-						      tree *, tree *);
-static gimple *vect_recog_divmod_pattern (vec<gimple *> *,
-					 tree *, tree *);
-
-static gimple *vect_recog_mult_pattern (vec<gimple *> *,
-				       tree *, tree *);
-
-static gimple *vect_recog_mixed_size_cond_pattern (vec<gimple *> *,
-						  tree *, tree *);
-static gimple *vect_recog_bool_pattern (vec<gimple *> *, tree *, tree *);
-static gimple *vect_recog_mask_conversion_pattern (vec<gimple *> *, tree *, tree *);
-static gimple *vect_recog_gather_scatter_pattern (vec<gimple *> *, tree *,
-						  tree *);
-
-struct vect_recog_func
-{
-  vect_recog_func_ptr fn;
-  const char *name;
-};
-
-/* Note that ordering matters - the first pattern matching on a stmt
-   is taken which means usually the more complex one needs to preceed
-   the less comples onex (widen_sum only after dot_prod or sad for example).  */
-static vect_recog_func vect_vect_recog_func_ptrs[NUM_PATTERNS] = {
-      { vect_recog_widen_mult_pattern, "widen_mult" },
-      { vect_recog_dot_prod_pattern, "dot_prod" },
-      { vect_recog_sad_pattern, "sad" },
-      { vect_recog_widen_sum_pattern, "widen_sum" },
-      { vect_recog_pow_pattern, "pow" },
-      { vect_recog_widen_shift_pattern, "widen_shift" },
-      { vect_recog_over_widening_pattern, "over_widening" },
-      { vect_recog_rotate_pattern, "rotate" },
-      { vect_recog_vector_vector_shift_pattern, "vector_vector_shift" },
-      {	vect_recog_divmod_pattern, "divmod" },
-      {	vect_recog_mult_pattern, "mult" },
-      {	vect_recog_mixed_size_cond_pattern, "mixed_size_cond" },
-      {	vect_recog_bool_pattern, "bool" },
-      /* This must come before mask conversion, and includes the parts
-	 of mask conversion that are needed for gather and scatter
-	 internal functions.  */
-      { vect_recog_gather_scatter_pattern, "gather_scatter" },
-      {	vect_recog_mask_conversion_pattern, "mask_conversion" }
-};
-
 /* Report that we've found an instance of pattern PATTERN in
    statement STMT.  */
 
@@ -134,6 +74,49 @@ new_pattern_def_seq (stmt_vec_info stmt_info, gimple *stmt)
   append_pattern_def_seq (stmt_info, stmt);
 }
 
+/* Return true if the target supports a vector version of CODE,
+   where CODE is known to map to a direct optab.  ITYPE specifies
+   the type of (some of) the scalar inputs and OTYPE specifies the
+   type of the scalar result.
+
+   If CODE allows the inputs and outputs to have different type
+   (such as for WIDEN_SUM_EXPR), it is the input mode rather
+   than the output mode that determines the appropriate target pattern.
+   Operand 0 of the target pattern then specifies the mode that the output
+   must have.
+
+   When returning true, set *VECOTYPE_OUT to the vector version of OTYPE.
+   Also set *VECITYPE_OUT to the vector version of ITYPE if VECITYPE_OUT
+   is nonnull.  */
+
+static bool
+vect_supportable_direct_optab_p (tree otype, tree_code code,
+				 tree itype, tree *vecotype_out,
+				 tree *vecitype_out = NULL)
+{
+  tree vecitype = get_vectype_for_scalar_type (itype);
+  if (!vecitype)
+    return false;
+
+  tree vecotype = get_vectype_for_scalar_type (otype);
+  if (!vecotype)
+    return false;
+
+  optab optab = optab_for_tree_code (code, vecitype, optab_default);
+  if (!optab)
+    return false;
+
+  insn_code icode = optab_handler (optab, TYPE_MODE (vecitype));
+  if (icode == CODE_FOR_nothing
+      || insn_data[icode].operand[0].mode != TYPE_MODE (vecotype))
+    return false;
+
+  *vecotype_out = vecotype;
+  if (vecitype_out)
+    *vecitype_out = vecitype;
+  return true;
+}
+
 /* Check whether STMT2 is in the same loop or basic block as STMT1.
    Which of the two applies depends on whether we're currently doing
    loop-based or basic-block-based vectorization, as determined by
@@ -293,8 +276,6 @@ vect_reassociating_reduction_p (stmt_vec_info stmt_vinfo)
 
    Output:
 
-   * TYPE_IN: The type of the input arguments to the pattern.
-
    * TYPE_OUT: The type of the output  of this pattern.
 
    * Return value: A new stmt that will be used to replace the sequence of
@@ -310,8 +291,7 @@ vect_reassociating_reduction_p (stmt_vec_info stmt_vinfo)
          inner-loop nested in an outer-loop that us being vectorized).  */
 
 static gimple *
-vect_recog_dot_prod_pattern (vec<gimple *> *stmts, tree *type_in,
-			     tree *type_out)
+vect_recog_dot_prod_pattern (vec<gimple *> *stmts, tree *type_out)
 {
   gimple *stmt, *last_stmt = (*stmts)[0];
   tree oprnd0, oprnd1;
@@ -448,8 +428,9 @@ vect_recog_dot_prod_pattern (vec<gimple *> *stmts, tree *type_in,
   vect_pattern_detected ("vect_recog_dot_prod_pattern", last_stmt);
 
   half_type = TREE_TYPE (oprnd00);
-  *type_in = half_type;
-  *type_out = type;
+  if (!vect_supportable_direct_optab_p (type, DOT_PROD_EXPR, half_type,
+					type_out))
+    return NULL;
 
   var = vect_recog_temp_ssa_var (type, NULL);
   pattern_stmt = gimple_build_assign (var, DOT_PROD_EXPR,
@@ -489,8 +470,6 @@ vect_recog_dot_prod_pattern (vec<gimple *> *stmts, tree *type_in,
 
    Output:
 
-   * TYPE_IN: The type of the input arguments to the pattern.
-
    * TYPE_OUT: The type of the output of this pattern.
 
    * Return value: A new stmt that will be used to replace the sequence of
@@ -499,8 +478,7 @@ vect_recog_dot_prod_pattern (vec<gimple *> *stmts, tree *type_in,
   */
 
 static gimple *
-vect_recog_sad_pattern (vec<gimple *> *stmts, tree *type_in,
-			     tree *type_out)
+vect_recog_sad_pattern (vec<gimple *> *stmts, tree *type_out)
 {
   gimple *last_stmt = (*stmts)[0];
   tree sad_oprnd0, sad_oprnd1;
@@ -641,8 +619,9 @@ vect_recog_sad_pattern (vec<gimple *> *stmts, tree *type_in,
 
   vect_pattern_detected ("vect_recog_sad_pattern", last_stmt);
 
-  *type_in = TREE_TYPE (sad_oprnd0);
-  *type_out = sum_type;
+  if (!vect_supportable_direct_optab_p (sum_type, SAD_EXPR,
+					TREE_TYPE (sad_oprnd0), type_out))
+    return NULL;
 
   tree var = vect_recog_temp_ssa_var (sum_type, NULL);
   gimple *pattern_stmt = gimple_build_assign (var, SAD_EXPR, sad_oprnd0,
@@ -778,8 +757,6 @@ vect_handle_widen_op_by_const (gimple *stmt, enum tree_code code,
 
    Output:
 
-   * TYPE_IN: The type of the input arguments to the pattern.
-
    * TYPE_OUT: The type of the output of this pattern.
 
    * Return value: A new stmt that will be used to replace the sequence of
@@ -790,8 +767,7 @@ vect_handle_widen_op_by_const (gimple *stmt, enum tree_code code,
 */
 
 static gimple *
-vect_recog_widen_mult_pattern (vec<gimple *> *stmts,
-                               tree *type_in, tree *type_out)
+vect_recog_widen_mult_pattern (vec<gimple *> *stmts, tree *type_out)
 {
   gimple *last_stmt = stmts->pop ();
   gimple *def_stmt0, *def_stmt1;
@@ -933,8 +909,9 @@ vect_recog_widen_mult_pattern (vec<gimple *> *stmts,
 					  &dummy_int, &dummy_vec))
     return NULL;
 
-  *type_in = vectype;
   *type_out = get_vectype_for_scalar_type (type);
+  if (!*type_out)
+    return NULL;
 
   /* Pattern supported. Create a stmt to be used to replace the pattern: */
   var = vect_recog_temp_ssa_var (itype, NULL);
@@ -989,8 +966,6 @@ vect_recog_widen_mult_pattern (vec<gimple *> *stmts,
 
    Output:
 
-   * TYPE_IN: The type of the input arguments to the pattern.
-
    * TYPE_OUT: The type of the output of this pattern.
 
    * Return value: A new stmt that will be used to replace the sequence of
@@ -1001,8 +976,7 @@ vect_recog_widen_mult_pattern (vec<gimple *> *stmts,
 */
 
 static gimple *
-vect_recog_pow_pattern (vec<gimple *> *stmts, tree *type_in,
-			tree *type_out)
+vect_recog_pow_pattern (vec<gimple *> *stmts, tree *type_out)
 {
   gimple *last_stmt = (*stmts)[0];
   tree base, exp;
@@ -1072,12 +1046,13 @@ vect_recog_pow_pattern (vec<gimple *> *stmts, tree *type_in,
 		  if (node->simd_clones == NULL)
 		    return NULL;
 		}
+	      *type_out = get_vectype_for_scalar_type (TREE_TYPE (base));
+	      if (!*type_out)
+		return NULL;
 	      stmt_vec_info stmt_vinfo = vinfo_for_stmt (last_stmt);
 	      tree def = vect_recog_temp_ssa_var (TREE_TYPE (base), NULL);
 	      gimple *g = gimple_build_assign (def, MULT_EXPR, exp, logc);
 	      new_pattern_def_seq (stmt_vinfo, g);
-	      *type_in = TREE_TYPE (base);
-	      *type_out = NULL_TREE;
 	      tree res = vect_recog_temp_ssa_var (TREE_TYPE (base), NULL);
 	      g = gimple_build_call (exp_decl, 1, def);
 	      gimple_call_set_lhs (g, res);
@@ -1091,15 +1066,15 @@ vect_recog_pow_pattern (vec<gimple *> *stmts, tree *type_in,
   /* We now have a pow or powi builtin function call with a constant
      exponent.  */
 
-  *type_out = NULL_TREE;
-
   /* Catch squaring.  */
   if ((tree_fits_shwi_p (exp)
        && tree_to_shwi (exp) == 2)
       || (TREE_CODE (exp) == REAL_CST
           && real_equal (&TREE_REAL_CST (exp), &dconst2)))
     {
-      *type_in = TREE_TYPE (base);
+      if (!vect_supportable_direct_optab_p (TREE_TYPE (base), MULT_EXPR,
+					    TREE_TYPE (base), type_out))
+	return NULL;
 
       var = vect_recog_temp_ssa_var (TREE_TYPE (base), NULL);
       stmt = gimple_build_assign (var, MULT_EXPR, base, base);
@@ -1110,9 +1085,9 @@ vect_recog_pow_pattern (vec<gimple *> *stmts, tree *type_in,
   if (TREE_CODE (exp) == REAL_CST
       && real_equal (&TREE_REAL_CST (exp), &dconsthalf))
     {
-      *type_in = get_vectype_for_scalar_type (TREE_TYPE (base));
-      if (*type_in
-	  && direct_internal_fn_supported_p (IFN_SQRT, *type_in,
+      *type_out = get_vectype_for_scalar_type (TREE_TYPE (base));
+      if (*type_out
+	  && direct_internal_fn_supported_p (IFN_SQRT, *type_out,
 					     OPTIMIZE_FOR_SPEED))
 	{
 	  gcall *stmt = gimple_build_call_internal (IFN_SQRT, 1, base);
@@ -1150,8 +1125,6 @@ vect_recog_pow_pattern (vec<gimple *> *stmts, tree *type_in,
 
    Output:
 
-   * TYPE_IN: The type of the input arguments to the pattern.
-
    * TYPE_OUT: The type of the output of this pattern.
 
    * Return value: A new stmt that will be used to replace the sequence of
@@ -1167,8 +1140,7 @@ vect_recog_pow_pattern (vec<gimple *> *stmts, tree *type_in,
 	 inner-loop nested in an outer-loop that us being vectorized).  */
 
 static gimple *
-vect_recog_widen_sum_pattern (vec<gimple *> *stmts, tree *type_in,
-			      tree *type_out)
+vect_recog_widen_sum_pattern (vec<gimple *> *stmts, tree *type_out)
 {
   gimple *stmt, *last_stmt = (*stmts)[0];
   tree oprnd0, oprnd1;
@@ -1223,14 +1195,15 @@ vect_recog_widen_sum_pattern (vec<gimple *> *stmts, tree *type_in,
   if (!type_conversion_p (oprnd0, last_stmt, true, &half_type, &stmt,
                           &promotion)
       || !promotion)
-     return NULL;
+    return NULL;
 
   oprnd0 = gimple_assign_rhs1 (stmt);
 
   vect_pattern_detected ("vect_recog_widen_sum_pattern", last_stmt);
 
-  *type_in = half_type;
-  *type_out = type;
+  if (!vect_supportable_direct_optab_p (type, WIDEN_SUM_EXPR, half_type,
+					type_out))
+    return NULL;
 
   var = vect_recog_temp_ssa_var (type, NULL);
   pattern_stmt = gimple_build_assign (var, WIDEN_SUM_EXPR, oprnd0, oprnd1);
@@ -1473,8 +1446,7 @@ vect_operation_fits_smaller_type (gimple *stmt, tree def, tree *new_type,
    demotion operation.  We also check that S3 and S4 have only one use.  */
 
 static gimple *
-vect_recog_over_widening_pattern (vec<gimple *> *stmts,
-                                  tree *type_in, tree *type_out)
+vect_recog_over_widening_pattern (vec<gimple *> *stmts, tree *type_out)
 {
   gimple *stmt = stmts->pop ();
   gimple *pattern_stmt = NULL, *new_def_stmt, *prev_stmt = NULL,
@@ -1562,14 +1534,15 @@ vect_recog_over_widening_pattern (vec<gimple *> *stmts,
       if (TYPE_UNSIGNED (new_type) != TYPE_UNSIGNED (use_type)
           || TYPE_PRECISION (new_type) != TYPE_PRECISION (use_type))
         {
+	  *type_out = get_vectype_for_scalar_type (use_type);
+	  if (!*type_out)
+	    return NULL;
+
           /* Create NEW_TYPE->USE_TYPE conversion.  */
 	  new_oprnd = make_ssa_name (use_type);
 	  pattern_stmt = gimple_build_assign (new_oprnd, NOP_EXPR, var);
           STMT_VINFO_RELATED_STMT (vinfo_for_stmt (use_stmt)) = pattern_stmt;
 
-          *type_in = get_vectype_for_scalar_type (new_type);
-          *type_out = get_vectype_for_scalar_type (use_type);
-
           /* We created a pattern statement for the last statement in the
              sequence, so we don't need to associate it with the pattern
              statement created for PREV_STMT.  Therefore, we add PREV_STMT
@@ -1583,8 +1556,7 @@ vect_recog_over_widening_pattern (vec<gimple *> *stmts,
 	    STMT_VINFO_PATTERN_DEF_SEQ (vinfo_for_stmt (use_stmt))
 	       = STMT_VINFO_PATTERN_DEF_SEQ (vinfo_for_stmt (prev_stmt));
 
-          *type_in = vectype;
-          *type_out = NULL_TREE;
+	  *type_out = vectype;
         }
 
       stmts->safe_push (use_stmt);
@@ -1648,8 +1620,6 @@ vect_recog_over_widening_pattern (vec<gimple *> *stmts,
 
   Output:
 
-  * TYPE_IN: The type of the input arguments to the pattern.
-
   * TYPE_OUT: The type of the output of this pattern.
 
   * Return value: A new stmt that will be used to replace the sequence of
@@ -1657,8 +1627,7 @@ vect_recog_over_widening_pattern (vec<gimple *> *stmts,
     WIDEN_LSHIFT_EXPR <a_t, CONST>.  */
 
 static gimple *
-vect_recog_widen_shift_pattern (vec<gimple *> *stmts,
-				tree *type_in, tree *type_out)
+vect_recog_widen_shift_pattern (vec<gimple *> *stmts, tree *type_out)
 {
   gimple *last_stmt = stmts->pop ();
   gimple *def_stmt0;
@@ -1740,7 +1709,6 @@ vect_recog_widen_shift_pattern (vec<gimple *> *stmts,
 					  &dummy_int, &dummy_vec))
     return NULL;
 
-  *type_in = vectype;
   *type_out = vectype_out;
 
   /* Pattern supported.  Create a stmt to be used to replace the pattern.  */
@@ -1783,15 +1751,13 @@ vect_recog_widen_shift_pattern (vec<gimple *> *stmts,
 
   Output:
 
-  * TYPE_IN: The type of the input arguments to the pattern.
-
   * TYPE_OUT: The type of the output of this pattern.
 
   * Return value: A new stmt that will be used to replace the rotate
     S0 stmt.  */
 
 static gimple *
-vect_recog_rotate_pattern (vec<gimple *> *stmts, tree *type_in, tree *type_out)
+vect_recog_rotate_pattern (vec<gimple *> *stmts, tree *type_out)
 {
   gimple *last_stmt = stmts->pop ();
   tree oprnd0, oprnd1, lhs, var, var1, var2, vectype, type, stype, def, def2;
@@ -1876,10 +1842,7 @@ vect_recog_rotate_pattern (vec<gimple *> *stmts, tree *type_in, tree *type_out)
 	return NULL;
     }
 
-  *type_in = vectype;
   *type_out = vectype;
-  if (*type_in == NULL_TREE)
-    return NULL;
 
   if (dt == vect_external_def
       && TREE_CODE (oprnd1) == SSA_NAME
@@ -2034,16 +1997,13 @@ vect_recog_rotate_pattern (vec<gimple *> *stmts, tree *type_in, tree *type_out)
 
   Output:
 
-  * TYPE_IN: The type of the input arguments to the pattern.
-
   * TYPE_OUT: The type of the output of this pattern.
 
   * Return value: A new stmt that will be used to replace the shift/rotate
     S3 stmt.  */
 
 static gimple *
-vect_recog_vector_vector_shift_pattern (vec<gimple *> *stmts,
-					tree *type_in, tree *type_out)
+vect_recog_vector_vector_shift_pattern (vec<gimple *> *stmts, tree *type_out)
 {
   gimple *last_stmt = stmts->pop ();
   tree oprnd0, oprnd1, lhs, var;
@@ -2085,9 +2045,8 @@ vect_recog_vector_vector_shift_pattern (vec<gimple *> *stmts,
   if (!def_vinfo)
     return NULL;
 
-  *type_in = get_vectype_for_scalar_type (TREE_TYPE (oprnd0));
-  *type_out = *type_in;
-  if (*type_in == NULL_TREE)
+  *type_out = get_vectype_for_scalar_type (TREE_TYPE (oprnd0));
+  if (*type_out == NULL_TREE)
     return NULL;
 
   tree def = NULL_TREE;
@@ -2449,16 +2408,13 @@ vect_synth_mult_by_constant (tree op, tree val,
 
  Output:
 
-  * TYPE_IN: The type of the input arguments to the pattern.
-
   * TYPE_OUT: The type of the output of this pattern.
 
   * Return value: A new stmt that will be used to replace
     the multiplication.  */
 
 static gimple *
-vect_recog_mult_pattern (vec<gimple *> *stmts,
-			 tree *type_in, tree *type_out)
+vect_recog_mult_pattern (vec<gimple *> *stmts, tree *type_out)
 {
   gimple *last_stmt = stmts->pop ();
   tree oprnd0, oprnd1, vectype, itype;
@@ -2504,7 +2460,6 @@ vect_recog_mult_pattern (vec<gimple *> *stmts,
   vect_pattern_detected ("vect_recog_mult_pattern", last_stmt);
 
   stmts->safe_push (last_stmt);
-  *type_in = vectype;
   *type_out = vectype;
 
   return pattern_stmt;
@@ -2543,16 +2498,13 @@ vect_recog_mult_pattern (vec<gimple *> *stmts,
 
   Output:
 
-  * TYPE_IN: The type of the input arguments to the pattern.
-
   * TYPE_OUT: The type of the output of this pattern.
 
   * Return value: A new stmt that will be used to replace the division
     S1 or modulo S4 stmt.  */
 
 static gimple *
-vect_recog_divmod_pattern (vec<gimple *> *stmts,
-			   tree *type_in, tree *type_out)
+vect_recog_divmod_pattern (vec<gimple *> *stmts, tree *type_out)
 {
   gimple *last_stmt = stmts->pop ();
   tree oprnd0, oprnd1, vectype, itype, cond;
@@ -2703,7 +2655,6 @@ vect_recog_divmod_pattern (vec<gimple *> *stmts,
 
       stmts->safe_push (last_stmt);
 
-      *type_in = vectype;
       *type_out = vectype;
       return pattern_stmt;
     }
@@ -2955,7 +2906,6 @@ vect_recog_divmod_pattern (vec<gimple *> *stmts,
 
   stmts->safe_push (last_stmt);
 
-  *type_in = vectype;
   *type_out = vectype;
   return pattern_stmt;
 }
@@ -2980,8 +2930,6 @@ vect_recog_divmod_pattern (vec<gimple *> *stmts,
 
    Output:
 
-   * TYPE_IN: The type of the input arguments to the pattern.
-
    * TYPE_OUT: The type of the output of this pattern.
 
    * Return value: A new stmt that will be used to replace the pattern.
@@ -2991,8 +2939,7 @@ vect_recog_divmod_pattern (vec<gimple *> *stmts,
 	a_T = (TYPE) a_it;  */
 
 static gimple *
-vect_recog_mixed_size_cond_pattern (vec<gimple *> *stmts, tree *type_in,
-				    tree *type_out)
+vect_recog_mixed_size_cond_pattern (vec<gimple *> *stmts, tree *type_out)
 {
   gimple *last_stmt = (*stmts)[0];
   tree cond_expr, then_clause, else_clause;
@@ -3108,7 +3055,6 @@ vect_recog_mixed_size_cond_pattern (vec<gimple *> *stmts, tree *type_in,
   def_stmt_info = new_stmt_vec_info (def_stmt, vinfo);
   set_vinfo_for_stmt (def_stmt, def_stmt_info);
   STMT_VINFO_VECTYPE (def_stmt_info) = vecitype;
-  *type_in = vecitype;
   *type_out = vectype;
 
   vect_pattern_detected ("vect_recog_mixed_size_cond_pattern", last_stmt);
@@ -3581,8 +3527,6 @@ search_type_for_mask (tree var, vec_info *vinfo)
 
    Output:
 
-   * TYPE_IN: The type of the input arguments to the pattern.
-
    * TYPE_OUT: The type of the output of this pattern.
 
    * Return value: A new stmt that will be used to replace the pattern.
@@ -3602,8 +3546,7 @@ search_type_for_mask (tree var, vec_info *vinfo)
 	but the above is more efficient.  */
 
 static gimple *
-vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_in,
-			 tree *type_out)
+vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_out)
 {
   gimple *last_stmt = stmts->pop ();
   enum tree_code rhs_code;
@@ -3679,7 +3622,6 @@ vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_in,
 	}
 
       *type_out = vectype;
-      *type_in = vectype;
       stmts->safe_push (last_stmt);
       vect_pattern_detected ("vect_recog_bool_pattern", last_stmt);
 
@@ -3718,7 +3660,6 @@ vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_in,
 				 gimple_assign_rhs2 (last_stmt),
 				 gimple_assign_rhs3 (last_stmt));
       *type_out = vectype;
-      *type_in = vectype;
       stmts->safe_push (last_stmt);
       vect_pattern_detected ("vect_recog_bool_pattern", last_stmt);
 
@@ -3775,7 +3716,6 @@ vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_in,
       STMT_VINFO_DR_WRT_VEC_LOOP (pattern_stmt_info)
 	= STMT_VINFO_DR_WRT_VEC_LOOP (stmt_vinfo);
       *type_out = vectype;
-      *type_in = vectype;
       stmts->safe_push (last_stmt);
       vect_pattern_detected ("vect_recog_bool_pattern", last_stmt);
 
@@ -3839,8 +3779,7 @@ build_mask_conversion (tree mask, tree vectype, stmt_vec_info stmt_vinfo,
    S4'  c_1' = m_3'' ? c_2 : c_3;  */
 
 static gimple *
-vect_recog_mask_conversion_pattern (vec<gimple *> *stmts, tree *type_in,
-				    tree *type_out)
+vect_recog_mask_conversion_pattern (vec<gimple *> *stmts, tree *type_out)
 {
   gimple *last_stmt = stmts->pop ();
   enum tree_code rhs_code;
@@ -3911,7 +3850,6 @@ vect_recog_mask_conversion_pattern (vec<gimple *> *stmts, tree *type_in,
 	= STMT_VINFO_DR_WRT_VEC_LOOP (stmt_vinfo);
 
       *type_out = vectype1;
-      *type_in = vectype1;
       stmts->safe_push (last_stmt);
       vect_pattern_detected ("vect_recog_mask_conversion_pattern", last_stmt);
 
@@ -4035,7 +3973,6 @@ vect_recog_mask_conversion_pattern (vec<gimple *> *stmts, tree *type_in,
 					  gimple_assign_rhs3 (last_stmt));
 
       *type_out = vectype1;
-      *type_in = vectype1;
       stmts->safe_push (last_stmt);
       vect_pattern_detected ("vect_recog_mask_conversion_pattern", last_stmt);
 
@@ -4081,7 +4018,6 @@ vect_recog_mask_conversion_pattern (vec<gimple *> *stmts, tree *type_in,
   pattern_stmt = gimple_build_assign (lhs, rhs_code, rhs1, rhs2);
 
   *type_out = vectype1;
-  *type_in = vectype1;
   stmts->safe_push (last_stmt);
   vect_pattern_detected ("vect_recog_mask_conversion_pattern", last_stmt);
 
@@ -4180,14 +4116,14 @@ vect_add_conversion_to_patterm (tree type, tree value,
 
 /* Try to convert STMT into a call to a gather load or scatter store
    internal function.  Return the final statement on success and set
-   *TYPE_IN and *TYPE_OUT to the vector type being loaded or stored.
+   *TYPE_OUT to the vector type being loaded or stored.
 
    This function only handles gathers and scatters that were recognized
    as such from the outset (indicated by STMT_VINFO_GATHER_SCATTER_P).  */
 
 static gimple *
 vect_try_gather_scatter_pattern (gimple *stmt, stmt_vec_info last_stmt_info,
-				 tree *type_in, tree *type_out)
+				 tree *type_out)
 {
   /* Currently we only support this for loop vectorization.  */
   stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
@@ -4264,7 +4200,6 @@ vect_try_gather_scatter_pattern (gimple *stmt, stmt_vec_info last_stmt_info,
 
   tree vectype = STMT_VINFO_VECTYPE (stmt_info);
   *type_out = vectype;
-  *type_in = vectype;
   vect_pattern_detected ("gather/scatter pattern", stmt);
 
   return pattern_stmt;
@@ -4273,19 +4208,52 @@ vect_try_gather_scatter_pattern (gimple *stmt, stmt_vec_info last_stmt_info,
 /* Pattern wrapper around vect_try_gather_scatter_pattern.  */
 
 static gimple *
-vect_recog_gather_scatter_pattern (vec<gimple *> *stmts, tree *type_in,
-				   tree *type_out)
+vect_recog_gather_scatter_pattern (vec<gimple *> *stmts, tree *type_out)
 {
   gimple *last_stmt = stmts->pop ();
   stmt_vec_info last_stmt_info = vinfo_for_stmt (last_stmt);
   gimple *pattern_stmt = vect_try_gather_scatter_pattern (last_stmt,
 							  last_stmt_info,
-							  type_in, type_out);
+							  type_out);
   if (pattern_stmt)
     stmts->safe_push (last_stmt);
   return pattern_stmt;
 }
 
+typedef gimple *(*vect_recog_func_ptr) (vec<gimple *> *, tree *);
+
+struct vect_recog_func
+{
+  vect_recog_func_ptr fn;
+  const char *name;
+};
+
+/* Note that ordering matters - the first pattern matching on a stmt is
+   taken which means usually the more complex one needs to preceed the
+   less comples onex (widen_sum only after dot_prod or sad for example).  */
+static vect_recog_func vect_vect_recog_func_ptrs[] = {
+  { vect_recog_widen_mult_pattern, "widen_mult" },
+  { vect_recog_dot_prod_pattern, "dot_prod" },
+  { vect_recog_sad_pattern, "sad" },
+  { vect_recog_widen_sum_pattern, "widen_sum" },
+  { vect_recog_pow_pattern, "pow" },
+  { vect_recog_widen_shift_pattern, "widen_shift" },
+  { vect_recog_over_widening_pattern, "over_widening" },
+  { vect_recog_rotate_pattern, "rotate" },
+  { vect_recog_vector_vector_shift_pattern, "vector_vector_shift" },
+  { vect_recog_divmod_pattern, "divmod" },
+  { vect_recog_mult_pattern, "mult" },
+  { vect_recog_mixed_size_cond_pattern, "mixed_size_cond" },
+  { vect_recog_bool_pattern, "bool" },
+  /* This must come before mask conversion, and includes the parts
+     of mask conversion that are needed for gather and scatter
+     internal functions.  */
+  { vect_recog_gather_scatter_pattern, "gather_scatter" },
+  { vect_recog_mask_conversion_pattern, "mask_conversion" }
+};
+
+const unsigned int NUM_PATTERNS = ARRAY_SIZE (vect_vect_recog_func_ptrs);
+
 /* Mark statements that are involved in a pattern.  */
 
 static inline void
@@ -4337,17 +4305,13 @@ vect_mark_pattern_stmts (gimple *orig_stmt, gimple *pattern_stmt,
         computation pattern.
    STMT: A stmt from which the pattern search should start.
 
-   If PATTERN_RECOG_FUNC successfully detected the pattern, it creates an
-   expression that computes the same functionality and can be used to
-   replace the sequence of stmts that are involved in the pattern.
-
-   Output:
-   This function checks if the expression returned by PATTERN_RECOG_FUNC is
-   supported in vector form by the target.  We use 'TYPE_IN' to obtain the
-   relevant vector type. If 'TYPE_IN' is already a vector type, then this
-   indicates that target support had already been checked by PATTERN_RECOG_FUNC.
-   If 'TYPE_OUT' is also returned by PATTERN_RECOG_FUNC, we check that it fits
-   to the available target pattern.
+   If PATTERN_RECOG_FUNC successfully detected the pattern, it creates
+   a sequence of statements that has the same functionality and can be
+   used to replace STMT.  It returns the last statement in the sequence
+   and adds any earlier statements to STMT's STMT_VINFO_PATTERN_DEF_SEQ.
+   PATTERN_RECOG_FUNC also sets *TYPE_OUT to the vector type of the final
+   statement, having first checked that the target supports the new operation
+   in that type.
 
    This function also does some bookkeeping, as explained in the documentation
    for vect_recog_pattern.  */
@@ -4361,13 +4325,11 @@ vect_pattern_recog_1 (vect_recog_func *recog_func,
   stmt_vec_info stmt_info;
   loop_vec_info loop_vinfo;
   tree pattern_vectype;
-  tree type_in, type_out;
-  enum tree_code code;
   int i;
 
   stmts_to_replace->truncate (0);
   stmts_to_replace->quick_push (stmt);
-  pattern_stmt = recog_func->fn (stmts_to_replace, &type_in, &type_out);
+  pattern_stmt = recog_func->fn (stmts_to_replace, &pattern_vectype);
   if (!pattern_stmt)
     {
       /* Clear related stmt info that analysis might have noted for
@@ -4385,43 +4347,8 @@ vect_pattern_recog_1 (vect_recog_func *recog_func,
   stmt = stmts_to_replace->last ();
   stmt_info = vinfo_for_stmt (stmt);
   loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
+  gcc_assert (pattern_vectype);
  
-  if (VECTOR_BOOLEAN_TYPE_P (type_in)
-      || VECTOR_TYPE_P (type_in))
-    {
-      /* No need to check target support (already checked by the pattern
-         recognition function).  */
-      pattern_vectype = type_out ? type_out : type_in;
-    }
-  else
-    {
-      /* Check target support  */
-      type_in = get_vectype_for_scalar_type (type_in);
-      if (!type_in)
-	return false;
-      if (type_out)
-	type_out = get_vectype_for_scalar_type (type_out);
-      else
-	type_out = type_in;
-      if (!type_out)
-	return false;
-      pattern_vectype = type_out;
-
-      if (is_gimple_assign (pattern_stmt))
-	{
-	  enum insn_code icode;
-	  code = gimple_assign_rhs_code (pattern_stmt);
-	  optab optab = optab_for_tree_code (code, type_in, optab_default);
-	  machine_mode vec_mode = TYPE_MODE (type_in);
-	  if (!optab
-	      || (icode = optab_handler (optab, vec_mode)) == CODE_FOR_nothing
-	      || (insn_data[icode].operand[0].mode != TYPE_MODE (type_out)))
-	    return false;
-	}
-      else
-	gcc_assert (is_gimple_call (pattern_stmt));
-    }
-
   /* Found a vectorizable pattern.  */
   if (dump_enabled_p ())
     {

gcc/tree-vectorizer.h

@@ -1628,8 +1628,6 @@ extern int vect_get_place_in_interleaving_chain (gimple *, gimple *);
 /* Pattern recognition functions.
    Additional pattern recognition functions can (and will) be added
    in the future.  */
-typedef gimple *(* vect_recog_func_ptr) (vec<gimple *> *, tree *, tree *);
-#define NUM_PATTERNS 15
 void vect_pattern_recog (vec_info *);
 
 /* In tree-vectorizer.c.  */