cgraph.c (cgraph_clone_edge): New UPDATE_ORIGINAL argument.

	* cgraph.c (cgraph_clone_edge): New UPDATE_ORIGINAL argument.
	(cgraph_clone_node): Likewise.
	* cgraph.h (cgraph_clone_edge): Update prototype.
	(cgraph_clone_node): Likewise.
	* ipa-inline.c (cgraph_clone_inlined_nodes): Update call of
	cgraph_clone_node.
	(lookup_recursive_calls): Consider profile.
	(cgraph_decide_recursive_inlining): Fix updating; use new
	probability argument; use profile.
	* params.def (PARAM_MIN_INLINE_RECURSIVE_PROBABILITY): New.
	* tree-inline.c (copy_bb): Update clal of clone_edge.
	* tree-optimize.c (tree_rest_of_compilation): UPdate cal of clone_node.

	* invoke.texi (min-inline-recursive-probability): Document.

From-SVN: r102521

gcc/ChangeLog

+2005-07-28  Jan Hubicka  <jh@suse.cz>
+
+	* cgraph.c (cgraph_clone_edge): New UPDATE_ORIGINAL argument.
+	(cgraph_clone_node): Likewise.
+	* cgraph.h (cgraph_clone_edge): Update prototype.
+	(cgraph_clone_node): Likewise.
+	* ipa-inline.c (cgraph_clone_inlined_nodes): Update call of
+	cgraph_clone_node.
+	(lookup_recursive_calls): Consider profile.
+	(cgraph_decide_recursive_inlining): Fix updating; use new
+	probability argument; use profile.
+	* params.def (PARAM_MIN_INLINE_RECURSIVE_PROBABILITY): New.
+	* tree-inline.c (copy_bb): Update clal of clone_edge.
+	* tree-optimize.c (tree_rest_of_compilation): UPdate cal of clone_node.
+
+	* invoke.texi (min-inline-recursive-probability): Document.
+
 2005-07-28  Gerald Pfeifer  <gerald@pfeifer.com>
 
 	* doc/install.texi (Configuration): Update Valgrind homepage.

gcc/cgraph.c

@@ -884,7 +884,8 @@ cgraph_function_possibly_inlined_p (tree decl)
 /* Create clone of E in the node N represented by CALL_EXPR the callgraph.  */
 struct cgraph_edge *
 cgraph_clone_edge (struct cgraph_edge *e, struct cgraph_node *n,
-		   tree call_stmt, int count_scale, int loop_nest)
+		   tree call_stmt, int count_scale, int loop_nest,
+		   bool update_original)
 {
   struct cgraph_edge *new;
 
@@ -893,14 +894,20 @@ cgraph_clone_edge (struct cgraph_edge *e, struct cgraph_node *n,
 		            e->loop_nest + loop_nest);
 
   new->inline_failed = e->inline_failed;
+  if (update_original)
     e->count -= new->count;
   return new;
 }
 
 /* Create node representing clone of N executed COUNT times.  Decrease
-   the execution counts from original node too.  */
+   the execution counts from original node too. 
+
+   When UPDATE_ORIGINAL is true, the counts are subtracted from the original
+   function's profile to reflect the fact that part of execution is handled
+   by node.  */
 struct cgraph_node *
-cgraph_clone_node (struct cgraph_node *n, gcov_type count, int loop_nest)
+cgraph_clone_node (struct cgraph_node *n, gcov_type count, int loop_nest,
+		   bool update_original)
 {
   struct cgraph_node *new = cgraph_create_node ();
   struct cgraph_edge *e;
@@ -923,10 +930,12 @@ cgraph_clone_node (struct cgraph_node *n, gcov_type count, int loop_nest)
     count_scale = new->count * REG_BR_PROB_BASE / n->count;
   else
     count_scale = 0;
+  if (update_original)
     n->count -= count;
 
   for (e = n->callees;e; e=e->next_callee)
-    cgraph_clone_edge (e, new, e->call_stmt, count_scale, loop_nest);
+    cgraph_clone_edge (e, new, e->call_stmt, count_scale, loop_nest,
+		       update_original);
 
   new->next_clone = n->next_clone;
   new->prev_clone = n;

gcc/cgraph.h

@@ -240,8 +240,11 @@ struct cgraph_local_info *cgraph_local_info (tree);
 struct cgraph_global_info *cgraph_global_info (tree);
 struct cgraph_rtl_info *cgraph_rtl_info (tree);
 const char * cgraph_node_name (struct cgraph_node *);
-struct cgraph_edge * cgraph_clone_edge (struct cgraph_edge *, struct cgraph_node *, tree, int, int);
-struct cgraph_node * cgraph_clone_node (struct cgraph_node *, gcov_type, int);
+struct cgraph_edge * cgraph_clone_edge (struct cgraph_edge *,
+				        struct cgraph_node *,
+				        tree, int, int, bool);
+struct cgraph_node * cgraph_clone_node (struct cgraph_node *, gcov_type,
+					int, bool);
 
 struct cgraph_varpool_node *cgraph_varpool_node (tree);
 struct cgraph_varpool_node *cgraph_varpool_node_for_asm (tree asmname);

gcc/doc/invoke.texi

@@ -5790,6 +5790,18 @@ happens only when @option{-finline-functions} (included in @option{-O3}) is
 enabled and @option{--param max-inline-recursive-depth-auto} is used.  The
 default value is 450.
 
+@item min-inline-recursive-probability
+Recursive inlining is profitable only for function having deep recursion
+in average and can hurt for function having little recursion depth by
+increasing the prologue size or complexity of function body to other
+optimizers.
+
+When profile feedback is available (see @option{-fprofile-generate}) the actual
+recursion depth can be guessed from probability that function will recurse via
+given call expression.  This parameter limits inlining only to call expression
+whose probability exceeds given threshold (in percents).  The default value is
+10.
+
 @item inline-call-cost
 Specify cost of call instruction relative to simple arithmetics operations
 (having cost of 1).  Increasing this cost disqualifies inlining of non-leaf

gcc/ipa-inline.c

@@ -131,7 +131,7 @@ cgraph_clone_inlined_nodes (struct cgraph_edge *e, bool duplicate)
     }
    else if (duplicate)
     {
-      n = cgraph_clone_node (e->callee, e->count, e->loop_nest);
+      n = cgraph_clone_node (e->callee, e->count, e->loop_nest, true);
       cgraph_redirect_edge_callee (e, n);
     }
 
@@ -456,10 +456,13 @@ lookup_recursive_calls (struct cgraph_node *node, struct cgraph_node *where,
   for (e = where->callees; e; e = e->next_callee)
     if (e->callee == node)
       {
-	/* FIXME: Once counts and frequencies are available we should drive the
-	   order by these.  For now force the order to be simple queue since
-	   we get order dependent on recursion depth for free by this.  */
-        fibheap_insert (heap, priority++, e);
+	/* When profile feedback is available, prioritize by expected number
+	   of calls.  Without profile feedback we maintain simple queue
+	   to order candidates via recursive depths.  */
+        fibheap_insert (heap,
+			!max_count ? priority++
+		        : -(e->count / ((max_count + (1<<24) - 1) / (1<<24))),
+		        e);
       }
   for (e = where->callees; e; e = e->next_callee)
     if (!e->inline_failed)
@@ -533,6 +536,7 @@ cgraph_decide_recursive_inlining (struct cgraph_node *node)
 {
   int limit = PARAM_VALUE (PARAM_MAX_INLINE_INSNS_RECURSIVE_AUTO);
   int max_depth = PARAM_VALUE (PARAM_MAX_INLINE_RECURSIVE_DEPTH_AUTO);
+  int probability = PARAM_VALUE (PARAM_MIN_INLINE_RECURSIVE_PROBABILITY);
   fibheap_t heap;
   struct cgraph_edge *e;
   struct cgraph_node *master_clone;
@@ -563,7 +567,7 @@ cgraph_decide_recursive_inlining (struct cgraph_node *node)
 	     cgraph_node_name (node));
 
   /* We need original clone to copy around.  */
-  master_clone = cgraph_clone_node (node, 0, 1);
+  master_clone = cgraph_clone_node (node, node->count, 1, false);
   master_clone->needed = true;
   for (e = master_clone->callees; e; e = e->next_callee)
     if (!e->inline_failed)
@@ -571,27 +575,60 @@ cgraph_decide_recursive_inlining (struct cgraph_node *node)
 
   /* Do the inlining and update list of recursive call during process.  */
   while (!fibheap_empty (heap)
-	 && cgraph_estimate_size_after_inlining (1, node, master_clone) <= limit)
+	 && (cgraph_estimate_size_after_inlining (1, node, master_clone)
+	     <= limit))
     {
       struct cgraph_edge *curr = fibheap_extract_min (heap);
-      struct cgraph_node *node;
+      struct cgraph_node *cnode;
 
-      depth = 0;
-      for (node = curr->caller;
-	   node; node = node->global.inlined_to)
+      depth = 1;
+      for (cnode = curr->caller;
+	   cnode->global.inlined_to; cnode = cnode->callers->caller)
 	if (node->decl == curr->callee->decl)
 	  depth++;
       if (depth > max_depth)
+	{
+          if (dump_file)
+	    fprintf (dump_file, 
+		     "   maxmal depth reached\n");
+	  continue;
+	}
+
+      if (max_count)
+	{
+          if (!cgraph_maybe_hot_edge_p (curr))
+	    {
+	      if (dump_file)
+		fprintf (dump_file, "   Not inlining cold call\n");
 	      continue;
+	    }
+          if (curr->count * 100 / node->count < probability)
+	    {
+	      if (dump_file)
+		fprintf (dump_file, 
+			 "   Probability of edge is too small\n");
+	      continue;
+	    }
+	}
 
       if (dump_file)
+	{
 	  fprintf (dump_file, 
-		 "   Inlining call of depth %i\n", depth);
+		   "   Inlining call of depth %i", depth);
+	  if (node->count)
+	    {
+	      fprintf (dump_file, " called approx. %.2f times per call",
+		       (double)curr->count / node->count);
+	    }
+	  fprintf (dump_file, "\n");
+	}
       cgraph_redirect_edge_callee (curr, master_clone);
       cgraph_mark_inline_edge (curr);
       lookup_recursive_calls (node, curr->callee, heap);
       n++;
     }
+  if (!fibheap_empty (heap) && dump_file)
+    fprintf (dump_file, "    Recursive inlining growth limit met.\n");
 
   fibheap_delete (heap);
   if (dump_file)
@@ -607,6 +644,10 @@ cgraph_decide_recursive_inlining (struct cgraph_node *node)
     if (node->global.inlined_to == master_clone)
       cgraph_remove_node (node);
   cgraph_remove_node (master_clone);
+  /* FIXME: Recursive inlining actually reduces number of calls of the
+     function.  At this place we should probably walk the function and
+     inline clones and compensate the counts accordingly.  This probably
+     doesn't matter much in practice.  */
   return true;
 }
 

gcc/params.def

@@ -116,6 +116,11 @@ DEFPARAM (PARAM_MAX_INLINE_RECURSIVE_DEPTH_AUTO,
 	  "The maximum depth of recursive inlining for non-inline functions",
 	  8, 0, 0)
 
+DEFPARAM (PARAM_MIN_INLINE_RECURSIVE_PROBABILITY,
+	  "min-inline-recursive-probability",
+	  "Inline recursively only when the probability of call being executed exceeds the parameter",
+	  10, 0, 0)
+
 /* Limit the number of expansions created by the variable expansion
    optimization to avoid register pressure.  */
 DEFPARAM (PARAM_MAX_VARIABLE_EXPANSIONS,

gcc/tree-inline.c

@@ -752,7 +752,7 @@ copy_bb (inline_data *id, basic_block bb, int frequency_scale, int count_scale)
 		  edge = cgraph_edge (id->current_node, orig_stmt);
 		  if (edge)
 		    cgraph_clone_edge (edge, id->node, stmt,
-				       REG_BR_PROB_BASE, 1);
+				       REG_BR_PROB_BASE, 1, true);
 		}
 	    }
 	  /* If you think we can abort here, you are wrong.

gcc/tree-optimize.c

@@ -372,7 +372,7 @@ tree_rest_of_compilation (tree fndecl)
 	{
 	  struct cgraph_edge *e;
 
-	  saved_node = cgraph_clone_node (node, node->count, 1);
+	  saved_node = cgraph_clone_node (node, node->count, 1, false);
 	  for (e = saved_node->callees; e; e = e->next_callee)
 	    if (!e->inline_failed)
 	      cgraph_clone_inlined_nodes (e, true);