cgraphbuild.c (compute_call_stmt_bb_frequency): Use proper ENTRY_BLOCK_PTR.

	* cgraphbuild.c (compute_call_stmt_bb_frequency): Use proper ENTRY_BLOCK_PTR.
	* cgraph.c (cgraph_clone_edge): Avoid freq_scale 0 to completely zero out all
	callees.
	* cgraphunit.c (verify_cgraph_node): Verify cgraph nodes for frequency and count match.
	* ipa-inline.c (update_noncloned_frequencies): New function.
	(cgraph_clone_inlined_nodes): Use it.
	* tree-inline.c (copy_bb): Fix frequency scaling; output
	diagnostic on frequency mismatches to dump file.
	(initialize_cfun): Do not scale frequency; fix count scaling;
	initialize entry and exit block frequencies; copy profile
	info.
	(copy_cfg_body): Use frequency_scale as argument;
	fix count scaling.
	(copy_body): Use frequency_scale as argument.
	(expand_call_inline): Compute frequency scale and output diagnostic
	to dump file.
	(delete_unreachable_blocks_update_callgrah): Remove checking that
	has to be done after edge redirection.
	(tree_function_versioning): Update initialize_cfun and copy_body call.

From-SVN: r154205

gcc/ChangeLog

 2009-11-14  Jan Hubicka  <jh@suse.cz>
 
+	* cgraphbuild.c (compute_call_stmt_bb_frequency): Use proper ENTRY_BLOCK_PTR.
+	* cgraph.c (cgraph_clone_edge): Avoid freq_scale 0 to completely zero out all
+	callees.
+	* cgraphunit.c (verify_cgraph_node): Verify cgraph nodes for frequency and count match.
+	* ipa-inline.c (update_noncloned_frequencies): New function.
+	(cgraph_clone_inlined_nodes): Use it.
+	* tree-inline.c (copy_bb): Fix frequency scaling; output
+	diagnostic on frequency mismatches to dump file.
+	(initialize_cfun): Do not scale frequency; fix count scaling;
+	initialize entry and exit block frequencies; copy profile
+	info.
+	(copy_cfg_body): Use frequency_scale as argument;
+	fix count scaling.
+	(copy_body): Use frequency_scale as argument.
+	(expand_call_inline): Compute frequency scale and output diagnostic
+	to dump file.
+	(delete_unreachable_blocks_update_callgrah): Remove checking that
+	has to be done after edge redirection.
+	(tree_function_versioning): Update initialize_cfun and copy_body call.
+
+2009-11-14  Jan Hubicka  <jh@suse.cz>
+
 	* cgraph.c (cgraph_release_function_body): Update use of
 	ipa_transforms_to_apply.
 	(cgraph_remove_node): Remove ipa_transforms_to_apply.

gcc/cgraph.c

@@ -1641,8 +1641,12 @@ cgraph_clone_edge (struct cgraph_edge *e, struct cgraph_node *n,
 {
   struct cgraph_edge *new_edge;
   gcov_type count = e->count * count_scale / REG_BR_PROB_BASE;
-  gcov_type freq = e->frequency * (gcov_type) freq_scale / CGRAPH_FREQ_BASE;
+  gcov_type freq;
 
+  /* We do not want to ignore loop nest after frequency drops to 0.  */
+  if (!freq_scale)
+    freq_scale = 1;
+  freq = e->frequency * (gcov_type) freq_scale / CGRAPH_FREQ_BASE;
   if (freq > CGRAPH_FREQ_MAX)
     freq = CGRAPH_FREQ_MAX;
   new_edge = cgraph_create_edge (n, e->callee, call_stmt, count, freq,

gcc/cgraphbuild.c

@@ -109,7 +109,8 @@ reset_inline_failed (struct cgraph_node *node)
 int
 compute_call_stmt_bb_frequency (tree decl, basic_block bb)
 {
-  int entry_freq = ENTRY_BLOCK_PTR->frequency;
+  int entry_freq = ENTRY_BLOCK_PTR_FOR_FUNCTION
+  		     (DECL_STRUCT_FUNCTION (decl))->frequency;
   int freq = bb->frequency;
 
   if (profile_status_for_function (DECL_STRUCT_FUNCTION (decl)) == PROFILE_ABSENT)

gcc/ipa-inline.c

@@ -207,6 +207,29 @@ cgraph_estimate_size_after_inlining (int times, struct cgraph_node *to,
   return size;
 }
 
+/* Scale frequency of NODE edges by FREQ_SCALE and increase loop nest
+   by NEST.  */
+
+static void
+update_noncloned_frequencies (struct cgraph_node *node,
+			      int freq_scale, int nest)
+{
+  struct cgraph_edge *e;
+
+  /* We do not want to ignore high loop nest after freq drops to 0.  */
+  if (!freq_scale)
+    freq_scale = 1;
+  for (e = node->callees; e; e = e->next_callee)
+    {
+      e->loop_nest += nest;
+      e->frequency = e->frequency * (gcov_type) freq_scale / CGRAPH_FREQ_BASE;
+      if (e->frequency > CGRAPH_FREQ_MAX)
+        e->frequency = CGRAPH_FREQ_MAX;
+      if (!e->inline_failed)
+        update_noncloned_frequencies (e->callee, freq_scale, nest);
+    }
+}
+
 /* E is expected to be an edge being inlined.  Clone destination node of
    the edge and redirect it to the new clone.
    DUPLICATE is used for bookkeeping on whether we are actually creating new
@@ -234,6 +257,7 @@ cgraph_clone_inlined_nodes (struct cgraph_edge *e, bool duplicate,
 	    }
 	  duplicate = false;
 	  e->callee->local.externally_visible = false;
+          update_noncloned_frequencies (e->callee, e->frequency, e->loop_nest);
 	}
       else
 	{

gcc/tree-inline.c

@@ -1472,6 +1472,7 @@ copy_bb (copy_body_data *id, basic_block bb, int frequency_scale,
   gimple_stmt_iterator gsi, copy_gsi, seq_gsi;
   basic_block copy_basic_block;
   tree decl;
+  gcov_type freq;
 
   /* create_basic_block() will append every new block to
      basic_block_info automatically.  */
@@ -1481,11 +1482,12 @@ copy_bb (copy_body_data *id, basic_block bb, int frequency_scale,
 
   /* We are going to rebuild frequencies from scratch.  These values
      have just small importance to drive canonicalize_loop_headers.  */
-  copy_basic_block->frequency = ((gcov_type)bb->frequency
-				 * frequency_scale / REG_BR_PROB_BASE);
+  freq = ((gcov_type)bb->frequency * frequency_scale / REG_BR_PROB_BASE);
 
-  if (copy_basic_block->frequency > BB_FREQ_MAX)
-    copy_basic_block->frequency = BB_FREQ_MAX;
+  /* We recompute frequencies after inlining, so this is quite safe.  */
+  if (freq > BB_FREQ_MAX)
+    freq = BB_FREQ_MAX;
+  copy_basic_block->frequency = freq;
 
   copy_gsi = gsi_start_bb (copy_basic_block);
 
@@ -1631,10 +1633,34 @@ copy_bb (copy_body_data *id, basic_block bb, int frequency_scale,
 		case CB_CGE_DUPLICATE:
 		  edge = cgraph_edge (id->src_node, orig_stmt);
 		  if (edge)
-		    edge = cgraph_clone_edge (edge, id->dst_node, stmt,
-					      gimple_uid (stmt),
-					      REG_BR_PROB_BASE, 1,
-					      edge->frequency, true);
+		    {
+		      int edge_freq = edge->frequency;
+		      edge = cgraph_clone_edge (edge, id->dst_node, stmt,
+					        gimple_uid (stmt),
+					        REG_BR_PROB_BASE, CGRAPH_FREQ_BASE,
+					        edge->frequency, true);
+		      /* We could also just rescale the frequency, but
+		         doing so would introduce roundoff errors and make
+			 verifier unhappy.  */
+		      edge->frequency 
+		        = compute_call_stmt_bb_frequency (id->dst_node->decl,
+							  copy_basic_block);
+		      if (dump_file
+		      	  && profile_status_for_function (cfun) != PROFILE_ABSENT
+			  && (edge_freq > edge->frequency + 10
+			      || edge_freq < edge->frequency - 10))
+			{
+			  fprintf (dump_file, "Edge frequency estimated by "
+			           "cgraph %i diverge from inliner's estimate %i\n",
+			  	   edge_freq,
+				   edge->frequency);
+			  fprintf (dump_file,
+			  	   "Orig bb: %i, orig bb freq %i, new bb freq %i\n",
+				   bb->index,
+				   bb->frequency,
+				   copy_basic_block->frequency);
+			}
+		    }
 		  break;
 
 		case CB_CGE_MOVE_CLONES:
@@ -1674,7 +1700,8 @@ copy_bb (copy_body_data *id, basic_block bb, int frequency_scale,
 		  if (id->transform_call_graph_edges == CB_CGE_MOVE_CLONES)
 		    cgraph_create_edge_including_clones
 		      (id->dst_node, dest, stmt, bb->count,
-		       compute_call_stmt_bb_frequency (id->dst_node->decl, bb),
+		       compute_call_stmt_bb_frequency (id->dst_node->decl, 
+		       				       copy_basic_block),
 		       bb->loop_depth, CIF_ORIGINALLY_INDIRECT_CALL);
 		  else
 		    cgraph_create_edge (id->dst_node, dest, stmt,
@@ -1948,24 +1975,16 @@ remap_decl_1 (tree decl, void *data)
    NEW_FNDECL to be build.  CALLEE_FNDECL is the original */
 
 static void
-initialize_cfun (tree new_fndecl, tree callee_fndecl, gcov_type count,
-		 int frequency)
+initialize_cfun (tree new_fndecl, tree callee_fndecl, gcov_type count)
 {
   struct function *src_cfun = DECL_STRUCT_FUNCTION (callee_fndecl);
-  gcov_type count_scale, frequency_scale;
+  gcov_type count_scale;
 
   if (ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->count)
     count_scale = (REG_BR_PROB_BASE * count
 		   / ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->count);
   else
-    count_scale = 1;
-
-  if (ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->frequency)
-    frequency_scale = (REG_BR_PROB_BASE * frequency
-		       /
-		       ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->frequency);
-  else
-    frequency_scale = count_scale;
+    count_scale = REG_BR_PROB_BASE;
 
   /* Register specific tree functions.  */
   gimple_register_cfg_hooks ();
@@ -1998,18 +2017,17 @@ initialize_cfun (tree new_fndecl, tree callee_fndecl, gcov_type count,
 
   init_empty_tree_cfg ();
 
+  profile_status_for_function (cfun) = profile_status_for_function (src_cfun);
   ENTRY_BLOCK_PTR->count =
     (ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->count * count_scale /
      REG_BR_PROB_BASE);
-  ENTRY_BLOCK_PTR->frequency =
-    (ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->frequency *
-     frequency_scale / REG_BR_PROB_BASE);
+  ENTRY_BLOCK_PTR->frequency
+    = ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->frequency;
   EXIT_BLOCK_PTR->count =
     (EXIT_BLOCK_PTR_FOR_FUNCTION (src_cfun)->count * count_scale /
      REG_BR_PROB_BASE);
   EXIT_BLOCK_PTR->frequency =
-    (EXIT_BLOCK_PTR_FOR_FUNCTION (src_cfun)->frequency *
-     frequency_scale / REG_BR_PROB_BASE);
+    EXIT_BLOCK_PTR_FOR_FUNCTION (src_cfun)->frequency;
   if (src_cfun->eh)
     init_eh_for_function ();
 
@@ -2026,7 +2044,7 @@ initialize_cfun (tree new_fndecl, tree callee_fndecl, gcov_type count,
    another function.  Walks FN via CFG, returns new fndecl.  */
 
 static tree
-copy_cfg_body (copy_body_data * id, gcov_type count, int frequency,
+copy_cfg_body (copy_body_data * id, gcov_type count, int frequency_scale,
 	       basic_block entry_block_map, basic_block exit_block_map)
 {
   tree callee_fndecl = id->src_fn;
@@ -2035,21 +2053,14 @@ copy_cfg_body (copy_body_data * id, gcov_type count, int frequency,
   struct function *cfun_to_copy;
   basic_block bb;
   tree new_fndecl = NULL;
-  gcov_type count_scale, frequency_scale;
+  gcov_type count_scale;
   int last;
 
   if (ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->count)
     count_scale = (REG_BR_PROB_BASE * count
 		   / ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->count);
   else
-    count_scale = 1;
-
-  if (ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->frequency)
-    frequency_scale = (REG_BR_PROB_BASE * frequency
-		       /
-		       ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->frequency);
-  else
-    frequency_scale = count_scale;
+    count_scale = REG_BR_PROB_BASE;
 
   /* Register specific tree functions.  */
   gimple_register_cfg_hooks ();
@@ -2204,7 +2215,7 @@ copy_tree_body (copy_body_data *id)
    another function.  */
 
 static tree
-copy_body (copy_body_data *id, gcov_type count, int frequency,
+copy_body (copy_body_data *id, gcov_type count, int frequency_scale,
 	   basic_block entry_block_map, basic_block exit_block_map)
 {
   tree fndecl = id->src_fn;
@@ -2212,7 +2223,7 @@ copy_body (copy_body_data *id, gcov_type count, int frequency,
 
   /* If this body has a CFG, walk CFG and copy.  */
   gcc_assert (ENTRY_BLOCK_PTR_FOR_FUNCTION (DECL_STRUCT_FUNCTION (fndecl)));
-  body = copy_cfg_body (id, count, frequency, entry_block_map, exit_block_map);
+  body = copy_cfg_body (id, count, frequency_scale, entry_block_map, exit_block_map);
   copy_debug_stmts (id);
 
   return body;
@@ -3732,12 +3743,23 @@ expand_call_inline (basic_block bb, gimple stmt, copy_body_data *id)
 				       cfun->local_decls);
     }
 
+  if (dump_file && (dump_flags & TDF_DETAILS))
+    {
+      fprintf (dump_file, "Inlining ");
+      print_generic_expr (dump_file, id->src_fn, 0); 
+      fprintf (dump_file, " to ");
+      print_generic_expr (dump_file, id->dst_fn, 0); 
+      fprintf (dump_file, " with frequency %i\n", cg_edge->frequency);
+    }
+
   /* This is it.  Duplicate the callee body.  Assume callee is
      pre-gimplified.  Note that we must not alter the caller
      function in any way before this point, as this CALL_EXPR may be
      a self-referential call; if we're calling ourselves, we need to
      duplicate our body before altering anything.  */
-  copy_body (id, bb->count, bb->frequency, bb, return_block);
+  copy_body (id, bb->count,
+  	     cg_edge->frequency * REG_BR_PROB_BASE / CGRAPH_FREQ_BASE,
+	     bb, return_block);
 
   /* Reset the escaped and callused solutions.  */
   if (cfun->gimple_df)
@@ -4732,30 +4754,6 @@ delete_unreachable_blocks_update_callgraph (copy_body_data *id)
 
   if (changed)
     tidy_fallthru_edges ();
-#ifdef ENABLE_CHECKING0
-  verify_cgraph_node (id->dst_node);
-  if (id->transform_call_graph_edges == CB_CGE_MOVE_CLONES
-      && id->dst_node->clones)
-    {
-      struct cgraph_node *node;
-      for (node = id->dst_node->clones; node != id->dst_node;)
-	{
-	  verify_cgraph_node (node);
-	   
-	  if (node->clones)
-	    node = node->clones;
-	  else if (node->next_sibling_clone)
-	    node = node->next_sibling_clone;
-	  else
-	    {
-	      while (node != id->dst_node && !node->next_sibling_clone)
-		node = node->clone_of;
-	      if (node != id->dst_node)
-		node = node->next_sibling_clone;
-	    }
-	}
-     }
-#endif
   return changed;
 }
 
@@ -4876,8 +4874,7 @@ tree_function_versioning (tree old_decl, tree new_decl,
   old_entry_block = ENTRY_BLOCK_PTR_FOR_FUNCTION
     (DECL_STRUCT_FUNCTION (old_decl));
   initialize_cfun (new_decl, old_decl,
-		   old_entry_block->count,
-		   old_entry_block->frequency);
+		   old_entry_block->count);
   push_cfun (DECL_STRUCT_FUNCTION (new_decl));
   
   /* Copy the function's static chain.  */
@@ -4947,7 +4944,7 @@ tree_function_versioning (tree old_decl, tree new_decl,
       }
   
   /* Copy the Function's body.  */
-  copy_body (&id, old_entry_block->count, old_entry_block->frequency,
+  copy_body (&id, old_entry_block->count, REG_BR_PROB_BASE,
 	     ENTRY_BLOCK_PTR, EXIT_BLOCK_PTR);
   
   if (DECL_RESULT (old_decl) != NULL_TREE)