auto-profile.c (AFDO_EINFO): New macro.

	* auto-profile.c (AFDO_EINFO): New macro.
	(class edge_info): New class.
	(is_edge_annotated, set_edge_annotated): Delete.
	(afdo_propagate_edge, afdo_propagate_circuit, afdo_propagate): Remove
	parameter.  Adjust edge count computation and annotation using class
	edge_info.
	(afdo_calculate_branch_prob, afdo_annotate_cfg): Likewise.

From-SVN: r267119

gcc/ChangeLog

+2018-12-14  Bin Cheng  <bin.cheng@linux.alibaba.com>
+
+	* auto-profile.c (AFDO_EINFO): New macro.
+	(class edge_info): New class.
+	(is_edge_annotated, set_edge_annotated): Delete.
+	(afdo_propagate_edge, afdo_propagate_circuit, afdo_propagate): Remove
+	parameter.  Adjust edge count computation and annotation using class
+	edge_info.
+	(afdo_calculate_branch_prob, afdo_annotate_cfg): Likewise.
+
 2018-12-13  Michael Ploujnikov  <michael.ploujnikov@oracle.com>
 
 	* ipa-cp.c (print_all_lattices): Skip cp clones.

gcc/auto-profile.c

@@ -101,6 +101,23 @@ along with GCC; see the file COPYING3.  If not see
 namespace autofdo
 {
 
+/* Intermediate edge info used when propagating AutoFDO profile information.
+   We can't edge->count() directly since it's computed from edge's probability
+   while probability is yet not decided during propagation.  */
+#define AFDO_EINFO(e)                     ((struct edge_info *) e->aux)
+class edge_info
+{
+public:
+  edge_info () : count_ (profile_count::zero ().afdo ()), annotated_ (false) {}
+  bool is_annotated () const { return annotated_; }
+  void set_annotated () { annotated_ = true; }
+  profile_count get_count () const { return count_; }
+  void set_count (profile_count count) { count_ = count; }
+private:
+  profile_count count_;
+  bool annotated_;
+};
+
 /* Represent a source location: (function_decl, lineno).  */
 typedef std::pair<tree, unsigned> decl_lineno;
 
@@ -1067,18 +1084,6 @@ set_bb_annotated (basic_block bb, bb_set *annotated)
   annotated->insert (bb);
 }
 
-static bool
-is_edge_annotated (const edge e, const edge_set &annotated)
-{
-  return annotated.find (e) != annotated.end ();
-}
-
-static void
-set_edge_annotated (edge e, edge_set *annotated)
-{
-  annotated->insert (e);
-}
-
 /* For a given BB, set its execution count. Attach value profile if a stmt
    is not in PROMOTED, because we only want to promote an indirect call once.
    Return TRUE if BB is annotated.  */
@@ -1188,12 +1193,11 @@ afdo_find_equiv_class (bb_set *annotated_bb)
    edges' counts are known, then the basic block's unknown count can also be
    calculated.
    IS_SUCC is true if out edges of a basic blocks are examined.
-   Update ANNOTATED_BB and ANNOTATED_EDGE accordingly.
+   Update ANNOTATED_BB accordingly.
    Return TRUE if any basic block/edge count is changed.  */
 
 static bool
-afdo_propagate_edge (bool is_succ, bb_set *annotated_bb,
-                     edge_set *annotated_edge)
+afdo_propagate_edge (bool is_succ, bb_set *annotated_bb)
 {
   basic_block bb;
   bool changed = false;
@@ -1206,29 +1210,29 @@ afdo_propagate_edge (bool is_succ, bb_set *annotated_bb,
     profile_count total_known_count = profile_count::zero ().afdo ();
 
     FOR_EACH_EDGE (e, ei, is_succ ? bb->succs : bb->preds)
-      if (!is_edge_annotated (e, *annotated_edge))
+      {
+	gcc_assert (AFDO_EINFO (e) != NULL);
+	if (! AFDO_EINFO (e)->is_annotated ())
 	  num_unknown_edge++, unknown_edge = e;
 	else
-	total_known_count += e->count ();
+	  total_known_count += AFDO_EINFO (e)->get_count ();
+      }
 
-    if (num_unknown_edge == 0)
-      {
-        if (total_known_count > bb->count)
+    /* Be careful not to annotate block with no successor in special cases.  */
+    if (num_unknown_edge == 0 && total_known_count > bb->count)
       {
 	bb->count = total_known_count;
-            changed = true;
-          }
 	if (!is_bb_annotated (bb, *annotated_bb))
-          {
 	  set_bb_annotated (bb, annotated_bb);
 	changed = true;
       }
-      }
     else if (num_unknown_edge == 1 && is_bb_annotated (bb, *annotated_bb))
       {
-        unknown_edge->probability
-	  = total_known_count.probability_in (bb->count);
-        set_edge_annotated (unknown_edge, annotated_edge);
+	if (bb->count > total_known_count)
+	  AFDO_EINFO (unknown_edge)->set_count (bb->count - total_known_count);
+	else
+	  AFDO_EINFO (unknown_edge)->set_count (profile_count::zero().afdo ());
+	AFDO_EINFO (unknown_edge)->set_annotated ();
 	changed = true;
       }
   }
@@ -1265,11 +1269,10 @@ afdo_propagate_edge (bool is_succ, bb_set *annotated_bb,
        goto BB3
 
    In this case, we need to propagate through PHI to determine the edge
-   count of BB1->BB.t1, BB.t1->BB.t2.
-   Update ANNOTATED_EDGE accordingly.  */
+   count of BB1->BB.t1, BB.t1->BB.t2.  */
 
 static void
-afdo_propagate_circuit (const bb_set &annotated_bb, edge_set *annotated_edge)
+afdo_propagate_circuit (const bb_set &annotated_bb)
 {
   basic_block bb;
   FOR_ALL_BB_FN (bb, cfun)
@@ -1308,7 +1311,7 @@ afdo_propagate_circuit (const bb_set &annotated_bb, edge_set *annotated_edge)
       bool check_value_one = (((integer_onep (cmp_rhs))
                                ^ (gimple_cond_code (cmp_stmt) == EQ_EXPR))
                               ^ ((e->flags & EDGE_TRUE_VALUE) != 0));
-      if (!is_edge_annotated (e, *annotated_edge))
+      if (! AFDO_EINFO (e)->is_annotated ())
         continue;
       for (i = 0; i < gimple_phi_num_args (phi_stmt); i++)
         {
@@ -1322,17 +1325,17 @@ afdo_propagate_circuit (const bb_set &annotated_bb, edge_set *annotated_edge)
             continue;
           total++;
           only_one = ep;
-          if (!e->probability.initialized_p ()
-	      && !is_edge_annotated (ep, *annotated_edge))
+          if (! (AFDO_EINFO (e)->get_count ()).nonzero_p ()
+	      && ! AFDO_EINFO (ep)->is_annotated ())
 	    {
-              ep->probability = profile_probability::never ().afdo ();
-              set_edge_annotated (ep, annotated_edge);
+	      AFDO_EINFO (ep)->set_count (profile_count::zero ().afdo ());
+	      AFDO_EINFO (ep)->set_annotated ();
 	    }
 	}
-      if (total == 1 && !is_edge_annotated (only_one, *annotated_edge))
+      if (total == 1 && ! AFDO_EINFO (only_one)->is_annotated ())
 	{
-          only_one->probability = e->probability;
-          set_edge_annotated (only_one, annotated_edge);
+	  AFDO_EINFO (only_one)->set_count (AFDO_EINFO (e)->get_count ());
+	  AFDO_EINFO (only_one)->set_annotated ();
 	}
     }
   }
@@ -1342,7 +1345,7 @@ afdo_propagate_circuit (const bb_set &annotated_bb, edge_set *annotated_edge)
    graph. We do the propagation iteratively until stablize.  */
 
 static void
-afdo_propagate (bb_set *annotated_bb, edge_set *annotated_edge)
+afdo_propagate (bb_set *annotated_bb)
 {
   basic_block bb;
   bool changed = true;
@@ -1359,11 +1362,11 @@ afdo_propagate (bb_set *annotated_bb, edge_set *annotated_edge)
     {
       changed = false;
 
-      if (afdo_propagate_edge (true, annotated_bb, annotated_edge))
+      if (afdo_propagate_edge (true, annotated_bb))
         changed = true;
-      if (afdo_propagate_edge (false, annotated_bb, annotated_edge))
+      if (afdo_propagate_edge (false, annotated_bb))
         changed = true;
-      afdo_propagate_circuit (*annotated_bb, annotated_edge);
+      afdo_propagate_circuit (*annotated_bb);
     }
 }
 
@@ -1371,52 +1374,59 @@ afdo_propagate (bb_set *annotated_bb, edge_set *annotated_edge)
    probabilities.  */
 
 static void
-afdo_calculate_branch_prob (bb_set *annotated_bb, edge_set *annotated_edge)
+afdo_calculate_branch_prob (bb_set *annotated_bb)
 {
+  edge e;
+  edge_iterator ei;
   basic_block bb;
-  bool has_sample = false;
 
-  FOR_EACH_BB_FN (bb, cfun)
+  calculate_dominance_info (CDI_POST_DOMINATORS);
+  calculate_dominance_info (CDI_DOMINATORS);
+  loop_optimizer_init (0);
+
+  FOR_ALL_BB_FN (bb, cfun)
     {
-    if (bb->count > profile_count::zero ())
+      gcc_assert (bb->aux == NULL);
+      FOR_EACH_EDGE (e, ei, bb->succs)
 	{
-	has_sample = true;
-	break;
+	  gcc_assert (e->aux == NULL);
+	  e->aux = new edge_info ();
 	}
     }
 
-  if (!has_sample)
-    return;
-
-  calculate_dominance_info (CDI_POST_DOMINATORS);
-  calculate_dominance_info (CDI_DOMINATORS);
-  loop_optimizer_init (0);
-
   afdo_find_equiv_class (annotated_bb);
-  afdo_propagate (annotated_bb, annotated_edge);
+  afdo_propagate (annotated_bb);
 
   FOR_EACH_BB_FN (bb, cfun)
   {
-    edge e;
-    edge_iterator ei;
     int num_unknown_succ = 0;
     profile_count total_count = profile_count::zero ().afdo ();
 
     FOR_EACH_EDGE (e, ei, bb->succs)
     {
-      if (!is_edge_annotated (e, *annotated_edge))
+      gcc_assert (AFDO_EINFO (e) != NULL);
+      if (! AFDO_EINFO (e)->is_annotated ())
         num_unknown_succ++;
       else
-        total_count += e->count ();
+        total_count += AFDO_EINFO (e)->get_count ();
     }
     if (num_unknown_succ == 0 && total_count > profile_count::zero ())
       {
 	FOR_EACH_EDGE (e, ei, bb->succs)
-          e->probability = e->count ().probability_in (total_count);
+	  e->probability
+	    = AFDO_EINFO (e)->get_count ().probability_in (total_count);
       }
   }
   FOR_ALL_BB_FN (bb, cfun)
+    {
       bb->aux = NULL;
+      FOR_EACH_EDGE (e, ei, bb->succs)
+	if (AFDO_EINFO (e) != NULL)
+	  {
+	    delete AFDO_EINFO (e);
+	    e->aux = NULL;
+	  }
+    }
 
   loop_optimizer_finalize ();
   free_dominance_info (CDI_DOMINATORS);
@@ -1496,7 +1506,6 @@ afdo_annotate_cfg (const stmt_set &promoted_stmts)
 {
   basic_block bb;
   bb_set annotated_bb;
-  edge_set annotated_edge;
   const function_instance *s
       = afdo_source_profile->get_function_instance_by_decl (
           current_function_decl);
@@ -1512,15 +1521,9 @@ afdo_annotate_cfg (const stmt_set &promoted_stmts)
 
   FOR_EACH_BB_FN (bb, cfun)
     {
-    edge e;
-    edge_iterator ei;
-
       /* As autoFDO uses sampling approach, we have to assume that all
 	 counters are zero when not seen by autoFDO.  */
       bb->count = profile_count::zero ().afdo ();
-    FOR_EACH_EDGE (e, ei, bb->succs)
-      e->probability = profile_probability::uninitialized ();
-
       if (afdo_set_bb_count (bb, promoted_stmts))
 	set_bb_annotated (bb, &annotated_bb);
       if (bb->count > max_count)
@@ -1546,7 +1549,8 @@ afdo_annotate_cfg (const stmt_set &promoted_stmts)
   afdo_source_profile->mark_annotated (cfun->function_end_locus);
   if (max_count > profile_count::zero ())
     {
-      afdo_calculate_branch_prob (&annotated_bb, &annotated_edge);
+      /* Calculate, propagate count and probability information on CFG.  */
+      afdo_calculate_branch_prob (&annotated_bb);
       update_max_bb_count ();
       profile_status_for_fn (cfun) = PROFILE_READ;
     }