Commit e7b655e8 by Jan Hubicka Committed by Jan Hubicka

cfgloopmanip.c (duplicate_loop_to_header_edge): Cleanup profile manipulation.

	* cfgloopmanip.c (duplicate_loop_to_header_edge): Cleanup profile
	manipulation.

From-SVN: r254806
parent 1554ed7e
2017-11-14 Jan Hubicka <hubicka@ucw.cz>
* cfgloopmanip.c (duplicate_loop_to_header_edge): Cleanup profile
manipulation.
2017-11-16 Chung-Ju Wu <jasonwucj@gmail.com> 2017-11-16 Chung-Ju Wu <jasonwucj@gmail.com>
Kito Cheng <kito.cheng@gmail.com> Kito Cheng <kito.cheng@gmail.com>
...@@ -1096,14 +1096,16 @@ duplicate_loop_to_header_edge (struct loop *loop, edge e, ...@@ -1096,14 +1096,16 @@ duplicate_loop_to_header_edge (struct loop *loop, edge e,
basic_block new_bb, bb, first_active_latch = NULL; basic_block new_bb, bb, first_active_latch = NULL;
edge ae, latch_edge; edge ae, latch_edge;
edge spec_edges[2], new_spec_edges[2]; edge spec_edges[2], new_spec_edges[2];
#define SE_LATCH 0 const int SE_LATCH = 0;
#define SE_ORIG 1 const int SE_ORIG = 1;
unsigned i, j, n; unsigned i, j, n;
int is_latch = (latch == e->src); int is_latch = (latch == e->src);
int scale_act = 0, *scale_step = NULL, scale_main = 0; profile_probability *scale_step = NULL;
int scale_after_exit = 0; profile_probability scale_main = profile_probability::always ();
int p, freq_in, freq_le, freq_out_orig; profile_probability scale_act = profile_probability::always ();
int prob_pass_thru, prob_pass_wont_exit, prob_pass_main; profile_count after_exit_num = profile_count::zero (),
after_exit_den = profile_count::zero ();
bool scale_after_exit = false;
int add_irreducible_flag; int add_irreducible_flag;
basic_block place_after; basic_block place_after;
bitmap bbs_to_scale = NULL; bitmap bbs_to_scale = NULL;
...@@ -1142,33 +1144,26 @@ duplicate_loop_to_header_edge (struct loop *loop, edge e, ...@@ -1142,33 +1144,26 @@ duplicate_loop_to_header_edge (struct loop *loop, edge e,
if (flags & DLTHE_FLAG_UPDATE_FREQ) if (flags & DLTHE_FLAG_UPDATE_FREQ)
{ {
/* Calculate coefficients by that we have to scale frequencies /* Calculate coefficients by that we have to scale counts
of duplicated loop bodies. */ of duplicated loop bodies. */
freq_in = header->count.to_frequency (cfun); profile_count count_in = header->count;
freq_le = EDGE_FREQUENCY (latch_edge); profile_count count_le = latch_edge->count ();
if (freq_in == 0) profile_count count_out_orig = orig ? orig->count () : count_in - count_le;
freq_in = 1; profile_probability prob_pass_thru = count_le.probability_in (count_in);
if (freq_in < freq_le) profile_probability prob_pass_wont_exit =
freq_in = freq_le; (count_le + count_out_orig).probability_in (count_in);
freq_out_orig = orig ? EDGE_FREQUENCY (orig) : freq_in - freq_le;
if (freq_out_orig > freq_in - freq_le)
freq_out_orig = freq_in - freq_le;
prob_pass_thru = RDIV (REG_BR_PROB_BASE * freq_le, freq_in);
prob_pass_wont_exit =
RDIV (REG_BR_PROB_BASE * (freq_le + freq_out_orig), freq_in);
if (orig && orig->probability.initialized_p () if (orig && orig->probability.initialized_p ()
&& !(orig->probability == profile_probability::always ())) && !(orig->probability == profile_probability::always ()))
{ {
/* The blocks that are dominated by a removed exit edge ORIG have /* The blocks that are dominated by a removed exit edge ORIG have
frequencies scaled by this. */ frequencies scaled by this. */
if (orig->probability.initialized_p ()) if (orig->count ().initialized_p ())
scale_after_exit {
= GCOV_COMPUTE_SCALE (REG_BR_PROB_BASE, after_exit_num = orig->src->count;
REG_BR_PROB_BASE after_exit_den = after_exit_num - orig->count ();
- orig->probability.to_reg_br_prob_base ()); scale_after_exit = true;
else }
scale_after_exit = REG_BR_PROB_BASE;
bbs_to_scale = BITMAP_ALLOC (NULL); bbs_to_scale = BITMAP_ALLOC (NULL);
for (i = 0; i < n; i++) for (i = 0; i < n; i++)
{ {
...@@ -1178,7 +1173,7 @@ duplicate_loop_to_header_edge (struct loop *loop, edge e, ...@@ -1178,7 +1173,7 @@ duplicate_loop_to_header_edge (struct loop *loop, edge e,
} }
} }
scale_step = XNEWVEC (int, ndupl); scale_step = XNEWVEC (profile_probability, ndupl);
for (i = 1; i <= ndupl; i++) for (i = 1; i <= ndupl; i++)
scale_step[i - 1] = bitmap_bit_p (wont_exit, i) scale_step[i - 1] = bitmap_bit_p (wont_exit, i)
...@@ -1189,52 +1184,48 @@ duplicate_loop_to_header_edge (struct loop *loop, edge e, ...@@ -1189,52 +1184,48 @@ duplicate_loop_to_header_edge (struct loop *loop, edge e,
copy becomes 1. */ copy becomes 1. */
if (flags & DLTHE_FLAG_COMPLETTE_PEEL) if (flags & DLTHE_FLAG_COMPLETTE_PEEL)
{ {
int wanted_freq = EDGE_FREQUENCY (e); profile_count wanted_count = e->count ();
if (wanted_freq > freq_in)
wanted_freq = freq_in;
gcc_assert (!is_latch); gcc_assert (!is_latch);
/* First copy has frequency of incoming edge. Each subsequent /* First copy has count of incoming edge. Each subsequent
frequency should be reduced by prob_pass_wont_exit. Caller count should be reduced by prob_pass_wont_exit. Caller
should've managed the flags so all except for original loop should've managed the flags so all except for original loop
has won't exist set. */ has won't exist set. */
scale_act = GCOV_COMPUTE_SCALE (wanted_freq, freq_in); scale_act = wanted_count.probability_in (count_in);
/* Now simulate the duplication adjustments and compute header /* Now simulate the duplication adjustments and compute header
frequency of the last copy. */ frequency of the last copy. */
for (i = 0; i < ndupl; i++) for (i = 0; i < ndupl; i++)
wanted_freq = combine_probabilities (wanted_freq, scale_step[i]); wanted_count = wanted_count.apply_probability (scale_step [i]);
scale_main = GCOV_COMPUTE_SCALE (wanted_freq, freq_in); scale_main = wanted_count.probability_in (count_in);
} }
/* Here we insert loop bodies inside the loop itself (for loop unrolling).
First iteration will be original loop followed by duplicated bodies.
It is necessary to scale down the original so we get right overall
number of iterations. */
else if (is_latch) else if (is_latch)
{ {
prob_pass_main = bitmap_bit_p (wont_exit, 0) profile_probability prob_pass_main = bitmap_bit_p (wont_exit, 0)
? prob_pass_wont_exit ? prob_pass_wont_exit
: prob_pass_thru; : prob_pass_thru;
p = prob_pass_main; profile_probability p = prob_pass_main;
scale_main = REG_BR_PROB_BASE; profile_count scale_main_den = count_in;
for (i = 0; i < ndupl; i++) for (i = 0; i < ndupl; i++)
{ {
scale_main += p; scale_main_den += count_in.apply_probability (p);
p = combine_probabilities (p, scale_step[i]); p = p * scale_step[i];
} }
scale_main = GCOV_COMPUTE_SCALE (REG_BR_PROB_BASE, scale_main); /* If original loop is executed COUNT_IN times, the unrolled
scale_act = combine_probabilities (scale_main, prob_pass_main); loop will account SCALE_MAIN_DEN times. */
scale_main = count_in.probability_in (scale_main_den);
scale_act = scale_main * prob_pass_main;
} }
else else
{ {
int preheader_freq = EDGE_FREQUENCY (e); profile_count preheader_count = e->count ();
scale_main = REG_BR_PROB_BASE;
for (i = 0; i < ndupl; i++) for (i = 0; i < ndupl; i++)
scale_main = combine_probabilities (scale_main, scale_step[i]); scale_main = scale_main * scale_step[i];
if (preheader_freq > freq_in) scale_act = preheader_count.probability_in (count_in);
preheader_freq = freq_in;
scale_act = GCOV_COMPUTE_SCALE (preheader_freq, freq_in);
} }
for (i = 0; i < ndupl; i++)
gcc_assert (scale_step[i] >= 0 && scale_step[i] <= REG_BR_PROB_BASE);
gcc_assert (scale_main >= 0 && scale_main <= REG_BR_PROB_BASE
&& scale_act >= 0 && scale_act <= REG_BR_PROB_BASE);
} }
/* Loop the new bbs will belong to. */ /* Loop the new bbs will belong to. */
...@@ -1327,13 +1318,11 @@ duplicate_loop_to_header_edge (struct loop *loop, edge e, ...@@ -1327,13 +1318,11 @@ duplicate_loop_to_header_edge (struct loop *loop, edge e,
force_edge_cold (new_spec_edges[SE_ORIG], true); force_edge_cold (new_spec_edges[SE_ORIG], true);
/* Scale the frequencies of the blocks dominated by the exit. */ /* Scale the frequencies of the blocks dominated by the exit. */
if (bbs_to_scale) if (bbs_to_scale && scale_after_exit)
{ {
EXECUTE_IF_SET_IN_BITMAP (bbs_to_scale, 0, i, bi) EXECUTE_IF_SET_IN_BITMAP (bbs_to_scale, 0, i, bi)
{ scale_bbs_frequencies_profile_count (new_bbs + i, 1, after_exit_num,
scale_bbs_frequencies_int (new_bbs + i, 1, scale_after_exit, after_exit_den);
REG_BR_PROB_BASE);
}
} }
} }
...@@ -1348,8 +1337,8 @@ duplicate_loop_to_header_edge (struct loop *loop, edge e, ...@@ -1348,8 +1337,8 @@ duplicate_loop_to_header_edge (struct loop *loop, edge e,
/* Set counts and frequencies. */ /* Set counts and frequencies. */
if (flags & DLTHE_FLAG_UPDATE_FREQ) if (flags & DLTHE_FLAG_UPDATE_FREQ)
{ {
scale_bbs_frequencies_int (new_bbs, n, scale_act, REG_BR_PROB_BASE); scale_bbs_frequencies (new_bbs, n, scale_act);
scale_act = combine_probabilities (scale_act, scale_step[j]); scale_act = scale_act * scale_step[j];
} }
} }
free (new_bbs); free (new_bbs);
...@@ -1363,13 +1352,11 @@ duplicate_loop_to_header_edge (struct loop *loop, edge e, ...@@ -1363,13 +1352,11 @@ duplicate_loop_to_header_edge (struct loop *loop, edge e,
force_edge_cold (orig, true); force_edge_cold (orig, true);
/* Scale the frequencies of the blocks dominated by the exit. */ /* Scale the frequencies of the blocks dominated by the exit. */
if (bbs_to_scale) if (bbs_to_scale && scale_after_exit)
{ {
EXECUTE_IF_SET_IN_BITMAP (bbs_to_scale, 0, i, bi) EXECUTE_IF_SET_IN_BITMAP (bbs_to_scale, 0, i, bi)
{ scale_bbs_frequencies_profile_count (bbs + i, 1, after_exit_num,
scale_bbs_frequencies_int (bbs + i, 1, scale_after_exit, after_exit_den);
REG_BR_PROB_BASE);
}
} }
} }
...@@ -1378,7 +1365,7 @@ duplicate_loop_to_header_edge (struct loop *loop, edge e, ...@@ -1378,7 +1365,7 @@ duplicate_loop_to_header_edge (struct loop *loop, edge e,
set_immediate_dominator (CDI_DOMINATORS, e->dest, e->src); set_immediate_dominator (CDI_DOMINATORS, e->dest, e->src);
if (flags & DLTHE_FLAG_UPDATE_FREQ) if (flags & DLTHE_FLAG_UPDATE_FREQ)
{ {
scale_bbs_frequencies_int (bbs, n, scale_main, REG_BR_PROB_BASE); scale_bbs_frequencies (bbs, n, scale_main);
free (scale_step); free (scale_step);
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment