Commit 87741e51 by Martin Liska Committed by Martin Liska

Add zstd support for LTO bytecode compression.

2019-07-03  Martin Liska  <mliska@suse.cz>

	* Makefile.in: Define ZSTD_LIB.
	* common.opt: Adjust compression level
	to support also zstd levels.
	* config.in: Regenerate.
	* configure: Likewise.
	* configure.ac: Add --with-zstd and --with-zstd-include options
	and detect ZSTD.
	* doc/install.texi: Mention zstd dependency.
	* gcc.c: Print supported LTO compression algorithms.
	* lto-compress.c (lto_normalized_zstd_level): Likewise.
	(lto_compression_zstd): Likewise.
	(lto_uncompression_zstd): Likewise.
	(lto_end_compression): Dispatch in between zlib and zstd.
	(lto_compression_zlib): Mark with ATTRIBUTE_UNUSED.
	(lto_uncompression_zlib): Make it static.
	* lto-compress.h (lto_end_uncompression): Fix GNU coding style.
	* lto-section-in.c (lto_get_section_data): Pass info
	about used compression.
	* lto-streamer-out.c: By default use zstd when possible.
	* timevar.def (TV_IPA_LTO_DECOMPRESS): Rename to decompression
	(TV_IPA_LTO_COMPRESS): Likewise for compression.

From-SVN: r272996
parent 88614dfa
2019-07-03 Martin Liska <mliska@suse.cz>
* Makefile.in: Define ZSTD_LIB.
* common.opt: Adjust compression level
to support also zstd levels.
* config.in: Regenerate.
* configure: Likewise.
* configure.ac: Add --with-zstd and --with-zstd-include options
and detect ZSTD.
* doc/install.texi: Mention zstd dependency.
* gcc.c: Print supported LTO compression algorithms.
* lto-compress.c (lto_normalized_zstd_level): Likewise.
(lto_compression_zstd): Likewise.
(lto_uncompression_zstd): Likewise.
(lto_end_compression): Dispatch in between zlib and zstd.
(lto_compression_zlib): Mark with ATTRIBUTE_UNUSED.
(lto_uncompression_zlib): Make it static.
* lto-compress.h (lto_end_uncompression): Fix GNU coding style.
* lto-section-in.c (lto_get_section_data): Pass info
about used compression.
* lto-streamer-out.c: By default use zstd when possible.
* timevar.def (TV_IPA_LTO_DECOMPRESS): Rename to decompression
(TV_IPA_LTO_COMPRESS): Likewise for compression.
2019-07-03 Martin Liska <mliska@suse.cz>
* lto-section-in.c (lto_get_section_data): Add "lto" section.
* lto-section-out.c (lto_destroy_simple_output_block): Never
compress LTO_section_lto section.
......
......@@ -1065,7 +1065,7 @@ BUILD_LIBDEPS= $(BUILD_LIBIBERTY)
LIBS = @LIBS@ libcommon.a $(CPPLIB) $(LIBINTL) $(LIBICONV) $(LIBBACKTRACE) \
$(LIBIBERTY) $(LIBDECNUMBER) $(HOST_LIBS)
BACKENDLIBS = $(ISLLIBS) $(GMPLIBS) $(PLUGINLIBS) $(HOST_LIBS) \
$(ZLIB)
$(ZLIB) $(ZSTD_LIB)
# Any system libraries needed just for GNAT.
SYSLIBS = @GNAT_LIBEXC@
......@@ -1076,6 +1076,8 @@ GNATMAKE = @GNATMAKE@
# Libs needed (at present) just for jcf-dump.
LDEXP_LIB = @LDEXP_LIB@
ZSTD_LIB = @ZSTD_LIB@
# Likewise, for use in the tools that must run on this machine
# even if we are cross-building GCC.
BUILD_LIBS = $(BUILD_LIBIBERTY)
......
......@@ -1888,8 +1888,8 @@ Specify the algorithm to partition symbols and vars at linktime.
; The initial value of -1 comes from Z_DEFAULT_COMPRESSION in zlib.h.
flto-compression-level=
Common Joined RejectNegative UInteger Var(flag_lto_compression_level) Init(-1) IntegerRange(0, 9)
-flto-compression-level=<number> Use zlib compression level <number> for IL.
Common Joined RejectNegative UInteger Var(flag_lto_compression_level) Init(-1) IntegerRange(0, 19)
-flto-compression-level=<number> Use zlib/zstd compression level <number> for IL.
flto-odr-type-merging
Common Ignore
......
......@@ -1926,6 +1926,12 @@
#endif
/* Define if you have a working <zstd.h> header file. */
#ifndef USED_FOR_TARGET
#undef HAVE_ZSTD_H
#endif
/* Define if isl is in use. */
#ifndef USED_FOR_TARGET
#undef HAVE_isl
......
......@@ -782,6 +782,8 @@ manext
LIBICONV_DEP
LTLIBICONV
LIBICONV
ZSTD_LIB
ZSTD_INCLUDE
DL_LIB
LDEXP_LIB
EXTRA_GCC_LIBS
......@@ -959,6 +961,9 @@ with_pkgversion
with_bugurl
enable_languages
with_multilib_list
with_zstd
with_zstd_include
with_zstd_lib
enable_rpath
with_libiconv_prefix
enable_sjlj_exceptions
......@@ -1783,6 +1788,12 @@ Optional Packages:
--with-pkgversion=PKG Use PKG in the version string in place of "GCC"
--with-bugurl=URL Direct users to URL to report a bug
--with-multilib-list select multilibs (AArch64, SH and x86-64 only)
--with-zstd=PATH specify prefix directory for installed zstd library.
Equivalent to --with-zstd-include=PATH/include plus
--with-zstd-lib=PATH/lib
--with-zstd-include=PATH
specify directory for installed zstd include files
--with-zstd-lib=PATH specify directory for the installed zstd library
--with-gnu-ld assume the C compiler uses GNU ld default=no
--with-libiconv-prefix[=DIR] search for libiconv in DIR/include and DIR/lib
--without-libiconv-prefix don't search for libiconv in includedir and libdir
......@@ -9838,6 +9849,154 @@ $as_echo "#define HAVE_INTTYPES_H 1" >>confdefs.h
fi
# Look for the ZSTD package.
ZSTD_INCLUDE=
ZSTD_LIB=
ZSTD_CPPFLAGS=
ZSTD_LDFLAGS=
# Check whether --with-zstd was given.
if test "${with_zstd+set}" = set; then :
withval=$with_zstd;
fi
# Check whether --with-zstd-include was given.
if test "${with_zstd_include+set}" = set; then :
withval=$with_zstd_include;
fi
# Check whether --with-zstd-lib was given.
if test "${with_zstd_lib+set}" = set; then :
withval=$with_zstd_lib;
fi
case "x$with_zstd" in
x) ;;
xno)
ZSTD_INCLUDE=no
ZSTD_LIB=no
;;
*) ZSTD_INCLUDE=$with_zstd/include
ZSTD_LIB=$with_zstd/lib
;;
esac
if test "x$with_zstd_include" != x; then
ZSTD_INCLUDE=$with_zstd_include
fi
if test "x$with_zstd_lib" != x; then
ZSTD_LIB=$with_zstd_lib
fi
if test "x$ZSTD_INCLUDE" != x \
&& test "x$ZSTD_INCLUDE" != xno; then
ZSTD_CPPFLAGS=-I$ZSTD_INCLUDE
fi
if test "x$ZSTD_LIB" != x \
&& test "x$ZSTD_LIB" != xno; then
ZSTD_LDFLAGS=-L$ZSTD_LIB
fi
CXXFLAGS="$CXXFLAGS $ZSTD_CPPFLAGS"
LDFLAGS="$LDFLAGS $ZSTD_LDFLAGS"
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for zstd.h" >&5
$as_echo_n "checking for zstd.h... " >&6; }
if ${gcc_cv_header_zstd_h+:} false; then :
$as_echo_n "(cached) " >&6
else
cat confdefs.h - <<_ACEOF >conftest.$ac_ext
/* end confdefs.h. */
#include <zstd.h>
int
main ()
{
;
return 0;
}
_ACEOF
if ac_fn_cxx_try_compile "$LINENO"; then :
gcc_cv_header_zstd_h=yes
else
gcc_cv_header_zstd_h=no
fi
rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
fi
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $gcc_cv_header_zstd_h" >&5
$as_echo "$gcc_cv_header_zstd_h" >&6; }
if test $gcc_cv_header_zstd_h = yes; then
$as_echo "#define HAVE_ZSTD_H 1" >>confdefs.h
fi
# LTO can use zstd compression algorithm
save_LIBS="$LIBS"
LIBS=
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing ZSTD_compress" >&5
$as_echo_n "checking for library containing ZSTD_compress... " >&6; }
if ${ac_cv_search_ZSTD_compress+:} false; then :
$as_echo_n "(cached) " >&6
else
ac_func_search_save_LIBS=$LIBS
cat confdefs.h - <<_ACEOF >conftest.$ac_ext
/* end confdefs.h. */
/* Override any GCC internal prototype to avoid an error.
Use char because int might match the return type of a GCC
builtin and then its argument prototype would still apply. */
#ifdef __cplusplus
extern "C"
#endif
char ZSTD_compress ();
int
main ()
{
return ZSTD_compress ();
;
return 0;
}
_ACEOF
for ac_lib in '' zstd; do
if test -z "$ac_lib"; then
ac_res="none required"
else
ac_res=-l$ac_lib
LIBS="-l$ac_lib $ac_func_search_save_LIBS"
fi
if ac_fn_cxx_try_link "$LINENO"; then :
ac_cv_search_ZSTD_compress=$ac_res
fi
rm -f core conftest.err conftest.$ac_objext \
conftest$ac_exeext
if ${ac_cv_search_ZSTD_compress+:} false; then :
break
fi
done
if ${ac_cv_search_ZSTD_compress+:} false; then :
else
ac_cv_search_ZSTD_compress=no
fi
rm conftest.$ac_ext
LIBS=$ac_func_search_save_LIBS
fi
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_ZSTD_compress" >&5
$as_echo "$ac_cv_search_ZSTD_compress" >&6; }
ac_res=$ac_cv_search_ZSTD_compress
if test "$ac_res" != no; then :
test "$ac_res" = "none required" || LIBS="$ac_res $LIBS"
fi
ZSTD_LIB="$LIBS"
LIBS="$save_LIBS"
for ac_func in times clock kill getrlimit setrlimit atoq \
......@@ -18655,7 +18814,7 @@ else
lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
lt_status=$lt_dlunknown
cat > conftest.$ac_ext <<_LT_EOF
#line 18658 "configure"
#line 18817 "configure"
#include "confdefs.h"
#if HAVE_DLFCN_H
......@@ -18761,7 +18920,7 @@ else
lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
lt_status=$lt_dlunknown
cat > conftest.$ac_ext <<_LT_EOF
#line 18764 "configure"
#line 18923 "configure"
#include "confdefs.h"
#if HAVE_DLFCN_H
......
......@@ -1258,6 +1258,72 @@ if test $gcc_cv_header_inttypes_h = yes; then
[Define if you have a working <inttypes.h> header file.])
fi
# Look for the ZSTD package.
ZSTD_INCLUDE=
ZSTD_LIB=
AC_SUBST(ZSTD_INCLUDE)
AC_SUBST(ZSTD_LIB)
ZSTD_CPPFLAGS=
ZSTD_LDFLAGS=
AC_ARG_WITH(zstd,
[AS_HELP_STRING([--with-zstd=PATH],
[specify prefix directory for installed zstd library.
Equivalent to --with-zstd-include=PATH/include
plus --with-zstd-lib=PATH/lib])])
AC_ARG_WITH(zstd-include,
[AS_HELP_STRING([--with-zstd-include=PATH],
[specify directory for installed zstd include files])])
AC_ARG_WITH(zstd-lib,
[AS_HELP_STRING([--with-zstd-lib=PATH],
[specify directory for the installed zstd library])])
case "x$with_zstd" in
x) ;;
xno)
ZSTD_INCLUDE=no
ZSTD_LIB=no
;;
*) ZSTD_INCLUDE=$with_zstd/include
ZSTD_LIB=$with_zstd/lib
;;
esac
if test "x$with_zstd_include" != x; then
ZSTD_INCLUDE=$with_zstd_include
fi
if test "x$with_zstd_lib" != x; then
ZSTD_LIB=$with_zstd_lib
fi
if test "x$ZSTD_INCLUDE" != x \
&& test "x$ZSTD_INCLUDE" != xno; then
ZSTD_CPPFLAGS=-I$ZSTD_INCLUDE
fi
if test "x$ZSTD_LIB" != x \
&& test "x$ZSTD_LIB" != xno; then
ZSTD_LDFLAGS=-L$ZSTD_LIB
fi
CXXFLAGS="$CXXFLAGS $ZSTD_CPPFLAGS"
LDFLAGS="$LDFLAGS $ZSTD_LDFLAGS"
AC_MSG_CHECKING(for zstd.h)
AC_CACHE_VAL(gcc_cv_header_zstd_h,
[AC_COMPILE_IFELSE([AC_LANG_PROGRAM(
[[#include <zstd.h>]])],
[gcc_cv_header_zstd_h=yes],
[gcc_cv_header_zstd_h=no])])
AC_MSG_RESULT($gcc_cv_header_zstd_h)
if test $gcc_cv_header_zstd_h = yes; then
AC_DEFINE(HAVE_ZSTD_H, 1,
[Define if you have a working <zstd.h> header file.])
fi
# LTO can use zstd compression algorithm
save_LIBS="$LIBS"
LIBS=
AC_SEARCH_LIBS(ZSTD_compress, zstd)
ZSTD_LIB="$LIBS"
LIBS="$save_LIBS"
AC_SUBST(ZSTD_LIB)
dnl Disabled until we have a complete test for buggy enum bitfields.
dnl gcc_AC_C_ENUM_BF_UNSIGNED
......
......@@ -396,6 +396,12 @@ built together with GCC. Alternatively, the @option{--with-isl} configure
option should be used if isl is not installed in your default library
search path.
@item zstd Library.
Necessary to build GCC with zstd compression used for LTO bytecode.
The library is searched in your default library patch search.
Alternatively, the @option{--with-zstd} configure option should be used.
@end table
@heading Tools/packages necessary for modifying GCC
......
......@@ -6791,6 +6791,11 @@ print_configuration (FILE *file)
#endif
fnotice (file, "Thread model: %s\n", thrmod);
fnotice (file, "Supported LTO compression algorithms: zlib");
#ifdef HAVE_ZSTD_H
fnotice (file, " zstd");
#endif
fnotice (file, "\n");
/* compiler_version is truncated at the first space when initialized
from version string, so truncate version_string at the first space
......
......@@ -35,6 +35,10 @@ along with GCC; see the file COPYING3. If not see
#include "lto-compress.h"
#include "timevar.h"
#ifdef HAVE_ZSTD_H
#include <zstd.h>
#endif
/* Compression stream structure, holds the flush callback and opaque token,
the buffered data, and a note of whether compressing or uncompressing. */
......@@ -92,6 +96,91 @@ lto_normalized_zlib_level (void)
return level;
}
/* Free the buffer and memory associated with STREAM. */
static void
lto_destroy_compression_stream (struct lto_compression_stream *stream)
{
free (stream->buffer);
free (stream);
}
#ifdef HAVE_ZSTD_H
/* Return a zstd compression level that zstd will not reject. Normalizes
the compression level from the command line flag, clamping non-default
values to the appropriate end of their valid range. */
static int
lto_normalized_zstd_level (void)
{
int level = flag_lto_compression_level;
if (level != ZSTD_CLEVEL_DEFAULT)
{
if (level < 1)
level = 1;
else if (level > ZSTD_maxCLevel ())
level = ZSTD_maxCLevel ();
}
return level;
}
/* Compress STREAM using ZSTD algorithm. */
static void
lto_compression_zstd (struct lto_compression_stream *stream)
{
unsigned char *cursor = (unsigned char *) stream->buffer;
size_t size = stream->bytes;
timevar_push (TV_IPA_LTO_COMPRESS);
size_t const outbuf_length = ZSTD_compressBound (size);
char *outbuf = (char *) xmalloc (outbuf_length);
size_t const csize = ZSTD_compress (outbuf, outbuf_length, cursor, size,
lto_normalized_zstd_level ());
if (ZSTD_isError (csize))
internal_error ("compressed stream: %s", ZSTD_getErrorName (csize));
stream->callback (outbuf, csize, NULL);
lto_destroy_compression_stream (stream);
free (outbuf);
timevar_pop (TV_IPA_LTO_COMPRESS);
}
/* Uncompress STREAM using ZSTD algorithm. */
static void
lto_uncompression_zstd (struct lto_compression_stream *stream)
{
unsigned char *cursor = (unsigned char *) stream->buffer;
size_t size = stream->bytes;
timevar_push (TV_IPA_LTO_DECOMPRESS);
unsigned long long const rsize = ZSTD_getFrameContentSize (cursor, size);
if (rsize == ZSTD_CONTENTSIZE_ERROR)
internal_error ("original not compressed with zstd");
else if (rsize == ZSTD_CONTENTSIZE_UNKNOWN)
internal_error ("original size unknown");
char *outbuf = (char *) xmalloc (rsize);
size_t const dsize = ZSTD_decompress (outbuf, rsize, cursor, size);
if (ZSTD_isError (dsize))
internal_error ("decompressed stream: %s", ZSTD_getErrorName (dsize));
stream->callback (outbuf, dsize, stream->opaque);
lto_destroy_compression_stream (stream);
free (outbuf);
timevar_pop (TV_IPA_LTO_DECOMPRESS);
}
#endif
/* Create a new compression stream, with CALLBACK flush function passed
OPAQUE token, IS_COMPRESSION indicates if compressing or uncompressing. */
......@@ -132,15 +221,6 @@ lto_append_to_compression_stream (struct lto_compression_stream *stream,
stream->bytes += num_chars;
}
/* Free the buffer and memory associated with STREAM. */
static void
lto_destroy_compression_stream (struct lto_compression_stream *stream)
{
free (stream->buffer);
free (stream);
}
/* Return a new compression stream, with CALLBACK flush function passed
OPAQUE token. */
......@@ -163,10 +243,8 @@ lto_compress_block (struct lto_compression_stream *stream,
lto_stats.num_output_il_bytes += num_chars;
}
/* Finalize STREAM compression, and free stream allocations. */
void
lto_end_compression (struct lto_compression_stream *stream)
static void ATTRIBUTE_UNUSED
lto_compression_zlib (struct lto_compression_stream *stream)
{
unsigned char *cursor = (unsigned char *) stream->buffer;
size_t remaining = stream->bytes;
......@@ -226,6 +304,16 @@ lto_end_compression (struct lto_compression_stream *stream)
timevar_pop (TV_IPA_LTO_COMPRESS);
}
void
lto_end_compression (struct lto_compression_stream *stream)
{
#ifdef HAVE_ZSTD_H
lto_compression_zstd (stream);
#else
lto_compression_zlib (stream);
#endif
}
/* Return a new uncompression stream, with CALLBACK flush function passed
OPAQUE token. */
......@@ -248,14 +336,8 @@ lto_uncompress_block (struct lto_compression_stream *stream,
lto_stats.num_input_il_bytes += num_chars;
}
/* Finalize STREAM uncompression, and free stream allocations.
Because of the way LTO IL streams are compressed, there may be several
concatenated compressed segments in the accumulated data, so for this
function we iterate decompressions until no data remains. */
void
lto_end_uncompression (struct lto_compression_stream *stream)
static void
lto_uncompression_zlib (struct lto_compression_stream *stream)
{
unsigned char *cursor = (unsigned char *) stream->buffer;
size_t remaining = stream->bytes;
......@@ -318,3 +400,20 @@ lto_end_uncompression (struct lto_compression_stream *stream)
free (outbuf);
timevar_pop (TV_IPA_LTO_DECOMPRESS);
}
void
lto_end_uncompression (struct lto_compression_stream *stream,
lto_compression compression)
{
#ifdef HAVE_ZSTD_H
if (compression == ZSTD)
{
lto_uncompression_zstd (stream);
return;
}
#endif
if (compression == ZSTD)
internal_error ("compiler does not support ZSTD LTO compression");
lto_uncompression_zlib (stream);
}
......@@ -37,6 +37,7 @@ extern struct lto_compression_stream
void *opaque);
extern void lto_uncompress_block (struct lto_compression_stream *stream,
const char *base, size_t num_chars);
extern void lto_end_uncompression (struct lto_compression_stream *stream);
extern void lto_end_uncompression (struct lto_compression_stream *stream,
lto_compression compression);
#endif /* GCC_LTO_COMPRESS_H */
......@@ -161,7 +161,7 @@ lto_get_section_data (struct lto_file_decl_data *file_data,
stream = lto_start_uncompression (lto_append_data, &buffer);
lto_uncompress_block (stream, data, *len);
lto_end_uncompression (stream);
lto_end_uncompression (stream, file_data->lto_section_header.compression);
*len = buffer.length - header_length;
data = buffer.data + header_length;
......
......@@ -2395,7 +2395,11 @@ produce_lto_section ()
lto_begin_section (section_name, false);
free (section_name);
#ifdef HAVE_ZSTD_H
lto_compression compression = ZSTD;
#else
lto_compression compression = ZLIB;
#endif
bool slim_object = flag_generate_lto && !flag_fat_lto_objects;
lto_section s
......
......@@ -78,8 +78,8 @@ DEFTIMEVAR (TV_IPA_INLINING , "ipa inlining heuristics")
DEFTIMEVAR (TV_IPA_FNSPLIT , "ipa function splitting")
DEFTIMEVAR (TV_IPA_COMDATS , "ipa comdats")
DEFTIMEVAR (TV_IPA_OPT , "ipa various optimizations")
DEFTIMEVAR (TV_IPA_LTO_DECOMPRESS , "lto stream inflate")
DEFTIMEVAR (TV_IPA_LTO_COMPRESS , "lto stream deflate")
DEFTIMEVAR (TV_IPA_LTO_DECOMPRESS , "lto stream decompression")
DEFTIMEVAR (TV_IPA_LTO_COMPRESS , "lto stream compression")
DEFTIMEVAR (TV_IPA_LTO_OUTPUT , "lto stream output")
DEFTIMEVAR (TV_IPA_LTO_GIMPLE_IN , "ipa lto gimple in")
DEFTIMEVAR (TV_IPA_LTO_GIMPLE_OUT , "ipa lto gimple out")
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment