builder: msys2 fixes + misc improvements

- Fix leading spaces in build env vars.

- Pass -Dcc="$CC" to the c2man metaconfig, it tries to use cc on msys2
  which fails.

- Pass LDFLAGS to openssl make to fix link errors.

- Change the host_env, target_env and related functions to output a list
  of commands for eval.

- Add a --host-env flag to output the host env for cross build
  environments.

- Ignore symlink failures when installing dists, symlinks fail on msys2
  sometimes for some reason.

- Put libicu in the list of both host and target dists only in
  mingw-cross, the host libicu does not build on msys2, and everything
  works fine without it.

- Add -j1 to openssl make args to disable parallel builds for the msys2
  and cygwin host builds, because they fail otherwise.

- Add "-ldl -lcatgets -lws2_32" to libicu LDFLAGS for the target build,
  ws2_32 may be necessary for 32 bit builds.

tools/builder/core.sh

@@ -43,11 +43,11 @@ esac
 export CC_ORIG="\${CC_ORIG:-\$CC}"
 export CXX_ORIG="\${CXX_ORIG:-\$CXX}"
 
-export CPPFLAGS="$CPPFLAGS -I$BUILD_ROOT/root/include"
+export CPPFLAGS="$CPPFLAGS${CPPFLAGS:+ }-I$BUILD_ROOT/root/include"
-export CFLAGS="$CFLAGS -fPIC -I$BUILD_ROOT/root/include -L$BUILD_ROOT/root/lib -pthread -lm"
+export CFLAGS="$CFLAGS${CFLAGS:+ }-fPIC -I$BUILD_ROOT/root/include -L$BUILD_ROOT/root/lib -pthread -lm"
-export CXXFLAGS="$CXXFLAGS -fPIC -I$BUILD_ROOT/root/include -L$BUILD_ROOT/root/lib -std=gnu++11 -fpermissive -pthread -lm"
+export CXXFLAGS="$CXXFLAGS${CXXFLAGS:+ }-fPIC -I$BUILD_ROOT/root/include -L$BUILD_ROOT/root/lib -std=gnu++11 -fpermissive -pthread -lm"
-export OBJCXXFLAGS="$OBJCXXFLAGS -fPIC -I$BUILD_ROOT/root/include -L$BUILD_ROOT/root/lib -std=gnu++11 -fpermissive -pthread -lm"
+export OBJCXXFLAGS="$OBJCXXFLAGS${OBJCXXFLAGS:+ }-fPIC -I$BUILD_ROOT/root/include -L$BUILD_ROOT/root/lib -std=gnu++11 -fpermissive -pthread -lm"
-export LDFLAGS="$LDFLAGS -fPIC -L$BUILD_ROOT/root/lib -pthread -lm"
+export LDFLAGS="$LDFLAGS${LDFLAGS:+ }-fPIC -L$BUILD_ROOT/root/lib -pthread -lm"
 export STRIP="${STRIP:-strip}"
 
 if [ -z "\$OPENMP" ] && echo "\$CC" | grep -Eq gcc; then
@@ -336,7 +336,7 @@ DIST_CONFIGURE_OVERRIDES="$DIST_CONFIGURE_OVERRIDES
 "
 
 DIST_BUILD_OVERRIDES="$DIST_BUILD_OVERRIDES
-    c2man          ./Configure -de -Dprefix=/usr -Dmansrc=/usr/share/man/man1; \
+    c2man          ./Configure -de -Dprefix=/usr -Dmansrc=/usr/share/man/man1 -Dcc=\"\$CC\"; \
                    sed -i.bak 's|/[^ ][^ ]*/libfl\\.[^ ]*|-L$BUILD_ROOT/root/lib -lfl|' Makefile; \
                    make -j\$NUM_CPUS; \
                    make install bin='$BUILD_ROOT/root/bin' mansrc='$BUILD_ROOT/root/share/man/man1' privlib='$BUILD_ROOT/root/lib/c2man'
@@ -405,7 +405,7 @@ ALL_MAKE_ARGS='V=1 VERBOSE=1'
 
 # have to disable ccache for openssl
 DIST_MAKE_ARGS="$DIST_MAKE_ARGS
-    openssl     CC=\"\$CC_ORIG -fPIC\" CXX=\"\$CXX_ORIG -fPIC\"
+    openssl     CC=\"\$CC\" CXX=\"\$CXX\" LDFLAGS=\"\$LDFLAGS\"
     getopt      LDFLAGS=\"\$LDFLAGS -lintl -liconv\" CFLAGS=\"\$CFLAGS\"
     bzip2       libbz2.a bzip2 bzip2recover CFLAGS=\"\$CFLAGS\" LDFLAGS=\"\$LDFLAGS\"
     unzip       generic2
@@ -461,10 +461,15 @@ builder() {
 
 read_command_line() {
     case "$1" in
-        --env)
+        --env|--target-env)
             puts "$BUILD_ENV"
             exit 0
             ;;
+        --host-env)
+            puts "$BUILD_ENV"
+            host_env 2>/dev/null || :
+            exit 0
+            ;;
         --clean)
             rm -rf "$BUILD_ROOT/dists/*"
             unpack_dists
@@ -1871,12 +1876,12 @@ EOF
             link_dest=$(fully_resolve_link "$link_dest")
 
             if [ -e "$link_dest" ]; then
-                echo_run ln -sf "$link_dest" "$dest_f"
+                echo_run ln -sf "$link_dest" "$dest_f" || :
             else
                 # this is for windows as well, where symlinks can't point to a
                 # file that doesn't (yet) exist
                 defer_cmds="$defer_cmds
-ln -sf \"$link_dest\" \"$dest_f\"
+ln -sf \"$link_dest\" \"$dest_f\" || :
 "
             fi
 
@@ -1954,12 +1959,16 @@ ln -sf \"$link_dest\" \"$dest_f\"
     wait_all_jobs running_jobs
     cleanup_jobs
 
+    if [ -n "$defer_cmds" ]; then
+        message "making deferred links..."
+
         IFS=$NL
         for cmd in $defer_cmds; do
             IFS=$OIFS
             eval echo_run "$cmd"
         done
         IFS=$OIFS
+    fi
 
     cd "$OLDPWD"
 
@@ -2548,6 +2557,10 @@ warn() {
     puts >&2 "${NL}WARNING: $@${NL}${NL}"
 }
 
+message() {
+    puts >&2 "${NL}INFO: $@${NL}${NL}"
+}
+
 die() {
     error "$@"
     exit 1

tools/builder/mingw-cross.sh

@@ -35,7 +35,7 @@ host_dists='
 
 both_dists='
     openssl zlib bzip2 libiconv gettext xz libxml2 expat libpng freetype
-    fontconfig
+    fontconfig libicu
 '
 
 [ -n "$BUILD_ENV" ] && eval "$BUILD_ENV"
@@ -46,10 +46,10 @@ export CC='${target_arch}-gcc'
 export CXX='${target_arch}-g++'
 export STRIP='${target_arch}-strip'
 
-export CFLAGS="\$CFLAGS -L/usr/${target_arch}/usr/lib${lib_suffix}"
 export CPPFLAGS="\$CPPFLAGS"
-export CXXFLAGS="\$CXXFLAGS -L/usr/${target_arch}/usr/lib${lib_suffix}"
+export CFLAGS="\$CFLAGS\${CFLAGS:+ }-L/usr/${target_arch}/usr/lib${lib_suffix}"
-export OBJCXXFLAGS="\$OBJCXXFLAGS -L/usr/${target_arch}/usr/lib${lib_suffix}"
+export CXXFLAGS="\$CXXFLAGS\${CXXFLAGS:+ }-L/usr/${target_arch}/usr/lib${lib_suffix}"
+export OBJCXXFLAGS="\$OBJCXXFLAGS\${OBJCXXFLAGS:+ }-L/usr/${target_arch}/usr/lib${lib_suffix}"
 export LDFLAGS="-L/usr/${target_arch}/usr/lib${lib_suffix} \$LDFLAGS"
 
 EOF
@@ -68,6 +68,8 @@ table_line_replace DIST_CONFIGURE_OVERRIDES openssl-target "./Configure $openssl
 
 table_line_append DIST_PRE_BUILD bzip2-target ':; sed -i.bak '\''s,include <sys\\stat.h>,include <sys/stat.h>,g'\'' *.c;'
 
+table_line_append DIST_ARGS libicu-target "--with-cross-build=$BUILD_ROOT/dists/libicu/source"
+
 table_line_replace DIST_POST_BUILD harfbuzz "$(table_line DIST_POST_BUILD harfbuzz | sed 's/rebuild_dist freetype /rebuild_dist freetype-target /')"
 
 table_line_replace DIST_POST_BUILD glib     "$(table_line DIST_POST_BUILD glib     | sed 's/rebuild_dist gettext /rebuild_dist gettext-target /')"

tools/builder/mingw.sh

@@ -8,11 +8,11 @@ CROSS_OS=windows
 
 BUILD_ENV=$BUILD_ENV$(cat <<EOF
 
-export CPPFLAGS="$CPPFLAGS -DMINGW_HAS_SECURE_API"
+export CPPFLAGS="$CPPFLAGS${CPPFLAGS:+ }-DMINGW_HAS_SECURE_API"
-export CFLAGS="$CFLAGS -static-libgcc -static-libstdc++ -static -lpthread -DMINGW_HAS_SECURE_API -lm"
+export CFLAGS="$CFLAGS${CFLAGS:+ }-static-libgcc -static-libstdc++ -static -lpthread -DMINGW_HAS_SECURE_API -lm"
-export CXXFLAGS="$CXXFLAGS -static-libgcc -static-libstdc++ -static -lpthread -DMINGW_HAS_SECURE_API -lm"
+export CXXFLAGS="$CXXFLAGS${CXXFLAGS:+ }-static-libgcc -static-libstdc++ -static -lpthread -DMINGW_HAS_SECURE_API -lm"
-export OBJCXXFLAGS="$OBJCXXFLAGS -static-libgcc -static-libstdc++ -static -lpthread -DMINGW_HAS_SECURE_API -lm"
+export OBJCXXFLAGS="$OBJCXXFLAGS${OBJCXXFLAGS:+ }-static-libgcc -static-libstdc++ -static -lpthread -DMINGW_HAS_SECURE_API -lm"
-export LDFLAGS="$LDFLAGS -static-libgcc -static-libstdc++ -static -lpthread -DMINGW_HAS_SECURE_API -lm"
+export LDFLAGS="$LDFLAGS${LDFLAGS:+ }-static-libgcc -static-libstdc++ -static -lpthread -DMINGW_HAS_SECURE_API -lm"
 export LIBS="-lpthread -lm"
 
 export UUID_LIBS="-luuid_mingw -luuid"
@@ -63,7 +63,7 @@ host_dists="$host_dists autoconf autoconf-archive automake m4 gsed bison \
                         flex-2.6.3 flex c2man docbook2x ccache"
 host_dists=$(list_remove_duplicates $host_dists)
 
-both_dists="$both_dists openssl zlib bzip2 libiconv libicu"
+both_dists="$both_dists openssl zlib bzip2 libiconv"
 
 if [ "$os" != windows ]; then
     both_dists="$both_dists libuuid"
@@ -71,10 +71,11 @@ fi
 
 both_dists=$(list_remove_duplicates $both_dists)
 
-set_host_env() {
+host_env() {
     rm -f "$BUILD_ROOT/root"
     ln -sf "$BUILD_ROOT/host" "$BUILD_ROOT/root"
     if [ -z "$OCC" ]; then
+        cat <<EOF
 OCC=$CC
 OCXX=$CXX
 OCC_ORIG=$CC_ORIG
@@ -108,16 +109,18 @@ set_host_env() {
 
 REQUIRED_CONFIGURE_ARGS=$(puts "$REQUIRED_CONFIGURE_ARGS" | sed 's/--host[^ ]*//g')
 REQUIRED_CMAKE_ARGS=$(puts "$REQUIRED_CMAKE_ARGS" | sed 's/-DCMAKE_TOOLCHAIN_FILE=[^ ]*//g')
+EOF
     fi
 
-    set_host_env_hook 2>/dev/null || :
+    host_env_hook 2>/dev/null || :
 }
 
-unset_host_env() {
+target_env() {
     rm -f "$BUILD_ROOT/root"
     ln -sf "$BUILD_ROOT/target" "$BUILD_ROOT/root"
 
     if [ -n "$OCC" ]; then
+        cat <<EOF
 export CC="$OCC"
 export CXX="$OCXX"
 export CC_ORIG="$OCC_ORIG"
@@ -136,6 +139,7 @@ unset_host_env() {
 REQUIRED_CONFIGURE_ARGS=$OREQUIRED_CONFIGURE_ARGS
 REQUIRED_CMAKE_ARGS=$OREQUIRED_CMAKE_ARGS
 OREQUIRED_CONFIGURE_ARGS= OREQUIRED_CMAKE_ARGS=
+EOF
     fi
 
     # make links to executables in the target as well
@@ -149,7 +153,7 @@ unset_host_env() {
     done
     IFS=$OIFS
 
-    unset_host_env_hook 2>/dev/null || :
+    target_env_hook 2>/dev/null || :
 }
 
 # replace install artifact paths with absolute paths into host and target trees
@@ -163,7 +167,7 @@ pre_build_all() {
         set -- $dist
 
         case "$(table_line DIST_PRE_BUILD "$1")" in
-            *set_host_env*)
+            *host_env*)
                 path="$BUILD_ROOT/host/$3"
                 ;;
             *)
@@ -179,15 +183,15 @@ pre_build_all() {
 }
 
 for dist in $host_dists $perl_dists; do
-    table_line_append  DIST_PRE_BUILD  $dist ':; set_host_env;'
+    table_line_append  DIST_PRE_BUILD  $dist ':; eval "$(host_env)";'
-    table_line_replace DIST_POST_BUILD $dist "unset_host_env; $(table_line DIST_POST_BUILD $dist)"
+    table_line_replace DIST_POST_BUILD $dist "eval \"\$(target_env)\"; $(table_line DIST_POST_BUILD $dist)"
 done
 
 for dist in $both_dists; do
     duplicate_dist $dist "${dist}-target"
 
-    table_line_append  DIST_PRE_BUILD  $dist ':; set_host_env;'
+    table_line_append  DIST_PRE_BUILD  $dist ':; eval "$(host_env)";'
-    table_line_replace DIST_POST_BUILD $dist "unset_host_env; $(table_line DIST_POST_BUILD $dist)"
+    table_line_replace DIST_POST_BUILD $dist "eval \"\$(target_env)\"; $(table_line DIST_POST_BUILD $dist)"
 done
 
 remove_dists='graphviz python2 python3 swig libxml2-python doxygen bakefile setuptools pip meson XML-Parser intltool ninja libsecret shared-mime-info'
@@ -236,9 +240,13 @@ table_insert_after DISTS zlib "dlfcn https://github.com/dlfcn-win32/dlfcn-win32/
 
 table_line_replace DIST_CONFIGURE_TYPES dlfcn cmake
 
-table_line_append DIST_ARGS libicu-target "--with-cross-build=$BUILD_ROOT/dists/libicu/source"
+libicu=libicu
 
-table_line_append DIST_PATCHES libicu-target " \
+if [ -n "$(table_line DISTS libicu-target || :)" ]; then
+    libicu=libicu-target
+fi
+
+table_line_append DIST_PATCHES $libicu " \
     https://raw.githubusercontent.com/Alexpux/MINGW-packages/master/mingw-w64-icu/0004-move-to-bin.mingw.patch \
     https://raw.githubusercontent.com/Alexpux/MINGW-packages/master/mingw-w64-icu/0007-actually-move-to-bin.mingw.patch \
     https://raw.githubusercontent.com/Alexpux/MINGW-packages/master/mingw-w64-icu/0008-data-install-dir.mingw.patch \
@@ -252,6 +260,8 @@ table_line_append DIST_PATCHES libicu-target " \
     https://raw.githubusercontent.com/Alexpux/MINGW-packages/master/mingw-w64-icu/0021-mingw-static-libraries-without-s.patch \
 "
 
+table_line_append DIST_EXTRA_LDFLAGS $libicu "-ldl -lcatgets -lws2_32"
+
 table_insert_after DISTS libiconv-target '
     catgets         https://downloads.sourceforge.net/project/mingw/MinGW/Extension/catgets/mingw-catgets-1.0.1/mingw-catgets-1.0.1-src.tar.gz    include/langinfo.h
 '

tools/win/cygwin-cross-builder

@@ -30,6 +30,9 @@ REQUIRED_CMAKE_ARGS="-DGETTEXT_MSGFMT_EXECUTABLE:FILEPATH=\"\$BUILD_ROOT/host/bi
 
 table_line_append DIST_MAKE_ARGS unzip 'CFLAGS="$CFLAGS -DNO_LCHMOD"'
 
+# parallel build of openssl doesn't work for cygwin layer
+table_line_append DIST_MAKE_ARGS openssl -j1
+
 # FIXME: problems with autopoint that need to be sorted
 table_line_remove DISTS flex
 

tools/win/mac-cross-builder

@@ -6,7 +6,8 @@ TAR=tar
 
 . "${0%/*}/../builder/mingw-cross.sh"
 
-set_host_env_hook() {
+host_env_hook() {
+    cat <<EOF
 export COMMAND_MODE=unix2003
 export CC='ccache clang'
 export CXX='ccache clang++'
@@ -17,10 +18,13 @@ set_host_env_hook() {
 export CXXFLAGS="-fPIC -I$BUILD_ROOT/root/include -L$BUILD_ROOT/root/lib -std=gnu++11 -fpermissive -stdlib=libc++ -framework Carbon -framework Foundation -framework CoreServices -Wno-unused-command-line-argument -DICONV_CONST= -Wl,-no_compact_unwind"
 export OBJCXXFLAGS="-fPIC -I$BUILD_ROOT/root/include -L$BUILD_ROOT/root/lib -std=gnu++11 -fpermissive -stdlib=libc++ -framework Carbon -framework Foundation -framework CoreServices -Wno-unused-command-line-argument -DICONV_CONST= -Wl,-no_compact_unwind"
 export LDFLAGS="-fPIC -L$BUILD_ROOT/root/lib -framework Carbon -framework Foundation -framework CoreServices -Wno-unused-command-line-argument -Wl,-no_compact_unwind"
+EOF
 }
 
-unset_host_env_hook() {
+target_env_hook() {
+    cat <<EOF
 unset COMMAND_MODE
+EOF
 }
 
 table_line_remove DISTS flex

tools/win/msys2-builder

@@ -104,25 +104,35 @@ DIST_TAR_ARGS="
 
 . "${0%/*}/../builder/mingw.sh"
 
-set_host_env() {
+host_env() {
     ln -sf "$BUILD_ROOT/host" "$BUILD_ROOT/root"
 
+    cat <<EOF
 unset MSYSTEM_PREFIX MSYSTEM_CARCH MSYSTEM_CHOST MINGW_CHOST MINGW_PREFIX MINGW_PACKAGE_PREFIX CONFIG_SITE ACLOCAL_PATH PKG_CONFIG_PATH
 
 OPWD=$PWD
-    MSYSTEM=MSYS . /etc/profile || :
+export MSYSTEM=MSYS
-    eval "$BUILD_ENV"
+. /etc/profile || :
+. ~/.bashrc 2>/dev/null || :
+$BUILD_ENV
+EOF
+
     cd "$OPWD"
 }
 
-unset_host_env() {
+target_env() {
     ln -sf "$BUILD_ROOT/target" "$BUILD_ROOT/root"
 
+    cat <<EOF
 unset MSYSTEM_PREFIX MSYSTEM_CARCH MSYSTEM_CHOST MINGW_CHOST MINGW_PREFIX MINGW_PACKAGE_PREFIX CONFIG_SITE ACLOCAL_PATH PKG_CONFIG_PATH
 
 OPWD=$PWD
-    MSYSTEM=$OMSYSTEM . /etc/profile || :
+export MSYSTEM=$OMSYSTEM
-    eval "$BUILD_ENV"
+. /etc/profile || :
+. ~/.bashrc 2>/dev/null || :
+$BUILD_ENV
+EOF
+
     cd "$OPWD"
 }
 
@@ -171,4 +181,7 @@ table_line_replace DIST_POST_BUILD unzip ":; \
 
 table_line_replace DIST_CONFIGURE_OVERRIDES openssl "$(table_line DIST_CONFIGURE_OVERRIDES openssl | sed 's,^./config ,./Configure Cygwin-x86_64 ,')"
 
+# parallel build of openssl doesn't work for msys2 layer
+table_line_append DIST_MAKE_ARGS openssl -j1
+
 builder "$@"