From 349bbd13bccea348d2abae1408a5c5739825deee Mon Sep 17 00:00:00 2001
From: Rafael Kitover <rkitover@gmail.com>
Date: Mon, 21 Nov 2016 12:37:21 -0800
Subject: [PATCH] add LTO support, fix libssp link, add -ggdb3 -Og

Add support for Link Time Optimization (LTO) on gcc and Clang.

Link libssp only when it is on the system, some toolchains like Ubuntu's
do not require explicitly linking to libssp. On Win32 try to link it
statically.

In debug builds with GCC, use -ggdb3 -Og, otherwise use -g .

Two new cmake modules were written for this:

* FindSSP.cmake -- this is for finding libssp on the system.

* UseGCCBinUtilsWrappers.cmake -- this is for using gcc binutils
wrappers such as gcc-ar, which is required for gcc LTO to work.

These will be distributed separately as well.
---
 CMakeLists.txt                            | 56 +++++++++++---
 CMakeScripts/FindSSP.cmake                | 87 +++++++++++++++++++++
 CMakeScripts/UseGCCBinUtilsWrappers.cmake | 92 +++++++++++++++++++++++
 3 files changed, 226 insertions(+), 9 deletions(-)
 create mode 100644 CMakeScripts/FindSSP.cmake
 create mode 100644 CMakeScripts/UseGCCBinUtilsWrappers.cmake

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 01a1a3d7..a549855a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -7,6 +7,8 @@ IF(CMAKE_BUILD_TYPE STREQUAL "")
     SET(CMAKE_BUILD_TYPE "Release" CACHE STRING "Build Type" FORCE)
 ENDIF()
 
+SET(ALL_TARGETS fex gvbam visualboyadvance-m vbamcore vbam)
+
 if( COMMAND cmake_policy )
     cmake_policy( SET CMP0003 NEW )
     cmake_policy( SET CMP0005 OLD )
@@ -289,8 +291,21 @@ ENDIF()
 # Compiler flags
 
 IF(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
+    # common optimization flags
+    SET(MY_C_AND_CXX_OPT_FLAGS      -O2 -mtune=generic -fomit-frame-pointer -flto=10)
+    SET(MY_C_AND_CXX_OPT_FLAGS_STR "-O2 -mtune=generic -fomit-frame-pointer -flto=10")
+
+    # common debug flags
+    IF(CMAKE_COMPILER_IS_GNUCXX)
+        SET(MY_C_AND_CXX_DBG_FLAGS      -ggdb3 -Og)
+        SET(MY_C_AND_CXX_DBG_FLAGS_STR "-ggdb3 -Og")
+    ELSE()
+        SET(MY_C_AND_CXX_DBG_FLAGS      -g)
+        SET(MY_C_AND_CXX_DBG_FLAGS_STR "-g")
+    ENDIF()
+
     # common flags
-    SET(MY_C_AND_CXX_FLAGS -mtune=generic -pipe -fPIC -Wformat -Wformat-security -fomit-frame-pointer -fstack-protector-strong --param ssp-buffer-size=4 -fexceptions -D_FORTIFY_SOURCE=2 -feliminate-unused-debug-types)
+    SET(MY_C_AND_CXX_FLAGS -pipe -fPIC -Wformat -Wformat-security -fstack-protector-strong --param ssp-buffer-size=4 -fexceptions -D_FORTIFY_SOURCE=2 -feliminate-unused-debug-types)
 
     SET(MY_C_FLAGS   ${MY_C_FLAGS}   ${MY_C_AND_CXX_FLAGS})
     SET(MY_CXX_FLAGS ${MY_CXX_FLAGS} ${MY_C_AND_CXX_FLAGS})
@@ -300,12 +315,12 @@ IF(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
         SET(MY_CXX_FLAGS ${MY_CXX_FLAGS} -static-libgcc -static-libstdc++)
     ENDIF(MINGW)
 
-    IF(CMAKE_BUILD_TYPE STREQUAL "Debug")
-        SET(MY_C_FLAGS   ${MY_C_FLAGS}   -g2 -Wall)
-        SET(MY_CXX_FLAGS ${MY_CXX_FLAGS} -g2 -Wall)
+    IF(CMAKE_BUILD_TYPE STREQUAL Debug)
+        SET(MY_C_FLAGS   ${MY_C_FLAGS}   ${MY_C_AND_CXX_DBG_FLAGS} -Wall)
+        SET(MY_CXX_FLAGS ${MY_CXX_FLAGS} ${MY_C_AND_CXX_DBG_FLAGS} -Wall)
     ELSE()
-        SET(MY_C_FLAGS   ${MY_C_FLAGS}   -O2 -Wno-error)
-        SET(MY_CXX_FLAGS ${MY_CXX_FLAGS} -O2 -Wno-error)
+        SET(MY_C_FLAGS   ${MY_C_FLAGS}   ${MY_C_AND_CXX_OPT_FLAGS} -Wno-error)
+        SET(MY_CXX_FLAGS ${MY_CXX_FLAGS} ${MY_C_AND_CXX_OPT_FLAGS} -Wno-error)
     ENDIF()
 
     FOREACH(C_COMPILE_FLAG ${MY_C_FLAGS})
@@ -316,11 +331,34 @@ IF(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
         ADD_COMPILE_OPTIONS($<$<COMPILE_LANGUAGE:CXX>:${CXX_COMPILE_FLAG}>)
     ENDFOREACH()
 
+    # need opt flags for link step for LTO, including -flto
+    # but don't use LTO for debug builds, too slow
+    IF(CMAKE_BUILD_TYPE STREQUAL Debug)
+        SET(CMAKE_CXX_LINK_EXECUTABLE "${CMAKE_CXX_LINK_EXECUTABLE} ${MY_C_AND_CXX_DBG_FLAGS_STR}")
+        SET(CMAKE_C_LINK_EXECUTABLE   "${CMAKE_C_LINK_EXECUTABLE}   ${MY_C_AND_CXX_DBG_FLAGS_STR}")
+    ELSE()
+        SET(CMAKE_CXX_LINK_EXECUTABLE "${CMAKE_CXX_LINK_EXECUTABLE} ${MY_C_AND_CXX_OPT_FLAGS_STR}")
+        SET(CMAKE_C_LINK_EXECUTABLE   "${CMAKE_C_LINK_EXECUTABLE}   ${MY_C_AND_CXX_OPT_FLAGS_STR}")
+    ENDIF()
+
     # for the gcc -fstack-protector* flags we need libssp
-    # clang does not have this
+    # we also have to use the gcc- binutils for LTO to work
     IF(CMAKE_COMPILER_IS_GNUCXX)
-        SET(CMAKE_CXX_LINK_EXECUTABLE "${CMAKE_CXX_LINK_EXECUTABLE} -lssp")
-        SET(CMAKE_C_LINK_EXECUTABLE   "${CMAKE_C_LINK_EXECUTABLE}   -lssp")
+        INCLUDE(UseGCCBinUtilsWrappers)
+
+        IF(WIN32)
+            SET(SSP_STATIC ON)
+        ENDIF(WIN32)
+
+        FIND_PACKAGE(SSP)
+
+        IF(SSP_LIBRARY)
+            SET(CMAKE_CXX_LINK_EXECUTABLE "${CMAKE_CXX_LINK_EXECUTABLE} ${SSP_LIBRARY}")
+            SET(CMAKE_C_LINK_EXECUTABLE   "${CMAKE_C_LINK_EXECUTABLE}   ${SSP_LIBRARY}")
+        ENDIF(SSP_LIBRARY)
+
+        SET(CMAKE_CXX_LINK_EXECUTABLE "${CMAKE_CXX_LINK_EXECUTABLE} -Wl,-allow-multiple-definition")
+        SET(CMAKE_C_LINK_EXECUTABLE   "${CMAKE_C_LINK_EXECUTABLE}   -Wl,-allow-multiple-definition")
     ENDIF()
 ENDIF()
 
diff --git a/CMakeScripts/FindSSP.cmake b/CMakeScripts/FindSSP.cmake
new file mode 100644
index 00000000..3008f3a8
--- /dev/null
+++ b/CMakeScripts/FindSSP.cmake
@@ -0,0 +1,87 @@
+# FindSSP.cmake
+#
+# Find libssp necessary when using gcc with e.g. -fstack-protector=strong
+#
+# See: http://wiki.osdev.org/Stack_Smashing_Protector
+#
+# To use:
+#
+# put a copy into your <project_root>/CMakeScripts/
+#
+# In your main CMakeLists.txt do something like this:
+#
+# IF(WIN32)
+#     SET(SSP_STATIC ON)
+# ENDIF(WIN32)
+#
+# FIND_PACKAGE(SSP)
+#
+# IF(SSP_LIBRARY)
+#     SET(CMAKE_CXX_LINK_EXECUTABLE "${CMAKE_CXX_LINK_EXECUTABLE} ${SSP_LIBRARY}")
+#     SET(CMAKE_C_LINK_EXECUTABLE   "${CMAKE_C_LINK_EXECUTABLE}   ${SSP_LIBRARY}")
+# ENDIF(SSP_LIBRARY)
+#
+# BSD 2-Clause License
+#
+# Copyright (c) 2016, Rafael Kitover
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright notice, this
+#   list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+# only do this when compiling with gcc/g++
+IF(NOT (CMAKE_COMPILER_IS_GNUCXX OR CMAKE_COMPILER_IS_GNUC))
+    RETURN()
+ENDIF()
+
+GET_FILENAME_COMPONENT(GCC_DIRNAME ${CMAKE_C_COMPILER} DIRECTORY)
+
+EXECUTE_PROCESS(COMMAND ${CMAKE_C_COMPILER} --print-libgcc-file-name OUTPUT_VARIABLE LIBGCC_FILE OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+GET_FILENAME_COMPONENT(LIBGCC_DIRNAME ${LIBGCC_FILE} DIRECTORY)
+
+SET(SSP_SEARCH_PATHS ${GCC_DIRNAME} ${LIBGCC_DIRNAME})
+
+SET(CURRENT_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES})
+
+IF(SSP_STATIC)
+    IF(WIN32)
+        SET(CMAKE_FIND_LIBRARY_SUFFIXES .lib .a)
+    ELSE(WIN32)
+        SET(CMAKE_FIND_LIBRARY_SUFFIXES .a)
+    ENDIF(WIN32)
+ENDIF(SSP_STATIC)
+
+FIND_LIBRARY(SSP_LIBRARY
+    NAMES ssp libssp
+    HINTS ${SSP_SEARCH_PATHS}
+    PATH_SUFFIXES lib64 lib lib/x64 lib/x86
+)
+
+SET(CMAKE_FIND_LIBRARY_SUFFIXES ${CURRENT_FIND_LIBRARY_SUFFIXES})
+
+FOREACH(VAR GCC_DIRNAME LIBGCC_FILE LIBGCC_DIRNAME SSP_SEARCH_PATHS CURRENT_FIND_LIBRARY_SUFFIXES)
+    UNSET(${VAR})
+ENDFOREACH()
+
+INCLUDE(FindPackageHandleStandardArgs)
+
+FIND_PACKAGE_HANDLE_STANDARD_ARGS(SSP REQUIRED_VARS SSP_LIBRARY)
diff --git a/CMakeScripts/UseGCCBinUtilsWrappers.cmake b/CMakeScripts/UseGCCBinUtilsWrappers.cmake
new file mode 100644
index 00000000..bae1a24b
--- /dev/null
+++ b/CMakeScripts/UseGCCBinUtilsWrappers.cmake
@@ -0,0 +1,92 @@
+# UseGCCBinUtilsWrappers.cmake
+#
+# Use gcc binutils wrappers such as gcc-ar, this may be necessary for LTO.
+#
+# To use:
+#
+# put a copy into your <project_root>/CMakeScripts/
+#
+# In your main CMakeLists.txt add the command:
+#
+# INCLUDE(UseGCCBinUtilsWrappers)
+#
+# BSD 2-Clause License
+#
+# Copyright (c) 2016, Rafael Kitover
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright notice, this
+#   list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+# only do this when compiling with gcc/g++
+IF(NOT (CMAKE_COMPILER_IS_GNUCXX OR CMAKE_COMPILER_IS_GNUC))
+    RETURN()
+ENDIF()
+
+# first try appending -util to basename of compiler
+STRING(REGEX MATCH   "\\.(exe|EXE)$"    GCC_EXE_SUFFIX ${CMAKE_C_COMPILER})
+STRING(REGEX REPLACE "\\.(exe|EXE)$" "" GCC_BASENAME   ${CMAKE_C_COMPILER})
+
+SET(GCC_AR      "${GCC_BASENAME}-ar${GCC_EXE_SUFFIX}")
+SET(GCC_NM      "${GCC_BASENAME}-nm${GCC_EXE_SUFFIX}")
+SET(GCC_RANLIB  "${GCC_BASENAME}-ranlib${GCC_EXE_SUFFIX}")
+
+# if that does not work, try looking for gcc-util in the compiler directory,
+# and failing that in the PATH
+
+GET_FILENAME_COMPONENT(GCC_DIRNAME ${CMAKE_C_COMPILER} DIRECTORY)
+
+IF(NOT EXISTS ${GCC_AR})
+    UNSET(GCC_AR)
+
+    FIND_PROGRAM(GCC_AR NAMES gcc-ar gcc-ar.exe GCC-AR.EXE HINTS ${GCC_DIRNAME})
+ENDIF()
+
+IF(NOT EXISTS ${GCC_NM})
+    UNSET(GCC_NM)
+
+    FIND_PROGRAM(GCC_NM NAMES gcc-nm gcc-nm.exe GCC-NM.EXE HINTS ${GCC_DIRNAME})
+ENDIF()
+
+IF(NOT EXISTS ${GCC_RANLIB})
+    UNSET(GCC_RANLIB)
+
+    FIND_PROGRAM(GCC_RANLIB NAMES gcc-ranlib gcc-ranlib.exe GCC-RANLIB.EXE HINTS ${GCC_DIRNAME})
+ENDIF()
+
+IF(EXISTS ${GCC_AR})
+    SET(CMAKE_AR ${GCC_AR})
+    MESSAGE("-- Found gcc-ar: ${CMAKE_AR}")
+ENDIF()
+
+IF(EXISTS ${GCC_NM})
+    SET(CMAKE_NM ${GCC_NM})
+    MESSAGE("-- Found gcc-nm: ${CMAKE_NM}")
+ENDIF()
+
+IF(EXISTS ${GCC_RANLIB})
+    SET(CMAKE_RANLIB ${GCC_RANLIB})
+    MESSAGE("-- Found gcc-ranlib: ${CMAKE_RANLIB}")
+ENDIF()
+
+FOREACH(VAR "GCC_AR" "GCC_NM" "GCC_RANLIB" "GCC_DIRNAME" "GCC_BASENAME" "GCC_EXE_SUFFIX")
+    UNSET(${VAR})
+ENDFOREACH()