Bug#27470071 ERROR C1083: CANNOT OPEN INCLUDE FILE: 'LEX_TOKEN.H'

This is a bacport to 5.6 of four bugfixes in 5.7

    Bug#20678411 BROKEN MAKEFILE DEPENDENCY: SQL_YACC.YY AND LEX_TOKEN.H
    Add explicit dependency sql_yacc.cc.o => lex_token.h

    Bug#19060850: SQL_YACC.CC REGENERATED TWICE IN WINDOWS BUILD, CAN LEAD TO BUILD BREAK
    Run Bison separately for libmysqld so that independent copies of
    sql_yacc.h/sql_yacc.cc are generated. This avoids the Windows
    build break because of libmysqld compilation of sql_yacc.cc while
    it is being generated.

    Bug#21680733 WINDOWS (10) BUILD MAY COMPILE SQL_YACC.CC BEFORE LEX_TOKEN.H HAS BEEN GENERATED
    Problem: build may fail non-deterministically when when building in parallel.
    Solution: build local copy lex_token.h, and add explicit dependency for it.

    Bug#24931655 RACE CONDITION BUILDING LEX_HASH.H
    Mysql cluster is experiencing race conditions generating lex_hash.h
    Fix:
    Generate lex_hash.h locally for the embedded server,
    rather than picking it up from the sql/ build directory.

Author: Tor Didriksen

cmake/bison.cmake

@@ -1,5 +1,4 @@
-# Copyright (c) 2009 Sun Microsystems, Inc.
-# Use is subject to license terms.
+# Copyright (c) 2009, 2018, Oracle and/or its affiliates. All rights reserved.
 # 
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -12,18 +11,13 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA 
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA 
 
-IF(CMAKE_SYSTEM_NAME MATCHES "SunOS")
- # On Solaris, /opt/csw often contains a newer bison 
- IF(NOT BISON_EXECUTABLE AND EXISTS /opt/csw/bin/bison)
-   SET(BISON_EXECUTABLE /opt/csw/bin/bison)
- ENDIF()
-ENDIF()
 FIND_PROGRAM(BISON_EXECUTABLE bison DOC "path to the bison executable")
 MARK_AS_ADVANCED(BISON_EXECUTABLE "")
+
 IF(NOT BISON_EXECUTABLE)
-  MESSAGE("Warning: Bison executable not found in PATH")
+  MESSAGE(WARNING "Bison executable not found in PATH")
 ELSEIF(BISON_EXECUTABLE AND NOT BISON_USABLE)
   # Check version as well
   EXEC_PROGRAM(${BISON_EXECUTABLE} ARGS --version OUTPUT_VARIABLE BISON_VERSION_STR)
@@ -33,19 +27,26 @@ ELSEIF(BISON_EXECUTABLE AND NOT BISON_USABLE)
    STRING(REGEX REPLACE ".* ([0-9]+)\\.([0-9]+)" "\\1" BISON_VERSION_MAJOR "${FIRST_LINE}")
    STRING(REGEX REPLACE ".* ([0-9]+)\\.([0-9]+)" "\\2" BISON_VERSION_MINOR "${FIRST_LINE}")
    IF (BISON_VERSION_MAJOR LESS 2)
-     MESSAGE("Warning: bison version is old. please update to version 2")
+     MESSAGE(WARNING "Bison version is old. please update to version 2")
    ELSE()
      SET(BISON_USABLE 1 CACHE INTERNAL "Bison version 2 or higher")
    ENDIF()
 ENDIF()
 
+
+# Handle out-of-source build from source package with possibly broken 
+# bison. Copy bison output to from source to build directory, if not already 
+# there
+MACRO(COPY_BISON_OUTPUT input_cc input_h output_cc output_h)
+  IF(EXISTS ${input_cc} AND NOT EXISTS ${output_cc})
+    CONFIGURE_FILE(${input_cc} ${output_cc} COPYONLY)
+    CONFIGURE_FILE(${input_h}  ${output_h}  COPYONLY)
+  ENDIF()
+ENDMACRO()
+
+
 # Use bison to generate C++ and header file
 MACRO (RUN_BISON input_yy output_cc output_h)
-  IF(BISON_TOO_OLD)
-    IF(EXISTS ${output_cc} AND EXISTS ${output_h})
-      SET(BISON_USABLE FALSE)
-    ENDIF()
-  ENDIF()
   IF(BISON_USABLE)
     ADD_CUSTOM_COMMAND(
       OUTPUT ${output_cc}
@@ -61,7 +62,7 @@ MACRO (RUN_BISON input_yy output_cc output_h)
     IF(EXISTS  ${output_cc} AND EXISTS ${output_h})
       IF(${input_yy} IS_NEWER_THAN ${output_cc}  OR  ${input_yy} IS_NEWER_THAN ${output_h})
         # Possibly timestamps are messed up in source distribution.
-        MESSAGE("Warning: no usable bison found, ${input_yy} will not be rebuilt.")
+        MESSAGE(WARNING "No usable bison found, ${input_yy} will not be rebuilt.")
       ENDIF()
     ELSE()
       # Output files are missing, bail out.

libmysqld/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2006, 2014, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2006, 2018, Oracle and/or its affiliates. All rights reserved.
 # 
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -29,14 +29,26 @@ INCLUDE_DIRECTORIES(
   ${CMAKE_SOURCE_DIR}/sql/backup
 )
 
+# We should generate these separately for libmysqld to avoid
+# compiling them for libmysqld while they are generated for sql.
+SET(GEN_YACC_SOURCES
+  ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc.h 
+  ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc.cc
+)
+
 SET(GEN_SOURCES
-  ${CMAKE_BINARY_DIR}/sql/sql_yacc.h 
-  ${CMAKE_BINARY_DIR}/sql/sql_yacc.cc
   ${CMAKE_BINARY_DIR}/sql/sql_builtin.cc
-  ${CMAKE_BINARY_DIR}/sql/lex_hash.h 
 )
 
-SET_SOURCE_FILES_PROPERTIES(${GEN_SOURCES} PROPERTIES GENERATED TRUE)
+ADD_CUSTOM_COMMAND(
+  OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/lex_hash.h
+  COMMAND gen_lex_hash > lex_hash.h
+  DEPENDS gen_lex_hash
+)
+
+SET_SOURCE_FILES_PROPERTIES(${GEN_YACC_SOURCES}
+                            ${GEN_SOURCES}
+                            PROPERTIES GENERATED TRUE)
 
 FOREACH(file ${SQL_EXPORTED_SOURCES})
   LIST(APPEND IMPORTED_SOURCES "../sql/${file}")
@@ -48,6 +60,7 @@ SET(SQL_EMBEDDED_SOURCES
   libmysqld.c
   ${GEN_SOURCES}
   ${MYSYS_LIBWRAP_SOURCE}
+  ${GEN_YACC_SOURCES}
   ../client/get_password.c
   ../libmysql/errmsg.c
   ../libmysql/libmysql.c
@@ -79,9 +92,45 @@ SET(SQL_EMBEDDED_SOURCES
 )
 
 
+
+# Handle out-of-source build from source package with possibly broken 
+# bison. Copy bison output to from source to build directory, if not already 
+# there
+INCLUDE(${CMAKE_SOURCE_DIR}/cmake/bison.cmake)
+COPY_BISON_OUTPUT(
+  ${CMAKE_SOURCE_DIR}/sql/sql_yacc.cc
+  ${CMAKE_SOURCE_DIR}/sql/sql_yacc.h
+  ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc.cc
+  ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc.h
+)
+
+RUN_BISON(
+  ${CMAKE_SOURCE_DIR}/sql/sql_yacc.yy 
+  ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc.cc
+  ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc.h
+)
+
+ADD_CUSTOM_COMMAND(
+  OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/lex_token.h
+  COMMAND gen_lex_token > lex_token.h
+  DEPENDS gen_lex_token
+)
+
+SET_SOURCE_FILES_PROPERTIES(
+  ${CMAKE_CURRENT_BINARY_DIR}/sql/lex_token.h
+  PROPERTIES GENERATED 1)
+
+SET_SOURCE_FILES_PROPERTIES(
+  ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc.cc
+  ${CMAKE_SOURCE_DIR}/sql/sql_digest.cc
+  PROPERTIES OBJECT_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/lex_token.h
+)
+
+ADD_CUSTOM_TARGET(GenYaccEmbeddedSource DEPENDS ${GEN_YACC_SOURCES})
+
 ADD_CONVENIENCE_LIBRARY(sql_embedded ${SQL_EMBEDDED_SOURCES})
 DTRACE_INSTRUMENT(sql_embedded)
-ADD_DEPENDENCIES(sql_embedded GenError GenServerSource)
+ADD_DEPENDENCIES(sql_embedded GenError GenServerSource GenYaccEmbeddedSource)
 
 # On Windows, static embedded server library is called mysqlserver.lib
 # On Unix, it is libmysqld.a
@@ -93,8 +142,8 @@ ENDIF()
 
 
 SET(LIBS 
-  dbug strings regex mysys mysys_ssl vio 
-  ${ZLIB_LIBRARY} ${SSL_LIBRARIES} 
+  dbug strings regex mysys mysys_ssl vio
+  ${ZLIB_LIBRARY} ${SSL_LIBRARIES}
   ${LIBWRAP} ${LIBCRYPT} ${LIBDL}
   ${MYSQLD_STATIC_PLUGIN_LIBS}
   sql_embedded
@@ -112,6 +161,8 @@ FOREACH(LIB ${LIBS})
   ENDIF()
 ENDFOREACH()
 
+LIST(REMOVE_DUPLICATES EMBEDDED_LIBS)
+
 MERGE_LIBRARIES(mysqlserver STATIC ${EMBEDDED_LIBS} 
   OUTPUT_NAME ${MYSQLSERVER_OUTPUT_NAME} COMPONENT Embedded)
 
@@ -145,7 +196,7 @@ IF(WITH_EMBEDDED_SHARED_LIBRARY)
     )
 
   GET_TARGET_PROPERTY(libmysqld_link_flags libmysqld LINK_FLAGS)
-  IF(NOT libmysqld_link_flag)
+  IF(NOT libmysqld_link_flags)
     SET(libmysqld_link_flags)
   ENDIF()
   SET(libmysqld_link_flags

sql/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2006, 2017, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2006, 2018, Oracle and/or its affiliates. All rights reserved.
 # 
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -13,6 +13,7 @@
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 
+
 INCLUDE_DIRECTORIES(
   ${CMAKE_SOURCE_DIR}/include 
   ${CMAKE_SOURCE_DIR}/sql 
@@ -35,8 +36,8 @@ SET(GEN_DIGEST_SOURCES
 )
 
 SET_SOURCE_FILES_PROPERTIES(${GEN_SOURCES}
-                            ${CONF_SOURCES}
                             ${GEN_DIGEST_SOURCES}
+                            ${CONF_SOURCES}
                             PROPERTIES GENERATED 1)
 
 # Gen_lex_token
@@ -193,8 +194,8 @@ SET(SQL_EXPORTED_SOURCES ${SQL_SHARED_SOURCES} PARENT_SCOPE)
 
 SET(SQL_SOURCE
   ${GEN_SOURCES}
-  ${CONF_SOURCES}
   ${GEN_DIGEST_SOURCES}
+  ${CONF_SOURCES}
   ${MYSYS_LIBWRAP_SOURCE}
   ${SQL_SHARED_SOURCES}
   ../libmysql/errmsg.c
@@ -277,7 +278,8 @@ ELSE()
   SET(MYSQLD_SOURCE main.cc ${DTRACE_PROBES_ALL})
 ENDIF()
 
-MYSQL_ADD_EXECUTABLE(mysqld ${MYSQLD_SOURCE} DESTINATION ${INSTALL_SBINDIR} COMPONENT Server)
+MYSQL_ADD_EXECUTABLE(mysqld
+  ${MYSQLD_SOURCE} DESTINATION ${INSTALL_SBINDIR} COMPONENT Server)
 
 OPTION(DEBUG_EXTNAME "Build server as mysqld-debug (debug builds only)" OFF)
 MARK_AS_ADVANCED(DEBUG_EXTNAME)
@@ -304,9 +306,11 @@ IF(NOT WITHOUT_DYNAMIC_PLUGINS)
     SET_TARGET_PROPERTIES(mysqld PROPERTIES LINK_FLAGS "${mysqld_link_flags} -Wl,--export-all-symbols")
   ENDIF()
   IF(MSVC)
-    # Set module definition file. Also use non-incremental linker, 
-    # incremental appears to crash from time to time,if used with /DEF option
-    SET_TARGET_PROPERTIES(mysqld PROPERTIES LINK_FLAGS "${mysqld_link_flags} /DEF:mysqld.def /INCREMENTAL:NO")
+    # Set module definition file.
+    # Also use non-incremental linker, incremental appears to crash from
+    # time to time,if used with /DEF option
+    SET_TARGET_PROPERTIES(mysqld PROPERTIES LINK_FLAGS
+      "${mysqld_link_flags} /DEF:mysqld.def /INCREMENTAL:NO")
 
     FOREACH (CORELIB sql mysys mysys_ssl dbug strings)
       GET_TARGET_PROPERTY(LOC ${CORELIB} LOCATION)
@@ -367,25 +371,26 @@ ENDIF()
 # Handle out-of-source build from source package with possibly broken 
 # bison. Copy bison output to from source to build directory, if not already 
 # there
-IF (NOT ${CMAKE_CURRENT_SOURCE_DIR} STREQUAL ${CMAKE_CURRENT_BINARY_DIR})
-  IF(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/sql_yacc.cc)
-    IF(NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc.cc)
-      CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/sql_yacc.cc 
-        ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc.cc COPYONLY)
-      CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/sql_yacc.h
-        ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc.h COPYONLY)
-    ENDIF()
-  ENDIF()
-ENDIF()
-
-
 INCLUDE(${CMAKE_SOURCE_DIR}/cmake/bison.cmake)
+COPY_BISON_OUTPUT(
+  ${CMAKE_CURRENT_SOURCE_DIR}/sql_yacc.cc
+  ${CMAKE_CURRENT_SOURCE_DIR}/sql_yacc.h
+  ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc.cc
+  ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc.h
+)
+
 RUN_BISON(
   ${CMAKE_CURRENT_SOURCE_DIR}/sql_yacc.yy 
   ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc.cc
   ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc.h
 )
 
+SET_SOURCE_FILES_PROPERTIES(
+  ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc.cc
+  ${CMAKE_CURRENT_SOURCE_DIR}/sql_digest.cc
+  PROPERTIES OBJECT_DEPENDS ${GEN_DIGEST_SOURCES}
+)
+
 # Gen_lex_hash
 ADD_EXECUTABLE(gen_lex_hash gen_lex_hash.cc)
 
@@ -409,16 +414,15 @@ ADD_CUSTOM_TARGET(
         DEPENDS ${GEN_DIGEST_SOURCES}
 )
 
-#Need this only for embedded
-SET_TARGET_PROPERTIES(GenServerSource PROPERTIES EXCLUDE_FROM_ALL TRUE)
 
 IF(WIN32 OR HAVE_DLOPEN AND NOT DISABLE_SHARED)
   ADD_LIBRARY(udf_example MODULE udf_example.cc)
   SET_TARGET_PROPERTIES(udf_example PROPERTIES PREFIX "")
   # udf_example depends on strings 
   IF(WIN32)
     IF(MSVC)
-     SET_TARGET_PROPERTIES(udf_example PROPERTIES LINK_FLAGS "/DEF:${CMAKE_CURRENT_SOURCE_DIR}/udf_example.def")
+      SET_TARGET_PROPERTIES(udf_example
+        PROPERTIES LINK_FLAGS "/DEF:${CMAKE_CURRENT_SOURCE_DIR}/udf_example.def")
     ENDIF()
     TARGET_LINK_LIBRARIES(udf_example strings)
   ELSE()
@@ -502,4 +506,3 @@ ENDIF()
 ADD_CUSTOM_TARGET(show-dist-name
   COMMAND ${CMAKE_COMMAND} -E echo "${CPACK_PACKAGE_FILE_NAME}"
 )
-