Migrate z/OS to a CMake build system

CMakeLists.txt

@@ -84,6 +84,24 @@ target_include_directories(omr_base
 		$<INSTALL_INTERFACE:${OMR_INSTALL_INC_DIR}>
 )
 
+if(OMR_OS_ZOS)
+	if(OMR_USE_NATIVE_ENCODING)
+		target_link_libraries(omr_base INTERFACE omr_ebcdic)
+		target_compile_definitions(omr_base
+			INTERFACE
+				-DOMR_EBCDIC
+		)
+		install(TARGETS omr_ebcdic
+			EXPORT OmrTargets
+		)
+	else()
+		target_link_libraries(omr_base INTERFACE omr_ascii)
+		install(TARGETS omr_ascii
+			EXPORT OmrTargets
+		)
+	endif()
+endif()
+
 install(
 	TARGETS omr_base
 	EXPORT OmrTargets

buildenv/jenkins/jobs/pull-requests/PullRequest-zos_390-64

@@ -14,41 +14,23 @@ pipeline {
         stage('Build') {
             steps {
                 timestamps {
-                    //echo 'Output CCACHE stats before running and clear them'
-                    //echo '''ccache -s -z'''
-
-                    echo 'Configure...'
-                    sh'''make -f run_configure.mk OMRGLUE=./example/glue SPEC=zos_390-64'''
-
-                    echo 'Convert HDF files to EBCDIC'
-                    //sh'''for file in `find . -name '*.hdf'`;do;mv "$file" "$file.iso88591";iconv -f ISO8859-1 -t IBM-1047 "$file.iso88591" >"$file" && chtag -tc IBM-1047 "$file";done'''
-                    //sh'''for file in `find . -name '*.hdf'`;do; chtag -p "$file" | grep -q "ISO8859-1"; if [[ $? == 0 ]]; then; mv "$file" "$file.iso88591";iconv -f ISO8859-1 -t IBM-1047 "$file.iso88591" >"$file" && chtag -tc IBM-1047 "$file"; fi; done'''
-                    sh'''for file in `find . -name '*.hdf'`;do; chtag -p "$file" | grep -q "ISO8859-1"; if [ $? -eq 0 ]; then; mv "$file" "$file.iso88591";iconv -f ISO8859-1 -t IBM-1047 "$file.iso88591" >"$file" && chtag -tc IBM-1047 "$file"; fi; done'''
-
-                    echo 'Convert TDF files to EBCDIC'
-                    //sh'''for file in `find . -name '*.tdf'`;do;mv "$file" "$file.iso88591";iconv -f ISO8859-1 -t IBM-1047 "$file.iso88591" >"$file" && chtag -tc IBM-1047 "$file";done'''
-                    //sh'''for file in `find . -name '*.tdf'`;do; chtag -p "$file" | grep -q "ISO8859-1"; if [[ $? == 0 ]]; then; mv "$file" "$file.iso88591";iconv -f ISO8859-1 -t IBM-1047 "$file.iso88591" >"$file" && chtag -tc IBM-1047 "$file"; fi; done
-                    sh'''for file in `find . -name '*.tdf'`;do; chtag -p "$file" | grep -q "ISO8859-1"; if [ $? -eq 0 ]; then; mv "$file" "$file.iso88591";iconv -f ISO8859-1 -t IBM-1047 "$file.iso88591" >"$file" && chtag -tc IBM-1047 "$file"; fi; done'''
-
-                    echo 'Compile...'
-                    sh'''make -j4'''
-
-                    //echo 'Output CCACHE stats after running'
-                    //echo '''ccache -s'''
+                    dir('build') {
+                        echo 'Configure...'
+                        sh '''cmake -Wdev -C../cmake/caches/Travis.cmake -DCMAKE_C_COMPILER=/bin/c89 -DCMAKE_CXX_COMPILER=/bin/xlc -DOMR_DDR=0 -DOMR_THR_FORK_SUPPORT=0 ..''' 
+
+                        echo 'Compile...'
+                        sh '''make -j4'''
+                    }
                 }
             }
         }
         stage('Test') {
             steps {
                 timestamps {
-                    echo "Sanity Test..."
-
-                    // Workaround: Convert XML files to EBCDIC 
-                    //sh'''for file in `find . -name '*.xml'`;do;mv "$file" "$file.iso88591";iconv -f ISO8859-1 -t IBM-1047 "$file.iso88591" >"$file" && chtag -tc IBM-1047 "$file";done'''
-                    sh'''for file in `find . -name '*.xml'`;do; chtag -p "$file" | grep -q "ISO8859-1"; if [ $? -eq 0 ]; then; mv "$file" "$file.iso88591";iconv -f ISO8859-1 -t IBM-1047 "$file.iso88591" >"$file" && chtag -tc IBM-1047 "$file"; fi; done'''
-
-                    // Perform a subset of the testing until while we resolve the issues with the other tests
-                    sh'''set; make -f fvtest/omrtest.mk -j1 SPEC=zos_390-64 omr_algotest omr_vmtest omr_utiltest omr_sigtest omr_rastest'''
+                    dir('build') {
+                        echo "Sanity Test..."
+                        sh'''./omralgotest -avltest:fvtest/algotest/avltest.lst && ./omrvmtest && ./omrutiltest && ./omrsigtest && ./omrrastest && ./omrsubscribertest && ./omrtraceoptiontest'''
+                    }
                 }
             }
         }

cmake/config.cmake

@@ -48,6 +48,9 @@ set(OMR_TEST_COMPILER OFF CACHE BOOL "Enable building the test compiler")
 set(OMR_GC ON CACHE BOOL "Enable the GC")
 set(OMR_GC_TEST ${OMR_GC} CACHE BOOL "Enable the GC tests.")
 
+set(OMR_USE_NATIVE_ENCODING ON CACHE BOOL
+	"Indicates that runtime components should use the systems native encoding (currently only defined for z/OS)"
+)
 ## OMR_COMPILER is required for OMR_JITBUILDER and OMR_TEST_COMPILER
 if(NOT OMR_COMPILER)
 	if(OMR_JITBUILDER)
@@ -82,6 +85,10 @@ set(OMR_TOOLS_IMPORTFILE "IMPORTFILE-NOTFOUND" CACHE FILEPATH
 	"Point it to the ImportTools.cmake file of a native build"
 )
 
+set(OMR_TOOLS_USE_NATIVE_ENCODING ON CACHE BOOL
+	"Indicates if omr tooling should use system native character encoding (currently only defined for z/OS)"
+)
+
 ###
 ### Library names
 ###

cmake/modules/FindLEX.cmake

@@ -0,0 +1,80 @@
+###############################################################################
+# Copyright (c) 2019, 2019 IBM Corp. and others
+#
+# This program and the accompanying materials are made available under
+# the terms of the Eclipse Public License 2.0 which accompanies this
+# distribution and is available at http://eclipse.org/legal/epl-2.0
+# or the Apache License, Version 2.0 which accompanies this distribution
+# and is available at https://www.apache.org/licenses/LICENSE-2.0.
+#
+# This Source Code may also be made available under the following Secondary
+# Licenses when the conditions for such availability set forth in the
+# Eclipse Public License, v. 2.0 are satisfied: GNU General Public License,
+# version 2 with the GNU Classpath Exception [1] and GNU General Public
+# License, version 2 with the OpenJDK Assembly Exception [2].
+#
+# [1] https://www.gnu.org/software/classpath/license.html
+# [2] http://openjdk.java.net/legal/assembly-exception.html
+#
+# SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 OR LicenseRef-GPL-2.0 WITH Assembly-exception
+#############################################################################
+
+find_program(LEX_EXECUTABLE NAMES lex DOC "Path to the lex executable")
+mark_as_advanced(LEX_EXECUTABLE)
+
+if(LEX_EXECUTABLE)
+	macro(LEX_TARGET Name Input Output)
+		set(LEX_TARGET_outputs "${Output}")
+		set(LEX_EXECUTABLE_opts "")
+		set(LEX_TARGET_PARAM_OPTIONS)
+		set(LEX_TARGET_PARAM_ONE_VALUE_KEYWORDS
+			COMPILE_FLAGS
+		)
+		set(LEX_TARGET_PARAM_MULTI_VALUE_KEYWORDS)
+
+		cmake_parse_arguments(
+			LEX_TARGET_ARG
+			"${LEX_TARGET_PARAM_OPTIONS}"
+			"${LEX_TARGET_PARAM_ONE_VALUE_KEYWORDS}"
+			"${LEX_TARGET_MULTI_VALUE_KEYWORDS}"
+			${ARGN}
+		)
+
+		set(LEX_TARGET_usage "LEX_TARGET(<Name> <Input> <Output> [COMPILE_FLAGS <string>]")
+
+		if(NOT "${LEX_TARGET_ARG_UNPARSED_ARGUMENTS}" STREQUAL "")
+			message(SEND_ERROR ${LEX_TARGET_usage})
+		else()
+			if(NOT "${LEX_TARGET_ARG_COMPILE_FLAGS}" STREQUAL "")
+				set(LEX_EXECUTABLE_opts "${LEX_TARGET_ARG_COMPILE_FLAGS}")
+				separate_arguments(LEX_EXECUTABLE_opts)
+			endif()
+
+			add_custom_command(OUTPUT ${LEX_TARGET_outputs}
+				COMMAND ${LEX_EXECUTABLE} ${LEX_EXECUTABLE_opts} -o${Output} ${Input}
+				VERBATIM
+				DEPENDS ${Input}
+				COMMENT "[LEX][${Name}] Building scanner with LEX ${LEX_VERSION}"
+				WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+			)
+
+			set(LEX_${Name}_DEFINED TRUE)
+			set(LEX_${Name}_OUTPUTS ${Output})
+			set(LEX_${Name}_INPUT ${Input})
+			set(LEX_${Name}_COMPILE_FLAGS ${LEX_EXECUTABLE_opts})
+		endif()
+	endmacro()
+
+	macro(ADD_LEX_YACC_DEPENDENCY LEXTarget YACCTarget)
+		if(NOT LEX_${LEXTarget}_OUTPUTS)
+			message(SEND_ERROR "LEX target `${LEXTarget}' does not exist.")
+		endif()
+
+		if(NOT YACC_${YACCTarget}_OUTPUT_HEADER)
+			message(SEND_ERROR "YACC target `${YACCTarget}' does not exist.")
+		endif()
+
+		set_source_files_properties(${LEX_${LEXTarget}_OUTPUTS}
+			PROPERTIES OBJECT_DEPENDS ${YACC_${YACCTarget}_OUTPUT_HEADER})
+	endmacro()
+endif()

cmake/modules/FindYACC.cmake

@@ -0,0 +1,93 @@
+###############################################################################
+# Copyright (c) 2019, 2019 IBM Corp. and others
+#
+# This program and the accompanying materials are made available under
+# the terms of the Eclipse Public License 2.0 which accompanies this
+# distribution and is available at http://eclipse.org/legal/epl-2.0
+# or the Apache License, Version 2.0 which accompanies this distribution
+# and is available at https://www.apache.org/licenses/LICENSE-2.0.
+#
+# This Source Code may also be made available under the following Secondary
+# Licenses when the conditions for such availability set forth in the
+# Eclipse Public License, v. 2.0 are satisfied: GNU General Public License,
+# version 2 with the GNU Classpath Exception [1] and GNU General Public
+# License, version 2 with the OpenJDK Assembly Exception [2].
+#
+# [1] https://www.gnu.org/software/classpath/license.html
+# [2] http://openjdk.java.net/legal/assembly-exception.html
+#
+# SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 OR LicenseRef-GPL-2.0 WITH Assembly-exception
+#############################################################################
+
+find_program(YACC_EXECUTABLE NAMES yacc DOC "Path to the YACC executable")
+mark_as_advanced(YACC_EXECUTABLE)
+
+if(YACC_EXECUTABLE)
+	macro(YACC_TARGET_option_extraopts Options)
+		set(YACC_TARGET_cmdopt "")
+		set(YACC_TARGET_extraopts "${Options}")
+		separate_arguments(YACC_TARGET_extraopts)
+		list(APPEND YACC_TARGET_cmdopt ${YACC_TARGET_extraopts})
+	endmacro()
+
+	macro(YACC_TARGET_option_defines YACCOutput Header)
+		if("${Header}" STREQUAL "")
+			string(REGEX REPLACE "^(.*)(\\.[^.]*)$" "\\2" _fileext "${YACCOutput}")
+			string(REPLACE "c" "h" _fileext ${_fileext})
+			string(REGEX REPLACE "^(.*)(\\.[^.]*)$" "\\1${_fileext}" YACC_TARGET_output_header "${YACCOutput}")
+			list(APPEND YACC_TARGET_cmdopt "-d")
+		else()
+			set(YACC_TARGET_output_header "${Header}")
+
+			if(OMR_HOST_OS STREQUAL "zos")
+				list(APPEND YACC_TARGET_cmdopt "-D${YACC_TARGET_output_header}")
+			else()
+				list(APPEND YACC_TARGET_cmdopt "--defines=${YACC_TARGET_output_header}")
+			endif()
+		endif()
+	endmacro()
+
+	macro(YACC_TARGET Name YACCInput YACCOutput)
+		set(YACC_TARGET_outputs "${YACCOutput}")
+		set(YACC_TARGET_extraoutputs "")
+		set(YACC_TARGET_PARAM_OPTIONS)
+		set(YACC_TARGET_PARAM_ONE_VALUE_KEYWORDS
+			COMPILE_FLAGS
+			DEFINES_FILE
+		 )
+
+		cmake_parse_arguments(
+			YACC_TARGET_ARG
+			"${YACC_TARGET_PARAM_OPTIONS}"
+			"${YACC_TARGET_PARAM_ONE_VALUE_KEYWORDS}"
+			"${LEX_TARGET_MULTI_VALUE_KEYWORDS}"
+			${ARGN}
+		)
+
+		set(YACC_TARGET_usage "YACC_TARGET(<Name> <Input> <Output> [COMPILE_FLAGS <string>] [DEFINES_FILE <string>]")
+
+		if(NOT "${YACC_TARGET_ARG_UNPARSED_ARGUMENTS}" STREQUAL "")
+			message(SEND_ERROR ${YACC_TARGET_usage})
+		else()
+			YACC_TARGET_option_extraopts("${YACC_TARGET_ARG_COMPILE_FLAGS}")
+			YACC_TARGET_option_defines("${YACCOutput}" "${YACC_TARGET_ARG_DEFINES_FILE}")
+
+			list(APPEND YACC_TARGET_outputs "${YACC_TARGET_output_header}")
+
+			add_custom_command(OUTPUT ${YACC_TARGET_outputs}
+				COMMAND ${YACC_EXECUTABLE} ${YACC_TARGET_cmdopt} -o ${YACCOutput} ${YACCInput}
+				VERBATIM
+				DEPENDS ${YACCInput}
+				COMMENT "[YACC][${Name}] Building parser with YACC"
+				WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+			)
+
+			set(YACC_${Name}_DEFINED TRUE)
+			set(YACC_${Name}_INPUT ${YACCInput})
+			set(YACC_${Name}_OUTPUTS ${YACC_TARGET_outputs} ${YACC_TARGET_extraoutputs})
+			set(YACC_${Name}_COMPILE_FLAGS ${YACC_TARGET_cmdopt})
+			set(YACC_${Name}_OUTPUT_SOURCE "${YACCOutput}")
+			set(YACC_${Name}_OUTPUT_HEADER "${YACC_TARGET_output_header}")
+		endif()
+	endmacro()
+endif()

cmake/modules/OmrDetectSystemInformation.cmake

@@ -1,5 +1,5 @@
 ###############################################################################
-# Copyright (c) 2017, 2018 IBM Corp. and others
+# Copyright (c) 2017, 2019 IBM Corp. and others
 #
 # This program and the accompanying materials are made available under
 # the terms of the Eclipse Public License 2.0 which accompanies this
@@ -182,7 +182,7 @@ macro(omr_detect_system_information)
 				# Just use GNU config
 				set(_OMR_TOOLCONFIG "gnu")
 			endif()
-		elseif(CMAKE_C_COMPILER_ID STREQUAL "XL")
+		elseif(CMAKE_C_COMPILER_ID STREQUAL "XL" OR CMAKE_C_COMPILER_ID STREQUAL "zOS")
 			set(_OMR_TOOLCONFIG "xlc")
 		else()
 			message(FATAL_ERROR "OMR: Unknown compiler ID: '${CMAKE_CXX_COMPILER_ID}'")

cmake/modules/platform/arch/s390.cmake

@@ -1,5 +1,5 @@
 ###############################################################################
-# Copyright (c) 2017, 2018 IBM Corp. and others
+# Copyright (c) 2017, 2019 IBM Corp. and others
 #
 # This program and the accompanying materials are made available under
 # the terms of the Eclipse Public License 2.0 which accompanies this
@@ -19,10 +19,6 @@
 # SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 OR LicenseRef-GPL-2.0 WITH Assembly-exception
 ###############################################################################
 
-list(APPEND OMR_PLATFORM_DEFINITIONS
-	-DJ9VM_TIERED_CODE_CACHE
-)
-
 # Testarossa build variables. Longer term the distinction between TR and the rest
 # of the OMR code should be heavily reduced. In the mean time, we keep
 # the distinction

cmake/modules/platform/os/zos.cmake

@@ -1,5 +1,5 @@
 ###############################################################################
-# Copyright (c) 2017, 2017 IBM Corp. and others
+# Copyright (c) 2017, 2019 IBM Corp. and others
 #
 # This program and the accompanying materials are made available under
 # the terms of the Eclipse Public License 2.0 which accompanies this
@@ -20,12 +20,15 @@
 #############################################################################
 
 list(APPEND OMR_PLATFORM_DEFINITIONS
-	-DJ9ZOS390
-	-DLONGLONG
 	-D_ALL_SOURCE
-	-D_XOPEN_SOURCE_EXTENDED
-	-DIBM_ATOE
+	-D_OPEN_THREADS=2
 	-D_POSIX_SOURCE
+	-D_XOPEN_SOURCE_EXTENDED
+	-D_ISOC99_SOURCE
+	-DLONGLONG
+	-DJ9ZOS390
+	-DSUPPORTS_THREAD_LOCAL
+	-DZOS
 )
 
 list(APPEND OMR_PLATFORM_INCLUDE_DIRECTORIES
@@ -34,52 +37,14 @@ list(APPEND OMR_PLATFORM_INCLUDE_DIRECTORIES
 	/usr/include
 )
 
-list(APPEND OMR_PLATFORM_COMPILE_OPTIONS
-	"\"-Wc,xplink\""               # link with xplink calling convention
-	"\"-Wc,convlit(ISO8859-1)\""   # convert all string literals to a codepage
-	"\"-Wc,rostring\""             # place string literals in read only storage
-	"\"-Wc,FLOAT(IEEE,FOLD,AFP)\"" # Use IEEE (instead of IBM Hex Format) style floats
-	"\"-Wc,enum(4)\""              # Specifies how many bytes of storage enums occupy
-	"\"-Wa,goff\""                 # Assemble into GOFF object files
-	"\"-Wc,NOANSIALIAS\""          # Do not generate ALIAS binger control statements
-	"\"-Wc,TARGET(zOSV1R13)\""     # Generate code for the target operating system
-)
-
-list(APPEND OMR_PLATFORM_C_COMPILE_OPTIONS
-	"\"-Wc,ARCH(7)\""
-	"\"-Wc,langlvl(extc99)\""
-	"\"-qnosearch\""
-)
-
-list(APPEND OMR_PLATFORM_CXX_COMPILE_OPTIONS
-	"\"-Wc,ARCH(7)\""
-	"\"-Wc,langlvl(extended)\""
-	"\"-qnosearch\""
-)
-
-list(APPEND OMR_PLATFORM_SHARED_COMPILE_OPTIONS
-	"\"-Wc,DLL\""
-	"\"-Wc,EXPORTALL\""
-)
-
-list(APPEND OMR_PLATFORM_SHARED_LINKER_OPTIONS
-	"\"-Wl,xplink\""
-	"\"-Wl,dll\""
-)
+# Create helper targets for specifying ascii/ebcdic options
+add_library(omr_ascii INTERFACE)
+target_compile_definitions(omr_ascii INTERFACE -DIBM_ATOE)
+target_compile_options(omr_ascii INTERFACE "-Wc,convlit(ISO8859-1)")
+target_link_libraries(omr_ascii INTERFACE j9a2e)
 
-if(OMR_ENV_DATA64)
-	list(APPEND OMR_PLATFORM_DEFINITIONS
-		-DJ9ZOS39064
-	)
-	list(APPEND OMR_PLATFORM_COMPILE_OPTIONS
-		\"-Wc,lp64\"
-		\"-Wa,SYSPARM(BIT64)\"
-	)
-else()
-	list(APPEND OMR_PLATFORM_DEFINITIONS
-		-D_LARGE_FILES
-	)
-endif()
+add_library(omr_ebcdic INTERFACE)
+target_compile_definitions(omr_ebcdic INTERFACE -DOMR_EBCDIC)
 
 macro(omr_os_global_setup)