diff --git a/src/Compiler/CMakeLists.txt b/src/Compiler/CMakeLists.txt
index a7695282ab..6fc6d1bb56 100644
--- a/src/Compiler/CMakeLists.txt
+++ b/src/Compiler/CMakeLists.txt
@@ -188,6 +188,9 @@ add_dependencies(OMCompilerUtils cruntime)
 if (ONNX_MLIR_ENABLE_JNI)
   add_dependencies(OMCompilerUtils jniruntime)
 endif()
+if (TARGET omp)
+  add_dependencies(OMCompilerUtils libOMomp)
+endif()
 
 add_onnx_mlir_library(OMCompiler
   OnnxMlirCompiler.cpp
diff --git a/src/Runtime/CMakeLists.txt b/src/Runtime/CMakeLists.txt
index 09c5bdf7f1..74a8dfc0d4 100644
--- a/src/Runtime/CMakeLists.txt
+++ b/src/Runtime/CMakeLists.txt
@@ -2,6 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 add_subdirectory(jni)
+add_subdirectory(omp)
 
 # TODO: should add for each accelerator its subdirectory that implements InitAccel##name
 # and ShutdownAccel##name.
diff --git a/src/Runtime/omp/CMakeLists.txt b/src/Runtime/omp/CMakeLists.txt
new file mode 100644
index 0000000000..d1ca1b77d6
--- /dev/null
+++ b/src/Runtime/omp/CMakeLists.txt
@@ -0,0 +1,41 @@
+# SPDX-License-Identifier: Apache-2.0
+
+# Only if LLVM is built with openmp enabled
+if(TARGET omp)
+  set(OMP_TOPDIR ${CMAKE_CURRENT_BINARY_DIR})
+  set_directory_properties(PROPERTIES EP_BASE ${OMP_TOPDIR})
+
+  ExternalProject_Add(OMomp
+    DOWNLOAD_DIR "."
+    SOURCE_DIR "."
+    BINARY_DIR "."
+    INSTALL_DIR "."
+    STAMP_DIR "."
+    TMP_DIR "tmp"
+
+    DOWNLOAD_COMMAND ""
+    # Build with LIBOMP_ENABLE_SHARED=OFF so that libomp.a can be embedded
+    # into model.so
+    CONFIGURE_COMMAND sh -c "CC=clang CXX=clang++ \
+                             cmake -G Ninja ${LLVM_BUILD_MAIN_SRC_DIR} \
+                                   -DLLVM_ENABLE_PROJECTS=\"clang$<SEMICOLON>openmp\" \
+                                   -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
+                                   -DLIBOMP_ENABLE_SHARED=OFF"
+    BUILD_COMMAND sh -c "cmake --build . --target omp -- ${MAKEFLAGS}"
+    INSTALL_COMMAND ""
+    )
+
+  add_custom_target(libOMomp
+    COMMAND ${CMAKE_COMMAND} -E copy_if_different
+            ${OMP_TOPDIR}/lib/libomp.a ${ONNX_MLIR_LIBRARY_PATH}/libOMomp.a
+    DEPENDS OMomp
+    # BYPRODUCTS requires cmake 3.20+
+    BYPRODUCTS ${OMP_TOPDIR}/lib/libomp.a
+    )
+
+  install(FILES ${ONNX_MLIR_LIBRARY_PATH}/libOMomp.a DESTINATION lib)
+
+  message(STATUS "OpenMP support           : ON")
+else()
+  message(STATUS "OpenMP support           : OFF")
+endif()