Merge branch 'trunk' into possible-resource-leak

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/connect_cancel/CMakeLists.txt

@@ -23,5 +23,6 @@ set(LIBHDFSPP_DIR CACHE STRING ${CMAKE_INSTALL_PREFIX})
 include_directories( ${LIBHDFSPP_DIR}/include )
 link_directories( ${LIBHDFSPP_DIR}/lib )
 
-add_executable(connect_cancel_c connect_cancel.c)
-target_link_libraries(connect_cancel_c hdfspp_static uriparser2)
+add_executable(connect_cancel_c $<TARGET_OBJECTS:x_platform_utils_obj_c_api> connect_cancel.c)
+target_link_libraries(connect_cancel_c hdfspp_static uriparser2)
+target_include_directories(connect_cancel_c PRIVATE ../../lib)

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/connect_cancel/connect_cancel.c

@@ -26,10 +26,10 @@
 #include <stdlib.h>
 #include <string.h>
 #include <signal.h>
-#include <unistd.h>
 
 #include "hdfspp/hdfs_ext.h"
 #include "common/util_c.h"
+#include "x-platform/c_api.h"
 
 #define ERROR_BUFFER_SIZE 1024
 
@@ -43,10 +43,10 @@ const char *catch_exit   = "Exiting the signal handler.\n";
 // Print to stdout without calling malloc or otherwise indirectly modify userspace state.
 // Write calls to stdout may still interleave with stuff coming from elsewhere.
 static void sighandler_direct_stdout(const char *msg) {
-  if(!msg)
+  if(!msg) {
     return;
-  ssize_t res = write(1 /*posix stdout fd*/, msg, strlen(msg));
-  (void)res;
+  }
+  x_platform_syscall_write_to_stdout(msg);
 }
 
 static void sig_catch(int val) {

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/connect_cancel/CMakeLists.txt

@@ -23,5 +23,6 @@ set(LIBHDFSPP_DIR CACHE STRING ${CMAKE_INSTALL_PREFIX})
 include_directories( ${LIBHDFSPP_DIR}/include )
 link_directories( ${LIBHDFSPP_DIR}/lib )
 
-add_executable(connect_cancel connect_cancel.cc)
+add_executable(connect_cancel $<TARGET_OBJECTS:x_platform_utils_obj> connect_cancel.cc)
 target_link_libraries(connect_cancel hdfspp_static)
+target_include_directories(connect_cancel PRIVATE ../../lib)

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/connect_cancel/connect_cancel.cc

@@ -25,11 +25,12 @@
 #include <google/protobuf/stubs/common.h>
 
 #include <signal.h>
-#include <unistd.h>
 
 #include <thread>
 #include <iostream>
 
+#include "x-platform/syscall.h"
+
 // Simple example of how to cancel an async connect call.
 // Here Control-C (SIGINT) is caught in order to invoke the FS level cancel and
 // properly tear down the process.  Valgrind should show no leaked memory on exit
@@ -47,11 +48,10 @@ const std::string catch_exit("Exiting the signal handler.\n");
 // It's possible that the write interleaves with another write call,
 // but it won't corrupt the stack or heap.
 static void sighandler_direct_stdout(const std::string &msg) {
-  ssize_t res = ::write(1 /*posix stdout FD*/, msg.data(), msg.size());
-  // In production you'd want to check res, but error handling code will
-  // need to be fairly application specific if it's going to properly
-  // avoid reentrant calls to malloc.
-  (void)res;
+  XPlatform::Syscall::WriteToStdout(msg);
+  // In production you'd want to check the result of the above call,
+  // but error handling code will need to be fairly application
+  // specific if it's going to properly avoid reentrant calls to malloc.
 }
 
 // Signal handler to make a SIGINT call cancel rather than exit().

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/CMakeLists.txt

@@ -16,4 +16,11 @@
 # limitations under the License.
 #
 
-add_library(x_platform_utils_obj OBJECT utils.cc)
+if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows")
+  set(SYSCALL_SRC syscall_windows.cc)
+else()
+  set(SYSCALL_SRC syscall_linux.cc)
+endif()
+
+add_library(x_platform_utils_obj OBJECT ${SYSCALL_SRC} utils.cc)
+add_library(x_platform_utils_obj_c_api OBJECT $<TARGET_OBJECTS:x_platform_utils_obj> c_api.cc)

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/c_api.cc

@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "syscall.h"
+
+extern "C" int x_platform_syscall_write_to_stdout(const char* msg) {
+  return XPlatform::Syscall::WriteToStdout(msg) ? 1 : 0;
+}

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/c_api.h

@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef NATIVE_LIBHDFSPP_LIB_CROSS_PLATFORM_C_API_H
+#define NATIVE_LIBHDFSPP_LIB_CROSS_PLATFORM_C_API_H
+
+/**
+ * C APIs for accessing XPlatform
+ */
+
+int x_platform_syscall_write_to_stdout(const char* msg);
+
+#endif  // NATIVE_LIBHDFSPP_LIB_CROSS_PLATFORM_C_API_H

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall.h

@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef NATIVE_LIBHDFSPP_LIB_CROSS_PLATFORM_SYSCALL
+#define NATIVE_LIBHDFSPP_LIB_CROSS_PLATFORM_SYSCALL
+
+#include <string>
+
+/**
+ * The {@link XPlatform} namespace contains components that
+ * aid in writing cross-platform code.
+ */
+namespace XPlatform {
+class Syscall {
+ public:
+  /**
+   * Writes the given string to the application's
+   * standard output stream.
+   *
+   * @param message The string to write to stdout.
+   * @returns A boolean indicating whether the write
+   * was successful.
+   */
+  static bool WriteToStdout(const std::string& message);
+
+  /**
+   * Writes the given char pointer to the application's
+   * standard output stream.
+   *
+   * @param message The char pointer to write to stdout.
+   * @returns A boolean indicating whether the write
+   * was successful.
+   */
+  static int WriteToStdout(const char* message);
+
+ private:
+  static bool WriteToStdoutImpl(const char* message);
+};
+}  // namespace XPlatform
+
+#endif

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc

@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <unistd.h>
+
+#include <cstring>
+
+#include "syscall.h"
+
+bool XPlatform::Syscall::WriteToStdout(const std::string& message) {
+  return WriteToStdoutImpl(message.c_str());
+}
+
+int XPlatform::Syscall::WriteToStdout(const char* message) {
+  return WriteToStdoutImpl(message) ? 1 : 0;
+}
+
+bool XPlatform::Syscall::WriteToStdoutImpl(const char* message) {
+  const auto message_len = strlen(message);
+  const auto result = write(1, message, message_len);
+  return result == static_cast<ssize_t>(message_len);
+}

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_windows.cc

@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <Windows.h>
+
+#include "syscall.h"
+
+bool XPlatform::Syscall::WriteToStdout(const std::string& message) {
+  return WriteToStdoutImpl(message.c_str());
+}
+
+int XPlatform::Syscall::WriteToStdout(const char* message) {
+  return WriteToStdoutImpl(message) ? 1 : 0;
+}
+
+bool XPlatform::Syscall::WriteToStdoutImpl(const char* message) {
+  auto* const stdout_handle = GetStdHandle(STD_OUTPUT_HANDLE);
+  if (stdout_handle == INVALID_HANDLE_VALUE || stdout_handle == nullptr) {
+    return false;
+  }
+
+  unsigned long bytes_written = 0;
+  const auto message_len = lstrlen(message);
+  const auto result =
+      WriteFile(stdout_handle, message, message_len, &bytes_written, nullptr);
+  return result && static_cast<unsigned long>(message_len) == bytes_written;
+}

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/utils.cc

@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-#include "x-platform/utils.h"
+#include "utils.h"
 
 #include <filesystem>
 #include <string>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingMonitorManager.java

@@ -61,7 +61,7 @@ private void updateSchedulingMonitors(Configuration conf,
     }
 
     // When monitor is enabled, loading policies
-    String[] configuredPolicies = conf.getStrings(
+    String[] configuredPolicies = conf.getTrimmedStrings(
         YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES);
     if (configuredPolicies == null || configuredPolicies.length == 0) {
       return;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java

@@ -775,7 +775,7 @@ private float extractFloatValueFromWeightConfig(String configureValue) {
     if (!configuredWeightAsCapacity(configureValue)) {
       return -1f;
     } else {
-      return Float.valueOf(
+      return Float.parseFloat(
           configureValue.substring(0, configureValue.indexOf(WEIGHT_SUFFIX)));
     }
   }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java

@@ -571,7 +571,7 @@ public void testAutoQueueCreationFailsForEmptyPathWithAQCAndWeightMode()
 
       ApplicationId appId = BuilderUtils.newApplicationId(1, 1);
       SchedulerEvent addAppEvent = new AppAddedSchedulerEvent(
-          appId, "user", "root.");
+          appId, "user", USER0);
       newCS.handle(addAppEvent);
 
       RMAppEvent event = new RMAppEvent(appId, RMAppEventType.APP_REJECTED,