Merge pull request #35 from TAK-Product-Center/upstream/4.10-RELEASE-68

TAK Server 4.10-RELEASE-68

README.md

@@ -1,5 +1,5 @@
 # TAK Server Development
-*Requires Java 11*
+*Requires Java 17*
 
 * Linux or MacOS is recommended for development. If using Windows, replace "gradlew" with "gradlew.bat" in commands below. An x86-64 architecture CPU is required to build from source, including on MacOS. M1 or M2 Apple silicon is not supported.
 
@@ -58,28 +58,31 @@ See appendix B in src/docs/TAK_Server_Configuration_Guide.pdf for cert generatio
 
 ### Build and run TAK server locally for development
 
+Note that due to Java 17, there are a lot of '--add-opens' arguments in the JDK_JAVA_OPTIONS
 ```
 cd takserver-core
 ../gradlew clean bootWar bootJar
 cd example
-export JDK_JAVA_OPTIONS="-Dloader.path=WEB-INF/lib-provided,WEB-INF/lib,WEB-INF/classes,file:lib/ -Djava.net.preferIPv4Stack=true -Djava.security.egd=file:/dev/./urandom -DIGNITE_UPDATE_NOTIFIER=false -DIGNITE_QUIET=true"
+export IGNITE_HOME="$PWD/ignite"
+export JDK_JAVA_OPTIONS="-Dloader.path=WEB-INF/lib-provided,WEB-INF/lib,WEB-INF/classes,file:lib/ -Djava.net.preferIPv4Stack=true -Djava.security.egd=file:/dev/./urandom -DIGNITE_UPDATE_NOTIFIER=false -DIGNITE_QUIET=true -Dio.netty.tmpdir=$PWD -Djava.io.tmpdir=$PWD -Dio.netty.native.workdir=$PWD -Djdk.tls.client.protocols=TLSv1.2  --add-opens=java.base/sun.security.pkcs=ALL-UNNAMED --add-opens=java.base/sun.security.pkcs10=ALL-UNNAMED --add-opens=java.base/sun.security.util=ALL-UNNAMED --add-opens=java.base/sun.security.x509=ALL-UNNAMED --add-opens=java.base/sun.security.tools.keytool=ALL-UNNAMED --add-opens=java.base/jdk.internal.misc=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.management/com.sun.jmx.mbeanserver=ALL-UNNAMED --add-opens=jdk.internal.jvmstat/sun.jvmstat.monitor=ALL-UNNAMED --add-opens=java.base/sun.reflect.generics.reflectiveObjects=ALL-UNNAMED --add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.math=ALL-UNNAMED --add-opens=java.sql/java.sql=ALL-UNNAMED --add-opens=java.base/javax.net.ssl=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=jdk.unsupported/sun.misc=ALL-UNNAMED --add-opens=java.base/java.lang.ref=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.security=ALL-UNNAMED --add-opens=java.base/java.security.ssl=ALL-UNNAMED --add-opens=java.base/java.security.cert=ALL-UNNAMED --add-opens=java.base/sun.security.rsa=ALL-UNNAMED --add-opens=java.base/sun.security.ssl=ALL-UNNAMED --add-opens=java.base/sun.security.x500=ALL-UNNAMED --add-opens=java.base/sun.security.pkcs12=ALL-UNNAMED --add-opens=java.base/sun.security.provider=ALL-UNNAMED --add-opens=java.base/javax.security.auth.x500=ALL-UNNAMED"
+
 ```
 
 TAK server consists of two processes: Messaging and API. The messaging process can run independently, but the API process needs to connect to the ignite server that runs as a part of the messaging process. For both processes, -Xmx should always be specified.
 
 Run Messaging (note - this command and the following one to run api include the **duplicatelogs** profile. This turns off the filter that blocks duplicated log messages that cause log spam in operational deployments of TAK Server.
 ```
-java -Xmx<value> -Dspring.profiles.active=messaging,duplicatelogs -jar ../build/libs/takserver-core-xyz.war
+java -server -XX:+AlwaysPreTouch -XX:+UseG1GC -XX:+ScavengeBeforeFullGC -XX:+DisableExplicitGC -Xmx<value> -Dspring.profiles.active=messaging,duplicatelogs -jar ../build/libs/takserver-core-xyz.war
 ```
 
 Run API
 ```
-java -Xmx<value> -Dspring.profiles.active=api,duplicatelogs -Dkeystore.pkcs12.legacy -jar ../build/libs/takserver-core-xyz.war
+java -server -XX:+AlwaysPreTouch -XX:+UseG1GC -XX:+ScavengeBeforeFullGC -XX:+DisableExplicitGC -Xmx<value> -Dspring.profiles.active=api,duplicatelogs -Dkeystore.pkcs12.legacy -jar ../build/libs/takserver-core-xyz.war
 ```
 
 Run Plugin Manager (useful when working on plugin capability)
 ```
-java -Xmx<value> -jar ../../takserver-plugin-manager/build/libs/takserver-plugin-manager-xyz.jar 
+java -server -XX:+AlwaysPreTouch -XX:+UseG1GC -XX:+ScavengeBeforeFullGC -XX:+DisableExplicitGC -Xmx<value> -jar ../../takserver-plugin-manager/build/libs/takserver-plugin-manager-xyz.jar 
 ```
 
 ### RPM Generation

src/build.gradle

@@ -57,8 +57,82 @@ allprojects {
 //  apply plugin: 'pmd'
 
 
-  sourceCompatibility = 1.8
-  targetCompatibility = 1.8
+  sourceCompatibility = 17
+  targetCompatibility = 17
+
+
+// The java opens values. These must be managed in takserver-core/scripts/setenv.sh in addition to here!
+ext.java_internal_opens = [
+    // Compile/test takserver required opens
+    'java.base/sun.security.pkcs',
+    'java.base/sun.security.pkcs10',
+    'java.base/sun.security.util',
+    'java.base/sun.security.x509',
+    'java.base/sun.security.tools.keytool',
+
+    // Ignite required opens as per https://github.com/apache/ignite/blob/master/bin/include/jvmdefaults.sh
+    'java.base/jdk.internal.misc',
+    'java.base/sun.nio.ch',
+    'java.management/com.sun.jmx.mbeanserver',
+    'jdk.internal.jvmstat/sun.jvmstat.monitor',
+    'java.base/sun.reflect.generics.reflectiveObjects',
+    'jdk.management/com.sun.management.internal',
+    'java.base/java.io',
+    'java.base/java.nio',
+    'java.base/java.util',
+    'java.base/java.util.concurrent',
+    'java.base/java.util.concurrent.locks',
+    'java.base/java.util.concurrent.atomic',
+    'java.base/java.lang',
+    'java.base/java.lang.invoke',
+    'java.base/java.math',
+    'java.sql/java.sql',
+
+    // Confirmed runtime requires
+    'java.base/javax.net.ssl',
+	'java.base/java.net',
+
+    // Runtime requires and misc found from previous jdk17 upgrade attempt
+    // One was needed, so I'll just include them all instead of trial-and-error it
+    'jdk.unsupported/sun.misc',
+    'java.base/java.lang.ref',
+    'java.base/java.lang.reflect',
+    'java.base/java.security',
+    'java.base/java.security.ssl',
+    'java.base/java.security.cert',
+    'java.base/sun.security.rsa',
+    'java.base/sun.security.ssl',
+    'java.base/sun.security.x500',
+    'java.base/sun.security.pkcs12',
+    'java.base/sun.security.provider',
+    'java.base/javax.security.auth.x500'
+    ]
+
+ext.common_java_args = ext.java_internal_opens.collect { '--add-opens=' + it + '=ALL-UNNAMED' }
+//// 'exports' is not a typo. As per javac, "warning: [options] --add-opens has no effect at compile time"
+ext.common_javac_args = ext.java_internal_opens.collect { '--add-exports=' + it + '=ALL-UNNAMED' }
+ext.common_manifest = ['Add-Opens': ext.java_internal_opens.join(' ')]
+
+  // Apply compiler exports and opens
+  tasks.withType(JavaCompile.class).configureEach {
+    options.compilerArgs += common_javac_args
+  }
+
+
+  // Apply test exports and opens
+  tasks.withType(Test.class).configureEach {
+    jvmArgs += common_java_args
+  }
+
+  // Apply Jar (which BootJar and ShadowJar extends) exports and opens manifest entries
+  tasks.withType(Jar.class).configureEach {
+    manifest.attributes.putAll(common_manifest)
+  }
+
+  // Apply War (which BootWar extends) exports and opens manifest entries
+  tasks.withType(War.class).configureEach {
+    manifest.attributes.putAll(common_manifest)
+  }
 
 // alternate way to set target
 // https://github.com/gradle/gradle/issues/2510

src/federation-common/docker/Dockerfile.fedhub

@@ -1,4 +1,4 @@
-FROM openjdk:11-jdk-bullseye
+FROM eclipse-temurin:17-jammy
 RUN apt update && \
 	apt-get install -y emacs-nox net-tools netcat vim
 

src/federation-common/src/main/java/tak/server/federation/hub/broker/FederationHubServerConfig.java

@@ -59,7 +59,6 @@ public FederationHubServerConfig() {
     private String nonce;
     @JsonIgnore
     private String fullId;
-
 
     public int getOutgoingReconnectSeconds() {
 		return outgoingReconnectSeconds;

src/federation-common/src/main/java/tak/server/federation/hub/ui/FederationHubUIConfig.java

@@ -21,6 +21,20 @@ public class FederationHubUIConfig {
     private String authUsers = AUTH_USER_FILE_DEFAULT;
 
     private Integer port = 9100;
+
+    private boolean allowOauth  = false;
+    private Integer oauthPort;
+    private String keycloakServerName;
+    private String keycloakDerLocation;
+    private String keycloakClientId;
+    private String keycloakSecret;
+    private String keycloakrRedirectUri;
+    private String keycloakAuthEndpoint;
+    private String keycloakTokenEndpoint;
+    private String keycloakClaimName;
+    private String keycloakAdminClaimValue;
+    private String keycloakAccessTokenName = "access_token";
+    private String keycloakRefreshTokenName = "refresh_token";
 
     public String getKeystoreType() {
         return keystoreType;
@@ -83,27 +97,96 @@ public void setPort(Integer port) {
         this.port = port;
     }
 
-    @Override
-    public String toString() {
-        StringBuilder builder = new StringBuilder();
-        builder.append("FederationHubUIConfig [keystoreType=");
-        builder.append(keystoreType);
-        builder.append(", keystoreFile=");
-        builder.append(keystoreFile);
-        builder.append(", keystorePassword=");
-        builder.append(keystorePassword);
-        builder.append(", truststoreType=");
-        builder.append(truststoreType);
-        builder.append(", truststoreFile=");
-        builder.append(truststoreFile);
-        builder.append(", truststorePassword=");
-        builder.append(truststorePassword);
-        builder.append(", keyAlias=");
-        builder.append(keyAlias);
-        builder.append(", port=");
-        builder.append(port);
-        builder.append("]");
-
-        return builder.toString();
-    }
+    public Integer getOauthPort() {
+		return oauthPort;
+	}
+	public void setOauthPort(Integer oauthPort) {
+		this.oauthPort = oauthPort;
+	}
+	public String getKeycloakServerName() {
+		return keycloakServerName;
+	}
+	public void setKeycloakServerName(String keycloakServerName) {
+		this.keycloakServerName = keycloakServerName;
+	}
+	public String getKeycloakDerLocation() {
+		return keycloakDerLocation;
+	}
+	public void setKeycloakDerLocation(String keycloakDerLocation) {
+		this.keycloakDerLocation = keycloakDerLocation;
+	}
+	public String getKeycloakClientId() {
+		return keycloakClientId;
+	}
+	public void setKeycloakClientId(String keycloakClientId) {
+		this.keycloakClientId = keycloakClientId;
+	}
+	public String getKeycloakSecret() {
+		return keycloakSecret;
+	}
+	public void setKeycloakSecret(String keycloakSecret) {
+		this.keycloakSecret = keycloakSecret;
+	}
+	public String getKeycloakrRedirectUri() {
+		return keycloakrRedirectUri;
+	}
+	public void setKeycloakrRedirectUri(String keycloakrRedirectUri) {
+		this.keycloakrRedirectUri = keycloakrRedirectUri;
+	}
+	public String getKeycloakAuthEndpoint() {
+		return keycloakAuthEndpoint;
+	}
+	public void setKeycloakAuthEndpoint(String keycloakAuthEndpoint) {
+		this.keycloakAuthEndpoint = keycloakAuthEndpoint;
+	}
+	public String getKeycloakTokenEndpoint() {
+		return keycloakTokenEndpoint;
+	}
+	public void setKeycloakTokenEndpoint(String keycloakTokenEndpoint) {
+		this.keycloakTokenEndpoint = keycloakTokenEndpoint;
+	}
+	public boolean isAllowOauth() {
+		return allowOauth;
+	}
+	public void setAllowOauth(boolean allowOauth) {
+		this.allowOauth = allowOauth;
+	}
+	public String getKeycloakAccessTokenName() {
+		return keycloakAccessTokenName;
+	}
+	public void setKeycloakAccessTokenName(String keycloakAccessTokenName) {
+		this.keycloakAccessTokenName = keycloakAccessTokenName;
+	}
+	public String getKeycloakRefreshTokenName() {
+		return keycloakRefreshTokenName;
+	}
+	public void setKeycloakRefreshTokenName(String keycloakRefreshTokenName) {
+		this.keycloakRefreshTokenName = keycloakRefreshTokenName;
+	}
+	public String getKeycloakClaimName() {
+		return keycloakClaimName;
+	}
+	public void setKeycloakClaimName(String keycloakClaimName) {
+		this.keycloakClaimName = keycloakClaimName;
+	}
+
+	public String getKeycloakAdminClaimValue() {
+		return keycloakAdminClaimValue;
+	}
+	public void setKeycloakAdminClaimValue(String keycloakAdminClaimValue) {
+		this.keycloakAdminClaimValue = keycloakAdminClaimValue;
+	}
+	@Override
+	public String toString() {
+		return "FederationHubUIConfig [keystoreType=" + keystoreType + ", keystoreFile=" + keystoreFile
+				+ ", keystorePassword=" + keystorePassword + ", truststoreType=" + truststoreType + ", truststoreFile="
+				+ truststoreFile + ", keyAlias=" + keyAlias + ", authUsers=" + authUsers + ", port=" + port
+				+ ", allowOauth=" + allowOauth + ", oauthPort=" + oauthPort + ", keycloakServerName="
+				+ keycloakServerName + ", keycloakDerLocation=" + keycloakDerLocation + ", keycloakClientId="
+				+ keycloakClientId + ", keycloakSecret=" + keycloakSecret + ", keycloakrRedirectUri="
+				+ keycloakrRedirectUri + ", keycloakAuthEndpoint=" + keycloakAuthEndpoint + ", keycloakTokenEndpoint="
+				+ keycloakTokenEndpoint + ", keycloakAccessTokenName=" + keycloakAccessTokenName
+				+ ", keycloakRefreshTokenName=" + keycloakRefreshTokenName + ", keycloakClaimName=" + keycloakClaimName
+				+ ", keycloakAdminClaimValue=" + keycloakAdminClaimValue + "]";
+	}
 }

src/federation-common/src/main/java/tak/server/federation/message/Message.java

@@ -4,12 +4,9 @@
 
 import java.io.IOException;
 import java.net.URI;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.Locale;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
@@ -18,7 +15,7 @@
 
 import javax.security.auth.Subject;
 
-import org.apache.commons.codec.binary.Hex;
+//import org.apache.commons.codec.binary.Hex;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -516,33 +513,13 @@ public String printIFPnodeID() {
 		return result.toString();
 	}
 
-	/**
-	 * Returns a SHA-256 hash of the message payload
-	 */
-	public String printPayloadHash() {
-		if (this.payload == null) {
-			return null;
-		}
-		String result = null;
-		try {
-			MessageDigest hash  = MessageDigest.getInstance("SHA-256");
-			hash.update(this.payload.getBytes());
-			String lowercase = new String(Hex.encodeHex(hash.digest()));
-			result = lowercase.toUpperCase(Locale.getDefault());
-		} catch (NoSuchAlgorithmException e) {
-			System.err.println("Unable to generate message payload hash: " + e);
-		}
-		return result;
-	}
-
 	@Override
 	public String toString() {
 		StringBuilder sbuilder = new StringBuilder(100);
 
 		sbuilder.append("\n=== BEGIN Message ").append(getMessageID())
 		.append(NEWLINE_TAB).append(printIFPnodeID()).append(NEWLINE_TAB).append(printMetadata()).append(NEWLINE_TAB)
 		.append(payload.getClass().getName()).append('\n')
-		.append(printPayloadHash())
 		.append("\n=== END ===");
 
 		return sbuilder.toString();

src/federation-hub-broker/build.gradle

@@ -28,12 +28,16 @@ sourceSets {
   }
 }
 dependencies {
-  implementation group: 'org.apache.logging.log4j', name: 'log4j-api', version: log4j_api_version
-  implementation group: 'org.apache.logging.log4j', name: 'log4j-to-slf4j', version: log4j_api_version
+
+
+  implementation group: 'xerces', name: 'xercesImpl', version: xerces_version
   implementation group: 'org.slf4j', name: 'slf4j-api', version: slf4j_version
-  implementation group: 'org.slf4j', name: 'log4j-over-slf4j', version: slf4j_version
 
-  implementation project(':federation-common')
+  implementation(project(':federation-common')) {
+    exclude group: 'log4j', module: 'log4j'
+    exclude group: 'org.apache.logging.log4j'
+    exclude group: 'org.slf4j', module: 'log4j-over-slf4j'
+  }
 
   implementation group: 'com.h2database', name: 'h2', version: h2_version
 
@@ -52,7 +56,12 @@ dependencies {
   implementation "io.netty:netty-tcnative-boringssl-static:$netty_tcnative_version:osx-aarch_64"
   implementation "io.netty:netty-tcnative-boringssl-static:$netty_tcnative_version:windows-x86_64"
 
-  implementation group: 'org.springframework.boot', name: 'spring-boot-starter-actuator'
+  // exclude from actuator
+  implementation ("org.springframework.boot:spring-boot-starter-actuator:$spring_boot_version") {
+    exclude group: 'org.apache.logging.log4j'
+    exclude group: 'org.slf4j', module: 'log4j-over-slf4j'
+  }
+
   implementation group: 'org.springframework.boot', name: 'spring-boot-loader', version: spring_boot_version
   implementation group: 'org.springframework', name: 'spring-context'