feat(attach): implement basic dynamic attach (#234)

README.md

@@ -34,19 +34,43 @@ is an available version upgrade and ensure both your Agent and server match this
 
 ### Build Requirements
 - Git
-- JDK11+
+- OpenJDK11+
 - Maven 3+
 
 Run Requirements:
-- An application JVM to attach this agent to
+- A OpenJDK11+ application JVM to attach this agent to
 - Configuration for the application JVM to load this agent
+- A [Cryostat server](https://github.com/cryostatio/cryostat) instance
 
+## Run
 
 An example for configuring a Quarkus application to use this agent and enable JMX:
 ```
 JAVA_OPTIONS="-Dcom.sun.management.jmxremote.port=9091 -Dcom.sun.management.jmxremote.rmi.port=9091 -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -javaagent:/deployments/app/cryostat-agent-${CRYOSTAT_AGENT_VERSION}.jar"
 ```
-This assumes that the agent JAR has been included in the application image within `/deployments/app/`.
+This assumes that the agent JAR has been included in the application image within `/deployments/app/` or mounted as a
+volume at the same location.
+
+The agent JAR may also be loaded and dynamically attached to an already-running JVM. In this case, the agent JAR must
+again be already included in the application image mounted as a volume into the container at the same location. If this
+requirement is met, then the host JVM application may be started first without the Cryostat Agent with whatever its own
+standard launcher process looks like. Once that application is running, the Cryostat Agent can be launched as a separate
+process and asked to dynamically attach to the host application:
+```
+$ java -jar /path/to/cryostat-agent.jar -Dcryostat.agent.baseuri=http://cryostat.local
+```
+
+In this dynamic attachment mode, the agent [configuration](#configuration) options can be specified using the
+`-D`/`property` flag. These must be placed *after* the `-jar /path/to/cryostat-agent.jar` in order to be passed as
+arguments to the agent launcher - if they are passed *before* the `-jar` then they will be used by the `java` process
+as system properties on the agent launcher itself, rather than having them passed on to the injected instances.
+
+[Smart triggers](#smart-triggers) can be specified using `--smartTrigger`.
+
+The optional PID is a positional argument and may be ignored or set to: `0` to request that the Agent launcher attempt
+to find exactly one candidate JVM application to dynamically attach to, exiting if zero or more than one applications
+are found; `*` to request that the Agent launch attempt to dynamically attach to every JVM application it finds; or a
+specific PID to request that the Agent attempt to dynamically attach only to that one PID.
 
 ## Harvester
 
@@ -55,7 +79,7 @@ Recording using a given event template on Agent initialization, and to periodica
 it to the Agent's associated Cryostat server. The Agent will also attempt to push the tail end of this recording on JVM
 shutdown so that the cause of an unexpected JVM shutdown might be captured for later analysis.
 
-## SMART TRIGGERS
+## Smart Triggers
 
 `cryostat-agent` supports Smart Triggers that listen to the values of the MBean Counters and can start recordings based
 on a set of constraints specified by the user.
@@ -102,9 +126,11 @@ Smart Triggers may define more complex conditions that test multiple metrics:
 These may be passed as an argument to the Cryostat Agent, for example:
 
 ```
-JAVA_OPTIONS="-javaagent:/deployments/app/cryostat-agent-${CRYOSTAT_AGENT_VERSION}.jar=[ProcessCpuLoad>0.2]~profile
+JAVA_OPTIONS="-javaagent:-Dcryostat.agent.baseuri=http://cryostat.local!/deployments/app/cryostat-agent-${CRYOSTAT_AGENT_VERSION}.jar=[ProcessCpuLoad>0.2]~profile
 ```
 
+(note the '!' separator between system properties overrides and Smart Triggers)
+
 or as a [configuration property](#configuration):
 
 ```
@@ -130,7 +156,7 @@ Harvester period without a Harvester template, you can achieve a setup where dyn
 begin when trigger conditions are met, and their data is then periodically captured until the recording is manually
 stopped or the host JVM shuts down.
 
-## CONFIGURATION
+## Configuration
 
 `cryostat-agent` uses [smallrye-config](https://github.com/smallrye/smallrye-config) for configuration.
 Below is a list of configuration properties that can be used to influence how `cryostat-agent` runs

pom.xml

@@ -87,6 +87,7 @@
   <org.slf4j.version>2.0.12</org.slf4j.version>
   <org.projectnessie.cel.bom.version>0.4.4</org.projectnessie.cel.bom.version>
   <com.google.protobuf-java.version>3.22.3</com.google.protobuf-java.version>
+  <info.picocli.version>4.7.4</info.picocli.version>
   <com.redhat.insights.agent.version>0.9.1</com.redhat.insights.agent.version>
 
   <com.github.spotbugs.version>4.8.3</com.github.spotbugs.version>
@@ -177,6 +178,11 @@
       </exclusion>
     </exclusions>
   </dependency>
+  <dependency>
+    <groupId>info.picocli</groupId>
+    <artifactId>picocli</artifactId>
+    <version>${info.picocli.version}</version>
+  </dependency>
   <dependency>
     <groupId>org.projectnessie.cel</groupId>
     <artifactId>cel-tools</artifactId>
@@ -278,6 +284,7 @@
       <configuration>
         <archive>
           <manifest>
+            <addDefaultImplementationEntries>true</addDefaultImplementationEntries>
             <mainClass>${mainClass}</mainClass>
           </manifest>
           <manifestEntries>
@@ -349,6 +356,10 @@
                 <pattern>com.fasterxml.jackson</pattern>
                 <shadedPattern>${shade.prefix}.com.fasterxml.jackson</shadedPattern>
               </relocation>
+              <relocation>
+                <pattern>picocli</pattern>
+                <shadedPattern>${shade.prefix}.picocli</shadedPattern>
+              </relocation>
             </relocations>
             <transformers>
               <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />

src/main/java/io/cryostat/agent/Agent.java

@@ -15,17 +15,24 @@
  */
 package io.cryostat.agent;
 
+import java.io.IOException;
 import java.lang.instrument.Instrumentation;
 import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.file.Path;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Consumer;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
 
 import javax.inject.Named;
 import javax.inject.Singleton;
@@ -36,23 +43,175 @@
 import io.cryostat.agent.model.PluginInfo;
 import io.cryostat.agent.triggers.TriggerEvaluator;
 
+import com.sun.tools.attach.AgentInitializationException;
+import com.sun.tools.attach.AgentLoadException;
+import com.sun.tools.attach.AttachNotSupportedException;
+import com.sun.tools.attach.VirtualMachine;
+import com.sun.tools.attach.VirtualMachineDescriptor;
 import dagger.Component;
+import org.eclipse.microprofile.config.Config;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import picocli.CommandLine;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.IVersionProvider;
+import picocli.CommandLine.Option;
+import picocli.CommandLine.Parameters;
 import sun.misc.Signal;
 import sun.misc.SignalHandler;
 
-public class Agent {
+@Command(
+        name = "CryostatAgent",
+        mixinStandardHelpOptions = true,
+        showAtFileInUsageHelp = true,
+        versionProvider = Agent.VersionProvider.class,
+        description =
+                "Launcher for Cryostat Agent to self-inject and dynamically attach to workload"
+                        + " JVMs")
+public class Agent implements Callable<Integer>, Consumer<AgentArgs> {
 
     private static Logger log = LoggerFactory.getLogger(Agent.class);
     private static final AtomicBoolean needsCleanup = new AtomicBoolean(true);
+    private static final String ALL_PIDS = "*";
+    static final String AUTO_ATTACH_PID = "0";
+
+    @Parameters(
+            index = "0",
+            defaultValue = "0",
+            arity = "0..1",
+            description =
+                    "The PID to attach to and attempt to self-inject the Cryostat Agent. If not"
+                        + " specified, the Agent will look to find exactly one candidate and attach"
+                        + " to that, failing if none or more than one are found. Otherwise, this"
+                        + " should be a process ID, or the '*' wildcard to request the Agent"
+                        + " attempt to attach to all discovered JVMs.")
+    private String pid;
+
+    @Option(
+            names = {"-D", "--property"},
+            description =
+                    "Optional property definitions to supply to the injected Agent copies to add or"
+                        + " override property definitions once the Agent is running in the workload"
+                        + " JVM. These should be specified as key=value pairs, ex."
+                        + " -Dcryostat.agent.baseuri=http://cryostat.service.local . May be"
+                        + " specified more than once.")
+    private Map<String, String> properties;
+
+    @Option(
+            names = "--smartTrigger",
+            description = "Smart Triggers definition. May be specified more than once.")
+    private List<String> smartTriggers;
+
+    // standalone entry point, Agent is started separately and should self-inject and dynamic attach
+    // to the host JVM application. After the agent is dynamically attached we should begin
+    // execution again in agentmain
+    public static void main(String[] args) {
+        int exitCode = new CommandLine(new Agent()).execute(args);
+        System.exit(exitCode);
+    }
+
+    @Override
+    public Integer call()
+            throws IOException,
+                    AttachNotSupportedException,
+                    AgentInitializationException,
+                    AgentLoadException,
+                    URISyntaxException {
+        log.trace("main");
+        List<String> pids = getAttachPid(pid);
+        if (pids.isEmpty()) {
+            throw new IllegalStateException("No candidate JVM PIDs");
+        }
+        String agentmainArg =
+                new AgentArgs(
+                                properties,
+                                String.join(",", smartTriggers != null ? smartTriggers : List.of()))
+                        .toAgentMain();
+        log.trace("agentmain arg: \"{}\"", agentmainArg);
+        for (String pid : pids) {
+            VirtualMachine vm = VirtualMachine.attach(pid);
+            log.info("Injecting agent into PID {}", pid);
+            try {
+                vm.loadAgent(Path.of(selfJarLocation()).toAbsolutePath().toString(), agentmainArg);
+            } finally {
+                vm.detach();
+            }
+        }
+        return 0;
+    }
 
-    private static InsightsAgentHelper insights;
+    // -javaagent entry point, Agent is starting before the host JVM application
+    public static void premain(String args, Instrumentation instrumentation) {
+        log.trace("premain");
+        agentmain(args, instrumentation);
+    }
 
-    public static void main(String[] args) {
+    // dynamic attach entry point, Agent is starting after being loaded and attached to a running
+    // JVM application
+    public static void agentmain(String args, Instrumentation instrumentation) {
+        log.trace("agentmain");
+        AgentArgs aa = AgentArgs.from(instrumentation, args);
+        Agent agent = new Agent();
+        Thread t =
+                new Thread(
+                        () -> {
+                            log.info("Cryostat Agent starting...");
+                            agent.accept(aa);
+                        });
+        t.setDaemon(true);
+        t.setName("cryostat-agent-main");
+        t.start();
+    }
+
+    private static List<String> getAttachPid(String pidSpec) {
+        List<VirtualMachineDescriptor> vms = VirtualMachine.list();
+        Predicate<VirtualMachineDescriptor> vmFilter;
+        if (ALL_PIDS.equals(pidSpec)) {
+            vmFilter = vmd -> true;
+        } else if (pidSpec == null || AUTO_ATTACH_PID.equals(pidSpec)) {
+            if (vms.size() > 2) { // one of them is ourself
+                throw new IllegalStateException(
+                        String.format(
+                                "Too many available virtual machines. Auto-attach only progresses"
+                                        + " if there is one candidate. VMs: %s",
+                                vms));
+            } else if (vms.size() < 2) {
+                throw new IllegalStateException(
+                        String.format(
+                                "Too few available virtual machines. Auto-attach only progresses if"
+                                        + " there is one candidate. VMs: %s",
+                                vms));
+            }
+            long ownId = ProcessHandle.current().pid();
+            vmFilter = vmd -> !Objects.equals(String.valueOf(ownId), vmd.id());
+        } else {
+            vmFilter = vmd -> pidSpec.equals(vmd.id());
+        }
+        return vms.stream()
+                .filter(vmFilter)
+                .peek(vmd -> log.info("Attaching to VM: {} {}", vmd.displayName(), vmd.id()))
+                .map(VirtualMachineDescriptor::id)
+                .collect(Collectors.toList());
+    }
+
+    static URI selfJarLocation() throws URISyntaxException {
+        return Agent.class.getProtectionDomain().getCodeSource().getLocation().toURI();
+    }
+
+    @Override
+    public void accept(AgentArgs args) {
+        log.info("Cryostat Agent starting...");
+        args.getProperties()
+                .forEach(
+                        (k, v) -> {
+                            log.info("Set system property {} = {}", k, v);
+                            System.setProperty(k, v);
+                        });
         AgentExitHandler agentExitHandler = null;
         try {
             final Client client = DaggerAgent_Client.builder().build();
+            InsightsAgentHelper insights =
+                    new InsightsAgentHelper(client.config(), args.getInstrumentation());
 
             URI baseUri = client.baseUri();
             URIRange uriRange = client.uriRange();
@@ -112,7 +271,7 @@ public static void main(String[] args) {
                     });
             webServer.start();
             registration.start();
-            client.triggerEvaluator().start(args);
+            client.triggerEvaluator().start(args.getSmartTriggers());
             log.info("Startup complete");
         } catch (Exception e) {
             log.error(Agent.class.getSimpleName() + " startup failure", e);
@@ -144,23 +303,6 @@ private static AgentExitHandler installSignalHandlers(
         return agentExitHandler;
     }
 
-    public static void agentmain(String args, Instrumentation instrumentation) {
-        insights = new InsightsAgentHelper(instrumentation);
-        Thread t =
-                new Thread(
-                        () -> {
-                            log.info("Cryostat Agent starting...");
-                            main(args == null ? new String[0] : args.split("\\s"));
-                        });
-        t.setDaemon(true);
-        t.setName("cryostat-agent-main");
-        t.start();
-    }
-
-    public static void premain(String args, Instrumentation instrumentation) {
-        agentmain(args, instrumentation);
-    }
-
     private static void setupInsightsIfEnabled(
             InsightsAgentHelper insights, PluginInfo pluginInfo) {
         if (insights != null && insights.isInsightsEnabled(pluginInfo)) {
@@ -176,6 +318,8 @@ private static void setupInsightsIfEnabled(
     @Singleton
     @Component(modules = {MainModule.class})
     interface Client {
+        Config config();
+
         @Named(ConfigModule.CRYOSTAT_AGENT_BASEURI)
         URI baseUri();
 
@@ -287,4 +431,12 @@ private void safeCall(Runnable r) {
             }
         }
     }
+
+    static class VersionProvider implements IVersionProvider {
+
+        @Override
+        public String[] getVersion() throws Exception {
+            return new String[] {Agent.class.getPackage().getImplementationVersion()};
+        }
+    }
 }