Composite Actions Support for Multiple Run Steps

src/Runner.Worker/ActionManager.cs

@@ -395,6 +395,11 @@ public Definition LoadAction(IExecutionContext executionContext, Pipelines.Actio
                             Trace.Info($"Action cleanup plugin: {plugin.PluginTypeName}.");
                         }
                     }
+                    else if (definition.Data.Execution.ExecutionType == ActionExecutionType.Composite && !String.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA")))
+                    {
+                        var compositeAction = definition.Data.Execution as CompositeActionExecutionData;
+                        Trace.Info($"Action steps: {compositeAction.Steps}.");
+                    }
                     else
                     {
                         throw new NotSupportedException(definition.Data.Execution.ExecutionType.ToString());
@@ -1101,6 +1106,11 @@ private ActionContainer PrepareRepositoryActionAsync(IExecutionContext execution
                     Trace.Info($"Action plugin: {(actionDefinitionData.Execution as PluginActionExecutionData).Plugin}, no more preparation.");
                     return null;
                 }
+                else if (actionDefinitionData.Execution.ExecutionType == ActionExecutionType.Composite && !String.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA")))
+                {
+                    Trace.Info($"Action composite: {(actionDefinitionData.Execution as CompositeActionExecutionData).Steps}, no more preparation.");
+                    return null;
+                }
                 else
                 {
                     throw new NotSupportedException(actionDefinitionData.Execution.ExecutionType.ToString());
@@ -1211,6 +1221,7 @@ public enum ActionExecutionType
         NodeJS,
         Plugin,
         Script,
+        Composite
     }
 
     public sealed class ContainerActionExecutionData : ActionExecutionData
@@ -1267,6 +1278,14 @@ public sealed class ScriptActionExecutionData : ActionExecutionData
         public override bool HasPost => false;
     }
 
+    public sealed class CompositeActionExecutionData : ActionExecutionData
+    {
+        public override ActionExecutionType ExecutionType => ActionExecutionType.Composite;
+        public override bool HasPre => false;
+        public override bool HasPost => false;
+        public List<Pipelines.ActionStep> Steps { get; set; }
+    }
+
     public abstract class ActionExecutionData
     {
         private string _initCondition = $"{Constants.Expressions.Always}()";

src/Runner.Worker/ActionManifestManager.cs

@@ -14,6 +14,7 @@
 using YamlDotNet.Core.Events;
 using System.Globalization;
 using System.Linq;
+using Pipelines = GitHub.DistributedTask.Pipelines;
 
 namespace GitHub.Runner.Worker
 {
@@ -92,7 +93,7 @@ public ActionDefinitionData Load(IExecutionContext executionContext, string mani
                             break;
 
                         case "runs":
-                            actionDefinition.Execution = ConvertRuns(context, actionPair.Value);
+                            actionDefinition.Execution = ConvertRuns(executionContext, context, actionPair.Value);
                             break;
                         default:
                             Trace.Info($"Ignore action property {propertyName}.");
@@ -284,7 +285,7 @@ private TemplateContext CreateContext(
             // Add the file table
             if (_fileTable?.Count > 0)
             {
-                for (var i = 0 ; i < _fileTable.Count ; i++)
+                for (var i = 0; i < _fileTable.Count; i++)
                 {
                     result.GetFileId(_fileTable[i]);
                 }
@@ -294,6 +295,7 @@ private TemplateContext CreateContext(
         }
 
         private ActionExecutionData ConvertRuns(
+            IExecutionContext executionContext,
             TemplateContext context,
             TemplateToken inputsToken)
         {
@@ -311,6 +313,8 @@ private ActionExecutionData ConvertRuns(
             var postToken = default(StringToken);
             var postEntrypointToken = default(StringToken);
             var postIfToken = default(StringToken);
+            var stepsLoaded = default(List<Pipelines.ActionStep>);
+
             foreach (var run in runsMapping)
             {
                 var runsKey = run.Key.AssertString("runs key").Value;
@@ -355,6 +359,15 @@ private ActionExecutionData ConvertRuns(
                     case "pre-if":
                         preIfToken = run.Value.AssertString("pre-if");
                         break;
+                    case "steps":
+                        if (!String.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA")))
+                        {
+                            var steps = run.Value.AssertSequence("steps");
+                            var evaluator = executionContext.ToPipelineTemplateEvaluator();
+                            stepsLoaded = evaluator.LoadCompositeSteps(steps);
+                            break;
+                        }
+                        throw new Exception("You aren't supposed to be using Composite Actions yet!");
                     default:
                         Trace.Info($"Ignore run property {runsKey}.");
                         break;
@@ -402,6 +415,20 @@ private ActionExecutionData ConvertRuns(
                         };
                     }
                 }
+                else if (string.Equals(usingToken.Value, "composite", StringComparison.OrdinalIgnoreCase) && !String.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA")))
+                {
+                    if (stepsLoaded == null)
+                    {
+                        throw new ArgumentNullException($"No steps provided.");
+                    }
+                    else
+                    {
+                        return new CompositeActionExecutionData()
+                        {
+                            Steps = stepsLoaded,
+                        };
+                    }
+                }
                 else
                 {
                     throw new ArgumentOutOfRangeException($"'using: {usingToken.Value}' is not supported, use 'docker' or 'node12' instead.");

src/Runner.Worker/ExecutionContext.cs

@@ -105,6 +105,8 @@ public interface IExecutionContext : IRunnerService
         // others
         void ForceTaskComplete();
         void RegisterPostJobStep(IStep step);
+        IStep RegisterCompositeStep(IStep step, DictionaryContextData inputsData);
+        void EnqueueAllCompositeSteps(Queue<IStep> steps);
     }
 
     public sealed class ExecutionContext : RunnerService, IExecutionContext
@@ -169,7 +171,6 @@ public sealed class ExecutionContext : RunnerService, IExecutionContext
 
         public bool EchoOnActionCommand { get; set; }
 
-
         public TaskResult? Result
         {
             get
@@ -266,6 +267,64 @@ public void RegisterPostJobStep(IStep step)
             Root.PostJobSteps.Push(step);
         }
 
+        /*
+            RegisterCompositeStep is a helper function used in CompositeActionHandler::RunAsync to 
+            add a child node, aka a step, to the current job to the front of the queue for processing. 
+        */
+        public IStep RegisterCompositeStep(IStep step, DictionaryContextData inputsData)
+        {
+            // ~Brute Force Method~
+            // There is no way to put this current job in front of the queue in < O(n) time where n = # of elements in JobSteps
+            // Everytime we add a new step, you could requeue every item to put those steps from that stack in JobSteps which 
+            // would result in O(n) for each time we add a composite action step where n = number of jobSteps which would compound 
+            // to O(n*m) where m = number of composite steps
+            // var temp = Root.JobSteps.ToArray();
+            // Root.JobSteps.Clear();
+            // Root.JobSteps.Enqueue(step);
+            // foreach(var s in temp)
+            //     Root.JobSteps.Enqueue(s);
+
+            // ~Optimized Method~
+            // Alterative solution: We add to another temp Queue
+            // After we add all the transformed composite steps to this temp queue, we requeue the whole JobSteps accordingly in EnqueueAllCompositeSteps()
+            // where the queued composite steps are at the front of the JobSteps Queue and the rest of the jobs maintain its order and are 
+            // placed after the queued composite steps
+            // This will take only O(n+m) time which would be just as efficient if we refactored the code of JobSteps to a PriorityQueue
+            // This temp Queue is created in the CompositeActionHandler. 
+            var newGuid = Guid.NewGuid();
+            step.ExecutionContext = Root.CreateChild(newGuid, step.DisplayName, newGuid.ToString("N"), null, null);
+            step.ExecutionContext.ExpressionValues["inputs"] = inputsData;
+            return step;
+        }
+
+        // Add Composite Steps first and then requeue the rest of the job steps. 
+        public void EnqueueAllCompositeSteps(Queue<IStep> steps)
+        {
+            // TODO: For UI purposes, look at figuring out how to condense steps in one node
+            //  maybe use "this" instead of "Root"?
+            if (Root.JobSteps != null)
+            {
+                var temp = Root.JobSteps.ToArray();
+                Root.JobSteps.Clear();
+                foreach (var cs in steps)
+                {
+                    Root.JobSteps.Enqueue(cs);
+                }
+                foreach (var s in temp)
+                {
+                    Root.JobSteps.Enqueue(s);
+                }
+            }
+            else
+            {
+                Root.JobSteps = new Queue<IStep>();
+                foreach (var cs in steps)
+                {
+                    Root.JobSteps.Enqueue(cs);
+                }
+            }
+        }
+
         public IExecutionContext CreateChild(Guid recordId, string displayName, string refName, string scopeName, string contextName, Dictionary<string, string> intraActionState = null, int? recordOrder = null)
         {
             Trace.Entering();

src/Runner.Worker/Handlers/CompositeActionHandler.cs

@@ -0,0 +1,97 @@
+using System.IO;
+using System.Text;
+using System.Threading.Tasks;
+using GitHub.Runner.Common;
+using GitHub.Runner.Sdk;
+using GitHub.DistributedTask.WebApi;
+using Pipelines = GitHub.DistributedTask.Pipelines;
+using System;
+using System.Linq;
+using GitHub.DistributedTask.ObjectTemplating.Tokens;
+using System.Collections.Generic;
+using GitHub.DistributedTask.Pipelines.ContextData;
+
+namespace GitHub.Runner.Worker.Handlers
+{
+    [ServiceLocator(Default = typeof(CompositeActionHandler))]
+    public interface ICompositeActionHandler : IHandler
+    {
+        CompositeActionExecutionData Data { get; set; }
+    }
+    public sealed class CompositeActionHandler : Handler, ICompositeActionHandler
+    {
+        public CompositeActionExecutionData Data { get; set; }
+
+        public Task RunAsync(ActionRunStage stage)
+        {
+            // Validate args.
+            Trace.Entering();
+            ArgUtil.NotNull(ExecutionContext, nameof(ExecutionContext));
+            ArgUtil.NotNull(Inputs, nameof(Inputs));
+
+            var githubContext = ExecutionContext.ExpressionValues["github"] as GitHubContext;
+            ArgUtil.NotNull(githubContext, nameof(githubContext));
+
+            var tempDirectory = HostContext.GetDirectory(WellKnownDirectory.Temp);
+
+            // Resolve action steps
+            var actionSteps = Data.Steps;
+
+            // Create Context Data to reuse for each composite action step
+            var inputsData = new DictionaryContextData();
+            foreach (var i in Inputs)
+            {
+                inputsData[i.Key] = new StringContextData(i.Value);
+            }
+
+            // Add each composite action step to the front of the queue
+            var compositeActionSteps = new Queue<IStep>();
+            foreach (Pipelines.ActionStep aStep in actionSteps)
+            {
+                // Ex: 
+                // runs:
+                //      using: "composite"
+                //      steps:
+                //          - uses: example/test-composite@v2 (a)
+                //          - run echo hello world (b)
+                //          - run echo hello world 2 (c)
+                // 
+                // ethanchewy/test-composite/action.yaml
+                // runs:
+                //      using: "composite"
+                //      steps: 
+                //          - run echo hello world 3 (d)
+                //          - run echo hello world 4 (e)
+                // 
+                // Stack (LIFO) [Bottom => Middle => Top]:
+                // | a |
+                // | a | => | d |
+                // (Run step d)
+                // | a | 
+                // | a | => | e |
+                // (Run step e)
+                // | a | 
+                // (Run step a)
+                // | b | 
+                // (Run step b)
+                // | c |
+                // (Run step c)
+                // Done.
+
+                var actionRunner = HostContext.CreateService<IActionRunner>();
+                actionRunner.Action = aStep;
+                actionRunner.Stage = stage;
+                actionRunner.Condition = aStep.Condition;
+                actionRunner.DisplayName = aStep.DisplayName;
+                // TODO: Do we need to add any context data from the job message?
+                // (See JobExtension.cs ~line 236)
+
+                compositeActionSteps.Enqueue(ExecutionContext.RegisterCompositeStep(actionRunner, inputsData));
+            }
+            ExecutionContext.EnqueueAllCompositeSteps(compositeActionSteps);
+
+            return Task.CompletedTask;
+        }
+
+    }
+}

src/Runner.Worker/Handlers/HandlerFactory.cs

@@ -66,6 +66,14 @@ public IHandler Create(
                 handler = HostContext.CreateService<IRunnerPluginHandler>();
                 (handler as IRunnerPluginHandler).Data = data as PluginActionExecutionData;
             }
+            else if (data.ExecutionType == ActionExecutionType.Composite)
+            {
+                // TODO
+                // Runner plugin
+                handler = HostContext.CreateService<ICompositeActionHandler>();
+                // handler = CompositeHandler;
+                (handler as ICompositeActionHandler).Data = data as CompositeActionExecutionData;
+            }
             else
             {
                 // This should never happen.

src/Runner.Worker/StepsRunner.cs

@@ -67,6 +67,11 @@ public async Task RunAsync(IExecutionContext jobContext)
 
                 var step = jobContext.JobSteps.Dequeue();
                 var nextStep = jobContext.JobSteps.Count > 0 ? jobContext.JobSteps.Peek() : null;
+                // TODO: Fix this temporary workaround for Composite Actions
+                if (!String.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA")))
+                {
+                    nextStep = null;
+                }
 
                 Trace.Info($"Processing step: DisplayName='{step.DisplayName}'");
                 ArgUtil.NotNull(step.ExecutionContext, nameof(step.ExecutionContext));
@@ -409,7 +414,11 @@ private bool InitializeScope(IStep step, Dictionary<string, PipelineContextData>
                     scope = scopesToInitialize.Pop();
                     executionContext.Debug($"Initializing scope '{scope.Name}'");
                     executionContext.ExpressionValues["steps"] = stepsContext.GetScope(scope.ParentName);
-                    executionContext.ExpressionValues["inputs"] = !String.IsNullOrEmpty(scope.ParentName) ? scopeInputs[scope.ParentName] : null;
+                    // TODO: Fix this temporary workaround for Composite Actions
+                    if (!executionContext.ExpressionValues.ContainsKey("inputs") && !String.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA")))
+                    {
+                        executionContext.ExpressionValues["inputs"] = !String.IsNullOrEmpty(scope.ParentName) ? scopeInputs[scope.ParentName] : null;
+                    }
                     var templateEvaluator = executionContext.ToPipelineTemplateEvaluator();
                     var inputs = default(DictionaryContextData);
                     try
@@ -432,7 +441,11 @@ private bool InitializeScope(IStep step, Dictionary<string, PipelineContextData>
             // Setup expression values
             var scopeName = executionContext.ScopeName;
             executionContext.ExpressionValues["steps"] = stepsContext.GetScope(scopeName);
-            executionContext.ExpressionValues["inputs"] = string.IsNullOrEmpty(scopeName) ? null : scopeInputs[scopeName];
+            // TODO: Fix this temporary workaround for Composite Actions
+            if (!executionContext.ExpressionValues.ContainsKey("inputs") && !String.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA")))
+            {
+                executionContext.ExpressionValues["inputs"] = string.IsNullOrEmpty(scopeName) ? null : scopeInputs[scopeName];
+            }
 
             return true;
         }