Skip to content

Commit

Permalink
Composite Run Steps
Browse files Browse the repository at this point in the history
  • Loading branch information
ethanchewy committed Jun 17, 2020
1 parent 5815819 commit c6c0571
Show file tree
Hide file tree
Showing 11 changed files with 661 additions and 16 deletions.
18 changes: 18 additions & 0 deletions src/Runner.Worker/ActionManager.cs
Original file line number Diff line number Diff line change
Expand Up @@ -395,6 +395,10 @@ public Definition LoadAction(IExecutionContext executionContext, Pipelines.Actio
Trace.Info($"Action cleanup plugin: {plugin.PluginTypeName}.");
}
}
else if (definition.Data.Execution.ExecutionType == ActionExecutionType.Composite && !String.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA")))
{
// Don't do anything for now
}
else
{
throw new NotSupportedException(definition.Data.Execution.ExecutionType.ToString());
Expand Down Expand Up @@ -1101,6 +1105,11 @@ private ActionContainer PrepareRepositoryActionAsync(IExecutionContext execution
Trace.Info($"Action plugin: {(actionDefinitionData.Execution as PluginActionExecutionData).Plugin}, no more preparation.");
return null;
}
else if (actionDefinitionData.Execution.ExecutionType == ActionExecutionType.Composite && !String.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA")))
{
Trace.Info($"Action composite: {(actionDefinitionData.Execution as CompositeActionExecutionData).Steps}, no more preparation.");
return null;
}
else
{
throw new NotSupportedException(actionDefinitionData.Execution.ExecutionType.ToString());
Expand Down Expand Up @@ -1211,6 +1220,7 @@ public enum ActionExecutionType
NodeJS,
Plugin,
Script,
Composite
}

public sealed class ContainerActionExecutionData : ActionExecutionData
Expand Down Expand Up @@ -1267,6 +1277,14 @@ public sealed class ScriptActionExecutionData : ActionExecutionData
public override bool HasPost => false;
}

public sealed class CompositeActionExecutionData : ActionExecutionData
{
public override ActionExecutionType ExecutionType => ActionExecutionType.Composite;
public override bool HasPre => false;
public override bool HasPost => false;
public List<Pipelines.ActionStep> Steps { get; set; }
}

public abstract class ActionExecutionData
{
private string _initCondition = $"{Constants.Expressions.Always}()";
Expand Down
31 changes: 29 additions & 2 deletions src/Runner.Worker/ActionManifestManager.cs
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
using YamlDotNet.Core.Events;
using System.Globalization;
using System.Linq;
using Pipelines = GitHub.DistributedTask.Pipelines;

namespace GitHub.Runner.Worker
{
Expand Down Expand Up @@ -92,7 +93,7 @@ public ActionDefinitionData Load(IExecutionContext executionContext, string mani
break;

case "runs":
actionDefinition.Execution = ConvertRuns(context, actionPair.Value);
actionDefinition.Execution = ConvertRuns(executionContext, context, actionPair.Value);
break;
default:
Trace.Info($"Ignore action property {propertyName}.");
Expand Down Expand Up @@ -284,7 +285,7 @@ private TemplateContext CreateContext(
// Add the file table
if (_fileTable?.Count > 0)
{
for (var i = 0 ; i < _fileTable.Count ; i++)
for (var i = 0; i < _fileTable.Count; i++)
{
result.GetFileId(_fileTable[i]);
}
Expand All @@ -294,6 +295,7 @@ private TemplateContext CreateContext(
}

private ActionExecutionData ConvertRuns(
IExecutionContext executionContext,
TemplateContext context,
TemplateToken inputsToken)
{
Expand All @@ -311,6 +313,8 @@ private ActionExecutionData ConvertRuns(
var postToken = default(StringToken);
var postEntrypointToken = default(StringToken);
var postIfToken = default(StringToken);
var stepsLoaded = default(List<Pipelines.ActionStep>);

foreach (var run in runsMapping)
{
var runsKey = run.Key.AssertString("runs key").Value;
Expand Down Expand Up @@ -355,6 +359,15 @@ private ActionExecutionData ConvertRuns(
case "pre-if":
preIfToken = run.Value.AssertString("pre-if");
break;
case "steps":
if (!String.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA")))
{
var steps = run.Value.AssertSequence("steps");
var evaluator = executionContext.ToPipelineTemplateEvaluator();
stepsLoaded = evaluator.LoadCompositeSteps(steps);
break;
}
throw new Exception("You aren't supposed to be using Composite Actions yet!");
default:
Trace.Info($"Ignore run property {runsKey}.");
break;
Expand Down Expand Up @@ -402,6 +415,20 @@ private ActionExecutionData ConvertRuns(
};
}
}
else if (string.Equals(usingToken.Value, "composite", StringComparison.OrdinalIgnoreCase) && !String.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA")))
{
if (stepsLoaded == null)
{
throw new ArgumentNullException($"No steps provided.");
}
else
{
return new CompositeActionExecutionData()
{
Steps = stepsLoaded,
};
}
}
else
{
throw new ArgumentOutOfRangeException($"'using: {usingToken.Value}' is not supported, use 'docker' or 'node12' instead.");
Expand Down
65 changes: 64 additions & 1 deletion src/Runner.Worker/ExecutionContext.cs
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,8 @@ public interface IExecutionContext : IRunnerService
// others
void ForceTaskComplete();
void RegisterPostJobStep(IStep step);
IStep RegisterCompositeStep(IStep step, Dictionary<string, string> inputs, DictionaryContextData inputsData);
void EnqueueAllCompositeSteps(Queue<IStep> steps);
}

public sealed class ExecutionContext : RunnerService, IExecutionContext
Expand Down Expand Up @@ -169,7 +171,6 @@ public sealed class ExecutionContext : RunnerService, IExecutionContext

public bool EchoOnActionCommand { get; set; }


public TaskResult? Result
{
get
Expand Down Expand Up @@ -266,6 +267,68 @@ public void RegisterPostJobStep(IStep step)
Root.PostJobSteps.Push(step);
}

/*
RegisterCompositeStep is a helper function used in CompositeActionHandler::RunAsync to
add a child node, aka a step, to the current job to the front of the queue for processing.
*/
public IStep RegisterCompositeStep(IStep step, Dictionary<string, string> inputs, DictionaryContextData inputsData)
{
// ~Brute Force Method~
// There is no way to put this current job in front of the queue in < O(n) time where n = # of elements in JobSteps
// Everytime we add a new step, you could requeue every item to put those steps from that stack in JobSteps which
// would result in O(n) for each time we add a composite action step where n = number of jobSteps which would compound
// to O(n*m) where m = number of composite steps
// var temp = Root.JobSteps.ToArray();
// Root.JobSteps.Clear();
// Root.JobSteps.Enqueue(step);
// foreach(var s in temp)
// Root.JobSteps.Enqueue(s);

// ~Optimized Method~
// Alterative solution: We add to another temp Queue
// After we add all the transformed composite steps to this temp queue, we requeue the whole JobSteps accordingly in EnqueueAllCompositeSteps()
// where the queued composite steps are at the front of the JobSteps Queue and the rest of the jobs maintain its order and are
// placed after the queued composite steps
// This will take only O(n+m) time which would be just as efficient if we refactored the code of JobSteps to a PriorityQueue
// This temp Queue is created in the CompositeActionHandler.

Trace.Info("Adding Composite Action Step");
var newGuid = Guid.NewGuid();
step.ExecutionContext = Root.CreateChild(newGuid, step.DisplayName, newGuid.ToString("N"), null, null);
step.ExecutionContext.ExpressionValues["inputs"] = inputsData;
return step;
}

// Add Composite Steps first and then requeue the rest of the job steps.
public void EnqueueAllCompositeSteps(Queue<IStep> steps)
{
// TODO: For UI purposes, look at figuring out how to condense steps in one node
// maybe use "this" instead of "Root"?
if (Root.JobSteps != null)
{
var temp = Root.JobSteps.ToArray();
Root.JobSteps.Clear();
foreach (var cs in steps)
{
Trace.Info($"EnqueueAllCompositeSteps : Adding Composite action step {cs}");
Root.JobSteps.Enqueue(cs);
}
foreach (var s in temp)
{
Root.JobSteps.Enqueue(s);
}
}
else
{
Root.JobSteps = new Queue<IStep>();
foreach (var cs in steps)
{
Trace.Info($"EnqueueAllCompositeSteps : Adding Composite action step {cs}");
Root.JobSteps.Enqueue(cs);
}
}
}

public IExecutionContext CreateChild(Guid recordId, string displayName, string refName, string scopeName, string contextName, Dictionary<string, string> intraActionState = null, int? recordOrder = null)
{
Trace.Entering();
Expand Down
105 changes: 105 additions & 0 deletions src/Runner.Worker/Handlers/CompositeActionHandler.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
using System.IO;
using System.Text;
using System.Threading.Tasks;
using GitHub.Runner.Common;
using GitHub.Runner.Sdk;
using GitHub.DistributedTask.WebApi;
using Pipelines = GitHub.DistributedTask.Pipelines;
using System;
using System.Linq;
using GitHub.DistributedTask.ObjectTemplating.Tokens;
using System.Collections.Generic;
using GitHub.DistributedTask.Pipelines.ContextData;

namespace GitHub.Runner.Worker.Handlers
{
[ServiceLocator(Default = typeof(CompositeActionHandler))]
public interface ICompositeActionHandler : IHandler
{
CompositeActionExecutionData Data { get; set; }
}
public sealed class CompositeActionHandler : Handler, ICompositeActionHandler
{
public CompositeActionExecutionData Data { get; set; }

public Task RunAsync(ActionRunStage stage)
{
// Validate args.
Trace.Entering();
ArgUtil.NotNull(ExecutionContext, nameof(ExecutionContext));
ArgUtil.NotNull(Inputs, nameof(Inputs));

var githubContext = ExecutionContext.ExpressionValues["github"] as GitHubContext;
ArgUtil.NotNull(githubContext, nameof(githubContext));

var tempDirectory = HostContext.GetDirectory(WellKnownDirectory.Temp);

// Resolve action steps
var actionSteps = Data.Steps;
if (actionSteps == null)
{
Trace.Error("Data.Steps in CompositeActionHandler is null");
}
else
{
Trace.Info($"Data Steps Value for Composite Actions is: {actionSteps}.");
}

// Create Context Data to reuse for each composite action step
var inputsData = new DictionaryContextData();
foreach (var i in Inputs)
{
inputsData[i.Key] = new StringContextData(i.Value);
}

// Add each composite action step to the front of the queue
var compositeActionSteps = new Queue<IStep>();
foreach (Pipelines.ActionStep aStep in actionSteps)
{
// Ex:
// runs:
// using: "composite"
// steps:
// - uses: example/test-composite@v2 (a)
// - run echo hello world (b)
// - run echo hello world 2 (c)
//
// ethanchewy/test-composite/action.yaml
// runs:
// using: "composite"
// steps:
// - run echo hello world 3 (d)
// - run echo hello world 4 (e)
//
// Stack (LIFO) [Bottom => Middle => Top]:
// | a |
// | a | => | d |
// (Run step d)
// | a |
// | a | => | e |
// (Run step e)
// | a |
// (Run step a)
// | b |
// (Run step b)
// | c |
// (Run step c)
// Done.

var actionRunner = HostContext.CreateService<IActionRunner>();
actionRunner.Action = aStep;
actionRunner.Stage = stage;
actionRunner.Condition = aStep.Condition;
actionRunner.DisplayName = aStep.DisplayName;
// TODO: Do we need to add any context data from the job message?
// (See JobExtension.cs ~line 236)

compositeActionSteps.Enqueue(ExecutionContext.RegisterCompositeStep(actionRunner, Inputs, inputsData));
}
ExecutionContext.EnqueueAllCompositeSteps(compositeActionSteps);

return Task.CompletedTask;
}

}
}
8 changes: 8 additions & 0 deletions src/Runner.Worker/Handlers/HandlerFactory.cs
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,14 @@ public IHandler Create(
handler = HostContext.CreateService<IRunnerPluginHandler>();
(handler as IRunnerPluginHandler).Data = data as PluginActionExecutionData;
}
else if (data.ExecutionType == ActionExecutionType.Composite)
{
// TODO
// Runner plugin
handler = HostContext.CreateService<ICompositeActionHandler>();
// handler = CompositeHandler;
(handler as ICompositeActionHandler).Data = data as CompositeActionExecutionData;
}
else
{
// This should never happen.
Expand Down
17 changes: 15 additions & 2 deletions src/Runner.Worker/StepsRunner.cs
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,11 @@ public async Task RunAsync(IExecutionContext jobContext)

var step = jobContext.JobSteps.Dequeue();
var nextStep = jobContext.JobSteps.Count > 0 ? jobContext.JobSteps.Peek() : null;
// TODO: Fix this temporary workaround for Composite Actions
if (!String.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA")))
{
nextStep = null;
}

Trace.Info($"Processing step: DisplayName='{step.DisplayName}'");
ArgUtil.NotNull(step.ExecutionContext, nameof(step.ExecutionContext));
Expand Down Expand Up @@ -409,7 +414,11 @@ private bool InitializeScope(IStep step, Dictionary<string, PipelineContextData>
scope = scopesToInitialize.Pop();
executionContext.Debug($"Initializing scope '{scope.Name}'");
executionContext.ExpressionValues["steps"] = stepsContext.GetScope(scope.ParentName);
executionContext.ExpressionValues["inputs"] = !String.IsNullOrEmpty(scope.ParentName) ? scopeInputs[scope.ParentName] : null;
// TODO: Fix this temporary workaround for Composite Actions
if (!executionContext.ExpressionValues.ContainsKey("inputs") && !String.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA")))
{
executionContext.ExpressionValues["inputs"] = !String.IsNullOrEmpty(scope.ParentName) ? scopeInputs[scope.ParentName] : null;
}
var templateEvaluator = executionContext.ToPipelineTemplateEvaluator();
var inputs = default(DictionaryContextData);
try
Expand All @@ -432,7 +441,11 @@ private bool InitializeScope(IStep step, Dictionary<string, PipelineContextData>
// Setup expression values
var scopeName = executionContext.ScopeName;
executionContext.ExpressionValues["steps"] = stepsContext.GetScope(scopeName);
executionContext.ExpressionValues["inputs"] = string.IsNullOrEmpty(scopeName) ? null : scopeInputs[scopeName];
// TODO: Fix this temporary workaround for Composite Actions
if (!executionContext.ExpressionValues.ContainsKey("inputs") && !String.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA")))
{
executionContext.ExpressionValues["inputs"] = string.IsNullOrEmpty(scopeName) ? null : scopeInputs[scopeName];
}

return true;
}
Expand Down
Loading

0 comments on commit c6c0571

Please sign in to comment.