From 3ce8c877ff863b00596089343d611a503cda9c60 Mon Sep 17 00:00:00 2001 From: Dimitar Venkov Date: Fri, 3 Aug 2018 14:25:03 +0800 Subject: [PATCH 01/23] Pull Dimitar'soriginal PR commits --- .../IronPythonCompletionProvider.cs | 634 +++++++++++++----- 1 file changed, 471 insertions(+), 163 deletions(-) diff --git a/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs b/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs index a272024033c..7ad06566401 100644 --- a/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs +++ b/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs @@ -1,19 +1,15 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Reflection; using System.Text.RegularExpressions; -using Dynamo; -using Dynamo.Interfaces; -using Dynamo.Utilities; using Dynamo.Logging; using ICSharpCode.AvalonEdit.CodeCompletion; -using IronPython.Hosting; using IronPython.Runtime; using IronPython.Runtime.Types; +using Microsoft.Scripting; using Microsoft.Scripting.Actions; using Microsoft.Scripting.Hosting; -using Microsoft.Scripting; -using System.Reflection; namespace Dynamo.Python { @@ -26,7 +22,7 @@ public class IronPythonCompletionProvider : LogSourceBase /// /// The engine used for autocompletion. This essentially keeps - /// track of the state of the editor, allowing access to variable types and + /// track of the state of the editor, allowing access to variable types and /// imported symbols. /// private ScriptEngine engine; @@ -64,28 +60,60 @@ public ScriptScope Scope /// /// Maps a regex to a particular python type. Useful for matching things like dicts, - /// floats, strings, etc. Initialized by + /// floats, strings, etc. Initialized by /// public Dictionary RegexToType = new Dictionary(); + /// + /// Maps a basic variable regex to a basic python type. + /// + public List> BasicVariableTypes; + + /// + /// Tracks already referenced CLR modules + /// + public HashSet clrModules { get; set; } + + /// + /// Keeps track of failed statements to avoid poluting the log + /// + public Dictionary badStatements { get; set; } + /// /// A bunch of regexes for use in introspaction /// - public static string commaDelimitedVariableNamesRegex = @"(([0-9a-zA-Z_]+,?\s*)+)"; - public static string variableName = @"([0-9a-zA-Z_]+(\.[a-zA-Z_0-9]+)*)"; - public static string doubleQuoteStringRegex = "(\"[^\"]*\")"; - public static string singleQuoteStringRegex = "(\'[^\']*\')"; + public static string commaDelimitedVariableNamesRegex = @"(([0-9a-zA-Z_]+,?\s?)+)"; + public static string variableName = @"([0-9a-zA-Z_]+(\.[a-zA-Z_0-9]+)*)"; + public static string quotesStringRegex = "[\"']([^\"']*)[\"']"; public static string arrayRegex = "(\\[.*\\])"; public static string spacesOrNone = @"(\s*)"; public static string atLeastOneSpaceRegex = @"(\s+)"; - public static string equals = @"(=)"; + public static string equalsRegex = @"(=)"; public static string dictRegex = "({.*})"; public static string doubleRegex = @"([-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?)"; public static string intRegex = @"([-+]?\d+)[\s\n]*$"; public static string basicImportRegex = @"(import)"; public static string fromImportRegex = @"^(from)"; -#endregion + private static readonly Regex MATCH_LAST_NAMESPACE = new Regex(@"[\w.]+$", RegexOptions.Compiled); + private static readonly Regex MATCH_LAST_WORD = new Regex(@"\w+$", RegexOptions.Compiled); + private static readonly Regex MATCH_FIRST_QUOTED_NAME = new Regex(quotesStringRegex, RegexOptions.Compiled); + private static readonly Regex MATCH_VALID_TYPE_NAME_CHARACTERS_ONLY = new Regex(@"^\w+", RegexOptions.Compiled); + private static readonly Regex TRIPPLE_QUOTE_STRINGS = new Regex(".*?\\\"{{3}}[\\s\\S]+?\\\"{{3}}", RegexOptions.Compiled); + + private static readonly Regex MATCH_IMPORT_STATEMENTS = new Regex(@"^import\s+?(.+)", RegexOptions.Compiled | RegexOptions.Multiline); + private static readonly Regex MATCH_FROM_IMPORT_STATEMENTS = new Regex(@"from\s+?([\w.]+)\s+?import\s+?([\w, *]+)", RegexOptions.Compiled | RegexOptions.Multiline); + private static readonly Regex MATCH_VARIABLE_ASSIGNMENTS = new Regex(@"^[ \t]*?(\w+(\s*?,\s*?\w+)*)\s*?=\s*(.+)", RegexOptions.Compiled | RegexOptions.Multiline); + + private static readonly Regex STRING_VARIABLE = new Regex("[\"']([^\"']*)[\"']", RegexOptions.Compiled); + private static readonly Regex DOUBLE_VARIABLE = new Regex("^-?\\d+\\.\\d+", RegexOptions.Compiled); + private static readonly Regex INT_VARIABLE = new Regex("^-?\\d+", RegexOptions.Compiled); + private static readonly Regex LIST_VARIABLE = new Regex("\\[.*\\]", RegexOptions.Compiled); + private static readonly Regex DICT_VARIABLE = new Regex("{.*}", RegexOptions.Compiled); + + private static readonly string BAD_ASSIGNEMNT_ENDS = ",([{"; + + #endregion /// /// Class constructor @@ -97,116 +125,154 @@ public IronPythonCompletionProvider() VariableTypes = new Dictionary(); ImportedTypes = new Dictionary(); + clrModules = new HashSet(); + badStatements = new Dictionary(); - RegexToType.Add(singleQuoteStringRegex, typeof(string)); - RegexToType.Add(doubleQuoteStringRegex, typeof(string)); - RegexToType.Add(doubleRegex, typeof(double)); - RegexToType.Add(intRegex, typeof(int)); - RegexToType.Add(arrayRegex, typeof(List)); - RegexToType.Add(dictRegex, typeof(PythonDictionary)); + //special case for python variables defined as null + ImportedTypes["None"] = null; - var assemblies = AppDomain.CurrentDomain.GetAssemblies(); + BasicVariableTypes = new List>(); - if (assemblies.Any(x => x.FullName.Contains("RevitAPI")) && assemblies.Any(x => x.FullName.Contains("RevitAPIUI"))) + BasicVariableTypes.Add(Tuple.Create(STRING_VARIABLE, typeof(string))); + BasicVariableTypes.Add(Tuple.Create(DOUBLE_VARIABLE, typeof(double))); + BasicVariableTypes.Add(Tuple.Create(INT_VARIABLE, typeof(int))); + BasicVariableTypes.Add(Tuple.Create(LIST_VARIABLE, typeof(IronPython.Runtime.List))); + BasicVariableTypes.Add(Tuple.Create(DICT_VARIABLE, typeof(PythonDictionary))); + + //main clr module + engine.CreateScriptSourceFromString("import clr\n", SourceCodeKind.SingleStatement).Execute(scope); + + var assemblies = AppDomain.CurrentDomain.GetAssemblies(); + if (assemblies.Any(x => x.GetName().Name == "RevitAPI")) { try { - scope.Engine.CreateScriptSourceFromString("import clr\n", SourceCodeKind.Statements).Execute(scope); - var revitImports = "clr.AddReference('RevitAPI')\nclr.AddReference('RevitAPIUI')\nfrom Autodesk.Revit.DB import *\nimport Autodesk\n"; - scope.Engine.CreateScriptSourceFromString(revitImports, SourceCodeKind.Statements).Execute(scope); + engine.CreateScriptSourceFromString(revitImports, SourceCodeKind.Statements).Execute(scope); + clrModules.Add("RevitAPI"); + clrModules.Add("RevitAPIUI"); } catch { - Log("Failed to load Revit types for autocomplete. Python autocomplete will not see Autodesk namespace types."); + Log("Failed to load Revit types for autocomplete. Python autocomplete will not see Autodesk namespace types."); } } - if (assemblies.Any(x => x.FullName.Contains("ProtoGeometry"))) + if (assemblies.Any(x => x.GetName().Name == "ProtoGeometry")) { try { - scope.Engine.CreateScriptSourceFromString("import clr\n", SourceCodeKind.Statements).Execute(scope); - var libGImports = - "import clr\nclr.AddReference('ProtoGeometry')\nfrom Autodesk.DesignScript.Geometry import *\n"; + "clr.AddReference('ProtoGeometry')\nfrom Autodesk.DesignScript.Geometry import *\n"; - scope.Engine.CreateScriptSourceFromString(libGImports, SourceCodeKind.Statements).Execute(scope); + engine.CreateScriptSourceFromString(libGImports, SourceCodeKind.Statements).Execute(scope); + clrModules.Add("ProtoGeometry"); } catch (Exception e) { Log(e.ToString()); - Log("Failed to load ProtoGeometry types for autocomplete. Python autocomplete will not see Autodesk namespace types."); + Log("Failed to load ProtoGeometry types for autocomplete. Python autocomplete will not see Autodesk namespace types."); } } + string pythonLibDir = System.IO.Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ProgramFilesX86), + "IronPython 2.7\\Lib"); + if (System.IO.Directory.Exists(pythonLibDir)) + { + try + { + var pyLibImports = String.Format("import sys\nsys.path.append(r'{0}')\n", pythonLibDir); + engine.CreateScriptSourceFromString(pyLibImports, SourceCodeKind.Statements).Execute(scope); + } + catch (Exception e) + { + Log(e.ToString()); + Log("Failed to register IronPython's native library. Python autocomplete will not see standard modules."); + } + } + + else + { + Log("Valid IronPython installation not found. Python autocomplete will not see native modules."); + } } /// - /// Generates completion data for the specified text, while import the given types into the + /// Generates completion data for the specified text, while import the given types into the /// scope and discovering variable assignments. /// - /// The code to parse + /// The code to parse + /// Optionally recurse and extend the search to the entire namespace /// Return a list of IronPythonCompletionData - public ICompletionData[] GetCompletionData(string line) + public ICompletionData[] GetCompletionData(string code, bool expand = false) { var items = new List(); - this.UpdateImportedTypes(line); - this.UpdateVariableTypes(line); // this is where hindley-milner could come into play + if (code.Contains("\"\"\"")) + { + code = StripDocStrings(code); + } - string name = GetName(line); + UpdateImportedTypes(code); + UpdateVariableTypes(code); // this is where hindley-milner could come into play + + string name = expand ? GetLastNameSpace(code) : GetLastName(code); if (!String.IsNullOrEmpty(name)) { try { AutocompletionInProgress = true; - - // is it a CLR type? - var type = TryGetType(name); + //is it a known type? + Type type = expand ? TryGetTypeFromFullName(name) : TryGetType(name); if (type != null) { items = EnumerateMembers(type, name); } // it's a variable? - else if (this.VariableTypes.ContainsKey(name) ) + else if (VariableTypes.TryGetValue(name, out type)) { - items = EnumerateMembers(this.VariableTypes[name], name); + items = EnumerateMembers(type, name); } // is it a namespace or python type? - else + else { var mem = LookupMember(name); - - if (mem is NamespaceTracker) - { - items = EnumerateMembers(mem as NamespaceTracker, name); - } - else if (mem is PythonModule) + var namespaceTracker = mem as NamespaceTracker; + if (namespaceTracker != null) { - items = EnumerateMembers(mem as PythonModule, name); + items = EnumerateMembers(namespaceTracker, name); } - else if (mem is PythonType) + else { - // shows static and instance methods in just the same way :( - var value = ClrModule.GetClrType(mem as PythonType); - if (value != null) + var pythonModule = mem as PythonModule; + if (pythonModule != null) { - items = EnumerateMembers(value, name); + items = EnumerateMembers(pythonModule, name); + } + else if (mem is PythonType) + { + // shows static and instance methods in just the same way :( + var value = ClrModule.GetClrType(mem as PythonType); + if (value != null) + { + items = EnumerateMembers(value, name); + } } } - } } - catch + catch (Exception ex) { - //Dynamo.this.logger.Log("EXCEPTION: GETTING COMPLETION DATA"); + Log(ex.ToString()); } AutocompletionInProgress = false; } - + if (!items.Any() && !expand) + { + return GetCompletionData(code, true); + } return items.ToArray(); } @@ -223,14 +289,8 @@ public List EnumerateMembers(PythonModule module, stri foreach (var member in d) { - if ( member.Value is BuiltinFunction ) - { - items.Add(new IronPythonCompletionData( (string) member.Key, name, false, IronPythonCompletionData.CompletionType.METHOD, this)); - } - else - { - items.Add(new IronPythonCompletionData((string)member.Key, name, false, IronPythonCompletionData.CompletionType.FIELD, this)); - } + var ct = member.Value is BuiltinFunction ? IronPythonCompletionData.CompletionType.METHOD : IronPythonCompletionData.CompletionType.FIELD; + items.Add(new IronPythonCompletionData((string)member.Key, name, false, ct, this)); } return items; } @@ -293,7 +353,7 @@ protected List EnumerateMembers(Type type, string name if (!completionsList.ContainsKey(methodInfoItem.Name)) completionsList.Add(methodInfoItem.Name, IronPythonCompletionData.CompletionType.METHOD); } - + } foreach (PropertyInfo propertyInfoItem in propertyInfo) @@ -308,6 +368,15 @@ protected List EnumerateMembers(Type type, string name completionsList.Add(fieldInfoItem.Name, IronPythonCompletionData.CompletionType.FIELD); } + if (type.IsEnum) + { + foreach (string en in type.GetEnumNames()) + { + if (!completionsList.ContainsKey(en)) + completionsList.Add(en, IronPythonCompletionData.CompletionType.FIELD); + } + } + foreach (var completionPair in completionsList) { items.Add(new IronPythonCompletionData(completionPair.Key, name, true, completionPair.Value, this)); @@ -336,8 +405,8 @@ public object LookupMember(string name, NamespaceTracker n) return null; } - var currentName = name.Substring(0,periodIndex); - var theRest = name.Substring(periodIndex+1); + var currentName = name.Substring(0, periodIndex); + var theRest = name.Substring(periodIndex + 1); if (n.TryGetValue(currentName, out varOutput)) { @@ -347,7 +416,6 @@ public object LookupMember(string name, NamespaceTracker n) } } return null; - } /// @@ -420,7 +488,7 @@ public string GetDescription(string stub, string item, bool isInstance) } catch { - + } } @@ -437,23 +505,40 @@ public string GetDescription(string stub, string item, bool isInstance) /// Traverse the given source code and define variable types based on /// the current scope /// - /// The source code to look through - public void UpdateVariableTypes(string line) + /// The source code to look through + public void UpdateVariableTypes(string code) { - this.VariableTypes.Clear(); // for now... + VariableTypes.Clear(); // for now... + VariableTypes = FindAllVariableAssignments(code); + } + + /// + /// A list of short assembly names used with the TryGetTypeFromFullName method + /// + private static string[] knownAssemblies = { + "mscorlib", + "RevitAPI", + "RevitAPIUI", + "ProtoGeometry" + }; - var vars = this.FindAllVariables(line); - foreach (var varData in vars) + /// + /// check if a full type name is found in one of the known pre-loaded assemblies and return the type + /// + /// a full type name + /// + private Type TryGetTypeFromFullName(string name) + { + Type foundType; + foreach (var asName in knownAssemblies) { - if (this.VariableTypes.ContainsKey(varData.Key)) + foundType = Type.GetType(String.Format("{0},{1}", name, asName)); + if (foundType != null) { - VariableTypes[varData.Key] = varData.Value.Item3; - } - else - { - VariableTypes.Add(varData.Key, varData.Value.Item3); + return foundType; } } + return null; } /// @@ -468,27 +553,153 @@ protected Type TryGetType(string name) return ImportedTypes[name]; } - if (VariableTypes.ContainsKey(name)) - { - return VariableTypes[name]; - } + //if the type name does noe exist in the local or built-in variables, then it is out of scope + string lookupScr = String.Format("clr.GetClrType({0}) if (\"{0}\" in locals() or \"{0}\" in __builtins__) and isinstance({0}, type) else None", name); - string tryGetType = name + ".GetType()"; dynamic type = null; try { - type = scope.Engine.CreateScriptSourceFromString(tryGetType, SourceCodeKind.Expression).Execute(scope); + type = engine.CreateScriptSourceFromString(lookupScr, SourceCodeKind.Expression).Execute(scope); } catch (Exception e) { Log(e.ToString()); - Log("Failed to look up type"); + Log(String.Format("Failed to look up type: {0}", name)); } - return type as Type; + + var foundType = type as Type; + if (foundType != null) + { + ImportedTypes[name] = foundType; + } + return foundType; } /// - /// Attempts to find import statements that look like + /// Attempts to find all import statements in the code + /// + /// The code to search + /// A list of tuples that contain the namespace, the module and the custom name + public static List> FindAllImportStatements(string code) + { + var statements = new List>(); + + //i.e. import math + //or import math, cmath as cm + var importMatches = MATCH_IMPORT_STATEMENTS.Matches(code); + foreach (Match m in importMatches) + { + if (m.Value.EndsWith(".")) + { + continue; //incomplete statement + } + + var names = m.Groups[1].Value.Trim().Split(new char[] { ',' }).Select(x => x.Trim()); + foreach (string n in names) + { + var parts = n.Split(new string[] { " as " }, 2, StringSplitOptions.RemoveEmptyEntries); + var name = parts[0]; + string asname = parts.Length > 1 ? parts[1] : null; + statements.Add(new Tuple(null, name, asname)); + } + } + + //i.e. from Autodesk.Revit.DB import * + //or from Autodesk.Revit.DB import XYZ, Line, Point as rvtPoint + var fromMatches = MATCH_FROM_IMPORT_STATEMENTS.Matches(code); + foreach (Match m in fromMatches) + { + var module = m.Groups[1].Value; + var names = m.Groups[2].Value.Trim().Split(new char[] { ',' }).Select(x => x.Trim()); + foreach (string n in names) + { + var parts = n.Split(new string[] { " as " }, 2, StringSplitOptions.RemoveEmptyEntries); + var name = parts[0]; + string asname = parts.Length > 1 ? parts[1] : null; + statements.Add(new Tuple(module, name, asname)); + } + } + return statements; + } + + /// + /// Attempts to find all variable assignments in the code. Has basic variable unpacking support. + /// We don't need to check the line indices because regex matches are ordered as per the code. + /// + /// The code to search + /// A dictionary of variable name and type pairs + public Dictionary FindAllVariableAssignments(string code) + { + var assignments = new Dictionary(); + + var varMatches = MATCH_VARIABLE_ASSIGNMENTS.Matches(code); + foreach (Match m in varMatches) + { + string _left = m.Groups[1].Value.Trim(), _right = m.Groups[3].Value.Trim(); + if (BAD_ASSIGNEMNT_ENDS.Contains(_right.Last())) + { + continue; //incomplete statement + } + + string[] left = _left.Split(new char[] { ',' }).Select(x => x.Trim()).ToArray(); + string[] right = _right.Split(new char[] { ',' }).Select(x => x.Trim()).ToArray(); + + if (right.Length < left.Length) + { + continue; // we can't resolve iterable unpacking + } + + if (left.Length == 1 && right.Length > 1) + { + //most likely we broke up an iterable assignment + right = new string[] { _right }; + } + + //try to resolve each variable, assignment pair + if (left.Length == right.Length) + { + for (int i = 0; i < left.Length; i++) + { + //check the basics first + bool foundBasicMatch = false; + foreach (Tuple rx in BasicVariableTypes) + { + if (rx.Item1.IsMatch(right[i])) + { + assignments[left[i]] = rx.Item2; + foundBasicMatch = true; + break; + } + } + + //check the scope for a possible match + if (!foundBasicMatch) + { + var possibleTypeName = GetFirstPossibleTypeName(right[i]); + if (!String.IsNullOrEmpty(possibleTypeName)) + { + Type t1; + //check if this is pointing to a predefined variable + if (!assignments.TryGetValue(possibleTypeName, out t1)) + { + //else proceed with a regular scope type check + t1 = TryGetType(possibleTypeName); + } + if (t1 != null) + { + assignments[left[i]] = t1; + } + } + } + } + } + } + + return assignments; + } + + /// + /// Attempts to find import statements that look like /// from lib import * /// /// The code to search @@ -496,10 +707,10 @@ protected Type TryGetType(string name) public static Dictionary FindAllTypeImportStatements(string code) { // matches the following types: - // from lib import * + // from lib import * var matches = Regex.Matches(code, fromImportRegex + atLeastOneSpaceRegex + variableName + - atLeastOneSpaceRegex + basicImportRegex + atLeastOneSpaceRegex + @"\*$", RegexOptions.Multiline); + atLeastOneSpaceRegex + basicImportRegex + atLeastOneSpaceRegex + @"\*$", RegexOptions.Multiline); var importMatches = new Dictionary(); @@ -518,7 +729,7 @@ public static Dictionary FindAllTypeImportStatements(string code } /// - /// Attempts to find import statements that look like + /// Attempts to find import statements that look like /// from lib import type1, type2 /// Doesn't currently match types with namespace qualifiers like Collections.ArrayList /// @@ -527,8 +738,8 @@ public static Dictionary FindAllTypeImportStatements(string code public static Dictionary FindTypeSpecificImportStatements(string code) { - var matches = Regex.Matches(code, fromImportRegex + atLeastOneSpaceRegex + variableName + - atLeastOneSpaceRegex + basicImportRegex + atLeastOneSpaceRegex + commaDelimitedVariableNamesRegex + "$", RegexOptions.Multiline); + var matches = Regex.Matches(code, fromImportRegex + atLeastOneSpaceRegex + variableName + + atLeastOneSpaceRegex + basicImportRegex + atLeastOneSpaceRegex + commaDelimitedVariableNamesRegex + "$", RegexOptions.Multiline); var importMatches = new Dictionary(); @@ -542,7 +753,7 @@ public static Dictionary FindTypeSpecificImportStatements(string foreach (var typeName in allTypes) { - if (importMatches.ContainsKey(libName)) + if (importMatches.ContainsKey(typeName)) continue; importMatches.Add(typeName, wholeLine.Replace(joinedTypeNames, typeName)); } @@ -552,7 +763,7 @@ public static Dictionary FindTypeSpecificImportStatements(string } /// - /// Attempts to find import statements that look like + /// Attempts to find import statements that look like /// import lib /// /// The code to search @@ -578,15 +789,15 @@ public static Dictionary FindBasicImportStatements(string code) } /// - /// Find a variable assignment of the form "varName = bla" where bla is matched by + /// Find a variable assignment of the form "varName = bla" where bla is matched by /// the given regex /// /// The code to search /// Your regex to match the type - /// A dictionary of name to assignment line pairs/returns> + /// A dictionary of name to assignment line pairs public static Dictionary FindVariableStatementWithRegex(string code, string valueRegex) { - var matches = Regex.Matches(code, variableName + spacesOrNone + equals + spacesOrNone + valueRegex); + var matches = Regex.Matches(code, variableName + spacesOrNone + equalsRegex + spacesOrNone + valueRegex); var paramMatches = new Dictionary(); @@ -597,54 +808,132 @@ public static Dictionary FindVariableStatementWithRegex(string c paramMatches.Add(name, val); } return paramMatches; + } + + public List findClrReferences(string code) + { + var statements = new List(); + foreach (var line in code.Split(new[] { '\n', ';' })) + { + if (line.Contains("clr.AddReference")) + { + statements.Add(line.Trim()); + } + } + return statements; } /// - /// Find all import statements and import into scope. If the type is already in the scope, this will be skipped. - /// The ImportedTypes dictionary is + /// Find all import statements and import into scope. If the type is already in the scope, this will be skipped. + /// The ImportedTypes dictionary is /// /// The code to discover the import statements. public void UpdateImportedTypes(string code) { - // look all import statements - var imports = FindBasicImportStatements(code) - .Union(FindTypeSpecificImportStatements(code)) - .Union(FindAllTypeImportStatements(code)); - - // try and load modules into python scope - foreach (var import in imports) + //detect all lib references prior to attempting to import anything + var refs = findClrReferences(code); + foreach (var statement in refs) { - if (scope.ContainsVariable(import.Key)) + int previousTries = 0; + badStatements.TryGetValue(statement, out previousTries); + if (previousTries > 3) { continue; } + try { - scope.Engine.CreateScriptSourceFromString(import.Value, SourceCodeKind.SingleStatement) - .Execute(this.scope); - var type = Type.GetType(import.Key); - this.ImportedTypes.Add(import.Key, type); + string libName = MATCH_FIRST_QUOTED_NAME.Match(statement).Groups[1].Value; + if (!clrModules.Contains(libName)) + { + if (statement.Contains("AddReferenceToFileAndPath")) + { + engine.CreateScriptSourceFromString(statement, SourceCodeKind.SingleStatement).Execute(scope); + //it's an assembly path, don't check the current appdomain + clrModules.Add(libName); + continue; + } + + if (AppDomain.CurrentDomain.GetAssemblies().Any(x => x.GetName().Name == libName)) + { + engine.CreateScriptSourceFromString(statement, SourceCodeKind.SingleStatement).Execute(scope); + clrModules.Add(libName); + } + } } - catch (Exception exception) + catch (Exception e) { - Console.WriteLine(exception.Message); + Log(e.ToString()); + Log(String.Format("Failed to reference library: {0}", statement)); + badStatements[statement] = previousTries + 1; } } + var importStatements = FindAllImportStatements(code); + foreach (var i in importStatements) + { + string module = i.Item1, memberName = i.Item2, asname = i.Item3; + string name = asname ?? memberName; + string statement = ""; + int previousTries = 0; + + if (name != "*" && (scope.ContainsVariable(name) || ImportedTypes.ContainsKey(name))) + { + continue; + } + + try + { + if (module == null) + { + statement = String.Format("import {0} as {1}", memberName, name); + } + else + { + if (memberName != "*") + { + statement = String.Format("from {0} import {1} as {2}", module, memberName, name); + } + else + { + statement = String.Format("from {0} import *", module); + } + } + + badStatements.TryGetValue(statement, out previousTries); + if (previousTries > 3) + { + continue; + } + + engine.CreateScriptSourceFromString(statement, SourceCodeKind.SingleStatement).Execute(scope); + if (memberName == "*") + { + continue; + } + string typeName = module == null ? memberName : String.Format("{0}.{1}", module, memberName); + var type = Type.GetType(typeName); + ImportedTypes.Add(name, type); + } + catch (Exception e) + { + Log(e.ToString()); + Log(String.Format("Failed to load module: {0}, with statement: {1}", memberName, statement)); + badStatements[statement] = previousTries + 1; + } + } } /// /// Find all variable assignments in the source code and attempt to discover their type /// - /// The code from whih to get the assignments + /// The code from which to get the assignments /// A dictionary matching the name of the variable to a tuple of typeName, character at which the assignment was found, and the CLR type - public Dictionary > FindAllVariables(string code) + public Dictionary> FindAllVariables(string code) { - // regex to collection var variables = new Dictionary>(); - - var variableStatements = Regex.Matches(code, variableName + spacesOrNone + equals + spacesOrNone + @"(.*)", RegexOptions.Multiline); + var variableStatements = Regex.Matches(code, variableName + spacesOrNone + equalsRegex + spacesOrNone + @"(.*)", RegexOptions.Multiline); for (var i = 0; i < variableStatements.Count; i++) { @@ -652,55 +941,47 @@ public Dictionary > FindAllVariables(string cod var typeString = variableStatements[i].Groups[6].Value.Trim(); // type var currentIndex = variableStatements[i].Index; - // check if matches typename(blabla) - in this case its a type we need to look up - var typeStringMatches = Regex.Matches(typeString, @"^(.*)\(.*\)$", RegexOptions.Singleline); - if (typeStringMatches.Count > 0) + var possibleTypeName = GetFirstPossibleTypeName(typeString); + if (!String.IsNullOrEmpty(possibleTypeName)) { - typeString = typeStringMatches[0].Groups[1].Value.Trim(); - var typeInScope = this.LookupMember(typeString); - - if (typeInScope is PythonType) // dictionary, enum, etc + var t1 = TryGetType(possibleTypeName); + if (t1 != null) { - var type = ClrModule.GetClrType(typeInScope as PythonType); - - //// if we already saw this var if (variables.ContainsKey(name)) { - var varInfo = variables[name]; - if (currentIndex > varInfo.Item2) + if (currentIndex > variables[name].Item2) { - variables[name] = new Tuple(typeString, currentIndex, type); + variables[name] = new Tuple(typeString, currentIndex, t1); } } else // we've never seen it, add the type { - variables.Add(name, new Tuple(typeString, currentIndex, type)); + variables.Add(name, new Tuple(typeString, currentIndex, t1)); } - } + continue; + } } - else // match default types (numbers, dicts, arrays, strings) + + // match default types (numbers, dicts, arrays, strings) + foreach (var pair in RegexToType) { - foreach (var pair in RegexToType) + var matches = Regex.Matches(typeString, "^" + pair.Key + "$", RegexOptions.Singleline); + if (matches.Count > 0) { - var matches = Regex.Matches(typeString, "^" + pair.Key + "$", RegexOptions.Singleline); - if (matches.Count > 0) + // if we already saw this var + if (variables.ContainsKey(name)) { - // if we already saw this var - if (variables.ContainsKey(name)) + if (currentIndex > variables[name].Item2) { - var varInfo = variables[name]; - if (currentIndex > varInfo.Item2) - { - variables[name] = new Tuple(typeString, currentIndex, pair.Value); - } + variables[name] = new Tuple(typeString, currentIndex, pair.Value); } - else // we've never seen it, add the type - { - variables.Add(name, new Tuple(typeString, currentIndex, pair.Value)); - } - break; } + else // we've never seen it, add the type + { + variables.Add(name, new Tuple(typeString, currentIndex, pair.Value)); + } + break; } } } @@ -709,19 +990,46 @@ public Dictionary > FindAllVariables(string cod } /// - /// Returns the name from the end of a string. Matches back to the first space and trims off spaces or ('s - /// from the end of the line + /// Returns the last name from the input line. The regex ignores tabs, spaces, the first new line, etc. /// /// /// - string GetName(string text) + string GetLastName(string text) { - text = text.Replace("\t", " "); - text = text.Replace("\n", " "); - text = text.Replace("\r", " "); - int startIndex = text.LastIndexOf(' '); - return text.Substring(startIndex + 1).Trim('.').Trim('('); + return MATCH_LAST_WORD.Match(text.Trim('.').Trim()).Value; } + /// + /// Returns the entire namespace from the end of the input line. The regex ignores tabs, spaces, the first new line, etc. + /// + /// + /// + string GetLastNameSpace(string text) + { + return MATCH_LAST_NAMESPACE.Match(text.Trim('.').Trim()).Value; + } + + /// + /// Returns the first possible type name from the type's declaration line. + /// + /// + /// + private static string GetFirstPossibleTypeName(string line) + { + var match = MATCH_VALID_TYPE_NAME_CHARACTERS_ONLY.Match(line); + string possibleTypeName = match.Success ? match.Value : ""; + return possibleTypeName; + } + + /// + /// Removes any docstring charactes from the source code + /// + /// + /// + private string StripDocStrings(string code) + { + var matches = TRIPPLE_QUOTE_STRINGS.Split(code); + return String.Join("", matches); + } } } \ No newline at end of file From e3d23961650758f6906529f74b639545b56776a6 Mon Sep 17 00:00:00 2001 From: Keith Alfaro Date: Tue, 8 Jan 2019 15:14:40 -0500 Subject: [PATCH 02/23] address existing PR comments --- .../IronPythonCompletionProvider.cs | 321 ++++++++++++------ .../Properties/AssemblyInfo.cs | 3 + 2 files changed, 215 insertions(+), 109 deletions(-) diff --git a/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs b/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs index 7ad06566401..dea4d8e57fd 100644 --- a/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs +++ b/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Reflection; using System.Text.RegularExpressions; @@ -43,6 +44,39 @@ public ScriptScope Scope set { scope = value; } } + /// + /// Cache storing a reference to the Dynamo Core directory + /// + private static string dynamoCoreDir { get; set; } + + /// + /// Get the location of the Python Standard Library + /// + private static string pythonLibDir + { + get + { + // Attempt to get and cache the Dynamo Core directory path + if (string.IsNullOrEmpty(dynamoCoreDir)) + { + // Gather executing assembly info + Assembly assembly = Assembly.GetExecutingAssembly(); + Version version = assembly.GetName().Version; + dynamoCoreDir = Path.GetDirectoryName(assembly.Location); + } + + // Return the standard library path + if (!string.IsNullOrEmpty(dynamoCoreDir)) + { + return dynamoCoreDir + @"\IronPython.StdLib.2.7.8"; + } + else + { + return null; + } + } + } + /// /// Already discovered variable types /// @@ -67,51 +101,52 @@ public ScriptScope Scope /// /// Maps a basic variable regex to a basic python type. /// - public List> BasicVariableTypes; + internal List> BasicVariableTypes; /// /// Tracks already referenced CLR modules /// - public HashSet clrModules { get; set; } + internal HashSet clrModules { get; set; } /// /// Keeps track of failed statements to avoid poluting the log /// - public Dictionary badStatements { get; set; } + internal Dictionary badStatements { get; set; } /// - /// A bunch of regexes for use in introspaction + /// A bunch of regexes for use in introspection /// - public static string commaDelimitedVariableNamesRegex = @"(([0-9a-zA-Z_]+,?\s?)+)"; - public static string variableName = @"([0-9a-zA-Z_]+(\.[a-zA-Z_0-9]+)*)"; - public static string quotesStringRegex = "[\"']([^\"']*)[\"']"; - public static string arrayRegex = "(\\[.*\\])"; - public static string spacesOrNone = @"(\s*)"; - public static string atLeastOneSpaceRegex = @"(\s+)"; - public static string equalsRegex = @"(=)"; - public static string dictRegex = "({.*})"; - public static string doubleRegex = @"([-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?)"; - public static string intRegex = @"([-+]?\d+)[\s\n]*$"; - public static string basicImportRegex = @"(import)"; - public static string fromImportRegex = @"^(from)"; - - private static readonly Regex MATCH_LAST_NAMESPACE = new Regex(@"[\w.]+$", RegexOptions.Compiled); - private static readonly Regex MATCH_LAST_WORD = new Regex(@"\w+$", RegexOptions.Compiled); - private static readonly Regex MATCH_FIRST_QUOTED_NAME = new Regex(quotesStringRegex, RegexOptions.Compiled); - private static readonly Regex MATCH_VALID_TYPE_NAME_CHARACTERS_ONLY = new Regex(@"^\w+", RegexOptions.Compiled); - private static readonly Regex TRIPPLE_QUOTE_STRINGS = new Regex(".*?\\\"{{3}}[\\s\\S]+?\\\"{{3}}", RegexOptions.Compiled); - - private static readonly Regex MATCH_IMPORT_STATEMENTS = new Regex(@"^import\s+?(.+)", RegexOptions.Compiled | RegexOptions.Multiline); - private static readonly Regex MATCH_FROM_IMPORT_STATEMENTS = new Regex(@"from\s+?([\w.]+)\s+?import\s+?([\w, *]+)", RegexOptions.Compiled | RegexOptions.Multiline); - private static readonly Regex MATCH_VARIABLE_ASSIGNMENTS = new Regex(@"^[ \t]*?(\w+(\s*?,\s*?\w+)*)\s*?=\s*(.+)", RegexOptions.Compiled | RegexOptions.Multiline); - - private static readonly Regex STRING_VARIABLE = new Regex("[\"']([^\"']*)[\"']", RegexOptions.Compiled); - private static readonly Regex DOUBLE_VARIABLE = new Regex("^-?\\d+\\.\\d+", RegexOptions.Compiled); - private static readonly Regex INT_VARIABLE = new Regex("^-?\\d+", RegexOptions.Compiled); - private static readonly Regex LIST_VARIABLE = new Regex("\\[.*\\]", RegexOptions.Compiled); - private static readonly Regex DICT_VARIABLE = new Regex("{.*}", RegexOptions.Compiled); - - private static readonly string BAD_ASSIGNEMNT_ENDS = ",([{"; + internal const string commaDelimitedVariableNamesRegex = @"(([0-9a-zA-Z_]+,?\s?)+)"; + internal const string variableName = @"([0-9a-zA-Z_]+(\.[a-zA-Z_0-9]+)*)"; + internal const string quotesStringRegex = "[\"']([^\"']*)[\"']"; + internal const string arrayRegex = "(\\[.*\\])"; + internal const string spacesOrNone = @"(\s*)"; + internal const string atLeastOneSpaceRegex = @"(\s+)"; + internal const string equalsRegex = @"(=)"; + internal const string dictRegex = "({.*})"; + internal const string doubleRegex = @"([-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?)"; + internal const string intRegex = @"([-+]?\d+)[\s\n]*$"; + internal const string basicImportRegex = @"(import)"; + internal const string fromImportRegex = @"^(from)"; + + internal static readonly Regex MATCH_LAST_NAMESPACE = new Regex(@"[\w.]+$", RegexOptions.Compiled); + internal static readonly Regex MATCH_LAST_WORD = new Regex(@"\w+$", RegexOptions.Compiled); + internal static readonly Regex MATCH_FIRST_QUOTED_NAME = new Regex(quotesStringRegex, RegexOptions.Compiled); + internal static readonly Regex MATCH_VALID_TYPE_NAME_CHARACTERS_ONLY = new Regex(@"^\w+", RegexOptions.Compiled); + internal static readonly Regex TRIPPLE_QUOTE_STRINGS = new Regex(".*?\\\"{{3}}[\\s\\S]+?\\\"{{3}}", RegexOptions.Compiled); + + internal static readonly Regex MATCH_IMPORT_STATEMENTS = new Regex(@"^import\s+?(.+)", RegexOptions.Compiled | RegexOptions.Multiline); + internal static readonly Regex MATCH_FROM_IMPORT_STATEMENTS = new Regex(@"from\s+?([\w.]+)\s+?import\s+?([\w, *]+)", RegexOptions.Compiled | RegexOptions.Multiline); + internal static readonly Regex MATCH_VARIABLE_ASSIGNMENTS = new Regex(@"^[ \t]*?(\w+(\s*?,\s*?\w+)*)\s*?=\s*(.+)", RegexOptions.Compiled | RegexOptions.Multiline); + + internal static readonly Regex STRING_VARIABLE = new Regex("[\"']([^\"']*)[\"']", RegexOptions.Compiled); + internal static readonly Regex DOUBLE_VARIABLE = new Regex("^-?\\d+\\.\\d+", RegexOptions.Compiled); + internal static readonly Regex INT_VARIABLE = new Regex("^-?\\d+", RegexOptions.Compiled); + internal static readonly Regex LIST_VARIABLE = new Regex("\\[.*\\]", RegexOptions.Compiled); + internal static readonly Regex DICT_VARIABLE = new Regex("{.*}", RegexOptions.Compiled); + + // TODO should this be Regex? + internal static readonly string BAD_ASSIGNEMNT_ENDS = ",([{"; #endregion @@ -128,7 +163,7 @@ public IronPythonCompletionProvider() clrModules = new HashSet(); badStatements = new Dictionary(); - //special case for python variables defined as null + // Special case for python variables defined as null ImportedTypes["None"] = null; BasicVariableTypes = new List>(); @@ -139,12 +174,15 @@ public IronPythonCompletionProvider() BasicVariableTypes.Add(Tuple.Create(LIST_VARIABLE, typeof(IronPython.Runtime.List))); BasicVariableTypes.Add(Tuple.Create(DICT_VARIABLE, typeof(PythonDictionary))); - //main clr module + // Main CLR module engine.CreateScriptSourceFromString("import clr\n", SourceCodeKind.SingleStatement).Execute(scope); var assemblies = AppDomain.CurrentDomain.GetAssemblies(); + + // Determine if the Revit API is available in the given context if (assemblies.Any(x => x.GetName().Name == "RevitAPI")) { + // Try to load Revit Type for autocomplete try { var revitImports = @@ -160,8 +198,10 @@ public IronPythonCompletionProvider() } } + // Determine if the ProtoGeometry is available in the given context if (assemblies.Any(x => x.GetName().Name == "ProtoGeometry")) { + // Try to load ProtoGeometry Type for autocomplete try { var libGImports = @@ -177,10 +217,10 @@ public IronPythonCompletionProvider() } } - string pythonLibDir = System.IO.Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ProgramFilesX86), - "IronPython 2.7\\Lib"); - if (System.IO.Directory.Exists(pythonLibDir)) + // Determine if the Python Standard Library is available in the given context + if (Directory.Exists(pythonLibDir)) { + // Try to load Python Standard Library Type for autocomplete try { var pyLibImports = String.Format("import sys\nsys.path.append(r'{0}')\n", pythonLibDir); @@ -192,22 +232,23 @@ public IronPythonCompletionProvider() Log("Failed to register IronPython's native library. Python autocomplete will not see standard modules."); } } - else { - Log("Valid IronPython installation not found. Python autocomplete will not see native modules."); + Log("Valid IronPython Standard Library not found. Python autocomplete will not see native modules."); } } /// - /// Generates completion data for the specified text, while import the given types into the + /// Generate completion data for the specified text, while import the given types into the /// scope and discovering variable assignments. /// /// The code to parse - /// Optionally recurse and extend the search to the entire namespace /// Return a list of IronPythonCompletionData - public ICompletionData[] GetCompletionData(string code, bool expand = false) + public ICompletionData[] GetCompletionData(string code) { + // TODO - add optional param in Dynamo 3.0 with default set to false + bool expand = false; + var items = new List(); if (code.Contains("\"\"\"")) @@ -216,7 +257,7 @@ public ICompletionData[] GetCompletionData(string code, bool expand = false) } UpdateImportedTypes(code); - UpdateVariableTypes(code); // this is where hindley-milner could come into play + UpdateVariableTypes(code); // Possibly use hindley-milner in the future string name = expand ? GetLastNameSpace(code) : GetLastName(code); if (!String.IsNullOrEmpty(name)) @@ -224,22 +265,26 @@ public ICompletionData[] GetCompletionData(string code, bool expand = false) try { AutocompletionInProgress = true; - //is it a known type? + + // Attempt to get type using naming Type type = expand ? TryGetTypeFromFullName(name) : TryGetType(name); + + // CLR type if (type != null) { items = EnumerateMembers(type, name); } - // it's a variable? + // Variable type else if (VariableTypes.TryGetValue(name, out type)) { items = EnumerateMembers(type, name); } - // is it a namespace or python type? else { var mem = LookupMember(name); var namespaceTracker = mem as NamespaceTracker; + + // Namespace type if (namespaceTracker != null) { items = EnumerateMembers(namespaceTracker, name); @@ -247,14 +292,18 @@ public ICompletionData[] GetCompletionData(string code, bool expand = false) else { var pythonModule = mem as PythonModule; + + // Python Module type if (pythonModule != null) { items = EnumerateMembers(pythonModule, name); } + // Python type else if (mem is PythonType) { - // shows static and instance methods in just the same way :( + // Shows static and instance methods in the same way :( var value = ClrModule.GetClrType(mem as PythonType); + if (value != null) { items = EnumerateMembers(value, name); @@ -267,12 +316,15 @@ public ICompletionData[] GetCompletionData(string code, bool expand = false) { Log(ex.ToString()); } + AutocompletionInProgress = false; } + if (!items.Any() && !expand) { - return GetCompletionData(code, true); + return GetCompletionData(code); } + return items.ToArray(); } @@ -289,9 +341,10 @@ public List EnumerateMembers(PythonModule module, stri foreach (var member in d) { - var ct = member.Value is BuiltinFunction ? IronPythonCompletionData.CompletionType.METHOD : IronPythonCompletionData.CompletionType.FIELD; - items.Add(new IronPythonCompletionData((string)member.Key, name, false, ct, this)); + var completionType = member.Value is BuiltinFunction ? IronPythonCompletionData.CompletionType.METHOD : IronPythonCompletionData.CompletionType.FIELD; + items.Add(new IronPythonCompletionData((string)member.Key, name, false, completionType, this)); } + return items; } @@ -315,15 +368,16 @@ public List EnumerateMembers(NamespaceTracker ns, stri { items.Add(new IronPythonCompletionData(member.Key, name, false, IronPythonCompletionData.CompletionType.FIELD, this)); } - else if (member.Value is Microsoft.Scripting.Actions.PropertyTracker) + else if (member.Value is PropertyTracker) { items.Add(new IronPythonCompletionData(member.Key, name, false, IronPythonCompletionData.CompletionType.PROPERTY, this)); } - else if (member.Value is Microsoft.Scripting.Actions.TypeTracker) + else if (member.Value is TypeTracker) { items.Add(new IronPythonCompletionData(member.Key, name, false, IronPythonCompletionData.CompletionType.CLASS, this)); } } + return items; } @@ -345,27 +399,32 @@ protected List EnumerateMembers(Type type, string name foreach (MethodInfo methodInfoItem in methodInfo) { - if ((methodInfoItem.IsPublic) + if ( methodInfoItem.IsPublic && (methodInfoItem.Name.IndexOf("get_") != 0) && (methodInfoItem.Name.IndexOf("set_") != 0) && (methodInfoItem.Name.IndexOf("add_") != 0) && (methodInfoItem.Name.IndexOf("remove_") != 0) - && (methodInfoItem.Name.IndexOf("__") != 0)) + && (methodInfoItem.Name.IndexOf("__") != 0) ) { if (!completionsList.ContainsKey(methodInfoItem.Name)) + { completionsList.Add(methodInfoItem.Name, IronPythonCompletionData.CompletionType.METHOD); + } } - } foreach (PropertyInfo propertyInfoItem in propertyInfo) { if (!completionsList.ContainsKey(propertyInfoItem.Name)) + { completionsList.Add(propertyInfoItem.Name, IronPythonCompletionData.CompletionType.PROPERTY); + } } foreach (FieldInfo fieldInfoItem in fieldInfo) { if (!completionsList.ContainsKey(fieldInfoItem.Name)) + { completionsList.Add(fieldInfoItem.Name, IronPythonCompletionData.CompletionType.FIELD); + } } if (type.IsEnum) @@ -373,7 +432,9 @@ protected List EnumerateMembers(Type type, string name foreach (string en in type.GetEnumNames()) { if (!completionsList.ContainsKey(en)) + { completionsList.Add(en, IronPythonCompletionData.CompletionType.FIELD); + } } } @@ -419,7 +480,7 @@ public object LookupMember(string name, NamespaceTracker n) } /// - /// Recursively lookup a variable in the _scope + /// Recursively lookup a variable in the _scope /// /// A name for a type, possibly delimited by periods. /// The type as an object @@ -479,12 +540,23 @@ public string GetDescription(string stub, string item, bool isInstance) //var des = _engine.Operations.GetDocumentation(value); string docCommand = ""; - if (isInstance) docCommand = "type(" + stub + ")" + "." + item + ".__doc__"; - else docCommand = stub + "." + item + ".__doc__"; + + if (isInstance) + { + docCommand = "type(" + stub + ")" + "." + item + ".__doc__"; + } + else + { + docCommand = stub + "." + item + ".__doc__"; + } + object value = engine.CreateScriptSourceFromString(docCommand, SourceCodeKind.Expression).Execute(scope); if (!String.IsNullOrEmpty((string)value)) + { description = (string)value; + } + } catch { @@ -502,13 +574,13 @@ public string GetDescription(string stub, string item, bool isInstance) public delegate void DescriptionUpdateDelegate(string description); /// - /// Traverse the given source code and define variable types based on - /// the current scope + /// Traverse the given source code and define variable types based on + /// the current scope /// /// The source code to look through public void UpdateVariableTypes(string code) { - VariableTypes.Clear(); // for now... + VariableTypes.Clear(); VariableTypes = FindAllVariableAssignments(code); } @@ -523,26 +595,26 @@ public void UpdateVariableTypes(string code) }; /// - /// check if a full type name is found in one of the known pre-loaded assemblies and return the type + /// Check if a full type name is found in one of the known pre-loaded assemblies and return the type /// /// a full type name /// private Type TryGetTypeFromFullName(string name) { - Type foundType; foreach (var asName in knownAssemblies) { - foundType = Type.GetType(String.Format("{0},{1}", name, asName)); + Type foundType = Type.GetType(String.Format("{0},{1}", name, asName)); if (foundType != null) { return foundType; } } + return null; } /// - /// Returns a type from a name. For example: System.Collections or System.Collections.ArrayList + /// Returns a type from a name. For example: System.Collections or System.Collections.ArrayList /// /// The name /// The type or null if its not a valid type @@ -553,7 +625,7 @@ protected Type TryGetType(string name) return ImportedTypes[name]; } - //if the type name does noe exist in the local or built-in variables, then it is out of scope + // If the type name does not exist in the local or built-in variables, then it is out of scope string lookupScr = String.Format("clr.GetClrType({0}) if (\"{0}\" in locals() or \"{0}\" in __builtins__) and isinstance({0}, type) else None", name); dynamic type = null; @@ -563,8 +635,8 @@ protected Type TryGetType(string name) } catch (Exception e) { - Log(e.ToString()); Log(String.Format("Failed to look up type: {0}", name)); + Log(e.ToString()); } var foundType = type as Type; @@ -572,6 +644,7 @@ protected Type TryGetType(string name) { ImportedTypes[name] = foundType; } + return foundType; } @@ -579,19 +652,19 @@ protected Type TryGetType(string name) /// Attempts to find all import statements in the code /// /// The code to search - /// A list of tuples that contain the namespace, the module and the custom name + /// A list of tuples that contain the namespace, the module, and the custom name public static List> FindAllImportStatements(string code) { var statements = new List>(); - //i.e. import math - //or import math, cmath as cm + // i.e. import math + // or import math, cmath as cm var importMatches = MATCH_IMPORT_STATEMENTS.Matches(code); foreach (Match m in importMatches) { if (m.Value.EndsWith(".")) { - continue; //incomplete statement + continue; // Incomplete statement } var names = m.Groups[1].Value.Trim().Split(new char[] { ',' }).Select(x => x.Trim()); @@ -604,8 +677,8 @@ public static List> FindAllImportStatements(string } } - //i.e. from Autodesk.Revit.DB import * - //or from Autodesk.Revit.DB import XYZ, Line, Point as rvtPoint + // i.e. from Autodesk.Revit.DB import * + // or from Autodesk.Revit.DB import XYZ, Line, Point as rvtPoint var fromMatches = MATCH_FROM_IMPORT_STATEMENTS.Matches(code); foreach (Match m in fromMatches) { @@ -638,7 +711,7 @@ public Dictionary FindAllVariableAssignments(string code) string _left = m.Groups[1].Value.Trim(), _right = m.Groups[3].Value.Trim(); if (BAD_ASSIGNEMNT_ENDS.Contains(_right.Last())) { - continue; //incomplete statement + continue; // Incomplete statement } string[] left = _left.Split(new char[] { ',' }).Select(x => x.Trim()).ToArray(); @@ -646,21 +719,21 @@ public Dictionary FindAllVariableAssignments(string code) if (right.Length < left.Length) { - continue; // we can't resolve iterable unpacking + continue; // Unable to resolve iterable unpacking } if (left.Length == 1 && right.Length > 1) { - //most likely we broke up an iterable assignment + // Likely an iterable assignment has been broken up right = new string[] { _right }; } - //try to resolve each variable, assignment pair + // Attempt to resolve each variable, assignment pair if (left.Length == right.Length) { for (int i = 0; i < left.Length; i++) { - //check the basics first + // Check the basics first bool foundBasicMatch = false; foreach (Tuple rx in BasicVariableTypes) { @@ -672,19 +745,20 @@ public Dictionary FindAllVariableAssignments(string code) } } - //check the scope for a possible match + // Check the scope for a possible match if (!foundBasicMatch) { var possibleTypeName = GetFirstPossibleTypeName(right[i]); if (!String.IsNullOrEmpty(possibleTypeName)) { Type t1; - //check if this is pointing to a predefined variable + // Check if this is pointing to a predefined variable if (!assignments.TryGetValue(possibleTypeName, out t1)) { - //else proceed with a regular scope type check + // Proceed with a regular scope type check t1 = TryGetType(possibleTypeName); } + if (t1 != null) { assignments[left[i]] = t1; @@ -706,11 +780,15 @@ public Dictionary FindAllVariableAssignments(string code) /// A dictionary matching the lib to the code where lib is the library being imported from public static Dictionary FindAllTypeImportStatements(string code) { - // matches the following types: - // from lib import * + var pattern = fromImportRegex + + atLeastOneSpaceRegex + + variableName + + atLeastOneSpaceRegex + + basicImportRegex + + atLeastOneSpaceRegex + + @"\*$"; - var matches = Regex.Matches(code, fromImportRegex + atLeastOneSpaceRegex + variableName + - atLeastOneSpaceRegex + basicImportRegex + atLeastOneSpaceRegex + @"\*$", RegexOptions.Multiline); + var matches = Regex.Matches(code, pattern, RegexOptions.Multiline); var importMatches = new Dictionary(); @@ -737,9 +815,16 @@ public static Dictionary FindAllTypeImportStatements(string code /// A dictionary matching the lib to the code where lib is the library being imported from public static Dictionary FindTypeSpecificImportStatements(string code) { + var pattern = fromImportRegex + + atLeastOneSpaceRegex + + variableName + + atLeastOneSpaceRegex + + basicImportRegex + + atLeastOneSpaceRegex + + commaDelimitedVariableNamesRegex + + "$"; - var matches = Regex.Matches(code, fromImportRegex + atLeastOneSpaceRegex + variableName + - atLeastOneSpaceRegex + basicImportRegex + atLeastOneSpaceRegex + commaDelimitedVariableNamesRegex + "$", RegexOptions.Multiline); + var matches = Regex.Matches(code, pattern, RegexOptions.Multiline); var importMatches = new Dictionary(); @@ -754,7 +839,10 @@ public static Dictionary FindTypeSpecificImportStatements(string foreach (var typeName in allTypes) { if (importMatches.ContainsKey(typeName)) + { continue; + } + importMatches.Add(typeName, wholeLine.Replace(joinedTypeNames, typeName)); } } @@ -770,7 +858,9 @@ public static Dictionary FindTypeSpecificImportStatements(string /// A dictionary matching the lib to the code where lib is the library being imported from public static Dictionary FindBasicImportStatements(string code) { - var matches = Regex.Matches(code, "^" + basicImportRegex + spacesOrNone + variableName, RegexOptions.Multiline); + var pattern = "^" + basicImportRegex + spacesOrNone + variableName; + + var matches = Regex.Matches(code, pattern, RegexOptions.Multiline); var importMatches = new Dictionary(); @@ -780,7 +870,9 @@ public static Dictionary FindBasicImportStatements(string code) var libName = matches[i].Groups[3].Value.Trim(); if (importMatches.ContainsKey(libName)) + { continue; + } importMatches.Add(libName, wholeLine); } @@ -797,7 +889,9 @@ public static Dictionary FindBasicImportStatements(string code) /// A dictionary of name to assignment line pairs public static Dictionary FindVariableStatementWithRegex(string code, string valueRegex) { - var matches = Regex.Matches(code, variableName + spacesOrNone + equalsRegex + spacesOrNone + valueRegex); + var pattern = variableName + spacesOrNone + equalsRegex + spacesOrNone + valueRegex; + + var matches = Regex.Matches(code, pattern); var paramMatches = new Dictionary(); @@ -807,10 +901,10 @@ public static Dictionary FindVariableStatementWithRegex(string c var val = matches[i].Groups[6].Value.Trim(); paramMatches.Add(name, val); } + return paramMatches; } - public List findClrReferences(string code) { var statements = new List(); @@ -821,22 +915,23 @@ public List findClrReferences(string code) statements.Add(line.Trim()); } } + return statements; } /// /// Find all import statements and import into scope. If the type is already in the scope, this will be skipped. - /// The ImportedTypes dictionary is /// /// The code to discover the import statements. public void UpdateImportedTypes(string code) { - //detect all lib references prior to attempting to import anything + // Detect all lib references prior to attempting to import anything var refs = findClrReferences(code); foreach (var statement in refs) { - int previousTries = 0; + var previousTries = 0; badStatements.TryGetValue(statement, out previousTries); + // TODO - why is this 3? Should this be a constant? if (previousTries > 3) { continue; @@ -845,16 +940,18 @@ public void UpdateImportedTypes(string code) try { string libName = MATCH_FIRST_QUOTED_NAME.Match(statement).Groups[1].Value; + if (!clrModules.Contains(libName)) { if (statement.Contains("AddReferenceToFileAndPath")) { engine.CreateScriptSourceFromString(statement, SourceCodeKind.SingleStatement).Execute(scope); - //it's an assembly path, don't check the current appdomain clrModules.Add(libName); continue; } + // TODO - should there be a map of assemblies which can be used to verify + // the name exists as this lookup appears to occur frequently if (AppDomain.CurrentDomain.GetAssemblies().Any(x => x.GetName().Name == libName)) { engine.CreateScriptSourceFromString(statement, SourceCodeKind.SingleStatement).Execute(scope); @@ -864,19 +961,21 @@ public void UpdateImportedTypes(string code) } catch (Exception e) { - Log(e.ToString()); Log(String.Format("Failed to reference library: {0}", statement)); + Log(e.ToString()); badStatements[statement] = previousTries + 1; } } var importStatements = FindAllImportStatements(code); + + // TODO - add comment foreach (var i in importStatements) { string module = i.Item1, memberName = i.Item2, asname = i.Item3; string name = asname ?? memberName; string statement = ""; - int previousTries = 0; + var previousTries = 0; if (name != "*" && (scope.ContainsVariable(name) || ImportedTypes.ContainsKey(name))) { @@ -902,24 +1001,27 @@ public void UpdateImportedTypes(string code) } badStatements.TryGetValue(statement, out previousTries); + if (previousTries > 3) { continue; } engine.CreateScriptSourceFromString(statement, SourceCodeKind.SingleStatement).Execute(scope); + if (memberName == "*") { continue; } + string typeName = module == null ? memberName : String.Format("{0}.{1}", module, memberName); var type = Type.GetType(typeName); ImportedTypes.Add(name, type); } catch (Exception e) { - Log(e.ToString()); Log(String.Format("Failed to load module: {0}, with statement: {1}", memberName, statement)); + Log(e.ToString()); badStatements[statement] = previousTries + 1; } } @@ -944,32 +1046,32 @@ public Dictionary> FindAllVariables(string code var possibleTypeName = GetFirstPossibleTypeName(typeString); if (!String.IsNullOrEmpty(possibleTypeName)) { - var t1 = TryGetType(possibleTypeName); - if (t1 != null) + var variableType = TryGetType(possibleTypeName); + if (variableType != null) { if (variables.ContainsKey(name)) { if (currentIndex > variables[name].Item2) { - variables[name] = new Tuple(typeString, currentIndex, t1); + variables[name] = new Tuple(typeString, currentIndex, variableType); } } else // we've never seen it, add the type { - variables.Add(name, new Tuple(typeString, currentIndex, t1)); + variables.Add(name, new Tuple(typeString, currentIndex, variableType)); } continue; } } - // match default types (numbers, dicts, arrays, strings) + // Match default types (numbers, dicts, arrays, strings) foreach (var pair in RegexToType) { var matches = Regex.Matches(typeString, "^" + pair.Key + "$", RegexOptions.Singleline); if (matches.Count > 0) { - // if we already saw this var + // If variable has already been encountered if (variables.ContainsKey(name)) { if (currentIndex > variables[name].Item2) @@ -977,10 +1079,11 @@ public Dictionary> FindAllVariables(string code variables[name] = new Tuple(typeString, currentIndex, pair.Value); } } - else // we've never seen it, add the type + else // New type, add it { variables.Add(name, new Tuple(typeString, currentIndex, pair.Value)); } + break; } } @@ -994,7 +1097,7 @@ public Dictionary> FindAllVariables(string code /// /// /// - string GetLastName(string text) + private string GetLastName(string text) { return MATCH_LAST_WORD.Match(text.Trim('.').Trim()).Value; } @@ -1004,7 +1107,7 @@ string GetLastName(string text) /// /// /// - string GetLastNameSpace(string text) + private string GetLastNameSpace(string text) { return MATCH_LAST_NAMESPACE.Match(text.Trim('.').Trim()).Value; } @@ -1022,7 +1125,7 @@ private static string GetFirstPossibleTypeName(string line) } /// - /// Removes any docstring charactes from the source code + /// Removes any docstring characters from the source code /// /// /// diff --git a/src/Libraries/PythonNodeModelsWpf/Properties/AssemblyInfo.cs b/src/Libraries/PythonNodeModelsWpf/Properties/AssemblyInfo.cs index 1f18dc248f9..f830d751b1e 100644 --- a/src/Libraries/PythonNodeModelsWpf/Properties/AssemblyInfo.cs +++ b/src/Libraries/PythonNodeModelsWpf/Properties/AssemblyInfo.cs @@ -1,4 +1,5 @@ using System.Reflection; +using System.Runtime.CompilerServices; using System.Runtime.InteropServices; // General Information about an assembly is controlled through the following @@ -9,3 +10,5 @@ [assembly: AssemblyCulture("")] // The following GUID is for the ID of the typelib if this project is exposed to COM [assembly: Guid("5df40db3-627c-43fb-9a0e-0ee422d92485")] + +[assembly: InternalsVisibleTo("DynamoPythonTests")] \ No newline at end of file From ec6a098f6f1468750829d997befdcd007f63d05a Mon Sep 17 00:00:00 2001 From: Keith Alfaro Date: Wed, 9 Jan 2019 12:43:14 -0500 Subject: [PATCH 03/23] address python std lib path --- .../IronPythonCompletionProvider.cs | 33 +++++++++++-------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs b/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs index dea4d8e57fd..e4d9243260e 100644 --- a/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs +++ b/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs @@ -62,13 +62,14 @@ private static string pythonLibDir // Gather executing assembly info Assembly assembly = Assembly.GetExecutingAssembly(); Version version = assembly.GetName().Version; - dynamoCoreDir = Path.GetDirectoryName(assembly.Location); + string nodePath = Path.GetDirectoryName(assembly.Location); + dynamoCoreDir = Path.GetFullPath(Path.Combine(nodePath, @"..\")); } // Return the standard library path if (!string.IsNullOrEmpty(dynamoCoreDir)) { - return dynamoCoreDir + @"\IronPython.StdLib.2.7.8"; + return dynamoCoreDir + @"IronPython.StdLib.2.7.8"; } else { @@ -114,20 +115,24 @@ private static string pythonLibDir internal Dictionary badStatements { get; set; } /// - /// A bunch of regexes for use in introspection + /// Maps a basic variable regex to a basic python type. /// - internal const string commaDelimitedVariableNamesRegex = @"(([0-9a-zA-Z_]+,?\s?)+)"; - internal const string variableName = @"([0-9a-zA-Z_]+(\.[a-zA-Z_0-9]+)*)"; + // TODO - these can all be internal constants - Dynamo 3.0 + public static string commaDelimitedVariableNamesRegex = @"(([0-9a-zA-Z_]+,?\s?)+)"; + public static string variableName = @"([0-9a-zA-Z_]+(\.[a-zA-Z_0-9]+)*)"; + public static string doubleQuoteStringRegex = "(\"[^\"]*\")"; // TODO - Remove in Dynamo 3.0 + public static string singleQuoteStringRegex = "(\'[^\']*\')"; // TODO - Remove in Dynamo 3.0 + public static string arrayRegex = "(\\[.*\\])"; + public static string spacesOrNone = @"(\s*)"; + public static string atLeastOneSpaceRegex = @"(\s+)"; + public static string equalsRegex = @"(=)"; + public static string dictRegex = "({.*})"; + public static string doubleRegex = @"([-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?)"; + public static string intRegex = @"([-+]?\d+)[\s\n]*$"; + public static string basicImportRegex = @"(import)"; + public static string fromImportRegex = @"^(from)"; + internal const string quotesStringRegex = "[\"']([^\"']*)[\"']"; - internal const string arrayRegex = "(\\[.*\\])"; - internal const string spacesOrNone = @"(\s*)"; - internal const string atLeastOneSpaceRegex = @"(\s+)"; - internal const string equalsRegex = @"(=)"; - internal const string dictRegex = "({.*})"; - internal const string doubleRegex = @"([-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?)"; - internal const string intRegex = @"([-+]?\d+)[\s\n]*$"; - internal const string basicImportRegex = @"(import)"; - internal const string fromImportRegex = @"^(from)"; internal static readonly Regex MATCH_LAST_NAMESPACE = new Regex(@"[\w.]+$", RegexOptions.Compiled); internal static readonly Regex MATCH_LAST_WORD = new Regex(@"\w+$", RegexOptions.Compiled); From 3db1edf9e398f13708e0cf2475aa4ddaa5da2ece Mon Sep 17 00:00:00 2001 From: Keith Alfaro Date: Wed, 9 Jan 2019 12:52:17 -0500 Subject: [PATCH 04/23] cleanup --- .../PythonNodeModelsWpf/IronPythonCompletionProvider.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs b/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs index e4d9243260e..64af6a9c03b 100644 --- a/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs +++ b/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs @@ -59,10 +59,8 @@ private static string pythonLibDir // Attempt to get and cache the Dynamo Core directory path if (string.IsNullOrEmpty(dynamoCoreDir)) { - // Gather executing assembly info - Assembly assembly = Assembly.GetExecutingAssembly(); - Version version = assembly.GetName().Version; - string nodePath = Path.GetDirectoryName(assembly.Location); + // Gather executing assembly location + string nodePath = Assembly.GetExecutingAssembly().Location; dynamoCoreDir = Path.GetFullPath(Path.Combine(nodePath, @"..\")); } From 77b2dc4764135bf6e0f9c624bb94a20350a584d4 Mon Sep 17 00:00:00 2001 From: Keith Alfaro Date: Wed, 9 Jan 2019 13:38:52 -0500 Subject: [PATCH 05/23] address API changes --- .../IronPythonCompletionProvider.cs | 90 +++++++++++++++++-- .../ScriptEditorWindow.xaml.cs | 2 +- 2 files changed, 86 insertions(+), 6 deletions(-) diff --git a/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs b/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs index 64af6a9c03b..623d661e44d 100644 --- a/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs +++ b/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs @@ -241,17 +241,81 @@ public IronPythonCompletionProvider() } } + /// + /// Generates completion data for the specified text, while import the given types into the + /// scope and discovering variable assignments. + /// + /// The code to parse + /// Return a list of IronPythonCompletionData + [Obsolete("Please use GetCompletionData with additional parameters, this method will be removed in Dynamo 3.0.")] + public ICompletionData[] GetCompletionData(string line) + { + var items = new List(); + + this.UpdateImportedTypes(line); + this.UpdateVariableTypes(line); // this is where hindley-milner could come into play + + string name = GetName(line); + if (!String.IsNullOrEmpty(name)) + { + try + { + AutocompletionInProgress = true; + + // is it a CLR type? + var type = TryGetType(name); + if (type != null) + { + items = EnumerateMembers(type, name); + } + // it's a variable? + else if (this.VariableTypes.ContainsKey(name)) + { + items = EnumerateMembers(this.VariableTypes[name], name); + } + // is it a namespace or python type? + else + { + var mem = LookupMember(name); + + if (mem is NamespaceTracker) + { + items = EnumerateMembers(mem as NamespaceTracker, name); + } + else if (mem is PythonModule) + { + items = EnumerateMembers(mem as PythonModule, name); + } + else if (mem is PythonType) + { + // shows static and instance methods in just the same way :( + var value = ClrModule.GetClrType(mem as PythonType); + if (value != null) + { + items = EnumerateMembers(value, name); + } + } + + } + } + catch + { + //Dynamo.this.logger.Log("EXCEPTION: GETTING COMPLETION DATA"); + } + AutocompletionInProgress = false; + } + + return items.ToArray(); + } + /// /// Generate completion data for the specified text, while import the given types into the /// scope and discovering variable assignments. /// /// The code to parse /// Return a list of IronPythonCompletionData - public ICompletionData[] GetCompletionData(string code) + public ICompletionData[] GetCompletionData(string code, bool expand = false) { - // TODO - add optional param in Dynamo 3.0 with default set to false - bool expand = false; - var items = new List(); if (code.Contains("\"\"\"")) @@ -325,7 +389,7 @@ public ICompletionData[] GetCompletionData(string code) if (!items.Any() && !expand) { - return GetCompletionData(code); + return GetCompletionData(code, true); } return items.ToArray(); @@ -1137,5 +1201,21 @@ private string StripDocStrings(string code) var matches = TRIPPLE_QUOTE_STRINGS.Split(code); return String.Join("", matches); } + + /// + /// Returns the name from the end of a string. Matches back to the first space and trims off spaces or ('s + /// from the end of the line + /// + /// + /// + [Obsolete("This method will be removed in Dynamo 3.0")] + string GetName(string text) + { + text = text.Replace("\t", " "); + text = text.Replace("\n", " "); + text = text.Replace("\r", " "); + int startIndex = text.LastIndexOf(' '); + return text.Substring(startIndex + 1).Trim('.').Trim('('); + } } } \ No newline at end of file diff --git a/src/Libraries/PythonNodeModelsWpf/ScriptEditorWindow.xaml.cs b/src/Libraries/PythonNodeModelsWpf/ScriptEditorWindow.xaml.cs index 6145d6885c0..61a0711bb1b 100644 --- a/src/Libraries/PythonNodeModelsWpf/ScriptEditorWindow.xaml.cs +++ b/src/Libraries/PythonNodeModelsWpf/ScriptEditorWindow.xaml.cs @@ -94,7 +94,7 @@ private void OnTextAreaTextEntered(object sender, TextCompositionEventArgs e) if (e.Text == ".") { var subString = editText.Text.Substring(0, editText.CaretOffset); - var completions = completionProvider.GetCompletionData(subString); + var completions = completionProvider.GetCompletionData(subString, false); if (completions.Length == 0) return; From 844d2b36a1a1e6381de535a62e0ad4253021f4ab Mon Sep 17 00:00:00 2001 From: Keith Alfaro Date: Wed, 9 Jan 2019 16:02:03 -0500 Subject: [PATCH 06/23] autocomplete for standard lib --- .../IronPythonCompletionProvider.cs | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs b/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs index 623d661e44d..2cfd7c6a452 100644 --- a/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs +++ b/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs @@ -61,7 +61,7 @@ private static string pythonLibDir { // Gather executing assembly location string nodePath = Assembly.GetExecutingAssembly().Location; - dynamoCoreDir = Path.GetFullPath(Path.Combine(nodePath, @"..\")); + dynamoCoreDir = Path.GetFullPath(Path.Combine(nodePath, @"..\\..\\")); } // Return the standard library path @@ -118,12 +118,12 @@ private static string pythonLibDir // TODO - these can all be internal constants - Dynamo 3.0 public static string commaDelimitedVariableNamesRegex = @"(([0-9a-zA-Z_]+,?\s?)+)"; public static string variableName = @"([0-9a-zA-Z_]+(\.[a-zA-Z_0-9]+)*)"; - public static string doubleQuoteStringRegex = "(\"[^\"]*\")"; // TODO - Remove in Dynamo 3.0 - public static string singleQuoteStringRegex = "(\'[^\']*\')"; // TODO - Remove in Dynamo 3.0 + public static string doubleQuoteStringRegex = "(\"[^\"]*\")"; // Replaced w/ quotesStringRegex - Remove in Dynamo 3.0 + public static string singleQuoteStringRegex = "(\'[^\']*\')"; // Replaced w/ quotesStringRegex - Remove in Dynamo 3.0 public static string arrayRegex = "(\\[.*\\])"; public static string spacesOrNone = @"(\s*)"; public static string atLeastOneSpaceRegex = @"(\s+)"; - public static string equalsRegex = @"(=)"; + public static string equals = @"(=)"; // Not CLS compliant naming - Remove in Dynamo 3.0 public static string dictRegex = "({.*})"; public static string doubleRegex = @"([-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?)"; public static string intRegex = @"([-+]?\d+)[\s\n]*$"; @@ -131,6 +131,7 @@ private static string pythonLibDir public static string fromImportRegex = @"^(from)"; internal const string quotesStringRegex = "[\"']([^\"']*)[\"']"; + internal const string equalsRegex = @"(=)"; internal static readonly Regex MATCH_LAST_NAMESPACE = new Regex(@"[\w.]+$", RegexOptions.Compiled); internal static readonly Regex MATCH_LAST_WORD = new Regex(@"\w+$", RegexOptions.Compiled); @@ -148,7 +149,6 @@ private static string pythonLibDir internal static readonly Regex LIST_VARIABLE = new Regex("\\[.*\\]", RegexOptions.Compiled); internal static readonly Regex DICT_VARIABLE = new Regex("{.*}", RegexOptions.Compiled); - // TODO should this be Regex? internal static readonly string BAD_ASSIGNEMNT_ENDS = ",([{"; #endregion @@ -221,12 +221,14 @@ public IronPythonCompletionProvider() } // Determine if the Python Standard Library is available in the given context - if (Directory.Exists(pythonLibDir)) + var pythonStdLib = pythonLibDir; + + if (pythonStdLib != null && Directory.Exists(pythonStdLib)) { // Try to load Python Standard Library Type for autocomplete try { - var pyLibImports = String.Format("import sys\nsys.path.append(r'{0}')\n", pythonLibDir); + var pyLibImports = String.Format("import sys\nsys.path.append(r'{0}')\n", pythonStdLib); engine.CreateScriptSourceFromString(pyLibImports, SourceCodeKind.Statements).Execute(scope); } catch (Exception e) @@ -1039,7 +1041,9 @@ public void UpdateImportedTypes(string code) // TODO - add comment foreach (var i in importStatements) { - string module = i.Item1, memberName = i.Item2, asname = i.Item3; + string module = i.Item1; + string memberName = i.Item2; + string asname = i.Item3; string name = asname ?? memberName; string statement = ""; var previousTries = 0; From 5d56ebf08f882d4d8a71533491e352f3b4ef983d Mon Sep 17 00:00:00 2001 From: Keith Alfaro Date: Wed, 9 Jan 2019 16:46:08 -0500 Subject: [PATCH 07/23] comments and cleanup --- .../IronPythonCompletionProvider.cs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs b/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs index 2cfd7c6a452..45f5d2ccb17 100644 --- a/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs +++ b/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs @@ -315,6 +315,7 @@ public ICompletionData[] GetCompletionData(string line) /// scope and discovering variable assignments. /// /// The code to parse + /// Determines if the entire namespace should be used /// Return a list of IronPythonCompletionData public ICompletionData[] GetCompletionData(string code, bool expand = false) { @@ -328,6 +329,8 @@ public ICompletionData[] GetCompletionData(string code, bool expand = false) UpdateImportedTypes(code); UpdateVariableTypes(code); // Possibly use hindley-milner in the future + // If expand param is true use the entire namespace from the line of code + // Else just return the last name of the namespace string name = expand ? GetLastNameSpace(code) : GetLastName(code); if (!String.IsNullOrEmpty(name)) { @@ -389,6 +392,8 @@ public ICompletionData[] GetCompletionData(string code, bool expand = false) AutocompletionInProgress = false; } + // If unable to find matching results and expand was set to false, + // try again using the full namespace (set expand to true) if (!items.Any() && !expand) { return GetCompletionData(code, true); @@ -849,6 +854,9 @@ public Dictionary FindAllVariableAssignments(string code) /// A dictionary matching the lib to the code where lib is the library being imported from public static Dictionary FindAllTypeImportStatements(string code) { + // matches the following types: + // from lib import * + var pattern = fromImportRegex + atLeastOneSpaceRegex + variableName + @@ -1010,6 +1018,7 @@ public void UpdateImportedTypes(string code) { string libName = MATCH_FIRST_QUOTED_NAME.Match(statement).Groups[1].Value; + // If the library name cannot be found in the loaded clr modules if (!clrModules.Contains(libName)) { if (statement.Contains("AddReferenceToFileAndPath")) @@ -1019,8 +1028,6 @@ public void UpdateImportedTypes(string code) continue; } - // TODO - should there be a map of assemblies which can be used to verify - // the name exists as this lookup appears to occur frequently if (AppDomain.CurrentDomain.GetAssemblies().Any(x => x.GetName().Name == libName)) { engine.CreateScriptSourceFromString(statement, SourceCodeKind.SingleStatement).Execute(scope); @@ -1038,7 +1045,7 @@ public void UpdateImportedTypes(string code) var importStatements = FindAllImportStatements(code); - // TODO - add comment + // Format import statements based on available data foreach (var i in importStatements) { string module = i.Item1; @@ -1120,6 +1127,7 @@ public Dictionary> FindAllVariables(string code var variableType = TryGetType(possibleTypeName); if (variableType != null) { + // If variable has already been encountered if (variables.ContainsKey(name)) { if (currentIndex > variables[name].Item2) @@ -1127,7 +1135,7 @@ public Dictionary> FindAllVariables(string code variables[name] = new Tuple(typeString, currentIndex, variableType); } } - else // we've never seen it, add the type + else // New type, add it { variables.Add(name, new Tuple(typeString, currentIndex, variableType)); } From 03660c5df82fc0916b729c173a63d7e37ea209cd Mon Sep 17 00:00:00 2001 From: Keith Alfaro Date: Wed, 9 Jan 2019 19:20:13 -0500 Subject: [PATCH 08/23] InternalsVisibleTo not required until refactor --- .../PythonNodeModelsWpf/IronPythonCompletionProvider.cs | 9 +++++---- .../PythonNodeModelsWpf/Properties/AssemblyInfo.cs | 4 +--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs b/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs index 45f5d2ccb17..9d782017610 100644 --- a/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs +++ b/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs @@ -123,7 +123,7 @@ private static string pythonLibDir public static string arrayRegex = "(\\[.*\\])"; public static string spacesOrNone = @"(\s*)"; public static string atLeastOneSpaceRegex = @"(\s+)"; - public static string equals = @"(=)"; // Not CLS compliant naming - Remove in Dynamo 3.0 + public static string equals = @"(=)"; // Not CLS compliant - replaced with equalsRegex - Remove in Dynamo 3.0 public static string dictRegex = "({.*})"; public static string doubleRegex = @"([-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?)"; public static string intRegex = @"([-+]?\d+)[\s\n]*$"; @@ -264,18 +264,19 @@ public ICompletionData[] GetCompletionData(string line) { AutocompletionInProgress = true; - // is it a CLR type? var type = TryGetType(name); + + // Is it a CLR type? if (type != null) { items = EnumerateMembers(type, name); } - // it's a variable? + // Is it a variable? else if (this.VariableTypes.ContainsKey(name)) { items = EnumerateMembers(this.VariableTypes[name], name); } - // is it a namespace or python type? + // Else it is a namespace or python type else { var mem = LookupMember(name); diff --git a/src/Libraries/PythonNodeModelsWpf/Properties/AssemblyInfo.cs b/src/Libraries/PythonNodeModelsWpf/Properties/AssemblyInfo.cs index f830d751b1e..233960f4d54 100644 --- a/src/Libraries/PythonNodeModelsWpf/Properties/AssemblyInfo.cs +++ b/src/Libraries/PythonNodeModelsWpf/Properties/AssemblyInfo.cs @@ -9,6 +9,4 @@ [assembly: AssemblyTitle("PythonNodeModelsWpf")] [assembly: AssemblyCulture("")] // The following GUID is for the ID of the typelib if this project is exposed to COM -[assembly: Guid("5df40db3-627c-43fb-9a0e-0ee422d92485")] - -[assembly: InternalsVisibleTo("DynamoPythonTests")] \ No newline at end of file +[assembly: Guid("5df40db3-627c-43fb-9a0e-0ee422d92485")] \ No newline at end of file From 5ab51a26a9ea7a905e4a58a1a6f55e58296283a4 Mon Sep 17 00:00:00 2001 From: Keith Alfaro Date: Thu, 10 Jan 2019 09:37:33 -0500 Subject: [PATCH 09/23] restore unnecessary change --- src/Libraries/PythonNodeModelsWpf/Properties/AssemblyInfo.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Libraries/PythonNodeModelsWpf/Properties/AssemblyInfo.cs b/src/Libraries/PythonNodeModelsWpf/Properties/AssemblyInfo.cs index 233960f4d54..a39d34439a9 100644 --- a/src/Libraries/PythonNodeModelsWpf/Properties/AssemblyInfo.cs +++ b/src/Libraries/PythonNodeModelsWpf/Properties/AssemblyInfo.cs @@ -1,5 +1,4 @@ using System.Reflection; -using System.Runtime.CompilerServices; using System.Runtime.InteropServices; // General Information about an assembly is controlled through the following From 3372ea71335573a85e54a7009fc57c7c73fcc641 Mon Sep 17 00:00:00 2001 From: Keith Alfaro Date: Thu, 10 Jan 2019 10:19:30 -0500 Subject: [PATCH 10/23] remove unnecessary delta --- src/Libraries/PythonNodeModelsWpf/Properties/AssemblyInfo.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Libraries/PythonNodeModelsWpf/Properties/AssemblyInfo.cs b/src/Libraries/PythonNodeModelsWpf/Properties/AssemblyInfo.cs index a39d34439a9..1f18dc248f9 100644 --- a/src/Libraries/PythonNodeModelsWpf/Properties/AssemblyInfo.cs +++ b/src/Libraries/PythonNodeModelsWpf/Properties/AssemblyInfo.cs @@ -8,4 +8,4 @@ [assembly: AssemblyTitle("PythonNodeModelsWpf")] [assembly: AssemblyCulture("")] // The following GUID is for the ID of the typelib if this project is exposed to COM -[assembly: Guid("5df40db3-627c-43fb-9a0e-0ee422d92485")] \ No newline at end of file +[assembly: Guid("5df40db3-627c-43fb-9a0e-0ee422d92485")] From 7efab0359df7a4a5751b65bc07339836cc9fb3bc Mon Sep 17 00:00:00 2001 From: Keith Alfaro Date: Thu, 10 Jan 2019 15:13:45 -0500 Subject: [PATCH 11/23] change access modifiers for new methods --- .../PythonNodeModelsWpf/IronPythonCompletionProvider.cs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs b/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs index 9d782017610..ebdf35c60a6 100644 --- a/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs +++ b/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs @@ -728,7 +728,7 @@ protected Type TryGetType(string name) /// /// The code to search /// A list of tuples that contain the namespace, the module, and the custom name - public static List> FindAllImportStatements(string code) + private static List> FindAllImportStatements(string code) { var statements = new List>(); @@ -983,7 +983,12 @@ public static Dictionary FindVariableStatementWithRegex(string c return paramMatches; } - public List findClrReferences(string code) + /// + /// Detect all library references given the provided code + /// + /// Script code to search for CLR references + /// + private List findClrReferences(string code) { var statements = new List(); foreach (var line in code.Split(new[] { '\n', ';' })) From 20c2b691f2a9c639586794aacffbf4aa12933cd6 Mon Sep 17 00:00:00 2001 From: Keith Alfaro Date: Thu, 10 Jan 2019 21:19:10 -0500 Subject: [PATCH 12/23] get dynamo core path via the model --- .../IronPythonCompletionProvider.cs | 49 +++++++------------ .../ScriptEditorWindow.xaml.cs | 2 +- 2 files changed, 18 insertions(+), 33 deletions(-) diff --git a/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs b/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs index ebdf35c60a6..382c8711957 100644 --- a/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs +++ b/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs @@ -47,34 +47,7 @@ public ScriptScope Scope /// /// Cache storing a reference to the Dynamo Core directory /// - private static string dynamoCoreDir { get; set; } - - /// - /// Get the location of the Python Standard Library - /// - private static string pythonLibDir - { - get - { - // Attempt to get and cache the Dynamo Core directory path - if (string.IsNullOrEmpty(dynamoCoreDir)) - { - // Gather executing assembly location - string nodePath = Assembly.GetExecutingAssembly().Location; - dynamoCoreDir = Path.GetFullPath(Path.Combine(nodePath, @"..\\..\\")); - } - - // Return the standard library path - if (!string.IsNullOrEmpty(dynamoCoreDir)) - { - return dynamoCoreDir + @"IronPython.StdLib.2.7.8"; - } - else - { - return null; - } - } - } + private string pythonLibDir { get; set; } /// /// Already discovered variable types @@ -156,7 +129,16 @@ private static string pythonLibDir /// /// Class constructor /// - public IronPythonCompletionProvider() + [Obsolete("Use additional constructor passing the Dynamo Core Directory to enable Python Std Lib autocomplete.")] + public IronPythonCompletionProvider() : this("") + { + // TODO - remove this constructor in Dynamo 3.0) + } + + /// + /// Class constructor + /// + public IronPythonCompletionProvider(string dynamoCoreDir) { engine = IronPython.Hosting.Python.CreateEngine(); scope = engine.CreateScope(); @@ -221,14 +203,17 @@ public IronPythonCompletionProvider() } // Determine if the Python Standard Library is available in the given context - var pythonStdLib = pythonLibDir; + if(!String.IsNullOrEmpty(dynamoCoreDir)) + { + pythonLibDir = dynamoCoreDir + @"\IronPython.StdLib.2.7.8"; + } - if (pythonStdLib != null && Directory.Exists(pythonStdLib)) + if (!String.IsNullOrEmpty(pythonLibDir) && Directory.Exists(pythonLibDir)) { // Try to load Python Standard Library Type for autocomplete try { - var pyLibImports = String.Format("import sys\nsys.path.append(r'{0}')\n", pythonStdLib); + var pyLibImports = String.Format("import sys\nsys.path.append(r'{0}')\n", pythonLibDir); engine.CreateScriptSourceFromString(pyLibImports, SourceCodeKind.Statements).Execute(scope); } catch (Exception e) diff --git a/src/Libraries/PythonNodeModelsWpf/ScriptEditorWindow.xaml.cs b/src/Libraries/PythonNodeModelsWpf/ScriptEditorWindow.xaml.cs index 61a0711bb1b..e1e7084c663 100644 --- a/src/Libraries/PythonNodeModelsWpf/ScriptEditorWindow.xaml.cs +++ b/src/Libraries/PythonNodeModelsWpf/ScriptEditorWindow.xaml.cs @@ -39,7 +39,7 @@ ref ModelessChildWindow.WindowRect windowRect this.dynamoViewModel = dynamoViewModel; this.nodeModel = nodeModel; - completionProvider = new IronPythonCompletionProvider(); + completionProvider = new IronPythonCompletionProvider(dynamoViewModel.Model.PathManager.DynamoCoreDirectory); completionProvider.MessageLogged += dynamoViewModel.Model.Logger.Log; InitializeComponent(); From 945e659515ea597be9117219bc21f5da954377ed Mon Sep 17 00:00:00 2001 From: Keith Alfaro Date: Fri, 11 Jan 2019 09:19:10 -0500 Subject: [PATCH 13/23] cleanup obsolete redundancy --- .../IronPythonCompletionProvider.cs | 58 +------------------ 1 file changed, 1 insertion(+), 57 deletions(-) diff --git a/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs b/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs index 382c8711957..5b0e1cdcf45 100644 --- a/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs +++ b/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs @@ -237,63 +237,7 @@ public IronPythonCompletionProvider(string dynamoCoreDir) [Obsolete("Please use GetCompletionData with additional parameters, this method will be removed in Dynamo 3.0.")] public ICompletionData[] GetCompletionData(string line) { - var items = new List(); - - this.UpdateImportedTypes(line); - this.UpdateVariableTypes(line); // this is where hindley-milner could come into play - - string name = GetName(line); - if (!String.IsNullOrEmpty(name)) - { - try - { - AutocompletionInProgress = true; - - var type = TryGetType(name); - - // Is it a CLR type? - if (type != null) - { - items = EnumerateMembers(type, name); - } - // Is it a variable? - else if (this.VariableTypes.ContainsKey(name)) - { - items = EnumerateMembers(this.VariableTypes[name], name); - } - // Else it is a namespace or python type - else - { - var mem = LookupMember(name); - - if (mem is NamespaceTracker) - { - items = EnumerateMembers(mem as NamespaceTracker, name); - } - else if (mem is PythonModule) - { - items = EnumerateMembers(mem as PythonModule, name); - } - else if (mem is PythonType) - { - // shows static and instance methods in just the same way :( - var value = ClrModule.GetClrType(mem as PythonType); - if (value != null) - { - items = EnumerateMembers(value, name); - } - } - - } - } - catch - { - //Dynamo.this.logger.Log("EXCEPTION: GETTING COMPLETION DATA"); - } - AutocompletionInProgress = false; - } - - return items.ToArray(); + return GetCompletionData(line, false); } /// From 28f9bb24b26d1cd8eb00de556ecd6a7bc6a6c754 Mon Sep 17 00:00:00 2001 From: Keith Alfaro Date: Fri, 11 Jan 2019 11:11:41 -0500 Subject: [PATCH 14/23] step count with new imported type --- test/Libraries/DynamoPythonTests/CodeCompletionTests.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/Libraries/DynamoPythonTests/CodeCompletionTests.cs b/test/Libraries/DynamoPythonTests/CodeCompletionTests.cs index e7d043b51e5..185b96b0644 100644 --- a/test/Libraries/DynamoPythonTests/CodeCompletionTests.cs +++ b/test/Libraries/DynamoPythonTests/CodeCompletionTests.cs @@ -237,7 +237,7 @@ public void CanImportLibrary() var completionProvider = new IronPythonCompletionProvider(); completionProvider.UpdateImportedTypes(str); - Assert.AreEqual(1, completionProvider.ImportedTypes.Count); + Assert.AreEqual(2, completionProvider.ImportedTypes.Count); Assert.IsTrue(completionProvider.ImportedTypes.ContainsKey("System")); } @@ -249,7 +249,7 @@ public void DuplicateCallsToImportShouldBeFine() var completionProvider = new IronPythonCompletionProvider(); completionProvider.UpdateImportedTypes(str); - Assert.AreEqual(1, completionProvider.ImportedTypes.Count); + Assert.AreEqual(2, completionProvider.ImportedTypes.Count); Assert.IsTrue(completionProvider.ImportedTypes.ContainsKey("System")); } @@ -266,7 +266,7 @@ public void CanImportSystemLibraryAndGetCompletionData() var completionList = completionData.Select(d => d.Text); Assert.IsTrue(completionList.Any()); Assert.IsTrue(completionList.Intersect(new[] { "IO", "Console", "Reflection" }).Count() == 3); - Assert.AreEqual(1, completionProvider.ImportedTypes.Count); + Assert.AreEqual(2, completionProvider.ImportedTypes.Count); Assert.IsTrue(completionProvider.ImportedTypes.ContainsKey("System")); } From 321f0897beeac3e4b4e0f4e0a68e97a0e36e5407 Mon Sep 17 00:00:00 2001 From: Keith Alfaro Date: Fri, 11 Jan 2019 16:52:24 -0500 Subject: [PATCH 15/23] update summary tag --- .../PythonNodeModelsWpf/IronPythonCompletionProvider.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs b/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs index 5b0e1cdcf45..156fab3414b 100644 --- a/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs +++ b/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs @@ -45,7 +45,7 @@ public ScriptScope Scope } /// - /// Cache storing a reference to the Dynamo Core directory + /// Store a reference to the Dynamo Core Python Standard Library directory /// private string pythonLibDir { get; set; } From 23b6b271942ea658b2cf7c495993c56a034036f4 Mon Sep 17 00:00:00 2001 From: Keith Alfaro Date: Fri, 11 Jan 2019 18:15:19 -0500 Subject: [PATCH 16/23] use Path.Combine --- .../PythonNodeModelsWpf/IronPythonCompletionProvider.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs b/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs index 156fab3414b..26bee844937 100644 --- a/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs +++ b/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs @@ -205,7 +205,7 @@ public IronPythonCompletionProvider(string dynamoCoreDir) // Determine if the Python Standard Library is available in the given context if(!String.IsNullOrEmpty(dynamoCoreDir)) { - pythonLibDir = dynamoCoreDir + @"\IronPython.StdLib.2.7.8"; + pythonLibDir = Path.Combine(dynamoCoreDir + @"\IronPython.StdLib.2.7.8"); } if (!String.IsNullOrEmpty(pythonLibDir) && Directory.Exists(pythonLibDir)) From 68e99ed92b6ffc351d7302c1f691cd8f9a3bcec8 Mon Sep 17 00:00:00 2001 From: Keith Alfaro Date: Mon, 14 Jan 2019 12:04:53 -0500 Subject: [PATCH 17/23] fix path combine --- .../PythonNodeModelsWpf/IronPythonCompletionProvider.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs b/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs index 26bee844937..f674e384ddc 100644 --- a/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs +++ b/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs @@ -205,7 +205,7 @@ public IronPythonCompletionProvider(string dynamoCoreDir) // Determine if the Python Standard Library is available in the given context if(!String.IsNullOrEmpty(dynamoCoreDir)) { - pythonLibDir = Path.Combine(dynamoCoreDir + @"\IronPython.StdLib.2.7.8"); + pythonLibDir = Path.Combine(dynamoCoreDir, @"\IronPython.StdLib.2.7.8"); } if (!String.IsNullOrEmpty(pythonLibDir) && Directory.Exists(pythonLibDir)) From e7c5bcbf8b61795c6842ba297f5398dab06b63a0 Mon Sep 17 00:00:00 2001 From: Keith Alfaro Date: Tue, 15 Jan 2019 12:42:44 -0500 Subject: [PATCH 18/23] pull Mike K's changes --- .../IronPythonCompletionProvider.cs | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs b/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs index f674e384ddc..d2fd1e8d301 100644 --- a/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs +++ b/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs @@ -999,13 +999,22 @@ public void UpdateImportedTypes(string code) { if (module == null) { - statement = String.Format("import {0} as {1}", memberName, name); + if (asname == null) + { + statement = String.Format("import {0}", memberName); + + } + else + { + statement = String.Format("import {0} as {1}", memberName, asname); + + } } else { - if (memberName != "*") + if (memberName != "*" && asname != null) { - statement = String.Format("from {0} import {1} as {2}", module, memberName, name); + statement = String.Format("from {0} import {1} as {2}", module, memberName, asname); } else { @@ -1034,7 +1043,7 @@ public void UpdateImportedTypes(string code) catch (Exception e) { Log(String.Format("Failed to load module: {0}, with statement: {1}", memberName, statement)); - Log(e.ToString()); + // Log(e.ToString()); badStatements[statement] = previousTries + 1; } } From 55c5ad0a67c36b05994d5bc839f51eba58dc34b8 Mon Sep 17 00:00:00 2001 From: Keith Alfaro Date: Tue, 15 Jan 2019 12:43:54 -0500 Subject: [PATCH 19/23] add regions to large cs file --- .../IronPythonCompletionProvider.cs | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs b/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs index d2fd1e8d301..9d6340b78f5 100644 --- a/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs +++ b/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs @@ -49,6 +49,16 @@ public ScriptScope Scope /// private string pythonLibDir { get; set; } + /// + /// A list of short assembly names used with the TryGetTypeFromFullName method + /// + private static string[] knownAssemblies = { + "mscorlib", + "RevitAPI", + "RevitAPIUI", + "ProtoGeometry" + }; + /// /// Already discovered variable types /// @@ -126,6 +136,7 @@ public ScriptScope Scope #endregion + #region Constructors /// /// Class constructor /// @@ -227,7 +238,9 @@ public IronPythonCompletionProvider(string dynamoCoreDir) Log("Valid IronPython Standard Library not found. Python autocomplete will not see native modules."); } } + #endregion + #region Methods /// /// Generates completion data for the specified text, while import the given types into the /// scope and discovering variable assignments. @@ -588,16 +601,6 @@ public void UpdateVariableTypes(string code) VariableTypes = FindAllVariableAssignments(code); } - /// - /// A list of short assembly names used with the TryGetTypeFromFullName method - /// - private static string[] knownAssemblies = { - "mscorlib", - "RevitAPI", - "RevitAPIUI", - "ProtoGeometry" - }; - /// /// Check if a full type name is found in one of the known pre-loaded assemblies and return the type /// @@ -1173,5 +1176,6 @@ string GetName(string text) int startIndex = text.LastIndexOf(' '); return text.Substring(startIndex + 1).Trim('.').Trim('('); } + #endregion } } \ No newline at end of file From 1e78023e361bc72c439b42f23779e205a84dd2d3 Mon Sep 17 00:00:00 2001 From: Keith Alfaro Date: Tue, 15 Jan 2019 15:16:46 -0500 Subject: [PATCH 20/23] lots of cleanup! --- .../IronPythonCompletionProvider.cs | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs b/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs index 9d6340b78f5..62b50c25ebc 100644 --- a/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs +++ b/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs @@ -78,6 +78,7 @@ public ScriptScope Scope /// Maps a regex to a particular python type. Useful for matching things like dicts, /// floats, strings, etc. Initialized by /// + [Obsolete("This property has been replaced with BasicVariableTypes and will be removed in Dynamo 3.0")] public Dictionary RegexToType = new Dictionary(); /// @@ -216,7 +217,7 @@ public IronPythonCompletionProvider(string dynamoCoreDir) // Determine if the Python Standard Library is available in the given context if(!String.IsNullOrEmpty(dynamoCoreDir)) { - pythonLibDir = Path.Combine(dynamoCoreDir, @"\IronPython.StdLib.2.7.8"); + pythonLibDir = Path.Combine(dynamoCoreDir, @"IronPython.StdLib.2.7.8"); } if (!String.IsNullOrEmpty(pythonLibDir) && Directory.Exists(pythonLibDir)) @@ -533,6 +534,7 @@ public object LookupMember(string name) /// the dot character that triggered the completion. The text can contain the command line prompt /// '>>>' as this will be ignored. /// + [Obsolete("Please use additional GetDescription method as this one will be removed in Dynamo 3.0.")] public void GetDescription(string stub, string item, DescriptionUpdateDelegate updateDescription, bool isInstance) { string description = this.GetDescription(stub, item, isInstance); @@ -946,7 +948,7 @@ public void UpdateImportedTypes(string code) { var previousTries = 0; badStatements.TryGetValue(statement, out previousTries); - // TODO - why is this 3? Should this be a constant? + // TODO - Why is this 3? Should this be a constant? Is it related to knownAssembies.Length? if (previousTries > 3) { continue; @@ -1005,12 +1007,10 @@ public void UpdateImportedTypes(string code) if (asname == null) { statement = String.Format("import {0}", memberName); - } else { statement = String.Format("import {0} as {1}", memberName, asname); - } } else @@ -1060,7 +1060,8 @@ public void UpdateImportedTypes(string code) public Dictionary> FindAllVariables(string code) { var variables = new Dictionary>(); - var variableStatements = Regex.Matches(code, variableName + spacesOrNone + equalsRegex + spacesOrNone + @"(.*)", RegexOptions.Multiline); + var pattern = variableName + spacesOrNone + equalsRegex + spacesOrNone + @"(.*)"; + var variableStatements = Regex.Matches(code, pattern, RegexOptions.Multiline); for (var i = 0; i < variableStatements.Count; i++) { @@ -1092,9 +1093,9 @@ public Dictionary> FindAllVariables(string code } // Match default types (numbers, dicts, arrays, strings) - foreach (var pair in RegexToType) + foreach (var pair in BasicVariableTypes) { - var matches = Regex.Matches(typeString, "^" + pair.Key + "$", RegexOptions.Singleline); + var matches = Regex.Matches(typeString, "^" + pair.Item1 + "$", RegexOptions.Singleline); if (matches.Count > 0) { // If variable has already been encountered @@ -1102,12 +1103,12 @@ public Dictionary> FindAllVariables(string code { if (currentIndex > variables[name].Item2) { - variables[name] = new Tuple(typeString, currentIndex, pair.Value); + variables[name] = new Tuple(typeString, currentIndex, pair.Item2); } } else // New type, add it { - variables.Add(name, new Tuple(typeString, currentIndex, pair.Value)); + variables.Add(name, new Tuple(typeString, currentIndex, pair.Item2)); } break; From dea96524fd95f8908400a1b0723582a748f5ec13 Mon Sep 17 00:00:00 2001 From: Keith Alfaro Date: Wed, 16 Jan 2019 20:53:27 -0500 Subject: [PATCH 21/23] fix final failing test --- .../IronPythonCompletionProvider.cs | 33 +++++++++++++++++-- .../DynamoPythonTests/CodeCompletionTests.cs | 4 +-- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs b/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs index 62b50c25ebc..08ecf310c01 100644 --- a/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs +++ b/src/Libraries/PythonNodeModelsWpf/IronPythonCompletionProvider.cs @@ -671,12 +671,41 @@ private static List> FindAllImportStatements(strin var importMatches = MATCH_IMPORT_STATEMENTS.Matches(code); foreach (Match m in importMatches) { + var names = new List(); + + // If the match ends with '.' if (m.Value.EndsWith(".")) { - continue; // Incomplete statement + // For each group in mathces + foreach (Group item in m.Groups) + { + // Clone + var text = m.Value; + + // Reformat statment + text = text.Replace("\t", " ") + .Replace("\n", " ") + .Replace("\r", " "); + var spaceIndex = text.LastIndexOf(' '); + var equalsIndex = text.LastIndexOf('='); + var clean = text.Substring(Math.Max(spaceIndex, equalsIndex) + 1).Trim('.').Trim('('); + + // Check for multi-line statement + var allStatements = clean.Trim().Split(new char[] { ',' }).Select(x => x.Trim()).ToList(); + + // Build names output + foreach(string statement in allStatements) + { + names.Add(statement); + } + } + } + else + { + // Check for multi-line statement + names = m.Groups[1].Value.Trim().Split(new char[] { ',' }).Select(x => x.Trim()).ToList(); } - var names = m.Groups[1].Value.Trim().Split(new char[] { ',' }).Select(x => x.Trim()); foreach (string n in names) { var parts = n.Split(new string[] { " as " }, 2, StringSplitOptions.RemoveEmptyEntries); diff --git a/test/Libraries/DynamoPythonTests/CodeCompletionTests.cs b/test/Libraries/DynamoPythonTests/CodeCompletionTests.cs index 185b96b0644..ea8ba7a72fc 100644 --- a/test/Libraries/DynamoPythonTests/CodeCompletionTests.cs +++ b/test/Libraries/DynamoPythonTests/CodeCompletionTests.cs @@ -268,7 +268,6 @@ public void CanImportSystemLibraryAndGetCompletionData() Assert.IsTrue(completionList.Intersect(new[] { "IO", "Console", "Reflection" }).Count() == 3); Assert.AreEqual(2, completionProvider.ImportedTypes.Count); Assert.IsTrue(completionProvider.ImportedTypes.ContainsKey("System")); - } [Test] @@ -293,6 +292,7 @@ public void CanMatchImportSystemLibraryWithComment() var matches = IronPythonCompletionProvider.FindBasicImportStatements(str); Assert.AreEqual(1, matches.Count); + Assert.IsTrue(matches.ContainsKey("System")); } [Test] @@ -303,7 +303,7 @@ public void CanMatchImportSystemAndLoadLibraryAndWithComment() var completionProvider = new IronPythonCompletionProvider(); completionProvider.UpdateImportedTypes(str); - Assert.AreEqual(1, completionProvider.ImportedTypes.Count); + Assert.AreEqual(2, completionProvider.ImportedTypes.Count); Assert.IsTrue(completionProvider.ImportedTypes.ContainsKey("System")); } From 30eea8a76715e4001c2b68c86f92db471e6c97d9 Mon Sep 17 00:00:00 2001 From: Keith Alfaro Date: Thu, 17 Jan 2019 14:22:53 -0500 Subject: [PATCH 22/23] verify collections of imported types with more than 1 type contain all expected defaults --- .../DynamoPythonTests/CodeCompletionTests.cs | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/test/Libraries/DynamoPythonTests/CodeCompletionTests.cs b/test/Libraries/DynamoPythonTests/CodeCompletionTests.cs index ea8ba7a72fc..5e59e7e3e78 100644 --- a/test/Libraries/DynamoPythonTests/CodeCompletionTests.cs +++ b/test/Libraries/DynamoPythonTests/CodeCompletionTests.cs @@ -250,7 +250,7 @@ public void DuplicateCallsToImportShouldBeFine() completionProvider.UpdateImportedTypes(str); Assert.AreEqual(2, completionProvider.ImportedTypes.Count); - Assert.IsTrue(completionProvider.ImportedTypes.ContainsKey("System")); + verifyImportedTypes(completionProvider.ImportedTypes); } [Test] @@ -267,7 +267,7 @@ public void CanImportSystemLibraryAndGetCompletionData() Assert.IsTrue(completionList.Any()); Assert.IsTrue(completionList.Intersect(new[] { "IO", "Console", "Reflection" }).Count() == 3); Assert.AreEqual(2, completionProvider.ImportedTypes.Count); - Assert.IsTrue(completionProvider.ImportedTypes.ContainsKey("System")); + verifyImportedTypes(completionProvider.ImportedTypes); } [Test] @@ -304,7 +304,7 @@ public void CanMatchImportSystemAndLoadLibraryAndWithComment() completionProvider.UpdateImportedTypes(str); Assert.AreEqual(2, completionProvider.ImportedTypes.Count); - Assert.IsTrue(completionProvider.ImportedTypes.ContainsKey("System")); + verifyImportedTypes(completionProvider.ImportedTypes); } [Test] @@ -317,7 +317,6 @@ public void CanIdentifyVariableTypeAndGetCompletionData() var completionData = completionProvider.GetCompletionData(str); Assert.AreNotEqual(0, completionData.Length); - } [Test] @@ -331,7 +330,6 @@ public void CanFindTypeSpecificImportsMultipleTypesSingleLine() Assert.AreEqual("from math import sin", imports["sin"]); Assert.IsTrue( imports.ContainsKey("cos") ); Assert.AreEqual("from math import cos", imports["cos"]); - } [Test] @@ -343,7 +341,6 @@ public void CanFindTypeSpecificImportsSingleTypeSingleLine() var imports = IronPythonCompletionProvider.FindTypeSpecificImportStatements(str); Assert.IsTrue(imports.ContainsKey("sin")); Assert.AreEqual("from math import sin", imports["sin"]); - } [Test] @@ -355,7 +352,6 @@ public void CanFindTypeSpecificAutodeskImportsSingleTypeSingleLine() var imports = IronPythonCompletionProvider.FindTypeSpecificImportStatements(str); Assert.IsTrue(imports.ContainsKey("Events")); Assert.AreEqual("from Autodesk.Revit.DB import Events", imports["Events"]); - } [Test] @@ -367,7 +363,6 @@ public void CanFindAllTypeImports() var imports = IronPythonCompletionProvider.FindAllTypeImportStatements(str); Assert.IsTrue(imports.ContainsKey("Autodesk.Revit.DB")); Assert.AreEqual("from Autodesk.Revit.DB import *", imports["Autodesk.Revit.DB"]); - } [Test] @@ -434,5 +429,19 @@ public void VerifyIronPythonLoadedAssemblies() } } } + + private void verifyImportedTypes(Dictionary importedTypes) + { + var defaultImports = new string[] + { + "System", + "None" + }; + + foreach (var type in importedTypes) + { + Assert.Contains(type.Key, defaultImports); + } + } } } From 6d7268e8ca248fe634134becab02697b13145d09 Mon Sep 17 00:00:00 2001 From: Keith Alfaro Date: Fri, 18 Jan 2019 10:10:33 -0500 Subject: [PATCH 23/23] switch to SequenceEquals for collection comparison --- .../DynamoPythonTests/CodeCompletionTests.cs | 27 +++++++------------ 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/test/Libraries/DynamoPythonTests/CodeCompletionTests.cs b/test/Libraries/DynamoPythonTests/CodeCompletionTests.cs index 5e59e7e3e78..5ab4c60b425 100644 --- a/test/Libraries/DynamoPythonTests/CodeCompletionTests.cs +++ b/test/Libraries/DynamoPythonTests/CodeCompletionTests.cs @@ -77,6 +77,13 @@ public string Warning private ILogger logger; + // List of expected default imported types + private List defaultImports = new List() + { + "None", + "System" + }; + [SetUp] public void SetupPythonTests() { @@ -250,7 +257,7 @@ public void DuplicateCallsToImportShouldBeFine() completionProvider.UpdateImportedTypes(str); Assert.AreEqual(2, completionProvider.ImportedTypes.Count); - verifyImportedTypes(completionProvider.ImportedTypes); + Assert.IsTrue(defaultImports.SequenceEqual(completionProvider.ImportedTypes.Keys.ToList())); } [Test] @@ -267,7 +274,7 @@ public void CanImportSystemLibraryAndGetCompletionData() Assert.IsTrue(completionList.Any()); Assert.IsTrue(completionList.Intersect(new[] { "IO", "Console", "Reflection" }).Count() == 3); Assert.AreEqual(2, completionProvider.ImportedTypes.Count); - verifyImportedTypes(completionProvider.ImportedTypes); + Assert.IsTrue(defaultImports.SequenceEqual(completionProvider.ImportedTypes.Keys.ToList())); } [Test] @@ -304,7 +311,7 @@ public void CanMatchImportSystemAndLoadLibraryAndWithComment() completionProvider.UpdateImportedTypes(str); Assert.AreEqual(2, completionProvider.ImportedTypes.Count); - verifyImportedTypes(completionProvider.ImportedTypes); + Assert.IsTrue(defaultImports.SequenceEqual(completionProvider.ImportedTypes.Keys.ToList())); } [Test] @@ -429,19 +436,5 @@ public void VerifyIronPythonLoadedAssemblies() } } } - - private void verifyImportedTypes(Dictionary importedTypes) - { - var defaultImports = new string[] - { - "System", - "None" - }; - - foreach (var type in importedTypes) - { - Assert.Contains(type.Key, defaultImports); - } - } } }