Merge pull request #1546 from mahrud/quickfix/capture

Fix capture + several minor bug fixes

M2/Macaulay2/d/actors5.d

@@ -1565,17 +1565,17 @@ dictionaryPath(e:Expr):Expr := (
      is t:List do (					    -- set the current globalDictionary list
 	  s := t.v;
 	  n := length(s);
-	  if n == 0 then return WrongArg("expected a nonempty list of dictionaries");
+	  if n == 0 then return WrongArg("a nonempty list of dictionaries");
           sawUnprotected := false;
 	  foreach x in s do 
 	  when x is dc:DictionaryClosure do (
 	       d := dc.dictionary;
 	       if !d.Protected then sawUnprotected = true;
-	       if d.frameID != 0 || d.transient then return WrongArg("expected a list of global dictionaries")
+	       if d.frameID != 0 || d.transient then return WrongArg("a list of global dictionaries")
 	       )
-	  else return WrongArg("expected a list of dictionaries");
-	  for i from 0 to n-2 do for j from i+1 to n-1 do if s.i == s.j then return WrongArg("expected a list of dictionaries with no duplicate entries");
-          if !sawUnprotected then return WrongArg("expected a list of dictionaries, not all protected");
+	  else return WrongArg("a list of dictionaries");
+	  for i from 0 to n-2 do for j from i+1 to n-1 do if s.i == s.j then return WrongArg("a list of dictionaries with no duplicate entries");
+          if !sawUnprotected then return WrongArg("a list of dictionaries, not all protected");
      	  a := new array(Dictionary) len n do foreach x in s do when x is d:DictionaryClosure do provide d.dictionary else nothing;
      	  a.(n-1).outerDictionary = a.(n-1);
      	  for i from 0 to n-2 do a.i.outerDictionary = a.(i+1);

M2/Macaulay2/d/actors6.dd

@@ -27,7 +27,7 @@ showElapsedTimefun(a:Code):Expr := (
      time4 := Ccode(Chrono, "std::chrono::steady_clock::now()");
      d := Ccode(long, "(time4-time3).count()");
      x := double(d)/1000000000.;
-     stdIO << "     -- " << x << " seconds elapsed" << endl;
+     stdError << " -- " << x << " seconds elapsed" << endl;
      ret);
 setupop(elapsedTimeS,showElapsedTimefun);
 

M2/Macaulay2/d/interp.dd

@@ -529,6 +529,9 @@ capture(e:Expr):Expr := (
      when e
      is s:stringCell do (
      	  flush(stdIO);
+	  flush(stdError);
+	  oldStdError := stdError; -- doesn't quite work, see stderrS in actors2.dd
+	  stdError = stdIO;
 	  oldDebuggingMode := debuggingMode;
 	  setDebuggingMode(false);
           foss := getFileFOSS(stdIO);
@@ -552,6 +555,7 @@ capture(e:Expr):Expr := (
 	  foss.outindex = oldindex;
 	  setLineNumber(oldLineNumber);
 	  setDebuggingMode(oldDebuggingMode);
+	  stdError = oldStdError;
 	  previousLineNumber = -1;
 	  Expr(Sequence( when r is err:Error do True else False, toExpr(out) )))
      else WrongArg(1,"a string"));

M2/Macaulay2/m2/examples.m2

@@ -22,15 +22,44 @@ separateM2output = str -> drop(drop(separate(M2outputRE, str),1),-1)
 -- TODO: the output format is provisional
 -- TODO: does't capture stderr
 capture' := capture
-capture = method()
-capture Net    := s -> capture toString s
-capture List   := s -> capture demark_newline s
---capture String := s -> capture' s -- output is (Boolean, String) => (Err?, Output)
+capture = method(Options => { UserMode => true })
+capture Net    := opts -> s -> capture(toString s,       opts)
+capture List   := opts -> s -> capture(demark_newline s, opts)
 -- TODO: do this in interp.dd instead
-capture String := s -> (
-    pushvar(symbol interpreterDepth, 1);
+-- TODO: alternatively, change the setup to do this in a clean thread
+capture String := opts -> s -> if opts.UserMode then capture' s else (
+    -- output is (Boolean, String) => (Err?, Output)
+    -- TODO: this should eventually be unnecessary
+    oldThreadLocalVars := (gbTrace, debugLevel, errorDepth, interpreterDepth, debuggingMode, stopIfError, notify);
+    interpreterDepth = 1;
+
+    oldPrivateDictionary := User#"private dictionary";
+    oldDictionaryPath := dictionaryPath;
+    oldLoadedPackages := loadedPackages;
+    oldCurrentPackage := currentPackage;
+    pushvar(symbol OutputDictionary, new GlobalDictionary);
+
+    User#"private dictionary" = new Dictionary;
+    OutputDictionary = new GlobalDictionary;
+    dictionaryPath = {
+	User#"private dictionary", -- this is necessary mainly due to indeterminates.m2
+	oldPrivateDictionary,
+	Core.Dictionary,
+	OutputDictionary,
+	PackageDictionary};
+    scan(Core#"pre-installed packages", needsPackage);
+    needsPackage toString currentPackage;
+    currentPackage = User;
+
     ret := capture' s;
-    popvar symbol interpreterDepth;
+
+    User#"private dictionary" = oldPrivateDictionary;
+    dictionaryPath = oldDictionaryPath;
+    loadedPackages = oldLoadedPackages;
+    currentPackage = oldCurrentPackage;
+    popvar symbol OutputDictionary;
+
+    (gbTrace, debugLevel, errorDepth, interpreterDepth, debuggingMode, stopIfError, notify) = oldThreadLocalVars;
     ret)
 protect symbol capture
 
@@ -80,7 +109,7 @@ getExampleOutput := (pkg, fkey) -> (
     output := if fileExists filename
     then ( verboseLog("info: reading cached example results from ", filename); get filename )
     else if width (ex := examples fkey) =!= 0
-    then ( verboseLog("info: capturing example results on-demand"); last capture ex );
+    then ( verboseLog("info: capturing example results on-demand"); last capture(ex, UserMode => false) );
     pkg#"example results"#fkey = if output === null then {} else separateM2output output)
 
 -- used in installPackage.m2
@@ -94,15 +123,30 @@ storeExampleOutput = (pkg, fkey, outf, verboseLog) -> (
 
 -- used in installPackage.m2
 captureExampleOutput = (pkg, fkey, inputs, cacheFunc, inf, outf, errf, inputhash, changeFunc, usermode, verboseLog) -> (
-    desc := "example results for " | fkey;
---    printerr("capturing ", desc);
---    (err, output) := evaluateWithPackage(pkg, inputs, capture);
---    if not err then ( outf << "-- -*- M2-comint -*- hash: " << inputhash << endl << output << close ) else
-    (
-	data := if pkg#"example data files"#?fkey then pkg#"example data files"#fkey else {};
-	inf << inputs << endl << close;
-	if runFile(inf, inputhash, outf, errf, desc, pkg, changeFunc fkey, usermode, data)
-	then ( removeFile inf; cacheFunc fkey )))
+    stdio << flush; -- just in case previous timing information hasn't been flushed yet
+    desc := "example results for " | format fkey;
+    desc  = concatenate(desc, 62 - #desc);
+    -- try capturing in the same process
+    if  not match("no-capture-flag", inputs) -- this flag is really necessary, but only sometimes
+    -- FIXME: these are workarounds to prevent bugs, in order of priority for being fixed:
+    and not match("(gbTrace|read|run|stderr|stdio|print|<<)", inputs) -- stderr and prints are not handled correctly
+    and not match("(notify|stopIfError|debuggingMode)", inputs) -- stopIfError and debuggingMode may be fixable
+    and not match("([Cc]ommand|schedule|thread|Task)", inputs) -- remove when threads work more predictably
+    and not match("(temporaryFileName)", inputs) -- this is sometimes bug prone
+    and not match("(installMethod|load|export|newPackage)", inputs) -- exports may land in the package User
+    and not match("(GlobalAssignHook|GlobalReleaseHook)", inputs) -- same as above
+    and not match({"ThreadedGB", "RunExternalM2"}, pkg#"pkgname") -- TODO: eventually remove
+    and false -- TODO: this is temporarily here, to be removed after v1.17 is released
+    then (
+	stderr << commentize ("capturing ", desc) << flush; -- the timing info will appear at the end
+	(err, output) := evaluateWithPackage(pkg, inputs, capture_(UserMode => false));
+	if not err then return outf << "-- -*- M2-comint -*- hash: " << inputhash << endl << output << close);
+    -- fallback to using an external process
+    desc  = concatenate(desc, 65 - #desc);
+    data := if pkg#"example data files"#?fkey then pkg#"example data files"#fkey else {};
+    inf << replace("-\\* no-capture-flag \\*-", "", inputs) << endl << close;
+    if runFile(inf, inputhash, outf, errf, desc, pkg, changeFunc fkey, usermode, data)
+    then ( removeFile inf; cacheFunc fkey ))
 
 -----------------------------------------------------------------------------
 -- process examples

M2/Macaulay2/m2/exports.m2

@@ -292,7 +292,6 @@ export {
 	"Options",
 	"Order",
 	"OrderedMonoid",
-	"OutputDictionary",
 	"Outputs",
 	"POSIX",
 	"Package",
@@ -1214,6 +1213,7 @@ export {
 }
 
 exportMutable {
+	"OutputDictionary",
 	"allowableThreads",
 	"applicationDirectorySuffix",
 	"backtrace",

M2/Macaulay2/m2/help.m2

@@ -148,7 +148,7 @@ documentationValue := method(TypicalValue => Hypertext)
 documentationValue(Symbol, Thing) := (S, X) -> ()
 -- e.g. Macaulay2Doc :: MethodFunction
 documentationValue(Symbol, Type)  := (S, T) -> (
-    syms := unique flatten(values \ dictionaryPath);
+    syms := unique flatten apply(dictionaryPath, dict -> if mutable dict then {} else values dict);
     -- constructors of T
     a := smenu apply(select(pairs typicalValues, (key, Y) -> Y === T and isDocumentableMethod key), (key, Y) -> key);
     -- types that inherit from T

M2/Macaulay2/m2/indeterminates.m2

@@ -1,7 +1,5 @@
 --		Copyright 1993-1999 by Daniel R. Grayson
 
-indeterminates =  new MutableHashTable
-
 varIndices := new MutableHashTable
 
 varName := i -> (
@@ -12,11 +10,9 @@ varName := i -> (
      else "x" | toString(i-52))
 
 vars ZZ := i -> (
-     if indeterminates#?i then indeterminates#i else (
-	  x := getSymbol varName i;
-	  indeterminates#i = x;
-	  varIndices#x = i;
-	  x))
+    x := getSymbol varName i;
+    varIndices#x = i;
+    x)
 
 vars List := vars Sequence := args -> apply(flatten splice args, j -> vars j)
 

M2/Macaulay2/m2/installPackage.m2

@@ -514,7 +514,7 @@ generateExampleResults := (pkg, rawDocumentationCache, exampleDir, exampleOutput
 	    and not opts.RerunExamples and fileExists outf' and gethash outf' === inputhash then (
 		if fileExists errf then removeFile errf; copyFile(outf', outf))
 	    -- run and capture example results
-	    else captureExampleOutput(
+	    else elapsedTime captureExampleOutput(
 		pkg, fkey, demark_newline inputs,
 		possiblyCache(outf, outf'),
 		inpf, outf, errf,
@@ -555,7 +555,7 @@ installPackage = method(
 	RunExamples            => true,
 	SeparateExec           => false,
 	UserMode               => null,
-	Verbose                => true
+	Verbose                => false
 	})
 
 installPackage String := opts -> pkg -> (

M2/Macaulay2/m2/localring.m2

@@ -25,7 +25,7 @@
 protect MaximalIdeal
 
 LocalRing = new Type of EngineRing
-LocalRing.synonym = "Local ring"
+LocalRing.synonym = "local ring"
 LocalRing#{Standard,AfterPrint} = RP -> (
      << endl << concatenate(interpreterDepth:"o") << lineNumber << " : "; -- standard template
      << "LocalRing, maximal " << RP.MaximalIdeal << endl;

M2/Macaulay2/m2/packages.m2

@@ -385,7 +385,6 @@ newPackage String := opts -> pkgname -> (
     setAttribute(newpkg.Dictionary,           PrintNames, pkgname | ".Dictionary");
     setAttribute(newpkg#"private dictionary", PrintNames, pkgname | "#\"private dictionary\"");
     debuggingMode = opts.DebuggingMode;		    -- last step before turning control back to code of package
-    -- if pkgname =!= "SimpleDoc" and pkgname =!= "Core" and pkgname =!= "Text" then needsPackage "SimpleDoc";
     scan(opts.PackageImports, needsPackage);
     scan(opts.PackageExports, needsPackage);
     newpkg.loadDepth = loadDepth;
@@ -499,17 +498,14 @@ endPackage String := title -> (
 
 
 beginDocumentation = () -> (
-    if loadPackageOptions#?(currentPackage#"pkgname") and not loadPackageOptions#(currentPackage#"pkgname").LoadDocumentation
+    pkgname := currentPackage#"pkgname";
+    if loadPackageOptions#?pkgname and not loadPackageOptions#pkgname.LoadDocumentation
     and currentPackage#?rawKeyDB and isOpen currentPackage#rawKeyDB then (
-	if notify then stderr << "--beginDocumentation: using documentation database, skipping the rest of " << currentFileName << endl;
+	if notify then printerr("beginDocumentation: using documentation database, skipping the rest of ", currentFileName);
 	currentPackage#"documentation not loaded" = true;
 	return end);
-     if notify then stderr << "--beginDocumentation: reading the rest of " << currentFileName << endl;
-     if currentPackage#"pkgname" != "Text" and  currentPackage#"pkgname" != "SimpleDoc" then (
-	 needsPackage "Text";
-	 needsPackage "SimpleDoc";
-	  );
-     )
+    if notify then printerr("beginDocumentation: reading the rest of ", currentFileName);
+    if not member(pkgname, {"Text", "SimpleDoc"}) then (needsPackage "SimpleDoc"; needsPackage "Text";))
 
 ---------------------------------------------------------------------
 
@@ -535,18 +531,12 @@ package Sequence  := s -> youngest (package \ s)
 package Array     := s -> package toSequence s
 
 use Package := pkg -> (
-     a := member(pkg, loadedPackages);
-     b := member(pkg.Dictionary, dictionaryPath);
-     if a and not b then error("use: package ", toString pkg, " appears in loadedPackages, but its dictionary is missing from dictionaryPath");
-     if b and not a then error("use: package ", toString pkg, " does not appear in loadedPackages, but its dictionary appears in dictionaryPath");
-     if not a and not b then (
-     	  scan(pkg.Options.PackageExports, needsPackage);
-     	  loadedPackages = prepend(pkg, loadedPackages);
-	  --- if mutable pkg.Dictionary then error("package ", toString pkg, " not completely loaded yet");
-	  dictionaryPath = prepend(pkg.Dictionary, dictionaryPath);
-	  checkShadow();
-	  );
-     if pkg.?use then pkg.use pkg else pkg)
+    -- TODO: where is this ever used? make sure these simplifications are okay
+    scan(pkg.Options.PackageExports, needsPackage);
+    if not member(pkg,            loadedPackages) then loadedPackages = prepend(pkg,            loadedPackages);
+    if not member(pkg.Dictionary, dictionaryPath) then dictionaryPath = prepend(pkg.Dictionary, dictionaryPath);
+    checkShadow();
+    if pkg.?use then pkg.use pkg else pkg)
 
 debug = method()
 debug ZZ      := i   -> debugWarningHashcode = i

M2/Macaulay2/m2/run.m2

@@ -60,7 +60,7 @@ SetOutputFile  := 1 << 31 -* add >>tmpf *-
 SetCaptureErr  := 1 << 32 -* add 2>&1 *-
 
 -* by default, the following commandline fixtures are used *-
-defaultMode := (SetUlimit + GCMAXHEAP + ArgQ + ArgInt
+defaultMode := (SetUlimit + GCMAXHEAP + ArgQ + ArgInt -- + ArgNoPreload
     + ArgNoRandomize + ArgNoReadline + ArgSilent + ArgStop
     + ArgPrintWidth + SetInputFile + SetOutputFile + SetCaptureErr)
 -* making this global, so it can be edited after entering debug Core *-
@@ -78,7 +78,7 @@ argumentMode = defaultMode
 -- returns false if error
 runFile = (inf, inputhash, outf, tmpf, desc, pkg, announcechange, usermode, examplefiles) -> (
      announcechange();
-     stderr << " -- making " << desc << ( if debugLevel > 0 then " in file " | outf else "" ) << endl;
+     stderr << commentize ("making ", desc, if debugLevel > 0 then " in file " | outf) << flush; -- skipping endl on purpose
      if fileExists outf then removeFile outf;
      pkgname := toString pkg;
      tmpf << "-- -*- M2-comint -*- hash: " << inputhash << endl << close; -- must match regular expression below
@@ -100,6 +100,7 @@ runFile = (inf, inputhash, outf, tmpf, desc, pkg, announcechange, usermode, exam
      cmd = cmd | readmode(ArgInt,         "--int");
      cmd = cmd | readmode(ArgNoBacktrace, "--no-backtrace");
      cmd = cmd | readmode(ArgNoDebug,     "--no-debug");
+     if pkgname != "Macaulay2Doc" then -- TODO: eventually remove this line
      cmd = cmd | readmode(ArgNoPreload,   "--no-preload");
      cmd = cmd | readmode(ArgNoRandomize, "--no-randomize");
      cmd = cmd | readmode(ArgNoReadline,  "--no-readline");
@@ -112,32 +113,21 @@ runFile = (inf, inputhash, outf, tmpf, desc, pkg, announcechange, usermode, exam
      cmd = cmd | readmode(ArgStop,        "--stop");
      cmd = cmd | readmode(ArgPrintWidth,  "--print-width 77");
      cmd = cmd | concatenate apply(srcdirs, d -> (" --srcdir ", format d));
+     -- TODO: fix capture, add preloaded packages to Macaulay2Doc, then delete the following two lines
      needsline := concatenate(" -e 'needsPackage(\"",pkgname,"\", Reload => true, FileName => \"",pkg#"source file","\")'");
      cmd = cmd | if pkgname != "Macaulay2Doc" then needsline else "";
      cmd = cmd | readmode(SetInputFile,   "<" | format inf);
      cmd = cmd | readmode(SetOutputFile,  ">>" | format toAbsolutePath tmpf);
      cmd = cmd | readmode(SetCaptureErr,  "2>&1");
-     if debugLevel > 0 then stderr << cmd << endl;
+     if debugLevel > 0 then stderr << endl << cmd << endl;
      for fn in examplefiles do copyFile(fn,rundir | baseFilename fn);
      r := run cmd;
      if r == 0 then (
-	  scan(reverse findFiles rundir, f -> if isDirectory f then (
-		    -- under cygwin, it seems to take a random amount of time before the system knows the directory is no longer in use:
-		    try removeDirectory f
-		    else (
-			 stderr << "--warning: *** removing a directory failed, waiting..." << endl;
-			 sleep 1;
-			 try removeDirectory f
-			 else (
-			      stderr << "--warning: *** removing a directory failed again, waiting..." << endl;
-			      sleep 4;
-			      removeDirectory f
-			      )
-			 )
-		    ) else removeFile f);
+	  scan(reverse findFiles rundir, f -> if isDirectory f then removeDirectory f else removeFile f);
 	  moveFile(tmpf,outf);
 	  return true;
 	  );
+     if debugLevel == 0 then stderr << endl;
      stderr << cmd << endl;
      stderr << tmpf << ":0:1: (output file) error: Macaulay2 " << describeReturnCode r << endl;
      stderr << aftermatch(M2errorRegexp,get tmpf);

M2/Macaulay2/packages/CMakeLists.txt

@@ -158,6 +158,11 @@ foreach(package IN LISTS PACKAGES)
     endforeach()
   endif()
 
+  ## TODO: remove this; see https://github.com/Macaulay2/M2/pull/1483
+  if(package MATCHES "(ThreadedGB)")
+    set(USES_TERMINAL USES_TERMINAL)
+  endif()
+
   ## Custom command for getting package information
   # TODO: currently only number of tests is stored, but we can do more
   add_custom_command(OUTPUT info-${package}

M2/Macaulay2/packages/Complexes.m2

@@ -12,6 +12,7 @@ newPackage(
             HomePage => "http://www.math.cornell.edu/~mike"
             }},
     Headline => "development package for beta testing new version of chain complexes",
+    Keywords => {"Homological Algebra"},
     PackageExports => {"Truncations"},
     AuxiliaryFiles => true,
     DebuggingMode => false

M2/Macaulay2/packages/Cremona.m2

@@ -13,7 +13,7 @@ newPackage(
         Date => "November 20, 2020",
     	Authors => {{Name => "Giovanni Staglianò", Email => "[email protected]" }},
     	Headline => "rational maps between projective varieties",
-	Keywords => {"Algebraic geometry"},
+	Keywords => {"Algebraic Geometry"},
         AuxiliaryFiles => true,
 	Certification => {
 	     "journal name" => "The Journal of Software for Algebra and Geometry",