FIX: MaximalSubgroupClassReps with options.

The routine for `MaximalSubgroupClassReps allows options to calculate
maximal subgroups only up to specified limits, or if it is not too
difficult. This causes problems is such a partial list is stored as
attribute. Thus separate into two attributes: One to calculate the
guaranteed full list, one to calculate a potentially partial list subject to
restrictions. Also make sure that limiting options do not get accidentally
inherited. (In the big scheme of things we might want to revisit the
question of ``cheap attributes'' more generally, as composition tree uses
similar paradigms.)

Finally allow a soft fallback to old intermediate subgroup routines, if
maximal subgroups are not available. (This can go away once proper maximal
subgroups code is available.)

Also added test file

In the best of all worlds this should be backported to 4.9

lib/csetgrp.gi

@@ -209,7 +209,7 @@ end);
 ##  the operation of G on the Right Cosets of U.
 ##
 InstallGlobalFunction( IntermediateGroup, function(G,U)
-local o,b,img,G1,c,m,hardlimit,gens,t,k,intersize;
+local o,b,img,G1,c,m,mt,hardlimit,gens,t,k,intersize;
 
   if U=G then
     return fail;
@@ -221,34 +221,38 @@ local o,b,img,G1,c,m,hardlimit,gens,t,k,intersize;
     return fail; # avoid infinite recursion
   fi;
 
-  # use maximals
-  m:=MaximalSubgroupClassReps(G:cheap,intersize:=intersize);
-
-  m:=Filtered(m,x->Size(x) mod Size(U)=0 and Size(x)>Size(U));
-  SortBy(m,x->Size(G)/Size(x));
-
-  gens:=SmallGeneratingSet(U);
-  for c in m do
-    if Index(G,c)<50000 then
-      t:=RightTransversal(G,c:noascendingchain); # conjugates
-      for k in t do
-        if ForAll(gens,x->k*x/k in c) then
-	  Info(InfoCoset,2,"Found Size ",Size(c),"\n");
-          # U is contained in c^k
-          return c^k;
+  # use maximals, use `Try` as we call with limiting options
+  IsNaturalAlternatingGroup(G);
+  IsNaturalSymmetricGroup(G);
+  m:=TryMaximalSubgroupClassReps(G:cheap,intersize:=intersize,nolattice);
+  if m<>fail and Length(m)>0 then
+
+    m:=Filtered(m,x->Size(x) mod Size(U)=0 and Size(x)>Size(U));
+    SortBy(m,x->Size(G)/Size(x));
+
+    gens:=SmallGeneratingSet(U);
+    for c in m do
+      if Index(G,c)<50000 then
+        t:=RightTransversal(G,c:noascendingchain); # conjugates
+        for k in t do
+          if ForAll(gens,x->k*x/k in c) then
+            Info(InfoCoset,2,"Found Size ",Size(c),"\n");
+            # U is contained in c^k
+            return c^k;
+          fi;
+        od;
+      else
+        t:=DoConjugateInto(G,c,U,true:intersize:=intersize,onlyone:=true);
+        if t<>fail and t<>[] then 
+          Info(InfoCoset,2,"Found Size ",Size(c),"\n");
+          return c^(Inverse(t));
         fi;
-      od;
-    else
-      t:=DoConjugateInto(G,c,U,true:intersize:=intersize,onlyone:=true);
-      if t<>fail and t<>[] then 
-	Info(InfoCoset,2,"Found Size ",Size(c),"\n");
-        return c^(Inverse(t));
       fi;
-    fi;
-  od;
+    od;
 
-  Info(InfoCoset,2,"Found no intermediate subgroup ",Size(G)," ",Size(U));
-  return fail;
+    Info(InfoCoset,2,"Found no intermediate subgroup ",Size(G)," ",Size(U));
+    return fail;
+  fi;
 
   # old code -- obsolete
 
@@ -821,7 +825,7 @@ local c, flip, maxidx, refineChainActionLimit, cano, tryfct, p, r, t,
 	fi;
       end;
 
-      for i in MaximalSubgroupClassReps(G:cheap) do
+      for i in TryMaximalSubgroupClassReps(G:cheap) do
 	if Index(G,i)<maxidx(c) and Index(G,i)<badlimit then
 	  p:=Intersection(a,i);
 	  if Index(a,p)<uplimit then
@@ -843,7 +847,7 @@ local c, flip, maxidx, refineChainActionLimit, cano, tryfct, p, r, t,
 
     if maxidx(c)>10*actlimit then
 
-      r:=ShallowCopy(MaximalSubgroupClassReps(a:cheap));
+      r:=ShallowCopy(TryMaximalSubgroupClassReps(a:cheap));
       r:=Filtered(r,x->Index(a,x)<uplimit);
 
       Sort(r,function(a,b) return Size(a)<Size(b);end);

lib/factgrp.gi

@@ -753,7 +753,7 @@ totalcnt, interupt, u, nu, cor, zzz,bigperm,perm,badcores,max,i;
 	  # only affine ones are needed, rest will have wrong kernel
 	  max:=DoMaxesTF(u,["1"]:inmax,cheap);
 	else
-	  max:=MaximalSubgroupClassReps(u:inmax,cheap);
+	  max:=TryMaximalSubgroupClassReps(u:inmax,cheap);
 	fi;
         max:=Filtered(max,x->IndexNC(G,x)<knowi and IsSubset(x,N)); 
         for i in max do

lib/gpprmsya.gi

@@ -2409,7 +2409,7 @@ local G,max,dom,n,A,S,issn,p,i,j,m,k,powdec,pd,gps,v,invol,sel,mf,l,prim;
   return max;
 end);
 
-InstallMethod( MaximalSubgroupClassReps, "symmetric", true,
+InstallMethod( TryMaximalSubgroupClassReps, "symmetric", true,
     [ IsNaturalSymmetricGroup and IsFinite], OVERRIDENICE,
 function ( G )
 local m;
@@ -2421,7 +2421,7 @@ local m;
   fi;
 end);
 
-InstallMethod( MaximalSubgroupClassReps, "alternating", true,
+InstallMethod( TryMaximalSubgroupClassReps, "alternating", true,
     [ IsNaturalAlternatingGroup and IsFinite], OVERRIDENICE,
 function ( G )
 local m;

lib/grp.gd

@@ -1146,8 +1146,9 @@ DeclareAttribute( "ConjugacyClasses", IsGroup );
 ##  <Ref Func="MaximalSubgroupClassReps"/>.
 ##  <Example><![CDATA[
 ##  gap> ConjugacyClassesMaximalSubgroups(g);
-##  [ AlternatingGroup( [ 1 .. 4 ] )^G, Group( [ (1,2,3), (1,2) ] )^G, 
-##    Group( [ (1,2), (3,4), (1,3)(2,4) ] )^G ]
+##  [ Group( [ (2,4,3), (1,4)(2,3), (1,3)(2,4) ] )^G,
+##    Group( [ (3,4), (1,4)(2,3), (1,3)(2,4) ] )^G,
+##    Group( [ (3,4), (2,4,3) ] )^G ]
 ##  ]]></Example>
 ##  </Description>
 ##  </ManSection>
@@ -1193,16 +1194,20 @@ DeclareAttribute( "MaximalSubgroups", IsGroup );
 ##  of <A>G</A>.
 ##  <Example><![CDATA[
 ##  gap> MaximalSubgroupClassReps(g);
-##  [ Alt( [ 1 .. 4 ] ), Group([ (1,2,3), (1,2) ]), Group([ (1,2), (3,4),
-##      (1,3)(2,4) ]) ]
+##  [ Group([ (2,4,3), (1,4)(2,3), (1,3)(2,4) ]), Group([ (3,4), (1,4)
+##    (2,3), (1,3)(2,4) ]), Group([ (3,4), (2,4,3) ]) ]
 ##  ]]></Example>
 ##  </Description>
 ##  </ManSection>
 ##  <#/GAPDoc>
 ##
 DeclareAttribute("MaximalSubgroupClassReps",IsGroup);
 
+# utility attribute: Allow use with limiting options, so could hold `fail'.
+DeclareAttribute("TryMaximalSubgroupClassReps",IsGroup,"mutable");
+
 # utility function in maximal subgroups code
+DeclareGlobalFunction("TryMaxSubgroupTainter");
 DeclareGlobalFunction("DoMaxesTF");
 DeclareGlobalFunction("MaxesAlmostSimple");
 

lib/grp.gi

@@ -1342,6 +1342,50 @@ end);
 ##
 #M  MaximalSubgroups( <G> )
 ##
+InstallMethod(MaximalSubgroupClassReps,"default, catch dangerous options",
+  true,[IsGroup],0,
+function(G)
+local H,a,m,i,l;
+  # easy case, go without options
+  if not HasTryMaximalSubgroupClassReps(G) then
+    return TryMaximalSubgroupClassReps(G:
+           # as if options were unset
+           cheap:=fail,intersize:=fail,inmax:=fail,nolattice:=fail);
+  fi;
+
+  # hard case -- `Try` is stored
+  if not IsBound(G!.maxsubtrytaint) or G!.maxsubtrytaint=false then
+    # stored and untainted, just go on
+    return TryMaximalSubgroupClassReps(G); 
+  fi;
+
+  # compute anew for new group to avoid taint
+  H:=Group(GeneratorsOfGroup(G));
+  for i in [Size,IsNaturalAlternatingGroup,IsNaturalSymmetricGroup] do
+    if Tester(i)(G) then Setter(i)(H,i(G));fi;
+  od;
+  m:=TryMaximalSubgroupClassReps(H:
+          cheap:=false,intersize:=false,inmax:=false,nolattice:=false);
+  l:=[];
+  for i in m do
+    a:=SubgroupNC(G,GeneratorsOfGroup(i));
+    if HasSize(i) then SetSize(a,Size(i));fi;
+    Add(l,a);
+  od;
+
+  # now we know list is untained, store 
+  Unbind(G!.maxsubtrytaint);
+  G!.TryMaximalSubgroupClassReps:=l;
+  return l;
+
+end);
+
+InstallGlobalFunction(TryMaxSubgroupTainter,function(G)
+  if ForAny(["cheap","intersize","inmax","nolattice"],
+       x->not ValueOption(x) in [fail,false]) then
+       G!.maxsubtrytaint:=true;
+  fi;
+end);
 
 #############################################################################
 ##

lib/grplatt.gi

@@ -1767,10 +1767,12 @@ end);
 #F  MaximalSubgroupClassReps(<G>) . . . . reps of conjugacy classes of
 #F                                                          maximal subgroups
 ##
-InstallMethod(MaximalSubgroupClassReps,"using lattice",true,[IsGroup],0,
+InstallMethod(TryMaximalSubgroupClassReps,"using lattice",true,[IsGroup],0,
 function (G)
     local   maxs,lat;
 
+    TryMaxSubgroupTainter(G);
+    if ValueOption("nolattice")=true then return fail;fi;
     #AH special AG treatment
     if not HasIsSolvableGroup(G) and IsSolvableGroup(G) then
       return MaximalSubgroupClassReps(G);
@@ -2345,6 +2347,7 @@ InstallMethod(IntermediateSubgroups,"using maximal subgroups",
   1, # better than previous if index larger
 function(G,U)
 local uind,subs,incl,i,j,k,m,gens,t,c,p,conj,bas,basl,r;
+
   if (not IsFinite(G)) and Index(G,U)=infinity then
     TryNextMethod();
   fi;
@@ -2359,11 +2362,13 @@ local uind,subs,incl,i,j,k,m,gens,t,c,p,conj,bas,basl,r;
   gens:=SmallGeneratingSet(U);
   while i<=Length(subs) do
     if conj[i]<>fail then
-      m:=MaximalSubgroupClassReps(subs[conj[i][1]]); # fetch attribute
+      m:=TryMaximalSubgroupClassReps(subs[conj[i][1]]:nolattice); # fetch
+      if m=fail then TryNextMethod();fi;
       m:=List(m,x->x^conj[i][2]);
     else
       # find all maximals containing U
-      m:=MaximalSubgroupClassReps(subs[i]);
+      m:=TryMaximalSubgroupClassReps(subs[i]:nolattice);
+      if m=fail then TryNextMethod();fi;
     fi;
     m:=Filtered(m,x->IndexNC(subs[i],U) mod IndexNC(subs[i],x)=0);
 

lib/grpnice.gi

@@ -682,7 +682,20 @@ GroupSeriesMethodByNiceMonomorphism( LowerCentralSeriesOfGroup,
 ##
 #M  MaximalSubgroupClassReps( <G> )
 ##
-SubgroupsMethodByNiceMonomorphism( MaximalSubgroupClassReps, [ IsGroup ] );
+InstallOtherMethod( TryMaximalSubgroupClassReps,
+  "handled by nice monomorphism, transfer tainter", true, [IsGroup], 0,
+function( G )
+local   nice,  img,  sub,i;
+  TryMaxSubgroupTainter(G);
+  nice := NiceMonomorphism(G);
+  img  := ShallowCopy(TryMaximalSubgroupClassReps( NiceObject(G) ));
+  for i in [1..Length(img)] do
+    sub  := GroupByNiceMonomorphism( nice, img[i] );
+    SetParent( sub, G );
+    img[i]:=sub;
+  od;
+  return img;
+end );
 
 
 #############################################################################

lib/grppcatr.gi

@@ -369,6 +369,7 @@ end;
 MAXSUBS_BY_PCGS:=function( G )
     local spec, first, max, i, new;
 
+    TryMaxSubgroupTainter(G);
     spec  := SpecialPcgs(G);
     first := LGFirst( spec );
     max   := [];
@@ -384,15 +385,15 @@ end;
 ##
 #M  MaximalSubgroupClassReps( <G> )
 ##
-InstallMethod( MaximalSubgroupClassReps,
+InstallMethod( TryMaximalSubgroupClassReps,
     "pcgs computable groups using special pcgs",
     true, 
     [ IsGroup and CanEasilyComputePcgs and IsFinite ],
     0,
     MAXSUBS_BY_PCGS);
 
 #fallback
-InstallMethod( MaximalSubgroupClassReps,
+InstallMethod( TryMaximalSubgroupClassReps,
     "pcgs computable groups using special pcgs",
     true, 
     [ IsGroup and IsSolvableGroup and IsFinite ],

lib/maxsub.gi

@@ -804,6 +804,7 @@ local G,types,ff,maxes,lmax,q,d,dorb,dorbt,i,dorbc,dorba,dn,act,comb,smax,soc,
   a1emb,a2emb,anew,wnew,e1,e2,emb,a1,a2,mm;
 
   G:=arg[1];
+  TryMaxSubgroupTainter(G);
 
   # which kinds of maxes do we want to get
   if Length(arg)>1 then
@@ -991,6 +992,9 @@ end);
 InstallMethod(MaximalSubgroupClassReps,"TF method",true,
   [IsGroup and IsFinite and CanComputeFittingFree],OVERRIDENICE,DoMaxesTF);
 
+InstallMethod(TryMaximalSubgroupClassReps,"TF method",true,
+  [IsGroup and IsFinite and CanComputeFittingFree],OVERRIDENICE,DoMaxesTF);
+
 #InstallMethod(MaximalSubgroupClassReps,"perm group",true,
 #  [IsPermGroup and IsFinite],0,DoMaxesTF);
 

lib/pcgsperm.gi

@@ -1430,10 +1430,11 @@ end);
 ##
 ##  method for solvable perm groups -- it is cheaper to translate to a pc
 ##  group
-InstallMethod( MaximalSubgroupClassReps,"solvable perm group",true, 
+InstallMethod( TryMaximalSubgroupClassReps,"solvable perm group",true, 
     [ IsPermGroup and CanEasilyComputePcgs and IsFinite ], 0,
 function(G)
 local hom,m;
+  TryMaxSubgroupTainter(G);
   hom:=IsomorphismPcGroup(G);
   m:=MaximalSubgroupClassReps(Image(hom));
   List(m,Size); # force

tst/testbugfix/2018-05-24-IntermediateSubgroups.tst

@@ -0,0 +1,16 @@
+# test for MaximalSubgroupClassReps with options (reported through
+# observation by S.Alavi with IntermediateGroup
+
+gap> g:= AtlasGroup( "U4(4)" );;  Size( g );
+1018368000
+gap> h:= Image( IsomorphismPermGroup( PSL(2,16) ) );;
+gap> l:= IsomorphicSubgroups( g, h );;
+gap> Length( l );
+2
+gap> s1:= Image( l[1] );;  Size( s1 );
+4080
+gap> n1:= Normalizer( g, s1 );;  Size( n1 );
+24480
+gap> int:=IntermediateGroup(g,s1);;
+gap> IsGroup(int);
+true