Fix stabilizer computation in alternating groups (#5811)

lib/gpprmsya.gi

@@ -933,21 +933,25 @@ end);
 ##
 BindGlobal( "SYMGP_STABILIZER", function(sym, arg...)
     local  k, act, pt, mov, stab, nat, diff, int, bls, mov1, parts,
-           part, bl, i, gens, size;
+           part, bl, i, gens, size, alt;
     k := Length(arg);
     act := arg[k];
     pt := arg[k-3];
 
     if arg[k-1] <> arg[k-2] then
         TryNextMethod();
     fi;
+    alt := IsNaturalAlternatingGroup(sym);
+    if alt then
+      sym := SymmetricParentGroup(sym);
+    fi;
     mov := MovedPoints(sym);
     if act = OnPoints and IsPosInt(pt) then
         if pt in mov then
             stab := SymmetricGroup(Difference(mov,[pt]));
         else
             stab := sym;
-        fi ;
+        fi;
         nat := true;
     elif (act = OnTuples or act = OnPairs) and IsList(pt) and ForAll(pt, IsPosInt) then
         stab := SymmetricGroup(Difference(mov, Set(pt)));
@@ -1005,6 +1009,9 @@ BindGlobal( "SYMGP_STABILIZER", function(sym, arg...)
     if nat then
         SetIsNaturalSymmetricGroup(stab,true);
     fi;
+    if alt then
+        stab := AlternatingSubgroup(stab);
+    fi;
     return stab;
 end );
 
@@ -1033,25 +1040,15 @@ InstallOtherMethod( StabilizerOp,"alternating group", true,
   # the objects might be a group element: rank up
         {} -> RankFilter(IsMultiplicativeElementWithInverse) +
         RankFilter(IsSolvableGroup),
-function(g, arg...)
-local s;
-  s:=SymmetricParentGroup(g);
-  # we cannot go to the symmetric group if the acting elements are different
-  if arg[2]<>arg[3] or not IsSubset(s,arg[2]) then
-    TryNextMethod();
-  fi;
-  return AlternatingSubgroup(Stabilizer(s,arg[1],GeneratorsOfGroup(s),GeneratorsOfGroup(s),arg[4]));
-end);
+        SYMGP_STABILIZER);
 
 
 InstallOtherMethod( StabilizerOp,"alternating group", true,
     [ IsNaturalAlternatingGroup, IsDomain, IsObject, IsList, IsList, IsFunction ],
   # the objects might be a group element: rank up
         {} -> RankFilter(IsMultiplicativeElementWithInverse) +
         RankFilter(IsSolvableGroup),
-        function(g, arg...)
-    return AlternatingSubgroup(CallFuncList(Stabilizer, Concatenation([SymmetricParentGroup(g)], arg)));
-end);
+        SYMGP_STABILIZER);
 
 InstallMethod( CentralizerOp,
     "element in natural alternating group",

tst/testbugfix/2024-10-11-alt-stab.tst

@@ -0,0 +1,26 @@
+# Fix for https://github.com/gap-system/gap/issues/5808
+# See also https://github.com/gap-system/gap/pull/5811
+gap> G := AlternatingGroup( 6 );;
+gap> hom1 := GroupHomomorphismByImages( G, G, [ (1,2,3,4,5), (4,5,6) ], [ (1,2,6,3,5), (1,4,5) ] );;
+gap> hom2 := GroupHomomorphismByImages( G, G, [ (1,2,3,4,5), (4,5,6) ], [ (1,2,3,4,5), (4,5,6) ] );;
+gap> tc := function ( g, h ) return (h^hom2)^-1 * g * h^hom1; end;;
+gap> stab := Stabilizer( G, One(G), tc );;
+gap> Size(stab);
+5
+gap> Set( stab ) = Set( Filtered( G, g -> tc( One(G), g ) = One(G) ) );
+true
+gap> Set(stab);
+[ (), (1,2,6,4,5), (1,4,2,5,6), (1,5,4,6,2), (1,6,5,2,4) ]
+
+#
+gap> G := AlternatingGroup( 6 );;
+gap> hom1 := GroupHomomorphismByImages( G, G, [ (1,2,3,4,5), (4,5,6) ], [ (1,2,6,3,5), (1,4,5) ] );;
+gap> hom2 := IdentityMapping( G );;
+gap> tc := function ( g, h ) return (h^hom2)^-1 * g * h^hom1; end;;
+gap> stab := Stabilizer( G, One(G), tc );;
+gap> Size(stab);
+5
+gap> Set( stab ) = Set( Filtered( G, g -> tc( One(G), g ) = One(G) ) );
+true
+gap> Set(stab);
+[ (), (1,2,6,4,5), (1,4,2,5,6), (1,5,4,6,2), (1,6,5,2,4) ]