Skip to content

Commit

Permalink
Merge branch 'cachedQuery'
Browse files Browse the repository at this point in the history 
# Conflicts:
#	Signum.React.Extensions/Dashboard/View/ValueUserQueryListPart.tsx
#	Signum.React/Scripts/Frames/FramePage.tsx
  • Loading branch information
@olmobrutall
olmobrutall committed Nov 29, 2021
2 parents 451ed5e + e1186be commit 641293c
Show file tree
Hide file tree
Showing 69 changed files with 2,458 additions and 984 deletions.
56 changes: 13 additions & 43 deletions Signum.Engine.Extensions/Chart/ChartLogic.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,57 +22,27 @@ public static void Start(SchemaBuilder sb, bool googleMapsChartScripts, string[]

public static Task<ResultTable> ExecuteChartAsync(ChartRequestModel request, CancellationToken token)
{
IDynamicQueryCore core = QueryLogic.Queries.GetQuery(request.QueryName).Core.Value;

return miExecuteChartAsync.GetInvoker(core.GetType().GetGenericArguments()[0])(request, core, token);
}

static GenericInvoker<Func<ChartRequestModel, IDynamicQueryCore, CancellationToken, Task<ResultTable>>> miExecuteChartAsync =
new((req, dq, token) => ExecuteChartAsync<int>(req, (DynamicQueryCore<int>)dq, token));
static async Task<ResultTable> ExecuteChartAsync<T>(ChartRequestModel request, DynamicQueryCore<T> dq, CancellationToken token)
{
using (ExecutionMode.UserInterface())
{
var result = await dq.ExecuteQueryAsync(new QueryRequest
{
GroupResults = request.HasAggregates(),
QueryName = request.QueryName,
Columns = request.GetQueryColumns(),
Filters = request.Filters,
Orders = request.GetQueryOrders(),
Pagination = request.MaxRows.HasValue ? new Pagination.Firsts(request.MaxRows.Value + 1) : new Pagination.All(),
}, token);


if (request.MaxRows.HasValue && result.Rows.Length == request.MaxRows.Value)
throw new InvalidOperationException($"The chart request for ${request.QueryName} exceeded the max rows ({request.MaxRows})");

return result;
}
return QueryLogic.Queries.ExecuteQueryAsync(request.ToQueryRequest(), token);
}


public static ResultTable ExecuteChart(ChartRequestModel request)
{
IDynamicQueryCore core = QueryLogic.Queries.GetQuery(request.QueryName).Core.Value;
return QueryLogic.Queries.ExecuteQuery(request.ToQueryRequest());

return miExecuteChart.GetInvoker(core.GetType().GetGenericArguments()[0])(request, core);
}

static GenericInvoker<Func<ChartRequestModel, IDynamicQueryCore, ResultTable>> miExecuteChart =
new((req, dq) => ExecuteChart<int>(req, (DynamicQueryCore<int>)dq));
static ResultTable ExecuteChart<T>(ChartRequestModel request, DynamicQueryCore<T> dq)
public static QueryRequest ToQueryRequest(this ChartRequestModel request)
{
using (ExecutionMode.UserInterface())
return new QueryRequest
{
return dq.ExecuteQuery(new QueryRequest
{
GroupResults = request.HasAggregates(),
QueryName = request.QueryName,
Columns = request.GetQueryColumns(),
Filters = request.Filters,
Orders = request.GetQueryOrders(),
Pagination = new Pagination.All(),
});
}
QueryName = request.QueryName,
GroupResults = request.HasAggregates(),
Columns = request.GetQueryColumns(),
Filters = request.Filters,
Orders = request.GetQueryOrders(),
Pagination = request.MaxRows.HasValue ? new Pagination.Firsts(request.MaxRows.Value + 1) : new Pagination.All(),
};
}

}
2 changes: 1 addition & 1 deletion Signum.Engine.Extensions/Chart/UserChartLogic.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ public static UserChartEntity RetrieveUserChart(this Lite<UserChartEntity> userC
}
}

internal static ChartRequestModel ToChartRequest(UserChartEntity userChart)
internal static ChartRequestModel ToChartRequest(this UserChartEntity userChart)
{
var cr = new ChartRequestModel(userChart.Query.ToQueryName())
{
Expand Down
376 changes: 367 additions & 9 deletions Signum.Engine.Extensions/Dashboard/DashboardLogic.cs
View file

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions Signum.Engine.Extensions/MachineLearning/PredictorLogic.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ public static void RegisterPublication(PredictorPublicationSymbol publication, P
return Trainings.TryGetC(lite)?.Context;
}

public static void Start(SchemaBuilder sb, Func<IFileTypeAlgorithm> predictorFileAlgorithm)
public static void Start(SchemaBuilder sb, IFileTypeAlgorithm predictorFileAlgorithm)
{
if (sb.NotDefined(MethodInfo.GetCurrentMethod()))
{
Expand Down Expand Up @@ -143,7 +143,7 @@ public static void Start(SchemaBuilder sb, Func<IFileTypeAlgorithm> predictorFil
e.AccuracyValidation,
});

FileTypeLogic.Register(PredictorFileType.PredictorFile, predictorFileAlgorithm());
FileTypeLogic.Register(PredictorFileType.PredictorFile, predictorFileAlgorithm);

SymbolLogic<PredictorAlgorithmSymbol>.Start(sb, () => Algorithms.Keys);
SymbolLogic<PredictorColumnEncodingSymbol>.Start(sb, () => Algorithms.Values.SelectMany(a => a.GetRegisteredEncodingSymbols()).Distinct());
Expand Down
43 changes: 33 additions & 10 deletions Signum.Engine.Extensions/UserQueries/UserQueryLogic.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,21 +72,44 @@ public static QueryRequest ToQueryRequest(this UserQueryEntity userQuery)
return qr;
}

public static QueryRequest ToQueryRequestValue(this UserQueryEntity userQuery, QueryToken? valueToken = null)
{
var qn = userQuery.Query.ToQueryName();

if (valueToken == null)
{
var qd = QueryLogic.Queries.QueryDescription(qn);
valueToken = QueryUtils.Parse("Count", qd, SubTokensOptions.CanAggregate);
}

var qr = new QueryRequest()
{
QueryName = qn,
GroupResults = userQuery.GroupResults || valueToken is AggregateToken,
};

qr.Filters = userQuery.Filters.ToFilterList();
qr.Columns = new List<Column> { new Column(valueToken, null) };
qr.Orders = valueToken is AggregateToken ? new List<Order>() : userQuery.Orders.Select(qo => new Order(qo.Token.Token, qo.OrderType)).ToList();

qr.Pagination = userQuery.GetPagination() ?? new Pagination.All();

return qr;
}

static List<Column> MergeColumns(UserQueryEntity uq)
{
QueryDescription qd = QueryLogic.Queries.QueryDescription(uq.Query.ToQueryName());

switch (uq.ColumnsMode)
var result = uq.ColumnsMode switch
{
case ColumnOptionsMode.Add:
return qd.Columns.Where(cd => !cd.IsEntity).Select(cd => new Column(cd, qd.QueryName)).Concat(uq.Columns.Select(co => ToColumn(co))).ToList();
case ColumnOptionsMode.Remove:
return qd.Columns.Where(cd => !cd.IsEntity && !uq.Columns.Any(co => co.Token.TokenString == cd.Name)).Select(cd => new Column(cd, qd.QueryName)).ToList();
case ColumnOptionsMode.Replace:
return uq.Columns.Select(co => ToColumn(co)).ToList();
default:
throw new InvalidOperationException("{0} is not a valid ColumnOptionMode".FormatWith(uq.ColumnsMode));
}
ColumnOptionsMode.Add => qd.Columns.Where(cd => !cd.IsEntity).Select(cd => new Column(cd, qd.QueryName)).Concat(uq.Columns.Select(co => ToColumn(co))).ToList(),
ColumnOptionsMode.Remove => qd.Columns.Where(cd => !cd.IsEntity && !uq.Columns.Any(co => co.Token.TokenString == cd.Name)).Select(cd => new Column(cd, qd.QueryName)).ToList(),
ColumnOptionsMode.Replace => uq.Columns.Select(co => ToColumn(co)).ToList(),
_ => throw new InvalidOperationException("{0} is not a valid ColumnOptionMode".FormatWith(uq.ColumnsMode))
};

return result;
}

private static Column ToColumn(QueryColumnEmbedded co)
Expand Down
15 changes: 0 additions & 15 deletions Signum.Engine/DynamicQuery/AutoDynamicQuery.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -213,21 +213,6 @@ public override IQueryable<Entity> GetEntitiesFull(QueryEntitiesRequest request)
return result.TryTake(request.Count);
}

public override DQueryable<object> GetDQueryable(DQueryableRequest request)
{
request.Columns.Insert(0, new _EntityColumn(EntityColumnFactory().BuildColumnDescription(), QueryName));

DQueryable<T> query = Query
.ToDQueryable(GetQueryDescription())
.SelectMany(request.Multiplications)
.OrderBy(request.Orders)
.Where(request.Filters)
.Select(request.Columns)
.TryTake(request.Count);

return new DQueryable<object>(query.Query, query.Context);
}

public override Expression? Expression
{
get { return Query.Expression; }
Expand Down
33 changes: 18 additions & 15 deletions Signum.Engine/DynamicQuery/DynamicQuery.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,6 @@ public interface IDynamicQueryCore

IQueryable<Lite<Entity>> GetEntitiesLite(QueryEntitiesRequest request);
IQueryable<Entity> GetEntitiesFull(QueryEntitiesRequest request);
DQueryable<object> GetDQueryable(DQueryableRequest request);
}


Expand Down Expand Up @@ -130,7 +129,6 @@ public abstract class DynamicQueryCore<T> : IDynamicQueryCore

public abstract IQueryable<Lite<Entity>> GetEntitiesLite(QueryEntitiesRequest request);
public abstract IQueryable<Entity> GetEntitiesFull(QueryEntitiesRequest request);
public abstract DQueryable<object> GetDQueryable(DQueryableRequest request);


protected virtual ColumnDescriptionFactory[] InitializeColumns()
Expand Down Expand Up @@ -1122,22 +1120,27 @@ public static ResultTable ToResultTable<T>(this DEnumerableCount<T> collection,
{
object[] array = collection.Collection as object[] ?? collection.Collection.ToArray();

var columnAccesors = req.Columns.Select(c => (
column: c,
lambda: Expression.Lambda(c.Token.BuildExpression(collection.Context), collection.Context.Parameter)
)).ToList();
var isMultiKeyGrupping = req.GroupResults && req.Columns.Count(col => col.Token is not AggregateToken) >= 2;

return ToResultTable(array, columnAccesors, collection.TotalElements, req.Pagination);
}
var rows = collection.Collection.ToArray();

public static ResultTable ToResultTable(object[] result, List<(Column column, LambdaExpression lambda)> columnAccesors, int? totalElements, Pagination pagination)
{
var columnValues = columnAccesors.Select(c => new ResultColumn(
c.column,
miGetValues.GetInvoker(c.column.Type)(result, c.lambda.Compile()))
).ToArray();
var columnAccesors = req.Columns.Select(c =>
{
var expression = Expression.Lambda(c.Token.BuildExpression(collection.Context), collection.Context.Parameter);

var lambda = expression.Compile();

var values = miGetValues.GetInvoker(c.Token.Type)(rows, lambda);

var rc = new ResultColumn(c, values);

if (c.Token.Type.IsLite() || isMultiKeyGrupping && c.Token is not AggregateToken)
rc.CompressUniqueValues = true;

return rc;
}).ToArray();

return new ResultTable(columnValues, totalElements, pagination);
return new ResultTable(columnAccesors, collection.TotalElements, req.Pagination);
}

static readonly GenericInvoker<Func<object[], Delegate, Array>> miGetValues = new((objs, del) => GetValues<int>(objs, (Func<object, int>)del));
Expand Down
5 changes: 0 additions & 5 deletions Signum.Engine/DynamicQuery/DynamicQueryContainer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,11 +162,6 @@ public IQueryable<Entity> GetEntitiesFull(QueryEntitiesRequest request)
return Execute(ExecuteType.GetEntities, request.QueryName, null, dqb => dqb.Core.Value.GetEntitiesFull(request));
}

public DQueryable<object> GetDQueryable(DQueryableRequest request)
{
return Execute(ExecuteType.GetDQueryable, request.QueryName, null, dqb => dqb.Core.Value.GetDQueryable(request));
}

public event Func<object, bool, bool>? AllowQuery;

public bool QueryAllowed(object queryName, bool fullScreen)
Expand Down
7 changes: 1 addition & 6 deletions Signum.Engine/DynamicQuery/ManualDynamicQuery.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ public override async Task<ResultTable> ExecuteQueryGroupAsync(QueryRequest requ
{
req.Columns.Add(new Column(request.ValueToken, request.ValueToken.NiceName()));
var result = await Execute(req, GetQueryDescription(), cancellationToken);
return result.SelectOne(request.ValueToken).Unique(UniqueType.Single);
return result.SelectOne(request.ValueToken).Unique(UniqueType.SingleOrDefault);
}
}

Expand Down Expand Up @@ -129,11 +129,6 @@ public override IQueryable<Lite<Entity>> GetEntitiesLite(QueryEntitiesRequest re
throw new NotImplementedException();
}

public override DQueryable<object> GetDQueryable(DQueryableRequest request)
{
throw new NotImplementedException();
}

public override IQueryable<Entity> GetEntitiesFull(QueryEntitiesRequest request)
{
throw new NotImplementedException();
Expand Down
1 change: 1 addition & 0 deletions Signum.Engine/Engine/SchemaSynchronizer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -625,6 +625,7 @@ public static string GetDefaultValue(ITable table, IColumn column, Replacements
column.DbType.IsString() ? "''" :
column.DbType.IsDate() ? "GetDate()" :
column.DbType.IsGuid() ? "NEWID()" :
column.DbType.IsTime() ? "'00:00'" :
"?");

string defaultValue = rep.Interactive ? SafeConsole.AskString($"Default value for '{table.Name.Name}.{column.Name}'? ([Enter] for {typeDefault} or 'force' if there are no {(forNewColumn ? "rows" : "nulls")}) ", stringValidator: str => null) : "";
Expand Down
Loading

3 comments on commit 641293c

@olmobrutall
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Presenting CachedQueries

About a month ago we presented the possibility of making the different widgets in a Dashboard interact with each other using InteractionGroups.

But if static dashboards could already be quite demanding for the database, interactive dashboards are even worst... clicking on the different chart elements is addictive!

The solution to this problem is to cache the result of all the queries in the dashboard, save them in files, and let the dashboard use this cached data.... easy.

How to use it

1. Enable caching in your dashboard.

In your dashboard there is a new option Cache query configuration
image
when you add this option, there are a few new values to configure:

  • Timeout for queries: Maximum time that we let each queries that will be stored in a file . Default 300s (5 mins)
  • Max rows: Maximum number of rows that we let each query return, otherwise an exception will be thrown. Default 1 million.

2. Select which queries to cache.

Once you have activated "Cache query configuration", most of the widgets will have one (or many) new check boxes to enable caching data.
image

Choose queries with globally visible information: IMPORTANT: Cached queries do not respect TypeConditions!!.

Also choose queries that won't result in huge number of rows (prioritize group by with aggregates).

Pagination works, but gets disabled when InteractionGroups are involved (See How it works below).

3. Configure when to regenerate cached queries.

There are three options

  • Use Auto regenerate older than: If set will automatically regenerate the queries if the last cached queries are older than this value. This is the recommended setting for applications without too much users / data, since it won't be regenerating dashboards when no one is looking at it.
  • Create an Schedule Task to update the dashboard. DashboardEntity implementes ITaskEntity so you just need to override the implementations of SchduledTaskEntity and ScheduledTaskLogEntity in your Starter.cs. This is the recommenced way of updating heavyweight dashboards when no one is using the application (during the night).
  • You can manually generate the cached queries by clicking in the [Regenerate cached queries] button.

4. Enjoy your dashboard while your DB Rests.

image

No queries!! And check the "29 minutes ago" indicator on the top indicating the user that he is viewing potentially outdated data.

How it works

I typically don't get into too much details of how things work, but in this case I think is necessary to have an understanding of the inner workings to build cached dashboards that wont create huge queries that will take too long to execute and download.

There are two parts working together, the server side and the client side:

The server side

The server side will be responsible to save JSON files to the File System / Azure blob storage / etc.. each file contains an object with:

export interface CachedQueryJS {
  creationDate: string;
  queryRequest: QueryRequest; //The query definition
  resultTable: ResultTable; //The actual data
}

In order to generate this files, the following steps are followed:

  1. Initial generation: When the operation Regenerate cached queries is executed on a Dashboard, each part will translate his UserQueryEntity or UserChartEntity to a QueryRequest, like a server-side FindOptions with filters, columns, orders and pagination.

Simplified example:

//UserChart 1, BubblePack 
{ 
   queryName: "Order",
   groupResults: true,
   filters: []
   columns: [
      { token: "Customer"},
      { token: "Count" }
   ],
   orders: [{token: "Count", mode: "Descening"}],
   pagination: { mode: "First", elementsPerPage: 50}
}

//UserChart 2, Line chart
{ 
   queryName: "Order",
   groupResults: true,
   filters: []
   columns: [
      { token: "OrderDate.MonthStart"}
      { token: "TotalPrice.Sum" }
   ],
   orders: [{token: "OrderDate.MonthStart", mode: "Ascending"}],
   pagination: { mode: "First", elementsPerPage: 50 }
}
  1. Query generalization: All the queries that belong to the same InteractionGroup could affect each other, that's why the keys columns (not aggregates) of one query need to be added as column in all the other queries in the same group, so the results can be filtered later. This is the essence of an OLAP cube!

Assuming that both UserCharts belong to the same UserGroup, they will generalize like this:

//UserChart 1, BubblePack 
{ 
   queryName: "Order",
   groupResults: true,
   filters: []
   columns: [
      { token: "Customer"},
       { token: "OrderDate.MonthStart"}, //<-- New Key Column
      { token: "Count" }
   ],
   orders: [{token: "Count", mode: "Descening"}],
   pagination: { mode: "All" } //<-- Pagination needs to be removed
}

//UserChart 2, Line chart
{ 
   queryName: "Order",
   groupResults: true,
   filters: []
   columns: [
      { token: "OrderDate.MonthStart"},
      { token: "Customer"}, //<-- New Key Column
      { token: "TotalPrice.Sum" }
   ],
   orders: [{token: "OrderDate.MonthStart", mode: "Ascending"}],
   pagination: { mode: "All" } //<-- Pagination needs to be removed
}
  1. Query combination: After this generalization, many queries of the same group end up being very similar, if not identical, this step tries to combine similar queries so the end user doesn't need to download duplicated data. Currently two queries will be combined only if they have the same filters and key columns (for groping queries), but aggregates can be combined.
//UserChart 1, BubblePack   
//UserChart 2, Line chart    //<-- One file, two UserCharts
{ 
   queryName: "Order",
   groupResults: true,
   filters: []
   columns: [
      { token: "OrderDate.MonthStart"},   //<-- Keys stay are identical
      { token: "Customer"}, 
      { token: "Count" },                    //<-- Aggregates can be combined
      { token: "TotalPrice.Sum" },      
   ],
   orders: [{token: "OrderDate.MonthStart", mode: "Ascending"}],
   pagination: { mode: "All" }
}

Check the result

You can check the results of all this transformations by clicking in the Cheched Queries counter inside Cached Query Configuration:

image

There you can see how many queries are made, each with the number of columns and rows, how long took the query and uploading the file and for how many user assets it will be re-used.

You can also check the content itself by opening the CachedQueryEntity.

image

If you click in Format JSON you can see that the ResultTable format has been optimized to avoid duplicating big values, like Lite<T> or dates, that are repeated in many rows.

The client side

When the Dasboard is loaded, the response contains together with the DashboardEntity , a list of CachedQueryEntity with the FilePathEmbedded to download the files. Each file will be downloaded only once even if used by many widgets.

Now UserQueryPart.tsx, ValueUserQueryListPart.tsx and UserChartPart.tsx have been extended to implement some new extension points in SearchControl, ValueSearchControl, etc..

export interface SearchControlProps {
  findOptions: FindOptions;
  //...
  customRequest?: (req: QueryRequest, fop: FindOptionsParsed) => Promise<ResultTable>, //<-- NEW
}

This allows custom code to be used to resolve the QueryRequest in a ResultTable without calling the server API that makes a DB query.

In this case, the custom code will do the filtering, grouping, column selection, ordering and pagination, all locally in JavaScript in the client machine, liberating the web server of any work.

I've tried to make this code fast using simple loops, some eval tricks and doing strict equality (===) but maybe someone comes with better ideas. If you want to take a look how this work, check here.

Mixing cached and non-cached information

I'm very happy with the current solution's ability to mix and match cached and non-cached data in the same dashboard.

For example you can have some global widgets with "Sales by month" or "Top producs" of the company visible to all the users cached, but have a "My Orders" search control that is non cached because doing it will be inefficient and TypeConditions don't work in cached queries!.

You can even have them in the same InteractionGroup, so you click in one on the "Top producs" and it filters the orders that contains this products in My Orders.

And of course you can open one char's data by Alt+Click on a chart's element. This will make a DB query (very concrete filters and different columns) so some inconsistencies should be expected.

Conclusion

Dashboards are getting quite more powerful and scalable!

Let's see what else we can do in the future :)

@rezanos
Copy link
Contributor

@rezanos rezanos commented on 641293c Dec 4, 2021 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Excellent improvement! Great! 👌 👏
Thanks 🙏

@MehdyKarimpour
Copy link
Contributor

@MehdyKarimpour MehdyKarimpour commented on 641293c Dec 5, 2021 via email

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please sign in to comment.