From f51d95b9ac866e33977c3c15925cb0a8898aea2d Mon Sep 17 00:00:00 2001 From: Mario de Frutos Dieguez Date: Thu, 31 Jan 2019 17:58:03 +0100 Subject: [PATCH 1/3] Patch for load times with pg_type loading in AR - See https://github.com/CartoDB/cartodb/issues/14615 - See https://github.com/rails/rails/issues/19578 --- .../zz_patch_activerecord_type_loading.rb | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 config/initializers/zz_patch_activerecord_type_loading.rb diff --git a/config/initializers/zz_patch_activerecord_type_loading.rb b/config/initializers/zz_patch_activerecord_type_loading.rb new file mode 100644 index 000000000000..2c685ac2ec3a --- /dev/null +++ b/config/initializers/zz_patch_activerecord_type_loading.rb @@ -0,0 +1,38 @@ +# Activerecord every time it mades a connection brings all the `pg_type` +# data into a cache in order to do the relation between pg types and rails +# types +# +# What is the problem? +# +# Well in our users database we have 300K type. AR picks ~142K which leads to +# spent ~1.3s picking the data and ~0.5s in processing those records so we end +# spending about ~2s or more without needed in all our in_database operations +# +# How this fix works? +# +# Well, we filter all the table types (table, analysis tables, overviews) because +# in our app they aren't going to be used and in case they're need, Rails is going +# to query the database for it, so no problem. +module ActiveRecord + module ConnectionAdapters + module PostgreSQL + module OID # :nodoc: + class TypeMapInitializer # :nodoc: + def query_conditions_for_initial_load(type_map) + known_type_names = type_map.keys.map { |n| "'#{n}'" } + known_type_types = %w('r' 'e' 'd') + <<-SQL % [known_type_names.join(", "), known_type_types.join(", ")] + LEFT JOIN pg_type as tt ON (tt.typtype = 'c' AND tt.typarray = t.oid AND tt.typinput = 'record_in(cstring,oid,integer)'::regprocedure) + WHERE + tt.oid is null + AND (t.typname IN (%s) + OR t.typtype IN (%s) + OR t.typinput = 'array_in(cstring,oid,integer)'::regprocedure + OR t.typelem != 0) + SQL + end + end + end + end + end +end From 42f11d0c15797b54a7e1ee188cb99708d7990176 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Dieguez Date: Fri, 1 Feb 2019 18:24:42 +0100 Subject: [PATCH 2/3] Updated NEWS.md --- NEWS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS.md b/NEWS.md index 42cc96338cf1..4e301d0a9862 100644 --- a/NEWS.md +++ b/NEWS.md @@ -9,6 +9,7 @@ Development ### Bug fixes / enhancements - Add base URL to lockout redirection in static pages ([#14617](https://github.com/CartoDB/cartodb/pull/14617)) +- Improve the in_database operations fixing some rails behaviors that were problematic for us ([#14642]https://github.com/CartoDB/cartodb/pull/14642) - Makes maps listing go faster with related tables (user db size cache issue, #14165) - Do not redirect to /login by default when error is unknown in network interceptor ([#14616](https://github.com/CartoDB/cartodb/pull/14616)) - Update CARTO.js to v4.1.10 From dc335f963016ce5abf4e808a4d292fe042716601 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Dieguez Date: Wed, 6 Feb 2019 13:18:09 +0100 Subject: [PATCH 3/3] Comment corrections --- .../zz_patch_activerecord_type_loading.rb | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/config/initializers/zz_patch_activerecord_type_loading.rb b/config/initializers/zz_patch_activerecord_type_loading.rb index 2c685ac2ec3a..5d839eef03a3 100644 --- a/config/initializers/zz_patch_activerecord_type_loading.rb +++ b/config/initializers/zz_patch_activerecord_type_loading.rb @@ -1,18 +1,20 @@ -# Activerecord every time it mades a connection brings all the `pg_type` -# data into a cache in order to do the relation between pg types and rails -# types +# Every time Activerecord makes a connection, it brings all the `pg_type` +# data into a cache in order to do mapping between pg types and rails types # # What is the problem? # # Well in our users database we have 300K type. AR picks ~142K which leads to -# spent ~1.3s picking the data and ~0.5s in processing those records so we end -# spending about ~2s or more without needed in all our in_database operations +# spending ~1.3s picking the data and ~0.5s in processing those records so we end +# spending about ~2s or more without a need in all our in_database operations # -# How this fix works? +# How does this fix work? # # Well, we filter all the table types (table, analysis tables, overviews) because -# in our app they aren't going to be used and in case they're need, Rails is going +# in our app they aren't going to be used and in case they're needed, Rails is going # to query the database for it, so no problem. +# +# This patch was developed for rails 4.2.10, in case of update you need to review it +# to verify if it needs to be modified or even removed. module ActiveRecord module ConnectionAdapters module PostgreSQL