diff --git a/.github/workflows/rubocop.yml b/.github/workflows/rubocop.yml index 1aa0eb9a1ebe..88bf2116fed1 100644 --- a/.github/workflows/rubocop.yml +++ b/.github/workflows/rubocop.yml @@ -10,11 +10,12 @@ jobs: uses: actions/checkout@v2 - uses: ruby/setup-ruby@v1 with: - ruby-version: 2.5 + ruby-version: 2.5.8 + - run: 'gem install activesupport -v 6.1.4.4' - name: Rubocop linter uses: reviewdog/action-rubocop@v1 with: - rubocop_version: 1.12.1 + rubocop_version: 1.23.0 github_token: ${{ secrets.github_token }} reporter: github-pr-check level: error diff --git a/.gitmodules b/.gitmodules index 65729fda11c9..f18825c0e781 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,12 +1,12 @@ [submodule "lib/assets/javascripts/cdb"] path = lib/assets/javascripts/cdb - url = git://github.com/CartoDB/carto.js.git + url = git@github.com:CartoDB/carto.js.git [submodule "app/assets/stylesheets/old_common"] path = app/assets/stylesheets/old_common - url = git://github.com/CartoDB/cartodb.css.git + url = git@github.com:CartoDB/cartodb.css.git [submodule "lib/sql"] path = lib/sql - url = git://github.com/CartoDB/cartodb-postgresql.git + url = git@github.com:CartoDB/cartodb-postgresql.git [submodule "private"] path = private url = git@github.com:CartoDB/cartodb-private.git diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 000000000000..3a4ff0a9352a --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,4 @@ +version: 2 + +submodules: + exclude: all diff --git a/.rubocop.yml b/.rubocop.yml index f400f7f90414..5c202476ec93 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -6,7 +6,7 @@ require: - rubocop-rspec AllCops: - TargetRubyVersion: 2.4 + TargetRubyVersion: 2.5 Exclude: - "vendor/**/*" - "db/schema.rb" diff --git a/NEWS.md b/NEWS.md index 0c097761286e..d3410de707cd 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,9 +2,10 @@ Development ----------- ### NOTICES -- None yet +- None yet ### Features +* Display GeoParquet link in Catalog [16453](https://github.com/CartoDB/cartodb/pull/16453) * Display notifications about the new CARTO platform release [16352](https://github.com/CartoDB/cartodb/pull/16352) * Upgrade to deck.gl 8.5.6 [16338](https://github.com/CartoDB/cartodb/pull/16338) * Update DO Catalog dependencies and some changes to use bundle on CARTO Workspace [#16325](https://github.com/CartoDB/cartodb/pull/16325) @@ -17,24 +18,41 @@ Development * Upgrade to CARTO Viewer v1.0.8 [16347](https://github.com/CartoDB/cartodb/pull/16347) * Show user's database location in profile [16349](https://github.com/CartoDB/cartodb/pull/16349) * Setting to enable/disable import notifications [16354](https://github.com/CartoDB/cartodb/pull/16354) +* Setting to enable/disable random username generation on SAML authentication process [16372](https://github.com/CartoDB/cartodb/pull/16372) +* Add type guessing capabilities to the ArcGIS connector [#16385](https://github.com/CartoDB/cartodb/pull/16385) +* Add notification about data migrations to CARTO 3 [#16405](https://github.com/CartoDB/cartodb/pull/16405) +* Update banner to notify about data migrations to CARTO 3 [#16420](https://github.com/CartoDB/cartodb/pull/16420) ### Bug fixes / enhancements +- Removing the full path from urls with filter parameters in the Spatial Data Catalog [#16426](https://github.com/CartoDB/cartodb/pull/16426) +- Fix rubocop integration [#16382](https://github.com/CartoDB/cartodb/pull/16382) - Add marginTop to Page when notification is displayed [#16355](https://github.com/CartoDB/cartodb/pull/16355) - Add "element" param to DO-Catalog entry function [#16343](https://github.com/CartoDB/cartodb/pull/16343) - Add new DO Catalog route for internal usage [#16342](https://github.com/CartoDB/cartodb/pull/16342) +- Propagate 'invitation_token' when there is an error signing-up with Google [#16391](https://github.com/CartoDB/cartodb/pull/16391) +- Reverse analysis selection order on new widget form [#16412](https://github.com/CartoDB/cartodb/pull/16412) +- Improve info for :update_user command [#16363](https://github.com/CartoDB/cartodb/pull/16363) - Disable email validation in DO Premium Subscriptions [#16309](https://github.com/CartoDB/cartodb/pull/16309) +- Invalidate sessions on 'session_salt' issue [#16376](https://github.com/CartoDB/cartodb/pull/16376) - Hide sharing tab from viewer in on-premises [#16299](https://github.com/CartoDB/cartodb/pull/16299) +- Update browser version checker to allow Firefox/100.0 [#16415](https://github.com/CartoDB/cartodb/pull/16415) +- Update analysis schemas after giving required permissions on user promotion [#16390](https://github.com/CartoDB/cartodb/pull/16390) +- Add timeout for SQL API exports [#16377](https://github.com/CartoDB/cartodb/pull/16377) +- Avoid deleting a user if it has shared entities [#16424](https://github.com/CartoDB/cartodb/pull/16424) - Remove all references to Spatial Data Catalog and Kepler GL maps in on-premises [#16293](https://github.com/CartoDB/cartodb/pull/16293) +- Increase hard-limit of MAX_TABLES_PER_IMPORT [#16374](https://github.com/CartoDB/cartodb/pull/16374) - Guard code for vizjson users [#16267](https://github.com/CartoDB/cartodb/pull/16267) - Guard code for Users and Visualizations [#16265](https://github.com/CartoDB/cartodb/pull/16265) - Use the organization user's data while editing a user from organization settings [#16280](https://github.com/CartoDB/cartodb/pull/16280) - Fix schema name in layers created by free users [#16307](https://github.com/CartoDB/cartodb/pull/16307) - Limit start parameter of Dropbox connector [#16264](https://github.com/CartoDB/cartodb/pull/16264) - Fix messages about layer limit being reached [#16360](https://github.com/CartoDB/cartodb/pull/16360) +- Fix 404.html page [#16369](https://github.com/CartoDB/cartodb/pull/16369) - Fix deck.gl dependency conflicts [#16339](https://github.com/CartoDB/cartodb/pull/16339) - Migrate Redis DO subscription information in inter-cloud migrations [#16315](https://github.com/CartoDB/cartodb/pull/16315) - OauthApps restricted by default [#16304](https://github.com/CartoDB/cartodb/pull/16304) - Support staging hostname in the catalog [#16258](https://github.com/CartoDB/cartodb/pull/16258) +- Add custom redirection on `developers.carto.com/login` [#16383](https://github.com/CartoDB/cartodb/pull/16383) - Fix user migration export/import logs [#16298](https://github.com/CartoDB/cartodb/pull/16298) - Fix race condition when DO subscriptions are created [#16311](https://github.com/CartoDB/cartodb/pull/16311) - Allow the usage of WMTS URLs with parameters to create custom basemaps [#16271](https://github.com/CartoDB/cartodb/pull/16271) @@ -46,6 +64,7 @@ Development - Fix verification process for active users [#16337](https://github.com/CartoDB/cartodb/pull/16337) - Avoid updating analysis nodes more than once when moving layers in Builder [#16279](https://github.com/CartoDB/cartodb/pull/16279) - Fix subscription/sample filter for datasets [#16254](https://github.com/CartoDB/cartodb/pull/16254) +- Fix form to search dataset when generating a new API key [#16378](https://github.com/CartoDB/cartodb/pull/16378) - Use fully qualified table name while creating a new map from a shared dataset [#16241](https://github.com/CartoDB/cartodb/pull/16241) - Render tileset viewer features in front of basemap [#16333](https://github.com/CartoDB/cartodb/pull/16333) - Rake task to migrate legacy synchronizations [#16353](https://github.com/CartoDB/cartodb/pull/16353) @@ -70,6 +89,19 @@ Development - Fix Auth URL generation while establishing a connection with Google Drive [#16357](https://github.com/CartoDB/cartodb/pull/16357) - Fix adding license metadata to a dataset [#16356](https://github.com/CartoDB/cartodb/pull/16356) - Fix notifications when organization seats limit is reached [#16359](https://github.com/CartoDB/cartodb/pull/16359) +- Notify Support when a user is reaching the named maps limit [#16368](https://github.com/CartoDB/cartodb/pull/16368) +- Remove old named maps when a user is reaching the named maps limit [#16368](https://github.com/CartoDB/cartodb/pull/16368) +- Fix privacy dropdown when user is editing a map [#16367](https://github.com/CartoDB/cartodb/pull/16367) +- Add a new rake to update a user username [#16370](https://github.com/CartoDB/cartodb/pull/16370) +- Add a check before destroying user tables in order to avoid deleting dependent maps [#16381](https://github.com/CartoDB/cartodb/pull/16381) +- Fix duplicated attributions in datasets [#16384](https://github.com/CartoDB/cartodb/pull/16384) +- Moving assets cdn domain from global.ssl.fastly.net to libs.cartocdn.com [#16399](https://github.com/CartoDB/cartodb/pull/16399) +- Fix error while rolling back a user migration from one cloud to another [#16421](https://github.com/CartoDB/cartodb/pull/16421) +- Add retry if a timeout is thrown when swapping the tables related with a sync process [#16430](https://github.com/CartoDB/cartodb/pull/16430) +- Add AUTODETECT_SIZE_LIMIT to ogr2ogr process when guessing CSV file column types [#16431](https://github.com/CartoDB/cartodb/pull/16431) +- Log pg locks if there is any problem during a sync table import process [#16432](https://github.com/CartoDB/cartodb/pull/16432) +- Check pg locks during sync table swap and terminate locking queries [#16433](https://github.com/CartoDB/cartodb/pull/16433) +- Add deprecation notice in docs [#16446](https://github.com/CartoDB/cartodb/pull/16446) 4.45.0 (2021-04-14) ------------------- diff --git a/README.md b/README.md index 959e85980fe4..8c633a00c3d9 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,10 @@ +# [DEPRECATED] + +Hey! This content applies only to previous CARTO products. + +Please check if it's relevant to your use case. On October 2021 we released the current version of our platform. +You can learn more and read the latest documentation at docs.carto.com + # What is CARTO? [![Code Climate](https://codeclimate.com/github/CartoDB/cartodb20.png)](https://codeclimate.com/github/CartoDB/cartodb20) diff --git a/app/commands/central_user_commands.rb b/app/commands/central_user_commands.rb index ec122cdb984b..5bc89aa79125 100644 --- a/app/commands/central_user_commands.rb +++ b/app/commands/central_user_commands.rb @@ -14,7 +14,12 @@ def initialize(notifications_topic:, logger:) def update_user(message) payload = message.payload Carto::Common::CurrentRequest.with_request_id(message.request_id) do - logger.info(message: 'Processing :update_user', class_name: self.class.name) + logger.info( + message: 'Processing :update_user', + remote_user_id: payload['remote_user_id'], + class_name: self.class.name + ) + user_id = payload.delete('remote_user_id') return unless user_id.present? && payload.any? diff --git a/app/controllers/admin/organizations_controller.rb b/app/controllers/admin/organizations_controller.rb index 12831be4c892..6df791e1cb69 100644 --- a/app/controllers/admin/organizations_controller.rb +++ b/app/controllers/admin/organizations_controller.rb @@ -159,6 +159,7 @@ def auth_update @organization.auth_github_enabled = attributes[:auth_github_enabled] @organization.strong_passwords_enabled = attributes[:strong_passwords_enabled] @organization.password_expiration_in_d = attributes[:password_expiration_in_d] + @organization.random_saml_username = attributes[:random_saml_username] @organization.update_in_central @organization.save(raise_on_failure: true) diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index bd4a3bf58798..9937a539ecef 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -81,6 +81,7 @@ def current_viewer @current_viewer rescue Carto::ExpiredSessionError => e request.reset_session + current_user.try(:invalidate_all_sessions!) not_authorized(e) end @@ -415,7 +416,7 @@ def browser_is_html5_compliant? return true if user_agent.nil? banned_regex = [ - /msie [0-9]\./, /safari\/[0-4][0-2][0-2]/, /opera\/[0-8].[0-7]/, /firefox\/[0-2].[0-5]/ + /msie [0-9]\./, /safari\/[0-4][0-2][0-2]/, /opera\/[0-8].[0-7]/, /firefox\/[0-2]\.[0-5]/ ] if banned_regex.map { |re| user_agent.match(re) }.compact.first diff --git a/app/controllers/carto/api/organization_presenter.rb b/app/controllers/carto/api/organization_presenter.rb index 62bcef39ed20..56a6c263425a 100644 --- a/app/controllers/carto/api/organization_presenter.rb +++ b/app/controllers/carto/api/organization_presenter.rb @@ -35,8 +35,8 @@ def to_poro mapzen_routing_block_price: @organization.mapzen_routing_block_price, geocoder_provider: @organization.geocoder_provider, isolines_provider: @organization.isolines_provider, - routing_provider: @organization.routing_provider, - map_views_quota: @organization.map_views_quota, + routing_provider: @organization.routing_provider, + map_views_quota: @organization.map_views_quota, twitter_datasource_quota: @organization.twitter_datasource_quota, map_view_block_price: @organization.map_view_block_price, geocoding_block_price: @organization.geocoding_block_price, @@ -49,7 +49,8 @@ def to_poro admin_email: @organization.admin_email, avatar_url: @organization.avatar_url, user_count: @organization.users.count, - password_expiration_in_d: @organization.password_expiration_in_d + password_expiration_in_d: @organization.password_expiration_in_d, + random_saml_username: @organization.random_saml_username } end diff --git a/app/controllers/carto/api/organization_users_controller.rb b/app/controllers/carto/api/organization_users_controller.rb index 16819887445a..10c59b460d79 100644 --- a/app/controllers/carto/api/organization_users_controller.rb +++ b/app/controllers/carto/api/organization_users_controller.rb @@ -121,8 +121,8 @@ def destroy force_destroy = params[:force].present? if !force_destroy && @user.has_shared_entities? - error_message = "Can't delete @user. 'Has shared entities" - render_jsonp(error_message, 410 ) and return + error_message = "Can't delete user. Has shared entities" + render_jsonp(error_message, 401) and return end @user.set_force_destroy if force_destroy diff --git a/app/controllers/carto/api/users_controller.rb b/app/controllers/carto/api/users_controller.rb index bb2d7225ae21..9ad389ce7104 100644 --- a/app/controllers/carto/api/users_controller.rb +++ b/app/controllers/carto/api/users_controller.rb @@ -111,7 +111,13 @@ def delete_me deletion_password_confirmation = params[:deletion_password_confirmation] if user.needs_password_confirmation? && !user.validate_old_password(deletion_password_confirmation) - render_jsonp({ message: "Error deleting user: #{PASSWORD_DOES_NOT_MATCH_MESSAGE}" }, 400) and return + render_jsonp({ message: "Error deleting user: #{PASSWORD_DOES_NOT_MATCH_MESSAGE}" }, 400) + return + end + + if user.has_shared_entities? + render_jsonp({ message: "User can't be deleted because there are shared entities. Please, unshare or delete them and try again." }, 401) + return end user.destroy_account diff --git a/app/controllers/sessions_controller.rb b/app/controllers/sessions_controller.rb index e1b3ae250769..568cda7bf11a 100644 --- a/app/controllers/sessions_controller.rb +++ b/app/controllers/sessions_controller.rb @@ -39,6 +39,7 @@ class SessionsController < ApplicationController before_filter :load_organization before_filter :initialize_oauth_config before_filter :api_authorization_required, only: :show + before_action :custom_redirect, only: :new after_action :set_last_mfa_activity, only: [:multifactor_authentication, :multifactor_authentication_verify_code] PLEASE_LOGIN = 'Please, log in to continue using CARTO.'.freeze @@ -186,10 +187,17 @@ def saml_user_not_in_carto return end + organization_id = warden.env['warden.options'][:organization_id] + organization = Carto::Organization.find(organization_id) saml_email = warden.env['warden.options'][:saml_email] - username = CartoDB::UserAccountCreator.email_to_username(saml_email) + + if organization.random_saml_username + username = CartoDB::UserAccountCreator.random_saml_username + else + username = CartoDB::UserAccountCreator.email_to_username(saml_email) + end + unique_username = Carto::UsernameProposer.find_unique(username) - organization_id = warden.env['warden.options'][:organization_id] create_user( username: unique_username, @@ -309,6 +317,12 @@ def saml_user? private + # HACK: CARTO internals custom redirection + # https://app.shortcut.com/cartoteam/story/198852/update-developers-carto-com-login-redirection-to-carto-com-signin + def custom_redirect + redirect_to('https://carto.com/signin/') if CartoDB.extract_subdomain(request) == 'developers' + end + def mfa_request? params[:code].presence || params[:skip].presence end diff --git a/app/mailers/reporter_mailer.rb b/app/mailers/reporter_mailer.rb index dc970537047c..33e63fdaf8fe 100644 --- a/app/mailers/reporter_mailer.rb +++ b/app/mailers/reporter_mailer.rb @@ -8,4 +8,11 @@ def trending_maps_report(mail_to, trending_visualizations) mail to: mail_to, subject: @subject end + + def named_maps_near_the_limit(message) + mail_to = Cartodb.get_config(:mailer, 'support_email') + mail_from = Cartodb.get_config(:mailer, 'internal_notifications_email') + + mail from: mail_from, to: mail_to, subject: message if mail_to && mail_from + end end diff --git a/app/models/carto/helpers/user_commons.rb b/app/models/carto/helpers/user_commons.rb index 244bb6daf25d..b51fdfc62336 100644 --- a/app/models/carto/helpers/user_commons.rb +++ b/app/models/carto/helpers/user_commons.rb @@ -42,7 +42,7 @@ module Carto::UserCommons salesforce_datasource_enabled builder_enabled geocoder_provider isolines_provider routing_provider engine_enabled mapzen_routing_quota mapzen_routing_block_price soft_mapzen_routing_limit no_map_logo org_admin - user_render_timeout database_render_timeout frontend_version asset_host + user_render_timeout database_render_timeout export_timeout frontend_version asset_host state rate_limit_id public_map_quota regular_api_key_quota maintenance_mode private_map_quota public_dataset_quota] diff --git a/app/models/carto/user_migration_import.rb b/app/models/carto/user_migration_import.rb index 3748900a1365..1a78d81f0284 100644 --- a/app/models/carto/user_migration_import.rb +++ b/app/models/carto/user_migration_import.rb @@ -109,6 +109,7 @@ def do_import_metadata(package, service) raise e rescue StandardError => e log.append('=== Error importing metadata. Rollback! ===') + log.append(e.backtrace.join("\n")) service.rollback_import_from_directory(package.meta_dir) raise e end diff --git a/app/models/carto/user_table.rb b/app/models/carto/user_table.rb index 9e385d2f2fb8..4014ebc60821 100644 --- a/app/models/carto/user_table.rb +++ b/app/models/carto/user_table.rb @@ -13,6 +13,9 @@ def min_value module Carto class UserTable < ActiveRecord::Base + + attr_accessor :skip_destroy_dependent_visualizations + PRIVACY_PRIVATE = 0 PRIVACY_PUBLIC = 1 PRIVACY_LINK = 2 @@ -68,7 +71,7 @@ class UserTable < ActiveRecord::Base before_destroy :ensure_not_viewer before_destroy :cache_dependent_visualizations, unless: :destroyed? before_destroy :backup_visualizations, unless: :destroyed? - after_destroy :destroy_dependent_visualizations + after_destroy :destroy_dependent_visualizations, unless: :skip_destroy_dependent_visualizations after_destroy :service_after_destroy def geometry_types diff --git a/app/models/concerns/cartodb_central_synchronizable.rb b/app/models/concerns/cartodb_central_synchronizable.rb index 9ed2ccdf78db..ec27c1d3f913 100644 --- a/app/models/concerns/cartodb_central_synchronizable.rb +++ b/app/models/concerns/cartodb_central_synchronizable.rb @@ -84,7 +84,7 @@ def allowed_attributes_from_central(action) salesforce_datasource_enabled geocoder_provider isolines_provider routing_provider engine_enabled builder_enabled mapzen_routing_quota mapzen_routing_block_price no_map_logo auth_github_enabled - password_expiration_in_d inherit_owner_ffs) + password_expiration_in_d inherit_owner_ffs random_saml_username) when :update %i(seats viewer_seats quota_in_bytes display_name description website discus_shortname twitter_username geocoding_quota map_views_quota @@ -96,7 +96,7 @@ def allowed_attributes_from_central(action) salesforce_datasource_enabled geocoder_provider isolines_provider routing_provider engine_enabled builder_enabled mapzen_routing_quota mapzen_routing_block_price no_map_logo auth_github_enabled - password_expiration_in_d inherit_owner_ffs) + password_expiration_in_d inherit_owner_ffs random_saml_username) end elsif user? %i(account_type admin org_admin crypted_password database_host @@ -115,7 +115,7 @@ def allowed_attributes_from_central(action) salesforce_datasource_enabled viewer geocoder_provider isolines_provider routing_provider engine_enabled builder_enabled mapzen_routing_quota mapzen_routing_block_price soft_mapzen_routing_limit no_map_logo - user_render_timeout database_render_timeout state industry company phone job_role + user_render_timeout database_render_timeout export_timeout state industry company phone job_role password_reset_token password_reset_sent_at maintenance_mode company_employees use_case private_map_quota session_salt public_dataset_quota dashboard_viewed_at email_verification_token email_verification_sent_at) end @@ -129,7 +129,7 @@ def allowed_attributes_to_central(action) when :update allowed_attributes = %i(seats viewer_seats display_name description website discus_shortname twitter_username auth_username_password_enabled auth_google_enabled password_expiration_in_d - inherit_owner_ffs) + inherit_owner_ffs random_saml_username) attributes.symbolize_keys.slice(*allowed_attributes).merge(name: name) end elsif user? diff --git a/app/models/synchronization/adapter.rb b/app/models/synchronization/adapter.rb index c8d94ec4afdc..d9b67fd8bc2f 100644 --- a/app/models/synchronization/adapter.rb +++ b/app/models/synchronization/adapter.rb @@ -11,7 +11,7 @@ class Adapter THE_GEOM = 'the_geom'.freeze OVERWRITE_ERROR = 2013 - def initialize(table_name, runner, database, user, overviews_creator, synchronization_id) + def initialize(table_name, runner, database, user, overviews_creator, synchronization_id, logger = nil) @table_name = table_name @runner = runner @database = database @@ -25,6 +25,7 @@ def initialize(table_name, runner, database, user, overviews_creator, synchroniz ) @error_code = nil @synchronization_id = synchronization_id + @logger = logger end def run(&tracker) @@ -140,11 +141,7 @@ def overwrite_replace(schema, table_name, result) table_statements = @table_setup.generate_table_statements(schema, table_name) temporary_name = temporary_name_for(result.table_name) - database.transaction do - rename(table_name, temporary_name) if exists?(table_name) - drop(temporary_name) if exists?(temporary_name) - rename(result.table_name, table_name) - end + swap_tables(table_name, temporary_name, result) @table_setup.fix_oid(table_name) @table_setup.update_cdb_tablemetadata(table_name) @table_setup.run_table_statements(table_statements, @database) @@ -389,6 +386,43 @@ def temporary_name_for(table_name) private + def swap_tables(table_name, temporary_name, result) + database.transaction do + rename(table_name, temporary_name) if exists?(table_name) + drop(temporary_name) if exists?(temporary_name) + rename(result.table_name, table_name) + end + rescue Exception => exception + if exception.message.include?('canceling statement due to statement timeout') + # Check if the table has any lock and cancel locking queries + locks = user.in_database(as: :superuser).fetch(%Q{ + SELECT pid, query + FROM pg_stat_activity + WHERE pid in ( + SELECT pid FROM pg_locks l + JOIN pg_class t ON l.relation = t.oid + AND t.relkind = 'r' + WHERE t.relname IN ('#{table_name}') + ); + }).all + @logger.append_and_store "Transaction timed out as the table is blocked by other queries. Terminating locking queries and retrying in 60 seconds..." if @logger && locks.present? + locks.each do |lock| + @logger.append_and_store "Terminating query: #{lock[:query]}" if @logger + user.in_database(as: :superuser).execute %Q{ + SELECT pg_terminate_backend(#{lock[:pid]}); + } + end + sleep(60) # wait 60 seconds and retry the swap + database.transaction do + rename(table_name, temporary_name) if exists?(table_name) + drop(temporary_name) if exists?(temporary_name) + rename(result.table_name, table_name) + end + else + raise exception + end + end + def valid_cartodb_id_candidate?(user, table_name, qualified_table_name, col_name) return false unless column_names(user, table_name).include?(col_name) user.transaction_with_timeout(statement_timeout: STATEMENT_TIMEOUT, as: :superuser) do |db| diff --git a/app/models/synchronization/member.rb b/app/models/synchronization/member.rb index a5cd0a8d1010..e0a8bc7e89ea 100644 --- a/app/models/synchronization/member.rb +++ b/app/models/synchronization/member.rb @@ -192,7 +192,7 @@ def run database = user.in_database overviews_creator = CartoDB::Importer2::Overviews.new(runner, user) - importer = CartoDB::Synchronization::Adapter.new(name, runner, database, user, overviews_creator, id) + importer = CartoDB::Synchronization::Adapter.new(name, runner, database, user, overviews_creator, id, @log) importer.run self.ran_at = Time.now diff --git a/app/models/user.rb b/app/models/user.rb index bb18ca95c536..915eb7dd2369 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -787,7 +787,8 @@ def save_metadata 'db', user_timeout, 'db_public', database_timeout, 'render', user_render_timeout, - 'render_public', database_render_timeout + 'render_public', database_render_timeout, + 'export', export_timeout save_rate_limits end diff --git a/app/models/user/db_service.rb b/app/models/user/db_service.rb index 70dcaed0b5b0..3f17f19d194c 100644 --- a/app/models/user/db_service.rb +++ b/app/models/user/db_service.rb @@ -1212,7 +1212,6 @@ def move_to_own_schema create_public_db_user set_database_search_path - update_analyses_schema end rescue StandardError => e # Undo metadata changes if process fails diff --git a/app/models/user/user_organization.rb b/app/models/user/user_organization.rb index eecd1bbab7fe..49e5d4c0b405 100644 --- a/app/models/user/user_organization.rb +++ b/app/models/user/user_organization.rb @@ -32,6 +32,7 @@ def promote_user_to_admin @owner.db_service.reset_user_schema_permissions @owner.db_service.setup_organization_user_schema + @owner.db_service.update_analyses_schema @owner.update organization_id: @organization.id @owner.db_service.monitor_user_notification @active = true diff --git a/app/presenters/organization_presenter.rb b/app/presenters/organization_presenter.rb index e4483bd72e40..55431db85034 100644 --- a/app/presenters/organization_presenter.rb +++ b/app/presenters/organization_presenter.rb @@ -52,7 +52,8 @@ def common_attributes twitter_username: twitter_username, seats: seats, avatar_url: avatar_url, - password_expiration_in_d: password_expiration_in_d + password_expiration_in_d: password_expiration_in_d, + random_saml_username: random_saml_username } end diff --git a/app/services/carto/organization_metadata_export_service.rb b/app/services/carto/organization_metadata_export_service.rb index 4d048df11b3c..1aabda3d193f 100644 --- a/app/services/carto/organization_metadata_export_service.rb +++ b/app/services/carto/organization_metadata_export_service.rb @@ -24,7 +24,7 @@ module OrganizationMetadataExportServiceConfiguration :auth_google_enabled, :location, :here_isolines_quota, :here_isolines_block_price, :strong_passwords_enabled, :salesforce_datasource_enabled, :viewer_seats, :geocoder_provider, :isolines_provider, :routing_provider, :auth_github_enabled, :engine_enabled, :mapzen_routing_quota, :mapzen_routing_block_price, :builder_enabled, - :auth_saml_configuration, :no_map_logo, :password_expiration_in_d, :inherit_owner_ffs + :auth_saml_configuration, :no_map_logo, :password_expiration_in_d, :inherit_owner_ffs, :random_saml_username ].freeze def compatible_version?(version) @@ -266,7 +266,7 @@ def rollback_import_from_directory(meta_path) Carto::RedisExportService.new.remove_redis_from_json_export(File.read(organization_redis_file)) organization = load_organization_from_directory(meta_path) - user_list = organization.non_owner_users + [organization.owner] + user_list = organization.non_owner_users + [organization.owner].compact user_list.map do |user| Carto::UserMetadataExportService.new.rollback_import_from_directory("#{meta_path}/user_#{user.id}") end diff --git a/app/services/carto/user_metadata_export_service.rb b/app/services/carto/user_metadata_export_service.rb index 6cfd62a86c95..b6b731e8c28b 100644 --- a/app/services/carto/user_metadata_export_service.rb +++ b/app/services/carto/user_metadata_export_service.rb @@ -50,7 +50,7 @@ module UserMetadataExportServiceConfiguration mobile_gis_extension mobile_max_open_users mobile_max_private_users viewer salesforce_datasource_enabled builder_enabled geocoder_provider isolines_provider routing_provider github_user_id engine_enabled mapzen_routing_quota mapzen_routing_block_price soft_mapzen_routing_limit - no_map_logo org_admin last_name user_render_timeout database_render_timeout frontend_version + no_map_logo org_admin last_name user_render_timeout database_render_timeout export_timeout frontend_version asset_host state company phone industry job_role password_reset_token password_reset_sent_at maintenance_mode company_employees use_case private_map_quota session_salt public_dataset_quota email_verification_token email_verification_sent_at diff --git a/app/views/admin/organizations/auth.html.erb b/app/views/admin/organizations/auth.html.erb index 8bb0320dff88..cbf655760642 100644 --- a/app/views/admin/organizations/auth.html.erb +++ b/app/views/admin/organizations/auth.html.erb @@ -86,6 +86,21 @@ +
+
+ +
+
+
+ <%= f.check_box :random_saml_username, :id => "random_saml_username" %> + <%= label_tag(:random_saml_username, '') %> +
+
+

Generate random usernames for new SAML users.

+
+
+
+
diff --git a/app/views/reporter_mailer/named_maps_near_the_limit.html.erb b/app/views/reporter_mailer/named_maps_near_the_limit.html.erb new file mode 100644 index 000000000000..c6ca18dc195b --- /dev/null +++ b/app/views/reporter_mailer/named_maps_near_the_limit.html.erb @@ -0,0 +1,8 @@ + + +

+ There is one username that is reaching the named maps limit. Response Team should check if it's possible to + remove a few named maps! +

+ + diff --git a/app/views/sessions/new.html.erb b/app/views/sessions/new.html.erb index ce479d696914..3c17878e6025 100644 --- a/app/views/sessions/new.html.erb +++ b/app/views/sessions/new.html.erb @@ -31,7 +31,7 @@ <% if @organization.nil? || @organization.auth_username_password_enabled %>
- <%= text_field_tag :email, CartoDB.extract_subdomain(request), :title => "Email or username", placeholder: "email or username", class: "CDB-Size-medium #{@organization.present? ? 'Sessions-input' : 'Sessions-navy-input'} topBorderRadius", autofocus: true %> + <%= text_field_tag :email, (CartoDB.extract_subdomain(request) if @organization.blank?), :title => "Email or username", placeholder: "email or username", class: "CDB-Size-medium #{@organization.present? ? 'Sessions-input' : 'Sessions-navy-input'} topBorderRadius", autofocus: true %> <% if @login_error %>
!
diff --git a/app/views/signup/signup.html.erb b/app/views/signup/signup.html.erb index 89048147108d..81c20201bf82 100644 --- a/app/views/signup/signup.html.erb +++ b/app/views/signup/signup.html.erb @@ -33,7 +33,7 @@ <%= form_for @user, url: CartoDB.url(self, 'signup_organization_user'), html: { class: "js-Loading-form" } do |f| %> - +
<% if @organization.auth_username_password_enabled || duplicated_username_prompt? %> diff --git a/db/migrate/20211110171603_add_random_username_saml_feature_flag.rb b/db/migrate/20211110171603_add_random_username_saml_feature_flag.rb new file mode 100644 index 000000000000..7eb2d4b298b7 --- /dev/null +++ b/db/migrate/20211110171603_add_random_username_saml_feature_flag.rb @@ -0,0 +1,12 @@ +require 'carto/db/migration_helper' + +include Carto::Db::MigrationHelper + +migration( + Proc.new do + add_column :organizations, :random_saml_username, :bool, default: false + end, + Proc.new do + drop_column :organizations, :random_saml_username + end +) diff --git a/db/migrate/20211122155410_add_export_timeout_to_users.rb b/db/migrate/20211122155410_add_export_timeout_to_users.rb new file mode 100644 index 000000000000..4951470c6096 --- /dev/null +++ b/db/migrate/20211122155410_add_export_timeout_to_users.rb @@ -0,0 +1,15 @@ +require 'carto/db/migration_helper' + +# rubocop:disable Style/MixinUsage +include Carto::Db::MigrationHelper +# rubocop:enable Style/MixinUsage + +migration( + proc do + # The 0 value means: "apply default export timeout" (defined by the SQL API) + add_column :users, :export_timeout, :integer, default: 0, null: false + end, + proc do + drop_column :users, :export_timeout + end +) diff --git a/doc/manual/source/_static/deprecation_popup.js b/doc/manual/source/_static/deprecation_popup.js new file mode 100644 index 000000000000..5b1cc3b471da --- /dev/null +++ b/doc/manual/source/_static/deprecation_popup.js @@ -0,0 +1,19 @@ +function ready() { + var div = document.createElement("div"); + div.style.position = 'absolute', + div.style.bottom = '40px' + div.style.left = '0' + div.style.right = '0' + div.style.with = '100%' + div.style.padding = '10px' + div.innerHTML = ` +

Hey! This content applies only to previous CARTO products

+

Please check if it's relevant to your use case. On October 2021 we released a new version of our platform.

+

You can learn more and read the latest documentation at docs.carto.com

+ `; + + document.getElementsByTagName("nav")[0].appendChild(div); +} + + +document.addEventListener("DOMContentLoaded", ready); diff --git a/doc/manual/source/conf.py b/doc/manual/source/conf.py index fea00a5f63f4..7dd9c8dfe5ab 100644 --- a/doc/manual/source/conf.py +++ b/doc/manual/source/conf.py @@ -267,3 +267,7 @@ # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = {'http://docs.python.org/': None} + +html_js_files = [ + 'deprecation_popup.js' +] diff --git a/doc/manual/source/install.rst b/doc/manual/source/install.rst index a99d478e2a33..0dff811ee9b9 100644 --- a/doc/manual/source/install.rst +++ b/doc/manual/source/install.rst @@ -202,7 +202,7 @@ SQL API .. code-block:: bash - git clone git://github.com/CartoDB/CartoDB-SQL-API.git + git clone git@github.com:CartoDB/CartoDB-SQL-API.git cd CartoDB-SQL-API * Install npm dependencies @@ -232,7 +232,7 @@ MAPS API .. code-block:: bash - git clone git://github.com/CartoDB/Windshaft-cartodb.git + git clone git@github.com:CartoDB/Windshaft-cartodb.git cd Windshaft-cartodb * Install yarn dependencies diff --git a/lib/assets/javascripts/builder/components/modals/add-widgets/layer-selector-view.js b/lib/assets/javascripts/builder/components/modals/add-widgets/layer-selector-view.js index 94750480dcf5..50a78933e73e 100755 --- a/lib/assets/javascripts/builder/components/modals/add-widgets/layer-selector-view.js +++ b/lib/assets/javascripts/builder/components/modals/add-widgets/layer-selector-view.js @@ -58,7 +58,7 @@ module.exports = CoreView.extend({ mouseOutAction: this._onMouseOut.bind(this) }); - this._selectView.setValue(this.model.get('layer_index') || 0); + this._selectView.setValue(this.model.get('layer_index') || (options[0] || {}).val || 0); this.$el.html(this._selectView.render().el); }, @@ -92,7 +92,7 @@ module.exports = CoreView.extend({ color: layerDefModel.getColor(), isSourceType: nodeDefModel.isSourceType() }; - }); + }).reverse(); }, _bindEvents: function () { diff --git a/lib/assets/javascripts/builder/components/modals/publish/create-privacy-options.js b/lib/assets/javascripts/builder/components/modals/publish/create-privacy-options.js index a3a917371e7c..e788a2e225aa 100755 --- a/lib/assets/javascripts/builder/components/modals/publish/create-privacy-options.js +++ b/lib/assets/javascripts/builder/components/modals/publish/create-privacy-options.js @@ -57,8 +57,7 @@ module.exports = function (visDefinitionModel, userModel) { var publicMustBeDisabled = (option.privacy !== 'PRIVATE' && publicSharingIsDisabled); - var enabled = (option.alwaysEnabled || premiumEnabled) && !publicMustBeDisabled; - + var enabled = ((option.alwaysEnabled || premiumEnabled) && !publicMustBeDisabled) || option.privacy === 'PRIVATE'; return _.defaults({ selected: option.privacy === currentPrivacy, disabled: !enabled diff --git a/lib/assets/javascripts/cdb b/lib/assets/javascripts/cdb index 05916e8a628d..5203f3352f0f 160000 --- a/lib/assets/javascripts/cdb +++ b/lib/assets/javascripts/cdb @@ -1 +1 @@ -Subproject commit 05916e8a628d7493ccf10cead69a7d85a589b00e +Subproject commit 5203f3352f0f0977c188aa9e9bdd03843f7b778b diff --git a/lib/assets/javascripts/dashboard/components/table-grants/table-grants-view.js b/lib/assets/javascripts/dashboard/components/table-grants/table-grants-view.js index f22a7813147b..7ca7bc736392 100644 --- a/lib/assets/javascripts/dashboard/components/table-grants/table-grants-view.js +++ b/lib/assets/javascripts/dashboard/components/table-grants/table-grants-view.js @@ -121,5 +121,5 @@ module.exports = CoreView.extend({ _onSearchChanged: _.debounce(function (event) { this._userTablesModel.setQuery(event.target.value); - }, 500, this) + }, 500) }); diff --git a/lib/assets/javascripts/do-catalog/router.js b/lib/assets/javascripts/do-catalog/router.js index 713a1631918e..b1cb6994fa5e 100644 --- a/lib/assets/javascripts/do-catalog/router.js +++ b/lib/assets/javascripts/do-catalog/router.js @@ -94,7 +94,7 @@ router.afterEach(to => { if (!to.meta.titleInComponent) { document.title = to.meta.title(to); } - addCanonical(`https://carto.com/spatial-data-catalog/browser${to.fullPath}`); + addCanonical(`https://carto.com/spatial-data-catalog/browser${to.path}`); }); }); diff --git a/lib/assets/javascripts/new-dashboard/App.vue b/lib/assets/javascripts/new-dashboard/App.vue index 2a3387453635..a4d3bb9239b0 100644 --- a/lib/assets/javascripts/new-dashboard/App.vue +++ b/lib/assets/javascripts/new-dashboard/App.vue @@ -52,14 +52,14 @@ export default { MamufasImportView }, data: () => ({ - displayCarto3ReleaseNotification: notificationIsVisible(window.localStorage.getItem('carto3ReleaseVisible')) + displayCarto3ReleaseNotification: notificationIsVisible(window.localStorage.getItem('carto3ReleaseVisibleV2')) }), created () { sendMetric(MetricsTypes.VISITED_PRIVATE_PAGE, { page: 'dashboard' }); }, methods: { closeCarto3Release () { - window.localStorage.setItem('carto3ReleaseVisible', new Date().getTime()); + window.localStorage.setItem('carto3ReleaseVisibleV2', new Date().getTime()); this.displayCarto3ReleaseNotification = false; } }, diff --git a/lib/assets/javascripts/new-dashboard/components/NavigationBar/NavigationBar.vue b/lib/assets/javascripts/new-dashboard/components/NavigationBar/NavigationBar.vue index dd39cc6986b2..165776248220 100644 --- a/lib/assets/javascripts/new-dashboard/components/NavigationBar/NavigationBar.vue +++ b/lib/assets/javascripts/new-dashboard/components/NavigationBar/NavigationBar.vue @@ -39,11 +39,19 @@ + + +
diff --git a/lib/assets/javascripts/new-dashboard/components/Popups/NotificationPopup.vue b/lib/assets/javascripts/new-dashboard/components/Popups/NotificationPopup.vue index 090bb67ef848..21109c03ffc5 100644 --- a/lib/assets/javascripts/new-dashboard/components/Popups/NotificationPopup.vue +++ b/lib/assets/javascripts/new-dashboard/components/Popups/NotificationPopup.vue @@ -7,11 +7,10 @@ {{ title }}

-

- -

- {{ message }} -

+
@@ -21,7 +20,7 @@ export default { name: 'NotificationPopup', props: { title: String, - message: String, + message: Array, messageHasHTML: { type: Boolean, default: false diff --git a/lib/assets/javascripts/new-dashboard/i18n/locales/en.json b/lib/assets/javascripts/new-dashboard/i18n/locales/en.json index 18cb12fa0a49..64d3a0055d39 100644 --- a/lib/assets/javascripts/new-dashboard/i18n/locales/en.json +++ b/lib/assets/javascripts/new-dashboard/i18n/locales/en.json @@ -1,8 +1,9 @@ { "Carto3Release": { - "title": "New release", - "message": "Announcing the launch of the cloud native Spatial Extension for BigQuery, {link}!", - "link": "find out more" + "title": "Try out the new CARTO platform!", + "message": "Create your account {link}. {contact} so we can match your current plan and help you migrate your data.", + "link": "here", + "contactUs": "Contact us" }, "MapCard": { "shared": { @@ -557,6 +558,13 @@ "maps": "0 maps | {maps} map | {maps} maps", "datasets": "0 datasets | {datasets} dataset | {datasets} datasets" }, + "NewPlatformMessage": { + "title": "New version of the CARTO Platform", + "message": [ + "To start using the latest version of our platform, please create your account here. We will then align it to your current subscription plan.", + "If you need help migrating your data to the new version of CARTO, reach out to our Support team." + ] + }, "FeedbackMessage": { "title": "Let us know what you think.", "message": "Missing something? Should we be doing something differently?" diff --git a/lib/assets/javascripts/new-dashboard/pages/Data/CatalogDatasetSummary.vue b/lib/assets/javascripts/new-dashboard/pages/Data/CatalogDatasetSummary.vue index 4b7a94d9d829..eaeb3ae36e84 100644 --- a/lib/assets/javascripts/new-dashboard/pages/Data/CatalogDatasetSummary.vue +++ b/lib/assets/javascripts/new-dashboard/pages/Data/CatalogDatasetSummary.vue @@ -101,6 +101,17 @@

+
  • +

    + GeoParquet +

    +

    + Download +

    +
    • @@ -112,6 +123,7 @@ import { temporalAggregationName } from 'new-dashboard/utils/catalog/temporal-ag import { geometryTypeName } from 'new-dashboard/utils/catalog/geometry-type-name'; import { updateFrequencyName } from 'new-dashboard/utils/catalog/update-frequency-name'; import { sendCustomDimensions } from 'new-dashboard/utils/catalog/custom-dimensions-ga'; +import { checkGeoparquetBucket } from 'new-dashboard/utils/catalog/geoparquet'; import CatalogMapPreview from 'new-dashboard/components/Catalog/CatalogMapPreview'; export default { @@ -119,6 +131,12 @@ export default { components: { CatalogMapPreview }, + data () { + return { + hasGeoparquetAvailable: false, + geoparquetUrl: undefined + }; + }, watch: { dataset: { handler (value) { @@ -169,10 +187,16 @@ export default { id: this.$route.params.entity_id, type: this.$route.params.entity_type }); + }, + async checkIfGeoparquetBucketExists () { + const { ok, url } = await checkGeoparquetBucket(this.$route.params.entity_id); + this.hasGeoparquetAvailable = ok; + this.geoparquetUrl = url; } }, mounted () { this.fetchKeyVariables(); + this.checkIfGeoparquetBucketExists(); } }; diff --git a/lib/assets/javascripts/new-dashboard/pages/Home/WelcomeSection/Welcome.vue b/lib/assets/javascripts/new-dashboard/pages/Home/WelcomeSection/Welcome.vue index 0f5226a3582e..9b25eeb1df16 100644 --- a/lib/assets/javascripts/new-dashboard/pages/Home/WelcomeSection/Welcome.vue +++ b/lib/assets/javascripts/new-dashboard/pages/Home/WelcomeSection/Welcome.vue @@ -1,7 +1,7 @@