From 563c41c68dc6f64dc9146eb530d4aea5cb1c0135 Mon Sep 17 00:00:00 2001 From: Oliver Valls <199462+tramuntanal@users.noreply.github.com> Date: Tue, 11 Jan 2022 12:56:37 +0100 Subject: [PATCH 1/5] Update Scopes instead of creating new ones if code is the same --- lib/tasks/civi_crm.rake | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/tasks/civi_crm.rake b/lib/tasks/civi_crm.rake index 815a263..0bfa4ad 100644 --- a/lib/tasks/civi_crm.rake +++ b/lib/tasks/civi_crm.rake @@ -114,11 +114,12 @@ namespace :civi_crm do next if display_name[/\d/] scope_name = display_name.strip - Decidim::Scope.find_or_create_by!( + scope= Decidim::Scope.find_or_initialize_by( organization: organization, - name: { "ca" => scope_name, "en" => scope_name, "es" => scope_name }, code: contact_id ) + scope.name= { "ca" => scope_name, "en" => scope_name, "es" => scope_name } + scope.save! end end end From 3dde3c4b2e5d7bf3c6df51456465ab1d5cbb947a Mon Sep 17 00:00:00 2001 From: Oliver Valls <199462+tramuntanal@users.noreply.github.com> Date: Tue, 11 Jan 2022 13:19:20 +0100 Subject: [PATCH 2/5] Rubocopify --- lib/tasks/civi_crm.rake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/tasks/civi_crm.rake b/lib/tasks/civi_crm.rake index 0bfa4ad..2579685 100644 --- a/lib/tasks/civi_crm.rake +++ b/lib/tasks/civi_crm.rake @@ -114,11 +114,11 @@ namespace :civi_crm do next if display_name[/\d/] scope_name = display_name.strip - scope= Decidim::Scope.find_or_initialize_by( + scope = Decidim::Scope.find_or_initialize_by( organization: organization, code: contact_id ) - scope.name= { "ca" => scope_name, "en" => scope_name, "es" => scope_name } + scope.name = { "ca" => scope_name, "en" => scope_name, "es" => scope_name } scope.save! end end From 08764276cfbbb66d8b9172cb138cd9202b6cdc0f Mon Sep 17 00:00:00 2001 From: Laura Jaime Date: Wed, 12 Jan 2022 09:59:34 +0100 Subject: [PATCH 3/5] Create ymls with csvs --- lib/tasks/civi_crm.rake | 74 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/lib/tasks/civi_crm.rake b/lib/tasks/civi_crm.rake index 0bfa4ad..21815fc 100644 --- a/lib/tasks/civi_crm.rake +++ b/lib/tasks/civi_crm.rake @@ -1,4 +1,5 @@ # frozen_string_literal: true +require 'csv' namespace :civi_crm do task init: ["import:all", "generate:all", "create:scopes"] @@ -63,6 +64,79 @@ namespace :civi_crm do end end + namespace :import_by_csv do + task all: [:comarcals, :regionals, :locals, :local_comarcal_relationships, :local_regional_relationships] + + desc "Generates a YAML file with the CSV with 'Comarcal' tag" + task comarcals: :environment do + csv_text = File.read('tmp/scopes_codes.csv') + csv = CSV.parse(csv_text, headers: true) + + csv = csv.select { |row| row['type'] == 'FC' } + csv = csv.map { |row| row.to_h } + + result = get_code_and_display_name(csv) + write_config_yaml!('comarcals', result) + end + + desc "Generates a YAML file with the CSV with 'Regional' tag" + task regionals: :environment do + csv_text = File.read('tmp/scopes_codes.csv') + csv = CSV.parse(csv_text, headers: true) + + csv = csv.select { |row| row['type'] == 'FR' } + csv = csv.map { |row| row.to_h } + + result = get_code_and_display_name(csv) + write_config_yaml!('regionals', result) + end + + desc "Generates a YAML file with the CSV with 'Locals' tag" + task locals: :environment do + csv_text = File.read('tmp/scopes_codes.csv') + csv = CSV.parse(csv_text, headers: true) + + csv = csv.select { |row| row['type'] == 'SL' } + csv = csv.map { |row| row.to_h } + + result = get_code_and_display_name(csv) + write_config_yaml!('locals', result) + end + + desc "Generates a YAML file with the relationship between CSV 'Local' and 'Comarcal'" + task local_comarcal_relationships: :environment do + local_ids = load_config_yaml("locals").keys + comarcal_ids = load_config_yaml("comarcals").keys + result = get_code_relationships(local_ids, comarcal_ids, 'comarcal') + write_config_yaml!("local_comarcal_relationships", result) + end + + desc "Generates a YAML file with the relationship between CSV 'Local' and 'Regional'" + task local_regional_relationships: :environment do + local_ids = load_config_yaml("locals").keys + regional_ids = load_config_yaml("regionals").keys + result = get_code_relationships(local_ids, regional_ids, 'regional') + write_config_yaml!("local_regional_relationships", result) + end + + def get_code_and_display_name(body) + body.map! { |element| { element['code'] => element['name'] } }.reduce(Hash.new, :merge) + end + + def get_code_relationships(local_ids, filter_ids, scope_type) + local_ids.each_with_object({}) do |element, memo| + csv_text = File.read('tmp/scopes_hierarchy.csv') + csv = CSV.parse(csv_text, headers: true) + + csv = csv.select { |row| row["SL"] == element }.first.to_h + next unless csv.present? + + memo[csv['SL']] = csv["FC"] if scope_type == 'comarcal' + memo[csv['SL']] = csv["FR"] if scope_type == 'regional' + end + end + end + namespace :generate do task all: [:comarcal_exceptions, :decidim_scopes_mapping] From 4738f920526fb74459772640fba66a9c8314ca83 Mon Sep 17 00:00:00 2001 From: Laura Jaime Date: Wed, 12 Jan 2022 10:01:14 +0100 Subject: [PATCH 4/5] Fix Rubocop offenses --- lib/tasks/civi_crm.rake | 49 +++++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/lib/tasks/civi_crm.rake b/lib/tasks/civi_crm.rake index 8c57557..8aafdcc 100644 --- a/lib/tasks/civi_crm.rake +++ b/lib/tasks/civi_crm.rake @@ -1,5 +1,6 @@ # frozen_string_literal: true -require 'csv' + +require "csv" namespace :civi_crm do task init: ["import:all", "generate:all", "create:scopes"] @@ -69,45 +70,45 @@ namespace :civi_crm do desc "Generates a YAML file with the CSV with 'Comarcal' tag" task comarcals: :environment do - csv_text = File.read('tmp/scopes_codes.csv') + csv_text = File.read("tmp/scopes_codes.csv") csv = CSV.parse(csv_text, headers: true) - - csv = csv.select { |row| row['type'] == 'FC' } - csv = csv.map { |row| row.to_h } + + csv = csv.select { |row| row["type"] == "FC" } + csv = csv.map(&:to_h) result = get_code_and_display_name(csv) - write_config_yaml!('comarcals', result) + write_config_yaml!("comarcals", result) end desc "Generates a YAML file with the CSV with 'Regional' tag" task regionals: :environment do - csv_text = File.read('tmp/scopes_codes.csv') + csv_text = File.read("tmp/scopes_codes.csv") csv = CSV.parse(csv_text, headers: true) - - csv = csv.select { |row| row['type'] == 'FR' } - csv = csv.map { |row| row.to_h } + + csv = csv.select { |row| row["type"] == "FR" } + csv = csv.map(&:to_h) result = get_code_and_display_name(csv) - write_config_yaml!('regionals', result) + write_config_yaml!("regionals", result) end desc "Generates a YAML file with the CSV with 'Locals' tag" task locals: :environment do - csv_text = File.read('tmp/scopes_codes.csv') + csv_text = File.read("tmp/scopes_codes.csv") csv = CSV.parse(csv_text, headers: true) - - csv = csv.select { |row| row['type'] == 'SL' } - csv = csv.map { |row| row.to_h } + + csv = csv.select { |row| row["type"] == "SL" } + csv = csv.map(&:to_h) result = get_code_and_display_name(csv) - write_config_yaml!('locals', result) + write_config_yaml!("locals", result) end desc "Generates a YAML file with the relationship between CSV 'Local' and 'Comarcal'" task local_comarcal_relationships: :environment do local_ids = load_config_yaml("locals").keys comarcal_ids = load_config_yaml("comarcals").keys - result = get_code_relationships(local_ids, comarcal_ids, 'comarcal') + result = get_code_relationships(local_ids, comarcal_ids, "comarcal") write_config_yaml!("local_comarcal_relationships", result) end @@ -115,24 +116,24 @@ namespace :civi_crm do task local_regional_relationships: :environment do local_ids = load_config_yaml("locals").keys regional_ids = load_config_yaml("regionals").keys - result = get_code_relationships(local_ids, regional_ids, 'regional') + result = get_code_relationships(local_ids, regional_ids, "regional") write_config_yaml!("local_regional_relationships", result) end def get_code_and_display_name(body) - body.map! { |element| { element['code'] => element['name'] } }.reduce(Hash.new, :merge) + body.map! { |element| { element["code"] => element["name"] } }.reduce({}, :merge) end - def get_code_relationships(local_ids, filter_ids, scope_type) + def get_code_relationships(local_ids, _filter_ids, scope_type) local_ids.each_with_object({}) do |element, memo| - csv_text = File.read('tmp/scopes_hierarchy.csv') + csv_text = File.read("tmp/scopes_hierarchy.csv") csv = CSV.parse(csv_text, headers: true) csv = csv.select { |row| row["SL"] == element }.first.to_h - next unless csv.present? + next if csv.blank? - memo[csv['SL']] = csv["FC"] if scope_type == 'comarcal' - memo[csv['SL']] = csv["FR"] if scope_type == 'regional' + memo[csv["SL"]] = csv["FC"] if scope_type == "comarcal" + memo[csv["SL"]] = csv["FR"] if scope_type == "regional" end end end From dbaa3bdd3f50488417e747711e582d1f9efcbb8f Mon Sep 17 00:00:00 2001 From: Laura Jaime Date: Wed, 12 Jan 2022 11:03:33 +0100 Subject: [PATCH 5/5] Reorganize rake tasks --- lib/file_manager.rb | 17 ++++++++ lib/tasks/civi_crm.rake | 91 ++--------------------------------------- lib/tasks/erc_auth.rake | 85 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 105 insertions(+), 88 deletions(-) create mode 100644 lib/file_manager.rb create mode 100644 lib/tasks/erc_auth.rake diff --git a/lib/file_manager.rb b/lib/file_manager.rb new file mode 100644 index 0000000..c532d86 --- /dev/null +++ b/lib/file_manager.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module FileManager + def write_config_yaml!(filename, content) + File.write(filepath(filename), content.to_yaml) + puts "File generated 'config/civi_crm/#{filename}.yml'" + end + + def filepath(filename) + Dir.mkdir("config/civi_crm") unless File.directory?("config/civi_crm") + Rails.root.join("config", "civi_crm", "#{filename}.yml").to_s + end + + def load_config_yaml(filename) + YAML.load_file(filepath(filename)) + end +end diff --git a/lib/tasks/civi_crm.rake b/lib/tasks/civi_crm.rake index 8aafdcc..2c23a2a 100644 --- a/lib/tasks/civi_crm.rake +++ b/lib/tasks/civi_crm.rake @@ -1,8 +1,10 @@ # frozen_string_literal: true -require "csv" +require "file_manager" namespace :civi_crm do + include FileManager + task init: ["import:all", "generate:all", "create:scopes"] namespace :import do @@ -65,79 +67,6 @@ namespace :civi_crm do end end - namespace :import_by_csv do - task all: [:comarcals, :regionals, :locals, :local_comarcal_relationships, :local_regional_relationships] - - desc "Generates a YAML file with the CSV with 'Comarcal' tag" - task comarcals: :environment do - csv_text = File.read("tmp/scopes_codes.csv") - csv = CSV.parse(csv_text, headers: true) - - csv = csv.select { |row| row["type"] == "FC" } - csv = csv.map(&:to_h) - - result = get_code_and_display_name(csv) - write_config_yaml!("comarcals", result) - end - - desc "Generates a YAML file with the CSV with 'Regional' tag" - task regionals: :environment do - csv_text = File.read("tmp/scopes_codes.csv") - csv = CSV.parse(csv_text, headers: true) - - csv = csv.select { |row| row["type"] == "FR" } - csv = csv.map(&:to_h) - - result = get_code_and_display_name(csv) - write_config_yaml!("regionals", result) - end - - desc "Generates a YAML file with the CSV with 'Locals' tag" - task locals: :environment do - csv_text = File.read("tmp/scopes_codes.csv") - csv = CSV.parse(csv_text, headers: true) - - csv = csv.select { |row| row["type"] == "SL" } - csv = csv.map(&:to_h) - - result = get_code_and_display_name(csv) - write_config_yaml!("locals", result) - end - - desc "Generates a YAML file with the relationship between CSV 'Local' and 'Comarcal'" - task local_comarcal_relationships: :environment do - local_ids = load_config_yaml("locals").keys - comarcal_ids = load_config_yaml("comarcals").keys - result = get_code_relationships(local_ids, comarcal_ids, "comarcal") - write_config_yaml!("local_comarcal_relationships", result) - end - - desc "Generates a YAML file with the relationship between CSV 'Local' and 'Regional'" - task local_regional_relationships: :environment do - local_ids = load_config_yaml("locals").keys - regional_ids = load_config_yaml("regionals").keys - result = get_code_relationships(local_ids, regional_ids, "regional") - write_config_yaml!("local_regional_relationships", result) - end - - def get_code_and_display_name(body) - body.map! { |element| { element["code"] => element["name"] } }.reduce({}, :merge) - end - - def get_code_relationships(local_ids, _filter_ids, scope_type) - local_ids.each_with_object({}) do |element, memo| - csv_text = File.read("tmp/scopes_hierarchy.csv") - csv = CSV.parse(csv_text, headers: true) - - csv = csv.select { |row| row["SL"] == element }.first.to_h - next if csv.blank? - - memo[csv["SL"]] = csv["FC"] if scope_type == "comarcal" - memo[csv["SL"]] = csv["FR"] if scope_type == "regional" - end - end - end - namespace :generate do task all: [:comarcal_exceptions, :decidim_scopes_mapping] @@ -198,18 +127,4 @@ namespace :civi_crm do end end end - - def write_config_yaml!(filename, content) - File.write(filepath(filename), content.to_yaml) - puts "File generated 'config/civi_crm/#{filename}.yml'" - end - - def filepath(filename) - Dir.mkdir("config/civi_crm") unless File.directory?("config/civi_crm") - Rails.root.join("config", "civi_crm", "#{filename}.yml").to_s - end - - def load_config_yaml(filename) - YAML.load_file(filepath(filename)) - end end diff --git a/lib/tasks/erc_auth.rake b/lib/tasks/erc_auth.rake new file mode 100644 index 0000000..6e92d87 --- /dev/null +++ b/lib/tasks/erc_auth.rake @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +require "csv" +require "file_manager" + +namespace :erc_auth do + include FileManager + + namespace :csv_import do + task all: [:comarcals, :regionals, :locals, :local_comarcal_relationships, :local_regional_relationships] + + desc "Generates a YAML file with the CSV with 'Comarcal' tag" + task comarcals: :environment do + csv_text = File.read("tmp/scopes_codes.csv") + csv = CSV.parse(csv_text, headers: true) + + csv = csv.select { |row| row["type"] == "FC" } + csv = csv.map(&:to_h) + + result = get_code_and_display_name(csv) + write_config_yaml!("comarcals", result) + end + + desc "Generates a YAML file with the CSV with 'Regional' tag" + task regionals: :environment do + csv_text = File.read("tmp/scopes_codes.csv") + csv = CSV.parse(csv_text, headers: true) + + csv = csv.select { |row| row["type"] == "FR" } + csv = csv.map(&:to_h) + + result = get_code_and_display_name(csv) + write_config_yaml!("regionals", result) + end + + desc "Generates a YAML file with the CSV with 'Locals' tag" + task locals: :environment do + csv_text = File.read("tmp/scopes_codes.csv") + csv = CSV.parse(csv_text, headers: true) + + csv = csv.select { |row| row["type"] == "SL" } + csv = csv.map(&:to_h) + + result = get_code_and_display_name(csv) + write_config_yaml!("locals", result) + end + + desc "Generates a YAML file with the relationship between CSV 'Local' and 'Comarcal'" + task local_comarcal_relationships: :environment do + local_ids = load_config_yaml("locals").keys + comarcal_ids = load_config_yaml("comarcals").keys + result = get_code_relationships(local_ids, comarcal_ids, "comarcal") + write_config_yaml!("local_comarcal_relationships", result) + end + + desc "Generates a YAML file with the relationship between CSV 'Local' and 'Regional'" + task local_regional_relationships: :environment do + local_ids = load_config_yaml("locals").keys + regional_ids = load_config_yaml("regionals").keys + result = get_code_relationships(local_ids, regional_ids, "regional") + write_config_yaml!("local_regional_relationships", result) + end + + def get_code_and_display_name(body) + body.map! { |element| { element["code"] => element["name"] } }.reduce({}, :merge) + end + + def get_code_relationships(local_ids, _filter_ids, scope_type) + local_ids.each_with_object({}) do |element, memo| + csv_text = File.read("tmp/scopes_hierarchy.csv") + csv = CSV.parse(csv_text, headers: true) + + csv = csv.select { |row| row["SL"] == element }.first.to_h + next if csv.blank? + + case scope_type + when "comarcal" + memo[csv["SL"]] = csv["FC"] + when "regional" + memo[csv["SL"]] = csv["FR"] + end + end + end + end +end