issue with grow in notification and executions in mongo db

add to application.yml

'Setup::SystemNotification:attachment:uploader': EmbeddedUploader
'Setup::Execution:attachment:uploader': EmbeddedUploader

1- To migrate .files and .chunks collections attachment data after setting the above configuration in the file config/application.yml

migrate_attachment_storages_to_embedded_files.rb

script_tenant =
  if (script_tenant_id = state[:script_tenant_id])
    tenants = Tenant.where(:id.gt => state[:prev_tenant_id])
    Tenant.find(script_tenant_id)
  else
    tenants = Tenant.all
    state[:script_tenant_id] = Tenant.current.id
    state[:n_counter] = 0
    state[:tenant_counter] = 0
    Tenant.current
  end
​
models = Setup::CenitDataType.all.map(&:model).uniq.select { |m| m && m < Setup::AttachmentUploader }
models = models.select { |m| Cenit.attachment_uploader_for(m) == EmbeddedUploader }
prefixes = models.map { |m| "/#{m.to_s.underscore.gsub('/', '~')}/attachment/" }
​
done = true
tenants.each do |tenant|
  done = false
  state[:tenant_counter] += 1
  state[:prev_tenant_id] = tenant.id
  update(progress: 100 * (state[:tenant_counter] - 1).to_f / Tenant.all.count)
  prev_c = counter = 0
  tenant.switch do
    prefixes.each do |prefix|
      Setup::Storage.where(filename: Regexp.new("\\A#{prefix}", true)).each do |storage|
        if (obj = storage.storer_object)
          begin
            Cenit::EmbeddedStorage.store_on(obj, storage, filename: storage.filename.split('/').last)
            obj.save
          rescue Exception => ex
            Setup::SystemNotification.create_from(ex, "migrating storage for #{storage.filename}")
          end
        end
        counter += 1 if storage.destroy
        if counter > prev_c + 1000
          script_tenant.switch do
            Setup::SystemNotification.create(message: "#{counter} storages migrated in tenant #{tenant.label} (so far...)", type: :notice)
            prev_c = counter
          end
        end
      end
    end
    if counter > 0
      state[:n_counter] += counter
      Setup::SystemNotification.create(message: "#{counter} storages migrated", type: :notice)
    end
  end
  if counter > 0
    Setup::SystemNotification.create(message: "#{counter} storages migrated in tenant #{tenant.label}", type: :notice)
    if counter > 100
      break
    else
      save
    end
  end
end
if done
  msg = "#{state[:n_counter]} cross-tenant storages migrated"
  Setup::SystemNotification.create(message: msg, type: :notice)
else
  run_again
end

2- This should be executed periodically, for example every 2 days, to clean up old tasks and executions bulk_delete_old_taks.rb

 script_tenant =
  if (script_tenant_id = state[:script_tenant_id])
    tenants = Tenant.where(:id.gt => state[:prev_tenant_id])
    Tenant.find(script_tenant_id)
  else
    tenants = Tenant.all
    state[:script_tenant_id] = Tenant.current.id
    state[:task_counter] = 0
    state[:execution_counter] = 0
    state[:tenant_counter] = 0
    Tenant.current
  end
done = true
tenants_total = Tenant.all.count
tenants.each do |tenant|
  done = false
  state[:tenant_counter] += 1
  state[:prev_tenant_id] = tenant.id
  update(progress: 100 * (state[:tenant_counter] - 1).to_f / tenants_total)
  counter = 0
  e_counter = 0
  tenant.switch do
    dead_span = (Time.now - (Setup::SystemNotification.type_enum.map { |type| tenant.notification_span_for(type) }.max.seconds)).to_bson_id
    #Lookup for old tasks
    dead_tasks = Setup::Task.where(
      origin: :default,
      status: :completed,
      scheduler: nil,
      :id.lt => dead_span
    )
    # Lookup for old tasks
    c = dead_tasks.count
    if c > 0
      dead_tasks.delete_all
      counter += c
      state[:task_counter] += c
      Setup::SystemNotification.create(message: "#{counter} boring tasks deleted", type: :notice)
      script_tenant.switch do
        Setup::SystemNotification.create(message: "#{counter} boring tasks deleted in tenant #{tenant.label}", type: :notice)
      end
    else
      script_tenant.switch do
        Setup::SystemNotification.create(message: "No boring tasks detected in tenant #{tenant.label}", type: :notice)
      end
    end
    # Lookup for old executions
    dead_executions = Setup::Execution.where(
      status: :completed,
      :id.lt => dead_span
    )
    c = dead_executions.count
    if c > 0
      dead_executions.delete_all 
      e_counter += c
      state[:execution_counter] += c
      Setup::SystemNotification.create(message: "#{e_counter} old executions deleted", type: :notice)
      script_tenant.switch do
        Setup::SystemNotification.create(message: "#{e_counter} old executions deleted in tenant #{tenant.label}", type: :notice)
      end
    end
  end
  if counter + e_counter > 1000
    break
  else
    save
  end
end
if done
  msg =
    if state[:task_counter] > 0
      "#{state[:task_counter]} cross-tenant boring tasks deleted"
    else
      "No boring tasks detected"
    end
  Setup::SystemNotification.create(message: msg, type: :notice)
  msg =
    if state[:execution_counter] > 0
      "#{state[:execution_counter]} cross-tenant old executions deleted"
    else
      "No old executions detected"
    end
  Setup::SystemNotification.create(message: msg, type: :notice)
else
  run_again
end
nil

3- This should be executed periodically, for example every 2 days, to clean up old notifications

bulk_delete_old_notifications.rb

  script_tenant =
  if (script_tenant_id = state[:script_tenant_id])
    tenants = Tenant.where(:id.gt => state[:prev_tenant_id])
    Tenant.find(script_tenant_id)
  else
    tenants = Tenant.all
    state[:script_tenant_id] = Tenant.current.id
    state[:n_counter] = 0
    state[:tenant_counter] = 0
    Tenant.current
  end
done = true
tenants.each do |tenant|
  done = false
  state[:tenant_counter] += 1
  state[:prev_tenant_id] = tenant.id
  update(progress: 100 * (state[:tenant_counter] - 1).to_f / Tenant.all.count)
  prev_c = counter = 0
  tenant.switch do
    Setup::SystemNotification.type_enum.each do |type|
      lookup_id = (Time.now - tenant.notification_span_for(type).seconds).to_bson_id
      ns = Setup::SystemNotification.where(:id.lt => lookup_id, type: type)
      counter += (ns_count = ns.count)
      ns.delete_all if ns_count > 0
      if counter > prev_c + 1000
        script_tenant.switch do
          Setup::SystemNotification.create(message: "#{counter} old notifications deleted in tenant #{tenant.label} (so far...)", type: :notice)
          prev_c = counter
        end
      end
    end
    if counter > 0
      state[:n_counter] += counter
      Setup::SystemNotification.create(message: "#{counter} old notifications deleted", type: :notice)
    end
  end
  if counter > 0
    Setup::SystemNotification.create(message: "#{counter} old notifications deleted in tenant #{tenant.label}", type: :notice)
    if counter > 100
      break
    else
      save
    end
  end
end
if done
  msg =
    if state[:n_counter] > 0
      "#{state[:n_counter]} cross-tenant old notifications deleted"
    else
      "No old notifications detected"
    end
  Setup::SystemNotification.create(message: msg, type: :notice)
else
  run_again
end
nil

4- We also execute this one every 1h to delete expired tokens

delete_expired_tokens.rb

 script_account = Account.current
report = Hash.new { |h, k| h[k] = 0 }
destroying = true
count = 0
now = Time.now
while destroying
  destroying = false
  query = Cenit::Token.where(:expires_at.lte => now).limit(1000)
  if (c = query.count) > 0
    count += c
    destroying = true
    query.delete_all
  end
end
msg =
  if count > 0
    "#{count} expired tokens deleted at #{now}"
  else
    "No expired tokens detected at #{now}"
  end
Setup::SystemNotification.create(message: msg, type: :notice)