Add option to overwrite imported data (#9962)

* Add option to overwrite imported data

Fix #7465

* Add import for domain blocks
This commit is contained in:
Eugen Rochko 2019-02-03 03:59:51 +01:00 committed by GitHub
parent c5071f2d78
commit d14c276e58
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 148 additions and 43 deletions

View File

@ -12,6 +12,7 @@
class AccountDomainBlock < ApplicationRecord class AccountDomainBlock < ApplicationRecord
include Paginable include Paginable
include DomainNormalizable
belongs_to :account belongs_to :account
validates :domain, presence: true, uniqueness: { scope: :account_id } validates :domain, presence: true, uniqueness: { scope: :account_id }

View File

@ -10,6 +10,6 @@ module DomainNormalizable
private private
def normalize_domain def normalize_domain
self.domain = TagManager.instance.normalize_domain(domain) self.domain = TagManager.instance.normalize_domain(domain&.strip)
end end
end end

View File

@ -1,4 +1,5 @@
# frozen_string_literal: true # frozen_string_literal: true
require 'csv' require 'csv'
class Export class Export

View File

@ -13,20 +13,30 @@
# data_file_size :integer # data_file_size :integer
# data_updated_at :datetime # data_updated_at :datetime
# account_id :bigint(8) not null # account_id :bigint(8) not null
# overwrite :boolean default(FALSE), not null
# #
class Import < ApplicationRecord class Import < ApplicationRecord
FILE_TYPES = ['text/plain', 'text/csv'].freeze FILE_TYPES = %w(text/plain text/csv).freeze
MODES = %i(merge overwrite).freeze
self.inheritance_column = false self.inheritance_column = false
belongs_to :account belongs_to :account
enum type: [:following, :blocking, :muting] enum type: [:following, :blocking, :muting, :domain_blocking]
validates :type, presence: true validates :type, presence: true
has_attached_file :data has_attached_file :data
validates_attachment_content_type :data, content_type: FILE_TYPES validates_attachment_content_type :data, content_type: FILE_TYPES
validates_attachment_presence :data validates_attachment_presence :data
def mode
overwrite? ? :overwrite : :merge
end
def mode=(str)
self.overwrite = str.to_sym == :overwrite
end
end end

View File

@ -0,0 +1,90 @@
# frozen_string_literal: true
require 'csv'
class ImportService < BaseService
ROWS_PROCESSING_LIMIT = 20_000
def call(import)
@import = import
@account = @import.account
@data = CSV.new(import_data).reject(&:blank?)
case @import.type
when 'following'
import_follows!
when 'blocking'
import_blocks!
when 'muting'
import_mutes!
when 'domain_blocking'
import_domain_blocks!
end
end
private
def import_follows!
import_relationships!('follow', 'unfollow', @account.following, follow_limit)
end
def import_blocks!
import_relationships!('block', 'unblock', @account.blocking, ROWS_PROCESSING_LIMIT)
end
def import_mutes!
import_relationships!('mute', 'unmute', @account.muting, ROWS_PROCESSING_LIMIT)
end
def import_domain_blocks!
items = @data.take(ROWS_PROCESSING_LIMIT).map { |row| row.first.strip }
if @import.overwrite?
presence_hash = items.each_with_object({}) { |id, mapping| mapping[id] = true }
@account.domain_blocks.find_each do |domain_block|
if presence_hash[domain_block.domain]
items.delete(domain_block.domain)
else
@account.unblock_domain!(domain_block.domain)
end
end
end
items.each do |domain|
@account.block_domain!(domain)
end
AfterAccountDomainBlockWorker.push_bulk(items) do |domain|
[@account.id, domain]
end
end
def import_relationships!(action, undo_action, overwrite_scope, limit)
items = @data.take(limit).map { |row| row.first.strip }
if @import.overwrite?
presence_hash = items.each_with_object({}) { |id, mapping| mapping[id] = true }
overwrite_scope.find_each do |target_account|
if presence_hash[target_account.acct]
items.delete(target_account.acct)
else
Import::RelationshipWorker.perform_async(@account.id, target_account.acct, undo_action)
end
end
end
Import::RelationshipWorker.push_bulk(items) do |acct|
[@account.id, acct, action]
end
end
def import_data
Paperclip.io_adapters.for(@import.data).read
end
def follow_limit
FollowLimitValidator.limit_for_account(@account)
end
end

View File

@ -5,8 +5,11 @@
.field-group .field-group
= f.input :type, collection: Import.types.keys, wrapper: :with_block_label, include_blank: false, label_method: lambda { |type| I18n.t("imports.types.#{type}") }, hint: t('imports.preface') = f.input :type, collection: Import.types.keys, wrapper: :with_block_label, include_blank: false, label_method: lambda { |type| I18n.t("imports.types.#{type}") }, hint: t('imports.preface')
.field-group .fields-row
.fields-group.fields-row__column.fields-row__column-6
= f.input :data, wrapper: :with_block_label, hint: t('simple_form.hints.imports.data') = f.input :data, wrapper: :with_block_label, hint: t('simple_form.hints.imports.data')
.fields-group.fields-row__column.fields-row__column-6
= f.input :mode, as: :radio_buttons, collection: Import::MODES, label_method: lambda { |mode| safe_join([I18n.t("imports.modes.#{mode}"), content_tag(:span, I18n.t("imports.modes.#{mode}_long"), class: 'hint')]) }, collection_wrapper_tag: 'ul', item_wrapper_tag: 'li'
.actions .actions
= f.button :button, t('imports.upload'), type: :submit = f.button :button, t('imports.upload'), type: :submit

View File

@ -13,11 +13,17 @@ class Import::RelationshipWorker
case relationship case relationship
when 'follow' when 'follow'
FollowService.new.call(from_account, target_account.acct) FollowService.new.call(from_account, target_account)
when 'unfollow'
UnfollowService.new.call(from_account, target_account)
when 'block' when 'block'
BlockService.new.call(from_account, target_account) BlockService.new.call(from_account, target_account)
when 'unblock'
UnblockService.new.call(from_account, target_account)
when 'mute' when 'mute'
MuteService.new.call(from_account, target_account) MuteService.new.call(from_account, target_account)
when 'unmute'
UnmuteService.new.call(from_account, target_account)
end end
rescue ActiveRecord::RecordNotFound rescue ActiveRecord::RecordNotFound
true true

View File

@ -1,44 +1,14 @@
# frozen_string_literal: true # frozen_string_literal: true
require 'csv'
class ImportWorker class ImportWorker
include Sidekiq::Worker include Sidekiq::Worker
sidekiq_options queue: 'pull', retry: false sidekiq_options queue: 'pull', retry: false
attr_reader :import
def perform(import_id) def perform(import_id)
@import = Import.find(import_id) import = Import.find(import_id)
ImportService.new.call(import)
Import::RelationshipWorker.push_bulk(import_rows) do |row| ensure
[@import.account_id, row.first, relationship_type] import&.destroy
end
@import.destroy
end
private
def import_contents
Paperclip.io_adapters.for(@import.data).read
end
def relationship_type
case @import.type
when 'following'
'follow'
when 'blocking'
'block'
when 'muting'
'mute'
end
end
def import_rows
rows = CSV.new(import_contents).reject(&:blank?)
rows = rows.take(FollowLimitValidator.limit_for_account(@import.account)) if @import.type == 'following'
rows
end end
end end

View File

@ -628,10 +628,16 @@ en:
one: Something isn't quite right yet! Please review the error below one: Something isn't quite right yet! Please review the error below
other: Something isn't quite right yet! Please review %{count} errors below other: Something isn't quite right yet! Please review %{count} errors below
imports: imports:
modes:
merge: Merge
merge_long: Keep existing records and add new ones
overwrite: Overwrite
overwrite_long: Replace current records with the new ones
preface: You can import data that you have exported from another instance, such as a list of the people you are following or blocking. preface: You can import data that you have exported from another instance, such as a list of the people you are following or blocking.
success: Your data was successfully uploaded and will now be processed in due time success: Your data was successfully uploaded and will now be processed in due time
types: types:
blocking: Blocking list blocking: Blocking list
domain_blocking: Domain blocking list
following: Following list following: Following list
muting: Muting list muting: Muting list
upload: Upload upload: Upload

View File

@ -0,0 +1,17 @@
require Rails.root.join('lib', 'mastodon', 'migration_helpers')
class AddOverwriteToImports < ActiveRecord::Migration[5.2]
include Mastodon::MigrationHelpers
disable_ddl_transaction!
def up
safety_assured do
add_column_with_default :imports, :overwrite, :boolean, default: false, allow_null: false
end
end
def down
remove_column :imports, :overwrite, :boolean
end
end

View File

@ -10,7 +10,7 @@
# #
# It's strongly recommended that you check this file into your version control system. # It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 2019_01_17_114553) do ActiveRecord::Schema.define(version: 2019_02_01_012802) do
# These are extensions that must be enabled in order to support this database # These are extensions that must be enabled in order to support this database
enable_extension "plpgsql" enable_extension "plpgsql"
@ -290,6 +290,7 @@ ActiveRecord::Schema.define(version: 2019_01_17_114553) do
t.integer "data_file_size" t.integer "data_file_size"
t.datetime "data_updated_at" t.datetime "data_updated_at"
t.bigint "account_id", null: false t.bigint "account_id", null: false
t.boolean "overwrite", default: false, null: false
end end
create_table "invites", force: :cascade do |t| create_table "invites", force: :cascade do |t|

View File

@ -237,9 +237,9 @@ describe AccountInteractions do
end end
describe '#block_domain!' do describe '#block_domain!' do
let(:domain_block) { Fabricate(:domain_block) } let(:domain) { 'example.com' }
subject { account.block_domain!(domain_block) } subject { account.block_domain!(domain) }
it 'creates and returns AccountDomainBlock' do it 'creates and returns AccountDomainBlock' do
expect do expect do