Merge pull request #208 from yipdw/keyword-mute

Run keyword mutes on hashtags
This commit is contained in:
beatrix 2017-11-18 12:06:40 -05:00 committed by GitHub
commit dec960c828
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 163 additions and 45 deletions

View File

@ -149,7 +149,7 @@ class FeedManager
return false if receiver_id == status.account_id return false if receiver_id == status.account_id
return true if status.reply? && (status.in_reply_to_id.nil? || status.in_reply_to_account_id.nil?) return true if status.reply? && (status.in_reply_to_id.nil? || status.in_reply_to_account_id.nil?)
return true if keyword_filter?(status, Glitch::KeywordMute.matcher_for(receiver_id)) return true if keyword_filter?(status, receiver_id)
check_for_mutes = [status.account_id] check_for_mutes = [status.account_id]
check_for_mutes.concat(status.mentions.pluck(:account_id)) check_for_mutes.concat(status.mentions.pluck(:account_id))
@ -178,16 +178,23 @@ class FeedManager
false false
end end
def keyword_filter?(status, matcher) def keyword_filter?(status, receiver_id)
should_filter = matcher =~ status.text text_matcher = Glitch::KeywordMute.text_matcher_for(receiver_id)
should_filter ||= matcher =~ status.spoiler_text tag_matcher = Glitch::KeywordMute.tag_matcher_for(receiver_id)
should_filter = text_matcher.matches?(status.text)
should_filter ||= text_matcher.matches?(status.spoiler_text)
should_filter ||= tag_matcher.matches?(status.tags)
if status.reblog? if status.reblog?
should_filter ||= matcher =~ status.reblog.text reblog = status.reblog
should_filter ||= matcher =~ status.reblog.spoiler_text
should_filter ||= text_matcher.matches?(reblog.text)
should_filter ||= text_matcher.matches?(reblog.spoiler_text)
should_filter ||= tag_matcher.matches?(status.tags)
end end
!!should_filter should_filter
end end
def filter_from_mentions?(status, receiver_id) def filter_from_mentions?(status, receiver_id)
@ -199,7 +206,7 @@ class FeedManager
should_filter = Block.where(account_id: receiver_id, target_account_id: check_for_blocks).any? # Filter if it's from someone I blocked, in reply to someone I blocked, or mentioning someone I blocked should_filter = Block.where(account_id: receiver_id, target_account_id: check_for_blocks).any? # Filter if it's from someone I blocked, in reply to someone I blocked, or mentioning someone I blocked
should_filter ||= (status.account.silenced? && !Follow.where(account_id: receiver_id, target_account_id: status.account_id).exists?) # of if the account is silenced and I'm not following them should_filter ||= (status.account.silenced? && !Follow.where(account_id: receiver_id, target_account_id: status.account_id).exists?) # of if the account is silenced and I'm not following them
should_filter ||= keyword_filter?(status, Glitch::KeywordMute.matcher_for(receiver_id)) # or if the mention contains a muted keyword should_filter ||= keyword_filter?(status, receiver_id) # or if the mention contains a muted keyword
should_filter should_filter
end end

View File

@ -16,44 +16,37 @@ class Glitch::KeywordMute < ApplicationRecord
validates_presence_of :keyword validates_presence_of :keyword
after_commit :invalidate_cached_matcher after_commit :invalidate_cached_matchers
def self.matcher_for(account_id) def self.text_matcher_for(account_id)
Matcher.new(account_id) TextMatcher.new(account_id)
end
def self.tag_matcher_for(account_id)
TagMatcher.new(account_id)
end end
private private
def invalidate_cached_matcher def invalidate_cached_matchers
Rails.cache.delete("keyword_mutes:regex:#{account_id}") Rails.cache.delete(TextMatcher.cache_key(account_id))
Rails.cache.delete(TagMatcher.cache_key(account_id))
end end
class Matcher class RegexpMatcher
attr_reader :account_id attr_reader :account_id
attr_reader :regex attr_reader :regex
def initialize(account_id) def initialize(account_id)
@account_id = account_id @account_id = account_id
regex_text = Rails.cache.fetch("keyword_mutes:regex:#{account_id}") { regex_text_for_account } regex_text = Rails.cache.fetch(self.class.cache_key(account_id)) { make_regex_text }
@regex = /#{regex_text}/ @regex = /#{regex_text}/
end end
def =~(str) protected
regex =~ str
end
private
def keywords def keywords
Glitch::KeywordMute.where(account_id: account_id).select(:keyword, :id, :whole_word) Glitch::KeywordMute.where(account_id: account_id).pluck(:whole_word, :keyword)
end
def regex_text_for_account
kws = keywords.find_each.with_object([]) do |kw, a|
a << (kw.whole_word ? boundary_regex_for_keyword(kw.keyword) : kw.keyword)
end
Regexp.union(kws).source
end end
def boundary_regex_for_keyword(keyword) def boundary_regex_for_keyword(keyword)
@ -63,4 +56,45 @@ class Glitch::KeywordMute < ApplicationRecord
/(?mix:#{sb}#{Regexp.escape(keyword)}#{eb})/ /(?mix:#{sb}#{Regexp.escape(keyword)}#{eb})/
end end
end end
class TextMatcher < RegexpMatcher
def self.cache_key(account_id)
format('keyword_mutes:regex:text:%s', account_id)
end
def matches?(str)
!!(regex =~ str)
end
private
def make_regex_text
kws = keywords.map! do |whole_word, keyword|
whole_word ? boundary_regex_for_keyword(keyword) : keyword
end
Regexp.union(kws).source
end
end
class TagMatcher < RegexpMatcher
def self.cache_key(account_id)
format('keyword_mutes:regex:tag:%s', account_id)
end
def matches?(tags)
tags.pluck(:name).any? { |n| regex =~ n }
end
private
def make_regex_text
kws = keywords.map! do |whole_word, keyword|
term = (Tag::HASHTAG_RE =~ keyword) ? $1 : keyword
whole_word ? boundary_regex_for_keyword(term) : term
end
Regexp.union(kws).source
end
end
end end

View File

@ -164,6 +164,22 @@ RSpec.describe FeedManager do
expect(FeedManager.instance.filter?(:home, reblog, alice.id)).to be true expect(FeedManager.instance.filter?(:home, reblog, alice.id)).to be true
end end
it 'returns true for a status with a tag that matches a muted keyword' do
Fabricate('Glitch::KeywordMute', account: alice, keyword: 'jorts')
status = Fabricate(:status, account: bob)
status.tags << Fabricate(:tag, name: 'jorts')
expect(FeedManager.instance.filter?(:home, status, alice.id)).to be true
end
it 'returns true for a status with a tag that matches an octothorpe-prefixed muted keyword' do
Fabricate('Glitch::KeywordMute', account: alice, keyword: '#jorts')
status = Fabricate(:status, account: bob)
status.tags << Fabricate(:tag, name: 'jorts')
expect(FeedManager.instance.filter?(:home, status, alice.id)).to be true
end
end end
context 'for mentions feed' do context 'for mentions feed' do

View File

@ -4,8 +4,8 @@ RSpec.describe Glitch::KeywordMute, type: :model do
let(:alice) { Fabricate(:account, username: 'alice').tap(&:save!) } let(:alice) { Fabricate(:account, username: 'alice').tap(&:save!) }
let(:bob) { Fabricate(:account, username: 'bob').tap(&:save!) } let(:bob) { Fabricate(:account, username: 'bob').tap(&:save!) }
describe '.matcher_for' do describe '.text_matcher_for' do
let(:matcher) { Glitch::KeywordMute.matcher_for(alice) } let(:matcher) { Glitch::KeywordMute.text_matcher_for(alice.id) }
describe 'with no mutes' do describe 'with no mutes' do
before do before do
@ -13,7 +13,7 @@ RSpec.describe Glitch::KeywordMute, type: :model do
end end
it 'does not match' do it 'does not match' do
expect(matcher =~ 'This is a hot take').to be_falsy expect(matcher.matches?('This is a hot take')).to be_falsy
end end
end end
@ -21,75 +21,136 @@ RSpec.describe Glitch::KeywordMute, type: :model do
it 'does not match keywords set by a different account' do it 'does not match keywords set by a different account' do
Glitch::KeywordMute.create!(account: bob, keyword: 'take') Glitch::KeywordMute.create!(account: bob, keyword: 'take')
expect(matcher =~ 'This is a hot take').to be_falsy expect(matcher.matches?('This is a hot take')).to be_falsy
end end
it 'does not match if no keywords match the status text' do it 'does not match if no keywords match the status text' do
Glitch::KeywordMute.create!(account: alice, keyword: 'cold') Glitch::KeywordMute.create!(account: alice, keyword: 'cold')
expect(matcher =~ 'This is a hot take').to be_falsy expect(matcher.matches?('This is a hot take')).to be_falsy
end end
it 'considers word boundaries when matching' do it 'considers word boundaries when matching' do
Glitch::KeywordMute.create!(account: alice, keyword: 'bob', whole_word: true) Glitch::KeywordMute.create!(account: alice, keyword: 'bob', whole_word: true)
expect(matcher =~ 'bobcats').to be_falsy expect(matcher.matches?('bobcats')).to be_falsy
end end
it 'matches substrings if whole_word is false' do it 'matches substrings if whole_word is false' do
Glitch::KeywordMute.create!(account: alice, keyword: 'take', whole_word: false) Glitch::KeywordMute.create!(account: alice, keyword: 'take', whole_word: false)
expect(matcher =~ 'This is a shiitake mushroom').to be_truthy expect(matcher.matches?('This is a shiitake mushroom')).to be_truthy
end end
it 'matches keywords at the beginning of the text' do it 'matches keywords at the beginning of the text' do
Glitch::KeywordMute.create!(account: alice, keyword: 'take') Glitch::KeywordMute.create!(account: alice, keyword: 'take')
expect(matcher =~ 'Take this').to be_truthy expect(matcher.matches?('Take this')).to be_truthy
end end
it 'matches keywords at the end of the text' do it 'matches keywords at the end of the text' do
Glitch::KeywordMute.create!(account: alice, keyword: 'take') Glitch::KeywordMute.create!(account: alice, keyword: 'take')
expect(matcher =~ 'This is a hot take').to be_truthy expect(matcher.matches?('This is a hot take')).to be_truthy
end end
it 'matches if at least one keyword case-insensitively matches the text' do it 'matches if at least one keyword case-insensitively matches the text' do
Glitch::KeywordMute.create!(account: alice, keyword: 'hot') Glitch::KeywordMute.create!(account: alice, keyword: 'hot')
expect(matcher =~ 'This is a HOT take').to be_truthy expect(matcher.matches?('This is a HOT take')).to be_truthy
end end
it 'maintains case-insensitivity when combining keywords into a single matcher' do it 'maintains case-insensitivity when combining keywords into a single matcher' do
Glitch::KeywordMute.create!(account: alice, keyword: 'hot') Glitch::KeywordMute.create!(account: alice, keyword: 'hot')
Glitch::KeywordMute.create!(account: alice, keyword: 'cold') Glitch::KeywordMute.create!(account: alice, keyword: 'cold')
expect(matcher =~ 'This is a HOT take').to be_truthy expect(matcher.matches?('This is a HOT take')).to be_truthy
end end
it 'matches keywords surrounded by non-alphanumeric ornamentation' do it 'matches keywords surrounded by non-alphanumeric ornamentation' do
Glitch::KeywordMute.create!(account: alice, keyword: 'hot') Glitch::KeywordMute.create!(account: alice, keyword: 'hot')
expect(matcher =~ '(hot take)').to be_truthy expect(matcher.matches?('(hot take)')).to be_truthy
end end
it 'escapes metacharacters in keywords' do it 'escapes metacharacters in keywords' do
Glitch::KeywordMute.create!(account: alice, keyword: '(hot take)') Glitch::KeywordMute.create!(account: alice, keyword: '(hot take)')
expect(matcher =~ '(hot take)').to be_truthy expect(matcher.matches?('(hot take)')).to be_truthy
end end
it 'uses case-folding rules appropriate for more than just English' do it 'uses case-folding rules appropriate for more than just English' do
Glitch::KeywordMute.create!(account: alice, keyword: 'großeltern') Glitch::KeywordMute.create!(account: alice, keyword: 'großeltern')
expect(matcher =~ 'besuch der grosseltern').to be_truthy expect(matcher.matches?('besuch der grosseltern')).to be_truthy
end end
it 'matches keywords that are composed of multiple words' do it 'matches keywords that are composed of multiple words' do
Glitch::KeywordMute.create!(account: alice, keyword: 'a shiitake') Glitch::KeywordMute.create!(account: alice, keyword: 'a shiitake')
expect(matcher =~ 'This is a shiitake').to be_truthy expect(matcher.matches?('This is a shiitake')).to be_truthy
expect(matcher =~ 'This is shiitake').to_not be_truthy expect(matcher.matches?('This is shiitake')).to_not be_truthy
end
end
end
describe '.tag_matcher_for' do
let(:matcher) { Glitch::KeywordMute.tag_matcher_for(alice.id) }
let(:status) { Fabricate(:status) }
describe 'with no mutes' do
before do
Glitch::KeywordMute.delete_all
end
it 'does not match' do
status.tags << Fabricate(:tag, name: 'xyzzy')
expect(matcher.matches?(status.tags)).to be false
end
end
describe 'with mutes' do
it 'does not match keywords set by a different account' do
status.tags << Fabricate(:tag, name: 'xyzzy')
Glitch::KeywordMute.create!(account: bob, keyword: 'take')
expect(matcher.matches?(status.tags)).to be false
end
it 'matches #xyzzy when given the mute "#xyzzy"' do
status.tags << Fabricate(:tag, name: 'xyzzy')
Glitch::KeywordMute.create!(account: alice, keyword: '#xyzzy')
expect(matcher.matches?(status.tags)).to be true
end
it 'matches #thingiverse when given the non-whole-word mute "#thing"' do
status.tags << Fabricate(:tag, name: 'thingiverse')
Glitch::KeywordMute.create!(account: alice, keyword: '#thing', whole_word: false)
expect(matcher.matches?(status.tags)).to be true
end
it 'matches #hashtag when given the mute "##hashtag""' do
status.tags << Fabricate(:tag, name: 'hashtag')
Glitch::KeywordMute.create!(account: alice, keyword: '##hashtag')
expect(matcher.matches?(status.tags)).to be true
end
it 'matches #oatmeal when given the non-whole-word mute "oat"' do
status.tags << Fabricate(:tag, name: 'oatmeal')
Glitch::KeywordMute.create!(account: alice, keyword: 'oat', whole_word: false)
expect(matcher.matches?(status.tags)).to be true
end
it 'does not match #oatmeal when given the mute "#oat"' do
status.tags << Fabricate(:tag, name: 'oatmeal')
Glitch::KeywordMute.create!(account: alice, keyword: 'oat')
expect(matcher.matches?(status.tags)).to be false
end end
end end
end end