From 4a64181461cb02599da98166da4b527adbb705ad Mon Sep 17 00:00:00 2001 From: David Yip Date: Sun, 15 Oct 2017 19:49:22 -0500 Subject: [PATCH] Allow keywords to match either substrings or whole words. Word-boundary matching only works as intended in English and languages that use similar word-breaking characters; it doesn't work so well in (say) Japanese, Chinese, or Thai. It's unacceptable to have a feature that doesn't work as intended for some languages. (Moreso especially considering that it's likely that the largest contingent on the Mastodon bit of the fediverse speaks Japanese.) There are rules specified in Unicode TR29[1] for word-breaking across all languages supported by Unicode, but the rules deliberately do not cover all cases. In fact, TR29 states For example, reliable detection of word boundaries in languages such as Thai, Lao, Chinese, or Japanese requires the use of dictionary lookup, analogous to English hyphenation. So we aren't going to be able to make word detection work with regexes within Mastodon (or glitchsoc). However, for a first pass (even if it's kind of punting) we can allow the user to choose whether they want word or substring detection and warn about the limitations of this implementation in, say, docs. [1]: https://unicode.org/reports/tr29/ https://web.archive.org/web/20171001005125/https://unicode.org/reports/tr29/ --- app/models/keyword_mute.rb | 8 +++++--- db/migrate/20171009222537_create_keyword_mutes.rb | 1 + db/schema.rb | 1 + spec/models/keyword_mute_spec.rb | 12 +++++++++--- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/app/models/keyword_mute.rb b/app/models/keyword_mute.rb index 8b54ad696e..b0229923d8 100644 --- a/app/models/keyword_mute.rb +++ b/app/models/keyword_mute.rb @@ -6,6 +6,7 @@ # id :integer not null, primary key # account_id :integer not null # keyword :string not null +# whole_word :boolean default(TRUE), not null # created_at :datetime not null # updated_at :datetime not null # @@ -32,12 +33,13 @@ class KeywordMute < ApplicationRecord def initialize(account_id) re = [].tap do |arr| - KeywordMute.where(account_id: account_id).select(:keyword, :id).find_each do |m| - arr << Regexp.escape(m.keyword.strip) + KeywordMute.where(account_id: account_id).select(:keyword, :id, :whole_word).find_each do |m| + boundary = m.whole_word ? '\b' : '' + arr << "#{boundary}#{Regexp.escape(m.keyword.strip)}#{boundary}" end end.join('|') - @regex = /\b(?:#{re})\b/i unless re.empty? + @regex = /#{re}/i unless re.empty? end def =~(str) diff --git a/db/migrate/20171009222537_create_keyword_mutes.rb b/db/migrate/20171009222537_create_keyword_mutes.rb index ee690e799a..ec0c756fbf 100644 --- a/db/migrate/20171009222537_create_keyword_mutes.rb +++ b/db/migrate/20171009222537_create_keyword_mutes.rb @@ -3,6 +3,7 @@ class CreateKeywordMutes < ActiveRecord::Migration[5.1] create_table :keyword_mutes do |t| t.references :account, null: false t.string :keyword, null: false + t.boolean :whole_word, null: false, default: true t.timestamps end diff --git a/db/schema.rb b/db/schema.rb index 420bb0d2ee..c0704b13e3 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -170,6 +170,7 @@ ActiveRecord::Schema.define(version: 20171010025614) do create_table "keyword_mutes", force: :cascade do |t| t.bigint "account_id", null: false t.string "keyword", null: false + t.boolean "whole_word", default: true, null: false t.datetime "created_at", null: false t.datetime "updated_at", null: false t.index ["account_id"], name: "index_keyword_mutes_on_account_id" diff --git a/spec/models/keyword_mute_spec.rb b/spec/models/keyword_mute_spec.rb index de5d32bb42..c745051886 100644 --- a/spec/models/keyword_mute_spec.rb +++ b/spec/models/keyword_mute_spec.rb @@ -30,10 +30,16 @@ RSpec.describe KeywordMute, type: :model do expect(matcher =~ 'This is a hot take').to be_falsy end - it 'does not match substrings matching keywords' do - KeywordMute.create!(account: alice, keyword: 'take') + it 'considers word boundaries when matching' do + KeywordMute.create!(account: alice, keyword: 'bob', whole_word: true) - expect(matcher =~ 'This is a shiitake mushroom').to be_falsy + expect(matcher =~ 'bobcats').to be_falsy + end + + it 'matches substrings if whole_word is false' do + KeywordMute.create!(account: alice, keyword: 'take', whole_word: false) + + expect(matcher =~ 'This is a shiitake mushroom').to be_truthy end it 'matches keywords at the beginning of the text' do