Add search syntax for operators and phrases (#11411)

This commit is contained in:
Eugen Rochko 2019-07-27 04:42:08 +02:00 committed by GitHub
parent 501148ab91
commit b9fbcbfe4e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 112 additions and 6 deletions

View file

@ -64,6 +64,7 @@ gem 'nsa', '~> 0.2'
gem 'oj', '~> 3.8' gem 'oj', '~> 3.8'
gem 'ostatus2', '~> 2.0' gem 'ostatus2', '~> 2.0'
gem 'ox', '~> 2.11' gem 'ox', '~> 2.11'
gem 'parslet'
gem 'posix-spawn', git: 'https://github.com/rtomayko/posix-spawn', ref: '58465d2e213991f8afb13b984854a49fcdcc980c' gem 'posix-spawn', git: 'https://github.com/rtomayko/posix-spawn', ref: '58465d2e213991f8afb13b984854a49fcdcc980c'
gem 'pundit', '~> 2.0' gem 'pundit', '~> 2.0'
gem 'premailer-rails' gem 'premailer-rails'

View file

@ -404,6 +404,7 @@ GEM
parallel parallel
parser (2.6.3.0) parser (2.6.3.0)
ast (~> 2.4.0) ast (~> 2.4.0)
parslet (1.8.2)
pastel (0.7.2) pastel (0.7.2)
equatable (~> 0.5.0) equatable (~> 0.5.0)
tty-color (~> 0.4.0) tty-color (~> 0.4.0)
@ -724,6 +725,7 @@ DEPENDENCIES
paperclip (~> 6.0) paperclip (~> 6.0)
paperclip-av-transcoder (~> 0.6) paperclip-av-transcoder (~> 0.6)
parallel_tests (~> 2.29) parallel_tests (~> 2.29)
parslet
pg (~> 1.1) pg (~> 1.1)
pghero (~> 2.2) pghero (~> 2.2)
pkg-config (~> 1.3) pkg-config (~> 1.3)

View file

@ -0,0 +1,14 @@
# frozen_string_literal: true
class SearchQueryParser < Parslet::Parser
rule(:term) { match('[^\s":]').repeat(1).as(:term) }
rule(:quote) { str('"') }
rule(:colon) { str(':') }
rule(:space) { match('\s').repeat(1) }
rule(:operator) { (str('+') | str('-')).as(:operator) }
rule(:prefix) { (term >> colon).as(:prefix) }
rule(:phrase) { (quote >> (term >> space.maybe).repeat >> quote).as(:phrase) }
rule(:clause) { (prefix.maybe >> operator.maybe >> (phrase | term)).as(:clause) }
rule(:query) { (clause >> space.maybe).repeat.as(:query) }
root(:query)
end

View file

@ -0,0 +1,86 @@
# frozen_string_literal: true
class SearchQueryTransformer < Parslet::Transform
class Query
attr_reader :should_clauses, :must_not_clauses, :must_clauses
def initialize(clauses)
grouped = clauses.chunk(&:operator).to_h
@should_clauses = grouped.fetch(:should, [])
@must_not_clauses = grouped.fetch(:must_not, [])
@must_clauses = grouped.fetch(:must, [])
end
def apply(search)
should_clauses.each { |clause| search = search.query.should(clause_to_query(clause)) }
must_clauses.each { |clause| search = search.query.must(clause_to_query(clause)) }
must_not_clauses.each { |clause| search = search.query.must_not(clause_to_query(clause)) }
search.query.minimum_should_match(1)
end
private
def clause_to_query(clause)
case clause
when TermClause
{ multi_match: { type: 'most_fields', query: clause.term, fields: ['text', 'text.stemmed'] } }
when PhraseClause
{ match_phrase: { text: { query: clause.phrase } } }
else
raise "Unexpected clause type: #{clause}"
end
end
end
class Operator
class << self
def symbol(str)
case str
when '+'
:must
when '-'
:must_not
when nil
:should
else
raise "Unknown operator: #{str}"
end
end
end
end
class TermClause
attr_reader :prefix, :operator, :term
def initialize(prefix, operator, term)
@prefix = prefix
@operator = Operator.symbol(operator)
@term = term
end
end
class PhraseClause
attr_reader :prefix, :operator, :phrase
def initialize(prefix, operator, phrase)
@prefix = prefix
@operator = Operator.symbol(operator)
@phrase = phrase
end
end
rule(clause: subtree(:clause)) do
prefix = clause[:prefix][:term].to_s if clause[:prefix]
operator = clause[:operator]&.to_s
if clause[:term]
TermClause.new(prefix, operator, clause[:term].to_s)
elsif clause[:phrase]
PhraseClause.new(prefix, operator, clause[:phrase].map { |p| p[:term].to_s }.join(' '))
else
raise "Unexpected clause type: #{clause}"
end
end
rule(query: sequence(:clauses)) { Query.new(clauses) }
end

View file

@ -33,8 +33,7 @@ class SearchService < BaseService
end end
def perform_statuses_search! def perform_statuses_search!
definition = StatusesIndex.filter(term: { searchable_by: @account.id }) definition = parsed_query.apply(StatusesIndex.filter(term: { searchable_by: @account.id }))
.query(multi_match: { type: 'most_fields', query: @query, operator: 'and', fields: %w(text text.stemmed) })
if @options[:account_id].present? if @options[:account_id].present?
definition = definition.filter(term: { account_id: @options[:account_id] }) definition = definition.filter(term: { account_id: @options[:account_id] })
@ -70,7 +69,7 @@ class SearchService < BaseService
end end
def url_query? def url_query?
@options[:type].blank? && @query =~ /\Ahttps?:\/\// @resolve && @options[:type].blank? && @query =~ /\Ahttps?:\/\//
end end
def url_resource_results def url_resource_results
@ -120,4 +119,8 @@ class SearchService < BaseService
domain_blocking_by_domain: Account.domain_blocking_map_by_domain(domains, account.id), domain_blocking_by_domain: Account.domain_blocking_map_by_domain(domains, account.id),
} }
end end
def parsed_query
SearchQueryTransformer.new.apply(SearchQueryParser.new.parse(@query))
end
end end

View file

@ -27,7 +27,7 @@ describe SearchService, type: :service do
it 'returns the empty results' do it 'returns the empty results' do
service = double(call: nil) service = double(call: nil)
allow(ResolveURLService).to receive(:new).and_return(service) allow(ResolveURLService).to receive(:new).and_return(service)
results = subject.call(@query, nil, 10) results = subject.call(@query, nil, 10, resolve: true)
expect(service).to have_received(:call).with(@query, on_behalf_of: nil) expect(service).to have_received(:call).with(@query, on_behalf_of: nil)
expect(results).to eq empty_results expect(results).to eq empty_results
@ -40,7 +40,7 @@ describe SearchService, type: :service do
service = double(call: account) service = double(call: account)
allow(ResolveURLService).to receive(:new).and_return(service) allow(ResolveURLService).to receive(:new).and_return(service)
results = subject.call(@query, nil, 10) results = subject.call(@query, nil, 10, resolve: true)
expect(service).to have_received(:call).with(@query, on_behalf_of: nil) expect(service).to have_received(:call).with(@query, on_behalf_of: nil)
expect(results).to eq empty_results.merge(accounts: [account]) expect(results).to eq empty_results.merge(accounts: [account])
end end
@ -52,7 +52,7 @@ describe SearchService, type: :service do
service = double(call: status) service = double(call: status)
allow(ResolveURLService).to receive(:new).and_return(service) allow(ResolveURLService).to receive(:new).and_return(service)
results = subject.call(@query, nil, 10) results = subject.call(@query, nil, 10, resolve: true)
expect(service).to have_received(:call).with(@query, on_behalf_of: nil) expect(service).to have_received(:call).with(@query, on_behalf_of: nil)
expect(results).to eq empty_results.merge(statuses: [status]) expect(results).to eq empty_results.merge(statuses: [status])
end end