Tools/Notes
1
0
Fork 0
mirror of https://github.com/TriliumNext/Notes.git synced 2025-11-04 15:11:31 +08:00
Code Issues Packages Projects Releases Wiki Activity

search refactoring

Browse Source
This commit is contained in:
zadam 2020-07-21 00:01:07 +02:00
parent f03e4bc7d7
commit 32ecb43b5c
12 changed files with 76 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
spec/search/lexer.spec.js
View File

@ -1,63 +1,63 @@
const lexer = require('../../src/services/search/lexer.js'); const lex = require('../../src/services/search/services/lex.js');
describe("Lexer fulltext", () => { describe("Lexer fulltext", () => {
it("simple lexing", () => { it("simple lexing", () => {
expect(lexer("hello world").fulltextTokens.map(t => t.token)) expect(lex("hello world").fulltextTokens.map(t => t.token))
.toEqual(["hello", "world"]); .toEqual(["hello", "world"]);
}); });
it("use quotes to keep words together", () => { it("use quotes to keep words together", () => {
expect(lexer("'hello world' my friend").fulltextTokens.map(t => t.token)) expect(lex("'hello world' my friend").fulltextTokens.map(t => t.token))
.toEqual(["hello world", "my", "friend"]); .toEqual(["hello world", "my", "friend"]);
expect(lexer('"hello world" my friend').fulltextTokens.map(t => t.token)) expect(lex('"hello world" my friend').fulltextTokens.map(t => t.token))
.toEqual(["hello world", "my", "friend"]); .toEqual(["hello world", "my", "friend"]);
expect(lexer('`hello world` my friend').fulltextTokens.map(t => t.token)) expect(lex('`hello world` my friend').fulltextTokens.map(t => t.token))
.toEqual(["hello world", "my", "friend"]); .toEqual(["hello world", "my", "friend"]);
}); });
it("you can use different quotes and other special characters inside quotes", () => { it("you can use different quotes and other special characters inside quotes", () => {
expect(lexer("'i can use \" or ` or #~=*' without problem").fulltextTokens.map(t => t.token)) expect(lex("'i can use \" or ` or #~=*' without problem").fulltextTokens.map(t => t.token))
.toEqual(["i can use \" or ` or #~=*", "without", "problem"]); .toEqual(["i can use \" or ` or #~=*", "without", "problem"]);
}); });
it("if quote is not ended then it's just one long token", () => { it("if quote is not ended then it's just one long token", () => {
expect(lexer("'unfinished quote").fulltextTokens.map(t => t.token)) expect(lex("'unfinished quote").fulltextTokens.map(t => t.token))
.toEqual(["unfinished quote"]); .toEqual(["unfinished quote"]);
}); });
it("parenthesis and symbols in fulltext section are just normal characters", () => { it("parenthesis and symbols in fulltext section are just normal characters", () => {
expect(lexer("what's u=p <b(r*t)h>").fulltextTokens.map(t => t.token)) expect(lex("what's u=p <b(r*t)h>").fulltextTokens.map(t => t.token))
.toEqual(["what's", "u=p", "<b(r*t)h>"]); .toEqual(["what's", "u=p", "<b(r*t)h>"]);
}); });
it("escaping special characters", () => { it("escaping special characters", () => {
expect(lexer("hello \\#\\~\\'").fulltextTokens.map(t => t.token)) expect(lex("hello \\#\\~\\'").fulltextTokens.map(t => t.token))
.toEqual(["hello", "#~'"]); .toEqual(["hello", "#~'"]);
}); });
}); });
describe("Lexer expression", () => { describe("Lexer expression", () => {
it("simple attribute existence", () => { it("simple attribute existence", () => {
expect(lexer("#label ~relation").expressionTokens.map(t => t.token)) expect(lex("#label ~relation").expressionTokens.map(t => t.token))
.toEqual(["#label", "~relation"]); .toEqual(["#label", "~relation"]);
}); });
it("simple label operators", () => { it("simple label operators", () => {
expect(lexer("#label*=*text").expressionTokens.map(t => t.token)) expect(lex("#label*=*text").expressionTokens.map(t => t.token))
.toEqual(["#label", "*=*", "text"]); .toEqual(["#label", "*=*", "text"]);
}); });
it("simple label operator with in quotes and without", () => { it("simple label operator with in quotes and without", () => {
expect(lexer("#label*=*'text'").expressionTokens) expect(lex("#label*=*'text'").expressionTokens)
.toEqual([ .toEqual([
{token: "#label", inQuotes: false}, {token: "#label", inQuotes: false},
{token: "*=*", inQuotes: false}, {token: "*=*", inQuotes: false},
{token: "text", inQuotes: true} {token: "text", inQuotes: true}
]); ]);
expect(lexer("#label*=*text").expressionTokens) expect(lex("#label*=*text").expressionTokens)
.toEqual([ .toEqual([
{token: "#label", inQuotes: false}, {token: "#label", inQuotes: false},
{token: "*=*", inQuotes: false}, {token: "*=*", inQuotes: false},
@ -66,35 +66,35 @@ describe("Lexer expression", () => {
}); });
it("complex expressions with and, or and parenthesis", () => { it("complex expressions with and, or and parenthesis", () => {
expect(lexer(`# (#label=text OR #second=text) AND ~relation`).expressionTokens.map(t => t.token)) expect(lex(`# (#label=text OR #second=text) AND ~relation`).expressionTokens.map(t => t.token))
.toEqual(["#", "(", "#label", "=", "text", "or", "#second", "=", "text", ")", "and", "~relation"]); .toEqual(["#", "(", "#label", "=", "text", "or", "#second", "=", "text", ")", "and", "~relation"]);
}); });
it("dot separated properties", () => { it("dot separated properties", () => {
expect(lexer(`# ~author.title = 'Hugh Howey' AND note.'book title' = 'Silo'`).expressionTokens.map(t => t.token)) expect(lex(`# ~author.title = 'Hugh Howey' AND note.'book title' = 'Silo'`).expressionTokens.map(t => t.token))
.toEqual(["#", "~author", ".", "title", "=", "hugh howey", "and", "note", ".", "book title", "=", "silo"]); .toEqual(["#", "~author", ".", "title", "=", "hugh howey", "and", "note", ".", "book title", "=", "silo"]);
}); });
it("negation of label and relation", () => { it("negation of label and relation", () => {
expect(lexer(`#!capital ~!neighbor`).expressionTokens.map(t => t.token)) expect(lex(`#!capital ~!neighbor`).expressionTokens.map(t => t.token))
.toEqual(["#!capital", "~!neighbor"]); .toEqual(["#!capital", "~!neighbor"]);
}); });
it("negation of sub-expression", () => { it("negation of sub-expression", () => {
expect(lexer(`# not(#capital) and note.noteId != "root"`).expressionTokens.map(t => t.token)) expect(lex(`# not(#capital) and note.noteId != "root"`).expressionTokens.map(t => t.token))
.toEqual(["#", "not", "(", "#capital", ")", "and", "note", ".", "noteid", "!=", "root"]); .toEqual(["#", "not", "(", "#capital", ")", "and", "note", ".", "noteid", "!=", "root"]);
}); });
}); });
describe("Lexer invalid queries and edge cases", () => { describe("Lexer invalid queries and edge cases", () => {
it("concatenated attributes", () => { it("concatenated attributes", () => {
expect(lexer("#label~relation").expressionTokens.map(t => t.token)) expect(lex("#label~relation").expressionTokens.map(t => t.token))
.toEqual(["#label", "~relation"]); .toEqual(["#label", "~relation"]);
}); });
it("spaces in attribute names and values", () => { it("spaces in attribute names and values", () => {
// invalid but should be reported by parser as an error // invalid but should be reported by parser as an error
expect(lexer(`#'long label'="hello o' world" ~'long relation'`).expressionTokens.map(t => t.token)) expect(lex(`#'long label'="hello o' world" ~'long relation'`).expressionTokens.map(t => t.token))
.toEqual(["#long label", "=", "hello o' world", "~long relation"]); .toEqual(["#long label", "=", "hello o' world", "~long relation"]);
}); });
}); });

4
spec/search/parens.spec.js
View File

@ -1,11 +1,11 @@
const parens = require('../../src/services/search/parens.js'); const handleParens = require('../../src/services/search/services/handle_parens.js');
describe("Parens handler", () => { describe("Parens handler", () => {
it("handles parens", () => { it("handles parens", () => {
const input = ["(", "hello", ")", "and", "(", "(", "pick", "one", ")", "and", "another", ")"] const input = ["(", "hello", ")", "and", "(", "(", "pick", "one", ")", "and", "another", ")"]
.map(token => ({token})); .map(token => ({token}));
expect(parens(input)) expect(handleParens(input))
.toEqual([ .toEqual([
[ [
{token: "hello"} {token: "hello"}

32
spec/search/parser.spec.js
View File

@ -1,5 +1,5 @@
const ParsingContext = require("../../src/services/search/parsing_context.js"); const ParsingContext = require("../../src/services/search/parsing_context.js");
const parser = require('../../src/services/search/parser.js'); const parse = require('../../src/services/search/services/parse.js');
function tokens(...args) { function tokens(...args) {
return args.map(arg => { return args.map(arg => {
@ -17,7 +17,7 @@ function tokens(...args) {
describe("Parser", () => { describe("Parser", () => {
it("fulltext parser without content", () => { it("fulltext parser without content", () => {
const rootExp = parser({ const rootExp = parse({
fulltextTokens: tokens("hello", "hi"), fulltextTokens: tokens("hello", "hi"),
expressionTokens: [], expressionTokens: [],
parsingContext: new ParsingContext({includeNoteContent: false}) parsingContext: new ParsingContext({includeNoteContent: false})
@ -28,7 +28,7 @@ describe("Parser", () => {
}); });
it("fulltext parser with content", () => { it("fulltext parser with content", () => {
const rootExp = parser({ const rootExp = parse({
fulltextTokens: tokens("hello", "hi"), fulltextTokens: tokens("hello", "hi"),
expressionTokens: [], expressionTokens: [],
parsingContext: new ParsingContext({includeNoteContent: true}) parsingContext: new ParsingContext({includeNoteContent: true})
@ -48,7 +48,7 @@ describe("Parser", () => {
}); });
it("simple label comparison", () => { it("simple label comparison", () => {
const rootExp = parser({ const rootExp = parse({
fulltextTokens: [], fulltextTokens: [],
expressionTokens: tokens("#mylabel", "=", "text"), expressionTokens: tokens("#mylabel", "=", "text"),
parsingContext: new ParsingContext() parsingContext: new ParsingContext()
@ -61,7 +61,7 @@ describe("Parser", () => {
}); });
it("simple attribute negation", () => { it("simple attribute negation", () => {
let rootExp = parser({ let rootExp = parse({
fulltextTokens: [], fulltextTokens: [],
expressionTokens: tokens("#!mylabel"), expressionTokens: tokens("#!mylabel"),
parsingContext: new ParsingContext() parsingContext: new ParsingContext()
@ -72,7 +72,7 @@ describe("Parser", () => {
expect(rootExp.subExpression.attributeType).toEqual("label"); expect(rootExp.subExpression.attributeType).toEqual("label");
expect(rootExp.subExpression.attributeName).toEqual("mylabel"); expect(rootExp.subExpression.attributeName).toEqual("mylabel");
rootExp = parser({ rootExp = parse({
fulltextTokens: [], fulltextTokens: [],
expressionTokens: tokens("~!myrelation"), expressionTokens: tokens("~!myrelation"),
parsingContext: new ParsingContext() parsingContext: new ParsingContext()
@ -85,7 +85,7 @@ describe("Parser", () => {
}); });
it("simple label AND", () => { it("simple label AND", () => {
const rootExp = parser({ const rootExp = parse({
fulltextTokens: [], fulltextTokens: [],
expressionTokens: tokens("#first", "=", "text", "and", "#second", "=", "text"), expressionTokens: tokens("#first", "=", "text", "and", "#second", "=", "text"),
parsingContext: new ParsingContext(true) parsingContext: new ParsingContext(true)
@ -102,7 +102,7 @@ describe("Parser", () => {
}); });
it("simple label AND without explicit AND", () => { it("simple label AND without explicit AND", () => {
const rootExp = parser({ const rootExp = parse({
fulltextTokens: [], fulltextTokens: [],
expressionTokens: tokens("#first", "=", "text", "#second", "=", "text"), expressionTokens: tokens("#first", "=", "text", "#second", "=", "text"),
parsingContext: new ParsingContext() parsingContext: new ParsingContext()
@ -119,7 +119,7 @@ describe("Parser", () => {
}); });
it("simple label OR", () => { it("simple label OR", () => {
const rootExp = parser({ const rootExp = parse({
fulltextTokens: [], fulltextTokens: [],
expressionTokens: tokens("#first", "=", "text", "or", "#second", "=", "text"), expressionTokens: tokens("#first", "=", "text", "or", "#second", "=", "text"),
parsingContext: new ParsingContext() parsingContext: new ParsingContext()
@ -136,7 +136,7 @@ describe("Parser", () => {
}); });
it("fulltext and simple label", () => { it("fulltext and simple label", () => {
const rootExp = parser({ const rootExp = parse({
fulltextTokens: tokens("hello"), fulltextTokens: tokens("hello"),
expressionTokens: tokens("#mylabel", "=", "text"), expressionTokens: tokens("#mylabel", "=", "text"),
parsingContext: new ParsingContext() parsingContext: new ParsingContext()
@ -153,7 +153,7 @@ describe("Parser", () => {
}); });
it("label sub-expression", () => { it("label sub-expression", () => {
const rootExp = parser({ const rootExp = parse({
fulltextTokens: [], fulltextTokens: [],
expressionTokens: tokens("#first", "=", "text", "or", tokens("#second", "=", "text", "and", "#third", "=", "text")), expressionTokens: tokens("#first", "=", "text", "or", tokens("#second", "=", "text", "and", "#third", "=", "text")),
parsingContext: new ParsingContext() parsingContext: new ParsingContext()
@ -176,11 +176,11 @@ describe("Parser", () => {
}); });
}); });
describe("Invalid tokens", () => { describe("Invalid expressions", () => {
it("incomplete comparison", () => { it("incomplete comparison", () => {
const parsingContext = new ParsingContext(); const parsingContext = new ParsingContext();
parser({ parse({
fulltextTokens: [], fulltextTokens: [],
expressionTokens: tokens("#first", "="), expressionTokens: tokens("#first", "="),
parsingContext parsingContext
@ -192,7 +192,7 @@ describe("Invalid tokens", () => {
it("comparison between labels is impossible", () => { it("comparison between labels is impossible", () => {
let parsingContext = new ParsingContext(); let parsingContext = new ParsingContext();
parser({ parse({
fulltextTokens: [], fulltextTokens: [],
expressionTokens: tokens("#first", "=", "#second"), expressionTokens: tokens("#first", "=", "#second"),
parsingContext parsingContext
@ -202,7 +202,7 @@ describe("Invalid tokens", () => {
parsingContext = new ParsingContext(); parsingContext = new ParsingContext();
parser({ parse({
fulltextTokens: [], fulltextTokens: [],
expressionTokens: tokens("#first", "=", "note", ".", "relations", "second"), expressionTokens: tokens("#first", "=", "note", ".", "relations", "second"),
parsingContext parsingContext
@ -210,7 +210,7 @@ describe("Invalid tokens", () => {
expect(parsingContext.error).toEqual(`Error near token "note", it's possible to compare with constant only.`); expect(parsingContext.error).toEqual(`Error near token "note", it's possible to compare with constant only.`);
const rootExp = parser({ const rootExp = parse({
fulltextTokens: [], fulltextTokens: [],
expressionTokens: [ expressionTokens: [
{ token: "#first", inQuotes: false }, { token: "#first", inQuotes: false },

2
spec/search/search.spec.js
View File

@ -1,4 +1,4 @@
const searchService = require('../../src/services/search/search.js'); const searchService = require('../../src/services/search/services/search.js');
const Note = require('../../src/services/note_cache/entities/note.js'); const Note = require('../../src/services/note_cache/entities/note.js');
const Branch = require('../../src/services/note_cache/entities/branch.js'); const Branch = require('../../src/services/note_cache/entities/branch.js');
const Attribute = require('../../src/services/note_cache/entities/attribute.js'); const Attribute = require('../../src/services/note_cache/entities/attribute.js');

2
src/routes/api/autocomplete.js
View File

@ -1,7 +1,7 @@
"use strict"; "use strict";
const noteCacheService = require('../../services/note_cache/note_cache_service'); const noteCacheService = require('../../services/note_cache/note_cache_service');
const searchService = require('../../services/search/search'); const searchService = require('../../services/search/services/search.js');
const repository = require('../../services/repository'); const repository = require('../../services/repository');
const log = require('../../services/log'); const log = require('../../services/log');
const utils = require('../../services/utils'); const utils = require('../../services/utils');

2
src/routes/api/search.js
View File

@ -4,7 +4,7 @@ const repository = require('../../services/repository');
const noteCacheService = require('../../services/note_cache/note_cache.js'); const noteCacheService = require('../../services/note_cache/note_cache.js');
const log = require('../../services/log'); const log = require('../../services/log');
const scriptService = require('../../services/script'); const scriptService = require('../../services/script');
const searchService = require('../../services/search/search'); const searchService = require('../../services/search/services/search.js');
function searchNotes(req) { function searchNotes(req) {
const {count, results} = searchService.searchTrimmedNotes(req.params.searchString); const {count, results} = searchService.searchTrimmedNotes(req.params.searchString);

2
src/services/backend_script_api.js
View File

@ -12,7 +12,7 @@ const dayjs = require('dayjs');
const cloningService = require('./cloning'); const cloningService = require('./cloning');
const ws = require('./ws.js'); const ws = require('./ws.js');
const appInfo = require('./app_info'); const appInfo = require('./app_info');
const searchService = require('./search/search'); const searchService = require('./search/services/search.js');
/** /**
* This is the main backend API interface for scripts. It's published in the local "api" object. * This is the main backend API interface for scripts. It's published in the local "api" object.

0
src/services/search/comparator_builder.js → src/services/search/services/build_comparator.js
View File

6
src/services/search/parens.js → src/services/search/services/handle_parens.js
View File

@ -1,7 +1,7 @@
/** /**
* This will create a recursive object from list of tokens - tokens between parenthesis are grouped in a single array * This will create a recursive object from list of tokens - tokens between parenthesis are grouped in a single array
*/ */
function parens(tokens) { function handle_parens(tokens) {
if (tokens.length === 0) { if (tokens.length === 0) {
return []; return [];
} }
@ -34,10 +34,10 @@ function parens(tokens) {
tokens = [ tokens = [
...tokens.slice(0, leftIdx), ...tokens.slice(0, leftIdx),
parens(tokens.slice(leftIdx + 1, rightIdx)), handle_parens(tokens.slice(leftIdx + 1, rightIdx)),
...tokens.slice(rightIdx + 1) ...tokens.slice(rightIdx + 1)
]; ];
} }
} }
module.exports = parens; module.exports = handle_parens;

4
src/services/search/lexer.js → src/services/search/services/lex.js
View File

@ -1,4 +1,4 @@
function lexer(str) { function lex(str) {
str = str.toLowerCase(); str = str.toLowerCase();
const fulltextTokens = []; const fulltextTokens = [];
@ -130,4 +130,4 @@ function lexer(str) {
} }
} }
module.exports = lexer; module.exports = lex;

32
src/services/search/parser.js → src/services/search/services/parse.js
View File

@ -1,21 +1,21 @@
"use strict"; "use strict";
const AndExp = require('./expressions/and'); const AndExp = require('../expressions/and.js');
const OrExp = require('./expressions/or'); const OrExp = require('../expressions/or.js');
const NotExp = require('./expressions/not'); const NotExp = require('../expressions/not.js');
const ChildOfExp = require('./expressions/child_of'); const ChildOfExp = require('../expressions/child_of.js');
const DescendantOfExp = require('./expressions/descendant_of'); const DescendantOfExp = require('../expressions/descendant_of.js');
const ParentOfExp = require('./expressions/parent_of'); const ParentOfExp = require('../expressions/parent_of.js');
const RelationWhereExp = require('./expressions/relation_where'); const RelationWhereExp = require('../expressions/relation_where.js');
const PropertyComparisonExp = require('./expressions/property_comparison'); const PropertyComparisonExp = require('../expressions/property_comparison.js');
const AttributeExistsExp = require('./expressions/attribute_exists'); const AttributeExistsExp = require('../expressions/attribute_exists.js');
const LabelComparisonExp = require('./expressions/label_comparison'); const LabelComparisonExp = require('../expressions/label_comparison.js');
const NoteCacheFulltextExp = require('./expressions/note_cache_fulltext'); const NoteCacheFulltextExp = require('../expressions/note_cache_fulltext.js');
const NoteContentProtectedFulltextExp = require('./expressions/note_content_protected_fulltext'); const NoteContentProtectedFulltextExp = require('../expressions/note_content_protected_fulltext.js');
const NoteContentUnprotectedFulltextExp = require('./expressions/note_content_unprotected_fulltext'); const NoteContentUnprotectedFulltextExp = require('../expressions/note_content_unprotected_fulltext.js');
const OrderByAndLimitExp = require('./expressions/order_by_and_limit'); const OrderByAndLimitExp = require('../expressions/order_by_and_limit.js');
const comparatorBuilder = require('./comparator_builder'); const comparatorBuilder = require('./build_comparator.js');
const ValueExtractor = require('./value_extractor'); const ValueExtractor = require('../value_extractor.js');
function getFulltext(tokens, parsingContext) { function getFulltext(tokens, parsingContext) {
tokens = tokens.map(t => t.token); tokens = tokens.map(t => t.token);

28
src/services/search/search.js → src/services/search/services/search.js
View File

@ -1,16 +1,16 @@
"use strict"; "use strict";
const lexer = require('./lexer'); const lex = require('./lex.js');
const parens = require('./parens'); const handleParens = require('./handle_parens.js');
const parser = require('./parser'); const parse = require('./parse.js');
const NoteSet = require("./note_set"); const NoteSet = require("../note_set.js");
const SearchResult = require("./search_result"); const SearchResult = require("../search_result.js");
const ParsingContext = require("./parsing_context"); const ParsingContext = require("../parsing_context.js");
const noteCache = require('../note_cache/note_cache'); const noteCache = require('../../note_cache/note_cache.js');
const noteCacheService = require('../note_cache/note_cache_service'); const noteCacheService = require('../../note_cache/note_cache_service.js');
const hoistedNoteService = require('../hoisted_note'); const hoistedNoteService = require('../../hoisted_note.js');
const repository = require('../repository'); const repository = require('../../repository.js');
const utils = require('../utils'); const utils = require('../../utils.js');
/** /**
* @param {Expression} expression * @param {Expression} expression
@ -51,10 +51,10 @@ function findNotesWithExpression(expression) {
} }
function parseQueryToExpression(query, parsingContext) { function parseQueryToExpression(query, parsingContext) {
const {fulltextTokens, expressionTokens} = lexer(query); const {fulltextTokens, expressionTokens} = lex(query);
const structuredExpressionTokens = parens(expressionTokens); const structuredExpressionTokens = handleParens(expressionTokens);
const expression = parser({ const expression = parse({
fulltextTokens, fulltextTokens,
expressionTokens: structuredExpressionTokens, expressionTokens: structuredExpressionTokens,
parsingContext parsingContext