# This file is part of rhizi, a collaborative knowledge graph editor. # Copyright (C) 2014-2015 Rhizi # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published # by the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Affero General Public License for more details. # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . """ Cypher language parser - clear logical separation between lexing/parsing still missing - e_XXX class object should be considered internal """ from collections import defaultdict import logging import re # # Tokens # tok__quote__backquote = '`' # used to quote labels tok__quote__singlequote = '\'' # used to quote values tok_set__quote = [tok__quote__backquote, tok__quote__singlequote] tok_set__paren = ['(', ')', '[', ']', '{', '}'] tok_set__kw__write = [ 'create', 'set', 'delete', 'remove', 'foreach' ] tok_set__kw__supported = [ # order critical 'create', 'optional match', 'match', ] tok_set__kw__unsupported = [ # these are treated as generic clauses, capturing until the next keyword 'delete', 'foreach', 'limit', 'merge', 'order by', 'return', 'remove', 'set', 'skip', 'start', 'union', 'where', 'with', ] tok_set__kw__all = tok_set__kw__supported + tok_set__kw__unsupported log = logging.getLogger('rhizi') # # Parse Tree # class pt_abs_node(object): """ Abstract parse tree node """ def __init__(self): self.value = None self.parent = None def __iter__(self): return iter([]) # enable tree walking over pt_abs_node leaf nodes def collapse(self, n_type): return self.__collapse_common([n_type]) def collapse_set(self, n_type_set): return self.__collapse_common(n_type_set) def __collapse_common(self, n_type_set): def type_check(n): for n_type in n_type_set: if isinstance(n, n_type): return True return False ret = self while not type_check(ret): ret = ret.parent assert type_check(ret) return ret def rotate__pin_under_new_parent(self, n_type): # pin current node under a new parent node parent_cur = self.parent parent_new = n_type() parent_new.parent = parent_cur parent_new.sub_exp_set.append(self) for i in range(0, len(parent_cur.sub_exp_set)): if self != self.parent.sub_exp_set[i]: # lookup self in parent child set continue parent_cur.sub_exp_set[i] = parent_new self.parent = parent_new return parent_new def spawn_sibling(self, n_type=None, *args, **kwargs): if not n_type: # by default spawn siblig of the same type as self n_type = self.__class__ return self.parent.spawn_child(n_type, *args, **kwargs) def spawn_sibling__adjacent(self, n_type=None, *args, **kwargs): """ spawn sibling, immediately following self in the sub-exp order """ if not n_type: # by default spawn siblig of the same type as self n_type = self.__class__ i = 0 # position of self in parent sub-exp set p_exp_set = self.parent.sub_exp_set for exp in p_exp_set: if exp == self: break i += 1 child_node = n_type(*args, **kwargs) child_node.parent = self p_exp_set.insert(i + 1, child_node) return child_node def str__body(self): return self.value if self.value else '' def str__tok_open(self): return '' def str__tok_close(self): return '' class pt_abs_composite_node(pt_abs_node): def __init__(self): super(pt_abs_composite_node, self).__init__() self.sub_exp_set = [] def __repr__(self): return self.str__struct_tree() def __str__cypher_query(self): def f_pre(n, ctx): ctx[0] += n.str__tok_open() def f_visit(n, ctx, _depth): ctx[0] += n.str__body() def f_post(n, ctx): ctx[0] += n.str__tok_close() def f_recurse(n, ctx, depth): return True def f_inter(parent, n, n_next, ctx): ctx[0] += parent.str__tok_sibling_delim(n, n_next) ctx = [''] self.tree_walk__pre(f_pre, f_visit, f_post, f_recurse, f_inter, ctx) return ''.join(ctx) def __str__struct_tree(self, depth_delim=''): def f_visit(n, ctx, depth): line = '%s%s' % (depth_delim * depth, n.__class__.__name__) if n.value: line += ': \'%s\'' % (n.value) ctx += [line] ctx = [] self.tree_walk__pre(f_visit=f_visit, ctx=ctx) return ctx def __iter__(self): # iterate over sub nodes for exp in self.sub_exp_set: yield exp def assert_child_spawn_type(self, n_type): # provide hook for child spawning assertions pass def str__struct_tree(self): return '\n'.join(self.__str__struct_tree(depth_delim='. ')) def str__cypher_query(self): """ @return: a Cypher string representation of this parse tree """ return self.__str__cypher_query() def str__tok_sibling_delim(self, sib_a=None, sib_b=None): return '' def spawn_child(self, n_type, *args, **kwargs): """ Spawn child node: - set child's parent ref. to this object - add child node as sub exp. This method should be overridden by subclasses """ self.assert_child_spawn_type(n_type) child_node = n_type(*args, **kwargs) child_node.parent = self self.sub_exp_set.append(child_node) return child_node def sub_exp_set_by_type(self, exp_type_or_set, recurse=False): """ @param exp_type_or_set: type or set of types to check against using isinstance() @param recurse: whether to recursively search within sub expressions @return: set of matched sub expressions who match the given type set, possibly empty """ if isinstance(exp_type_or_set, list): exp_type_set = exp_type_or_set else: exp_type_set = [exp_type_or_set] def f_visit(n, ctx, depth): for exp_type in exp_type_set: if isinstance(n, exp_type): ctx += [n] if False == recurse: f_recurse = lambda _n, _ctx, depth: depth <= 1 # [!] visit child nodes only if True == recurse: f_recurse = lambda _n, _ctx, _depth: True # recurse return self.tree_walk__pre(f_visit=f_visit, f_recurse=f_recurse, ctx=[]) def tree_walk__pre(self, f_pre=lambda n, ctx: None, f_visit=lambda n, ctx, depth: None, f_post=lambda n, ctx: None, f_recurse=lambda n, ctx, depth: True, f_inter=lambda n, e_first, e_next, ctx: None, # called between each sibling pair ctx=None): # tree walk, preorder def walk__pre_rec(n, depth): f_pre(n, ctx) f_visit(n, ctx, depth) if f_recurse(n, ctx, depth): sub_n_set = [e for e in n] sub_exp_set_len = len(sub_n_set) for i in range(0, sub_exp_set_len): e = sub_n_set[i] walk__pre_rec(e, depth + 1) if i + 1 < sub_exp_set_len: # call f_inter in between siblings e_first = e e_next = sub_n_set[i + 1] f_inter(n, e_first, e_next, ctx) f_post(n, ctx) walk__pre_rec(self, 0) return ctx class e_keyword(pt_abs_node): def __init__(self, keyword): super(e_keyword, self).__init__() self.value = keyword def str__tok_close(self): return ' ' class e_value(pt_abs_node): def __init__(self): super(e_value, self).__init__() self.quoted = False self.quote_tok = None @classmethod def rgx__unquoted(self, g_name='value'): return '(?P<%s>[\w\d/|,_]+(\*\d*\.\.\d*)?)' % (g_name) @classmethod def rgx__quoted(self, g_name='value', quote_tok='\''): return '%s(?P<%s>[\w\d_/,|\-\s]+(\*\d*\.\.\d*)?)%s' % (quote_tok, g_name, quote_tok) def str__tok_open(self): if self.quoted: return self.quote_tok return '' def str__tok_close(self): if self.quoted: return self.quote_tok return '' class e_ident(pt_abs_node): # identifier def __init__(self): super(e_ident, self).__init__() @classmethod def rgx(self, g_name='ident'): return '(?P<%s>[\w_]+)' % (g_name) class e_param(e_ident): """ Cypher parameter @TODO: implement as bound e_ident """ def __init__(self): super(e_param, self).__init__() def str__tok_open(self): return '{' def str__tok_close(self): return '}' class e_function(pt_abs_node): # identifier def __init__(self): super(e_function, self).__init__() class e_set(pt_abs_composite_node): # identifier """ expression who's all sub nodes are of the same type: - may contain sub-expressions of different type: eg. 'match (n), ()-[r]-()' """ def __init__(self): super(e_set, self).__init__() def str__tok_sibling_delim(self, sib_a=None, sib_b=None): return ', ' class op_dot(pt_abs_composite_node): def str__tok_sibling_delim(self, sib_a=None, sib_b=None): return '.' # # Cypher patterns # class pt_root(pt_abs_composite_node): """ parse tree root node: - support keyword-to-clause mapping """ def __init__(self): super(pt_root, self).__init__() def assert_child_spawn_type(self, n_type): assert issubclass(n_type, e_clause) def index__kw_to_clause_set(self): ret = defaultdict(list) for e_clause in self: ret[e_clause.keyword_str].append(e_clause) return ret def clause_set_by_kw(self, keyword): """ select query clause set by keyword: eg. 'create' -> [e_clause, ...] """ return self.index__kw_to_clause_set()[keyword] def str__tok_sibling_delim(self, sib_a=None, sib_b=None): return ' ' class e_clause(pt_abs_composite_node): def __init__(self): super(e_clause, self).__init__() @property def keyword_str(self): ret = self.sub_exp_set[0] assert isinstance(ret, e_keyword) return ret.value class e_clause__match(e_clause): def __init__(self): super(e_clause__match, self).__init__() self.is_optional = False def assert_child_spawn_type(self, n_type): assert n_type in [e_keyword, p_node, p_path], n_type class e_clause__create(e_clause): def __init__(self): super(e_clause__create, self).__init__() def assert_child_spawn_type(self, n_type): assert n_type in [e_keyword, p_node, p_path] class e_clause__where(e_clause): def __init__(self): super(e_clause__where, self).__init__() @property def condition_value(self): assert len(self.sub_exp_set) == 2 return self.sub_exp_set[1].value def set_condition(self, cond_str): assert len(self.sub_exp_set) == 2 self.sub_exp_set[1].value = cond_str class e_attr_set(e_set): # attr-set pattern: {k:v, ...} def __init__(self): super(e_attr_set, self).__init__() def str__tok_open(self): return '{' def str__tok_close(self): return '}' def str__tok_sibling_delim(self, sib_a=None, sib_b=None): return ', ' def assert_child_spawn_type(self, n_type): assert n_type in [e_ident, e_kv_pair] class e_label_set(e_set): def __init__(self): super(e_label_set, self).__init__() def str__tok_open(self): return ':' # required when no identifier is present, eg. '()-[:A]-()' def str__tok_sibling_delim(self, sib_a=None, sib_b=None): return ':' def assert_child_spawn_type(self, n_type): assert n_type in [e_value] def add_label(self, label): lbl = self.spawn_child(e_value) lbl.value = label return lbl class e_kv_pair(pt_abs_composite_node): # key value pair def __init__(self): super(e_kv_pair, self).__init__() def assert_child_spawn_type(self, n_type): assert n_type in [e_value, e_ident, e_param] def spawn_child(self, n_type, *args, **kwargs): ret = pt_abs_composite_node.spawn_child(self, n_type, *args, **kwargs) assert len(self.sub_exp_set) <= 2 return ret def str__tok_sibling_delim(self, sib_a=None, sib_b=None): return ': ' def is_set__key(self): return len(self.sub_exp_set) > 0 and self.sub_exp_set[0].__class__ == e_ident def is_set__value(self): return len(self.sub_exp_set) == 2 class p_node(pt_abs_composite_node): # node pattern: '(...)' """ node pattern """ def __init__(self): super(p_node, self).__init__() def str__tok_sibling_delim(self, sib_a=None, sib_b=None): if isinstance(sib_a, e_ident) and isinstance(sib_b, e_label_set): return '' # handled by label_set return ' ' def str__tok_open(self): return '(' def str__tok_close(self): return ')' @classmethod def rgx(self, g_name): return '\((?P<%s>[^\(\)]*?)\)' % (g_name) @property def label_set(self): sub_exp_set = self.sub_exp_set_by_type(e_label_set) if 0 == len (sub_exp_set): return None assert 1 == len(sub_exp_set) return sub_exp_set.pop() def spawn_label_set(self): """ spawn label set adjacent to e_ident """ assert not self.label_set, 'label set already present' assert len(self.sub_exp_set) > 0 assert e_ident == self.sub_exp_set[0].__class__ e_id = self.sub_exp_set[0] lbl_set = e_id.spawn_sibling__adjacent(e_label_set) return lbl_set class p_rel(pt_abs_composite_node): # rel pattern: '[...]' """ relationship pattern """ def __init__(self): super(p_rel, self).__init__() self.is_directional = False def str__tok_sibling_delim(self, sib_a=None, sib_b=None): if isinstance(sib_a, e_ident) and isinstance(sib_b, e_label_set): return '' # handled by label_set return ' ' def str__tok_open(self): return '-[' def str__tok_close(self): if self.is_directional == True: return ']->' else: return ']-' @property def type(self): """ The single element equivalent of node label sets """ val_sub_n_set = self.sub_exp_set_by_type(e_value) assert len(val_sub_n_set) >= 1 if len(val_sub_n_set) == 1: return val_sub_n_set[0] return None @classmethod def rgx(self, g_name): return '-\[(?P<%s>[^\[\]]*?)\]->?' % (g_name) class p_path(pt_abs_composite_node): # path pattern: '()-[]-()' """ path pattern """ def __init__(self): super(p_path, self).__init__() class Cypher_Parser(object): """ Neo4J Cypher language parser - conventions: - read_xxx functions: should adjust argument node without modifying parse tree - parse_xxx functions: - caller should spawn correct current node - should handle current node closing - default parse case should recurse up via parent node """ def __init__(self): self.rgx__suffix = '(?P.*)$' # suffix regular expression, '$' terminated def parse_expression(self, input): """ @return: Cypher parse tree """ root_node = pt_root() try: self.__parse(input, root_node) except Exception as e: log.exception(e) log.debug('parse tree:\n%s\nq: %s' % (root_node.str__struct_tree(), input)) raise e return root_node def __match(self, rgx, input_str, flags=0): # error handling match ret = re.match(rgx, input_str, flags) if not ret: raise Exception('cypher parse error: rgx match failure: rgx: %s, input: "%s"' % (rgx, input_str)) return ret def first_sibling_root(self, n): ret = n while not ret.__class__ in [e_attr_set, e_clause]: ret = ret.parent return ret def cont(self, n): if isinstance(n, e_label_set): pass def read__e_ident(self, input, n_ident): rgx_ident = r'^%s%s' % (e_ident.rgx(), self.rgx__suffix) m = self.__match(rgx_ident, input, re.UNICODE) n_ident.value = m.group('ident') return m.group('suffix') def read__e_value(self, input, n_value): if input[0] in tok_set__quote: # quoted value quote_tok = input[0] rgx_value = r'^%s%s' % (e_value.rgx__quoted(quote_tok=quote_tok), self.rgx__suffix) n_value.quoted = True n_value.quote_tok = quote_tok else: # non quoted value rgx_value = r'^%s%s' % (e_value.rgx__unquoted(), self.rgx__suffix) m = self.__match(rgx_value, input, re.UNICODE) n_value.value = m.group('value') return m.group('suffix') def parse__e_attr_set(self, input, n_cur): if ' ' == input[0]: return self.__parse(input[1:], n_cur) # consume if '{' == input[0]: # open param n_cur = n_cur.spawn_child(e_param) return self.__parse(input[1:], n_cur) if '}' == input[0]: # close attr-set n_cur = n_cur.collapse(e_attr_set).parent return self.__parse(input[1:], n_cur) rgx_attr_set_or_param = r'^((%s:)|(%s))' % (e_ident.rgx('kv_pair__key'), e_ident.rgx('ident')) m = self.__match(rgx_attr_set_or_param, input, re.UNICODE) if m.group('ident'): n_cur = n_cur.spawn_child(e_ident) return self.__parse(input, n_cur) if m.group('kv_pair__key'): n_cur = n_cur.spawn_child(e_kv_pair) return self.__parse(input, n_cur) n_cur = n_cur.parent return self.__parse(input, n_cur) def parse__e_clause_create_or_match(self, input, n_cur): if ' ' == input[0]: return self.parse__e_clause_create_or_match(input[1:], n_cur) # consume if '(' == input[0]: return self.parse__node_or_rel(input, n_cur) # open node or path if ',' == input[0]: # open sibling assert len(n_cur.sub_exp_set) >= 2 and e_keyword == n_cur.sub_exp_set[0].__class__ last_child = n_cur.sub_exp_set[-1] assert last_child.__class__ in [p_path, p_node] n_cur = last_child.rotate__pin_under_new_parent(e_set) return self.__parse(input[1:], n_cur) n_cur = n_cur.parent return self.__parse(input, n_cur) def parse__e_clause__common(self, input, n_cur): assert len(n_cur.sub_exp_set) == 2 assert n_cur.sub_exp_set[1].__class__ == e_value n_val = n_cur.sub_exp_set[1] for kw in tok_set__kw__all: if input.startswith(kw + ' '): n_val.value = n_val.value.strip() # trim before handling new kw n_cur = n_cur.parent return self.__parse(input, n_cur) n_val.value += input[0] if len(input) == 1: # trim on EOI n_val.value = n_val.value.strip() return self.__parse(input[1:], n_cur) def parse__e_ident(self, input, n_cur): suffix = self.read__e_ident(input, n_cur) n_cur = n_cur.parent return self.__parse(suffix, n_cur) def parse__e_kv_pair(self, input, n_cur): if ' ' == input[0]: return self.__parse(input[1:], n_cur) # consume if ',' == input[0]: # open sibling n_cur = n_cur.spawn_sibling() return self.__parse(input[1:], n_cur) if ':' == input[0]: # open sibling return self.parse__e_val_or_param(input[1:], n_cur) if not n_cur.is_set__value(): # kv_pair key yet to be set n_cur = n_cur.spawn_child(e_ident) return self.__parse(input, n_cur) if n_cur.is_set__value() and n_cur.is_set__key(): n_cur = n_cur.parent return self.__parse(input, n_cur) # collapse def parse__e_val_or_param(self, input, n_cur): if ' ' == input[0]: return self.parse__e_val_or_param(input[1:], n_cur) # consume if '{' == input[0]: n_cur = n_cur.spawn_child(e_param) return self.parse__e_param(input[1:], n_cur) else: n_cur = n_cur.spawn_child(e_value) return self.__parse(input, n_cur) assert False def parse__e_label_set(self, input, n_cur): if ' ' == input[0]: # close label set return self.__parse(input[1:], n_cur.collapse_set([p_rel, p_node])) if ':' == input[0]: # append to label set n_cur = n_cur.spawn_child(e_value) return self.__parse(input[1:], n_cur) if input[0] in [')', ']']: n_cur = n_cur.parent return self.__parse(input, n_cur) assert False def parse__e_param(self, input, n_cur): if input.startswith('}.'): n_cur = n_cur.rotate__pin_under_new_parent(op_dot) return self.__parse(input[2:], n_cur) if '}' == input[0]: # close param n_cur = n_cur.parent return self.__parse(input[1:], n_cur) suffix = self.read__e_ident(input, n_cur) return self.__parse(suffix, n_cur) def parse__e_set(self, input, n_cur): if ' ' == input[0]: return self.__parse(input[1:], n_cur) # consume if '(' == input[0]: # open node or path return self.parse__node_or_rel(input, n_cur) n_cur = n_cur.parent return self.__parse(input, n_cur) def parse__e_value(self, input, n_cur): if ' ' == input[0]: return self.__parse(input[1:], n_cur) # consume suffix = self.read__e_value(input, n_cur) n_cur = n_cur.parent return self.__parse(suffix, n_cur) def parse__node_or_rel(self, input, n_cur): if ' ' == input[0]: return self.__parse(input[1:], n_cur) # consume if ':' == input[0]: # open label set n_cur = n_cur.spawn_child(e_label_set) n_cur = n_cur.spawn_child(e_value) return self.__parse(input[1:], n_cur) if '(' == input[0]: # open node n_cur = n_cur.spawn_child(p_node) return self.parse__node_or_rel(input[1:], n_cur) if ')' == input[0]: # close node/path if input.startswith(')-'): # open path n_cur = n_cur.rotate__pin_under_new_parent(p_path) return self.parse__node_or_rel(input[1:], n_cur) n_cur = n_cur.parent if n_cur.__class__ == p_path: n_cur = n_cur.parent return self.__parse(input[1:], n_cur) if input.startswith('-['): # open rel n_cur = n_cur.spawn_child(p_rel) return self.parse__node_or_rel(input[2:], n_cur) if input.startswith(']-'): # close directional-rel if input.startswith(']->'): # close directional-rel n_cur.is_directional = True n_cur = n_cur.parent return self.parse__node_or_rel(input[3:], n_cur) n_cur = n_cur.parent return self.parse__node_or_rel(input[2:], n_cur) if '{' == input[0]: # open attr-set n_cur = n_cur.spawn_child(e_attr_set) return self.__parse(input[1:], n_cur) rgx_opt_id = r'^%s' % (e_ident.rgx('ident')) m = re.match(rgx_opt_id, input) if m: n_cur = n_cur.spawn_child(e_ident) return self.__parse(input, n_cur) assert False def parse__op_dot(self, input, op_dot): n_ident = op_dot.spawn_child(e_ident) input_suffix = self.read__e_ident(input, n_ident) n_cur = op_dot.parent # collapse return self.__parse(input_suffix, n_cur) def parse__pt_root(self, input, n_root): # # keywords # for kw in tok_set__kw__all: # handle keywords if input.startswith(kw): clause_type = globals().get('e_clause__' + kw) if 'optional match' == kw: clause_type = e_clause__match if not clause_type: clause_type = e_clause n_cur = n_root.spawn_child(clause_type) if 'optional match' == kw: n_cur.is_optional = True n_cur.spawn_child(e_keyword, kw) if kw in tok_set__kw__unsupported: # generic e_clause case n_value = n_cur.spawn_child(e_value) n_value.value = '' return self.__parse(input[len(kw):], n_cur) def __parse(self, input, n_cur): if None == input or 0 == len(input): # end of input or opt regex group not found return # # order sensitive to specificity: # - e_clause_match, e_clause_create > e_clause # - e_param > e_ident # - e_label_set, e_attr_set > e_set # if isinstance(n_cur, e_clause__create): return self.parse__e_clause_create_or_match(input, n_cur) if isinstance(n_cur, e_clause__match): return self.parse__e_clause_create_or_match(input, n_cur) if isinstance(n_cur, e_clause__where): return self.parse__e_clause__common(input, n_cur) if isinstance(n_cur, e_clause): return self.parse__e_clause__common(input, n_cur) if isinstance(n_cur, p_rel): return self.parse__node_or_rel(input, n_cur) if isinstance(n_cur, p_node): return self.parse__node_or_rel(input, n_cur) if isinstance(n_cur, e_label_set): return self.parse__e_label_set(input, n_cur) if isinstance(n_cur, e_attr_set): return self.parse__e_attr_set(input, n_cur) if isinstance(n_cur, e_set): return self.parse__e_set(input, n_cur) f_parse = getattr(self, 'parse__' + n_cur.__class__.__name__) return f_parse(input, n_cur)