diff options
| author | Alon Levy <alon@pobox.com> | 2014-12-16 17:37:42 +0200 |
|---|---|---|
| committer | Alon Levy <alon@pobox.com> | 2014-12-16 17:37:42 +0200 |
| commit | ade19de410e0a789a921ec1666b278b2a64176d6 (patch) | |
| tree | c4c54b106de93609ca7bb70a20fd4be696715dbe /src/server | |
| parent | c7d026b1d504323a8c61d51b726e5e8d2337fc4b (diff) | |
moving files around after repository merger
Diffstat (limited to 'src/server')
| -rw-r--r-- | src/server/crypt_util.py | 46 | ||||
| -rw-r--r-- | src/server/db_controller.py | 495 | ||||
| -rw-r--r-- | src/server/db_driver.py | 91 | ||||
| -rw-r--r-- | src/server/model/__init__.py | 0 | ||||
| -rw-r--r-- | src/server/model/graph.py | 105 | ||||
| -rw-r--r-- | src/server/model/model.py | 37 | ||||
| -rw-r--r-- | src/server/neo4j_util.py | 260 | ||||
| -rw-r--r-- | src/server/rhizi_api.py | 256 | ||||
| -rw-r--r-- | src/server/rhizi_server.py | 263 | ||||
| -rwxr-xr-x | src/server/rhizi_server_fcgi.py | 25 | ||||
| -rw-r--r-- | src/server/util.py | 22 |
11 files changed, 1600 insertions, 0 deletions
diff --git a/src/server/crypt_util.py b/src/server/crypt_util.py new file mode 100644 index 00000000..464fac94 --- /dev/null +++ b/src/server/crypt_util.py @@ -0,0 +1,46 @@ +import pickle +import hashlib, uuid +import os +import logging + +log = logging.getLogger('rhizi') + +def add_user_login(config, u, p): + htpasswd_path = config.htpasswd_path + + if False == os.path.exists(htpasswd_path): + with open(htpasswd_path, 'wb') as f: + pickle.dump({}, f) + + with open(htpasswd_path, 'rb') as f: + data = f.read() + pw_db = pickle.loads(data) + + with open(htpasswd_path, 'wb') as f: + salt = config.secret_key + pw_db[u] = hash_pw(str(p), salt) + pickle.dump(pw_db, f) + + log.info('htpasswd db: added entry: user: %s, pw: %s...' % (u, pw_db[u][:5])) + +def hash_pw(pw_str, salt_str): + salt = hashlib.sha512(salt_str).hexdigest() + ret = hashlib.sha512(pw_str + salt).hexdigest() + return ret + +def validate_login(config, u, p): + htpasswd_path = config.htpasswd_path + + salt = config.secret_key + + with open(htpasswd_path) as f: + pw_db = pickle.load(f) + + existing_pw_hash = pw_db.get(u) + if None == existing_pw_hash: + raise Exception('Not autorhized') + + if hash_pw(p, salt) != existing_pw_hash: + raise Exception('Not autorhized') + + diff --git a/src/server/db_controller.py b/src/server/db_controller.py new file mode 100644 index 00000000..c17fd0cd --- /dev/null +++ b/src/server/db_controller.py @@ -0,0 +1,495 @@ +#!/usr/bin/python + +import json +import logging +import os +import re +import traceback + +from db_driver import DB_Driver_REST, DB_Driver_Base +from model.graph import Attr_Diff +from model.graph import Topo_Diff +from neo4j_util import DB_result_set +from neo4j_util import cfmt +import neo4j_util as db_util +from model.model import Link + +log = logging.getLogger('rhizi') + +class DB_op(object): + """ + tx wrapped DB operation possibly composing multiple DB queries + """ + def __init__(self): + self.statement_set = [] + self.result_set = [] + self.error_set = None + self.tx_id = None + self.tx_commit_url = None # cached from response to tx begin + + def parse_tx_id(self, tx_commit_url): + m = re.search('/(?P<id>\d+)/commit$', tx_commit_url) + id_str = m.group('id') + self.tx_id = int(id_str) + + def add_statement(self, query, query_params={}): + """ + add a DB query language statement + @return: statement index (zero based) + """ + s = db_util.statement_to_REST_form(query, query_params) + self.statement_set.append(s) + return len(self.statement_set) + + def __iter__(self): + """ + iterate over (statement_index, statement, result, error) + where result & error are mutually exclusive + + note: statement_index is zero based + + TODO: handle partial iteration due to error_set being non-empty + """ + i = 0 + r_set_len = len(self.result_set) + for s in self.statement_set: + r_set = None # row-set + if i < r_set_len: # support partial result recovery + r_set = DB_result_set(self.result_set[i]) + yield (i, s, r_set) + i = i + 1 + + def parse_multi_statement_response_data(self, data): + pass + + @property + def name(self): + return self.__class__.__name__ + + def process_result_set(self): + """ + DB op can issue complex sets of quries all at once - this helper method + assists in parsing response data from a single query. + """ + ret = [] + for _, _, r_set in self: + for row in r_set: + for col in row: + ret.append(col) + return ret + +class DB_composed_op(DB_op): + def __init__(self): + super(DB_composed_op, self).__init__() + self.sub_op_set = [] + + def __assert_false_statement_access(self): + assert False, "composed_op may not contain statements, only sub-ops" + + def add_statement(self, query, query_params={}): + self.__assert_false_statement_access() + + def add_sub_op(self, op): + self.sub_op_set.append(op) + + def __getattribute__(self, attr): + """ + intercept 'statement_set' attr get + """ + if attr == 'statement_set': + self.__assert_false_statement_access() + + return object.__getattribute__(self, attr) + + def __iter__(self): + """ + iterate over sub_op_set + """ + for s_op in self.sub_op_set: + yield s_op + + def process_result_set(self): + ret = [] + for s_op in self: + s_result_set = s_op.process_result_set() + ret.append(s_result_set) + return ret + +class DBO_cypher_query(DB_op): + """ + freeform cypher query + """ + def __init__(self, q, q_params={}): + super(DBO_cypher_query, self).__init__() + self.add_statement(q, q_params) + +class DBO_topo_diff_commit(DB_composed_op): + """ + commit a Topo_Diff + """ + def __init__(self, topo_diff): + super(DBO_topo_diff_commit, self).__init__() + + n_add_map = db_util.meta_attr_list_to_meta_attr_map(topo_diff.node_set_add) + l_add_map = db_util.meta_attr_list_to_meta_attr_map(topo_diff.link_set_add) + l_rm_set = topo_diff.link_set_rm + n_rm_set = topo_diff.node_set_rm + + # + # [!] order critical + # + if len(n_add_map) > 0: + op = DBO_add_node_set(n_add_map) + self.add_sub_op(op) + + if len(l_add_map) > 0: + op = DBO_add_link_set(l_add_map) + self.add_sub_op(op) + + if len(l_rm_set) > 0: + op = DBO_rm_link_set(l_rm_set) + self.add_sub_op(op) + + if len(n_rm_set) > 0: + op = DBO_rm_node_set(n_rm_set) + self.add_sub_op(op) + +class DBO_attr_diff_commit(DB_op): + """ + commit a Attr_Diff + """ + def __init__(self, attr_diff): + super(DBO_attr_diff_commit, self).__init__() + + for id_attr, n_attr_diff in attr_diff.type__node.items(): + # TODO parameterize multiple attr removal + r_attr_set = n_attr_diff['__attr_remove'] + w_attr_set = n_attr_diff['__attr_write'] + + assert len(r_attr_set) > 0 or len(w_attr_set) > 0 + + q_arr = ["match (n {id: {id}}) ", + "return n.id, n"] + q_param_set = {'id': id_attr} + + if len(r_attr_set) > 0: + stmt_attr_rm = "remove " + ', '.join(['n.' + attr for attr in r_attr_set]) + q_arr.insert(1, stmt_attr_rm) + + if len(w_attr_set) > 0: + stmt_attr_set = "set n += {attr_set}" + q_arr.insert(1, stmt_attr_set) + q_param_set['attr_set'] = w_attr_set + + q = " ".join(q_arr) + self.add_statement(q, q_param_set) + + for id_attr, n_attr_diff in attr_diff.type__link.items(): + pass # TODO: handl link attr_diffs + + def process_result_set(self): + ret = {} + for _, _, r_set in self: + for row in r_set: + n_id, n = [v for v in row] # we expect a [n_id, n] array + ret[n_id] = n + return ret + +class DBO_add_node_set(DB_op): + def __init__(self, node_map): + """ + DB op: add node set + + @param node_map: node-type to node-set map + @return: set of new node DB ids + """ + super(DBO_add_node_set, self).__init__() + for q, q_param_set in db_util.gen_query_create_from_node_map(node_map): + self.add_statement(q, q_param_set) + + def process_result_set(self): + id_set = [] + for _, _, row_set in self: + for row in row_set: + for clo in row: + id_set.append(clo) + + return id_set + +class DBO_add_link_set(DB_op): + def __init__(self, link_map): + """ + @param link_map: is a link-type to link-set map - see model.link + @return: set of new node DB ids + """ + super(DBO_add_link_set, self).__init__() + for q, q_params in db_util.gen_query_create_from_link_map(link_map): + self.add_statement(q, q_params) + + def process_result_set(self): + id_set = [] + for _, _, r_set in self: + for row in r_set: + for col_val in row: + id_set.append(col_val) + + return id_set + +class DBO_load_node_set_by_DB_id(DB_op): + def __init__(self, id_set): + """ + load a set of nodes whose DB id is in id_set + + @param id_set: DB node id set + @return: loaded node set or an empty set if no match was found + """ + super(DBO_load_node_set_by_DB_id, self).__init__() + q = "start n=node({id_set}) return n" + self.add_statement(q, { 'id_set': id_set}) + +class DBO_match_node_id_set(DB_op): + + def __init__(self, filter_label=None, filter_attr_map={}): + """ + match a set of nodes by type / attr_map + + @param filter_label: node type filter + @param filter_attr_map: is a filter_key to filter_value_set map of + possible attributes to match against, eg.: + { 'id':[0,1], 'color: ['red','blue'] } + @return: a set of node DB id's + """ + super(DBO_match_node_id_set, self).__init__() + + q = "match (n{filter_label}) {where_clause} return id(n)" + q = cfmt(q, filter_label="" if not filter_label else ":" + filter_label) + q = cfmt(q, where_clause=db_util.gen_clause_where_from_filter_attr_map(filter_attr_map)) + + q_params = filter_attr_map + + self.add_statement(q, q_params) + +class DBO_match_node_set_by_id_attribute(DBO_match_node_id_set): + def __init__(self, id_set): + """ + convenience op: load a set of nodes by their 'id' attribute != DB node id + """ + assert isinstance(id_set, list) + + super(DBO_match_node_set_by_id_attribute, self).__init__(filter_attr_map={'id': id_set}) + + +class DBO_load_link_set(DB_op): + def __init__(self, link_ptr_set): + """ + match a set of sets of links by source/target node id attributes + + This class should be instantiated through a static factory function + + @link_ptr_set link pointer set + @return: a set of loaded links + """ + super(DBO_load_link_set, self).__init__() + + for l_ptr in link_ptr_set: + if not l_ptr.src_id: + q = "match ()-[r]->({id: {dst_id}}) return r" + q_params = {'dst_id': l_ptr.dst_id} + elif not l_ptr.dst_id: + q = "match ({id: {src_id}})-[r]->() return r" + q_params = {'src_id': l_ptr.src_id} + else: + q = "match ({id: {src_id}})-[r]->({id: {dst_id}}) return r" + q_params = {'src_id': l_ptr.src_id, 'dst_id': l_ptr.dst_id} + + self.add_statement(q, q_params) + + @staticmethod + def init_from_link_ptr(l_ptr): + return DBO_load_link_set([l_ptr]) + + @staticmethod + def init_from_link_ptr_set(l_ptr_set): + return DBO_load_link_set(l_ptr_set) + +class DBO_match_link_id_set(DB_op): + def __init__(self, filter_label=None, filter_attr_map={}): + """ + load an id-set of links + + @param filter_label: link type filter + @param filter_attr_map: is a filter_key to filter_value_set map of + attributes to match link properties against + @return: a set of loaded link ids + """ + super(DBO_match_link_id_set, self).__init__() + + q_arr = ['match ()-[r{filter_label} {filter_attr}]->()', + 'return id(r)' + ] + q = ' '.join(q_arr) + q = cfmt(q, filter_label="" if not filter_label else ":" + filter_label) + q = cfmt(q, filter_attr=db_util.gen_clause_attr_filter_from_filter_attr_map(filter_attr_map)) + q_params = {k: v[0] for (k, v) in filter_attr_map.items()} # pass on only first value from each value set + + self.add_statement(q, q_params) + +class DBO_rm_node_set(DB_op): + def __init__(self, id_set, rm_links=False): + """ + remove node set + """ + assert len(id_set) > 0, __name__ + ': empty id set' + + super(DBO_rm_node_set, self).__init__() + + if rm_links: + q_arr = ['match (n)', + 'where n.id in {id_set}', + 'optional match (n)-[r]-()', + 'delete n,r', + 'return {id_set}' + ] + else: + q_arr = ['match (n)', + 'where n.id in {id_set}', + 'delete n', + 'return {id_set}' + ] + + q = ' '.join(q_arr) + q_params = {'id_set': id_set} + self.add_statement(q, q_params) + +class DBO_rm_link_set(DB_op): + def __init__(self, id_set): + """ + remove link set + + [!] when removing as a result of node removal, use DBO_rm_node_set + along with rm_links=True + """ + assert len(id_set) > 0, __name__ + ': empty id set' + + super(DBO_rm_link_set, self).__init__() + + q_arr = ['match ()-[r]->()', + 'where r.id in {id_set}', + 'delete r', + 'return {id_set}' + ] + + q = ' '.join(q_arr) + q_params = {'id_set': id_set} + self.add_statement(q, q_params) + +class DBO_rz_clone(DB_op): + def __init__(self, filter_label=None, limit=128): + """ + clone rhizi + + @return: a dict: {'node_set': n_set, + 'link_set': l_set } + where l_set is a list of (src.id, dst.id, link) tuples + """ + super(DBO_rz_clone, self).__init__() + + self.limit = limit + self.skip = 0 + + q_arr = ['match (n)' if not filter_label else 'match (n:%s)' % (filter_label), + 'optional match (n)-[r]->(m)', + 'with n,r,m', + 'order by n.id', + 'skip %d' % (self.skip), + 'limit %d' % (self.limit), + 'return n,labels(n),collect([m.id, r, type(r)])'] + + q = ' '.join(q_arr) + self.add_statement(q) + + def process_result_set(self): + ret_n_set = [] + ret_l_set = [] + for _, _, row_set in self: + for row in row_set: + n, n_lbl_set, l_set = row.items() # see query return statement + + # reconstruct nodes + assert None != n['id'] + + n['__label_set'] = n_lbl_set + ret_n_set.append(n) + + # reconstruct links from link tuples + for l_tuple in l_set: + assert 3 == len(l_tuple) # see query return statement + + if None == l_tuple[0]: # check if link dst is None + # as link matching is optional, collect may yield empty sets + continue + + l = l_tuple[1] + l['__src_id'] = n['id'] + l['__dst_id'] = l_tuple[0] + l['__label_set'] = [l_tuple[2]] # box single value returned by type() + + ret_l_set.append(l) + + return {'node_set': ret_n_set, + 'link_set': ret_l_set } + +class DB_Controller: + """ + neo4j DB controller + """ + def __init__(self, config, db_driver_class=None): + self.config = config + if not db_driver_class: + self.db_driver = DB_Driver_REST(self.config.db_base_url) + else: + self.db_driver = db_driver_class() + assert isinstance(self.db_driver, DB_Driver_Base) + + def exec_op(self, op): + """ + execute operation within a DB transaction + """ + if isinstance(op, DB_composed_op): + # construct a list comprehension composed of all sup_op statements + for s_op in op.sub_op_set: + self.exec_op(s_op) + return op.process_result_set() + + try: + self.db_driver.begin_tx(op) + self.db_driver.exec_statement_set(op) + self.db_driver.commit_tx(op) + + ret = op.process_result_set() + + log.debug('exec_op:' + op.name + ': return value: ' + str(ret)) + return ret + except Exception as e: + # here we watch for IOExecptions, etc - not db errors + # these are returned in the db response itself + log.error(e.message) + log.error(traceback.print_exc()) + raise e + + def create_db_op(self, f_work, f_cont): + ret = DB_op(f_work, f_cont) + return ret + + def exec_cypher_query(self, q): + """ + @deprecated: use DBO_cypher_query + """ + + # call post and not db_util.post_neo4j to avoid response key errors + try: + db_util.post(self.config.db_base_url + '/db/data/cypher', {"query" : q}) + except Exception as e: + log.error(e.message) + log.error(traceback.print_exc()) + raise e diff --git a/src/server/db_driver.py b/src/server/db_driver.py new file mode 100644 index 00000000..0532e512 --- /dev/null +++ b/src/server/db_driver.py @@ -0,0 +1,91 @@ +import logging + +from neo4j_util import Neo4JException +import neo4j_util as db_util + + +log = logging.getLogger('rhizi') + +class DB_Driver_Base(): + + def log_committed_queries(self, statement_set): + for sp_dict in statement_set['statements']: + if None != sp_dict['parameters']: + msg = '\tq: {0}\n\tp: {1}'.format(sp_dict['statement'], + sp_dict['parameters']) + else: + msg = '\tq: {0}'.format(sp_dict['statement']) + log.debug(msg) + +class DB_Driver_Embedded(DB_Driver_Base): + def __init__(self, db_base_url): + self.tx_base_url = db_base_url + '/db/data/transaction' + + from org.rhizi.db.neo4j.util import EmbeddedNeo4j + self.edb = EmbeddedNeo4j.createDb() + self.edb.createDb() + + def begin_tx(self, op): + pass + + def exec_statement_set(self, op): + s_set = op.statement_set + self.edb.executeCypherQury() + + def commit_tx(self, op): + pass + +class DB_Driver_REST(DB_Driver_Base): + def __init__(self, db_base_url): + self.tx_base_url = db_base_url + '/db/data/transaction' + + def begin_tx(self, op): + tx_open_url = self.tx_base_url + + try: + # + # [!] neo4j seems picky about receiving an additional empty statement list + # + data = data = db_util.statement_set_to_REST_form([]) + ret = db_util.post_neo4j(tx_open_url, data) + tx_commit_url = ret['commit'] + op.parse_tx_id(tx_commit_url) + + log.debug('tx-open: id: {0}, commit-url: {1}'.format(op.tx_id, tx_commit_url)) + except Exception as e: + raise Exception('failed to open transaction:' + e.message) + + def exec_statement_set(self, op): + + tx_url = "{0}/{1}".format(self.tx_base_url, op.tx_id) + statement_set = db_util.statement_set_to_REST_form(op.statement_set) + + try: + post_ret = db_util.post_neo4j(tx_url, statement_set) + op.result_set = post_ret['results'] + op.error_set = post_ret['errors'] + if 0 != len(op.error_set): + raise Neo4JException(op.error_set) + + self.log_committed_queries(statement_set) + except Neo4JException as e: + raise e + except Exception as e: + raise Exception('failed exec op statements: err: {0}, url: {1}'.format(e.message, tx_url)) + + def commit_tx(self, op): + tx_commit_url = "{0}/{1}/commit".format(self.tx_base_url, op.tx_id) + + try: + # + # [!] neo4j seems picky about receiving an additional empty statement list + # + data = db_util.statement_set_to_REST_form([]) + ret = db_util.post(tx_commit_url, data) + + log.debug('tx-commit: id: {0}, commit-url: {1}'.format(op.tx_id, tx_commit_url)) + + return ret + except Exception as e: + raise Exception('failed to commit transaction:' + e.message) + diff --git a/src/server/model/__init__.py b/src/server/model/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/src/server/model/__init__.py diff --git a/src/server/model/graph.py b/src/server/model/graph.py new file mode 100644 index 00000000..f197921d --- /dev/null +++ b/src/server/model/graph.py @@ -0,0 +1,105 @@ +class Attr_Diff(dict): + """ + Represents a change to note attributes, where nodes can represent + either logical nodes or logical links, and attributes can be added, + changed or removed + + Example: + attr_diff = {'__type_node' : {n_id: {'__attr_write': {'attr_0': 0, + 'attr_1': 'a'}, + '__attr_remove': ['attr_2'] }} + '__type_link' : {l_id: ... } + } + """ + def __init__(self): + self['__type_node'] = {} + self['__type_link'] = {} + + def init_node_attr_diff(self, n_id): + ret = {'__attr_write': {}, + '__attr_remove': []} + self['__type_node'][n_id] = ret + return ret + + @staticmethod + def from_json_dict(json_dict): + ret = Attr_Diff() + for obj_type in ret.keys(): + obj_ad_set = json_dict.get(obj_type) + if None != obj_ad_set: + for o_id, ad in obj_ad_set.items(): + if None != ad.get('__attr_write'): + for k, v in ad['__attr_write'].items(): + ret.add_node_attr_write(o_id, k, v) + if None != ad.get('__attr_remove'): + for k in ad['__attr_remove']: + ret.add_node_attr_rm(o_id, k) + return ret + + @property + def type__node(self): + return self['__type_node'] + + @property + def type__link(self): + return self['__type_link'] + + def add_node_attr_write(self, n_id, attr_name, attr_val): + + assert 'id' != attr_name.lower(), 'Attr_Diff: attempt to write to \'id\' attribute' + + n_attr_diff = self['__type_node'].get(n_id) + if None == n_attr_diff: + n_attr_diff = self.init_node_attr_diff(n_id) + n_attr_diff['__attr_write'][attr_name] = attr_val + + def add_node_attr_rm(self, n_id, attr_name): + n_attr_diff = self['__type_node'].get(n_id) + if None == n_attr_diff: + n_attr_diff = self.init_node_attr_diff(n_id) + n_attr_diff['__attr_remove'].append(attr_name) + + def add_link_attr_write(self, l_id, attr_name, attr_val): + assert False, 'unimplemented' + + def add_link_attr_rm(self, l_id, attr_name): + assert False, 'unimplemented' + +class Topo_Diff(object): + """ + Represents a change to the graph topology + """ + def __init__(self, link_set_rm=[], + node_set_rm=[], + node_set_add=[], + link_set_add=[]): + + self.link_set_rm = link_set_rm + self.node_set_rm = node_set_rm + self.node_set_add = node_set_add + self.link_set_add = link_set_add + + def __str__(self): + return __name__ + ': ' + ', '.join('%s: %s' % (k, v) for k, v in self.__dict__.items()) + + def check_validity(self, topo_diff_dict): + """ + Topo_Diff may represent invalid operations, eg. adding a link while + removing it's end-point - this stub should check for that + """ + pass + + @staticmethod + def from_json_dict(json_dict): + """ + construct from dict - no node/link constructor set must be provided + """ + ret = Topo_Diff() + + # merge keys - this allows constructor argument omission (link_set_rm, + # node_set_rm, etc.) such as when constructing from POST JSON data + for k, _ in ret.__dict__.items(): + v = json_dict.get(k) + if None != v: + ret.__dict__[k] = v + return ret diff --git a/src/server/model/model.py b/src/server/model/model.py new file mode 100644 index 00000000..33cf9e6a --- /dev/null +++ b/src/server/model/model.py @@ -0,0 +1,37 @@ +class Link(): + """ + documentation anchor - this class currently carries no implementation + and only acts as a documentation anchor + + link['__src'] - meta attribute for link source object + link['__dst'] - meta attribute for link destination object + """ + + def __init__(self, src=None, dst=None): + assert False, 'currently unused' + + class Link_Ptr(dict): + """ + link['__src_id'] - meta attribute for link source id + link['__dst_id'] - meta attribute for link destination id + """ + def __init__(self, src_id=None, dst_id=None): + assert None != src_id or None != dst_id + + self['__src_id'] = src_id + self['__dst_id'] = dst_id + + @property + def src_id(self): + return self['__src_id'] + + @property + def dst_id(self): + return self['__dst_id'] + + @staticmethod + def link_ptr(src_id=None, dst_id=None): + """ + init from src_id or dst_id attributes - at least one must be provided + """ + return Link.Link_Ptr(src_id, dst_id) diff --git a/src/server/neo4j_util.py b/src/server/neo4j_util.py new file mode 100644 index 00000000..cf2f962e --- /dev/null +++ b/src/server/neo4j_util.py @@ -0,0 +1,260 @@ +""" + Utility code in speaking the neo4j REST api +""" + +import json +import six +from six.moves.urllib import request +import six.moves.urllib_error as urllib_error +import model +import string +import time + +from util import debug_log_duration + +class Neo4JException(Exception): + def __init__(self, error_set): + self.error_set = error_set + + def __str__(self): + return 'neo4j error set: ' + str(self.error_set) + +class DB_row(object): + def __init__(self, data): + self.data = data + + def __iter__(self): + for column_val in self.data: + yield column_val + + def items(self): + return [x for x in self] + +class DB_result_set(object): + def __init__(self, data): + self.data = data + + def __iter__(self): + for db_row_dict in self.data['data']: + # example: dict: {u'row': [{u'title': u'foo'}]} + assert None != db_row_dict['row'] + + yield DB_row(db_row_dict['row']) + + def items(self): + return [x for x in self] + +class Cypher_String_Formatter(string.Formatter): + """ + Despite parameter support in Cypher, we sometimes do engage in query string building + - as both Cypher & Python use brackets to wrap parameters, escaping them in Python makes + queries less readable. This customized formatter will simply ignore unavailable keyworded + formatting arguments, allowing the use of non-escaped parameter designation, eg: + q = cfmt("match (a:{type} {cypher_param})", type='Book') + """ + + def get_field(self, field_name, args, kwargs): + # ignore key not found, return bracket wrapped key + try: + val = super(Cypher_String_Formatter, self).get_field(field_name, args, kwargs) + except (KeyError, AttributeError): + val = "{" + field_name + "}", field_name + return val + +def cfmt(fmt_str, *args, **kwargs): + return Cypher_String_Formatter().format(fmt_str, *args, **kwargs) + +def post_neo4j(url, data): + """ + @return dict object from the neo4j json POST response + """ + ret = post(url, data) + ret_data = json.load(ret) + + # [!] do not raise exception if ret_data['errors'] is not empty - + # this allows query-sets to partially succeed + + return ret_data + +def post(url, data): + assert(isinstance(data, dict)) # make sure we're not handed json strings + + post_data_json = json.dumps(data) + + req = request.Request(url) + req.add_header('User-Agent', 'rhizi-server/0.1') + req.add_header('Accept', 'application/json; charset=UTF-8') + req.add_header('Content-Type', 'application/json') + + req.add_header('X-Stream', 'true') # enable neo4j JSON streaming + + try: + ret = request.urlopen(req, post_data_json) + except urllib_error.HTTPError as e: + raise Exception('post request failed: code: {0}, reason: {1}'.format(e.code, e.reason)) + + return ret + +def statement_to_REST_form(query, parameters={}): + """ + turn cypher query to neo4j json API format + """ + assert isinstance(query, six.string_types) + if isinstance(parameters, list): + for v in parameters: + assert isinstance(v, dict) + else: + assert isinstance(parameters, dict) + + return {'statement' : query, 'parameters': parameters} + +def statement_set_to_REST_form(statement_set): + assert isinstance(statement_set, list) + + return {'statements': statement_set} + +def gen_clause_attr_filter_from_filter_attr_map(filter_attr_map, node_label="n"): + if not filter_attr_map: + return "{}" + + __type_check_filter_attr_map(filter_attr_map) + + filter_arr = [] + for attr_name in filter_attr_map.keys(): + # create a cypher query parameter place holder for each attr set + # eg. n.foo in {foo}, where foo is passed as a query parameter + f_attr = cfmt("{attr_name}: {{{attr}}}", attr_name=attr_name) + filter_arr.append(f_attr) + + filter_str = "{{{0}}}".format(', '.join(filter_arr)) + return filter_str + +def gen_clause_where_from_filter_attr_map(filter_attr_map, node_label="n"): + """ + convert a filter attribute map to a parameterized Cypher where clause, eg. + in: { 'att_foo': [ 'a', 'b' ], 'att_goo': [1,2] } + out: {att_foo: {att_foo}, att_goo: {att_goo}, ...} + + this function will essentially ignore all but the first value in the value list + + @param filter_attr_map: may be None or empty + """ + if not filter_attr_map: + return "" + + __type_check_filter_attr_map(filter_attr_map) + + filter_arr = [] + for attr in filter_attr_map.keys(): + # create a cypher query parameter place holder for each attr set + # eg. n.foo in {foo}, where foo is passed as a query parameter + f_attr = cfmt("{node_label}.{attr} in {{{attr}}}", node_label=node_label, attr=attr) + filter_arr.append(f_attr) + filter_str = "where {0}".format(' and '.join(filter_arr)) + return filter_str + +def gen_query_create_from_node_map(node_map, input_to_DB_property_map=lambda _: _): + """ + generate a set of node create queries + + @param node_map: is a node-type to node map + @input_to_DB_property_map: optional function which takes a map of input properties and returns a map of DB properties - use to map input schemas to DB schemas + + @return: a (query, query_parameteres) set of create queries + """ + __type_check_link_or_node_map(node_map) + + ret = [] + for label, n_set in node_map.items(): + + assert len(label) > 2 and label[0].isupper() and label[1:].islower(), 'malformed label: ' + label + + q_arr = ['create (n:%s {node_attr})' % (label), + 'return n.id' + ] + + q = ' '.join(q_arr) + q_params_set = [] + for n_prop_set in n_set: + + assert None != n_prop_set['id'], 'node create query: node id attribute not set' + + q_params = input_to_DB_property_map(n_prop_set) + q_params_set.append(q_params) + ret.append((q, {'node_attr': q_params_set})) + return ret + +def gen_query_create_from_link_map(link_map, input_to_DB_property_map=lambda _: _): + """ + generate a set of link create queries + + @param link_map: is a link-type to link map - see model.link + """ + __type_check_link_or_node_map(link_map) + + ret = [] + for l_type, l_set in link_map.items(): + q = "match (src {id: {src}.id}),(dst {id: {dst}.id}) " + \ + "create (src)-[r:%(__type)s {link_attr}]->(dst) " + \ + "return id(r)" + q = q % {'__type':l_type} + + for link in l_set: + __type_check_link(link) + + src_id = link['__src_id'] + dst_id = link['__dst_id'] + + # TODO: use object based link representation + prop_dict = link.copy() + del prop_dict['__dst_id'] + del prop_dict['__src_id'] + + q_params = {'src': { 'id': src_id} , + 'dst': { 'id': dst_id} , + 'link_attr' : input_to_DB_property_map(prop_dict)} + ret.append((q, q_params)) + + return ret + +def meta_attr_list_to_meta_attr_map(e_set, meta_attr='__label_set'): + """ + convert a list of maps each containing a meta_attr key into a + meta_attr-mapped collection of lists with the meta_attr removed - eg: + + in: [{'id':0, '__type': 'T'}, {'id':1, '__type': 'T'}] + out: { 'T', [{'id':0}, {'id':1}] } + """ + ret = {} + for v in e_set: + assert None != v[meta_attr], 'missing type meta-attribute' + assert 1 == len(v[meta_attr]), 'only single-type mapping currently suppoerted' + + v_type = v[meta_attr][0] + if None == ret.get(v_type): # init type list if necessary + ret[v_type] = [] + + v_no_meta = v.copy() + del v_no_meta[meta_attr] + + ret[v_type].append(v_no_meta) + + return ret + +def __type_check_link(link): + assert link.has_key('__src_id') + assert link.has_key('__dst_id') + +def __type_check_link_or_node_map(x_map): + for k, v in x_map.iteritems(): # do some type sanity checking + assert isinstance(k, six.string_types) + assert isinstance(v, list) + +def __type_check_filter_attr_map(filter_attr_map): + """ + # type sanity check an attribute filter map + """ + assert isinstance(filter_attr_map, dict) + for k, v in filter_attr_map.items(): + assert isinstance(k, six.string_types) + assert isinstance(v, list) diff --git a/src/server/rhizi_api.py b/src/server/rhizi_api.py new file mode 100644 index 00000000..579d41e1 --- /dev/null +++ b/src/server/rhizi_api.py @@ -0,0 +1,256 @@ +""" +Rhizi web API +""" +import os +import db_controller as dbc +import json +import logging +import traceback +import crypt_util + +import flask +from flask import jsonify +from flask import Flask +from flask import request +from flask import make_response +from flask import session +from flask import redirect +from flask import escape +from flask import url_for +from flask import render_template +from flask import send_from_directory + +from model.graph import Topo_Diff +from model.graph import Attr_Diff +from model.model import Link +from datetime import datetime + +log = logging.getLogger('rhizi') + +# injected: DB controller +db_ctl = None + +def __sanitize_input(*args, **kw_args): + pass + +def sanitize_input__node(n): + """ + provide a control point as to which node fields are persisted + """ + assert None != n['id'], 'invalid input: node: missing id' + +def sanitize_input__link(l): + """ + provide a control point as to which node fields are persisted + """ + assert None != l['id'], 'invalid input: link: missing id' + assert None != l['__src_id'], 'invalid input: link: missing src id' + assert None != l['__dst_id'], 'invalid input: link: missing dst id' + +def sanitize_input__topo_diff(topo_diff): + for n in topo_diff.node_set_add: + sanitize_input__node(n) + for l in topo_diff.link_set_add: + sanitize_input__link(l) + +def sanitize_input__attr_diff(attr_diff): + pass # TODO: impl + +def __response_wrap(data=None, error=None): + """ + wrap response data/errors as dict - this should always be used when returning + data to allow easy return of list objects, assist in error case distinction, etc. + """ + return dict(data=data, error=error) + +def __common_resp_handle(data=None, error=None): + """ + provide common response handling + """ + ret_data = __response_wrap(data, error) + resp = jsonify(ret_data) + + resp.headers['Access-Control-Allow-Origin'] = '*' + + # more response processing + + return resp + +def __common_exec(op, on_success=__common_resp_handle): + try: + op_ret = db_ctl.exec_op(op) + return on_success(op_ret) + except Exception as e: + log.error(e.message) + log.error(traceback.print_exc()) + return __common_resp_handle('error occurred') + +def load_node_set_by_id_attr(): + """ + load node-set by ID attribute + + @param id_set: list of node ids to match id attribute against + @return: a list of nodes whose id attribute matches 'id' or + an empty list if the requested node is not found + @raise exception: on error + """ + req_json = request.get_json() + id_set = req_json['id_set'] + + __sanitize_input(id_set) + + return __load_node_set_by_id_attr_common(id_set) + +def __load_node_set_by_id_attr_common(id_set): + """ + @param f_k: optional attribute filter key + @param f_vset: possible key values to match against + """ + op = dbc.DBO_match_node_set_by_id_attribute(id_set=id_set) + try: + n_set = db_ctl.exec_op(op) + return __common_resp_handle(data=n_set) + except Exception as e: + log.exception(e) + return __common_resp_handle(error='unable to load node with ids: {0}'.format(id_set)) + +def match_node_set_by_attr_filter_map(attr_filter_map): + """ + @param attr_filter_map + + @return: a set of node DB id's + """ + op = dbc.DBO_match_node_id_set(attr_filter_map) + return __common_exec(op) + +def load_link_set_by_link_ptr_set(): + + def deserialize_param_set(param_json): + l_ptr_set_raw = param_json['link_ptr_set'] + + __sanitize_input(l_ptr_set_raw) + + l_ptr_set = [] + for lptr_dict in l_ptr_set_raw: + src_id = lptr_dict.get('__src_id') + dst_id = lptr_dict.get('__dst_id') + l_ptr_set += [Link.Link_Ptr(src_id=src_id, dst_id=dst_id) ] + + return l_ptr_set + + l_ptr_set = deserialize_param_set(request.get_json()) + + op = dbc.DBO_load_link_set.init_from_link_ptr_set(l_ptr_set) + return __common_exec(op) + +def rz_clone(): + op = dbc.DBO_rz_clone() + return __common_exec(op) + +def diff_commit__set(): + """ + commit a diff set + """ + def sanitize_input(req): + diff_set_dict = request.get_json()['diff_set'] + topo_diff_dict = diff_set_dict['__diff_set_topo'][0] + topo_diff = Topo_Diff.from_json_dict(topo_diff_dict) + + sanitize_input__topo_diff(topo_diff) + return topo_diff; + + topo_diff = sanitize_input(request) + op = dbc.DBO_topo_diff_commit(topo_diff) + return __common_exec(op) + +def diff_commit__topo(): + """ + commit a graph topology diff + """ + def sanitize_input(req): + topo_diff_dict = request.get_json()['topo_diff'] + topo_diff = Topo_Diff.from_json_dict(topo_diff_dict) + + sanitize_input__topo_diff(topo_diff) + return topo_diff; + + topo_diff = sanitize_input(request) + op = dbc.DBO_topo_diff_commit(topo_diff) + return __common_exec(op) + +def diff_commit__attr(): + """ + commit a graph attribute diff + """ + def sanitize_input(req): + attr_diff_dict = request.get_json()['attr_diff'] + attr_diff = Attr_Diff.from_json_dict(attr_diff_dict) + + sanitize_input__attr_diff(attr_diff) + return attr_diff; + + attr_diff = sanitize_input(request) + op = dbc.DBO_attr_diff_commit(attr_diff) + return __common_exec(op) + +def diff_commit__vis(): + pass + +def add_node_set(): + """ + @deprecated: use topo_attr_commit + + @param node_map: node type to node map, eg. { 'Skill': { 'name': 'kung-fu' } } + """ + node_map = request.get_json()['node_map'] + __sanitize_input(node_map) + + op = dbc.DBO_add_node_set(node_map) + return __common_exec(op) + +def monitor__server_info(): + """ + server monitor stub + """ + dt = datetime.now() + return "<html><body>" + \ + "<h1>Rhizi Server v0.1</h1><p>" + \ + "date: " + dt.strftime("%Y-%m-%d") + "<br>" + \ + "time: " + dt.strftime("%H:%M:%S") + "<br>" + \ + "</p></body></html>" + +def index(): + username = escape(session.get('username')) + return render_template('index.html', username=username) + +def login(): + + def sanitize_input(req): + req_json = request.get_json() + u = req_json['username'] + p = req_json['password'] + return u, p + + if request.method == 'POST': + try: + u, p = sanitize_input(request) + crypt_util.validate_login(flask.current_app.rz_config, u, p) + except Exception as e: + # login failed + log.warn('login: unauthorized: user: %s' % (u)) + return render_template('login.html', login_failed=True) + + # login successful + session['username'] = u + log.debug('login: success: user: %s' % (u)) + return redirect(url_for('index')) + + if request.method == 'GET': + return render_template('login.html') + +def logout(): + # remove the username from the session if it's there + u = session.pop('username', None) + log.debug('logout: success: user: %s' % (u)) + return redirect(url_for('login')) + diff --git a/src/server/rhizi_server.py b/src/server/rhizi_server.py new file mode 100644 index 00000000..168e2f53 --- /dev/null +++ b/src/server/rhizi_server.py @@ -0,0 +1,263 @@ +#!/usr/bin/python + +import logging +import json +import util +import os +import neo4j_util +import argparse +import db_controller as dbc +import rhizi_api +import flask +import crypt_util +import re + +from flask import Flask +from flask import session +from flask import redirect +from flask import request +from flask import send_from_directory + +from functools import wraps + +class Config(object): + """ + rhizi-server configuration + + TODO: config option documentation + + htpasswd_path + listen_address + listen_port + neo4j_url + root_path + """ + + @staticmethod + def init_from_file(file_path): + + if False == os.path.exists(file_path): + raise Exception('config file not found: ' + file_path) + + # apply defaults + cfg = {} + cfg['access_control'] = True + cfg['config_dir'] = os.path.abspath(os.path.dirname(file_path)) # bypass prop restriction + cfg['development_mode'] = False + cfg['listen_address'] = '127.0.0.1' + cfg['listen_port'] = 8080 + cfg['root_path'] = os.getcwd() + cfg['static_url_path'] = '/static' + + # Flask keys + cfg['SECRET_KEY'] = '' + + with open(file_path, 'r') as f: + for line in f: + if re.match('(^#)|(\s+$)', line): + continue + + kv_arr = line.split('=') + if 2 != len(kv_arr): + raise Exception('failed to parse config line: ' + line) + + k, v = map(str.strip, kv_arr) + + if None != cfg.get(k): + # apply type conversion based on default value type + type_f = type(cfg[k]) + if bool == type_f: + v = v in ("True", "true") # workaround bool('false') = True + else: + v = type_f(v) + + # [!] we can't use k.lower() as we are loading Flask configuration + # keys which are expected to be capitalized + cfg[k] = v + + ret = Config() + ret.__dict__ = cfg # allows setting of @property attributes + + # validate config + if False == os.path.isabs(ret.root_path): + ret.root_path = os.path.abspath(ret.root_path) + + return ret + + def __str__(self): + return '\n'.join('%s: %s' % (k, v) for k, v in self.__dict__.items()) + + @property + def db_base_url(self): + return self.neo4j_url + + @property + def tx_api_path(self): + return '/db/data/transaction' + + @property + def config_dir_path(self): + return self.config_dir + + @property + def secret_key(self): + return self.SECRET_KEY + +class FlaskExt(Flask): + """ + Flask server customization + """ + + def __init__(self, import_name, *args, **kwargs): + """ + reserved for future use + """ + super(FlaskExt, self).__init__(import_name, *args, **kwargs) + + def before_request(self, *args, **kwargs): + # TODO impl + pass + + def make_default_options_response(self): + ret = Flask.make_default_options_response(self) + + ret.headers['Access-Control-Allow-Origin'] = 'http://rhizi.net' + ret.headers['Access-Control-Allow-Headers'] = "Accept, Authorization, Content-Type, Origin" + ret.headers['Access-Control-Allow-Credentials'] = 'true' + + # ret.headers['Access-Control-Allow-Methods'] = ', '.join(m_list) + return ret + +def init_log(cfg): + """ + init log file, location derived from configuration + """ + log = logging.getLogger('rhizi') + log.setLevel(logging.DEBUG) + log_handler_c = logging.StreamHandler() + log_handler_f = logging.FileHandler(cfg.log_path) + + log.addHandler(log_handler_c) + log.addHandler(log_handler_f) + return log + +def init_rest_api(cfg, flask_webapp): + """ + map REST API calls + """ + + def rest_entry(path, f, flask_args={'methods': ['POST']}): + return (path, f, flask_args) + + def dev_mode__resend_from_static(static_url): + """ + redirect broken-on-local-deploy links: + - /src -> '': handle root based files, eg. app.js + - /res, /lib -> res, lib + """ + static_folder = flask.current_app.static_folder + + static_path = request.path + if static_path.startswith('/src'): + # TODO: clean - /src/... links should not exist + static_path = static_path.replace('/src', '') + if static_path.startswith('/'): # convert to relative path + static_path = static_path[1:] + return send_from_directory(static_folder, static_path) + + def login_decorator(f): + """ + [!] security boundary: asserd logged-in user before executing REST api call + """ + @wraps(f) + def wrapped_function(*args, **kw): + if not 'username' in session: + return redirect('/login') + return f(*args, **kw) + + return wrapped_function + + rest_entry_set = [ + rest_entry('/add/node-set' , rhizi_api.add_node_set), + rest_entry('/graph/clone', rhizi_api.rz_clone), + rest_entry('/graph/diff-commit-set', rhizi_api.diff_commit__set), + rest_entry('/graph/diff-commit-topo', rhizi_api.diff_commit__topo), + rest_entry('/graph/diff-commit-attr', rhizi_api.diff_commit__attr), + rest_entry('/graph/diff-commit-vis', rhizi_api.diff_commit__vis), + rest_entry('/index', rhizi_api.index, {'methods': ['GET']}), + rest_entry('/load/node-set-by-id', rhizi_api.load_node_set_by_id_attr), + rest_entry('/load/link-set/by_link_ptr_set', rhizi_api.load_link_set_by_link_ptr_set), + rest_entry('/login', rhizi_api.login, {'methods': ['GET', 'POST']}), + rest_entry('/logout', rhizi_api.logout, {'methods': ['GET', 'POST']}), + rest_entry('/match/node-set', rhizi_api.match_node_set_by_attr_filter_map), + rest_entry('/monitor/server-info', rhizi_api.monitor__server_info), + ] + + if cfg.development_mode: + dev_path_set = ['/src', '/res', '/lib'] + rest_dev_entry_set = [] + for dev_path in dev_path_set: + rest_dev_entry_set.append(rest_entry(dev_path + '/<path:static_url>', + dev_mode__resend_from_static, + {'methods': ['GET']})) + rest_entry_set += rest_dev_entry_set + + if False == cfg.access_control: + log.warn('access control disabled, public access set on all URLs') + + for re_entry in rest_entry_set: + rest_path, f, flask_args = re_entry + + if cfg.access_control and '/login' != rest_path: + # currently require login on all but /login paths + f = login_decorator(f) + + # [!] order seems important - apply route decorator last + route_dec = flask_webapp.route(rest_path, **flask_args) + f = route_dec(f) + + flask_webapp.f = f # assign decorated function + +def init_webapp(cfg): + root_path = cfg.root_path + webapp = FlaskExt(__name__, + static_folder='static', + template_folder=os.path.join(root_path, 'templates'), + static_url_path=cfg.static_url_path) + webapp.config.from_object(cfg) + webapp.root_path = root_path # for some reason calling config.from_xxx() does not have effect + + db_ctl = dbc.DB_Controller(cfg) + rhizi_api.db_ctl = db_ctl + + webapp.rz_config = cfg + return webapp + +def init_config(cfg_dir): + cfg_path = os.path.join(cfg_dir, 'rhizi-server.conf') + cfg = Config.init_from_file(cfg_path) + return cfg + + +if __name__ == "__main__": + + p = argparse.ArgumentParser(description='rhizi-server') + p.add_argument('--config-dir', help='path to Rhizi config dir', default='res/etc') + p.add_argument('--init-htpasswd-db', help='init login htpasswd db', action='store_const', const=True) + args = p.parse_args() + + cfg = init_config(args.config_dir) + log = init_log(cfg) + log.debug('loaded configuration:\n%s' % cfg) + + if args.init_htpasswd_db: + init_pw_db(cfg) + exit(0) + + webapp = init_webapp(cfg) + init_rest_api(cfg, webapp) + + log.info('launching webapp via Flask development server') + webapp.run(host=cfg.listen_address, + port=cfg.listen_port) + diff --git a/src/server/rhizi_server_fcgi.py b/src/server/rhizi_server_fcgi.py new file mode 100755 index 00000000..f0b9a04b --- /dev/null +++ b/src/server/rhizi_server_fcgi.py @@ -0,0 +1,25 @@ +#!/usr/bin/python + +from flup.server.fcgi import WSGIServer +import os +import sys +import cgitb +import rhizi_server + +# sys.path.insert(0, '/srv/www/rhizi/rhizi.net/src-py') + +# enable debugging +cgitb.enable() + +if __name__ == '__main__': + cfg_dir = '/etc/rhizi' + + cfg = rhizi_server.init_config(os.path.join(cfg_dir, 'rhizi-server.conf')) + log = rhizi_server.init_log() + + webapp = rhizi_server.init_webapp(cfg) + rhizi_server.init_rest_api(cfg, webapp) + + log.info('launching webapp via flup.server.fcgi.WSGIServer') + + WSGIServer(webapp).run() diff --git a/src/server/util.py b/src/server/util.py new file mode 100644 index 00000000..e8840114 --- /dev/null +++ b/src/server/util.py @@ -0,0 +1,22 @@ +""" +code with no better place to go +""" +import time + +def debug_log_duration(method): + """ + dubug call durations - use example: + + neo4j_util.post = util.debug_log_duration(neo4j_util.post) + """ + + def timed(*args, **kw): + t_0 = time.time() + result = method(*args, **kw) + t_1 = time.time() + dt = t_1 - t_0 + + print ('%2.2f sec, function: %r' % (dt, method.__name__)) + return result + + return timed |
