diff options
| author | Alon Levy <alon@pobox.com> | 2014-12-16 17:37:42 +0200 |
|---|---|---|
| committer | Alon Levy <alon@pobox.com> | 2014-12-16 17:37:42 +0200 |
| commit | ade19de410e0a789a921ec1666b278b2a64176d6 (patch) | |
| tree | c4c54b106de93609ca7bb70a20fd4be696715dbe /src-py | |
| parent | c7d026b1d504323a8c61d51b726e5e8d2337fc4b (diff) | |
moving files around after repository merger
Diffstat (limited to 'src-py')
| -rw-r--r-- | src-py/crypt_util.py | 46 | ||||
| -rw-r--r-- | src-py/db_controller.py | 495 | ||||
| -rw-r--r-- | src-py/db_driver.py | 91 | ||||
| -rw-r--r-- | src-py/model/__init__.py | 0 | ||||
| -rw-r--r-- | src-py/model/graph.py | 105 | ||||
| -rw-r--r-- | src-py/model/model.py | 37 | ||||
| -rw-r--r-- | src-py/neo4j_util.py | 260 | ||||
| -rw-r--r-- | src-py/rhizi_api.py | 256 | ||||
| -rw-r--r-- | src-py/rhizi_server.py | 263 | ||||
| -rwxr-xr-x | src-py/rhizi_server_fcgi.py | 25 | ||||
| -rw-r--r-- | src-py/util.py | 22 |
11 files changed, 0 insertions, 1600 deletions
diff --git a/src-py/crypt_util.py b/src-py/crypt_util.py deleted file mode 100644 index 464fac94..00000000 --- a/src-py/crypt_util.py +++ /dev/null @@ -1,46 +0,0 @@ -import pickle -import hashlib, uuid -import os -import logging - -log = logging.getLogger('rhizi') - -def add_user_login(config, u, p): - htpasswd_path = config.htpasswd_path - - if False == os.path.exists(htpasswd_path): - with open(htpasswd_path, 'wb') as f: - pickle.dump({}, f) - - with open(htpasswd_path, 'rb') as f: - data = f.read() - pw_db = pickle.loads(data) - - with open(htpasswd_path, 'wb') as f: - salt = config.secret_key - pw_db[u] = hash_pw(str(p), salt) - pickle.dump(pw_db, f) - - log.info('htpasswd db: added entry: user: %s, pw: %s...' % (u, pw_db[u][:5])) - -def hash_pw(pw_str, salt_str): - salt = hashlib.sha512(salt_str).hexdigest() - ret = hashlib.sha512(pw_str + salt).hexdigest() - return ret - -def validate_login(config, u, p): - htpasswd_path = config.htpasswd_path - - salt = config.secret_key - - with open(htpasswd_path) as f: - pw_db = pickle.load(f) - - existing_pw_hash = pw_db.get(u) - if None == existing_pw_hash: - raise Exception('Not autorhized') - - if hash_pw(p, salt) != existing_pw_hash: - raise Exception('Not autorhized') - - diff --git a/src-py/db_controller.py b/src-py/db_controller.py deleted file mode 100644 index c17fd0cd..00000000 --- a/src-py/db_controller.py +++ /dev/null @@ -1,495 +0,0 @@ -#!/usr/bin/python - -import json -import logging -import os -import re -import traceback - -from db_driver import DB_Driver_REST, DB_Driver_Base -from model.graph import Attr_Diff -from model.graph import Topo_Diff -from neo4j_util import DB_result_set -from neo4j_util import cfmt -import neo4j_util as db_util -from model.model import Link - -log = logging.getLogger('rhizi') - -class DB_op(object): - """ - tx wrapped DB operation possibly composing multiple DB queries - """ - def __init__(self): - self.statement_set = [] - self.result_set = [] - self.error_set = None - self.tx_id = None - self.tx_commit_url = None # cached from response to tx begin - - def parse_tx_id(self, tx_commit_url): - m = re.search('/(?P<id>\d+)/commit$', tx_commit_url) - id_str = m.group('id') - self.tx_id = int(id_str) - - def add_statement(self, query, query_params={}): - """ - add a DB query language statement - @return: statement index (zero based) - """ - s = db_util.statement_to_REST_form(query, query_params) - self.statement_set.append(s) - return len(self.statement_set) - - def __iter__(self): - """ - iterate over (statement_index, statement, result, error) - where result & error are mutually exclusive - - note: statement_index is zero based - - TODO: handle partial iteration due to error_set being non-empty - """ - i = 0 - r_set_len = len(self.result_set) - for s in self.statement_set: - r_set = None # row-set - if i < r_set_len: # support partial result recovery - r_set = DB_result_set(self.result_set[i]) - yield (i, s, r_set) - i = i + 1 - - def parse_multi_statement_response_data(self, data): - pass - - @property - def name(self): - return self.__class__.__name__ - - def process_result_set(self): - """ - DB op can issue complex sets of quries all at once - this helper method - assists in parsing response data from a single query. - """ - ret = [] - for _, _, r_set in self: - for row in r_set: - for col in row: - ret.append(col) - return ret - -class DB_composed_op(DB_op): - def __init__(self): - super(DB_composed_op, self).__init__() - self.sub_op_set = [] - - def __assert_false_statement_access(self): - assert False, "composed_op may not contain statements, only sub-ops" - - def add_statement(self, query, query_params={}): - self.__assert_false_statement_access() - - def add_sub_op(self, op): - self.sub_op_set.append(op) - - def __getattribute__(self, attr): - """ - intercept 'statement_set' attr get - """ - if attr == 'statement_set': - self.__assert_false_statement_access() - - return object.__getattribute__(self, attr) - - def __iter__(self): - """ - iterate over sub_op_set - """ - for s_op in self.sub_op_set: - yield s_op - - def process_result_set(self): - ret = [] - for s_op in self: - s_result_set = s_op.process_result_set() - ret.append(s_result_set) - return ret - -class DBO_cypher_query(DB_op): - """ - freeform cypher query - """ - def __init__(self, q, q_params={}): - super(DBO_cypher_query, self).__init__() - self.add_statement(q, q_params) - -class DBO_topo_diff_commit(DB_composed_op): - """ - commit a Topo_Diff - """ - def __init__(self, topo_diff): - super(DBO_topo_diff_commit, self).__init__() - - n_add_map = db_util.meta_attr_list_to_meta_attr_map(topo_diff.node_set_add) - l_add_map = db_util.meta_attr_list_to_meta_attr_map(topo_diff.link_set_add) - l_rm_set = topo_diff.link_set_rm - n_rm_set = topo_diff.node_set_rm - - # - # [!] order critical - # - if len(n_add_map) > 0: - op = DBO_add_node_set(n_add_map) - self.add_sub_op(op) - - if len(l_add_map) > 0: - op = DBO_add_link_set(l_add_map) - self.add_sub_op(op) - - if len(l_rm_set) > 0: - op = DBO_rm_link_set(l_rm_set) - self.add_sub_op(op) - - if len(n_rm_set) > 0: - op = DBO_rm_node_set(n_rm_set) - self.add_sub_op(op) - -class DBO_attr_diff_commit(DB_op): - """ - commit a Attr_Diff - """ - def __init__(self, attr_diff): - super(DBO_attr_diff_commit, self).__init__() - - for id_attr, n_attr_diff in attr_diff.type__node.items(): - # TODO parameterize multiple attr removal - r_attr_set = n_attr_diff['__attr_remove'] - w_attr_set = n_attr_diff['__attr_write'] - - assert len(r_attr_set) > 0 or len(w_attr_set) > 0 - - q_arr = ["match (n {id: {id}}) ", - "return n.id, n"] - q_param_set = {'id': id_attr} - - if len(r_attr_set) > 0: - stmt_attr_rm = "remove " + ', '.join(['n.' + attr for attr in r_attr_set]) - q_arr.insert(1, stmt_attr_rm) - - if len(w_attr_set) > 0: - stmt_attr_set = "set n += {attr_set}" - q_arr.insert(1, stmt_attr_set) - q_param_set['attr_set'] = w_attr_set - - q = " ".join(q_arr) - self.add_statement(q, q_param_set) - - for id_attr, n_attr_diff in attr_diff.type__link.items(): - pass # TODO: handl link attr_diffs - - def process_result_set(self): - ret = {} - for _, _, r_set in self: - for row in r_set: - n_id, n = [v for v in row] # we expect a [n_id, n] array - ret[n_id] = n - return ret - -class DBO_add_node_set(DB_op): - def __init__(self, node_map): - """ - DB op: add node set - - @param node_map: node-type to node-set map - @return: set of new node DB ids - """ - super(DBO_add_node_set, self).__init__() - for q, q_param_set in db_util.gen_query_create_from_node_map(node_map): - self.add_statement(q, q_param_set) - - def process_result_set(self): - id_set = [] - for _, _, row_set in self: - for row in row_set: - for clo in row: - id_set.append(clo) - - return id_set - -class DBO_add_link_set(DB_op): - def __init__(self, link_map): - """ - @param link_map: is a link-type to link-set map - see model.link - @return: set of new node DB ids - """ - super(DBO_add_link_set, self).__init__() - for q, q_params in db_util.gen_query_create_from_link_map(link_map): - self.add_statement(q, q_params) - - def process_result_set(self): - id_set = [] - for _, _, r_set in self: - for row in r_set: - for col_val in row: - id_set.append(col_val) - - return id_set - -class DBO_load_node_set_by_DB_id(DB_op): - def __init__(self, id_set): - """ - load a set of nodes whose DB id is in id_set - - @param id_set: DB node id set - @return: loaded node set or an empty set if no match was found - """ - super(DBO_load_node_set_by_DB_id, self).__init__() - q = "start n=node({id_set}) return n" - self.add_statement(q, { 'id_set': id_set}) - -class DBO_match_node_id_set(DB_op): - - def __init__(self, filter_label=None, filter_attr_map={}): - """ - match a set of nodes by type / attr_map - - @param filter_label: node type filter - @param filter_attr_map: is a filter_key to filter_value_set map of - possible attributes to match against, eg.: - { 'id':[0,1], 'color: ['red','blue'] } - @return: a set of node DB id's - """ - super(DBO_match_node_id_set, self).__init__() - - q = "match (n{filter_label}) {where_clause} return id(n)" - q = cfmt(q, filter_label="" if not filter_label else ":" + filter_label) - q = cfmt(q, where_clause=db_util.gen_clause_where_from_filter_attr_map(filter_attr_map)) - - q_params = filter_attr_map - - self.add_statement(q, q_params) - -class DBO_match_node_set_by_id_attribute(DBO_match_node_id_set): - def __init__(self, id_set): - """ - convenience op: load a set of nodes by their 'id' attribute != DB node id - """ - assert isinstance(id_set, list) - - super(DBO_match_node_set_by_id_attribute, self).__init__(filter_attr_map={'id': id_set}) - - -class DBO_load_link_set(DB_op): - def __init__(self, link_ptr_set): - """ - match a set of sets of links by source/target node id attributes - - This class should be instantiated through a static factory function - - @link_ptr_set link pointer set - @return: a set of loaded links - """ - super(DBO_load_link_set, self).__init__() - - for l_ptr in link_ptr_set: - if not l_ptr.src_id: - q = "match ()-[r]->({id: {dst_id}}) return r" - q_params = {'dst_id': l_ptr.dst_id} - elif not l_ptr.dst_id: - q = "match ({id: {src_id}})-[r]->() return r" - q_params = {'src_id': l_ptr.src_id} - else: - q = "match ({id: {src_id}})-[r]->({id: {dst_id}}) return r" - q_params = {'src_id': l_ptr.src_id, 'dst_id': l_ptr.dst_id} - - self.add_statement(q, q_params) - - @staticmethod - def init_from_link_ptr(l_ptr): - return DBO_load_link_set([l_ptr]) - - @staticmethod - def init_from_link_ptr_set(l_ptr_set): - return DBO_load_link_set(l_ptr_set) - -class DBO_match_link_id_set(DB_op): - def __init__(self, filter_label=None, filter_attr_map={}): - """ - load an id-set of links - - @param filter_label: link type filter - @param filter_attr_map: is a filter_key to filter_value_set map of - attributes to match link properties against - @return: a set of loaded link ids - """ - super(DBO_match_link_id_set, self).__init__() - - q_arr = ['match ()-[r{filter_label} {filter_attr}]->()', - 'return id(r)' - ] - q = ' '.join(q_arr) - q = cfmt(q, filter_label="" if not filter_label else ":" + filter_label) - q = cfmt(q, filter_attr=db_util.gen_clause_attr_filter_from_filter_attr_map(filter_attr_map)) - q_params = {k: v[0] for (k, v) in filter_attr_map.items()} # pass on only first value from each value set - - self.add_statement(q, q_params) - -class DBO_rm_node_set(DB_op): - def __init__(self, id_set, rm_links=False): - """ - remove node set - """ - assert len(id_set) > 0, __name__ + ': empty id set' - - super(DBO_rm_node_set, self).__init__() - - if rm_links: - q_arr = ['match (n)', - 'where n.id in {id_set}', - 'optional match (n)-[r]-()', - 'delete n,r', - 'return {id_set}' - ] - else: - q_arr = ['match (n)', - 'where n.id in {id_set}', - 'delete n', - 'return {id_set}' - ] - - q = ' '.join(q_arr) - q_params = {'id_set': id_set} - self.add_statement(q, q_params) - -class DBO_rm_link_set(DB_op): - def __init__(self, id_set): - """ - remove link set - - [!] when removing as a result of node removal, use DBO_rm_node_set - along with rm_links=True - """ - assert len(id_set) > 0, __name__ + ': empty id set' - - super(DBO_rm_link_set, self).__init__() - - q_arr = ['match ()-[r]->()', - 'where r.id in {id_set}', - 'delete r', - 'return {id_set}' - ] - - q = ' '.join(q_arr) - q_params = {'id_set': id_set} - self.add_statement(q, q_params) - -class DBO_rz_clone(DB_op): - def __init__(self, filter_label=None, limit=128): - """ - clone rhizi - - @return: a dict: {'node_set': n_set, - 'link_set': l_set } - where l_set is a list of (src.id, dst.id, link) tuples - """ - super(DBO_rz_clone, self).__init__() - - self.limit = limit - self.skip = 0 - - q_arr = ['match (n)' if not filter_label else 'match (n:%s)' % (filter_label), - 'optional match (n)-[r]->(m)', - 'with n,r,m', - 'order by n.id', - 'skip %d' % (self.skip), - 'limit %d' % (self.limit), - 'return n,labels(n),collect([m.id, r, type(r)])'] - - q = ' '.join(q_arr) - self.add_statement(q) - - def process_result_set(self): - ret_n_set = [] - ret_l_set = [] - for _, _, row_set in self: - for row in row_set: - n, n_lbl_set, l_set = row.items() # see query return statement - - # reconstruct nodes - assert None != n['id'] - - n['__label_set'] = n_lbl_set - ret_n_set.append(n) - - # reconstruct links from link tuples - for l_tuple in l_set: - assert 3 == len(l_tuple) # see query return statement - - if None == l_tuple[0]: # check if link dst is None - # as link matching is optional, collect may yield empty sets - continue - - l = l_tuple[1] - l['__src_id'] = n['id'] - l['__dst_id'] = l_tuple[0] - l['__label_set'] = [l_tuple[2]] # box single value returned by type() - - ret_l_set.append(l) - - return {'node_set': ret_n_set, - 'link_set': ret_l_set } - -class DB_Controller: - """ - neo4j DB controller - """ - def __init__(self, config, db_driver_class=None): - self.config = config - if not db_driver_class: - self.db_driver = DB_Driver_REST(self.config.db_base_url) - else: - self.db_driver = db_driver_class() - assert isinstance(self.db_driver, DB_Driver_Base) - - def exec_op(self, op): - """ - execute operation within a DB transaction - """ - if isinstance(op, DB_composed_op): - # construct a list comprehension composed of all sup_op statements - for s_op in op.sub_op_set: - self.exec_op(s_op) - return op.process_result_set() - - try: - self.db_driver.begin_tx(op) - self.db_driver.exec_statement_set(op) - self.db_driver.commit_tx(op) - - ret = op.process_result_set() - - log.debug('exec_op:' + op.name + ': return value: ' + str(ret)) - return ret - except Exception as e: - # here we watch for IOExecptions, etc - not db errors - # these are returned in the db response itself - log.error(e.message) - log.error(traceback.print_exc()) - raise e - - def create_db_op(self, f_work, f_cont): - ret = DB_op(f_work, f_cont) - return ret - - def exec_cypher_query(self, q): - """ - @deprecated: use DBO_cypher_query - """ - - # call post and not db_util.post_neo4j to avoid response key errors - try: - db_util.post(self.config.db_base_url + '/db/data/cypher', {"query" : q}) - except Exception as e: - log.error(e.message) - log.error(traceback.print_exc()) - raise e diff --git a/src-py/db_driver.py b/src-py/db_driver.py deleted file mode 100644 index 0532e512..00000000 --- a/src-py/db_driver.py +++ /dev/null @@ -1,91 +0,0 @@ -import logging - -from neo4j_util import Neo4JException -import neo4j_util as db_util - - -log = logging.getLogger('rhizi') - -class DB_Driver_Base(): - - def log_committed_queries(self, statement_set): - for sp_dict in statement_set['statements']: - if None != sp_dict['parameters']: - msg = '\tq: {0}\n\tp: {1}'.format(sp_dict['statement'], - sp_dict['parameters']) - else: - msg = '\tq: {0}'.format(sp_dict['statement']) - log.debug(msg) - -class DB_Driver_Embedded(DB_Driver_Base): - def __init__(self, db_base_url): - self.tx_base_url = db_base_url + '/db/data/transaction' - - from org.rhizi.db.neo4j.util import EmbeddedNeo4j - self.edb = EmbeddedNeo4j.createDb() - self.edb.createDb() - - def begin_tx(self, op): - pass - - def exec_statement_set(self, op): - s_set = op.statement_set - self.edb.executeCypherQury() - - def commit_tx(self, op): - pass - -class DB_Driver_REST(DB_Driver_Base): - def __init__(self, db_base_url): - self.tx_base_url = db_base_url + '/db/data/transaction' - - def begin_tx(self, op): - tx_open_url = self.tx_base_url - - try: - # - # [!] neo4j seems picky about receiving an additional empty statement list - # - data = data = db_util.statement_set_to_REST_form([]) - ret = db_util.post_neo4j(tx_open_url, data) - tx_commit_url = ret['commit'] - op.parse_tx_id(tx_commit_url) - - log.debug('tx-open: id: {0}, commit-url: {1}'.format(op.tx_id, tx_commit_url)) - except Exception as e: - raise Exception('failed to open transaction:' + e.message) - - def exec_statement_set(self, op): - - tx_url = "{0}/{1}".format(self.tx_base_url, op.tx_id) - statement_set = db_util.statement_set_to_REST_form(op.statement_set) - - try: - post_ret = db_util.post_neo4j(tx_url, statement_set) - op.result_set = post_ret['results'] - op.error_set = post_ret['errors'] - if 0 != len(op.error_set): - raise Neo4JException(op.error_set) - - self.log_committed_queries(statement_set) - except Neo4JException as e: - raise e - except Exception as e: - raise Exception('failed exec op statements: err: {0}, url: {1}'.format(e.message, tx_url)) - - def commit_tx(self, op): - tx_commit_url = "{0}/{1}/commit".format(self.tx_base_url, op.tx_id) - - try: - # - # [!] neo4j seems picky about receiving an additional empty statement list - # - data = db_util.statement_set_to_REST_form([]) - ret = db_util.post(tx_commit_url, data) - - log.debug('tx-commit: id: {0}, commit-url: {1}'.format(op.tx_id, tx_commit_url)) - - return ret - except Exception as e: - raise Exception('failed to commit transaction:' + e.message) - diff --git a/src-py/model/__init__.py b/src-py/model/__init__.py deleted file mode 100644 index e69de29b..00000000 --- a/src-py/model/__init__.py +++ /dev/null diff --git a/src-py/model/graph.py b/src-py/model/graph.py deleted file mode 100644 index f197921d..00000000 --- a/src-py/model/graph.py +++ /dev/null @@ -1,105 +0,0 @@ -class Attr_Diff(dict): - """ - Represents a change to note attributes, where nodes can represent - either logical nodes or logical links, and attributes can be added, - changed or removed - - Example: - attr_diff = {'__type_node' : {n_id: {'__attr_write': {'attr_0': 0, - 'attr_1': 'a'}, - '__attr_remove': ['attr_2'] }} - '__type_link' : {l_id: ... } - } - """ - def __init__(self): - self['__type_node'] = {} - self['__type_link'] = {} - - def init_node_attr_diff(self, n_id): - ret = {'__attr_write': {}, - '__attr_remove': []} - self['__type_node'][n_id] = ret - return ret - - @staticmethod - def from_json_dict(json_dict): - ret = Attr_Diff() - for obj_type in ret.keys(): - obj_ad_set = json_dict.get(obj_type) - if None != obj_ad_set: - for o_id, ad in obj_ad_set.items(): - if None != ad.get('__attr_write'): - for k, v in ad['__attr_write'].items(): - ret.add_node_attr_write(o_id, k, v) - if None != ad.get('__attr_remove'): - for k in ad['__attr_remove']: - ret.add_node_attr_rm(o_id, k) - return ret - - @property - def type__node(self): - return self['__type_node'] - - @property - def type__link(self): - return self['__type_link'] - - def add_node_attr_write(self, n_id, attr_name, attr_val): - - assert 'id' != attr_name.lower(), 'Attr_Diff: attempt to write to \'id\' attribute' - - n_attr_diff = self['__type_node'].get(n_id) - if None == n_attr_diff: - n_attr_diff = self.init_node_attr_diff(n_id) - n_attr_diff['__attr_write'][attr_name] = attr_val - - def add_node_attr_rm(self, n_id, attr_name): - n_attr_diff = self['__type_node'].get(n_id) - if None == n_attr_diff: - n_attr_diff = self.init_node_attr_diff(n_id) - n_attr_diff['__attr_remove'].append(attr_name) - - def add_link_attr_write(self, l_id, attr_name, attr_val): - assert False, 'unimplemented' - - def add_link_attr_rm(self, l_id, attr_name): - assert False, 'unimplemented' - -class Topo_Diff(object): - """ - Represents a change to the graph topology - """ - def __init__(self, link_set_rm=[], - node_set_rm=[], - node_set_add=[], - link_set_add=[]): - - self.link_set_rm = link_set_rm - self.node_set_rm = node_set_rm - self.node_set_add = node_set_add - self.link_set_add = link_set_add - - def __str__(self): - return __name__ + ': ' + ', '.join('%s: %s' % (k, v) for k, v in self.__dict__.items()) - - def check_validity(self, topo_diff_dict): - """ - Topo_Diff may represent invalid operations, eg. adding a link while - removing it's end-point - this stub should check for that - """ - pass - - @staticmethod - def from_json_dict(json_dict): - """ - construct from dict - no node/link constructor set must be provided - """ - ret = Topo_Diff() - - # merge keys - this allows constructor argument omission (link_set_rm, - # node_set_rm, etc.) such as when constructing from POST JSON data - for k, _ in ret.__dict__.items(): - v = json_dict.get(k) - if None != v: - ret.__dict__[k] = v - return ret diff --git a/src-py/model/model.py b/src-py/model/model.py deleted file mode 100644 index 33cf9e6a..00000000 --- a/src-py/model/model.py +++ /dev/null @@ -1,37 +0,0 @@ -class Link(): - """ - documentation anchor - this class currently carries no implementation - and only acts as a documentation anchor - - link['__src'] - meta attribute for link source object - link['__dst'] - meta attribute for link destination object - """ - - def __init__(self, src=None, dst=None): - assert False, 'currently unused' - - class Link_Ptr(dict): - """ - link['__src_id'] - meta attribute for link source id - link['__dst_id'] - meta attribute for link destination id - """ - def __init__(self, src_id=None, dst_id=None): - assert None != src_id or None != dst_id - - self['__src_id'] = src_id - self['__dst_id'] = dst_id - - @property - def src_id(self): - return self['__src_id'] - - @property - def dst_id(self): - return self['__dst_id'] - - @staticmethod - def link_ptr(src_id=None, dst_id=None): - """ - init from src_id or dst_id attributes - at least one must be provided - """ - return Link.Link_Ptr(src_id, dst_id) diff --git a/src-py/neo4j_util.py b/src-py/neo4j_util.py deleted file mode 100644 index cf2f962e..00000000 --- a/src-py/neo4j_util.py +++ /dev/null @@ -1,260 +0,0 @@ -""" - Utility code in speaking the neo4j REST api -""" - -import json -import six -from six.moves.urllib import request -import six.moves.urllib_error as urllib_error -import model -import string -import time - -from util import debug_log_duration - -class Neo4JException(Exception): - def __init__(self, error_set): - self.error_set = error_set - - def __str__(self): - return 'neo4j error set: ' + str(self.error_set) - -class DB_row(object): - def __init__(self, data): - self.data = data - - def __iter__(self): - for column_val in self.data: - yield column_val - - def items(self): - return [x for x in self] - -class DB_result_set(object): - def __init__(self, data): - self.data = data - - def __iter__(self): - for db_row_dict in self.data['data']: - # example: dict: {u'row': [{u'title': u'foo'}]} - assert None != db_row_dict['row'] - - yield DB_row(db_row_dict['row']) - - def items(self): - return [x for x in self] - -class Cypher_String_Formatter(string.Formatter): - """ - Despite parameter support in Cypher, we sometimes do engage in query string building - - as both Cypher & Python use brackets to wrap parameters, escaping them in Python makes - queries less readable. This customized formatter will simply ignore unavailable keyworded - formatting arguments, allowing the use of non-escaped parameter designation, eg: - q = cfmt("match (a:{type} {cypher_param})", type='Book') - """ - - def get_field(self, field_name, args, kwargs): - # ignore key not found, return bracket wrapped key - try: - val = super(Cypher_String_Formatter, self).get_field(field_name, args, kwargs) - except (KeyError, AttributeError): - val = "{" + field_name + "}", field_name - return val - -def cfmt(fmt_str, *args, **kwargs): - return Cypher_String_Formatter().format(fmt_str, *args, **kwargs) - -def post_neo4j(url, data): - """ - @return dict object from the neo4j json POST response - """ - ret = post(url, data) - ret_data = json.load(ret) - - # [!] do not raise exception if ret_data['errors'] is not empty - - # this allows query-sets to partially succeed - - return ret_data - -def post(url, data): - assert(isinstance(data, dict)) # make sure we're not handed json strings - - post_data_json = json.dumps(data) - - req = request.Request(url) - req.add_header('User-Agent', 'rhizi-server/0.1') - req.add_header('Accept', 'application/json; charset=UTF-8') - req.add_header('Content-Type', 'application/json') - - req.add_header('X-Stream', 'true') # enable neo4j JSON streaming - - try: - ret = request.urlopen(req, post_data_json) - except urllib_error.HTTPError as e: - raise Exception('post request failed: code: {0}, reason: {1}'.format(e.code, e.reason)) - - return ret - -def statement_to_REST_form(query, parameters={}): - """ - turn cypher query to neo4j json API format - """ - assert isinstance(query, six.string_types) - if isinstance(parameters, list): - for v in parameters: - assert isinstance(v, dict) - else: - assert isinstance(parameters, dict) - - return {'statement' : query, 'parameters': parameters} - -def statement_set_to_REST_form(statement_set): - assert isinstance(statement_set, list) - - return {'statements': statement_set} - -def gen_clause_attr_filter_from_filter_attr_map(filter_attr_map, node_label="n"): - if not filter_attr_map: - return "{}" - - __type_check_filter_attr_map(filter_attr_map) - - filter_arr = [] - for attr_name in filter_attr_map.keys(): - # create a cypher query parameter place holder for each attr set - # eg. n.foo in {foo}, where foo is passed as a query parameter - f_attr = cfmt("{attr_name}: {{{attr}}}", attr_name=attr_name) - filter_arr.append(f_attr) - - filter_str = "{{{0}}}".format(', '.join(filter_arr)) - return filter_str - -def gen_clause_where_from_filter_attr_map(filter_attr_map, node_label="n"): - """ - convert a filter attribute map to a parameterized Cypher where clause, eg. - in: { 'att_foo': [ 'a', 'b' ], 'att_goo': [1,2] } - out: {att_foo: {att_foo}, att_goo: {att_goo}, ...} - - this function will essentially ignore all but the first value in the value list - - @param filter_attr_map: may be None or empty - """ - if not filter_attr_map: - return "" - - __type_check_filter_attr_map(filter_attr_map) - - filter_arr = [] - for attr in filter_attr_map.keys(): - # create a cypher query parameter place holder for each attr set - # eg. n.foo in {foo}, where foo is passed as a query parameter - f_attr = cfmt("{node_label}.{attr} in {{{attr}}}", node_label=node_label, attr=attr) - filter_arr.append(f_attr) - filter_str = "where {0}".format(' and '.join(filter_arr)) - return filter_str - -def gen_query_create_from_node_map(node_map, input_to_DB_property_map=lambda _: _): - """ - generate a set of node create queries - - @param node_map: is a node-type to node map - @input_to_DB_property_map: optional function which takes a map of input properties and returns a map of DB properties - use to map input schemas to DB schemas - - @return: a (query, query_parameteres) set of create queries - """ - __type_check_link_or_node_map(node_map) - - ret = [] - for label, n_set in node_map.items(): - - assert len(label) > 2 and label[0].isupper() and label[1:].islower(), 'malformed label: ' + label - - q_arr = ['create (n:%s {node_attr})' % (label), - 'return n.id' - ] - - q = ' '.join(q_arr) - q_params_set = [] - for n_prop_set in n_set: - - assert None != n_prop_set['id'], 'node create query: node id attribute not set' - - q_params = input_to_DB_property_map(n_prop_set) - q_params_set.append(q_params) - ret.append((q, {'node_attr': q_params_set})) - return ret - -def gen_query_create_from_link_map(link_map, input_to_DB_property_map=lambda _: _): - """ - generate a set of link create queries - - @param link_map: is a link-type to link map - see model.link - """ - __type_check_link_or_node_map(link_map) - - ret = [] - for l_type, l_set in link_map.items(): - q = "match (src {id: {src}.id}),(dst {id: {dst}.id}) " + \ - "create (src)-[r:%(__type)s {link_attr}]->(dst) " + \ - "return id(r)" - q = q % {'__type':l_type} - - for link in l_set: - __type_check_link(link) - - src_id = link['__src_id'] - dst_id = link['__dst_id'] - - # TODO: use object based link representation - prop_dict = link.copy() - del prop_dict['__dst_id'] - del prop_dict['__src_id'] - - q_params = {'src': { 'id': src_id} , - 'dst': { 'id': dst_id} , - 'link_attr' : input_to_DB_property_map(prop_dict)} - ret.append((q, q_params)) - - return ret - -def meta_attr_list_to_meta_attr_map(e_set, meta_attr='__label_set'): - """ - convert a list of maps each containing a meta_attr key into a - meta_attr-mapped collection of lists with the meta_attr removed - eg: - - in: [{'id':0, '__type': 'T'}, {'id':1, '__type': 'T'}] - out: { 'T', [{'id':0}, {'id':1}] } - """ - ret = {} - for v in e_set: - assert None != v[meta_attr], 'missing type meta-attribute' - assert 1 == len(v[meta_attr]), 'only single-type mapping currently suppoerted' - - v_type = v[meta_attr][0] - if None == ret.get(v_type): # init type list if necessary - ret[v_type] = [] - - v_no_meta = v.copy() - del v_no_meta[meta_attr] - - ret[v_type].append(v_no_meta) - - return ret - -def __type_check_link(link): - assert link.has_key('__src_id') - assert link.has_key('__dst_id') - -def __type_check_link_or_node_map(x_map): - for k, v in x_map.iteritems(): # do some type sanity checking - assert isinstance(k, six.string_types) - assert isinstance(v, list) - -def __type_check_filter_attr_map(filter_attr_map): - """ - # type sanity check an attribute filter map - """ - assert isinstance(filter_attr_map, dict) - for k, v in filter_attr_map.items(): - assert isinstance(k, six.string_types) - assert isinstance(v, list) diff --git a/src-py/rhizi_api.py b/src-py/rhizi_api.py deleted file mode 100644 index 579d41e1..00000000 --- a/src-py/rhizi_api.py +++ /dev/null @@ -1,256 +0,0 @@ -""" -Rhizi web API -""" -import os -import db_controller as dbc -import json -import logging -import traceback -import crypt_util - -import flask -from flask import jsonify -from flask import Flask -from flask import request -from flask import make_response -from flask import session -from flask import redirect -from flask import escape -from flask import url_for -from flask import render_template -from flask import send_from_directory - -from model.graph import Topo_Diff -from model.graph import Attr_Diff -from model.model import Link -from datetime import datetime - -log = logging.getLogger('rhizi') - -# injected: DB controller -db_ctl = None - -def __sanitize_input(*args, **kw_args): - pass - -def sanitize_input__node(n): - """ - provide a control point as to which node fields are persisted - """ - assert None != n['id'], 'invalid input: node: missing id' - -def sanitize_input__link(l): - """ - provide a control point as to which node fields are persisted - """ - assert None != l['id'], 'invalid input: link: missing id' - assert None != l['__src_id'], 'invalid input: link: missing src id' - assert None != l['__dst_id'], 'invalid input: link: missing dst id' - -def sanitize_input__topo_diff(topo_diff): - for n in topo_diff.node_set_add: - sanitize_input__node(n) - for l in topo_diff.link_set_add: - sanitize_input__link(l) - -def sanitize_input__attr_diff(attr_diff): - pass # TODO: impl - -def __response_wrap(data=None, error=None): - """ - wrap response data/errors as dict - this should always be used when returning - data to allow easy return of list objects, assist in error case distinction, etc. - """ - return dict(data=data, error=error) - -def __common_resp_handle(data=None, error=None): - """ - provide common response handling - """ - ret_data = __response_wrap(data, error) - resp = jsonify(ret_data) - - resp.headers['Access-Control-Allow-Origin'] = '*' - - # more response processing - - return resp - -def __common_exec(op, on_success=__common_resp_handle): - try: - op_ret = db_ctl.exec_op(op) - return on_success(op_ret) - except Exception as e: - log.error(e.message) - log.error(traceback.print_exc()) - return __common_resp_handle('error occurred') - -def load_node_set_by_id_attr(): - """ - load node-set by ID attribute - - @param id_set: list of node ids to match id attribute against - @return: a list of nodes whose id attribute matches 'id' or - an empty list if the requested node is not found - @raise exception: on error - """ - req_json = request.get_json() - id_set = req_json['id_set'] - - __sanitize_input(id_set) - - return __load_node_set_by_id_attr_common(id_set) - -def __load_node_set_by_id_attr_common(id_set): - """ - @param f_k: optional attribute filter key - @param f_vset: possible key values to match against - """ - op = dbc.DBO_match_node_set_by_id_attribute(id_set=id_set) - try: - n_set = db_ctl.exec_op(op) - return __common_resp_handle(data=n_set) - except Exception as e: - log.exception(e) - return __common_resp_handle(error='unable to load node with ids: {0}'.format(id_set)) - -def match_node_set_by_attr_filter_map(attr_filter_map): - """ - @param attr_filter_map - - @return: a set of node DB id's - """ - op = dbc.DBO_match_node_id_set(attr_filter_map) - return __common_exec(op) - -def load_link_set_by_link_ptr_set(): - - def deserialize_param_set(param_json): - l_ptr_set_raw = param_json['link_ptr_set'] - - __sanitize_input(l_ptr_set_raw) - - l_ptr_set = [] - for lptr_dict in l_ptr_set_raw: - src_id = lptr_dict.get('__src_id') - dst_id = lptr_dict.get('__dst_id') - l_ptr_set += [Link.Link_Ptr(src_id=src_id, dst_id=dst_id) ] - - return l_ptr_set - - l_ptr_set = deserialize_param_set(request.get_json()) - - op = dbc.DBO_load_link_set.init_from_link_ptr_set(l_ptr_set) - return __common_exec(op) - -def rz_clone(): - op = dbc.DBO_rz_clone() - return __common_exec(op) - -def diff_commit__set(): - """ - commit a diff set - """ - def sanitize_input(req): - diff_set_dict = request.get_json()['diff_set'] - topo_diff_dict = diff_set_dict['__diff_set_topo'][0] - topo_diff = Topo_Diff.from_json_dict(topo_diff_dict) - - sanitize_input__topo_diff(topo_diff) - return topo_diff; - - topo_diff = sanitize_input(request) - op = dbc.DBO_topo_diff_commit(topo_diff) - return __common_exec(op) - -def diff_commit__topo(): - """ - commit a graph topology diff - """ - def sanitize_input(req): - topo_diff_dict = request.get_json()['topo_diff'] - topo_diff = Topo_Diff.from_json_dict(topo_diff_dict) - - sanitize_input__topo_diff(topo_diff) - return topo_diff; - - topo_diff = sanitize_input(request) - op = dbc.DBO_topo_diff_commit(topo_diff) - return __common_exec(op) - -def diff_commit__attr(): - """ - commit a graph attribute diff - """ - def sanitize_input(req): - attr_diff_dict = request.get_json()['attr_diff'] - attr_diff = Attr_Diff.from_json_dict(attr_diff_dict) - - sanitize_input__attr_diff(attr_diff) - return attr_diff; - - attr_diff = sanitize_input(request) - op = dbc.DBO_attr_diff_commit(attr_diff) - return __common_exec(op) - -def diff_commit__vis(): - pass - -def add_node_set(): - """ - @deprecated: use topo_attr_commit - - @param node_map: node type to node map, eg. { 'Skill': { 'name': 'kung-fu' } } - """ - node_map = request.get_json()['node_map'] - __sanitize_input(node_map) - - op = dbc.DBO_add_node_set(node_map) - return __common_exec(op) - -def monitor__server_info(): - """ - server monitor stub - """ - dt = datetime.now() - return "<html><body>" + \ - "<h1>Rhizi Server v0.1</h1><p>" + \ - "date: " + dt.strftime("%Y-%m-%d") + "<br>" + \ - "time: " + dt.strftime("%H:%M:%S") + "<br>" + \ - "</p></body></html>" - -def index(): - username = escape(session.get('username')) - return render_template('index.html', username=username) - -def login(): - - def sanitize_input(req): - req_json = request.get_json() - u = req_json['username'] - p = req_json['password'] - return u, p - - if request.method == 'POST': - try: - u, p = sanitize_input(request) - crypt_util.validate_login(flask.current_app.rz_config, u, p) - except Exception as e: - # login failed - log.warn('login: unauthorized: user: %s' % (u)) - return render_template('login.html', login_failed=True) - - # login successful - session['username'] = u - log.debug('login: success: user: %s' % (u)) - return redirect(url_for('index')) - - if request.method == 'GET': - return render_template('login.html') - -def logout(): - # remove the username from the session if it's there - u = session.pop('username', None) - log.debug('logout: success: user: %s' % (u)) - return redirect(url_for('login')) - diff --git a/src-py/rhizi_server.py b/src-py/rhizi_server.py deleted file mode 100644 index 168e2f53..00000000 --- a/src-py/rhizi_server.py +++ /dev/null @@ -1,263 +0,0 @@ -#!/usr/bin/python - -import logging -import json -import util -import os -import neo4j_util -import argparse -import db_controller as dbc -import rhizi_api -import flask -import crypt_util -import re - -from flask import Flask -from flask import session -from flask import redirect -from flask import request -from flask import send_from_directory - -from functools import wraps - -class Config(object): - """ - rhizi-server configuration - - TODO: config option documentation - - htpasswd_path - listen_address - listen_port - neo4j_url - root_path - """ - - @staticmethod - def init_from_file(file_path): - - if False == os.path.exists(file_path): - raise Exception('config file not found: ' + file_path) - - # apply defaults - cfg = {} - cfg['access_control'] = True - cfg['config_dir'] = os.path.abspath(os.path.dirname(file_path)) # bypass prop restriction - cfg['development_mode'] = False - cfg['listen_address'] = '127.0.0.1' - cfg['listen_port'] = 8080 - cfg['root_path'] = os.getcwd() - cfg['static_url_path'] = '/static' - - # Flask keys - cfg['SECRET_KEY'] = '' - - with open(file_path, 'r') as f: - for line in f: - if re.match('(^#)|(\s+$)', line): - continue - - kv_arr = line.split('=') - if 2 != len(kv_arr): - raise Exception('failed to parse config line: ' + line) - - k, v = map(str.strip, kv_arr) - - if None != cfg.get(k): - # apply type conversion based on default value type - type_f = type(cfg[k]) - if bool == type_f: - v = v in ("True", "true") # workaround bool('false') = True - else: - v = type_f(v) - - # [!] we can't use k.lower() as we are loading Flask configuration - # keys which are expected to be capitalized - cfg[k] = v - - ret = Config() - ret.__dict__ = cfg # allows setting of @property attributes - - # validate config - if False == os.path.isabs(ret.root_path): - ret.root_path = os.path.abspath(ret.root_path) - - return ret - - def __str__(self): - return '\n'.join('%s: %s' % (k, v) for k, v in self.__dict__.items()) - - @property - def db_base_url(self): - return self.neo4j_url - - @property - def tx_api_path(self): - return '/db/data/transaction' - - @property - def config_dir_path(self): - return self.config_dir - - @property - def secret_key(self): - return self.SECRET_KEY - -class FlaskExt(Flask): - """ - Flask server customization - """ - - def __init__(self, import_name, *args, **kwargs): - """ - reserved for future use - """ - super(FlaskExt, self).__init__(import_name, *args, **kwargs) - - def before_request(self, *args, **kwargs): - # TODO impl - pass - - def make_default_options_response(self): - ret = Flask.make_default_options_response(self) - - ret.headers['Access-Control-Allow-Origin'] = 'http://rhizi.net' - ret.headers['Access-Control-Allow-Headers'] = "Accept, Authorization, Content-Type, Origin" - ret.headers['Access-Control-Allow-Credentials'] = 'true' - - # ret.headers['Access-Control-Allow-Methods'] = ', '.join(m_list) - return ret - -def init_log(cfg): - """ - init log file, location derived from configuration - """ - log = logging.getLogger('rhizi') - log.setLevel(logging.DEBUG) - log_handler_c = logging.StreamHandler() - log_handler_f = logging.FileHandler(cfg.log_path) - - log.addHandler(log_handler_c) - log.addHandler(log_handler_f) - return log - -def init_rest_api(cfg, flask_webapp): - """ - map REST API calls - """ - - def rest_entry(path, f, flask_args={'methods': ['POST']}): - return (path, f, flask_args) - - def dev_mode__resend_from_static(static_url): - """ - redirect broken-on-local-deploy links: - - /src -> '': handle root based files, eg. app.js - - /res, /lib -> res, lib - """ - static_folder = flask.current_app.static_folder - - static_path = request.path - if static_path.startswith('/src'): - # TODO: clean - /src/... links should not exist - static_path = static_path.replace('/src', '') - if static_path.startswith('/'): # convert to relative path - static_path = static_path[1:] - return send_from_directory(static_folder, static_path) - - def login_decorator(f): - """ - [!] security boundary: asserd logged-in user before executing REST api call - """ - @wraps(f) - def wrapped_function(*args, **kw): - if not 'username' in session: - return redirect('/login') - return f(*args, **kw) - - return wrapped_function - - rest_entry_set = [ - rest_entry('/add/node-set' , rhizi_api.add_node_set), - rest_entry('/graph/clone', rhizi_api.rz_clone), - rest_entry('/graph/diff-commit-set', rhizi_api.diff_commit__set), - rest_entry('/graph/diff-commit-topo', rhizi_api.diff_commit__topo), - rest_entry('/graph/diff-commit-attr', rhizi_api.diff_commit__attr), - rest_entry('/graph/diff-commit-vis', rhizi_api.diff_commit__vis), - rest_entry('/index', rhizi_api.index, {'methods': ['GET']}), - rest_entry('/load/node-set-by-id', rhizi_api.load_node_set_by_id_attr), - rest_entry('/load/link-set/by_link_ptr_set', rhizi_api.load_link_set_by_link_ptr_set), - rest_entry('/login', rhizi_api.login, {'methods': ['GET', 'POST']}), - rest_entry('/logout', rhizi_api.logout, {'methods': ['GET', 'POST']}), - rest_entry('/match/node-set', rhizi_api.match_node_set_by_attr_filter_map), - rest_entry('/monitor/server-info', rhizi_api.monitor__server_info), - ] - - if cfg.development_mode: - dev_path_set = ['/src', '/res', '/lib'] - rest_dev_entry_set = [] - for dev_path in dev_path_set: - rest_dev_entry_set.append(rest_entry(dev_path + '/<path:static_url>', - dev_mode__resend_from_static, - {'methods': ['GET']})) - rest_entry_set += rest_dev_entry_set - - if False == cfg.access_control: - log.warn('access control disabled, public access set on all URLs') - - for re_entry in rest_entry_set: - rest_path, f, flask_args = re_entry - - if cfg.access_control and '/login' != rest_path: - # currently require login on all but /login paths - f = login_decorator(f) - - # [!] order seems important - apply route decorator last - route_dec = flask_webapp.route(rest_path, **flask_args) - f = route_dec(f) - - flask_webapp.f = f # assign decorated function - -def init_webapp(cfg): - root_path = cfg.root_path - webapp = FlaskExt(__name__, - static_folder='static', - template_folder=os.path.join(root_path, 'templates'), - static_url_path=cfg.static_url_path) - webapp.config.from_object(cfg) - webapp.root_path = root_path # for some reason calling config.from_xxx() does not have effect - - db_ctl = dbc.DB_Controller(cfg) - rhizi_api.db_ctl = db_ctl - - webapp.rz_config = cfg - return webapp - -def init_config(cfg_dir): - cfg_path = os.path.join(cfg_dir, 'rhizi-server.conf') - cfg = Config.init_from_file(cfg_path) - return cfg - - -if __name__ == "__main__": - - p = argparse.ArgumentParser(description='rhizi-server') - p.add_argument('--config-dir', help='path to Rhizi config dir', default='res/etc') - p.add_argument('--init-htpasswd-db', help='init login htpasswd db', action='store_const', const=True) - args = p.parse_args() - - cfg = init_config(args.config_dir) - log = init_log(cfg) - log.debug('loaded configuration:\n%s' % cfg) - - if args.init_htpasswd_db: - init_pw_db(cfg) - exit(0) - - webapp = init_webapp(cfg) - init_rest_api(cfg, webapp) - - log.info('launching webapp via Flask development server') - webapp.run(host=cfg.listen_address, - port=cfg.listen_port) - diff --git a/src-py/rhizi_server_fcgi.py b/src-py/rhizi_server_fcgi.py deleted file mode 100755 index f0b9a04b..00000000 --- a/src-py/rhizi_server_fcgi.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/python - -from flup.server.fcgi import WSGIServer -import os -import sys -import cgitb -import rhizi_server - -# sys.path.insert(0, '/srv/www/rhizi/rhizi.net/src-py') - -# enable debugging -cgitb.enable() - -if __name__ == '__main__': - cfg_dir = '/etc/rhizi' - - cfg = rhizi_server.init_config(os.path.join(cfg_dir, 'rhizi-server.conf')) - log = rhizi_server.init_log() - - webapp = rhizi_server.init_webapp(cfg) - rhizi_server.init_rest_api(cfg, webapp) - - log.info('launching webapp via flup.server.fcgi.WSGIServer') - - WSGIServer(webapp).run() diff --git a/src-py/util.py b/src-py/util.py deleted file mode 100644 index e8840114..00000000 --- a/src-py/util.py +++ /dev/null @@ -1,22 +0,0 @@ -""" -code with no better place to go -""" -import time - -def debug_log_duration(method): - """ - dubug call durations - use example: - - neo4j_util.post = util.debug_log_duration(neo4j_util.post) - """ - - def timed(*args, **kw): - t_0 = time.time() - result = method(*args, **kw) - t_1 = time.time() - dt = t_1 - t_0 - - print ('%2.2f sec, function: %r' % (dt, method.__name__)) - return result - - return timed |
