From 6999761762aa773234de1bb642e91026475ca234 Mon Sep 17 00:00:00 2001 From: LV-426 Date: Mon, 6 Apr 2015 13:37:07 +0300 Subject: mving cri domain resources under cri.rhizi.net --- src/local/domain/cri.rhizi.net/__init__.py | 0 src/local/domain/cri.rhizi.net/domain_cri.py | 140 +++++++++++ src/local/domain/cri.rhizi.net/from_csv | 271 +++++++++++++++++++++ src/local/domain/cri.rhizi.net/test_domain__CRI.py | 58 +++++ src/local/domain/cri/__init__.py | 0 src/local/domain/cri/domain_cri.py | 140 ----------- src/local/domain/cri/from_csv | 271 --------------------- src/local/domain/cri/test_domain__CRI.py | 58 ----- 8 files changed, 469 insertions(+), 469 deletions(-) create mode 100644 src/local/domain/cri.rhizi.net/__init__.py create mode 100644 src/local/domain/cri.rhizi.net/domain_cri.py create mode 100755 src/local/domain/cri.rhizi.net/from_csv create mode 100644 src/local/domain/cri.rhizi.net/test_domain__CRI.py delete mode 100644 src/local/domain/cri/__init__.py delete mode 100644 src/local/domain/cri/domain_cri.py delete mode 100755 src/local/domain/cri/from_csv delete mode 100644 src/local/domain/cri/test_domain__CRI.py (limited to 'src') diff --git a/src/local/domain/cri.rhizi.net/__init__.py b/src/local/domain/cri.rhizi.net/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/local/domain/cri.rhizi.net/domain_cri.py b/src/local/domain/cri.rhizi.net/domain_cri.py new file mode 100644 index 00000000..db6ed925 --- /dev/null +++ b/src/local/domain/cri.rhizi.net/domain_cri.py @@ -0,0 +1,140 @@ +from db_op import DB_op + +class DBO_random_data_generation__domain__CRI(DB_op): + + def __init__(self, lim_n=50, + lim_r=10000, + prob_link_create=0.03): + """ + generate random data: CRI domain: Person x Skill + """ + super(DBO_random_data_generation__domain__CRI, self).__init__() + + n_attr_set__skill_raw = ['Cryptocurrency', + 'Database architecture', + 'Web-development', + 'Mobile-development', + 'Machine learning', + 'Guitar Playing', + 'Image processing', + 'Algebra', + 'Calculus', + 'Molecular-Biology', + 'Graphic design', + 'Geo-location services', + 'Drone-building', + 'Artificial-intelligence', + 'Distributed information systems', + 'Wordpress', + 'Woodworking', + 'Quality-control', + 'Video-editing', + 'Soldering', + 'Network engineering', + 'GIT', + 'Electronic music', + 'Network administration'] + + n_attr_set__skill = [] + for s in n_attr_set__skill_raw: + n_attr_set__skill.append(s) + + n_attr_set__name = ['John', + 'William', + 'James', + 'Charles', + 'George', + 'Frank', + 'Joseph', + 'Thomas', + 'Henry', + 'Robert', + 'Edward', + 'Harry', + 'Walter', + 'Arthur', + 'Fred', + 'Albert', + 'Samuel', + 'David', + 'Louis', + 'Joe', + 'Charlie', + 'Clarence', + 'Richard', + 'Andrew', + 'Daniel', + 'Ernest', + 'Mary', + 'Anna', + 'Emma', + 'Elizabeth', + 'Minnie', + 'Margaret', + 'Ida', + 'Alice', + 'Bertha', + 'Sarah', + 'Annie', + 'Clara', + 'Ella', + 'Florence', + 'Cora', + 'Martha', + 'Laura', + 'Nellie', + 'Grace', + 'Carrie', + 'Maude', + 'Mabel', + 'Bessie', + 'Jennie', + 'Gertrude', + 'Julia'] + + l_attr_set__level = ['Novice', 'Intermediate', 'Expert'] + + + min_id = 10000 # separate group ids with numerical perfix + + # create persons + q_arr = ['with {n_attr_set__name} as n_attr_set', # TODO clean: foreach triggers SyntaxException: otherwise + 'foreach (idx in range(0,%d)' % (lim_n - 1), + '|', + 'create (n:%s' % ('Person'), + '{id: \'test-id_\' + toString(%d + idx),' % (min_id), + 'name: n_attr_set[idx %% %d]}' % (len(n_attr_set__name)), + '))', + ] + q_param = {'n_attr_set__name': n_attr_set__name} + self.add_statement(q_arr, q_param) + min_id *= 2 + + # create skills + q_arr = ['with {skill_set} as skill_set', # TODO clean: foreach triggers SyntaxException: otherwise + 'foreach (idx in range(0,%d)' % (len(n_attr_set__skill) - 1), + '|', + 'create (n:%s' % ('Skill'), + '{id: \'test-id_\' + toString(idx + %d),' % (min_id), + 'name: skill_set[idx]}', + '))', + ] + q_param = {'skill_set': n_attr_set__skill} + self.add_statement(q_arr, q_param) + min_id *= 2 + + # create links + for skill_level in l_attr_set__level: + q_arr = ['match (n:%s),(m:%s)' % ('Person', 'Skill'), + 'with n, m, rand() as rand', + 'order by rand', + 'limit %d' % (lim_r - 1), + 'where rand() < %.2f' % (prob_link_create), + 'create (n)-[r:%s' % (skill_level), + '{id: \'test-id_\' + toString(%d + toInt(%d * rand())),' % (min_id, lim_r * 100000), # aim for low id collision probability, + 'proficiency: \'%s\'}' % (skill_level), + ']->(m)', + 'return collect(r.id)', + ] + + self.add_statement(q_arr, q_param) diff --git a/src/local/domain/cri.rhizi.net/from_csv b/src/local/domain/cri.rhizi.net/from_csv new file mode 100755 index 00000000..c11babd0 --- /dev/null +++ b/src/local/domain/cri.rhizi.net/from_csv @@ -0,0 +1,271 @@ +#!/usr/bin/python2.7 + +""" +Import CSV files into Rhizi Server via pythonic API. + +Tricky, since this is the only API user. Try to use it exactly as the REST/WS +API would, just without actually creating a socket connection. +""" + +import sys +import os +import argparse +import string +import csv + +root = os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..', '..', '..')) +print("root = %s" % root) +sys.path.append(os.path.join(root, 'src', 'server')) + +from rz_api_common import sanitize_input__topo_diff +from rz_kernel import RZ_Kernel +import db_controller as dbc +from rz_server import Config + +from model.graph import Topo_Diff + +def topo_diff_json(node_set_add=[], link_set_add=[]): + topo_diff_dict = ({ + u'link_id_set_rm': [], + u'link_set_add': link_set_add, + u'drop_conjugator_links': True, + u'node_set_add': node_set_add, + u'node_id_set_rm': [] + }) + topo_diff = Topo_Diff.from_json_dict(topo_diff_dict) + sanitize_input__topo_diff(topo_diff) + return topo_diff; + +def commit(topo_diff): + _, commit_ret = kernel.diff_commit__topo(topo_diff, ctx) + +# Rhizi constants - FIXME use API +PERSON_LABEL = 'Person' +INTEREST_LABEL = 'Interest' +INTERNSHIP_LABEL = "Internship" +LABEL_SET = '__label_set' + +# CSV file columns +PERSONAL_EMAIL = 'Personal e-mail' +INTEREST = 'Interests' + +id_count = 0 +def next_id(): + global id_count + id_count += 1 + return '%08d' % id_count + +def node_dict(the_id, **args): + """ FIXME use API """ + ret = dict(args) + ret['id'] = the_id + return ret + +def link_dict(source_id, target_id, label, the_id): + return ({ + u'__type': [label], + u'__src_id': source_id, + u'__dst_id': target_id, + u'id': the_id + }) + +def read_csv(filename): + for dialect in ['excel', 'excel-tab']: + lines = list(csv.reader(open(filename), dialect=dialect)) + lens = map(len, lines) + if max(lens) == min(lens) and max(lens) > 2: + return lines + raise Exception("no dialect found") + +class CSV(object): + def __init__(self, filename): + self.filename = filename + rows = read_csv(filename) + headers = rows[0] + self.parse_headers(headers) + #import pdb; pdb.set_trace() + self.rows = rows[1:] + self.row_dicts = map(lambda fields: dict(zip(headers, fields)), self.rows) + + def run(self): + print(" **************************** run on data from %r ****************" % self.filename) + self.nodes_dict = {} + self.node_set_add = [] + self.link_set_add = [] + self.generate_nodes_and_links() + #for node in node_set_add: + # print('committing %s' % repr(node)) + # commit(topo_diff_json(node_set_add=[node])) + commit(topo_diff_json(node_set_add=self.node_set_add)) + commit(topo_diff_json(link_set_add=self.link_set_add)) + + def append_id_node(self, the_id, node): + if the_id in self.nodes_dict: + return + print(repr(node)) + self.nodes_dict[the_id] = node + self.node_set_add.append(node) + return the_id + + def _csv_row_to_node(self, d, dict_gen): + the_id = next_id() + print(repr(d)) + return the_id, node_dict(the_id=the_id, **dict_gen(d)) + + def append_link(self, source_id, target_id, label): + the_id = next_id() + link = link_dict(source_id, target_id, label, the_id) + self.link_set_add.append(link) + print(repr(link)) + return the_id + +class Persons(object): + def __init__(self): + self.by_name = {} + self.by_email = {} + + def id_from_something(self, first_name, last_name, email): + return self.by_name.get(self.name_from_first_and_last(first_name, last_name), self.by_email.get(email, (None, None)))[0] + + def name_from_first_and_last(self, first_name, last_name): + return first_name + ' ' + last_name + + def addPersonNode(self, first_name, last_name, email): + ret = ret2 = None + name = self.name_from_first_and_last(first_name, last_name) + if name in self.by_name: + ret = self.by_name[name] + if email in self.by_email: + ret2 = self.by_email[email] + if ret != ret2: + print("person node similar but not same for: %r, %r, %r" % (first_name, last_name, email)) + print(ret2) + if ret2 and not ret: + ret = ret2 + if ret: + return ret + node_id = next_id() + node = node_dict(the_id=node_id, name=name, description=email, **{LABEL_SET: [PERSON_LABEL]}) + ret = (node_id, node) + self.by_name[name] = ret + self.by_email[email] = ret + return ret + +persons = Persons() +addPersonNode = persons.addPersonNode + +def cleanEmailField(unclean): + return unclean.lower().strip() + +class StudentCSV(CSV): + + def __init__(self, filename, **kw): + super(StudentCSV, self).__init__(filename=filename, **kw) + + def interests(self, d): + return d[self.interestsField] + + def generate_nodes_and_links(self): + for d in self.row_dicts: + d[self.interestsField] = map(string.strip, d[self.interestsField].split(',')) + person_id = next_id() + person_email = cleanEmailField(d[self.personalEmailField]) + person_id, person_node = addPersonNode(first_name=d[self.firstNameField], last_name=d[self.lastNameField], email=person_email) + self.append_id_node(person_id, person_node) + for interest in self.interests(d): + interest_id = next_id() + interest_node = node_dict(the_id=interest_id, + **{'name':interest, LABEL_SET:[INTEREST_LABEL]}) + interest_id = self.append_id_node(interest_id, interest_node) + self.append_link(person_id, interest_id, 'Is interested in') + + def parse_headers(self, headers): + self.firstNameField = headers[0] + self.lastNameField = headers[1] + self.personalEmailField = headers[2] + self.interestsField = headers[3] + +descriptionTemplate = """Nature: %s +-------- +Abstract: +%s""" + +class StudentInternshipsCSV(CSV): + + def __init__(self, filename, **kw): + super(StudentInternshipsCSV, self).__init__(filename=filename, **kw) + + def parse_headers(self, headers): + it = iter(headers) + self.firstNameField = it.next() + self.lastNameField = it.next() + self.emailField = it.next() + self.internshipTitleField = it.next() + self.internshipNatureField = it.next() + self.abstractField = it.next() + self.dateStartField = it.next() + self.dateEndField = it.next() + self.supervisorFirstNameField = it.next() + self.supervisorLastNameField = it.next() + self.supervisorTitleField = it.next() + self.supervisorEmailField = it.next() + self.laboratoryNameField = it.next() + self.laboratoryAffiliationField = it.next() + self.unitCodeField = it.next() # CNRS / INSERM unit code + self.streetField = it.next() # Street + self.streetSecondPartField = it.next() # Street (continued) + self.cityField = it.next() # City + self.countryField = it.next() # Country + + def generate_nodes_and_links(self): + for d in self.row_dicts: + internship_id = next_id() + # FIXME should be title. fix requires client change. need + # configurable fields, or a list of names and types for display per + # node (using a type node in the db, which could be cached + title = d[self.internshipTitleField].strip() + if len(title) == 0: + print("skipping empty internship field") + continue + input_dict = { + LABEL_SET:[INTERNSHIP_LABEL], + 'name': title, + 'description': descriptionTemplate % (d[self.internshipNatureField], d[self.abstractField]), + 'startdate': d[self.dateStartField], + 'enddate': d[self.dateEndField], + 'internship-type': d[self.internshipNatureField], + 'cnrs-inserm-unit-code': d[self.unitCodeField], + 'city': d[self.cityField], + 'country': d[self.countryField], + 'street-address': d[self.streetField] + d[self.streetSecondPartField], + 'facility': d[self.laboratoryNameField], + 'facility-affiliation': d[self.laboratoryAffiliationField], + } + internship_node = node_dict(the_id=internship_id, **input_dict) + self.append_id_node(internship_id, internship_node) + person_email = cleanEmailField(d[self.emailField]) + first_name = d[self.firstNameField] + last_name = d[self.lastNameField] + person_id = persons.id_from_something(first_name, last_name, person_email) + if None == person_id: + print("ERROR: missing %r (%r %r), creating person node with no interests" % (person_email, first_name, last_name)) + person_id, person_node = addPersonNode(first_name=first_name, last_name=last_name, email=person_email) + self.append_id_node(person_id, person_node) + self.append_link(source_id=person_id, target_id=internship_id, label='Did this internship') + +def main(args, internships): + cfg = Config.init_from_file(args.config) + kernel = RZ_Kernel() + kernel.db_ctl = dbc.DB_Controller(cfg) # yes, that. FIXME + globals()['kernel'] = kernel + globals()['ctx'] = {} # FIXME not logged it - fix later (also, don't do this here, put constructor in kernel) + StudentCSV(filename=args.student).run() + for internship in internships: + StudentInternshipsCSV(filename=internship).run() + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--student', required=True) + parser.add_argument('--config', default=os.path.join(root, 'res', 'etc', 'rhizi-server.conf')) + args, internships = parser.parse_known_args(sys.argv[1:]) + main(args, internships) diff --git a/src/local/domain/cri.rhizi.net/test_domain__CRI.py b/src/local/domain/cri.rhizi.net/test_domain__CRI.py new file mode 100644 index 00000000..54131bbd --- /dev/null +++ b/src/local/domain/cri.rhizi.net/test_domain__CRI.py @@ -0,0 +1,58 @@ +import inspect +import logging +import tempfile +import unittest + +import db_controller as dbc +from db_op import DBO_cypher_query +from domain_cri import DBO_random_data_generation__domain__CRI +from rz_server import Config + + +class Test_Domain_CRI(unittest.TestCase): + + @classmethod + def setUpClass(self): + cfg = Config.init_from_file('res/etc/rhizi-server.conf') + self.db_ctl = dbc.DB_Controller(cfg) + self.log = logging.getLogger('rhizi') + self.log.addHandler(logging.StreamHandler()) + + def test_random_data_generation__domain__CRI(self, export_as_csv=True): + """ + test: + - random CRI domain data generation + - DB dump in cvs format: person x skill x skill-level + """ + op = DBO_random_data_generation__domain__CRI() + self.db_ctl.exec_op(op) + + if False == export_as_csv: + return + + q_arr = ['match (n: Person)', + 'with n', + 'match (n)-[r:Novice|Intermediate|Expert]->(m:Skill)', # [!] expect link type to be proficiency level + 'return n.name, collect({skill_name: m.name, skill_level: r.proficiency})' + ] + op = DBO_cypher_query(q_arr) + + cur_f_name = inspect.stack()[0][3] + def q_process_result_set(): + with tempfile.NamedTemporaryFile(prefix='rz_%s' % (cur_f_name), dir='/tmp', suffix='.csv', delete=False) as f_out: + for _, _, r_set in op: + for row in r_set: + person_name, skill_dict_set = row # person to {sname: skill, s:pro: skill_level{ dict set + cvs_line_arr = [person_name] + for skill_dict in skill_dict_set: + skill_name = skill_dict['skill_name'] + skill_level = skill_dict['skill_level'] + cvs_line_arr += [skill_name, skill_level] + f_out.write(','.join(cvs_line_arr) + '\n') + f_out.write('\n') + + op.process_result_set = q_process_result_set + self.db_ctl.exec_op(op) + +if __name__ == "__main__": + unittest.main() diff --git a/src/local/domain/cri/__init__.py b/src/local/domain/cri/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/local/domain/cri/domain_cri.py b/src/local/domain/cri/domain_cri.py deleted file mode 100644 index db6ed925..00000000 --- a/src/local/domain/cri/domain_cri.py +++ /dev/null @@ -1,140 +0,0 @@ -from db_op import DB_op - -class DBO_random_data_generation__domain__CRI(DB_op): - - def __init__(self, lim_n=50, - lim_r=10000, - prob_link_create=0.03): - """ - generate random data: CRI domain: Person x Skill - """ - super(DBO_random_data_generation__domain__CRI, self).__init__() - - n_attr_set__skill_raw = ['Cryptocurrency', - 'Database architecture', - 'Web-development', - 'Mobile-development', - 'Machine learning', - 'Guitar Playing', - 'Image processing', - 'Algebra', - 'Calculus', - 'Molecular-Biology', - 'Graphic design', - 'Geo-location services', - 'Drone-building', - 'Artificial-intelligence', - 'Distributed information systems', - 'Wordpress', - 'Woodworking', - 'Quality-control', - 'Video-editing', - 'Soldering', - 'Network engineering', - 'GIT', - 'Electronic music', - 'Network administration'] - - n_attr_set__skill = [] - for s in n_attr_set__skill_raw: - n_attr_set__skill.append(s) - - n_attr_set__name = ['John', - 'William', - 'James', - 'Charles', - 'George', - 'Frank', - 'Joseph', - 'Thomas', - 'Henry', - 'Robert', - 'Edward', - 'Harry', - 'Walter', - 'Arthur', - 'Fred', - 'Albert', - 'Samuel', - 'David', - 'Louis', - 'Joe', - 'Charlie', - 'Clarence', - 'Richard', - 'Andrew', - 'Daniel', - 'Ernest', - 'Mary', - 'Anna', - 'Emma', - 'Elizabeth', - 'Minnie', - 'Margaret', - 'Ida', - 'Alice', - 'Bertha', - 'Sarah', - 'Annie', - 'Clara', - 'Ella', - 'Florence', - 'Cora', - 'Martha', - 'Laura', - 'Nellie', - 'Grace', - 'Carrie', - 'Maude', - 'Mabel', - 'Bessie', - 'Jennie', - 'Gertrude', - 'Julia'] - - l_attr_set__level = ['Novice', 'Intermediate', 'Expert'] - - - min_id = 10000 # separate group ids with numerical perfix - - # create persons - q_arr = ['with {n_attr_set__name} as n_attr_set', # TODO clean: foreach triggers SyntaxException: otherwise - 'foreach (idx in range(0,%d)' % (lim_n - 1), - '|', - 'create (n:%s' % ('Person'), - '{id: \'test-id_\' + toString(%d + idx),' % (min_id), - 'name: n_attr_set[idx %% %d]}' % (len(n_attr_set__name)), - '))', - ] - q_param = {'n_attr_set__name': n_attr_set__name} - self.add_statement(q_arr, q_param) - min_id *= 2 - - # create skills - q_arr = ['with {skill_set} as skill_set', # TODO clean: foreach triggers SyntaxException: otherwise - 'foreach (idx in range(0,%d)' % (len(n_attr_set__skill) - 1), - '|', - 'create (n:%s' % ('Skill'), - '{id: \'test-id_\' + toString(idx + %d),' % (min_id), - 'name: skill_set[idx]}', - '))', - ] - q_param = {'skill_set': n_attr_set__skill} - self.add_statement(q_arr, q_param) - min_id *= 2 - - # create links - for skill_level in l_attr_set__level: - q_arr = ['match (n:%s),(m:%s)' % ('Person', 'Skill'), - 'with n, m, rand() as rand', - 'order by rand', - 'limit %d' % (lim_r - 1), - 'where rand() < %.2f' % (prob_link_create), - 'create (n)-[r:%s' % (skill_level), - '{id: \'test-id_\' + toString(%d + toInt(%d * rand())),' % (min_id, lim_r * 100000), # aim for low id collision probability, - 'proficiency: \'%s\'}' % (skill_level), - ']->(m)', - 'return collect(r.id)', - ] - - self.add_statement(q_arr, q_param) diff --git a/src/local/domain/cri/from_csv b/src/local/domain/cri/from_csv deleted file mode 100755 index c11babd0..00000000 --- a/src/local/domain/cri/from_csv +++ /dev/null @@ -1,271 +0,0 @@ -#!/usr/bin/python2.7 - -""" -Import CSV files into Rhizi Server via pythonic API. - -Tricky, since this is the only API user. Try to use it exactly as the REST/WS -API would, just without actually creating a socket connection. -""" - -import sys -import os -import argparse -import string -import csv - -root = os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..', '..', '..')) -print("root = %s" % root) -sys.path.append(os.path.join(root, 'src', 'server')) - -from rz_api_common import sanitize_input__topo_diff -from rz_kernel import RZ_Kernel -import db_controller as dbc -from rz_server import Config - -from model.graph import Topo_Diff - -def topo_diff_json(node_set_add=[], link_set_add=[]): - topo_diff_dict = ({ - u'link_id_set_rm': [], - u'link_set_add': link_set_add, - u'drop_conjugator_links': True, - u'node_set_add': node_set_add, - u'node_id_set_rm': [] - }) - topo_diff = Topo_Diff.from_json_dict(topo_diff_dict) - sanitize_input__topo_diff(topo_diff) - return topo_diff; - -def commit(topo_diff): - _, commit_ret = kernel.diff_commit__topo(topo_diff, ctx) - -# Rhizi constants - FIXME use API -PERSON_LABEL = 'Person' -INTEREST_LABEL = 'Interest' -INTERNSHIP_LABEL = "Internship" -LABEL_SET = '__label_set' - -# CSV file columns -PERSONAL_EMAIL = 'Personal e-mail' -INTEREST = 'Interests' - -id_count = 0 -def next_id(): - global id_count - id_count += 1 - return '%08d' % id_count - -def node_dict(the_id, **args): - """ FIXME use API """ - ret = dict(args) - ret['id'] = the_id - return ret - -def link_dict(source_id, target_id, label, the_id): - return ({ - u'__type': [label], - u'__src_id': source_id, - u'__dst_id': target_id, - u'id': the_id - }) - -def read_csv(filename): - for dialect in ['excel', 'excel-tab']: - lines = list(csv.reader(open(filename), dialect=dialect)) - lens = map(len, lines) - if max(lens) == min(lens) and max(lens) > 2: - return lines - raise Exception("no dialect found") - -class CSV(object): - def __init__(self, filename): - self.filename = filename - rows = read_csv(filename) - headers = rows[0] - self.parse_headers(headers) - #import pdb; pdb.set_trace() - self.rows = rows[1:] - self.row_dicts = map(lambda fields: dict(zip(headers, fields)), self.rows) - - def run(self): - print(" **************************** run on data from %r ****************" % self.filename) - self.nodes_dict = {} - self.node_set_add = [] - self.link_set_add = [] - self.generate_nodes_and_links() - #for node in node_set_add: - # print('committing %s' % repr(node)) - # commit(topo_diff_json(node_set_add=[node])) - commit(topo_diff_json(node_set_add=self.node_set_add)) - commit(topo_diff_json(link_set_add=self.link_set_add)) - - def append_id_node(self, the_id, node): - if the_id in self.nodes_dict: - return - print(repr(node)) - self.nodes_dict[the_id] = node - self.node_set_add.append(node) - return the_id - - def _csv_row_to_node(self, d, dict_gen): - the_id = next_id() - print(repr(d)) - return the_id, node_dict(the_id=the_id, **dict_gen(d)) - - def append_link(self, source_id, target_id, label): - the_id = next_id() - link = link_dict(source_id, target_id, label, the_id) - self.link_set_add.append(link) - print(repr(link)) - return the_id - -class Persons(object): - def __init__(self): - self.by_name = {} - self.by_email = {} - - def id_from_something(self, first_name, last_name, email): - return self.by_name.get(self.name_from_first_and_last(first_name, last_name), self.by_email.get(email, (None, None)))[0] - - def name_from_first_and_last(self, first_name, last_name): - return first_name + ' ' + last_name - - def addPersonNode(self, first_name, last_name, email): - ret = ret2 = None - name = self.name_from_first_and_last(first_name, last_name) - if name in self.by_name: - ret = self.by_name[name] - if email in self.by_email: - ret2 = self.by_email[email] - if ret != ret2: - print("person node similar but not same for: %r, %r, %r" % (first_name, last_name, email)) - print(ret2) - if ret2 and not ret: - ret = ret2 - if ret: - return ret - node_id = next_id() - node = node_dict(the_id=node_id, name=name, description=email, **{LABEL_SET: [PERSON_LABEL]}) - ret = (node_id, node) - self.by_name[name] = ret - self.by_email[email] = ret - return ret - -persons = Persons() -addPersonNode = persons.addPersonNode - -def cleanEmailField(unclean): - return unclean.lower().strip() - -class StudentCSV(CSV): - - def __init__(self, filename, **kw): - super(StudentCSV, self).__init__(filename=filename, **kw) - - def interests(self, d): - return d[self.interestsField] - - def generate_nodes_and_links(self): - for d in self.row_dicts: - d[self.interestsField] = map(string.strip, d[self.interestsField].split(',')) - person_id = next_id() - person_email = cleanEmailField(d[self.personalEmailField]) - person_id, person_node = addPersonNode(first_name=d[self.firstNameField], last_name=d[self.lastNameField], email=person_email) - self.append_id_node(person_id, person_node) - for interest in self.interests(d): - interest_id = next_id() - interest_node = node_dict(the_id=interest_id, - **{'name':interest, LABEL_SET:[INTEREST_LABEL]}) - interest_id = self.append_id_node(interest_id, interest_node) - self.append_link(person_id, interest_id, 'Is interested in') - - def parse_headers(self, headers): - self.firstNameField = headers[0] - self.lastNameField = headers[1] - self.personalEmailField = headers[2] - self.interestsField = headers[3] - -descriptionTemplate = """Nature: %s --------- -Abstract: -%s""" - -class StudentInternshipsCSV(CSV): - - def __init__(self, filename, **kw): - super(StudentInternshipsCSV, self).__init__(filename=filename, **kw) - - def parse_headers(self, headers): - it = iter(headers) - self.firstNameField = it.next() - self.lastNameField = it.next() - self.emailField = it.next() - self.internshipTitleField = it.next() - self.internshipNatureField = it.next() - self.abstractField = it.next() - self.dateStartField = it.next() - self.dateEndField = it.next() - self.supervisorFirstNameField = it.next() - self.supervisorLastNameField = it.next() - self.supervisorTitleField = it.next() - self.supervisorEmailField = it.next() - self.laboratoryNameField = it.next() - self.laboratoryAffiliationField = it.next() - self.unitCodeField = it.next() # CNRS / INSERM unit code - self.streetField = it.next() # Street - self.streetSecondPartField = it.next() # Street (continued) - self.cityField = it.next() # City - self.countryField = it.next() # Country - - def generate_nodes_and_links(self): - for d in self.row_dicts: - internship_id = next_id() - # FIXME should be title. fix requires client change. need - # configurable fields, or a list of names and types for display per - # node (using a type node in the db, which could be cached - title = d[self.internshipTitleField].strip() - if len(title) == 0: - print("skipping empty internship field") - continue - input_dict = { - LABEL_SET:[INTERNSHIP_LABEL], - 'name': title, - 'description': descriptionTemplate % (d[self.internshipNatureField], d[self.abstractField]), - 'startdate': d[self.dateStartField], - 'enddate': d[self.dateEndField], - 'internship-type': d[self.internshipNatureField], - 'cnrs-inserm-unit-code': d[self.unitCodeField], - 'city': d[self.cityField], - 'country': d[self.countryField], - 'street-address': d[self.streetField] + d[self.streetSecondPartField], - 'facility': d[self.laboratoryNameField], - 'facility-affiliation': d[self.laboratoryAffiliationField], - } - internship_node = node_dict(the_id=internship_id, **input_dict) - self.append_id_node(internship_id, internship_node) - person_email = cleanEmailField(d[self.emailField]) - first_name = d[self.firstNameField] - last_name = d[self.lastNameField] - person_id = persons.id_from_something(first_name, last_name, person_email) - if None == person_id: - print("ERROR: missing %r (%r %r), creating person node with no interests" % (person_email, first_name, last_name)) - person_id, person_node = addPersonNode(first_name=first_name, last_name=last_name, email=person_email) - self.append_id_node(person_id, person_node) - self.append_link(source_id=person_id, target_id=internship_id, label='Did this internship') - -def main(args, internships): - cfg = Config.init_from_file(args.config) - kernel = RZ_Kernel() - kernel.db_ctl = dbc.DB_Controller(cfg) # yes, that. FIXME - globals()['kernel'] = kernel - globals()['ctx'] = {} # FIXME not logged it - fix later (also, don't do this here, put constructor in kernel) - StudentCSV(filename=args.student).run() - for internship in internships: - StudentInternshipsCSV(filename=internship).run() - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('--student', required=True) - parser.add_argument('--config', default=os.path.join(root, 'res', 'etc', 'rhizi-server.conf')) - args, internships = parser.parse_known_args(sys.argv[1:]) - main(args, internships) diff --git a/src/local/domain/cri/test_domain__CRI.py b/src/local/domain/cri/test_domain__CRI.py deleted file mode 100644 index 54131bbd..00000000 --- a/src/local/domain/cri/test_domain__CRI.py +++ /dev/null @@ -1,58 +0,0 @@ -import inspect -import logging -import tempfile -import unittest - -import db_controller as dbc -from db_op import DBO_cypher_query -from domain_cri import DBO_random_data_generation__domain__CRI -from rz_server import Config - - -class Test_Domain_CRI(unittest.TestCase): - - @classmethod - def setUpClass(self): - cfg = Config.init_from_file('res/etc/rhizi-server.conf') - self.db_ctl = dbc.DB_Controller(cfg) - self.log = logging.getLogger('rhizi') - self.log.addHandler(logging.StreamHandler()) - - def test_random_data_generation__domain__CRI(self, export_as_csv=True): - """ - test: - - random CRI domain data generation - - DB dump in cvs format: person x skill x skill-level - """ - op = DBO_random_data_generation__domain__CRI() - self.db_ctl.exec_op(op) - - if False == export_as_csv: - return - - q_arr = ['match (n: Person)', - 'with n', - 'match (n)-[r:Novice|Intermediate|Expert]->(m:Skill)', # [!] expect link type to be proficiency level - 'return n.name, collect({skill_name: m.name, skill_level: r.proficiency})' - ] - op = DBO_cypher_query(q_arr) - - cur_f_name = inspect.stack()[0][3] - def q_process_result_set(): - with tempfile.NamedTemporaryFile(prefix='rz_%s' % (cur_f_name), dir='/tmp', suffix='.csv', delete=False) as f_out: - for _, _, r_set in op: - for row in r_set: - person_name, skill_dict_set = row # person to {sname: skill, s:pro: skill_level{ dict set - cvs_line_arr = [person_name] - for skill_dict in skill_dict_set: - skill_name = skill_dict['skill_name'] - skill_level = skill_dict['skill_level'] - cvs_line_arr += [skill_name, skill_level] - f_out.write(','.join(cvs_line_arr) + '\n') - f_out.write('\n') - - op.process_result_set = q_process_result_set - self.db_ctl.exec_op(op) - -if __name__ == "__main__": - unittest.main() -- cgit v1.3.1