mving cri domain resources under cri.rhizi.net

4 files changed, 469 insertions, 0 deletions

diff --git a/src/local/domain/cri.rhizi.net/__init__.py b/src/local/domain/cri.rhizi.net/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/src/local/domain/cri.rhizi.net/__init__.py
diff --git a/src/local/domain/cri.rhizi.net/domain_cri.py b/src/local/domain/cri.rhizi.net/domain_cri.py
new file mode 100644
index 00000000..db6ed925
--- /dev/null
+++ b/src/local/domain/cri.rhizi.net/domain_cri.py
@@ -0,0 +1,140 @@
+from db_op import DB_op
+
+class DBO_random_data_generation__domain__CRI(DB_op):
+
+    def __init__(self, lim_n=50,
+                       lim_r=10000,
+                       prob_link_create=0.03):
+        """
+        generate random data: CRI domain: Person x Skill
+        """
+        super(DBO_random_data_generation__domain__CRI, self).__init__()
+
+        n_attr_set__skill_raw = ['Cryptocurrency',
+                             'Database architecture',
+                             'Web-development',
+                             'Mobile-development',
+                             'Machine learning',
+                             'Guitar Playing',
+                             'Image processing',
+                             'Algebra',
+                             'Calculus',
+                             'Molecular-Biology',
+                             'Graphic design',
+                             'Geo-location services',
+                             'Drone-building',
+                             'Artificial-intelligence',
+                             'Distributed information systems',
+                             'Wordpress',
+                             'Woodworking',
+                             'Quality-control',
+                             'Video-editing',
+                             'Soldering',
+                             'Network engineering',
+                             'GIT',
+                             'Electronic music',
+                             'Network administration']
+
+        n_attr_set__skill = []
+        for s in n_attr_set__skill_raw:
+            n_attr_set__skill.append(s)
+
+        n_attr_set__name = ['John',
+                            'William',
+                            'James',
+                            'Charles',
+                            'George',
+                            'Frank',
+                            'Joseph',
+                            'Thomas',
+                            'Henry',
+                            'Robert',
+                            'Edward',
+                            'Harry',
+                            'Walter',
+                            'Arthur',
+                            'Fred',
+                            'Albert',
+                            'Samuel',
+                            'David',
+                            'Louis',
+                            'Joe',
+                            'Charlie',
+                            'Clarence',
+                            'Richard',
+                            'Andrew',
+                            'Daniel',
+                            'Ernest',
+                            'Mary',
+                            'Anna',
+                            'Emma',
+                            'Elizabeth',
+                            'Minnie',
+                            'Margaret',
+                            'Ida',
+                            'Alice',
+                            'Bertha',
+                            'Sarah',
+                            'Annie',
+                            'Clara',
+                            'Ella',
+                            'Florence',
+                            'Cora',
+                            'Martha',
+                            'Laura',
+                            'Nellie',
+                            'Grace',
+                            'Carrie',
+                            'Maude',
+                            'Mabel',
+                            'Bessie',
+                            'Jennie',
+                            'Gertrude',
+                            'Julia']
+
+        l_attr_set__level = ['Novice', 'Intermediate', 'Expert']
+
+
+        min_id = 10000  # separate group ids with numerical perfix
+
+        # create persons
+        q_arr = ['with {n_attr_set__name} as n_attr_set',  # TODO clean: foreach triggers SyntaxException: otherwise
+                 'foreach (idx in range(0,%d)' % (lim_n - 1),
+                 '|',
+                 'create (n:%s' % ('Person'),
+                 '{id: \'test-id_\' + toString(%d + idx),' % (min_id),
+                  'name: n_attr_set[idx %% %d]}' % (len(n_attr_set__name)),
+                 '))',
+                ]
+        q_param = {'n_attr_set__name': n_attr_set__name}
+        self.add_statement(q_arr, q_param)
+        min_id *= 2
+
+        # create skills
+        q_arr = ['with {skill_set} as skill_set',  # TODO clean: foreach triggers SyntaxException: otherwise
+                 'foreach (idx in range(0,%d)' % (len(n_attr_set__skill) - 1),
+                 '|',
+                 'create (n:%s' % ('Skill'),
+                 '{id: \'test-id_\' + toString(idx + %d),' % (min_id),
+                  'name: skill_set[idx]}',
+                 '))',
+                ]
+        q_param = {'skill_set': n_attr_set__skill}
+        self.add_statement(q_arr, q_param)
+        min_id *= 2
+
+        # create links
+        for skill_level in l_attr_set__level:
+            q_arr = ['match (n:%s),(m:%s)' % ('Person', 'Skill'),
+                     'with n, m, rand() as rand',
+                     'order by rand',
+                     'limit %d' % (lim_r - 1),
+                     'where rand() < %.2f' % (prob_link_create),
+                     'create (n)-[r:%s' % (skill_level),
+                     '{id: \'test-id_\' + toString(%d + toInt(%d * rand())),' % (min_id, lim_r * 100000),  # aim for low id collision probability,
+                     'proficiency: \'%s\'}' % (skill_level),
+                     ']->(m)',
+                     'return collect(r.id)',
+                     ]
+
+            self.add_statement(q_arr, q_param)
diff --git a/src/local/domain/cri.rhizi.net/from_csv b/src/local/domain/cri.rhizi.net/from_csv
new file mode 100755
index 00000000..c11babd0
--- /dev/null
+++ b/src/local/domain/cri.rhizi.net/from_csv
@@ -0,0 +1,271 @@
+#!/usr/bin/python2.7
+
+"""
+Import CSV files into Rhizi Server via pythonic API.
+
+Tricky, since this is the only API user. Try to use it exactly as the REST/WS
+API would, just without actually creating a socket connection.
+"""
+
+import sys
+import os
+import argparse
+import string
+import csv
+
+root = os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..', '..', '..'))
+print("root = %s" % root)
+sys.path.append(os.path.join(root, 'src', 'server'))
+
+from rz_api_common import sanitize_input__topo_diff
+from rz_kernel import RZ_Kernel
+import db_controller as dbc
+from rz_server import Config
+
+from model.graph import Topo_Diff
+
+def topo_diff_json(node_set_add=[], link_set_add=[]):
+    topo_diff_dict = ({
+         u'link_id_set_rm': [],
+         u'link_set_add': link_set_add,
+         u'drop_conjugator_links': True,
+         u'node_set_add': node_set_add,
+         u'node_id_set_rm': []
+        })
+    topo_diff = Topo_Diff.from_json_dict(topo_diff_dict)
+    sanitize_input__topo_diff(topo_diff)
+    return topo_diff;
+
+def commit(topo_diff):
+    _, commit_ret = kernel.diff_commit__topo(topo_diff, ctx)
+
+# Rhizi constants - FIXME use API
+PERSON_LABEL = 'Person'
+INTEREST_LABEL = 'Interest'
+INTERNSHIP_LABEL = "Internship"
+LABEL_SET = '__label_set'
+
+# CSV file columns
+PERSONAL_EMAIL = 'Personal e-mail'
+INTEREST = 'Interests'
+
+id_count = 0
+def next_id():
+    global id_count
+    id_count += 1
+    return '%08d' % id_count
+
+def node_dict(the_id, **args):
+    """ FIXME use API """
+    ret = dict(args)
+    ret['id'] = the_id
+    return ret
+
+def link_dict(source_id, target_id, label, the_id):
+    return ({
+               u'__type': [label],
+               u'__src_id': source_id,
+               u'__dst_id': target_id,
+               u'id': the_id
+           })
+
+def read_csv(filename):
+    for dialect in ['excel', 'excel-tab']:
+        lines = list(csv.reader(open(filename), dialect=dialect))
+        lens = map(len, lines)
+        if max(lens) == min(lens) and max(lens) > 2:
+            return lines
+    raise Exception("no dialect found")
+
+class CSV(object):
+    def __init__(self, filename):
+        self.filename = filename
+        rows = read_csv(filename)
+        headers = rows[0]
+        self.parse_headers(headers)
+        #import pdb; pdb.set_trace()
+        self.rows = rows[1:]
+        self.row_dicts = map(lambda fields: dict(zip(headers, fields)), self.rows)
+
+    def run(self):
+        print("    ****************************    run on data from %r     ****************" % self.filename)
+        self.nodes_dict = {}
+        self.node_set_add = []
+        self.link_set_add = []
+        self.generate_nodes_and_links()
+        #for node in node_set_add:
+        #    print('committing %s' % repr(node))
+        #    commit(topo_diff_json(node_set_add=[node]))
+        commit(topo_diff_json(node_set_add=self.node_set_add))
+        commit(topo_diff_json(link_set_add=self.link_set_add))
+
+    def append_id_node(self, the_id, node):
+        if the_id in self.nodes_dict:
+            return
+        print(repr(node))
+        self.nodes_dict[the_id] = node
+        self.node_set_add.append(node)
+        return the_id
+
+    def _csv_row_to_node(self, d, dict_gen):
+        the_id = next_id()
+        print(repr(d))
+        return the_id, node_dict(the_id=the_id, **dict_gen(d))
+
+    def append_link(self, source_id, target_id, label):
+        the_id = next_id()
+        link = link_dict(source_id, target_id, label, the_id)
+        self.link_set_add.append(link)
+        print(repr(link))
+        return the_id
+
+class Persons(object):
+    def __init__(self):
+        self.by_name = {}
+        self.by_email = {}
+
+    def id_from_something(self, first_name, last_name, email):
+        return self.by_name.get(self.name_from_first_and_last(first_name, last_name), self.by_email.get(email, (None, None)))[0]
+
+    def name_from_first_and_last(self, first_name, last_name):
+        return first_name + ' ' + last_name
+
+    def addPersonNode(self, first_name, last_name, email):
+        ret = ret2 = None
+        name = self.name_from_first_and_last(first_name, last_name)
+        if name in self.by_name:
+            ret = self.by_name[name]
+        if email in self.by_email:
+            ret2 = self.by_email[email]
+        if ret != ret2:
+            print("person node similar but not same for: %r, %r, %r" % (first_name, last_name, email))
+            print(ret2)
+        if ret2 and not ret:
+            ret = ret2
+        if ret:
+            return ret
+        node_id = next_id()
+        node = node_dict(the_id=node_id, name=name, description=email, **{LABEL_SET: [PERSON_LABEL]})
+        ret = (node_id, node)
+        self.by_name[name] = ret
+        self.by_email[email] = ret
+        return ret
+
+persons = Persons()
+addPersonNode = persons.addPersonNode
+
+def cleanEmailField(unclean):
+    return unclean.lower().strip()
+
+class StudentCSV(CSV):
+
+    def __init__(self, filename, **kw):
+        super(StudentCSV, self).__init__(filename=filename, **kw)
+
+    def interests(self, d):
+        return d[self.interestsField]
+
+    def generate_nodes_and_links(self):
+        for d in self.row_dicts:
+            d[self.interestsField] = map(string.strip, d[self.interestsField].split(','))
+            person_id = next_id()
+            person_email = cleanEmailField(d[self.personalEmailField])
+            person_id, person_node = addPersonNode(first_name=d[self.firstNameField], last_name=d[self.lastNameField], email=person_email)
+            self.append_id_node(person_id, person_node)
+            for interest in self.interests(d):
+                interest_id = next_id()
+                interest_node = node_dict(the_id=interest_id,
+                    **{'name':interest, LABEL_SET:[INTEREST_LABEL]})
+                interest_id = self.append_id_node(interest_id, interest_node)
+                self.append_link(person_id, interest_id, 'Is interested in')
+
+    def parse_headers(self, headers):
+        self.firstNameField = headers[0]
+        self.lastNameField = headers[1]
+        self.personalEmailField = headers[2]
+        self.interestsField = headers[3]
+
+descriptionTemplate = """Nature: %s
+--------
+Abstract:
+%s"""
+
+class StudentInternshipsCSV(CSV):
+
+    def __init__(self, filename, **kw):
+        super(StudentInternshipsCSV, self).__init__(filename=filename, **kw)
+
+    def parse_headers(self, headers):
+        it = iter(headers)
+        self.firstNameField = it.next()
+        self.lastNameField = it.next()
+        self.emailField = it.next()
+        self.internshipTitleField = it.next()
+        self.internshipNatureField = it.next()
+        self.abstractField = it.next()
+        self.dateStartField = it.next()
+        self.dateEndField = it.next()
+        self.supervisorFirstNameField = it.next()
+        self.supervisorLastNameField = it.next()
+        self.supervisorTitleField = it.next()
+        self.supervisorEmailField = it.next()
+        self.laboratoryNameField = it.next()
+        self.laboratoryAffiliationField = it.next()
+        self.unitCodeField = it.next() # CNRS  / INSERM unit code
+        self.streetField = it.next() # Street
+        self.streetSecondPartField = it.next() # Street (continued)
+        self.cityField = it.next() # City
+        self.countryField = it.next() # Country
+
+    def generate_nodes_and_links(self):
+        for d in self.row_dicts:
+            internship_id = next_id()
+            # FIXME should be title. fix requires client change. need
+            # configurable fields, or a list of names and types for display per
+            # node (using a type node in the db, which could be cached
+            title = d[self.internshipTitleField].strip()
+            if len(title) == 0:
+                print("skipping empty internship field")
+                continue
+            input_dict = {
+                         LABEL_SET:[INTERNSHIP_LABEL],
+                         'name': title,
+                         'description': descriptionTemplate % (d[self.internshipNatureField], d[self.abstractField]),
+                         'startdate': d[self.dateStartField],
+                         'enddate': d[self.dateEndField],
+                         'internship-type': d[self.internshipNatureField],
+                         'cnrs-inserm-unit-code': d[self.unitCodeField],
+                         'city': d[self.cityField],
+                         'country': d[self.countryField],
+                         'street-address': d[self.streetField] + d[self.streetSecondPartField],
+                         'facility': d[self.laboratoryNameField],
+                         'facility-affiliation': d[self.laboratoryAffiliationField],
+                         }
+            internship_node = node_dict(the_id=internship_id, **input_dict)
+            self.append_id_node(internship_id, internship_node)
+            person_email = cleanEmailField(d[self.emailField])
+            first_name = d[self.firstNameField]
+            last_name = d[self.lastNameField]
+            person_id = persons.id_from_something(first_name, last_name, person_email)
+            if None == person_id:
+                print("ERROR: missing %r (%r %r), creating person node with no interests" % (person_email, first_name, last_name))
+                person_id, person_node = addPersonNode(first_name=first_name, last_name=last_name, email=person_email)
+                self.append_id_node(person_id, person_node)
+            self.append_link(source_id=person_id, target_id=internship_id, label='Did this internship')
+
+def main(args, internships):
+    cfg = Config.init_from_file(args.config)
+    kernel = RZ_Kernel()
+    kernel.db_ctl = dbc.DB_Controller(cfg) # yes, that. FIXME
+    globals()['kernel'] = kernel
+    globals()['ctx'] = {} # FIXME not logged it - fix later (also, don't do this here, put constructor in kernel)
+    StudentCSV(filename=args.student).run()
+    for internship in internships:
+        StudentInternshipsCSV(filename=internship).run()
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--student', required=True)
+    parser.add_argument('--config', default=os.path.join(root, 'res', 'etc', 'rhizi-server.conf'))
+    args, internships = parser.parse_known_args(sys.argv[1:])
+    main(args, internships)
diff --git a/src/local/domain/cri.rhizi.net/test_domain__CRI.py b/src/local/domain/cri.rhizi.net/test_domain__CRI.py
new file mode 100644
index 00000000..54131bbd
--- /dev/null
+++ b/src/local/domain/cri.rhizi.net/test_domain__CRI.py
@@ -0,0 +1,58 @@
+import inspect
+import logging
+import tempfile
+import unittest
+
+import db_controller as dbc
+from db_op import DBO_cypher_query
+from domain_cri import DBO_random_data_generation__domain__CRI
+from rz_server import Config
+
+
+class Test_Domain_CRI(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(self):
+        cfg = Config.init_from_file('res/etc/rhizi-server.conf')
+        self.db_ctl = dbc.DB_Controller(cfg)
+        self.log = logging.getLogger('rhizi')
+        self.log.addHandler(logging.StreamHandler())
+
+    def test_random_data_generation__domain__CRI(self, export_as_csv=True):
+        """
+        test:
+           - random CRI domain data generation
+           - DB dump in cvs format: person x skill x skill-level
+        """
+        op = DBO_random_data_generation__domain__CRI()
+        self.db_ctl.exec_op(op)
+
+        if False == export_as_csv:
+            return
+
+        q_arr = ['match (n: Person)',
+                 'with n',
+                 'match (n)-[r:Novice|Intermediate|Expert]->(m:Skill)',  # [!] expect link type to be proficiency level
+                 'return n.name, collect({skill_name: m.name, skill_level: r.proficiency})'
+                 ]
+        op = DBO_cypher_query(q_arr)
+
+        cur_f_name = inspect.stack()[0][3]
+        def q_process_result_set():
+            with tempfile.NamedTemporaryFile(prefix='rz_%s' % (cur_f_name), dir='/tmp', suffix='.csv', delete=False) as f_out:
+                for _, _, r_set in op:
+                    for row in r_set:
+                        person_name, skill_dict_set = row  # person to {sname: skill, s:pro: skill_level{ dict set
+                        cvs_line_arr = [person_name]
+                        for skill_dict in skill_dict_set:
+                            skill_name = skill_dict['skill_name']
+                            skill_level = skill_dict['skill_level']
+                            cvs_line_arr += [skill_name, skill_level]
+                        f_out.write(','.join(cvs_line_arr) + '\n')
+                f_out.write('\n')
+
+        op.process_result_set = q_process_result_set
+        self.db_ctl.exec_op(op)
+
+if __name__ == "__main__":
+    unittest.main()