cri/from_csv: working, but disconnected graph results, plus 5 emails need handling (hotmail/gmail etc.)

1 files changed, 203 insertions, 0 deletions

diff --git a/src/local/domain/cri/from_csv b/src/local/domain/cri/from_csv
new file mode 100755
index 00000000..593f1fc0
--- /dev/null
+++ b/src/local/domain/cri/from_csv
@@ -0,0 +1,203 @@
+#!/usr/bin/python2.7
+
+"""
+Import CSV files into Rhizi Server via pythonic API.
+
+Tricky, since this is the only API user. Try to use it exactly as the REST/WS
+API would, just without actually creating a socket connection.
+"""
+
+import sys
+import os
+import argparse
+import string
+import csv
+
+root = os.path.join(os.path.dirname(__file__), '..', '..')
+sys.path.append(os.path.join(root, 'src', 'server'))
+
+from rz_api_common import sanitize_input__topo_diff
+from rz_kernel import RZ_Kernel
+import db_controller as dbc
+from rz_server import Config
+
+from model.graph import Topo_Diff
+
+def topo_diff_json(node_set_add=[], link_set_add=[]):
+    topo_diff_dict = ({
+         u'link_id_set_rm': [],
+         u'link_set_add': link_set_add,
+         u'drop_conjugator_links': True,
+         u'node_set_add': node_set_add,
+         u'node_id_set_rm': []
+        })
+    topo_diff = Topo_Diff.from_json_dict(topo_diff_dict)
+    sanitize_input__topo_diff(topo_diff)
+    return topo_diff;
+
+cfg = Config.init_from_file(os.path.join(root, 'res', 'etc', 'rhizi-server.conf'))
+kernel = RZ_Kernel()
+kernel.db_ctl = dbc.DB_Controller(cfg) # yes, that. FIXME
+ctx = {} # FIXME not logged it - fix later (also, don't do this here, put constructor in kernel)
+
+def commit(topo_diff):
+    _, commit_ret = kernel.diff_commit__topo(topo_diff, ctx)
+
+# Rhizi constants - FIXME use API
+PERSON_LABEL = 'Person'
+INTEREST_LABEL = 'Interest'
+INTERNSHIP_LABEL = "Third-internship-proposal"
+LABEL_SET = '__label_set'
+
+# CSV file columns
+PERSONAL_EMAIL = 'Personal e-mail'
+INTEREST = 'Interests'
+
+id_count = 0
+def next_id():
+    global id_count
+    id_count += 1
+    return '%08d' % id_count
+
+def node_dict(the_id, **args):
+    """ FIXME use API """
+    ret = dict(args)
+    ret['id'] = the_id
+    return ret
+
+def link_dict(source_id, target_id, label, the_id):
+    return ({
+               u'__type': [label],
+               u'__src_id': source_id,
+               u'__dst_id': target_id,
+               u'id': the_id
+           })
+
+class CSV(object):
+    def __init__(self, filename):
+        rows = list(csv.reader(open(filename), 'excel-tab'))
+        headers = rows[0]
+        self.parse_headers(headers)
+        #import pdb; pdb.set_trace()
+        self.rows = rows[1:]
+        self.row_dicts = map(lambda fields: dict(zip(headers, fields)), self.rows)
+
+    def run(self):
+        self.nodes_dict = {}
+        self.node_set_add = []
+        self.link_set_add = []
+        self.generate_nodes_and_links()
+        #for node in node_set_add:
+        #    print('committing %s' % repr(node))
+        #    commit(topo_diff_json(node_set_add=[node]))
+        commit(topo_diff_json(node_set_add=self.node_set_add))
+        commit(topo_diff_json(link_set_add=self.link_set_add))
+
+    def append_id_node(self, the_id, node):
+        print(repr(node))
+        self.nodes_dict[the_id] = node
+        self.node_set_add.append(node)
+        return the_id
+
+    def _csv_row_to_node(self, d, dict_gen):
+        the_id = next_id()
+        print(repr(d))
+        return the_id, node_dict(the_id=the_id, **dict_gen(d))
+
+    def append_link(self, source_id, target_id, label):
+        the_id = next_id()
+        link = link_dict(source_id, target_id, label, the_id)
+        self.link_set_add.append(link)
+        print(repr(link))
+        return the_id
+
+class StudentCSV(CSV):
+
+    def __init__(self, filename):
+        super(StudentCSV, self).__init__(filename)
+        self.email_to_id = {}
+
+    def interests(self, d):
+        return d[self.interestsField]
+
+    def generate_nodes_and_links(self):
+        for d in self.row_dicts:
+            d[self.interestsField] = map(string.strip, d[self.interestsField].split(','))
+            person_id = next_id()
+            person_node = node_dict(the_id=person_id,
+                                **{'name':' '.join([d[self.firstNameField], d[self.lastNameField]]),
+                                LABEL_SET:[PERSON_LABEL]})
+            self.email_to_id[d[self.personalEmailField].lower()] = person_id
+            self.append_id_node(person_id, person_node)
+            for interest in self.interests(d):
+                interest_id = next_id()
+                interest_node = node_dict(the_id=interest_id,
+                    **{'name':interest, LABEL_SET:[INTEREST_LABEL]})
+                interest_id = self.append_id_node(interest_id, interest_node)
+                self.append_link(person_id, interest_id, 'Is interested in')
+
+    def parse_headers(self, headers):
+        self.firstNameField = headers[0]
+        self.lastNameField = headers[1]
+        self.personalEmailField = headers[2]
+        self.interestsField = headers[3]
+
+descriptionTemplate = """Nature: %s
+--------
+Abstract:
+%s"""
+
+class StudentInternshipsCSV(CSV):
+
+    def __init__(self, studentCSV, filename):
+        super(StudentInternshipsCSV, self).__init__(filename)
+        self.studentCSV = studentCSV
+        self.email_to_id = studentCSV.email_to_id
+
+    def parse_headers(self, headers):
+        it = iter(headers)
+        self.firstNameField = it.next()
+        self.lastNameField = it.next()
+        self.emailField = it.next()
+        self.internshipTitleField = it.next()
+        self.internshipNatureField = it.next()
+        self.abstractField = it.next()
+        self.dateStartField = it.next()
+        self.dateEndField = it.next()
+        self.supervisorFirstNameField = it.next()
+        self.supervisorLastNameField = it.next()
+        self.supervisorTitleField = it.next()
+        self.supervisorEmailField = it.next()
+        self.laboratoryNameField = it.next()
+        self.laboratoryAffiliationField = it.next()
+        self.unitCodeField = it.next() # CNRS  / INSERM unit code
+        self.streetField = it.next() # Street
+        self.streetSecondPartField = it.next() # Street (continued)
+        self.cityField = it.next() # City
+        self.countryField = it.next() # Country
+
+    def generate_nodes_and_links(self):
+        for d in self.row_dicts:
+            internship_id = next_id()
+            # FIXME should be title. fix requires client change. need
+            # configurable fields, or a list of names and types for display per
+            # node (using a type node in the db, which could be cached
+            input_dict = {LABEL_SET:[INTERNSHIP_LABEL],
+                         'name': d[self.internshipTitleField], 
+                         'description': descriptionTemplate % (d[self.internshipNatureField], d[self.abstractField]),
+                         'startdate': d[self.dateStartField],
+                         'enddate': d[self.dateEndField]}
+            internship_node = node_dict(the_id=internship_id, **input_dict)
+            self.append_id_node(internship_id, internship_node)
+            person_email = d[self.emailField].lower().strip()
+            person_id = self.email_to_id.get(person_email, None)
+            if None == person_id:
+                print("ERROR: missing %s" % person_email)
+            else:
+                self.append_link(source_id=person_id, target_id=internship_id, label='Did this internship')
+
+
+assert(len(sys.argv) == 3)
+studentCSV = StudentCSV(sys.argv[-2])
+studentCSV.run()
+StudentInternshipsCSV(studentCSV, sys.argv[-1]).run()