#!/usr/bin/python2.7 """ Import CSV files into Rhizi Server via pythonic API. Tricky, since this is the only API user. Try to use it exactly as the REST/WS API would, just without actually creating a socket connection. """ import sys import os import argparse import string import csv root = os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..', '..', '..')) print("root = %s" % root) sys.path.append(os.path.join(root, 'src', 'server')) from rz_api_common import sanitize_input__topo_diff from rz_kernel import RZ_Kernel import db_controller as dbc from rz_server import Config from model.graph import Topo_Diff def topo_diff_json(node_set_add=[], link_set_add=[]): topo_diff_dict = ({ u'link_id_set_rm': [], u'link_set_add': link_set_add, u'drop_conjugator_links': True, u'node_set_add': node_set_add, u'node_id_set_rm': [] }) topo_diff = Topo_Diff.from_json_dict(topo_diff_dict) sanitize_input__topo_diff(topo_diff) return topo_diff; def commit(topo_diff): _, commit_ret = kernel.diff_commit__topo(topo_diff, ctx) # Rhizi constants - FIXME use API PERSON_LABEL = 'Person' INTEREST_LABEL = 'Interest' INTERNSHIP_LABEL = "Internship" LABEL_SET = '__label_set' # CSV file columns PERSONAL_EMAIL = 'Personal e-mail' INTEREST = 'Interests' id_count = 0 def next_id(): global id_count id_count += 1 return '%08d' % id_count def node_dict(the_id, **args): """ FIXME use API """ ret = dict(args) ret['id'] = the_id return ret def link_dict(source_id, target_id, label, the_id): return ({ u'__type': [label], u'__src_id': source_id, u'__dst_id': target_id, u'id': the_id }) def read_csv(filename): for dialect in ['excel', 'excel-tab']: lines = list(csv.reader(open(filename), dialect=dialect)) lens = map(len, lines) if max(lens) == min(lens) and max(lens) > 2: return lines raise Exception("no dialect found") class CSV(object): def __init__(self, filename): self.filename = filename rows = read_csv(filename) headers = rows[0] self.parse_headers(headers) #import pdb; pdb.set_trace() self.rows = rows[1:] self.row_dicts = map(lambda fields: dict(zip(headers, fields)), self.rows) def run(self): print(" **************************** run on data from %r ****************" % self.filename) self.nodes_dict = {} self.node_set_add = [] self.link_set_add = [] self.generate_nodes_and_links() #for node in node_set_add: # print('committing %s' % repr(node)) # commit(topo_diff_json(node_set_add=[node])) commit(topo_diff_json(node_set_add=self.node_set_add)) commit(topo_diff_json(link_set_add=self.link_set_add)) def append_id_node(self, the_id, node): if the_id in self.nodes_dict: return print(repr(node)) self.nodes_dict[the_id] = node self.node_set_add.append(node) return the_id def _csv_row_to_node(self, d, dict_gen): the_id = next_id() print(repr(d)) return the_id, node_dict(the_id=the_id, **dict_gen(d)) def append_link(self, source_id, target_id, label): the_id = next_id() link = link_dict(source_id, target_id, label, the_id) self.link_set_add.append(link) print(repr(link)) return the_id class Persons(object): def __init__(self): self.by_name = {} self.by_email = {} def id_from_something(self, first_name, last_name, email): return self.by_name.get(self.name_from_first_and_last(first_name, last_name), self.by_email.get(email, (None, None)))[0] def name_from_first_and_last(self, first_name, last_name): return first_name + ' ' + last_name def addPersonNode(self, first_name, last_name, email): ret = ret2 = None name = self.name_from_first_and_last(first_name, last_name) if name in self.by_name: ret = self.by_name[name] if email in self.by_email: ret2 = self.by_email[email] if ret != ret2: print("person node similar but not same for: %r, %r, %r" % (first_name, last_name, email)) print(ret2) if ret2 and not ret: ret = ret2 if ret: return ret node_id = next_id() node = node_dict(the_id=node_id, name=name, description=email, **{LABEL_SET: [PERSON_LABEL]}) ret = (node_id, node) self.by_name[name] = ret self.by_email[email] = ret return ret persons = Persons() addPersonNode = persons.addPersonNode def cleanEmailField(unclean): return unclean.lower().strip() class StudentCSV(CSV): def __init__(self, filename, **kw): super(StudentCSV, self).__init__(filename=filename, **kw) def interests(self, d): return d[self.interestsField] def generate_nodes_and_links(self): for d in self.row_dicts: d[self.interestsField] = map(string.strip, d[self.interestsField].split(',')) person_id = next_id() person_email = cleanEmailField(d[self.personalEmailField]) person_id, person_node = addPersonNode(first_name=d[self.firstNameField], last_name=d[self.lastNameField], email=person_email) self.append_id_node(person_id, person_node) for interest in self.interests(d): interest_id = next_id() interest_node = node_dict(the_id=interest_id, **{'name':interest, LABEL_SET:[INTEREST_LABEL]}) interest_id = self.append_id_node(interest_id, interest_node) self.append_link(person_id, interest_id, 'Is interested in') def parse_headers(self, headers): self.firstNameField = headers[0] self.lastNameField = headers[1] self.personalEmailField = headers[2] self.interestsField = headers[3] descriptionTemplate = """Nature: %s -------- Abstract: %s""" class StudentInternshipsCSV(CSV): def __init__(self, filename, **kw): super(StudentInternshipsCSV, self).__init__(filename=filename, **kw) def parse_headers(self, headers): it = iter(headers) self.firstNameField = it.next() self.lastNameField = it.next() self.emailField = it.next() self.internshipTitleField = it.next() self.internshipNatureField = it.next() self.abstractField = it.next() self.dateStartField = it.next() self.dateEndField = it.next() self.supervisorFirstNameField = it.next() self.supervisorLastNameField = it.next() self.supervisorTitleField = it.next() self.supervisorEmailField = it.next() self.laboratoryNameField = it.next() self.laboratoryAffiliationField = it.next() self.unitCodeField = it.next() # CNRS / INSERM unit code self.streetField = it.next() # Street self.streetSecondPartField = it.next() # Street (continued) self.cityField = it.next() # City self.countryField = it.next() # Country def generate_nodes_and_links(self): for d in self.row_dicts: internship_id = next_id() # FIXME should be title. fix requires client change. need # configurable fields, or a list of names and types for display per # node (using a type node in the db, which could be cached title = d[self.internshipTitleField].strip() if len(title) == 0: print("skipping empty internship field") continue input_dict = { LABEL_SET:[INTERNSHIP_LABEL], 'name': title, 'description': descriptionTemplate % (d[self.internshipNatureField], d[self.abstractField]), 'startdate': d[self.dateStartField], 'enddate': d[self.dateEndField], 'internship-type': d[self.internshipNatureField], 'cnrs-inserm-unit-code': d[self.unitCodeField], 'city': d[self.cityField], 'country': d[self.countryField], 'street-address': d[self.streetField] + d[self.streetSecondPartField], 'facility': d[self.laboratoryNameField], 'facility-affiliation': d[self.laboratoryAffiliationField], } internship_node = node_dict(the_id=internship_id, **input_dict) self.append_id_node(internship_id, internship_node) person_email = cleanEmailField(d[self.emailField]) first_name = d[self.firstNameField] last_name = d[self.lastNameField] person_id = persons.id_from_something(first_name, last_name, person_email) if None == person_id: print("ERROR: missing %r (%r %r), creating person node with no interests" % (person_email, first_name, last_name)) person_id, person_node = addPersonNode(first_name=first_name, last_name=last_name, email=person_email) self.append_id_node(person_id, person_node) self.append_link(source_id=person_id, target_id=internship_id, label='Did this internship') def main(args, internships): cfg = Config.init_from_file(args.config) kernel = RZ_Kernel() kernel.db_ctl = dbc.DB_Controller(cfg) # yes, that. FIXME globals()['kernel'] = kernel globals()['ctx'] = {} # FIXME not logged it - fix later (also, don't do this here, put constructor in kernel) StudentCSV(filename=args.student).run() for internship in internships: StudentInternshipsCSV(filename=internship).run() if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('--student', required=True) parser.add_argument('--config', default=os.path.join(root, 'res', 'etc', 'rhizi-server.conf')) args, internships = parser.parse_known_args(sys.argv[1:]) main(args, internships)