#!/usr/bin/python2.7 """ Import CSV files into Rhizi Server via pythonic API. Tricky, since this is the only API user. Try to use it exactly as the REST/WS API would, just without actually creating a socket connection. """ import sys import os import argparse import string import csv root = os.path.join(os.path.dirname(__file__), '..', '..') sys.path.append(os.path.join(root, 'src', 'server')) from rz_api_common import sanitize_input__topo_diff from rz_kernel import RZ_Kernel import db_controller as dbc from rz_server import Config from model.graph import Topo_Diff def topo_diff_json(node_set_add=[], link_set_add=[]): topo_diff_dict = ({ u'link_id_set_rm': [], u'link_set_add': link_set_add, u'drop_conjugator_links': True, u'node_set_add': node_set_add, u'node_id_set_rm': [] }) topo_diff = Topo_Diff.from_json_dict(topo_diff_dict) sanitize_input__topo_diff(topo_diff) return topo_diff; cfg = Config.init_from_file(os.path.join(root, 'res', 'etc', 'rhizi-server.conf')) kernel = RZ_Kernel() kernel.db_ctl = dbc.DB_Controller(cfg) # yes, that. FIXME ctx = {} # FIXME not logged it - fix later (also, don't do this here, put constructor in kernel) def commit(topo_diff): _, commit_ret = kernel.diff_commit__topo(topo_diff, ctx) # Rhizi constants - FIXME use API PERSON_LABEL = 'Person' INTEREST_LABEL = 'Interest' INTERNSHIP_LABEL = "Third-internship-proposal" LABEL_SET = '__label_set' # CSV file columns PERSONAL_EMAIL = 'Personal e-mail' INTEREST = 'Interests' id_count = 0 def next_id(): global id_count id_count += 1 return '%08d' % id_count def node_dict(the_id, **args): """ FIXME use API """ ret = dict(args) ret['id'] = the_id return ret def link_dict(source_id, target_id, label, the_id): return ({ u'__type': [label], u'__src_id': source_id, u'__dst_id': target_id, u'id': the_id }) class CSV(object): def __init__(self, filename): rows = list(csv.reader(open(filename), 'excel-tab')) headers = rows[0] self.parse_headers(headers) #import pdb; pdb.set_trace() self.rows = rows[1:] self.row_dicts = map(lambda fields: dict(zip(headers, fields)), self.rows) def run(self): self.nodes_dict = {} self.node_set_add = [] self.link_set_add = [] self.generate_nodes_and_links() #for node in node_set_add: # print('committing %s' % repr(node)) # commit(topo_diff_json(node_set_add=[node])) commit(topo_diff_json(node_set_add=self.node_set_add)) commit(topo_diff_json(link_set_add=self.link_set_add)) def append_id_node(self, the_id, node): print(repr(node)) self.nodes_dict[the_id] = node self.node_set_add.append(node) return the_id def _csv_row_to_node(self, d, dict_gen): the_id = next_id() print(repr(d)) return the_id, node_dict(the_id=the_id, **dict_gen(d)) def append_link(self, source_id, target_id, label): the_id = next_id() link = link_dict(source_id, target_id, label, the_id) self.link_set_add.append(link) print(repr(link)) return the_id class StudentCSV(CSV): def __init__(self, filename): super(StudentCSV, self).__init__(filename) self.email_to_id = {} def interests(self, d): return d[self.interestsField] def generate_nodes_and_links(self): for d in self.row_dicts: d[self.interestsField] = map(string.strip, d[self.interestsField].split(',')) person_id = next_id() person_node = node_dict(the_id=person_id, **{'name':' '.join([d[self.firstNameField], d[self.lastNameField]]), LABEL_SET:[PERSON_LABEL]}) self.email_to_id[d[self.personalEmailField].lower()] = person_id self.append_id_node(person_id, person_node) for interest in self.interests(d): interest_id = next_id() interest_node = node_dict(the_id=interest_id, **{'name':interest, LABEL_SET:[INTEREST_LABEL]}) interest_id = self.append_id_node(interest_id, interest_node) self.append_link(person_id, interest_id, 'Is interested in') def parse_headers(self, headers): self.firstNameField = headers[0] self.lastNameField = headers[1] self.personalEmailField = headers[2] self.interestsField = headers[3] descriptionTemplate = """Nature: %s -------- Abstract: %s""" class StudentInternshipsCSV(CSV): def __init__(self, studentCSV, filename): super(StudentInternshipsCSV, self).__init__(filename) self.studentCSV = studentCSV self.email_to_id = studentCSV.email_to_id def parse_headers(self, headers): it = iter(headers) self.firstNameField = it.next() self.lastNameField = it.next() self.emailField = it.next() self.internshipTitleField = it.next() self.internshipNatureField = it.next() self.abstractField = it.next() self.dateStartField = it.next() self.dateEndField = it.next() self.supervisorFirstNameField = it.next() self.supervisorLastNameField = it.next() self.supervisorTitleField = it.next() self.supervisorEmailField = it.next() self.laboratoryNameField = it.next() self.laboratoryAffiliationField = it.next() self.unitCodeField = it.next() # CNRS / INSERM unit code self.streetField = it.next() # Street self.streetSecondPartField = it.next() # Street (continued) self.cityField = it.next() # City self.countryField = it.next() # Country def generate_nodes_and_links(self): for d in self.row_dicts: internship_id = next_id() # FIXME should be title. fix requires client change. need # configurable fields, or a list of names and types for display per # node (using a type node in the db, which could be cached input_dict = {LABEL_SET:[INTERNSHIP_LABEL], 'name': d[self.internshipTitleField], 'description': descriptionTemplate % (d[self.internshipNatureField], d[self.abstractField]), 'startdate': d[self.dateStartField], 'enddate': d[self.dateEndField]} internship_node = node_dict(the_id=internship_id, **input_dict) self.append_id_node(internship_id, internship_node) person_email = d[self.emailField].lower().strip() person_id = self.email_to_id.get(person_email, None) if None == person_id: print("ERROR: missing %s" % person_email) else: self.append_link(source_id=person_id, target_id=internship_id, label='Did this internship') assert(len(sys.argv) == 3) studentCSV = StudentCSV(sys.argv[-2]) studentCSV.run() StudentInternshipsCSV(studentCSV, sys.argv[-1]).run()