added jpype stuffjpype

7 files changed, 501 insertions, 0 deletions

diff --git a/stanford_parser/__init__.py b/stanford_parser/__init__.py
new file mode 100644
index 0000000..58d4807
--- /dev/null
+++ b/stanford_parser/__init__.py
@@ -0,0 +1,2 @@
+#from parser import ParserError, Parser
+
diff --git a/stanford_parser/dependencies.py b/stanford_parser/dependencies.py
new file mode 100644
index 0000000..4cda581
--- /dev/null
+++ b/stanford_parser/dependencies.py
@@ -0,0 +1,94 @@
+stanford_dependency_hierarchy = {"dep":
+                                 {"aux":{"auxpass":{},
+                                         "cop":{}},
+                                  "arg": {"agent":{},
+                                          "comp":{"acomp":{},
+                                                  "attr":{},
+                                                  "ccomp":{},
+                                                  "xcomp":{},
+                                                  "compl":{},
+                                                  "obj":{"dobj":{},
+                                                         "iobj":{},
+                                                         "pobj":{}},
+                                                  "mark":{},
+                                                  "rel":{}},
+                                          "subj":{"nsubj":{"nsubjpass":{}},
+                                                  "csubj":{}}},
+                                  "cc":{},
+                                  "conj":{},
+                                  "expl":{},
+                                  "mod":{"abbrev":{},
+                                         "amod":{},
+                                         "appos":{},
+                                         "advcl":{},
+                                         "purpcl":{},
+                                         "det":{},
+                                         "predet":{},
+                                         "preconj":{},
+                                         "infmod":{},
+                                         "partmod":{},
+                                         "advmod":{"neg":{}},
+                                         "rcmod":{},
+                                         "quantmod":{},
+                                         "tmod":{},
+                                         "measure":{},
+                                         "nn":{},
+                                         "num":{},
+                                         "number":{},
+                                         "prep":{},
+                                         "poss":{},
+                                         "possessive":{},
+                                         "prt":{}},
+                                  "parataxis":{},
+                                  "punct":{},
+                                  "ref":{},
+                                  "sdep":{"xsubj":{}}
+                                  }
+                                 }
+    
+
+class StanfordDependencyHierarchy:
+    """
+    Class that encodes the types of dependencies.
+    """
+
+
+    def __init__(self, hierarchy=stanford_dependency_hierarchy):
+        self.hierarchy=hierarchy
+
+        self.flatMap = {}
+
+        self.parentToChildren = {}
+
+        activeSet = [self.hierarchy]
+
+        while len(activeSet) != 0:
+            newActiveSet = []
+            for item in activeSet:
+                for key, mapValue in item.iteritems():
+                    self.flatMap[key] = mapValue
+                    self.parentToChildren[key] = sorted(list(mapValue.keys()))
+                    newActiveSet.append(mapValue)
+                
+            activeSet = newActiveSet
+
+        self.ancestorToDescendents = {}
+        
+        for key, mapValue in self.flatMap.iteritems():
+            descendents = []
+
+            activeSet = [mapValue]
+            while len(activeSet) != 0:
+                newActiveSet = []
+                for item in activeSet:
+                    for childKey, mapValue in item.iteritems():
+                        newActiveSet.extend(mapValue.values())
+                        descendents.append(childKey)
+                activeSet = newActiveSet
+                
+            self.ancestorToDescendents[key] = sorted(descendents)
+    def isa(self, relation, ancestor):
+        return relation in self.ancestorToDescendents[ancestor]
+
+            
+        
diff --git a/stanford_parser/dependencies_test.py b/stanford_parser/dependencies_test.py
new file mode 100644
index 0000000..6951828
--- /dev/null
+++ b/stanford_parser/dependencies_test.py
@@ -0,0 +1,17 @@
+import unittest
+
+
+class TestCase(unittest.TestCase):
+    def testChildren(self):
+        import dependencies
+        hierarchy = dependencies.StanfordDependencyHierarchy()
+        self.assertEqual(hierarchy.isa("agent", "arg"), True)
+
+        self.assertEqual(hierarchy.isa("ref", "dep"), True)
+        self.assertEqual(hierarchy.isa("dep", "dep"), False)
+
+        self.assertEqual(hierarchy.isa("predet", "mod"), True)
+
+        
+        
+    
diff --git a/stanford_parser/parser.py b/stanford_parser/parser.py
new file mode 100644
index 0000000..459ccf4
--- /dev/null
+++ b/stanford_parser/parser.py
@@ -0,0 +1,178 @@
+import jpype
+from standoff import TextStandoff
+
+class ParserError(Exception):
+    def __init__(self, *args, **margs):
+        Exception.__init__(self, *args,**margs)
+
+
+def standoffFromToken(txt, token):
+    return TextStandoff(txt, (token.beginPosition(), token.endPosition()))
+
+
+                   
+           
+
+
+class Dependencies:
+    def __init__(self, sentence, tokens, posTags, dependencies):
+        self.sentence = sentence
+
+        self.posTags = posTags        
+        
+        self.tokens = tokens
+
+        self.tokensToPosTags = dict(zip(self.tokens, self.posTags))
+
+        self.dependencies = dependencies
+        
+        self.govToDeps = {}
+        self.depToGov = {}
+        self.constituentsToRelation = {}
+
+        # there is a bug where sometimes there is a self dependency.
+        self.dependencies = [(relation, gov, dep) for relation, gov, dep in self.dependencies
+                             if gov != dep]
+
+
+        for relation, gov, dep in self.dependencies:
+
+            self.govToDeps.setdefault(gov, [])
+            self.govToDeps[gov].append(dep)
+            assert not dep in self.depToGov, (dep.text, [(key.text, value.text)
+                                                         for key, value in self.depToGov.iteritems()])
+            self.depToGov[dep] = gov
+            self.constituentsToRelation[(gov,dep)] = relation
+            
+        self.checkRep()
+
+    def tagForTokenStandoff(self, tokenStandoff):
+        return self.tokensToPosTags[tokenStandoff]
+        
+        
+    def checkRep(self):
+        assert len(self.posTags) == len(self.posTags)        
+        for t in self.tokens:
+            assert t.entireText == self.sentence
+
+
+
+    def govForDep(self, dep):
+        return self.depToGov[dep]
+    def depsForGov(self, gov):
+        return self.govToDeps[gov]
+
+    def relForConstituents(self, gov, dep):
+        return self.constituentsToRelation((gov, dep))
+    
+    def __str__(self):
+        result = ""
+        result += "sentence=" + repr(self.sentence) + "\n"
+        for relation, gov, dep in self.dependencies:
+            result += relation + "(" + gov.text + ", " + dep.text + ")\n"
+        return result
+
+stanford_parser_home = None
+
+def startJvm():
+    import os
+    os.environ.setdefault("STANFORD_PARSER_HOME", "/Users/yuval/Downloads/stanford-parser-python-r22186/3rdParty/stanford-parser")
+    global stanford_parser_home
+    stanford_parser_home = os.environ["STANFORD_PARSER_HOME"]
+    print "starting: %s/stanford-parser.jar" % (stanford_parser_home)
+    jpype.startJVM(jpype.getDefaultJVMPath(),
+                   "-ea",
+                   "-Djava.class.path=%s/stanford-parser.jar" % (stanford_parser_home),)
+startJvm() # one jvm per python instance.
+
+class Parser:
+
+    def __init__(self, pcfg_model_fname=None):
+        if pcfg_model_fname == None:
+            #self.pcfg_model_fname = "%s/englishPCFG.ser" % stanford_parser_home
+            #self.pcfg_model_fname = "%s/englishFactored.ser" % stanford_parser_home
+            self.pcfg_model_fname = "%s/../englishPCFG.July-2010.ser" % stanford_parser_home            
+        else:
+            self.pcfg_model_fname = pcfg_model_fname
+
+
+
+        self.package_lexparser = jpype.JPackage("edu.stanford.nlp.parser.lexparser")
+        
+        self.parser = self.package_lexparser.LexicalizedParser(self.pcfg_model_fname)
+        self.package = jpype.JPackage("edu.stanford.nlp")
+
+        tokenizerFactoryClass = self.package.process.__getattribute__("PTBTokenizer$PTBTokenizerFactory")
+        self.tokenizerFactory = tokenizerFactoryClass.newPTBTokenizerFactory(True, True)
+
+        self.documentPreprocessor = self.package.process.DocumentPreprocessor(self.tokenizerFactory)
+        
+        
+        self.parser.setOptionFlags(["-retainTmpSubcategories"])
+
+
+
+
+    def printInfo(self):
+
+        Numberer = self.package.util.Numberer
+        print ("Grammar\t" +
+               `Numberer.getGlobalNumberer("states").total()` + '\t' +
+               `Numberer.getGlobalNumberer("tags").total()` + '\t' +
+               `Numberer.getGlobalNumberer("words").total()` + '\t' +
+               `self.parser.pparser.ug.numRules()` + '\t' +
+               `self.parser.pparser.bg.numRules()` + '\t' +
+               `self.parser.pparser.lex.numRules()`)
+
+        print "ParserPack is ", self.parser.op.tlpParams.getClass()
+        print "Lexicon is ", self.parser.pd.lex.getClass()        
+        print "Tags are: ", Numberer.getGlobalNumberer("tags")
+        self.parser.op.display()
+        print "Test parameters"
+        self.parser.op.tlpParams.display();
+        self.package_lexparser.Test.display()
+    def parse(self, sentence):
+        """
+        Parses the sentence string, returning the tokens, and the parse tree as a tuple.
+        tokens, tree = parser.parse(sentence)
+        """
+
+        tokens = self.documentPreprocessor.getWordsFromString(sentence)
+        for token in tokens:
+            if token.word() in ["down"]:
+                print "setting tag"
+                token.setTag("IN")
+                pass
+            if token.word().lower() in ["bot"]:
+                token.setTag("NN")
+                pass
+        
+        wasParsed = self.parser.parse(tokens)
+        if not wasParsed:
+            raise ParserError("Could not parse " + sentence)
+        return tokens, self.parser.getBestParse()
+    
+    def parseToStanfordDependencies(self, sentence):
+
+        tokens, tree = self.parse(sentence)
+        standoffTokens = [standoffFromToken(sentence, token)
+                          for token in tokens]
+        posTags = [token.tag() for token in tree.taggedYield()]
+        print " ".join(["%s/%s" % (word.text, tag) for word, tag in zip(standoffTokens, posTags)])
+        #print tree.taggedYield().toString(False)
+        result = self.package.trees.EnglishGrammaticalStructure(tree)
+        
+        returnList = []
+        for dependency in result.typedDependenciesCollapsedTree():
+
+            govStandoff = standoffTokens[dependency.gov().index() - 1]
+            depStandoff = standoffTokens[dependency.dep().index() - 1]
+
+            returnList.append((str(dependency.reln()),
+                               govStandoff,
+                               depStandoff))
+
+
+
+        return Dependencies(sentence, standoffTokens, posTags, returnList)
+                              
diff --git a/stanford_parser/parser_test.py b/stanford_parser/parser_test.py
new file mode 100644
index 0000000..e5c90fa
--- /dev/null
+++ b/stanford_parser/parser_test.py
@@ -0,0 +1,86 @@
+import unittest
+
+
+class TestCase(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        from stanford_parser.parser import Parser        
+        cls.parser = Parser()
+    
+    def testParse(self):
+        
+        dependencies = self.parser.parseToStanfordDependencies("Pick up the tire pallet.")
+
+        tupleResult = [(rel, gov.text, dep.text) for rel, gov, dep in dependencies.dependencies]
+        self.assertEqual(tupleResult, [('prt', 'Pick', 'up'),
+                                       ('det', 'pallet', 'the'),
+                                       ('nn', 'pallet', 'tire'),
+                                       ('dobj', 'Pick', 'pallet')])
+
+        self.assertEqual(dependencies.tagForTokenStandoff(gov), "VB")
+        self.assertEqual(dependencies.tagForTokenStandoff(dep), "NN") 
+
+    def testParseRefexpNextTo(self):        
+        dependencies = self.parser.parseToStanfordDependencies("Pick up the tire pallet next to the truck.")
+        
+        tupleResult = [(rel, gov.text, dep.text) for rel, gov, dep in dependencies.dependencies]
+
+        
+        self.assertEqual(tupleResult,
+                         [('prt', 'Pick', 'up'),
+                          ('det', 'pallet', 'the'),
+                          ('nn', 'pallet', 'tire'),
+                          ('dobj', 'Pick', 'pallet'),
+                          ('det', 'truck', 'the'),
+                          ('prep_next_to', 'pallet', 'truck')])
+
+
+    def testParseRefexpNear(self):                
+        dependencies =self.parser.parseToStanfordDependencies("Pick up the tire pallet near the truck.")
+        
+        tupleResult = [(rel, gov.text, dep.text) for rel, gov, dep in dependencies.dependencies]
+        self.assertEqual(tupleResult,
+                         [('prt', 'Pick', 'up'),
+                          ('det', 'pallet', 'the'),
+                          ('nn', 'pallet', 'tire'),
+                          ('dobj', 'Pick', 'pallet'),
+                          ('det', 'truck', 'the'),
+                          ('prep_near', 'pallet', 'truck')])
+
+        
+
+    def testParseLong(self):                
+
+        # this sentence has a self dependency that the python code filters out.
+        # between drop and drop.
+        dependencies = self.parser.parseToStanfordDependencies("Grab the skid of tires right in front of you " +
+                                                               "and drop it off just in front and to the " +
+                                                               "right of the far skid of tires.")
+    
+        tupleResult = [(rel, gov.text, dep.text) for rel, gov, dep in dependencies.dependencies]
+        self.assertEqual(tupleResult,
+                         [('det', 'skid', 'the'), ('dobj', 'Grab', 'skid'),
+                          ('prep_of', 'skid', 'tires'), ('dep', 'Grab', 'right'),
+                          ('prep_in', 'Grab', 'front'), ('prep_of', 'front', 'you'),
+                          ('conj_and', 'Grab', 'drop'), ('dobj', 'drop', 'it'), ('prt', 'drop', 'off'),
+                          ('advmod', 'drop', 'just'), ('prep_in', 'drop', 'front'), ('det', 'right', 'the'),
+                          ('prep_to', 'drop', 'right'), ('det', 'skid', 'the'), ('amod', 'skid', 'far'),
+                          ('prep_of', 'right', 'skid'), ('prep_of', 'skid', 'tires')])
+                         
+       
+    
+    def testAllCaps(self):
+        dependencies = self.parser.parseToStanfordDependencies("GO TO THE TIRE PALLET NEXT TO THE TRUCK.")
+        tupleResult = [(rel, gov.text, dep.text) for rel, gov, dep in dependencies.dependencies]
+        self.assertEqual(tupleResult,
+                         [('nn', 'PALLET', 'GO'),
+                          ('nn', 'PALLET', 'TO'),
+                          ('nn', 'PALLET', 'THE'),
+                          ('nn', 'PALLET', 'TIRE'),
+                          ('nsubj', 'NEXT', 'PALLET'),
+                          ('dep', 'NEXT', 'TO'),
+                          ('det', 'TRUCK', 'THE'),
+                          ('dobj', 'TO', 'TRUCK')])
+        
+
+        
diff --git a/stanford_parser/standoff.py b/stanford_parser/standoff.py
new file mode 100644
index 0000000..76eef4d
--- /dev/null
+++ b/stanford_parser/standoff.py
@@ -0,0 +1,80 @@
+
+class TextStandoff:
+    def __init__(self, text, range):
+        self.entireText = text
+
+        self.range = range
+        
+    def asPrimitives(self):
+        return (self.entireText, self.range)
+    
+    @staticmethod
+    def fromPrimitives(args):
+        return TextStandoff(*args)
+        
+    def isNull(self):
+        return self.range == (0, 0)
+    
+    @property
+    def text(self):
+        start, end = self.range
+        return self.entireText[start:end]
+    @property
+    def length(self):
+        start, end = self.range
+        return end - start
+    @property
+    def end(self):
+        start, end = self.range
+        return end
+    @property
+    def start(self):
+        start, end = self.range
+        return start
+
+    def overlaps(self, standoff):
+        if self.start < standoff.end and standoff.start < self.end:
+            return True
+        else:
+            return False
+    def contains(self, standoff):
+        start, end = standoff
+        return self.start <= start and self.end >= end
+    def before(self, standoff):
+        if self.end <= standoff.start:
+            return True
+        else:
+            return False
+    def degreeOfOverlap(self, standoff):
+        """
+        Returns the size of the overlapping range of two tags. Returns
+        zero if they do not overlap.
+        """
+        start, end = standoff
+        if self.overlaps(standoff):
+            return min(end, self.end) - max(start, self.start)
+        else:
+            return 0
+
+    def __iter__(self):
+        return iter((self.start, self.end))
+    def toXml(self, standoff):
+        standoff.setAttribute("start", str(self.start))
+        standoff.setAttribute("end", str(self.end))
+
+    def __repr__(self):
+        return 'TextStandoff("%s", (%d, %d))' % (self.entireText, self.start, self.end)
+
+    def __str__(self):
+        return '("%s", (%d, %d))' % (self.text, self.start, self.end)
+
+    def __eq__(self, obj):
+        if isinstance(obj, TextStandoff):
+            if self.range == obj.range and self.entireText == obj.entireText:
+                return True
+        return False
+
+    def __hash__(self):
+        return hash(self.entireText) * 17 + hash(self.range)
+
+        
diff --git a/stanford_parser/standoff_test.py b/stanford_parser/standoff_test.py
new file mode 100644
index 0000000..5ff03f4
--- /dev/null
+++ b/stanford_parser/standoff_test.py
@@ -0,0 +1,44 @@
+import unittest
+from standoff import TextStandoff
+
+class StandoffTestCase(unittest.TestCase):
+    def testOverlaps(self):
+        self.assertEqual(TextStandoff("Testing 123", (0, 1)).overlaps(TextStandoff("Testing 123", (0, 1))), True)
+        self.assertEqual(TextStandoff("Testing 123", (0, 1)).overlaps(TextStandoff("Testing 123", (1, 2))), False)
+        self.assertEqual(TextStandoff("Testing 123", (0, 10)).overlaps(TextStandoff("Testing 123", (1, 2))), True)
+        self.assertEqual(TextStandoff("Testing 123", (0, 10)).overlaps(TextStandoff("Testing 123", (9, 10))), True)
+        self.assertEqual(TextStandoff("Testing 123", (0, 10)).overlaps(TextStandoff("Testing 123", (10, 11))), False)
+        self.assertEqual(TextStandoff("Testing 123", (10, 11)).overlaps(TextStandoff("Testing 123", (0, 10))), False)
+        
+        
+
+    def testBefore(self):
+        self.assertEqual(TextStandoff("Testing 123", (0, 1)).before(TextStandoff("Testing 123", (0, 1))), False)
+        self.assertEqual(TextStandoff("Testing 123", (0, 1)).before(TextStandoff("Testing 123", (1, 2))), True)
+        self.assertEqual(TextStandoff("Testing 123", (1, 2)).before(TextStandoff("Testing 123", (0, 1))), False)
+        self.assertEqual(TextStandoff("Testing 123", (0, 10)).before(TextStandoff("Testing 123", (1, 2))), False)
+        self.assertEqual(TextStandoff("Testing 123", (0, 10)).before(TextStandoff("Testing 123", (9, 10))), False)
+        self.assertEqual(TextStandoff("Testing 123", (0, 10)).before(TextStandoff("Testing 123", (10, 11))), True)
+
+        
+
+    def testContains(self):
+        self.assertEqual(TextStandoff("Testing 123", (0, 1)).contains(TextStandoff("Testing 123", (0, 1))), True)
+        self.assertEqual(TextStandoff("Testing 123", (0, 1)).contains(TextStandoff("Testing 123", (1, 2))), False)
+        self.assertEqual(TextStandoff("Testing 123", (1, 2)).contains(TextStandoff("Testing 123", (0, 1))), False)
+        self.assertEqual(TextStandoff("Testing 123", (0, 10)).contains(TextStandoff("Testing 123", (1, 2))), True)
+        self.assertEqual(TextStandoff("Testing 123", (0, 10)).contains(TextStandoff("Testing 123", (9, 10))), True)
+        self.assertEqual(TextStandoff("Testing 123", (0, 10)).contains(TextStandoff("Testing 123", (10, 11))), False)
+    def testDegreeOfOverlap(self):
+        self.assertEqual(TextStandoff("Testing 123", (0, 1)).degreeOfOverlap(TextStandoff("Testing 123", (0, 1))), 1)
+        self.assertEqual(TextStandoff("Testing 123", (0, 1)).degreeOfOverlap(TextStandoff("Testing 123", (1, 2))), 0)
+        self.assertEqual(TextStandoff("Testing 123", (1, 2)).degreeOfOverlap(TextStandoff("Testing 123", (0, 1))), 0)
+        self.assertEqual(TextStandoff("Testing 123", (0, 10)).degreeOfOverlap(TextStandoff("Testing 123", (1, 2))), 1)
+        self.assertEqual(TextStandoff("Testing 123", (0, 10)).degreeOfOverlap(TextStandoff("Testing 123", (9, 10))), 1)
+        self.assertEqual(TextStandoff("Testing 123", (0, 10)).degreeOfOverlap(TextStandoff("Testing 123", (10, 11))), 0)
+        self.assertEqual(TextStandoff("Testing 123", (0, 10)).degreeOfOverlap(TextStandoff("Testing 123", (8, 11))), 2)
+        self.assertEqual(TextStandoff("Testing 123", (8, 11)).degreeOfOverlap(TextStandoff("Testing 123", (0, 10))), 2)
+
+        self.assertEqual(TextStandoff("Testing 123", (0, 5)).degreeOfOverlap(TextStandoff("Testing 123", (-1, 10))), 5)
+
+        self.assertEqual(TextStandoff("Testing 123", (0, 5)).degreeOfOverlap(TextStandoff("Testing 123", (6, 125))), 0)