diff options
| -rw-r--r-- | stanford_parse.py | 15 | ||||
| -rw-r--r-- | stanford_parser/__init__.py | 2 | ||||
| -rw-r--r-- | stanford_parser/dependencies.py | 94 | ||||
| -rw-r--r-- | stanford_parser/dependencies_test.py | 17 | ||||
| -rw-r--r-- | stanford_parser/parser.py | 178 | ||||
| -rw-r--r-- | stanford_parser/parser_test.py | 86 | ||||
| -rw-r--r-- | stanford_parser/standoff.py | 80 | ||||
| -rw-r--r-- | stanford_parser/standoff_test.py | 44 |
8 files changed, 516 insertions, 0 deletions
diff --git a/stanford_parse.py b/stanford_parse.py new file mode 100644 index 0000000..37591d8 --- /dev/null +++ b/stanford_parse.py @@ -0,0 +1,15 @@ +from stanford_parser.parser import Parser +cls.parser = Parser() + +dependencies = self.parser.parseToStanfordDependencies("Pick up the tire pallet.") + +tupleResult = [(rel, gov.text, dep.text) for rel, gov, dep in dependencies.dependencies] +self.assertEqual(tupleResult, [('prt', 'Pick', 'up'), + ('det', 'pallet', 'the'), + ('nn', 'pallet', 'tire'), + ('dobj', 'Pick', 'pallet')]) + +self.assertEqual(dependencies.tagForTokenStandoff(gov), "VB") +self.assertEqual(dependencies.tagForTokenStandoff(dep), "NN") + +print tupleResult
\ No newline at end of file diff --git a/stanford_parser/__init__.py b/stanford_parser/__init__.py new file mode 100644 index 0000000..58d4807 --- /dev/null +++ b/stanford_parser/__init__.py @@ -0,0 +1,2 @@ +#from parser import ParserError, Parser + diff --git a/stanford_parser/dependencies.py b/stanford_parser/dependencies.py new file mode 100644 index 0000000..4cda581 --- /dev/null +++ b/stanford_parser/dependencies.py @@ -0,0 +1,94 @@ +stanford_dependency_hierarchy = {"dep": + {"aux":{"auxpass":{}, + "cop":{}}, + "arg": {"agent":{}, + "comp":{"acomp":{}, + "attr":{}, + "ccomp":{}, + "xcomp":{}, + "compl":{}, + "obj":{"dobj":{}, + "iobj":{}, + "pobj":{}}, + "mark":{}, + "rel":{}}, + "subj":{"nsubj":{"nsubjpass":{}}, + "csubj":{}}}, + "cc":{}, + "conj":{}, + "expl":{}, + "mod":{"abbrev":{}, + "amod":{}, + "appos":{}, + "advcl":{}, + "purpcl":{}, + "det":{}, + "predet":{}, + "preconj":{}, + "infmod":{}, + "partmod":{}, + "advmod":{"neg":{}}, + "rcmod":{}, + "quantmod":{}, + "tmod":{}, + "measure":{}, + "nn":{}, + "num":{}, + "number":{}, + "prep":{}, + "poss":{}, + "possessive":{}, + "prt":{}}, + "parataxis":{}, + "punct":{}, + "ref":{}, + "sdep":{"xsubj":{}} + } + } + + +class StanfordDependencyHierarchy: + """ + Class that encodes the types of dependencies. + """ + + + def __init__(self, hierarchy=stanford_dependency_hierarchy): + self.hierarchy=hierarchy + + self.flatMap = {} + + self.parentToChildren = {} + + activeSet = [self.hierarchy] + + while len(activeSet) != 0: + newActiveSet = [] + for item in activeSet: + for key, mapValue in item.iteritems(): + self.flatMap[key] = mapValue + self.parentToChildren[key] = sorted(list(mapValue.keys())) + newActiveSet.append(mapValue) + + activeSet = newActiveSet + + self.ancestorToDescendents = {} + + for key, mapValue in self.flatMap.iteritems(): + descendents = [] + + activeSet = [mapValue] + while len(activeSet) != 0: + newActiveSet = [] + for item in activeSet: + for childKey, mapValue in item.iteritems(): + newActiveSet.extend(mapValue.values()) + descendents.append(childKey) + activeSet = newActiveSet + + self.ancestorToDescendents[key] = sorted(descendents) + def isa(self, relation, ancestor): + return relation in self.ancestorToDescendents[ancestor] + + + diff --git a/stanford_parser/dependencies_test.py b/stanford_parser/dependencies_test.py new file mode 100644 index 0000000..6951828 --- /dev/null +++ b/stanford_parser/dependencies_test.py @@ -0,0 +1,17 @@ +import unittest + + +class TestCase(unittest.TestCase): + def testChildren(self): + import dependencies + hierarchy = dependencies.StanfordDependencyHierarchy() + self.assertEqual(hierarchy.isa("agent", "arg"), True) + + self.assertEqual(hierarchy.isa("ref", "dep"), True) + self.assertEqual(hierarchy.isa("dep", "dep"), False) + + self.assertEqual(hierarchy.isa("predet", "mod"), True) + + + + diff --git a/stanford_parser/parser.py b/stanford_parser/parser.py new file mode 100644 index 0000000..459ccf4 --- /dev/null +++ b/stanford_parser/parser.py @@ -0,0 +1,178 @@ +import jpype +from standoff import TextStandoff + +class ParserError(Exception): + def __init__(self, *args, **margs): + Exception.__init__(self, *args,**margs) + + +def standoffFromToken(txt, token): + return TextStandoff(txt, (token.beginPosition(), token.endPosition())) + + + + + + +class Dependencies: + def __init__(self, sentence, tokens, posTags, dependencies): + self.sentence = sentence + + self.posTags = posTags + + self.tokens = tokens + + self.tokensToPosTags = dict(zip(self.tokens, self.posTags)) + + self.dependencies = dependencies + + self.govToDeps = {} + self.depToGov = {} + self.constituentsToRelation = {} + + # there is a bug where sometimes there is a self dependency. + self.dependencies = [(relation, gov, dep) for relation, gov, dep in self.dependencies + if gov != dep] + + + for relation, gov, dep in self.dependencies: + + self.govToDeps.setdefault(gov, []) + self.govToDeps[gov].append(dep) + assert not dep in self.depToGov, (dep.text, [(key.text, value.text) + for key, value in self.depToGov.iteritems()]) + self.depToGov[dep] = gov + self.constituentsToRelation[(gov,dep)] = relation + + self.checkRep() + + def tagForTokenStandoff(self, tokenStandoff): + return self.tokensToPosTags[tokenStandoff] + + + def checkRep(self): + assert len(self.posTags) == len(self.posTags) + for t in self.tokens: + assert t.entireText == self.sentence + + + + def govForDep(self, dep): + return self.depToGov[dep] + def depsForGov(self, gov): + return self.govToDeps[gov] + + def relForConstituents(self, gov, dep): + return self.constituentsToRelation((gov, dep)) + + def __str__(self): + result = "" + result += "sentence=" + repr(self.sentence) + "\n" + for relation, gov, dep in self.dependencies: + result += relation + "(" + gov.text + ", " + dep.text + ")\n" + return result + +stanford_parser_home = None + +def startJvm(): + import os + os.environ.setdefault("STANFORD_PARSER_HOME", "/Users/yuval/Downloads/stanford-parser-python-r22186/3rdParty/stanford-parser") + global stanford_parser_home + stanford_parser_home = os.environ["STANFORD_PARSER_HOME"] + print "starting: %s/stanford-parser.jar" % (stanford_parser_home) + jpype.startJVM(jpype.getDefaultJVMPath(), + "-ea", + "-Djava.class.path=%s/stanford-parser.jar" % (stanford_parser_home),) +startJvm() # one jvm per python instance. + +class Parser: + + def __init__(self, pcfg_model_fname=None): + if pcfg_model_fname == None: + #self.pcfg_model_fname = "%s/englishPCFG.ser" % stanford_parser_home + #self.pcfg_model_fname = "%s/englishFactored.ser" % stanford_parser_home + self.pcfg_model_fname = "%s/../englishPCFG.July-2010.ser" % stanford_parser_home + else: + self.pcfg_model_fname = pcfg_model_fname + + + + self.package_lexparser = jpype.JPackage("edu.stanford.nlp.parser.lexparser") + + self.parser = self.package_lexparser.LexicalizedParser(self.pcfg_model_fname) + self.package = jpype.JPackage("edu.stanford.nlp") + + tokenizerFactoryClass = self.package.process.__getattribute__("PTBTokenizer$PTBTokenizerFactory") + self.tokenizerFactory = tokenizerFactoryClass.newPTBTokenizerFactory(True, True) + + self.documentPreprocessor = self.package.process.DocumentPreprocessor(self.tokenizerFactory) + + + self.parser.setOptionFlags(["-retainTmpSubcategories"]) + + + + + def printInfo(self): + + Numberer = self.package.util.Numberer + print ("Grammar\t" + + `Numberer.getGlobalNumberer("states").total()` + '\t' + + `Numberer.getGlobalNumberer("tags").total()` + '\t' + + `Numberer.getGlobalNumberer("words").total()` + '\t' + + `self.parser.pparser.ug.numRules()` + '\t' + + `self.parser.pparser.bg.numRules()` + '\t' + + `self.parser.pparser.lex.numRules()`) + + print "ParserPack is ", self.parser.op.tlpParams.getClass() + print "Lexicon is ", self.parser.pd.lex.getClass() + print "Tags are: ", Numberer.getGlobalNumberer("tags") + self.parser.op.display() + print "Test parameters" + self.parser.op.tlpParams.display(); + self.package_lexparser.Test.display() + def parse(self, sentence): + """ + Parses the sentence string, returning the tokens, and the parse tree as a tuple. + tokens, tree = parser.parse(sentence) + """ + + tokens = self.documentPreprocessor.getWordsFromString(sentence) + for token in tokens: + if token.word() in ["down"]: + print "setting tag" + token.setTag("IN") + pass + if token.word().lower() in ["bot"]: + token.setTag("NN") + pass + + wasParsed = self.parser.parse(tokens) + if not wasParsed: + raise ParserError("Could not parse " + sentence) + return tokens, self.parser.getBestParse() + + def parseToStanfordDependencies(self, sentence): + + tokens, tree = self.parse(sentence) + standoffTokens = [standoffFromToken(sentence, token) + for token in tokens] + posTags = [token.tag() for token in tree.taggedYield()] + print " ".join(["%s/%s" % (word.text, tag) for word, tag in zip(standoffTokens, posTags)]) + #print tree.taggedYield().toString(False) + result = self.package.trees.EnglishGrammaticalStructure(tree) + + returnList = [] + for dependency in result.typedDependenciesCollapsedTree(): + + govStandoff = standoffTokens[dependency.gov().index() - 1] + depStandoff = standoffTokens[dependency.dep().index() - 1] + + returnList.append((str(dependency.reln()), + govStandoff, + depStandoff)) + + + + return Dependencies(sentence, standoffTokens, posTags, returnList) + diff --git a/stanford_parser/parser_test.py b/stanford_parser/parser_test.py new file mode 100644 index 0000000..e5c90fa --- /dev/null +++ b/stanford_parser/parser_test.py @@ -0,0 +1,86 @@ +import unittest + + +class TestCase(unittest.TestCase): + @classmethod + def setUpClass(cls): + from stanford_parser.parser import Parser + cls.parser = Parser() + + def testParse(self): + + dependencies = self.parser.parseToStanfordDependencies("Pick up the tire pallet.") + + tupleResult = [(rel, gov.text, dep.text) for rel, gov, dep in dependencies.dependencies] + self.assertEqual(tupleResult, [('prt', 'Pick', 'up'), + ('det', 'pallet', 'the'), + ('nn', 'pallet', 'tire'), + ('dobj', 'Pick', 'pallet')]) + + self.assertEqual(dependencies.tagForTokenStandoff(gov), "VB") + self.assertEqual(dependencies.tagForTokenStandoff(dep), "NN") + + def testParseRefexpNextTo(self): + dependencies = self.parser.parseToStanfordDependencies("Pick up the tire pallet next to the truck.") + + tupleResult = [(rel, gov.text, dep.text) for rel, gov, dep in dependencies.dependencies] + + + self.assertEqual(tupleResult, + [('prt', 'Pick', 'up'), + ('det', 'pallet', 'the'), + ('nn', 'pallet', 'tire'), + ('dobj', 'Pick', 'pallet'), + ('det', 'truck', 'the'), + ('prep_next_to', 'pallet', 'truck')]) + + + def testParseRefexpNear(self): + dependencies =self.parser.parseToStanfordDependencies("Pick up the tire pallet near the truck.") + + tupleResult = [(rel, gov.text, dep.text) for rel, gov, dep in dependencies.dependencies] + self.assertEqual(tupleResult, + [('prt', 'Pick', 'up'), + ('det', 'pallet', 'the'), + ('nn', 'pallet', 'tire'), + ('dobj', 'Pick', 'pallet'), + ('det', 'truck', 'the'), + ('prep_near', 'pallet', 'truck')]) + + + + def testParseLong(self): + + # this sentence has a self dependency that the python code filters out. + # between drop and drop. + dependencies = self.parser.parseToStanfordDependencies("Grab the skid of tires right in front of you " + + "and drop it off just in front and to the " + + "right of the far skid of tires.") + + tupleResult = [(rel, gov.text, dep.text) for rel, gov, dep in dependencies.dependencies] + self.assertEqual(tupleResult, + [('det', 'skid', 'the'), ('dobj', 'Grab', 'skid'), + ('prep_of', 'skid', 'tires'), ('dep', 'Grab', 'right'), + ('prep_in', 'Grab', 'front'), ('prep_of', 'front', 'you'), + ('conj_and', 'Grab', 'drop'), ('dobj', 'drop', 'it'), ('prt', 'drop', 'off'), + ('advmod', 'drop', 'just'), ('prep_in', 'drop', 'front'), ('det', 'right', 'the'), + ('prep_to', 'drop', 'right'), ('det', 'skid', 'the'), ('amod', 'skid', 'far'), + ('prep_of', 'right', 'skid'), ('prep_of', 'skid', 'tires')]) + + + + def testAllCaps(self): + dependencies = self.parser.parseToStanfordDependencies("GO TO THE TIRE PALLET NEXT TO THE TRUCK.") + tupleResult = [(rel, gov.text, dep.text) for rel, gov, dep in dependencies.dependencies] + self.assertEqual(tupleResult, + [('nn', 'PALLET', 'GO'), + ('nn', 'PALLET', 'TO'), + ('nn', 'PALLET', 'THE'), + ('nn', 'PALLET', 'TIRE'), + ('nsubj', 'NEXT', 'PALLET'), + ('dep', 'NEXT', 'TO'), + ('det', 'TRUCK', 'THE'), + ('dobj', 'TO', 'TRUCK')]) + + + diff --git a/stanford_parser/standoff.py b/stanford_parser/standoff.py new file mode 100644 index 0000000..76eef4d --- /dev/null +++ b/stanford_parser/standoff.py @@ -0,0 +1,80 @@ + +class TextStandoff: + def __init__(self, text, range): + self.entireText = text + + self.range = range + + def asPrimitives(self): + return (self.entireText, self.range) + + @staticmethod + def fromPrimitives(args): + return TextStandoff(*args) + + def isNull(self): + return self.range == (0, 0) + + @property + def text(self): + start, end = self.range + return self.entireText[start:end] + @property + def length(self): + start, end = self.range + return end - start + @property + def end(self): + start, end = self.range + return end + @property + def start(self): + start, end = self.range + return start + + def overlaps(self, standoff): + if self.start < standoff.end and standoff.start < self.end: + return True + else: + return False + def contains(self, standoff): + start, end = standoff + return self.start <= start and self.end >= end + def before(self, standoff): + if self.end <= standoff.start: + return True + else: + return False + def degreeOfOverlap(self, standoff): + """ + Returns the size of the overlapping range of two tags. Returns + zero if they do not overlap. + """ + start, end = standoff + if self.overlaps(standoff): + return min(end, self.end) - max(start, self.start) + else: + return 0 + + def __iter__(self): + return iter((self.start, self.end)) + def toXml(self, standoff): + standoff.setAttribute("start", str(self.start)) + standoff.setAttribute("end", str(self.end)) + + def __repr__(self): + return 'TextStandoff("%s", (%d, %d))' % (self.entireText, self.start, self.end) + + def __str__(self): + return '("%s", (%d, %d))' % (self.text, self.start, self.end) + + def __eq__(self, obj): + if isinstance(obj, TextStandoff): + if self.range == obj.range and self.entireText == obj.entireText: + return True + return False + + def __hash__(self): + return hash(self.entireText) * 17 + hash(self.range) + + diff --git a/stanford_parser/standoff_test.py b/stanford_parser/standoff_test.py new file mode 100644 index 0000000..5ff03f4 --- /dev/null +++ b/stanford_parser/standoff_test.py @@ -0,0 +1,44 @@ +import unittest +from standoff import TextStandoff + +class StandoffTestCase(unittest.TestCase): + def testOverlaps(self): + self.assertEqual(TextStandoff("Testing 123", (0, 1)).overlaps(TextStandoff("Testing 123", (0, 1))), True) + self.assertEqual(TextStandoff("Testing 123", (0, 1)).overlaps(TextStandoff("Testing 123", (1, 2))), False) + self.assertEqual(TextStandoff("Testing 123", (0, 10)).overlaps(TextStandoff("Testing 123", (1, 2))), True) + self.assertEqual(TextStandoff("Testing 123", (0, 10)).overlaps(TextStandoff("Testing 123", (9, 10))), True) + self.assertEqual(TextStandoff("Testing 123", (0, 10)).overlaps(TextStandoff("Testing 123", (10, 11))), False) + self.assertEqual(TextStandoff("Testing 123", (10, 11)).overlaps(TextStandoff("Testing 123", (0, 10))), False) + + + + def testBefore(self): + self.assertEqual(TextStandoff("Testing 123", (0, 1)).before(TextStandoff("Testing 123", (0, 1))), False) + self.assertEqual(TextStandoff("Testing 123", (0, 1)).before(TextStandoff("Testing 123", (1, 2))), True) + self.assertEqual(TextStandoff("Testing 123", (1, 2)).before(TextStandoff("Testing 123", (0, 1))), False) + self.assertEqual(TextStandoff("Testing 123", (0, 10)).before(TextStandoff("Testing 123", (1, 2))), False) + self.assertEqual(TextStandoff("Testing 123", (0, 10)).before(TextStandoff("Testing 123", (9, 10))), False) + self.assertEqual(TextStandoff("Testing 123", (0, 10)).before(TextStandoff("Testing 123", (10, 11))), True) + + + + def testContains(self): + self.assertEqual(TextStandoff("Testing 123", (0, 1)).contains(TextStandoff("Testing 123", (0, 1))), True) + self.assertEqual(TextStandoff("Testing 123", (0, 1)).contains(TextStandoff("Testing 123", (1, 2))), False) + self.assertEqual(TextStandoff("Testing 123", (1, 2)).contains(TextStandoff("Testing 123", (0, 1))), False) + self.assertEqual(TextStandoff("Testing 123", (0, 10)).contains(TextStandoff("Testing 123", (1, 2))), True) + self.assertEqual(TextStandoff("Testing 123", (0, 10)).contains(TextStandoff("Testing 123", (9, 10))), True) + self.assertEqual(TextStandoff("Testing 123", (0, 10)).contains(TextStandoff("Testing 123", (10, 11))), False) + def testDegreeOfOverlap(self): + self.assertEqual(TextStandoff("Testing 123", (0, 1)).degreeOfOverlap(TextStandoff("Testing 123", (0, 1))), 1) + self.assertEqual(TextStandoff("Testing 123", (0, 1)).degreeOfOverlap(TextStandoff("Testing 123", (1, 2))), 0) + self.assertEqual(TextStandoff("Testing 123", (1, 2)).degreeOfOverlap(TextStandoff("Testing 123", (0, 1))), 0) + self.assertEqual(TextStandoff("Testing 123", (0, 10)).degreeOfOverlap(TextStandoff("Testing 123", (1, 2))), 1) + self.assertEqual(TextStandoff("Testing 123", (0, 10)).degreeOfOverlap(TextStandoff("Testing 123", (9, 10))), 1) + self.assertEqual(TextStandoff("Testing 123", (0, 10)).degreeOfOverlap(TextStandoff("Testing 123", (10, 11))), 0) + self.assertEqual(TextStandoff("Testing 123", (0, 10)).degreeOfOverlap(TextStandoff("Testing 123", (8, 11))), 2) + self.assertEqual(TextStandoff("Testing 123", (8, 11)).degreeOfOverlap(TextStandoff("Testing 123", (0, 10))), 2) + + self.assertEqual(TextStandoff("Testing 123", (0, 5)).degreeOfOverlap(TextStandoff("Testing 123", (-1, 10))), 5) + + self.assertEqual(TextStandoff("Testing 123", (0, 5)).degreeOfOverlap(TextStandoff("Testing 123", (6, 125))), 0) |