"""
This ipython notebook (Jupyter notebook) file provides the following information:
i) How to programmatically load the data from Mahadevan's digital corpus of Indus inscriptions. When the datafile is
kept one folder above the folder of python notebook, inside a folder called data, the relative path to be used is
"data/data_M77.txt". The relative path should be changed using the dataFileName variable.
ii) Once data is loaded, the corpus can be searched in various ways using following methods given.
A) searchDILsWithPhraseInPositions(startSequences,endSequences,middleSequences,
exactSequences,startsNots,endsNots,containsNots,
minLength=1, maxLength=14)
B) printILsWithPhraseInPositions(starts,ends,middles,exact,startsNots,endsNots,containsNots,minLen=1, maxLen=14)
C) searchCountsWithPhraseInPositions (starts,ends,middles,exact,startsNots,endsNots,containsNots,minLen=1, maxLen=14)
D) searchCompleteInscriptionsWithPhraseInPositions(starts,ends,middles,exact,startsNots,endsNots,containsNots,minLen=1, maxLen=14)
iii) Some other utility methods such as printArtefact(objID) and printArtefacts(objIDs) are provided, to print any artefact by providing its
IDs in a comma-separated way.
iv) The current file is divided into eight segments. The current segment is followed by a)All the metadata regarding the entities
used in the corpus; b) The python import statements used in the program; c) The data structures needed to load the corpus;
d) The method used to load the corpus; e)The utility methods useful for searching the corpus; f) The methods to create certain
statistical tables present in the article, g)And finally certain examples of how all these methods can be used to get
desired outputs along with the outputs generated.
"""
#All needed metadata about the corpus
artefactFullTypeDict=dict([(1, "Seal"), (2, "Sealing"), (3, "Miniature Tablet"),(4,"Pottery Graffiti"),(5,"Copper Tablet"),
(6,"Bronze Implement"),(7,"Ivory or Bone rod"),(9,"Miscellaneous")])
artefactTypeDict=dict([(1, "Seal"), (2, "Sealing"), (3, "MinTab"),(4,"PottGraff"),(5,"CoppTab"),
(6,"BronzImpl"),(7,"IvoryRod"),(9,"Misc")])
directionOfWritingDict=dict([(1 , "Right to Left"),(2 , "Left to Right"),(3 , "Single sign line"),(4 , "Top to Bottom"),
(5 , "Symmetrical arrangement"),(9 , "DW doubtful"),(0 , "No line of text")])
lineCodeDict=dict([(0,"Only-Line"),(1,"Line-1"),(2,"Line-2"),(3,"Line-3"),(9,"No-Inscription-on-this-side")])
sideCodeDict=dict([(0,"Only-Side"),(1,"Side-1"),(2,"Side-2"),(3,"Side-3"),(4,"Side-4"),(5,"Side-5"),(6,"Side-6")])
locationDict=dict([(1,"Mohenjodaro_MIC"),(2, "Mohenjodaro_FEM"),(3,"Mohenjodaro_Misc"),(4, "Harappa_EH"),(5, "Harappa_Misc"),
(6, "Chanhudaro_CE"),(7, "Lothal"),(8,"Kalibangan"),(900, "JHUKAR"),(901, "LOHUMJODARO"),(902, "RUPAR"),
(903, "TARKHANAWALA_DERA"),(904, "ROJDI"),(905, "KOT_DIJI"),(906, "ALANGIRPUR"),(907, "DESALPAR"),
(908, "AMRI"),(909, "SURKOTADA"),(910, "CHANDIGARH"),(911, "RAKHIGARHI"),(912, "DHOLAVIRA"),
(920, "BANAWALI"),(921, "BANAWALI"),(922, "BANAWALI"),(970,"Unknown_prob_Harappan"),(980, "SUSA"),
(981, "DJOKA_UMMA"),(982, "KISH"),(983, "UR"),(984, "UR"),(985,"TELLOH"),(990, "Unknown_prob_W_Asia")])
broadLocationDict={1:"Mohenjodaro", 2:"Harappa",3:"Chanhudaro", 4:"Lothal", 5:"Kalibangan", 6:"Other", 7:"West Asian"}
m77ColumnDict=dict([("OBJECT-NUMBER",0), ("SIDE_NUMBER-LINE_NUMBER",1), ( "LOCUS",2),("LEVEL",3),("Object_TYPE",4),
("Iconography",5), ("DIRECTION_OF_WRITING",6),("NO_OF_POSITIONS",7),("NO_OF_SIGNS",8),
("SIGN-SEQUENCE-START",9)])#9 onwards inscriptions
#All functional sign-classes defined in the article are listed below
PF1s="342,211,12,14,15,254"
PF2s="1,176,242,139,224,93"
PPFs="347,8,296,193,222,348,358,226,62,128"
CMs="99,123,402,343,344,345,97,98"
PCLs="267,391,293,286,150,216,395,195,412,305"
NUMs="86,87,89,93,94,95,96,100,102,104,105,106,107,108,109,110,112,114,115,116,118,121,171,287,312,313,314,315,406,407"
METs="249,328,204,343,344,345,346,329,330"
CROPs="162,167,169,190"
ENCs="26, 60, 61, 66, 68, 71, 73, 88, 207, 220, 250,278, 281, 285, 377, 388, 390, 392, 404"
#All import statements used in the program are kept here in one place.
import pprint, os, tokenize, nltk, re, functools, pandas as pd
from nltk.collocations import *
#The needed data structures to load the corpus data.
distinct_inscription_lines={} #refined Sequence mapped to InscriptionLine
inscribed_objects=dict()
class InscriptionLine:
"""
This class represents the unique sign-sequences of each inscription-line found in Indus corpus. The same
inscription-line is often repeated in many inscribed objects. Thus this class captures the information regarding
the list of objects where the inscription-line has occurred, in which objects a few of the signs of the inscription-line
were doubtfully read, in which objects the inscription-line was completely legible, type of the inscribed artefacts
where the inscription-line has occurred, and the locations from where the artefacts containing the inscription-line
were found.
"""
def __init__(self,refinedSignSequence, signSet):
self.objectsOfOccurrence=[] #IDs of artefacts where the inscription-line occurred.
self.accompanyingIconographies=[] #IDs of iconographies that accompanied the inscription-line.
self.objectWiseDamagedSignIndices={} # Objects where the inscription-line has some doubtfully read signs, with
# indices of those doubtfully read signs.
self.objectWiseHasAnyDamagedSign={} # Map of which objects contain the inscription-line with doubtfully read
# signs and which objects are having fully legible versions of it.
self.objectWiseDamagedSigns={} # Object wise set of doubtfully read signs.
self.locations=[] # Locations where the inscription-line are found on some artefact
self.locationWiseLevels={} # Levels of excavation where the inscribed artefacts are found.
self.objectWiseSideNo= {} # The sideNos in the objects, where the inscription-line occurred.
self.objectWiseLineNo= {} # The lineNos in the objects, where the inscription-line occurred.
self.objectWiseType = {} # The types of artefacts where the inscription-line occurred.
self.signSequence=refinedSignSequence #The refined sign sequences removes information
#about which signs are doubtfully read.
self.signSet=signSet #All signs of the inscription-line collected as a set.
self.hasAtleastOneUndamagedOccurrence=False #Whether the inscription-line has atleast one fully legible occurrence.
self.hasAtleastOneDamagedOccurrence=False #Whether the inscription-line has atleast one occurrence with some
#doubtful sign.
def addObjectSpecificAttributes(self,objNo,sideNo,lineNo,location,level,damagedSignIndices,damagedSigns,
hasDamagedSign,iconography,objType):
"""
This method adds all the attributes of the inscription-line, that are specific to the artefact where it occurred.
"""
objNo=int(objNo)
self.objectsOfOccurrence.append(objNo)
self.objectWiseSideNo.setdefault(objNo,[]).append(sideNo)
self.objectWiseLineNo.setdefault(objNo,[]).append(lineNo)
self.objectWiseDamagedSignIndices[objNo]=damagedSignIndices
self.objectWiseHasAnyDamagedSign[objNo]=hasDamagedSign
self.objectWiseDamagedSigns[objNo]= damagedSigns
self.objectWiseType[objNo]= objType
self.locations.append(location)
self.locationWiseLevels.setdefault(location,[]).append(level)
self.accompanyingIconographies.append(iconography)
if hasDamagedSign:
self.hasAtleastOneDamagedOccurrence = True
else:
self.hasAtleastOneUndamagedOccurrence= True
class InscribedObject:
"""
This class represents the inscribed artefacts. Here we capture the artefactID, artefact-type, and its
location of discovery. This class also contains information about which inscription-line occurred in
which side and line in the artefact.
"""
def __init__(self,objectNo,objectType,location):
self.objectNo=objectNo
self.objectType=objectType
self.location=location
self.sideLineWiseInscriptionLines={}
self.sideWiseNoInscriptionIconography={}
def addInscriptionLine(self,sideLineKey,inscriptionLine):
self.sideLineWiseInscriptionLines[sideLineKey]=inscriptionLine
def addIconographyOfNonInscribedSide(self,sideNo,iconography):
self.sideWiseNoInscriptionIconography[sideNo]=iconography
def printArtefact(objNo):
objNo=int(objNo)
obj= inscribed_objects[objNo]
if not obj:
return
print "Object# %s (%s - %s):" % (objNo, obj.location, obj.objectType)
for sideLineKey in obj.sideLineWiseInscriptionLines.keys():
insLine = obj.sideLineWiseInscriptionLines[sideLineKey]
hasDamagedSign= '(*)' if (insLine.objectWiseHasAnyDamagedSign[objNo]) else ''
damagedIndices=insLine.objectWiseDamagedSignIndices[objNo]
updatedSequences=[]
if insLine.objectWiseHasAnyDamagedSign[objNo]:
for ind, sign in enumerate(insLine.signSequence.split()):
if ind in damagedIndices:
sign='*'+str(sign)
updatedSequences.append(sign)
else:
updatedSequences=insLine.signSequence.split()
print " Side-Line %s%s: <%s> " % (sideLineKey,hasDamagedSign, ' '.join(updatedSequences))
def printArtefacts(objNosCommaSeparated):
objNos=objNosCommaSeparated.split(',')
for objNo in objNos:
printArtefact(objNo)
print '----------------------------------'
#Method used to load the corpus data.
dataFileName="data/data_M77.txt"
def loadCorpusData(dataFileName):
refinedInscriptionLinesVisited= {}
inscribedObjectsVisited= set()
dataFile= open(dataFileName)
for line in dataFile:
lineTokens = line.strip().split() # tokenize the string
objNo=int(lineTokens[m77ColumnDict.get("OBJECT-NUMBER")])
sideNo=int(lineTokens[m77ColumnDict.get("SIDE_NUMBER-LINE_NUMBER")][0])
lineNo=int(lineTokens[m77ColumnDict.get("SIDE_NUMBER-LINE_NUMBER")][1])
location=lineTokens[m77ColumnDict.get("LOCUS")]
level=lineTokens[m77ColumnDict.get("LEVEL")]
objType=artefactTypeDict[int(lineTokens[m77ColumnDict.get("Object_TYPE")])]
iconography=lineTokens[m77ColumnDict.get("Iconography")]
directionOfWriting=lineTokens[m77ColumnDict.get("DIRECTION_OF_WRITING")]
broadLocationId=-1
if objNo >= 1000 and objNo < 4000 :
broadLocationId =1
elif objNo >= 4000 and objNo < 6000:
broadLocationId = 2
elif objNo >= 6000 and objNo < 7000:
broadLocationId = 3
elif objNo >= 7000 and objNo < 8000:
broadLocationId = 4
elif objNo >= 8000 and objNo < 9000:
broadLocationId = 5
elif objNo >= 9000 and objNo < 9800:
broadLocationId = 6
elif objNo >= 9800 and objNo < 10000:
broadLocationId = 7
broadLocation=broadLocationDict[broadLocationId]
if lineNo != 9:
signSequence=lineTokens[m77ColumnDict.get("SIGN-SEQUENCE-START"):]
refinedSequence=[]
damagedSignIndices=[]
hasDamagedSign=False
signSet=set()
damagedSignSet=set()
#print "Original sign sequence: %s" %signSequence
for num, sign in enumerate(signSequence, start=0):
if sign.startswith("*"):
hasDamagedSign=True
#print "damaged inscription line", signSequence
damagedSignIndices.append(num)
sign= sign[1:]
while sign[0]=='0' and len(sign)>1:
sign= sign[1:]
damagedSignSet.add(sign)
while sign[0]=='0' and len(sign)>1:
sign= sign[1:]
refinedSequence.append(sign)
signSet.add(sign)
refinedSequenceStr= ' '.join(refinedSequence)
if refinedInscriptionLinesVisited.setdefault(refinedSequenceStr,0)==0:
currInscriptionLine= InscriptionLine(refinedSequenceStr,signSet)
refinedInscriptionLinesVisited[refinedSequenceStr]+= 1
distinct_inscription_lines[refinedSequenceStr]=currInscriptionLine
currInscriptionLine=distinct_inscription_lines[refinedSequenceStr]
currInscriptionLine.addObjectSpecificAttributes(objNo,sideNo,lineNo,broadLocation,
level,damagedSignIndices,damagedSignSet,hasDamagedSign,
iconography,objType)
distinct_inscription_lines[refinedSequenceStr]=currInscriptionLine
if objNo not in inscribedObjectsVisited:
inscribedObject= InscribedObject(objNo,objType,broadLocation)
inscribed_objects[objNo]=inscribedObject
inscribedObjectsVisited.add(objNo)
inscribedObject= inscribed_objects[objNo]
inscribedObject.addInscriptionLine(lineTokens[m77ColumnDict.get("SIDE_NUMBER-LINE_NUMBER")],
currInscriptionLine)
#Utility methods used for searching the corpus.
def searchDILsWithPhraseInPositions(startSequences,endSequences,middleSequences,
exactSequences,startsNots,endsNots,containsNots,
minLength=1, maxLength=14):
"""
This method helps in searching the inscription-lines that have certain sign-sequences in the start, end, or middle section of
the inscription-line. We can also search by giving negative inputs, such as which sign-sequences should not be at the start,
or end or in any part of the searched inscription-lines. Each input can have comma-separated list of signs which are searched
by OR relation. But each inputType is having a AND relationship with each other input-type. We also provide the minimum and
maximim number of signs of the inscription-lines as optional search inputs.
For example:
searchDILsWithPhraseInPositions('267,391,150','211,342 176','99,123','','293','254,162','102,104',4, 6)
#The above call to the method searches distinct inscription-lines that: starts with signs <267>, <391>, or <150>;
AND ends with signs <211>, or sign-seq. <342 176>; AND contains <99> or <123>, AND DOES NOT start with <293>, AND
DOES NOT end with <254> or <162>, AND DOES NOT contain <102> or <104>. Moreover the inscription-lines must have
atleast 4 signs and atmost 6 signs.
We get 29 such distinct inscription-lines that meet these criteria.
<211>,
"""
startSeqList = startSequences.strip().split(',')
hasStartSeqList = startSequences.strip() not in ['',',']
endSeqList = endSequences.strip().split(',')
hasEndSeqList = endSequences.strip() not in ['',',']
middleSeqList = middleSequences.strip().split(',')
hasMiddleSeqList = middleSequences.strip() not in ['',',']
exactSeqList = exactSequences.strip().split(',')
hasExactSeqList = exactSequences.strip() not in ['',',']
startsNotList = startsNots.strip().split(',')
hasStartsNotList = startsNots.strip() not in ['',',']
endsNotList = endsNots.strip().split(',')
hasEndsNotList = endsNots.strip() not in ['',',']
containsNotList = containsNots.strip().split(',')
hasContainsNotList = containsNots.strip() not in ['',',']
initialSet=distinct_inscription_lines.keys()
currentStepFilter=initialSet
exactMatches=[]
if hasExactSeqList :
for exactSeq in exactSeqList:
exactSeq=exactSeq.strip()
if exactSeq.strip() != '':
currFilter = filter(lambda x: x.strip()==exactSeq.strip(), currentStepFilter)
exactMatches.extend(currFilter)
else:
exactMatches = currentStepFilter
currentStepFilter=exactMatches
afterLengthFilter = filter(lambda x: (len(x.split())>= minLength and len(x.split())<= maxLength), currentStepFilter)
currentStepFilter=afterLengthFilter
allStartMatches= []
if hasStartSeqList:
for startSeq in startSeqList:
startSeq=startSeq.strip()+' '
if startSeq.strip() != '':
currFilter = filter(lambda x: x.startswith(startSeq) or x==startSeq.strip(), currentStepFilter)
allStartMatches.extend(currFilter)
else:
allStartMatches = currentStepFilter
currentStepFilter=allStartMatches
allStartEndMatches=[]
if hasEndSeqList:
for endSeq in endSeqList:
endSeq = ' '+ endSeq.strip()
if endSeq.strip() != '':
currFilter = filter(lambda x: x.endswith(endSeq) or x==endSeq.strip(), currentStepFilter)
allStartEndMatches.extend(currFilter)
else:
allStartEndMatches= currentStepFilter
currentStepFilter=allStartEndMatches
allStartEndMiddleMatches=[]
if hasMiddleSeqList:
for middleSeq in middleSeqList:
middleSeq = middleSeq.strip()
if middleSeq.strip() != '':
currFilter = filter(lambda x: ' '+middleSeq+' ' in x or x.endswith(' '+middleSeq)
or x.startswith(middleSeq+' ') or x.strip() == middleSeq.strip(),
currentStepFilter)
allStartEndMiddleMatches.extend(currFilter)
else:
allStartEndMiddleMatches= currentStepFilter
currentStepFilter=allStartEndMiddleMatches
allStartEndMiddleMatchesExcludeStartsNots= []
if hasStartsNotList:
for startNot in startsNotList:
startNot= startNot.strip()
if startNot.strip() != '':
currFilter = filter(lambda x: x.startswith(startNot+' ') or x==startNot, currentStepFilter)
allStartEndMiddleMatchesExcludeStartsNots.extend(currFilter)
allStartEndMiddleMatchesExcludeStartsNots= list(set(currentStepFilter)-set(allStartEndMiddleMatchesExcludeStartsNots))
else:
allStartEndMiddleMatchesExcludeStartsNots=currentStepFilter
currentStepFilter= allStartEndMiddleMatchesExcludeStartsNots
allStartEndMiddleMatchesExcludeStartsNotEndsNots= []
if hasEndsNotList:
for endsNot in endsNotList:
endsNot= endsNot.strip()
if endsNot.strip() != '':
currFilter = filter(lambda x: x.endswith(' '+endsNot) or x==endsNot, currentStepFilter)
allStartEndMiddleMatchesExcludeStartsNotEndsNots.extend(currFilter)
allStartEndMiddleMatchesExcludeStartsNotEndsNots= list(set(currentStepFilter)-set(allStartEndMiddleMatchesExcludeStartsNotEndsNots))
else:
allStartEndMiddleMatchesExcludeStartsNotEndsNots=currentStepFilter
currentStepFilter= allStartEndMiddleMatchesExcludeStartsNotEndsNots
allStartEndMiddleMatchesExcludeStartsNotEndsNotsContainsNots= []
if hasContainsNotList:
for containsNot in containsNotList:
containsNot= containsNot.strip()
if containsNot.strip() != '':
currFilter = filter(lambda x: ' '+containsNot+' ' in x or x==containsNot or x.startswith(containsNot+' ')
or x.endswith(' '+containsNot),
currentStepFilter)
allStartEndMiddleMatchesExcludeStartsNotEndsNotsContainsNots.extend(currFilter)
allStartEndMiddleMatchesExcludeStartsNotEndsNotsContainsNots= list(set(currentStepFilter)-set(allStartEndMiddleMatchesExcludeStartsNotEndsNotsContainsNots))
else:
allStartEndMiddleMatchesExcludeStartsNotEndsNotsContainsNots=currentStepFilter
currentStepFilter= allStartEndMiddleMatchesExcludeStartsNotEndsNotsContainsNots
return currentStepFilter
def printILsWithPhraseInPositions(startSequences,endSequences,middleSequences,
exactSequences,startsNots,endsNots,containsNots,
minLength=1, maxLength=14):
'''
printILsWithPhraseInPositions('','','48 342 176','','','','',3, 4)
'''
allDILsMatching=searchDILsWithPhraseInPositions(startSequences,endSequences,middleSequences,
exactSequences,startsNots,endsNots,containsNots, minLength, maxLength)
filtered_inscription_lines= {k : v for k,v in filter(lambda item: item[0] in allDILsMatching, distinct_inscription_lines.iteritems())}
lenInsMax=0
for ins_line_seq in filtered_inscription_lines.keys():
lenInsMax=max(len(ins_line_seq),lenInsMax)
count=0
for ins_line_seq,ins_line in filtered_inscription_lines.iteritems():
count+=1
damaged = ''
damagedSignsToPrint=''
insLineJust="<"+ins_line_seq+">"
insLineJust=insLineJust.ljust(lenInsMax+5,' ')#Justify inscription lines to the max insLine lenth
def addHash(x):
return '#'+str(x)
objectsStr=', '.join(list(map(addHash,ins_line.objectWiseType.keys())))
objectTypesStr=', '.join(set(ins_line.objectWiseType.values()))
locationStr= ', '.join(set(ins_line.locations))
#print functools.reduce(lambda a,b : len(a) if len(a) > len(b) else len(b),ins_line.locations)
if not ins_line.hasAtleastOneUndamagedOccurrence:
damaged = '\n Doubtful-'
for obj in ins_line.objectWiseDamagedSigns.keys():
objType=ins_line.objectWiseType[obj]
damagedSignsToPrint= objType+" #" + str(obj)+": Signs <*"+ '>, <*'.join(ins_line.objectWiseDamagedSigns[obj])+">;"
currStr= (str(count).rjust(3,'0')) + ") " + insLineJust + (" In: ("+objectTypesStr+") ").ljust(35,' ') + \
(" From:"+ locationStr).ljust(20,' ') + damaged + damagedSignsToPrint
print currStr
return ' '
def searchCountsWithPhraseInPositions(startSequences,endSequences,middleSequences,
exactSequences,startsNots,endsNots,containsNots, minLength=1, maxLength=14):
'''
This method provides the count of distinct inscriptione-lines, where the given search criteria match, and also provides
count of such occurrences where some signs are doubtfully read.
'''
allDILsMatching=searchDILsWithPhraseInPositions(startSequences,endSequences,middleSequences,
exactSequences,startsNots,endsNots,containsNots, minLength, maxLength)
filtered_inscription_lines= {k : v for k,v in filter(lambda item: item[0] in allDILsMatching, distinct_inscription_lines.iteritems())}
allSearchedSignSeqs=set(str(startSequences+','+endSequences+','+middleSequences+','+exactSequences).split(','))
allSearchedSignSeqs.remove('')
allSearchedSigns=set()
for seq in allSearchedSignSeqs:
signs=seq.split()
allSearchedSigns.update(signs)
count =0
damagedCount=0
#print filtered_inscription_lines
for ins_line in filtered_inscription_lines.values():
count+=1
if not ins_line.hasAtleastOneUndamagedOccurrence:
hasOneOccurrenceWhereSearchedSignsWereNotDamaged=False
for damagedList in ins_line.objectWiseDamagedSigns.values():
searchDamagedIntersect=list(set(damagedList) & allSearchedSigns)
if len(searchDamagedIntersect)==0:
hasOneOccurrenceWhereSearchedSignsWereNotDamaged=True
break
if not hasOneOccurrenceWhereSearchedSignsWereNotDamaged:
damagedCount+=1
returnCount= str(count) + ('' if damagedCount==0 else " ("+str(damagedCount)+"*)")
return returnCount
def createBigramsPrecedingFollowing(precedingSignsStr, followingSignsStr):
'''
This utility method helps to create bigrams where the signs given in precedingSignsStr, follow signs provided
in the followingSignsStr.
'''
precedingSigns=set(precedingSignsStr.split(','))
followingSigns=set(followingSignsStr.split(','))
if '' in precedingSigns:
precedingSigns.remove('')
if '' in followingSigns:
followingSigns.remove('')
bigrams=set()
for prec in precedingSigns:
for foll in followingSigns:
bigrams.add(prec+' '+foll)
#print bigrams
return bigrams
def searchCompleteInscriptionsWithPhraseInPositions(startSequences,endSequences,middleSequences,
exactSequences,startsNots,endsNots,containsNots,
minLength=1, maxLength=14):
allDILsMatching=searchDILsWithPhraseInPositions(startSequences,endSequences,middleSequences,
exactSequences,startsNots,endsNots,containsNots, minLength, maxLength)
allInsContent= set()
for insLineSeq in allDILsMatching:
insLine = distinct_inscription_lines.get(insLineSeq)
objectIDsOfOccurrence = insLine.objectsOfOccurrence
for objID in objectIDsOfOccurrence:
obj=inscribed_objects[objID]
insContent= str(obj.objectType+'('+obj.location+') ').ljust(25,' ')
for sideLineKey in sorted(obj.sideLineWiseInscriptionLines.keys()):
currInsLine = obj.sideLineWiseInscriptionLines[sideLineKey]
hasDamagedSign= '(*)' if (insLine.objectWiseHasAnyDamagedSign[objID]) else ''
damagedIndices=insLine.objectWiseDamagedSignIndices[objID]
updatedSequences=[]
if currInsLine.objectWiseHasAnyDamagedSign[objID]:
for ind, sign in enumerate(currInsLine.signSequence.split()):
if ind in damagedIndices:
sign='*'+str(sign)
updatedSequences.append(sign)
else:
updatedSequences=currInsLine.signSequence.split()
insContent+="SideLine-%s: <%s> | " % (sideLineKey, ' '.join(updatedSequences))
allInsContent.add(insContent)
return list(allInsContent)
def make_Table1_ofArticle() :
'''
This method helps to show the results included in TAble-1 of the article.
'''
PF1_PF2s=createBigramsPrecedingFollowing(PF1s,PF2s)
row_col_dict=dict()
dataframeDict=dict()
for PF1_PF2 in PF1_PF2s:
countStr=searchCountsWithPhraseInPositions('','',str(PF1_PF2),'','','','',1,14)
row_col_dict[PF1_PF2] = countStr
rowPF2=[]
for PF2 in sorted(PF2s.split(',')):
rowPF2.append('<'+PF2+'>')
for PF1 in sorted(PF1s.split(',')):
dataframeDict.setdefault('<'+PF1+'> ' ,[]).append(row_col_dict[PF1+' '+PF2])
dataframeDict.setdefault("All occurrences"+' ' ,[]).append(searchCountsWithPhraseInPositions('','',PF2,'','','','',1,14))
df = pd.DataFrame(dataframeDict, index=rowPF2)
print df
print "\nSince in certain split sequences, <342> and <12> occurs in end of first line and <93> occurs in the second line"
print "in the same inscribed side, those counts are added in table-1, and the count thus differs from the result above."
print "Below the occurrences of <93> are printed as evidence. \n\n"
completeInscriptionsWithSearchedSequence=searchCompleteInscriptionsWithPhraseInPositions('','','93','','','','',1, 14)
for num, ins in enumerate(completeInscriptionsWithSearchedSequence):
num+=1
print "%s %s" % ( (str(num)+')').ljust(5,' '), ins)
return ''
def make_Table2_ofArticle():
'''
This method helps to show the results included in TAble-2 of the article.
'''
PPF_list=list(set(PPFs.split(',')))
if '' in PPF_list:
PPF_list.remove('')
rowName=[]
row_col_dict=dict()
for PPF in PPF_list:
bigrams= createBigramsPrecedingFollowing(PPF,PF1s)
countOfPPFOcc= searchCountsWithPhraseInPositions('','',','.join(bigrams),'','','','',1,14)
allOccCount= searchCountsWithPhraseInPositions('','',PPF,'','','','',1,14)
rowName.append('<'+PPF+'>')
row_col_dict.setdefault('PrePhraseFinal occurrences count',[]).append(str(countOfPPFOcc))
row_col_dict.setdefault('All occurrences count',[]).append(str(allOccCount))
df = pd.DataFrame(row_col_dict, index=rowName)
print df
return ''
def preconnective_contexts(PCL_sign):
'''
This method searches the inscription-lines where the given sign occurs in a pre-connective context.
'''
allOccSet=searchDILsWithPhraseInPositions('','',PCL_sign,'','','','',1,14)
returnSet=set()
pclRegex = re.compile(' '+PCL_sign+' ')
cmRegex= re.compile('( 99 | 123 | 402 | 343 | 344 | 345 | 97 | 98 )')
PF1ENCsRegex= re.compile( '('+'|'.join((map(lambda x: ' '+str(x)+' ', (PF1s+','+ENCs+','+CROPs).split(','))))+')' )
for insLine in allOccSet:
insLineSigns= insLine.split()
pclIndex = None
cmIndex = None
terminalIndex=None
for m in pclRegex.finditer(' '+insLine+' '):
pclIndex = m.start()
break
for m in cmRegex.finditer(' '+insLine+' '):
cmIndex = m.start()
break
for m in PF1ENCsRegex.finditer(' '+insLine+' '):
terminalIndex = m.start()
break
if (pclIndex is not None) and (cmIndex is not None) and pclIndex<=cmIndex:
returnSet.add(insLine)
if terminalIndex and pclIndex> terminalIndex:
print "Possible prolematic case (check manually): ",insLine
return returnSet
def make_Table3_ofArticle():
'''
This method helps to show the results included in TAble-3 of the article.
'''
PCL_list=list(set(PCLs.split(',')))
if '' in PCL_list:
PCL_list.remove('')
rowName=[]
row_col_dict=dict()
for PCL in PCL_list:
preconnSet=preconnective_contexts(PCL)
allOccCount= searchCountsWithPhraseInPositions('','',PCL,'','','','',1,14)
rowName.append('<'+PCL+'>')
row_col_dict.setdefault('Preconn. occurrences count',[]).append(len(preconnSet))
row_col_dict.setdefault('All occurrences count',[]).append(str(allOccCount))
df = pd.DataFrame(row_col_dict, index=rowName)
print str('\n').ljust(100,'-')
print df
print str('\n').ljust(100,'-')
print "The problematic cases might have had a segmented inscription where"
print "the PCL and CM are in different semantic units. But manually checking"
print "it becomes evident that in each of them, the PCL sign is in preconnective context."
return ''
def count_unique_Indus_signs():
corpus= '\n'.join(distinct_inscription_lines.keys()) #Keep each inscription-line separated by newLine in the corpus
tokensList = nltk.word_tokenize(corpus) #repeated list of all sign occurrences in DILs
while '0' in tokensList:
tokensList.remove('0')
uniqueTokens=sorted(set(tokensList))
return len(uniqueTokens)
def print_top_n_bigrams_sorted_by_frq_descendin_order(n):
corpus= '\n'.join(distinct_inscription_lines.keys()) #Keep each inscription-line separated by newLine in the corpus
tokensList = nltk.word_tokenize(corpus) #repeated list of all sign occurrences in DILs
while '0' in tokensList:
tokensList.remove('0')
uniqueTokens=sorted(set(tokensList))
freqDistAllSigns = nltk.FreqDist(tokensList)
bigram_measures = nltk.collocations.BigramAssocMeasures()
bigram_fd = nltk.FreqDist(nltk.bigrams(tokensList))
finder = BigramCollocationFinder(freqDistAllSigns, bigram_fd)
#finder.apply_word_filter(lambda w: w in ('162', '169'))
scored = finder.score_ngrams(bigram_measures.raw_freq)
print finder.nbest(bigram_measures.raw_freq, n)
#Various output statements
print "Certain selected outputs that are generated using the programming done above."
print str('\n').ljust(110,'=')
#First load the corpus data
loadCorpusData(dataFileName)
print "The corpus is loaded."
#Now we can search the corpus in various ways.
print "Total number of unique Indus signs: ", count_unique_Indus_signs()
print str('\n').ljust(100,'=')
print "Number of distinct Inscription line in IDF-80: "+ str(len(distinct_inscription_lines)-1)
print str('\n').ljust(100,'=')
print "\nTable-1: certain statistics of the phrase-final signs.\n"
make_Table1_ofArticle()
print str('\n').ljust(100,'=')
print "\nTable-2: certain statistics of the pre-phrase-final signs.\n"
make_Table2_ofArticle()
print str('\n').ljust(100,'=')
print "\nTable-3: certain statistics of the frequently pre-connective lexeme signs.\n"
make_Table3_ofArticle()
print str('\n').ljust(100,'=')
print "\nPrint top n bigrams, sorted by their descending order of frequencies\n"
print_top_n_bigrams_sorted_by_frq_descendin_order(20)
print str('\n').ljust(100,'=')
print "Print details of any Indus Artefact/Object by giving their IDs as commaSeparated input:\n"
printArtefacts('1603,1602')
print str('\n').ljust(100,'=')
print "Search inscription-lines and print them along with their accompanying inscriptions, side and line of occurrences,"
print "and the type of objects they occurred in.\n"
completeInscriptionsWithSearchedSequence=searchCompleteInscriptionsWithPhraseInPositions('','','','48 342 176','','','',1, 14)
for num, ins in enumerate(completeInscriptionsWithSearchedSequence):
num+=1
print "%s %s" % ( (str(num)+')').ljust(5,' '), ins)
print str('\n').ljust(100,'=')
print "Search inscription-lines and print them along with artefact types, locations, and doubtful occurreces details,"
print "all collated in a precise manner.\n"
printILsWithPhraseInPositions('','','48 342 176','','','','',3, 5)