crawler.conf

#connection parameters 
fileType=text/html,text/plain
timeout=9000
soTimeout=9000
connMgrTimeout=9000
wait=0
maxFileSize=300000
usrAgent=TRSFC Crawler (Research purpose crawler)
usrFrom=tanaphol - at - akane.waseda.jp

#general parameter
maxPagePerSite=300
isOnlyStaticURL=true
ignoreRobot=true
canonicalCount=3
allowHttps=true
maxDepth=-1
pageMode=true


#limit downloaded page
limitDownloadedPages=20000

#crawler parameters
relevanceDegree=0.5
segmentThreshold=3
distanceFromRelevantSeg=1
filterNon=false
windowSize=100

#page classifier
checkerType=weka
targetLang=en
pageClassifierModel=/resources/classifiers/page-tourism.model
#pageClassifierModel=page-tourism.arff
relevantKeywordsPath=/resources/classifiers/thaiwords.txt

#predictor
predictorTrainingPath=
updateInterval=-100
isTrainingMode=false
useNeighborhood=false
useHistory=false
useDup=false

#linkClassifierAlgo=weka.classifiers.lazy.IBk
#linkClassifierParams=-K 5 -W 0

linkClassifierAlgo=weka.classifiers.bayes.NaiveBayesUpdateable
linkClassifierParams=


weightClassifiers=1,1,1,1

#tourism k5
#weightClassifiers=0.888,0.846,0.855
#tourism k3
#weightClassifiers=0.8,0.716,0.634
#tourism k5
#weightClassifiers=0.898,0.806,0.832


#thread
threads=1

#seed
seedPath=test.txt

#proxy
localProxyPath=
proxyServer=
proxyPort=


#db/dl parameters
downloadPath=logs/dl-tmp/
dbPath=db-tmp/
isGzip=true
collectDestinationUrl=false